2024-04-12T10:53:12.321 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-12T10:53:12.322 DEBUG:teuthology.run:Teuthology command: teuthology --name rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi --owner scheduled_rfriedma@teuthology --archive /home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791 --description upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --verbose -- /home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791/orig.config.yaml 2024-04-12T10:53:12.381 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-12T10:53:12.513 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791 branch: wip-rf-comb1 description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: null first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' first_in_suite: false job_id: '7653791' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-rf-comb1 ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-rf-comb1 sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 owner: scheduled_rfriedma@teuthology priority: 73 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 7341 sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade suite_branch: wip-rf-comb1 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_2cee1857f8068f7ca29db3a37937b8e9f919ba95/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 targets: smithi029.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBZ7FJKB+L6PIG6hkHMSXaF0oPRSlDO80Q8kJy72mhlk4GmXo9CD5wGZEMBwVpOk5ICmzPHSuiLYbmjWR5gpuuY= smithi087.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLdz/kUVoFbK0olMhupH5PXgGApvaUOeBM2pv4/kTcfQKbC9VpgdpNDQA/yOMVLBnY5CM+eB0qC5ZzwYlatsAjQ= tasks: - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-12_09:02:49 tube: smithi user: rfriedma verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-12T10:53:12.514 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_2cee1857f8068f7ca29db3a37937b8e9f919ba95/qa; will attempt to use it 2024-04-12T10:53:12.514 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_2cee1857f8068f7ca29db3a37937b8e9f919ba95/qa/tasks 2024-04-12T10:53:12.515 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-12T10:53:12.516 INFO:teuthology.task.internal:Checking packages... 2024-04-12T10:53:12.541 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '2cee1857f8068f7ca29db3a37937b8e9f919ba95' 2024-04-12T10:53:12.541 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T10:53:12.541 INFO:teuthology.packaging:ref: None 2024-04-12T10:53:12.541 INFO:teuthology.packaging:tag: None 2024-04-12T10:53:12.541 INFO:teuthology.packaging:branch: wip-rf-comb1 2024-04-12T10:53:12.541 INFO:teuthology.packaging:sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T10:53:12.541 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-rf-comb1 2024-04-12T10:53:12.763 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2874.g2cee1857 2024-04-12T10:53:12.764 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-12T10:53:12.771 INFO:teuthology.task.internal:no buildpackages task found 2024-04-12T10:53:12.771 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-12T10:53:12.783 INFO:teuthology.task.internal:Saving configuration 2024-04-12T10:53:12.803 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-12T10:53:12.810 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T10:53:12.835 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi029.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 10:48:06.617256', 'locked_by': 'scheduled_rfriedma@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBZ7FJKB+L6PIG6hkHMSXaF0oPRSlDO80Q8kJy72mhlk4GmXo9CD5wGZEMBwVpOk5ICmzPHSuiLYbmjWR5gpuuY='} 2024-04-12T10:53:12.862 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi087.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 10:48:06.618934', 'locked_by': 'scheduled_rfriedma@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLdz/kUVoFbK0olMhupH5PXgGApvaUOeBM2pv4/kTcfQKbC9VpgdpNDQA/yOMVLBnY5CM+eB0qC5ZzwYlatsAjQ='} 2024-04-12T10:53:12.863 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-12T10:53:12.869 INFO:teuthology.task.internal:roles: ubuntu@smithi029.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-12T10:53:12.869 INFO:teuthology.task.internal:roles: ubuntu@smithi087.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-12T10:53:12.869 INFO:teuthology.run_tasks:Running task console_log... 2024-04-12T10:53:12.940 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f82e4c760d0>, signals=[15]) 2024-04-12T10:53:12.941 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-12T10:53:12.948 INFO:teuthology.task.internal:Opening connections... 2024-04-12T10:53:12.948 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi029.front.sepia.ceph.com 2024-04-12T10:53:12.950 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T10:53:13.052 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi087.front.sepia.ceph.com 2024-04-12T10:53:13.053 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T10:53:13.143 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-12T10:53:13.149 DEBUG:teuthology.orchestra.run.smithi029:> uname -m 2024-04-12T10:53:13.167 INFO:teuthology.orchestra.run.smithi029.stdout:x86_64 2024-04-12T10:53:13.167 DEBUG:teuthology.orchestra.run.smithi029:> cat /etc/os-release 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:NAME="CentOS Stream" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:VERSION="9" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:ID="centos" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:ID_LIKE="rhel fedora" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:VERSION_ID="9" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:PLATFORM_ID="platform:el9" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:ANSI_COLOR="0;31" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:LOGO="fedora-logo-icon" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:HOME_URL="https://centos.org/" 2024-04-12T10:53:13.224 INFO:teuthology.orchestra.run.smithi029.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-12T10:53:13.225 INFO:teuthology.orchestra.run.smithi029.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-12T10:53:13.225 INFO:teuthology.orchestra.run.smithi029.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-12T10:53:13.225 INFO:teuthology.lock.ops:Updating smithi029.front.sepia.ceph.com on lock server 2024-04-12T10:53:13.254 DEBUG:teuthology.orchestra.run.smithi087:> uname -m 2024-04-12T10:53:13.273 INFO:teuthology.orchestra.run.smithi087.stdout:x86_64 2024-04-12T10:53:13.273 DEBUG:teuthology.orchestra.run.smithi087:> cat /etc/os-release 2024-04-12T10:53:13.330 INFO:teuthology.orchestra.run.smithi087.stdout:NAME="CentOS Stream" 2024-04-12T10:53:13.330 INFO:teuthology.orchestra.run.smithi087.stdout:VERSION="9" 2024-04-12T10:53:13.330 INFO:teuthology.orchestra.run.smithi087.stdout:ID="centos" 2024-04-12T10:53:13.330 INFO:teuthology.orchestra.run.smithi087.stdout:ID_LIKE="rhel fedora" 2024-04-12T10:53:13.330 INFO:teuthology.orchestra.run.smithi087.stdout:VERSION_ID="9" 2024-04-12T10:53:13.330 INFO:teuthology.orchestra.run.smithi087.stdout:PLATFORM_ID="platform:el9" 2024-04-12T10:53:13.330 INFO:teuthology.orchestra.run.smithi087.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-12T10:53:13.331 INFO:teuthology.orchestra.run.smithi087.stdout:ANSI_COLOR="0;31" 2024-04-12T10:53:13.331 INFO:teuthology.orchestra.run.smithi087.stdout:LOGO="fedora-logo-icon" 2024-04-12T10:53:13.331 INFO:teuthology.orchestra.run.smithi087.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-12T10:53:13.331 INFO:teuthology.orchestra.run.smithi087.stdout:HOME_URL="https://centos.org/" 2024-04-12T10:53:13.331 INFO:teuthology.orchestra.run.smithi087.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-12T10:53:13.331 INFO:teuthology.orchestra.run.smithi087.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-12T10:53:13.331 INFO:teuthology.orchestra.run.smithi087.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-12T10:53:13.331 INFO:teuthology.lock.ops:Updating smithi087.front.sepia.ceph.com on lock server 2024-04-12T10:53:13.353 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-12T10:53:13.363 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-12T10:53:13.373 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-12T10:53:13.373 DEBUG:teuthology.orchestra.run.smithi029:> test '!' -e /home/ubuntu/cephtest 2024-04-12T10:53:13.376 DEBUG:teuthology.orchestra.run.smithi087:> test '!' -e /home/ubuntu/cephtest 2024-04-12T10:53:13.391 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-12T10:53:13.397 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-12T10:53:13.397 DEBUG:teuthology.orchestra.run.smithi029:> test -z $(ls -A /var/lib/ceph) 2024-04-12T10:53:13.432 DEBUG:teuthology.orchestra.run.smithi087:> test -z $(ls -A /var/lib/ceph) 2024-04-12T10:53:13.458 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-12T10:53:13.509 INFO:teuthology.run_tasks:Running task kernel... 2024-04-12T10:53:13.524 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-04-12T10:53:13.524 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-04-12T10:53:13.524 DEBUG:teuthology.orchestra.run.smithi029:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-12T10:53:13.525 DEBUG:teuthology.orchestra.run.smithi087:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-12T10:53:13.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T10:53:13.540 DEBUG:teuthology.orchestra.run.smithi087:> uname -r 2024-04-12T10:53:13.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T10:53:13.541 DEBUG:teuthology.orchestra.run.smithi029:> uname -r 2024-04-12T10:53:13.596 INFO:teuthology.orchestra.run.smithi087.stdout:5.14.0-361.el9.x86_64 2024-04-12T10:53:13.597 INFO:teuthology.task.kernel:Running kernel on smithi087: 5.14.0-361.el9.x86_64 2024-04-12T10:53:13.597 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum install -y kernel 2024-04-12T10:53:13.598 INFO:teuthology.orchestra.run.smithi029.stdout:5.14.0-361.el9.x86_64 2024-04-12T10:53:13.598 INFO:teuthology.task.kernel:Running kernel on smithi029: 5.14.0-361.el9.x86_64 2024-04-12T10:53:13.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum install -y kernel 2024-04-12T10:53:15.939 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-9 - AppStream 11 MB/s | 8.1 MB 00:00 2024-04-12T10:53:16.850 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS-9 - AppStream 13 MB/s | 8.1 MB 00:00 2024-04-12T10:53:17.665 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-9 - BaseOS 6.0 MB/s | 2.5 MB 00:00 2024-04-12T10:53:18.560 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS-9 - BaseOS 7.5 MB/s | 2.5 MB 00:00 2024-04-12T10:53:19.497 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - BaseOS 5.8 MB/s | 8.0 MB 00:01 2024-04-12T10:53:20.020 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - BaseOS 8.3 MB/s | 8.0 MB 00:00 2024-04-12T10:53:32.296 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - AppStream 1.7 MB/s | 19 MB 00:11 2024-04-12T10:53:36.689 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - CRB 9.0 MB/s | 6.2 MB 00:00 2024-04-12T10:53:38.199 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - Extras packages 57 kB/s | 16 kB 00:00 2024-04-12T10:53:40.420 INFO:teuthology.orchestra.run.smithi029.stdout:Extra Packages for Enterprise Linux 10 MB/s | 21 MB 00:02 2024-04-12T10:53:44.080 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - AppStream 824 kB/s | 19 MB 00:23 2024-04-12T10:53:45.738 INFO:teuthology.orchestra.run.smithi029.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-04-12T10:53:48.082 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T10:53:48.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.086 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T10:53:48.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.089 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.090 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.091 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T10:53:48.091 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.091 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.091 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.091 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.091 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T10:53:48.091 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.091 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.091 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.091 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T10:53:48.538 INFO:teuthology.orchestra.run.smithi029.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-12T10:53:48.538 INFO:teuthology.orchestra.run.smithi029.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-12T10:53:48.660 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout: Package Architecture Version Repository Size 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout: kernel x86_64 5.14.0-435.el9 baseos 214 k 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-core x86_64 5.14.0-435.el9 baseos 15 M 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-modules x86_64 5.14.0-435.el9 baseos 34 M 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-modules-core x86_64 5.14.0-435.el9 baseos 28 M 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T10:53:48.675 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T10:53:48.676 INFO:teuthology.orchestra.run.smithi029.stdout:Install 4 Packages 2024-04-12T10:53:48.676 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T10:53:48.676 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 77 M 2024-04-12T10:53:48.676 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 123 M 2024-04-12T10:53:48.676 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T10:53:48.957 INFO:teuthology.orchestra.run.smithi029.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 1.3 MB/s | 214 kB 00:00 2024-04-12T10:53:49.375 INFO:teuthology.orchestra.run.smithi029.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 26 MB/s | 15 MB 00:00 2024-04-12T10:53:50.060 INFO:teuthology.orchestra.run.smithi029.stdout:(3/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 27 MB/s | 34 MB 00:01 2024-04-12T10:53:50.586 INFO:teuthology.orchestra.run.smithi029.stdout:(4/4): kernel-modules-core-5.14.0-435.el9.x86_6 17 MB/s | 28 MB 00:01 2024-04-12T10:53:50.586 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T10:53:50.586 INFO:teuthology.orchestra.run.smithi029.stdout:Total 40 MB/s | 77 MB 00:01 2024-04-12T10:53:51.288 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T10:53:51.738 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T10:53:51.739 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T10:53:52.777 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T10:53:52.778 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T10:53:55.404 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T10:53:55.587 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - CRB 808 kB/s | 6.2 MB 00:07 2024-04-12T10:53:56.099 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-12T10:53:56.258 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T10:53:57.313 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - Extras packages 28 kB/s | 16 kB 00:00 2024-04-12T10:53:59.165 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T10:53:59.385 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T10:53:59.667 INFO:teuthology.orchestra.run.smithi087.stdout:Extra Packages for Enterprise Linux 9.4 MB/s | 21 MB 00:02 2024-04-12T10:54:04.459 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T10:54:04.508 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:04.722 INFO:teuthology.orchestra.run.smithi087.stdout:lab-extras 41 kB/s | 1.7 kB 00:00 2024-04-12T10:54:06.990 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T10:54:06.990 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T10:54:06.990 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T10:54:06.990 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.990 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T10:54:06.990 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.990 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T10:54:06.990 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.991 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T10:54:06.992 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.993 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T10:54:06.994 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.995 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.996 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.997 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.998 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.998 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T10:54:06.998 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.998 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.998 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T10:54:06.998 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T10:54:07.422 INFO:teuthology.orchestra.run.smithi087.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-12T10:54:07.422 INFO:teuthology.orchestra.run.smithi087.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-12T10:54:07.544 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T10:54:07.557 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T10:54:07.557 INFO:teuthology.orchestra.run.smithi087.stdout: Package Architecture Version Repository Size 2024-04-12T10:54:07.557 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T10:54:07.557 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T10:54:07.557 INFO:teuthology.orchestra.run.smithi087.stdout: kernel x86_64 5.14.0-435.el9 baseos 214 k 2024-04-12T10:54:07.557 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2024-04-12T10:54:07.557 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-core x86_64 5.14.0-435.el9 baseos 15 M 2024-04-12T10:54:07.557 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-modules x86_64 5.14.0-435.el9 baseos 34 M 2024-04-12T10:54:07.557 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-modules-core x86_64 5.14.0-435.el9 baseos 28 M 2024-04-12T10:54:07.558 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T10:54:07.558 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T10:54:07.558 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T10:54:07.558 INFO:teuthology.orchestra.run.smithi087.stdout:Install 4 Packages 2024-04-12T10:54:07.558 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T10:54:07.558 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 77 M 2024-04-12T10:54:07.559 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 123 M 2024-04-12T10:54:07.559 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T10:54:07.974 INFO:teuthology.orchestra.run.smithi087.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 712 kB/s | 214 kB 00:00 2024-04-12T10:54:08.600 INFO:teuthology.orchestra.run.smithi087.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 16 MB/s | 15 MB 00:00 2024-04-12T10:54:09.606 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:09.866 INFO:teuthology.orchestra.run.smithi087.stdout:(3/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 15 MB/s | 34 MB 00:02 2024-04-12T10:54:10.141 INFO:teuthology.orchestra.run.smithi087.stdout:(4/4): kernel-modules-core-5.14.0-435.el9.x86_6 13 MB/s | 28 MB 00:02 2024-04-12T10:54:10.145 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T10:54:10.145 INFO:teuthology.orchestra.run.smithi087.stdout:Total 30 MB/s | 77 MB 00:02 2024-04-12T10:54:10.814 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T10:54:11.184 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T10:54:11.184 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T10:54:12.221 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T10:54:12.221 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T10:54:14.714 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T10:54:15.485 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-12T10:54:15.567 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T10:54:18.431 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T10:54:18.562 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T10:54:23.290 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T10:54:23.329 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:28.058 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:42.547 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:42.547 INFO:teuthology.orchestra.run.smithi029.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-12T10:54:42.547 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T10:54:42.595 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:43.038 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:43.038 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-12T10:54:43.038 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T10:54:43.038 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T10:54:43.500 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:43.500 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T10:54:43.500 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T10:54:43.500 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-12T10:54:43.500 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-12T10:54:43.500 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-12T10:54:43.500 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-12T10:54:43.500 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T10:54:43.501 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T10:54:43.941 DEBUG:teuthology.orchestra.run.smithi029:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T10:54:44.074 INFO:teuthology.orchestra.run.smithi029.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T10:54:44.074 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T10:54:44.074 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-12T10:54:44.074 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-12T10:54:44.074 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-12T10:54:44.075 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-12T10:54:44.075 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi029.front.sepia.ceph.com, path=None, version=distro) 2024-04-12T10:54:44.075 DEBUG:teuthology.orchestra.run.smithi029:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T10:54:44.159 INFO:teuthology.orchestra.run.smithi029.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T10:54:44.159 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T10:54:44.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo rpm -qi grub2-tools 2024-04-12T10:54:44.204 INFO:teuthology.orchestra.run.smithi029.stdout:Name : grub2-tools 2024-04-12T10:54:44.204 INFO:teuthology.orchestra.run.smithi029.stdout:Epoch : 1 2024-04-12T10:54:44.205 INFO:teuthology.orchestra.run.smithi029.stdout:Version : 2.06 2024-04-12T10:54:44.205 INFO:teuthology.orchestra.run.smithi029.stdout:Release : 68.el9 2024-04-12T10:54:44.205 INFO:teuthology.orchestra.run.smithi029.stdout:Architecture: x86_64 2024-04-12T10:54:44.205 INFO:teuthology.orchestra.run.smithi029.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-12T10:54:44.205 INFO:teuthology.orchestra.run.smithi029.stdout:Group : Unspecified 2024-04-12T10:54:44.206 INFO:teuthology.orchestra.run.smithi029.stdout:Size : 8264254 2024-04-12T10:54:44.206 INFO:teuthology.orchestra.run.smithi029.stdout:License : GPLv3+ 2024-04-12T10:54:44.206 INFO:teuthology.orchestra.run.smithi029.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-12T10:54:44.206 INFO:teuthology.orchestra.run.smithi029.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-12T10:54:44.206 INFO:teuthology.orchestra.run.smithi029.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-12T10:54:44.206 INFO:teuthology.orchestra.run.smithi029.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-12T10:54:44.206 INFO:teuthology.orchestra.run.smithi029.stdout:Packager : builder@centos.org 2024-04-12T10:54:44.206 INFO:teuthology.orchestra.run.smithi029.stdout:Vendor : CentOS 2024-04-12T10:54:44.207 INFO:teuthology.orchestra.run.smithi029.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-12T10:54:44.207 INFO:teuthology.orchestra.run.smithi029.stdout:Summary : Support tools for GRUB. 2024-04-12T10:54:44.207 INFO:teuthology.orchestra.run.smithi029.stdout:Description : 2024-04-12T10:54:44.207 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T10:54:44.207 INFO:teuthology.orchestra.run.smithi029.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-12T10:54:44.207 INFO:teuthology.orchestra.run.smithi029.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-12T10:54:44.207 INFO:teuthology.orchestra.run.smithi029.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-12T10:54:44.207 INFO:teuthology.orchestra.run.smithi029.stdout:hardware devices. 2024-04-12T10:54:44.207 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T10:54:44.207 INFO:teuthology.orchestra.run.smithi029.stdout:This subpackage provides tools for support of all platforms. 2024-04-12T10:54:44.208 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-12T10:54:44.208 INFO:teuthology.task.kernel:Updating grub on smithi029 to boot 5.14.0-435.el9.x86_64 2024-04-12T10:54:44.208 DEBUG:teuthology.orchestra.run.smithi029:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-12T10:54:45.015 INFO:teuthology.orchestra.run.smithi029.stderr:Generating grub configuration file ... 2024-04-12T10:54:46.668 INFO:teuthology.orchestra.run.smithi029.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-12T10:54:46.693 INFO:teuthology.orchestra.run.smithi029.stderr:done 2024-04-12T10:54:46.695 DEBUG:teuthology.orchestra.run.smithi029:> mktemp 2024-04-12T10:54:46.711 INFO:teuthology.orchestra.run.smithi029.stdout:/tmp/tmp.thxlFpOHFG 2024-04-12T10:54:46.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.thxlFpOHFG 2024-04-12T10:54:46.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod 0666 /tmp/tmp.thxlFpOHFG 2024-04-12T10:54:46.932 DEBUG:teuthology.orchestra.remote:smithi029:/tmp/tmp.thxlFpOHFG is 6KB 2024-04-12T10:54:46.980 DEBUG:teuthology.orchestra.run.smithi029:> rm -fr /tmp/tmp.thxlFpOHFG 2024-04-12T10:54:46.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo /bin/ls /boot/loader/entries || true 2024-04-12T10:54:47.063 INFO:teuthology.orchestra.run.smithi029.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-12T10:54:47.064 INFO:teuthology.orchestra.run.smithi029.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-12T10:54:47.064 INFO:teuthology.orchestra.run.smithi029.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-12T10:54:47.064 INFO:teuthology.orchestra.run.smithi029.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-12T10:54:47.065 DEBUG:teuthology.orchestra.run.smithi029:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-12T10:54:47.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo shutdown -r now 2024-04-12T10:54:57.035 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:57.035 INFO:teuthology.orchestra.run.smithi087.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-12T10:54:57.035 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T10:54:57.078 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:57.445 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:57.445 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-12T10:54:57.445 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T10:54:57.445 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T10:54:57.895 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T10:54:57.895 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T10:54:57.895 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T10:54:57.895 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-12T10:54:57.895 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-12T10:54:57.895 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-12T10:54:57.895 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-12T10:54:57.895 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T10:54:57.895 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T10:54:58.236 DEBUG:teuthology.orchestra.run.smithi087:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T10:54:58.335 INFO:teuthology.orchestra.run.smithi087.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T10:54:58.335 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T10:54:58.335 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-12T10:54:58.336 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-12T10:54:58.336 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-12T10:54:58.336 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-12T10:54:58.336 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi087.front.sepia.ceph.com, path=None, version=distro) 2024-04-12T10:54:58.336 DEBUG:teuthology.orchestra.run.smithi087:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T10:54:58.416 INFO:teuthology.orchestra.run.smithi087.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T10:54:58.416 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T10:54:58.416 DEBUG:teuthology.orchestra.run.smithi087:> sudo rpm -qi grub2-tools 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Name : grub2-tools 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Epoch : 1 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Version : 2.06 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Release : 68.el9 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Architecture: x86_64 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Group : Unspecified 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Size : 8264254 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:License : GPLv3+ 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-12T10:54:58.461 INFO:teuthology.orchestra.run.smithi087.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-12T10:54:58.462 INFO:teuthology.orchestra.run.smithi087.stdout:Packager : builder@centos.org 2024-04-12T10:54:58.462 INFO:teuthology.orchestra.run.smithi087.stdout:Vendor : CentOS 2024-04-12T10:54:58.462 INFO:teuthology.orchestra.run.smithi087.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-12T10:54:58.462 INFO:teuthology.orchestra.run.smithi087.stdout:Summary : Support tools for GRUB. 2024-04-12T10:54:58.462 INFO:teuthology.orchestra.run.smithi087.stdout:Description : 2024-04-12T10:54:58.462 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T10:54:58.462 INFO:teuthology.orchestra.run.smithi087.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-12T10:54:58.462 INFO:teuthology.orchestra.run.smithi087.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-12T10:54:58.463 INFO:teuthology.orchestra.run.smithi087.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-12T10:54:58.463 INFO:teuthology.orchestra.run.smithi087.stdout:hardware devices. 2024-04-12T10:54:58.463 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T10:54:58.463 INFO:teuthology.orchestra.run.smithi087.stdout:This subpackage provides tools for support of all platforms. 2024-04-12T10:54:58.464 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-12T10:54:58.464 INFO:teuthology.task.kernel:Updating grub on smithi087 to boot 5.14.0-435.el9.x86_64 2024-04-12T10:54:58.464 DEBUG:teuthology.orchestra.run.smithi087:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-12T10:54:59.025 INFO:teuthology.orchestra.run.smithi087.stderr:Generating grub configuration file ... 2024-04-12T10:55:00.405 INFO:teuthology.orchestra.run.smithi087.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-12T10:55:00.439 INFO:teuthology.orchestra.run.smithi087.stderr:done 2024-04-12T10:55:00.442 DEBUG:teuthology.orchestra.run.smithi087:> mktemp 2024-04-12T10:55:00.457 INFO:teuthology.orchestra.run.smithi087.stdout:/tmp/tmp.lTPpmPK1PV 2024-04-12T10:55:00.457 DEBUG:teuthology.orchestra.run.smithi087:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.lTPpmPK1PV 2024-04-12T10:55:00.525 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod 0666 /tmp/tmp.lTPpmPK1PV 2024-04-12T10:55:00.670 DEBUG:teuthology.orchestra.remote:smithi087:/tmp/tmp.lTPpmPK1PV is 6KB 2024-04-12T10:55:00.719 DEBUG:teuthology.orchestra.run.smithi087:> rm -fr /tmp/tmp.lTPpmPK1PV 2024-04-12T10:55:00.733 DEBUG:teuthology.orchestra.run.smithi087:> sudo /bin/ls /boot/loader/entries || true 2024-04-12T10:55:00.798 INFO:teuthology.orchestra.run.smithi087.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-12T10:55:00.798 INFO:teuthology.orchestra.run.smithi087.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-12T10:55:00.798 INFO:teuthology.orchestra.run.smithi087.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-12T10:55:00.798 INFO:teuthology.orchestra.run.smithi087.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-12T10:55:00.800 DEBUG:teuthology.orchestra.run.smithi087:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-12T10:55:00.944 DEBUG:teuthology.orchestra.run.smithi087:> sudo shutdown -r now 2024-04-12T10:55:17.246 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-12T10:55:17.246 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-04-12T10:55:17.247 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T10:55:30.957 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-12T10:55:30.958 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-12T10:55:30.959 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T10:55:35.714 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.29 2024-04-12T10:55:44.718 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-04-12T10:55:44.719 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T10:55:49.278 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.87 2024-04-12T10:55:58.280 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-12T10:55:58.281 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T10:56:01.342 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.87 2024-04-12T10:56:03.102 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.29 2024-04-12T10:56:13.348 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-12T10:56:13.349 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T10:56:15.105 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-04-12T10:56:15.106 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T10:56:18.178 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.29 2024-04-12T10:56:31.774 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.87 2024-04-12T10:56:33.180 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-04-12T10:56:33.181 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T10:56:33.868 DEBUG:teuthology.orchestra.run.smithi029:> true 2024-04-12T10:56:34.422 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-04-12T10:56:34.422 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-435.el9.x86_64"... 2024-04-12T10:56:34.423 DEBUG:teuthology.orchestra.run.smithi029:> uname -r 2024-04-12T10:56:34.447 INFO:teuthology.orchestra.run.smithi029.stdout:5.14.0-435.el9.x86_64 2024-04-12T10:56:34.448 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-12T10:56:34.448 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-12T10:56:34.448 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-12T10:56:35.448 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-12T10:56:35.448 DEBUG:teuthology.orchestra.run.smithi029:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-12T10:56:35.722 INFO:teuthology.orchestra.run.smithi029.stdout:ttyS1 2024-04-12T10:56:35.886 DEBUG:teuthology.parallel:result is None 2024-04-12T10:56:46.775 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-12T10:56:46.776 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T10:56:47.201 DEBUG:teuthology.orchestra.run.smithi087:> true 2024-04-12T10:56:47.619 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-12T10:56:47.620 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-435.el9.x86_64"... 2024-04-12T10:56:47.620 DEBUG:teuthology.orchestra.run.smithi087:> uname -r 2024-04-12T10:56:47.633 INFO:teuthology.orchestra.run.smithi087.stdout:5.14.0-435.el9.x86_64 2024-04-12T10:56:47.634 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-12T10:56:47.634 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-12T10:56:47.634 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-12T10:56:48.635 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-12T10:56:48.635 DEBUG:teuthology.orchestra.run.smithi087:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-12T10:56:48.695 INFO:teuthology.orchestra.run.smithi087.stdout:ttyS1 2024-04-12T10:56:48.724 DEBUG:teuthology.parallel:result is None 2024-04-12T10:56:48.724 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-12T10:56:48.732 INFO:teuthology.task.internal:Creating test directory... 2024-04-12T10:56:48.733 DEBUG:teuthology.orchestra.run.smithi029:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-12T10:56:48.735 DEBUG:teuthology.orchestra.run.smithi087:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-12T10:56:48.750 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-12T10:56:48.757 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-12T10:56:48.763 INFO:teuthology.task.internal:Creating archive directory... 2024-04-12T10:56:48.763 DEBUG:teuthology.orchestra.run.smithi029:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-12T10:56:48.793 DEBUG:teuthology.orchestra.run.smithi087:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-12T10:56:48.821 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-12T10:56:48.829 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-12T10:56:48.829 DEBUG:teuthology.orchestra.run.smithi029:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-12T10:56:48.858 DEBUG:teuthology.orchestra.run.smithi087:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-12T10:56:48.885 INFO:teuthology.orchestra.run.smithi087.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T10:56:48.892 INFO:teuthology.orchestra.run.smithi029.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T10:56:48.898 INFO:teuthology.orchestra.run.smithi087.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T10:56:48.906 INFO:teuthology.orchestra.run.smithi029.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T10:56:48.908 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-12T10:56:48.915 INFO:teuthology.task.internal:Configuring sudo... 2024-04-12T10:56:48.916 DEBUG:teuthology.orchestra.run.smithi029:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-12T10:56:48.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-12T10:56:48.982 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-12T10:56:48.993 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-12T10:56:48.993 DEBUG:teuthology.orchestra.run.smithi029:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-12T10:56:49.020 DEBUG:teuthology.orchestra.run.smithi087:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-12T10:56:49.035 DEBUG:teuthology.orchestra.run.smithi029:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T10:56:49.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T10:56:49.170 DEBUG:teuthology.orchestra.run.smithi029:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T10:56:49.251 DEBUG:teuthology.orchestra.run.smithi029:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T10:56:49.296 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T10:56:49.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-12T10:56:49.363 DEBUG:teuthology.orchestra.run.smithi087:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T10:56:49.406 DEBUG:teuthology.orchestra.run.smithi087:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T10:56:49.477 DEBUG:teuthology.orchestra.run.smithi087:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T10:56:49.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T10:56:49.556 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T10:56:49.556 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-12T10:56:49.622 DEBUG:teuthology.orchestra.run.smithi029:> sudo service rsyslog restart 2024-04-12T10:56:49.624 DEBUG:teuthology.orchestra.run.smithi087:> sudo service rsyslog restart 2024-04-12T10:56:49.672 INFO:teuthology.orchestra.run.smithi029.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T10:56:49.796 INFO:teuthology.orchestra.run.smithi087.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T10:56:50.058 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-12T10:56:50.066 INFO:teuthology.task.internal:Starting timer... 2024-04-12T10:56:50.066 INFO:teuthology.run_tasks:Running task pcp... 2024-04-12T10:56:50.076 INFO:teuthology.run_tasks:Running task selinux... 2024-04-12T10:56:50.084 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-12T10:56:50.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo service auditd rotate 2024-04-12T10:56:50.158 INFO:teuthology.orchestra.run.smithi029.stdout:Rotating logs: 2024-04-12T10:56:50.162 DEBUG:teuthology.orchestra.run.smithi087:> sudo service auditd rotate 2024-04-12T10:56:50.242 INFO:teuthology.orchestra.run.smithi087.stdout:Rotating logs: 2024-04-12T10:56:50.247 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-12T10:56:50.247 DEBUG:teuthology.orchestra.run.smithi029:> /usr/sbin/getenforce 2024-04-12T10:56:50.269 INFO:teuthology.orchestra.run.smithi029.stdout:Permissive 2024-04-12T10:56:50.269 DEBUG:teuthology.orchestra.run.smithi087:> /usr/sbin/getenforce 2024-04-12T10:56:50.305 INFO:teuthology.orchestra.run.smithi087.stdout:Permissive 2024-04-12T10:56:50.305 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi029.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi087.front.sepia.ceph.com': 'permissive'} 2024-04-12T10:56:50.306 DEBUG:teuthology.orchestra.run.smithi029:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-12T10:56:50.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T10:56:50.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-12T10:56:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T10:56:50.395 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-12T10:56:50.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo /usr/sbin/setenforce permissive 2024-04-12T10:56:50.429 DEBUG:teuthology.orchestra.run.smithi087:> sudo /usr/sbin/setenforce permissive 2024-04-12T10:56:50.500 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-12T10:56:50.510 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-12T10:56:50.517 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-12T10:56:50.943 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-12T10:56:50.959 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-12T10:56:50.960 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi029.front.sepia.ceph.com,smithi087.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-12T11:01:44.597 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi029.front.sepia.ceph.com'), Remote(name='ubuntu@smithi087.front.sepia.ceph.com')] 2024-04-12T11:01:44.599 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-04-12T11:01:44.600 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T11:01:44.690 DEBUG:teuthology.orchestra.run.smithi029:> true 2024-04-12T11:01:44.768 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-04-12T11:01:44.769 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-12T11:01:44.769 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T11:01:44.859 DEBUG:teuthology.orchestra.run.smithi087:> true 2024-04-12T11:01:44.937 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-12T11:01:44.938 INFO:teuthology.run_tasks:Running task clock... 2024-04-12T11:01:44.954 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-12T11:01:44.955 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-12T11:01:44.955 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T11:01:44.957 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-12T11:01:44.958 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T11:01:44.988 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-12T11:01:45.005 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-12T11:01:45.008 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-12T11:01:45.025 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-12T11:01:45.035 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: ntpd: command not found 2024-04-12T11:01:45.049 INFO:teuthology.orchestra.run.smithi029.stdout:506 Cannot talk to daemon 2024-04-12T11:01:45.054 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: ntpd: command not found 2024-04-12T11:01:45.065 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-12T11:01:45.068 INFO:teuthology.orchestra.run.smithi087.stdout:506 Cannot talk to daemon 2024-04-12T11:01:45.082 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-12T11:01:45.084 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-12T11:01:45.100 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-12T11:01:45.140 INFO:teuthology.orchestra.run.smithi029.stderr:bash: line 1: ntpq: command not found 2024-04-12T11:01:45.143 INFO:teuthology.orchestra.run.smithi029.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T11:01:45.143 INFO:teuthology.orchestra.run.smithi029.stdout:=============================================================================== 2024-04-12T11:01:45.143 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T11:01:45.143 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T11:01:45.143 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T11:01:45.143 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T11:01:45.159 INFO:teuthology.orchestra.run.smithi087.stderr:bash: line 1: ntpq: command not found 2024-04-12T11:01:45.162 INFO:teuthology.orchestra.run.smithi087.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T11:01:45.162 INFO:teuthology.orchestra.run.smithi087.stdout:=============================================================================== 2024-04-12T11:01:45.162 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T11:01:45.162 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T11:01:45.162 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T11:01:45.162 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T11:01:45.162 INFO:teuthology.run_tasks:Running task install... 2024-04-12T11:01:45.172 DEBUG:teuthology.task.install:project ceph 2024-04-12T11:01:45.172 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95'}} 2024-04-12T11:01:45.172 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95'} 2024-04-12T11:01:45.173 INFO:teuthology.task.install:Using flavor: default 2024-04-12T11:01:45.179 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-12T11:01:45.179 INFO:teuthology.task.install:extra packages: [] 2024-04-12T11:01:45.179 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95', 'tag': None, 'wait_for_package': False} 2024-04-12T11:01:45.179 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T11:01:45.179 INFO:teuthology.packaging:ref: None 2024-04-12T11:01:45.180 INFO:teuthology.packaging:tag: None 2024-04-12T11:01:45.180 INFO:teuthology.packaging:branch: reef 2024-04-12T11:01:45.180 INFO:teuthology.packaging:sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:01:45.180 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-04-12T11:01:45.182 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95', 'tag': None, 'wait_for_package': False} 2024-04-12T11:01:45.182 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T11:01:45.182 INFO:teuthology.packaging:ref: None 2024-04-12T11:01:45.182 INFO:teuthology.packaging:tag: None 2024-04-12T11:01:45.182 INFO:teuthology.packaging:branch: reef 2024-04-12T11:01:45.182 INFO:teuthology.packaging:sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:01:45.182 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-04-12T11:01:45.429 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/flavors/default/ 2024-04-12T11:01:45.430 INFO:teuthology.task.install.rpm:Package version is 18.2.2-1189.gc96c8d47 2024-04-12T11:01:45.461 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/flavors/default/ 2024-04-12T11:01:45.462 INFO:teuthology.task.install.rpm:Package version is 18.2.2-1189.gc96c8d47 2024-04-12T11:01:45.609 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-12T11:01:45.609 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:01:45.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-12T11:01:45.638 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-12T11:01:45.639 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T11:01:45.639 INFO:teuthology.packaging:ref: None 2024-04-12T11:01:45.639 INFO:teuthology.packaging:tag: None 2024-04-12T11:01:45.639 INFO:teuthology.packaging:branch: reef 2024-04-12T11:01:45.639 INFO:teuthology.packaging:sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:01:45.639 DEBUG:teuthology.orchestra.run.smithi029:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-12T11:01:45.651 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-12T11:01:45.652 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:01:45.652 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-12T11:01:45.680 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-12T11:01:45.680 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T11:01:45.680 INFO:teuthology.packaging:ref: None 2024-04-12T11:01:45.680 INFO:teuthology.packaging:tag: None 2024-04-12T11:01:45.680 INFO:teuthology.packaging:branch: reef 2024-04-12T11:01:45.680 INFO:teuthology.packaging:sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:01:45.681 DEBUG:teuthology.orchestra.run.smithi087:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-12T11:01:45.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-12T11:01:45.747 DEBUG:teuthology.orchestra.run.smithi087:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-12T11:01:45.790 DEBUG:teuthology.orchestra.run.smithi029:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-12T11:01:45.827 DEBUG:teuthology.orchestra.run.smithi087:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-12T11:01:45.857 INFO:teuthology.orchestra.run.smithi029.stdout:check_obsoletes = 1 2024-04-12T11:01:45.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum clean all 2024-04-12T11:01:45.894 INFO:teuthology.orchestra.run.smithi087.stdout:check_obsoletes = 1 2024-04-12T11:01:45.895 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum clean all 2024-04-12T11:01:46.103 INFO:teuthology.orchestra.run.smithi029.stdout:74 files removed 2024-04-12T11:01:46.130 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-radosgw 2024-04-12T11:01:46.141 INFO:teuthology.orchestra.run.smithi087.stdout:74 files removed 2024-04-12T11:01:46.168 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-radosgw 2024-04-12T11:01:46.788 INFO:teuthology.orchestra.run.smithi087.stdout:ceph packages for x86_64 241 kB/s | 77 kB 00:00 2024-04-12T11:01:46.930 INFO:teuthology.orchestra.run.smithi029.stdout:ceph packages for x86_64 156 kB/s | 77 kB 00:00 2024-04-12T11:01:47.214 INFO:teuthology.orchestra.run.smithi087.stdout:ceph noarch packages 30 kB/s | 11 kB 00:00 2024-04-12T11:01:47.380 INFO:teuthology.orchestra.run.smithi029.stdout:ceph noarch packages 28 kB/s | 11 kB 00:00 2024-04-12T11:01:47.598 INFO:teuthology.orchestra.run.smithi087.stdout:ceph source packages 5.4 kB/s | 1.9 kB 00:00 2024-04-12T11:01:47.815 INFO:teuthology.orchestra.run.smithi029.stdout:ceph source packages 4.8 kB/s | 1.9 kB 00:00 2024-04-12T11:01:48.353 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS-9 - AppStream 16 MB/s | 8.1 MB 00:00 2024-04-12T11:01:48.434 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-9 - AppStream 10 MB/s | 8.1 MB 00:00 2024-04-12T11:01:50.117 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-9 - BaseOS 8.0 MB/s | 2.5 MB 00:00 2024-04-12T11:01:50.146 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS-9 - BaseOS 5.8 MB/s | 2.5 MB 00:00 2024-04-12T11:01:51.296 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.0 MB 00:00 2024-04-12T11:01:55.075 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - BaseOS 1.8 MB/s | 8.0 MB 00:04 2024-04-12T11:01:55.907 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - AppStream 5.5 MB/s | 19 MB 00:03 2024-04-12T11:02:02.286 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - CRB 2.3 MB/s | 6.2 MB 00:02 2024-04-12T11:02:04.129 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - Extras packages 28 kB/s | 16 kB 00:00 2024-04-12T11:02:05.315 INFO:teuthology.orchestra.run.smithi029.stdout:Extra Packages for Enterprise Linux 20 MB/s | 21 MB 00:01 2024-04-12T11:02:10.651 INFO:teuthology.orchestra.run.smithi029.stdout:lab-extras 25 kB/s | 1.7 kB 00:00 2024-04-12T11:02:13.007 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - AppStream 1.1 MB/s | 19 MB 00:16 2024-04-12T11:02:13.335 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:02:13.335 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:02:13.335 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:02:13.335 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.335 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:02:13.335 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.335 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.336 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.337 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.338 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:02:13.339 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.339 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:02:13.339 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.339 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:02:13.339 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.339 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:02:13.339 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.339 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:02:13.340 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.341 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.342 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.343 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.344 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.345 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:02:13.936 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:02:13.956 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:02:13.956 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-04-12T11:02:13.956 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:02:13.956 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-radosgw x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 7.7 M 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout:Upgrading: 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: librados2 x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 3.3 M 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: librbd1 x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 3.0 M 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-base x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 5.1 M 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-common x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 18 M 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-selinux x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 25 k 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-04-12T11:02:13.957 INFO:teuthology.orchestra.run.smithi029.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs2 x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 708 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: libradosstriper1 x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 474 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: librgw2 x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 4.5 M 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-argparse x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 45 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-common x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 129 k 2024-04-12T11:02:13.958 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cephfs x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 162 k 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rados x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 321 k 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rbd x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 298 k 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rgw x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 99 k 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout:Install 28 Packages 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout:Upgrade 2 Packages 2024-04-12T11:02:13.959 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:13.960 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 53 M 2024-04-12T11:02:13.960 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:02:15.254 INFO:teuthology.orchestra.run.smithi029.stdout:(1/30): ceph-base-18.2.2-1189.gc96c8d47.el9.x86 6.4 MB/s | 5.1 MB 00:00 2024-04-12T11:02:15.346 INFO:teuthology.orchestra.run.smithi029.stdout:(2/30): ceph-selinux-18.2.2-1189.gc96c8d47.el9. 274 kB/s | 25 kB 00:00 2024-04-12T11:02:16.030 INFO:teuthology.orchestra.run.smithi029.stdout:(3/30): libcephfs2-18.2.2-1189.gc96c8d47.el9.x8 1.0 MB/s | 708 kB 00:00 2024-04-12T11:02:16.230 INFO:teuthology.orchestra.run.smithi029.stdout:(4/30): ceph-radosgw-18.2.2-1189.gc96c8d47.el9. 4.3 MB/s | 7.7 MB 00:01 2024-04-12T11:02:16.264 INFO:teuthology.orchestra.run.smithi029.stdout:(5/30): libradosstriper1-18.2.2-1189.gc96c8d47. 2.0 MB/s | 474 kB 00:00 2024-04-12T11:02:16.573 INFO:teuthology.orchestra.run.smithi029.stdout:(6/30): ceph-common-18.2.2-1189.gc96c8d47.el9.x 8.7 MB/s | 18 MB 00:02 2024-04-12T11:02:16.590 INFO:teuthology.orchestra.run.smithi029.stdout:(7/30): python3-ceph-argparse-18.2.2-1189.gc96c 138 kB/s | 45 kB 00:00 2024-04-12T11:02:16.641 INFO:teuthology.orchestra.run.smithi029.stdout:(8/30): python3-ceph-common-18.2.2-1189.gc96c8d 1.9 MB/s | 129 kB 00:00 2024-04-12T11:02:16.668 INFO:teuthology.orchestra.run.smithi029.stdout:(9/30): python3-cephfs-18.2.2-1189.gc96c8d47.el 2.0 MB/s | 162 kB 00:00 2024-04-12T11:02:16.785 INFO:teuthology.orchestra.run.smithi029.stdout:(10/30): librgw2-18.2.2-1189.gc96c8d47.el9.x86_ 8.0 MB/s | 4.5 MB 00:00 2024-04-12T11:02:16.819 INFO:teuthology.orchestra.run.smithi029.stdout:(11/30): python3-rbd-18.2.2-1189.gc96c8d47.el9. 1.9 MB/s | 298 kB 00:00 2024-04-12T11:02:16.846 INFO:teuthology.orchestra.run.smithi029.stdout:(12/30): python3-rados-18.2.2-1189.gc96c8d47.el 1.5 MB/s | 321 kB 00:00 2024-04-12T11:02:16.870 INFO:teuthology.orchestra.run.smithi029.stdout:(13/30): python3-rgw-18.2.2-1189.gc96c8d47.el9. 1.1 MB/s | 99 kB 00:00 2024-04-12T11:02:17.079 INFO:teuthology.orchestra.run.smithi029.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 412 kB/s | 107 kB 00:00 2024-04-12T11:02:17.113 INFO:teuthology.orchestra.run.smithi029.stdout:(15/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 598 kB/s | 160 kB 00:00 2024-04-12T11:02:17.138 INFO:teuthology.orchestra.run.smithi029.stdout:(16/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 169 kB/s | 45 kB 00:00 2024-04-12T11:02:17.197 INFO:teuthology.orchestra.run.smithi029.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 723 kB/s | 42 kB 00:00 2024-04-12T11:02:17.231 INFO:teuthology.orchestra.run.smithi029.stdout:(18/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.4 MB/s | 292 kB 00:00 2024-04-12T11:02:17.265 INFO:teuthology.orchestra.run.smithi029.stdout:(19/30): librdkafka-1.6.1-102.el9.x86_64.rpm 3.5 MB/s | 662 kB 00:00 2024-04-12T11:02:17.448 INFO:teuthology.orchestra.run.smithi029.stdout:(20/30): fmt-8.1.1-5.el9.x86_64.rpm 441 kB/s | 111 kB 00:00 2024-04-12T11:02:17.491 INFO:teuthology.orchestra.run.smithi029.stdout:(21/30): libarrow-doc-9.0.0-10.el9.noarch.rpm 622 kB/s | 26 kB 00:00 2024-04-12T11:02:17.525 INFO:teuthology.orchestra.run.smithi029.stdout:(22/30): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.0 MB/s | 309 kB 00:00 2024-04-12T11:02:17.542 INFO:teuthology.orchestra.run.smithi029.stdout:(23/30): liboath-2.6.7-2.el9.x86_64.rpm 965 kB/s | 49 kB 00:00 2024-04-12T11:02:17.575 INFO:teuthology.orchestra.run.smithi029.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-04-12T11:02:17.653 INFO:teuthology.orchestra.run.smithi029.stdout:(25/30): parquet-libs-9.0.0-10.el9.x86_64.rpm 7.4 MB/s | 838 kB 00:00 2024-04-12T11:02:17.687 INFO:teuthology.orchestra.run.smithi029.stdout:(26/30): re2-20211101-3.el9.x86_64.rpm 1.7 MB/s | 192 kB 00:00 2024-04-12T11:02:17.821 INFO:teuthology.orchestra.run.smithi029.stdout:(27/30): libarrow-9.0.0-10.el9.x86_64.rpm 8.8 MB/s | 4.4 MB 00:00 2024-04-12T11:02:17.871 INFO:teuthology.orchestra.run.smithi029.stdout:(28/30): thrift-0.15.0-2.el9.x86_64.rpm 7.4 MB/s | 1.6 MB 00:00 2024-04-12T11:02:18.130 INFO:teuthology.orchestra.run.smithi029.stdout:(29/30): librados2-18.2.2-1189.gc96c8d47.el9.x8 7.5 MB/s | 3.3 MB 00:00 2024-04-12T11:02:18.205 INFO:teuthology.orchestra.run.smithi029.stdout:(30/30): librbd1-18.2.2-1189.gc96c8d47.el9.x86_ 7.8 MB/s | 3.0 MB 00:00 2024-04-12T11:02:18.212 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:02:18.213 INFO:teuthology.orchestra.run.smithi029.stdout:Total 12 MB/s | 53 MB 00:04 2024-04-12T11:02:18.282 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:02:18.321 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:02:18.321 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:02:18.803 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:02:18.803 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:02:20.200 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:02:20.302 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-04-12T11:02:20.337 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-04-12T11:02:20.397 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-04-12T11:02:20.790 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-04-12T11:02:20.856 INFO:teuthology.orchestra.run.smithi029.stdout: Upgrading : librados2-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/32 2024-04-12T11:02:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librados2-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/32 2024-04-12T11:02:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rados-2:18.2.2-1189.gc96c8d47.el9.x86_64 6/32 2024-04-12T11:02:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libcephfs2-2:18.2.2-1189.gc96c8d47.el9.x86_64 7/32 2024-04-12T11:02:21.089 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libcephfs2-2:18.2.2-1189.gc96c8d47.el9.x86_64 7/32 2024-04-12T11:02:21.118 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-04-12T11:02:21.154 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-ceph-argparse-2:18.2.2-1189.gc96c8d47.el9. 9/32 2024-04-12T11:02:21.231 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cephfs-2:18.2.2-1189.gc96c8d47.el9.x86_64 10/32 2024-04-12T11:02:21.264 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libradosstriper1-2:18.2.2-1189.gc96c8d47.el9.x86_6 11/32 2024-04-12T11:02:21.338 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libradosstriper1-2:18.2.2-1189.gc96c8d47.el9.x86_6 11/32 2024-04-12T11:02:21.449 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : re2-1:20211101-3.el9.x86_64 12/32 2024-04-12T11:02:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 13/32 2024-04-12T11:02:21.531 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-04-12T11:02:21.565 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/32 2024-04-12T11:02:21.600 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 16/32 2024-04-12T11:02:21.638 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-04-12T11:02:21.671 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-04-12T11:02:21.986 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-04-12T11:02:22.019 INFO:teuthology.orchestra.run.smithi029.stdout: Upgrading : librbd1-2:18.2.2-1189.gc96c8d47.el9.x86_64 20/32 2024-04-12T11:02:22.110 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librbd1-2:18.2.2-1189.gc96c8d47.el9.x86_64 20/32 2024-04-12T11:02:22.149 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 21/32 2024-04-12T11:02:22.204 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-04-12T11:02:22.627 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 23/32 2024-04-12T11:02:22.667 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librgw2-2:18.2.2-1189.gc96c8d47.el9.x86_64 24/32 2024-04-12T11:02:22.721 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librgw2-2:18.2.2-1189.gc96c8d47.el9.x86_64 24/32 2024-04-12T11:02:22.801 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 25/32 2024-04-12T11:02:22.837 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-ceph-common-2:18.2.2-1189.gc96c8d47.el9.x8 26/32 2024-04-12T11:02:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 27/32 2024-04-12T11:02:25.075 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 27/32 2024-04-12T11:02:25.630 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 27/32 2024-04-12T11:02:25.725 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-base-2:18.2.2-1189.gc96c8d47.el9.x86_64 28/32 2024-04-12T11:02:25.821 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-base-2:18.2.2-1189.gc96c8d47.el9.x86_64 28/32 2024-04-12T11:02:25.822 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-12T11:02:25.822 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-12T11:02:25.822 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:25.860 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-selinux-2:18.2.2-1189.gc96c8d47.el9.x86_64 29/32 2024-04-12T11:02:33.232 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - CRB 386 kB/s | 6.2 MB 00:16 2024-04-12T11:02:34.799 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - Extras packages 58 kB/s | 16 kB 00:00 2024-04-12T11:02:37.708 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-selinux-2:18.2.2-1189.gc96c8d47.el9.x86_64 29/32 2024-04-12T11:02:37.708 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /sys 2024-04-12T11:02:37.708 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /proc 2024-04-12T11:02:37.708 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /mnt 2024-04-12T11:02:37.709 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /var/tmp 2024-04-12T11:02:37.709 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /home 2024-04-12T11:02:37.709 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /root 2024-04-12T11:02:37.709 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /tmp 2024-04-12T11:02:37.709 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:38.192 INFO:teuthology.orchestra.run.smithi087.stdout:Extra Packages for Enterprise Linux 6.4 MB/s | 21 MB 00:03 2024-04-12T11:02:38.504 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-radosgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 30/32 2024-04-12T11:02:38.528 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-radosgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 30/32 2024-04-12T11:02:38.528 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:02:38.529 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-12T11:02:38.529 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-12T11:02:38.529 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-12T11:02:38.529 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:38.531 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-04-12T11:02:38.608 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-04-12T11:02:38.608 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-04-12T11:02:40.011 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-04-12T11:02:40.011 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-base-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/32 2024-04-12T11:02:40.011 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 2/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-radosgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 3/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-selinux-2:18.2.2-1189.gc96c8d47.el9.x86_64 4/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libcephfs2-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libradosstriper1-2:18.2.2-1189.gc96c8d47.el9.x86_6 6/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librgw2-2:18.2.2-1189.gc96c8d47.el9.x86_64 7/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-ceph-argparse-2:18.2.2-1189.gc96c8d47.el9. 8/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-ceph-common-2:18.2.2-1189.gc96c8d47.el9.x8 9/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cephfs-2:18.2.2-1189.gc96c8d47.el9.x86_64 10/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rados-2:18.2.2-1189.gc96c8d47.el9.x86_64 11/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 12/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 13/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-04-12T11:02:40.012 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-04-12T11:02:40.013 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-04-12T11:02:40.013 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-04-12T11:02:40.013 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-04-12T11:02:40.013 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 21/32 2024-04-12T11:02:40.013 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 22/32 2024-04-12T11:02:40.013 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 23/32 2024-04-12T11:02:40.013 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-04-12T11:02:40.013 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-04-12T11:02:40.013 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 26/32 2024-04-12T11:02:40.013 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : re2-1:20211101-3.el9.x86_64 27/32 2024-04-12T11:02:40.014 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-04-12T11:02:40.014 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librados2-2:18.2.2-1189.gc96c8d47.el9.x86_64 29/32 2024-04-12T11:02:40.014 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-04-12T11:02:40.014 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librbd1-2:18.2.2-1189.gc96c8d47.el9.x86_64 31/32 2024-04-12T11:02:41.098 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout:Upgraded: 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: librados2-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: librbd1-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-base-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-radosgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-selinux-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: fmt-8.1.1-5.el9.x86_64 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-12T11:02:41.099 INFO:teuthology.orchestra.run.smithi029.stdout: libarrow-9.0.0-10.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs2-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: libradosstriper1-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: librgw2-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-argparse-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cephfs-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-12T11:02:41.101 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rados-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.101 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.101 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:41.101 INFO:teuthology.orchestra.run.smithi029.stdout: re2-1:20211101-3.el9.x86_64 2024-04-12T11:02:41.101 INFO:teuthology.orchestra.run.smithi029.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-12T11:02:41.101 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:41.101 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:02:41.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-test 2024-04-12T11:02:42.126 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:00:32 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:02:42.436 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:02:42.436 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:02:42.436 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:02:42.437 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:02:42.438 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.439 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.440 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.441 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:02:42.442 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.442 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:02:42.442 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.442 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:02:42.442 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.442 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:02:42.442 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.442 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:02:42.442 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.443 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.444 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.445 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.446 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:02:42.518 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:02:42.519 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:02:42.519 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-04-12T11:02:42.519 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:02:42.519 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:02:42.519 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-test x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 40 M 2024-04-12T11:02:42.519 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-04-12T11:02:42.519 INFO:teuthology.orchestra.run.smithi029.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-12T11:02:42.519 INFO:teuthology.orchestra.run.smithi029.stdout: libcephsqlite x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 166 k 2024-04-12T11:02:42.519 INFO:teuthology.orchestra.run.smithi029.stdout: oniguruma x86_64 6.9.6-1.el9.5 baseos 218 k 2024-04-12T11:02:42.520 INFO:teuthology.orchestra.run.smithi029.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-12T11:02:42.520 INFO:teuthology.orchestra.run.smithi029.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-12T11:02:42.520 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:42.520 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:02:42.520 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:02:42.520 INFO:teuthology.orchestra.run.smithi029.stdout:Install 6 Packages 2024-04-12T11:02:42.520 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:42.521 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 41 M 2024-04-12T11:02:42.521 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 164 M 2024-04-12T11:02:42.521 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:02:43.037 INFO:teuthology.orchestra.run.smithi029.stdout:(1/6): jq-1.6-16.el9.x86_64.rpm 1.2 MB/s | 187 kB 00:00 2024-04-12T11:02:43.106 INFO:teuthology.orchestra.run.smithi029.stdout:(2/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 3.1 MB/s | 218 kB 00:00 2024-04-12T11:02:43.131 INFO:teuthology.orchestra.run.smithi029.stdout:(3/6): libcephsqlite-18.2.2-1189.gc96c8d47.el9. 683 kB/s | 166 kB 00:00 2024-04-12T11:02:43.208 INFO:teuthology.orchestra.run.smithi087.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-04-12T11:02:43.282 INFO:teuthology.orchestra.run.smithi029.stdout:(4/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 423 kB/s | 64 kB 00:00 2024-04-12T11:02:43.315 INFO:teuthology.orchestra.run.smithi029.stdout:(5/6): socat-1.7.4.1-5.el9.x86_64.rpm 1.4 MB/s | 305 kB 00:00 2024-04-12T11:02:45.217 INFO:teuthology.orchestra.run.smithi029.stdout:(6/6): ceph-test-18.2.2-1189.gc96c8d47.el9.x86_ 17 MB/s | 40 MB 00:02 2024-04-12T11:02:45.219 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:02:45.219 INFO:teuthology.orchestra.run.smithi029.stdout:Total 15 MB/s | 41 MB 00:02 2024-04-12T11:02:45.272 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:02:45.293 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:02:45.294 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:02:45.644 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:02:45.644 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:02:45.835 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:02:45.835 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:02:45.835 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:02:45.835 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:02:45.836 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.837 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:02:45.837 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.837 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:02:45.837 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.837 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:02:45.837 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.837 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:02:45.837 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.837 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:02:45.837 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.838 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:02:45.838 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.838 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:02:45.838 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.838 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:02:45.838 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.838 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:02:45.838 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.838 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:02:45.838 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.839 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:02:45.839 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.839 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:02:45.839 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.839 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:02:45.839 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.839 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:02:45.839 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.839 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:02:45.840 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.840 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:02:45.840 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.840 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:02:45.840 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.840 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:02:45.840 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.840 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:02:45.840 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.840 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.841 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.842 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.843 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:02:45.844 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.845 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:02:45.846 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:02:46.197 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:02:46.294 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-12T11:02:46.350 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-12T11:02:46.354 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-radosgw x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 7.7 M 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout:Upgrading: 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout: librados2 x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 3.3 M 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout: librbd1 x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 3.0 M 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-base x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 5.1 M 2024-04-12T11:02:46.364 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-common x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 18 M 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-selinux x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 25 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: libarrow x86_64 9.0.0-10.el9 epel 4.4 M 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: libarrow-doc noarch 9.0.0-10.el9 epel 26 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: libcephfs2 x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 708 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: libradosstriper1 x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 474 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: librgw2 x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 4.5 M 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-12T11:02:46.365 INFO:teuthology.orchestra.run.smithi087.stdout: parquet-libs x86_64 9.0.0-10.el9 epel 838 k 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout: python3-ceph-argparse x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 45 k 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout: python3-ceph-common x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 129 k 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cephfs x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 162 k 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rados x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 321 k 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rbd x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 298 k 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rgw x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 99 k 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout:Install 28 Packages 2024-04-12T11:02:46.366 INFO:teuthology.orchestra.run.smithi087.stdout:Upgrade 2 Packages 2024-04-12T11:02:46.367 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:02:46.367 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 53 M 2024-04-12T11:02:46.367 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:02:46.385 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-12T11:02:46.439 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-12T11:02:46.466 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libcephsqlite-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/6 2024-04-12T11:02:49.852 INFO:teuthology.orchestra.run.smithi087.stdout:(1/30): ceph-base-18.2.2-1189.gc96c8d47.el9.x86 6.4 MB/s | 5.1 MB 00:00 2024-04-12T11:02:49.927 INFO:teuthology.orchestra.run.smithi087.stdout:(2/30): ceph-selinux-18.2.2-1189.gc96c8d47.el9. 336 kB/s | 25 kB 00:00 2024-04-12T11:02:50.127 INFO:teuthology.orchestra.run.smithi087.stdout:(3/30): libcephfs2-18.2.2-1189.gc96c8d47.el9.x8 3.5 MB/s | 708 kB 00:00 2024-04-12T11:02:50.294 INFO:teuthology.orchestra.run.smithi087.stdout:(4/30): libradosstriper1-18.2.2-1189.gc96c8d47. 2.8 MB/s | 474 kB 00:00 2024-04-12T11:02:50.408 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libcephsqlite-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/6 2024-04-12T11:02:50.469 INFO:teuthology.orchestra.run.smithi087.stdout:(5/30): ceph-radosgw-18.2.2-1189.gc96c8d47.el9. 5.4 MB/s | 7.7 MB 00:01 2024-04-12T11:02:50.498 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-test-2:18.2.2-1189.gc96c8d47.el9.x86_64 6/6 2024-04-12T11:02:50.562 INFO:teuthology.orchestra.run.smithi087.stdout:(6/30): python3-ceph-argparse-18.2.2-1189.gc96c 491 kB/s | 45 kB 00:00 2024-04-12T11:02:50.653 INFO:teuthology.orchestra.run.smithi087.stdout:(7/30): python3-ceph-common-18.2.2-1189.gc96c8d 1.4 MB/s | 129 kB 00:00 2024-04-12T11:02:50.728 INFO:teuthology.orchestra.run.smithi087.stdout:(8/30): python3-cephfs-18.2.2-1189.gc96c8d47.el 2.1 MB/s | 162 kB 00:00 2024-04-12T11:02:50.945 INFO:teuthology.orchestra.run.smithi087.stdout:(9/30): ceph-common-18.2.2-1189.gc96c8d47.el9.x 9.7 MB/s | 18 MB 00:01 2024-04-12T11:02:50.971 INFO:teuthology.orchestra.run.smithi087.stdout:(10/30): python3-rados-18.2.2-1189.gc96c8d47.el 1.3 MB/s | 321 kB 00:00 2024-04-12T11:02:51.029 INFO:teuthology.orchestra.run.smithi087.stdout:(11/30): python3-rbd-18.2.2-1189.gc96c8d47.el9. 3.5 MB/s | 298 kB 00:00 2024-04-12T11:02:51.063 INFO:teuthology.orchestra.run.smithi087.stdout:(12/30): python3-rgw-18.2.2-1189.gc96c8d47.el9. 1.1 MB/s | 99 kB 00:00 2024-04-12T11:02:51.121 INFO:teuthology.orchestra.run.smithi087.stdout:(13/30): librgw2-18.2.2-1189.gc96c8d47.el9.x86_ 5.4 MB/s | 4.5 MB 00:00 2024-04-12T11:02:51.172 INFO:teuthology.orchestra.run.smithi087.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 752 kB/s | 107 kB 00:00 2024-04-12T11:02:51.214 INFO:teuthology.orchestra.run.smithi087.stdout:(15/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 492 kB/s | 45 kB 00:00 2024-04-12T11:02:51.264 INFO:teuthology.orchestra.run.smithi087.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 798 kB/s | 160 kB 00:00 2024-04-12T11:02:51.289 INFO:teuthology.orchestra.run.smithi087.stdout:(17/30): librdkafka-1.6.1-102.el9.x86_64.rpm 5.5 MB/s | 662 kB 00:00 2024-04-12T11:02:51.315 INFO:teuthology.orchestra.run.smithi087.stdout:(18/30): python3-prettytable-0.7.2-27.el9.noarc 843 kB/s | 42 kB 00:00 2024-04-12T11:02:51.340 INFO:teuthology.orchestra.run.smithi087.stdout:(19/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.3 MB/s | 292 kB 00:00 2024-04-12T11:02:51.673 INFO:teuthology.orchestra.run.smithi087.stdout:(20/30): fmt-8.1.1-5.el9.x86_64.rpm 288 kB/s | 111 kB 00:00 2024-04-12T11:02:51.757 INFO:teuthology.orchestra.run.smithi087.stdout:(21/30): libarrow-doc-9.0.0-10.el9.noarch.rpm 312 kB/s | 26 kB 00:00 2024-04-12T11:02:51.782 INFO:teuthology.orchestra.run.smithi087.stdout:(22/30): gperftools-libs-2.9.1-2.el9.x86_64.rpm 661 kB/s | 309 kB 00:00 2024-04-12T11:02:51.841 INFO:teuthology.orchestra.run.smithi087.stdout:(23/30): liboath-2.6.7-2.el9.x86_64.rpm 581 kB/s | 49 kB 00:00 2024-04-12T11:02:51.866 INFO:teuthology.orchestra.run.smithi087.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 805 kB/s | 67 kB 00:00 2024-04-12T11:02:51.966 INFO:teuthology.orchestra.run.smithi087.stdout:(25/30): re2-20211101-3.el9.x86_64.rpm 1.9 MB/s | 192 kB 00:00 2024-04-12T11:02:52.059 INFO:teuthology.orchestra.run.smithi087.stdout:(26/30): libarrow-9.0.0-10.el9.x86_64.rpm 6.2 MB/s | 4.4 MB 00:00 2024-04-12T11:02:52.092 INFO:teuthology.orchestra.run.smithi087.stdout:(27/30): parquet-libs-9.0.0-10.el9.x86_64.rpm 3.3 MB/s | 838 kB 00:00 2024-04-12T11:02:52.184 INFO:teuthology.orchestra.run.smithi087.stdout:(28/30): thrift-0.15.0-2.el9.x86_64.rpm 7.3 MB/s | 1.6 MB 00:00 2024-04-12T11:02:52.451 INFO:teuthology.orchestra.run.smithi087.stdout:(29/30): librbd1-18.2.2-1189.gc96c8d47.el9.x86_ 8.4 MB/s | 3.0 MB 00:00 2024-04-12T11:02:52.526 INFO:teuthology.orchestra.run.smithi087.stdout:(30/30): librados2-18.2.2-1189.gc96c8d47.el9.x8 7.1 MB/s | 3.3 MB 00:00 2024-04-12T11:02:52.531 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:02:52.531 INFO:teuthology.orchestra.run.smithi087.stdout:Total 8.6 MB/s | 53 MB 00:06 2024-04-12T11:02:52.595 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:02:52.630 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:02:52.630 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:02:53.084 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:02:53.084 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:02:53.825 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-test-2:18.2.2-1189.gc96c8d47.el9.x86_64 6/6 2024-04-12T11:02:53.825 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-test-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/6 2024-04-12T11:02:53.825 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libcephsqlite-2:18.2.2-1189.gc96c8d47.el9.x86_64 2/6 2024-04-12T11:02:53.826 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : jq-1.6-16.el9.x86_64 3/6 2024-04-12T11:02:53.826 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 4/6 2024-04-12T11:02:53.826 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 5/6 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 6/6 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-test-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout: jq-1.6-16.el9.x86_64 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout: libcephsqlite-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:54.270 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:02:54.532 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:02:54.624 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph 2024-04-12T11:02:54.644 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-04-12T11:02:54.681 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-04-12T11:02:54.730 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-04-12T11:02:55.042 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-04-12T11:02:55.088 INFO:teuthology.orchestra.run.smithi087.stdout: Upgrading : librados2-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/32 2024-04-12T11:02:55.179 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librados2-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/32 2024-04-12T11:02:55.238 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:00:45 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:02:55.263 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-rados-2:18.2.2-1189.gc96c8d47.el9.x86_64 6/32 2024-04-12T11:02:55.298 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libcephfs2-2:18.2.2-1189.gc96c8d47.el9.x86_64 7/32 2024-04-12T11:02:55.360 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: libcephfs2-2:18.2.2-1189.gc96c8d47.el9.x86_64 7/32 2024-04-12T11:02:55.398 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-04-12T11:02:55.441 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-ceph-argparse-2:18.2.2-1189.gc96c8d47.el9. 9/32 2024-04-12T11:02:55.506 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cephfs-2:18.2.2-1189.gc96c8d47.el9.x86_64 10/32 2024-04-12T11:02:55.534 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libradosstriper1-2:18.2.2-1189.gc96c8d47.el9.x86_6 11/32 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:02:55.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:02:55.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:02:55.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.552 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.553 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:02:55.554 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.555 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.556 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:02:55.556 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.556 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.556 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.556 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.556 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:02:55.556 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.556 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.556 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.557 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.557 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:02:55.557 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.557 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.557 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.557 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:02:55.609 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: libradosstriper1-2:18.2.2-1189.gc96c8d47.el9.x86_6 11/32 2024-04-12T11:02:55.638 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:02:55.640 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout: ceph x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 6.5 k 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mds x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 2.1 M 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 1.5 M 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-modules-core noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 246 k 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mon x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 4.7 M 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-osd x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 17 M 2024-04-12T11:02:55.641 INFO:teuthology.orchestra.run.smithi029.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-collections 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-12T11:02:55.642 INFO:teuthology.orchestra.run.smithi029.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-12T11:02:55.643 INFO:teuthology.orchestra.run.smithi029.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-12T11:02:55.644 INFO:teuthology.orchestra.run.smithi029.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-12T11:02:55.644 INFO:teuthology.orchestra.run.smithi029.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-12T11:02:55.644 INFO:teuthology.orchestra.run.smithi029.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-12T11:02:55.644 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:55.644 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:02:55.644 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:02:55.644 INFO:teuthology.orchestra.run.smithi029.stdout:Install 33 Packages 2024-04-12T11:02:55.644 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:02:55.645 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 29 M 2024-04-12T11:02:55.645 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 104 M 2024-04-12T11:02:55.645 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:02:55.713 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : re2-1:20211101-3.el9.x86_64 12/32 2024-04-12T11:02:55.758 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libarrow-9.0.0-10.el9.x86_64 13/32 2024-04-12T11:02:55.798 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-04-12T11:02:55.831 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/32 2024-04-12T11:02:55.858 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libarrow-doc-9.0.0-10.el9.noarch 16/32 2024-04-12T11:02:55.894 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-04-12T11:02:55.931 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-04-12T11:02:56.146 INFO:teuthology.orchestra.run.smithi029.stdout:(1/33): ceph-18.2.2-1189.gc96c8d47.el9.x86_64.r 30 kB/s | 6.5 kB 00:00 2024-04-12T11:02:56.226 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-04-12T11:02:56.256 INFO:teuthology.orchestra.run.smithi087.stdout: Upgrading : librbd1-2:18.2.2-1189.gc96c8d47.el9.x86_64 20/32 2024-04-12T11:02:56.346 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librbd1-2:18.2.2-1189.gc96c8d47.el9.x86_64 20/32 2024-04-12T11:02:56.369 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-rbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 21/32 2024-04-12T11:02:56.413 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-04-12T11:02:56.429 INFO:teuthology.orchestra.run.smithi029.stdout:(2/33): ceph-mgr-18.2.2-1189.gc96c8d47.el9.x86_ 2.9 MB/s | 1.5 MB 00:00 2024-04-12T11:02:56.530 INFO:teuthology.orchestra.run.smithi029.stdout:(3/33): ceph-mds-18.2.2-1189.gc96c8d47.el9.x86_ 3.5 MB/s | 2.1 MB 00:00 2024-04-12T11:02:56.647 INFO:teuthology.orchestra.run.smithi029.stdout:(4/33): ceph-mgr-modules-core-18.2.2-1189.gc96c 2.1 MB/s | 246 kB 00:00 2024-04-12T11:02:56.789 INFO:teuthology.orchestra.run.smithi029.stdout:(5/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.7 MB/s | 253 kB 00:00 2024-04-12T11:02:56.812 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : parquet-libs-9.0.0-10.el9.x86_64 23/32 2024-04-12T11:02:56.850 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : librgw2-2:18.2.2-1189.gc96c8d47.el9.x86_64 24/32 2024-04-12T11:02:56.890 INFO:teuthology.orchestra.run.smithi029.stdout:(6/33): python3-cryptography-36.0.1-4.el9.x86_6 12 MB/s | 1.2 MB 00:00 2024-04-12T11:02:56.921 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librgw2-2:18.2.2-1189.gc96c8d47.el9.x86_64 24/32 2024-04-12T11:02:56.932 INFO:teuthology.orchestra.run.smithi029.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-04-12T11:02:56.983 INFO:teuthology.orchestra.run.smithi029.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 2.6 MB/s | 135 kB 00:00 2024-04-12T11:02:56.989 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-rgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 25/32 2024-04-12T11:02:57.019 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-ceph-common-2:18.2.2-1189.gc96c8d47.el9.x8 26/32 2024-04-12T11:02:57.033 INFO:teuthology.orchestra.run.smithi029.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 2.4 MB/s | 125 kB 00:00 2024-04-12T11:02:57.076 INFO:teuthology.orchestra.run.smithi029.stdout:(10/33): python3-urllib3-1.26.5-5.el9.noarch.rp 5.0 MB/s | 215 kB 00:00 2024-04-12T11:02:57.393 INFO:teuthology.orchestra.run.smithi029.stdout:(11/33): ceph-osd-18.2.2-1189.gc96c8d47.el9.x86 17 MB/s | 17 MB 00:00 2024-04-12T11:02:57.685 INFO:teuthology.orchestra.run.smithi029.stdout:(12/33): ceph-mon-18.2.2-1189.gc96c8d47.el9.x86 3.0 MB/s | 4.7 MB 00:01 2024-04-12T11:02:57.710 INFO:teuthology.orchestra.run.smithi029.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 109 kB/s | 35 kB 00:00 2024-04-12T11:02:57.744 INFO:teuthology.orchestra.run.smithi029.stdout:(14/33): python3-mako-1.1.4-6.el9.noarch.rpm 257 kB/s | 172 kB 00:00 2024-04-12T11:02:57.897 INFO:teuthology.orchestra.run.smithi029.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 224 kB/s | 42 kB 00:00 2024-04-12T11:02:57.972 INFO:teuthology.orchestra.run.smithi029.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 191 kB/s | 43 kB 00:00 2024-04-12T11:02:58.014 INFO:teuthology.orchestra.run.smithi029.stdout:(17/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.4 MB/s | 172 kB 00:00 2024-04-12T11:02:58.039 INFO:teuthology.orchestra.run.smithi029.stdout:(18/33): python3-pytz-2021.1-5.el9.noarch.rpm 144 kB/s | 51 kB 00:00 2024-04-12T11:02:58.065 INFO:teuthology.orchestra.run.smithi029.stdout:(19/33): python3-cherrypy-18.6.1-2.el9.noarch.r 3.8 MB/s | 358 kB 00:00 2024-04-12T11:02:58.091 INFO:teuthology.orchestra.run.smithi029.stdout:(20/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 141 kB/s | 11 kB 00:00 2024-04-12T11:02:58.116 INFO:teuthology.orchestra.run.smithi029.stdout:(21/33): python3-jaraco-classes-3.2.1-5.el9.noa 233 kB/s | 18 kB 00:00 2024-04-12T11:02:58.141 INFO:teuthology.orchestra.run.smithi029.stdout:(22/33): python3-jaraco-collections-3.0.0-8.el9 305 kB/s | 23 kB 00:00 2024-04-12T11:02:58.167 INFO:teuthology.orchestra.run.smithi029.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 256 kB/s | 19 kB 00:00 2024-04-12T11:02:58.209 INFO:teuthology.orchestra.run.smithi029.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 292 kB/s | 20 kB 00:00 2024-04-12T11:02:58.235 INFO:teuthology.orchestra.run.smithi029.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 499 kB/s | 46 kB 00:00 2024-04-12T11:02:58.260 INFO:teuthology.orchestra.run.smithi029.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 850 kB/s | 79 kB 00:00 2024-04-12T11:02:58.294 INFO:teuthology.orchestra.run.smithi029.stdout:(27/33): python3-pecan-1.4.2-3.el9.noarch.rpm 3.2 MB/s | 272 kB 00:00 2024-04-12T11:02:58.311 INFO:teuthology.orchestra.run.smithi029.stdout:(28/33): python3-portend-3.1.0-2.el9.noarch.rpm 218 kB/s | 16 kB 00:00 2024-04-12T11:02:58.345 INFO:teuthology.orchestra.run.smithi029.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-04-12T11:02:58.362 INFO:teuthology.orchestra.run.smithi029.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 528 kB/s | 36 kB 00:00 2024-04-12T11:02:58.396 INFO:teuthology.orchestra.run.smithi029.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.7 MB/s | 230 kB 00:00 2024-04-12T11:02:58.421 INFO:teuthology.orchestra.run.smithi029.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 5.5 MB/s | 427 kB 00:00 2024-04-12T11:02:58.448 INFO:teuthology.orchestra.run.smithi029.stdout:(33/33): python3-zc-lockfile-2.0-10.el9.noarch. 233 kB/s | 20 kB 00:00 2024-04-12T11:02:58.452 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:02:58.452 INFO:teuthology.orchestra.run.smithi029.stdout:Total 10 MB/s | 29 MB 00:02 2024-04-12T11:02:58.571 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:02:58.611 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:02:58.611 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:02:59.013 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:02:59.013 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:02:59.052 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 27/32 2024-04-12T11:02:59.091 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 27/32 2024-04-12T11:02:59.619 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 27/32 2024-04-12T11:02:59.653 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-base-2:18.2.2-1189.gc96c8d47.el9.x86_64 28/32 2024-04-12T11:02:59.735 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-base-2:18.2.2-1189.gc96c8d47.el9.x86_64 28/32 2024-04-12T11:02:59.735 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-12T11:02:59.735 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-12T11:02:59.735 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:02:59.770 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-selinux-2:18.2.2-1189.gc96c8d47.el9.x86_64 29/32 2024-04-12T11:03:00.288 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:03:00.360 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-12T11:03:00.394 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-12T11:03:00.435 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-12T11:03:00.531 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-12T11:03:00.609 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-12T11:03:00.659 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-12T11:03:00.702 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-12T11:03:00.741 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-12T11:03:00.776 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-12T11:03:00.810 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-12T11:03:00.860 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-12T11:03:00.903 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-12T11:03:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-12T11:03:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-12T11:03:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-12T11:03:01.214 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-12T11:03:01.303 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-12T11:03:01.363 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-12T11:03:01.410 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-12T11:03:01.470 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-12T11:03:01.540 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-12T11:03:01.744 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-12T11:03:01.811 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-12T11:03:01.891 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-12T11:03:01.991 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-12T11:03:02.037 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-12T11:03:02.150 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-12T11:03:02.322 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-modules-core-2:18.2.2-1189.gc96c8d47.el9. 28/33 2024-04-12T11:03:02.351 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-2:18.2.2-1189.gc96c8d47.el9.x86_64 29/33 2024-04-12T11:03:02.377 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-2:18.2.2-1189.gc96c8d47.el9.x86_64 29/33 2024-04-12T11:03:02.377 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:02.377 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-12T11:03:02.377 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-12T11:03:02.377 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-12T11:03:02.377 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:03.870 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-osd-2:18.2.2-1189.gc96c8d47.el9.x86_64 30/33 2024-04-12T11:03:03.896 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-osd-2:18.2.2-1189.gc96c8d47.el9.x86_64 30/33 2024-04-12T11:03:03.896 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:03.896 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-12T11:03:03.896 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-12T11:03:03.896 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-12T11:03:03.896 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:04.363 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mon-2:18.2.2-1189.gc96c8d47.el9.x86_64 31/33 2024-04-12T11:03:04.386 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mon-2:18.2.2-1189.gc96c8d47.el9.x86_64 31/33 2024-04-12T11:03:04.386 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:04.386 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-12T11:03:04.386 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-12T11:03:04.386 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-12T11:03:04.386 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:04.607 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mds-2:18.2.2-1189.gc96c8d47.el9.x86_64 32/33 2024-04-12T11:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mds-2:18.2.2-1189.gc96c8d47.el9.x86_64 32/33 2024-04-12T11:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-12T11:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-12T11:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-12T11:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-2:18.2.2-1189.gc96c8d47.el9.x86_64 33/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-2:18.2.2-1189.gc96c8d47.el9.x86_64 33/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mds-2:18.2.2-1189.gc96c8d47.el9.x86_64 2/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-2:18.2.2-1189.gc96c8d47.el9.x86_64 3/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mon-2:18.2.2-1189.gc96c8d47.el9.x86_64 4/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-osd-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-modules-core-2:18.2.2-1189.gc96c8d47.el9. 6/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-12T11:03:07.191 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-12T11:03:07.192 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-12T11:03:07.193 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-12T11:03:07.193 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-12T11:03:07.193 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-12T11:03:07.193 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-12T11:03:07.193 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-12T11:03:07.193 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-12T11:03:07.193 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-12T11:03:08.239 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-12T11:03:08.239 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:08.239 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:03:08.239 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:08.239 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mds-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:08.240 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:08.240 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-modules-core-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:08.240 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mon-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:08.240 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-osd-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:08.240 INFO:teuthology.orchestra.run.smithi029.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-12T11:03:08.240 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-12T11:03:08.240 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-12T11:03:08.241 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-12T11:03:08.241 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-12T11:03:08.241 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-12T11:03:08.241 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-12T11:03:08.241 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-12T11:03:08.241 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-12T11:03:08.241 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-12T11:03:08.241 INFO:teuthology.orchestra.run.smithi029.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-12T11:03:08.241 INFO:teuthology.orchestra.run.smithi029.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-12T11:03:08.241 INFO:teuthology.orchestra.run.smithi029.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ply-3.11-14.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-12T11:03:08.242 INFO:teuthology.orchestra.run.smithi029.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-12T11:03:08.243 INFO:teuthology.orchestra.run.smithi029.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-12T11:03:08.243 INFO:teuthology.orchestra.run.smithi029.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-12T11:03:08.243 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:08.244 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:03:08.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-base 2024-04-12T11:03:09.120 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-selinux-2:18.2.2-1189.gc96c8d47.el9.x86_64 29/32 2024-04-12T11:03:09.120 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /sys 2024-04-12T11:03:09.120 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /proc 2024-04-12T11:03:09.120 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /mnt 2024-04-12T11:03:09.120 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /var/tmp 2024-04-12T11:03:09.120 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /home 2024-04-12T11:03:09.120 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /root 2024-04-12T11:03:09.120 INFO:teuthology.orchestra.run.smithi087.stdout:skipping the directory /tmp 2024-04-12T11:03:09.120 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:09.333 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:00:59 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:03:09.673 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:03:09.673 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:03:09.673 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:03:09.673 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.673 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:03:09.674 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.675 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.676 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:03:09.677 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:03:09.678 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.679 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.680 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.681 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:09.682 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.682 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.682 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.682 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:09.682 INFO:teuthology.orchestra.run.smithi029.stdout:Package ceph-base-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:03:09.747 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:03:09.748 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-04-12T11:03:09.748 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:03:09.823 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install cephadm 2024-04-12T11:03:09.858 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-radosgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 30/32 2024-04-12T11:03:09.882 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-radosgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 30/32 2024-04-12T11:03:09.882 INFO:teuthology.orchestra.run.smithi087.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:09.882 INFO:teuthology.orchestra.run.smithi087.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-12T11:03:09.883 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-12T11:03:09.883 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-12T11:03:09.883 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:09.884 INFO:teuthology.orchestra.run.smithi087.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-04-12T11:03:09.994 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-04-12T11:03:09.994 INFO:teuthology.orchestra.run.smithi087.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-04-12T11:03:10.429 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:00 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:03:10.733 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:03:10.733 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:03:10.733 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:03:10.733 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.733 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:03:10.733 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:03:10.734 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.735 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.736 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.737 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.738 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.739 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.741 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:10.742 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.742 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.742 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.742 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.742 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:10.742 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.742 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.742 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.742 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:10.805 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:03:10.806 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:10.806 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-04-12T11:03:10.806 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:10.806 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:03:10.807 INFO:teuthology.orchestra.run.smithi029.stdout: cephadm noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 224 k 2024-04-12T11:03:10.807 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:10.807 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:03:10.807 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:10.807 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-04-12T11:03:10.807 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:10.807 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 224 k 2024-04-12T11:03:10.807 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 216 k 2024-04-12T11:03:10.807 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:03:11.083 INFO:teuthology.orchestra.run.smithi029.stdout:cephadm-18.2.2-1189.gc96c8d47.el9.noarch.rpm 812 kB/s | 224 kB 00:00 2024-04-12T11:03:11.084 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:03:11.084 INFO:teuthology.orchestra.run.smithi029.stdout:Total 807 kB/s | 224 kB 00:00 2024-04-12T11:03:11.084 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:03:11.090 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:03:11.090 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:03:11.092 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:03:11.093 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:03:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-base-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 2/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-radosgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 3/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-selinux-2:18.2.2-1189.gc96c8d47.el9.x86_64 4/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libcephfs2-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libradosstriper1-2:18.2.2-1189.gc96c8d47.el9.x86_6 6/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librgw2-2:18.2.2-1189.gc96c8d47.el9.x86_64 7/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-ceph-argparse-2:18.2.2-1189.gc96c8d47.el9. 8/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-ceph-common-2:18.2.2-1189.gc96c8d47.el9.x8 9/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cephfs-2:18.2.2-1189.gc96c8d47.el9.x86_64 10/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-rados-2:18.2.2-1189.gc96c8d47.el9.x86_64 11/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-rbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 12/32 2024-04-12T11:03:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-rgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 13/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 21/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libarrow-9.0.0-10.el9.x86_64 22/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libarrow-doc-9.0.0-10.el9.noarch 23/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : parquet-libs-9.0.0-10.el9.x86_64 26/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : re2-1:20211101-3.el9.x86_64 27/32 2024-04-12T11:03:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-04-12T11:03:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librados2-2:18.2.2-1189.gc96c8d47.el9.x86_64 29/32 2024-04-12T11:03:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-04-12T11:03:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librbd1-2:18.2.2-1189.gc96c8d47.el9.x86_64 31/32 2024-04-12T11:03:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:03:11.618 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 1/1 2024-04-12T11:03:11.714 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 1/1 2024-04-12T11:03:12.149 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 1/1 2024-04-12T11:03:12.300 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-04-12T11:03:12.300 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:12.300 INFO:teuthology.orchestra.run.smithi087.stdout:Upgraded: 2024-04-12T11:03:12.300 INFO:teuthology.orchestra.run.smithi087.stdout: librados2-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.300 INFO:teuthology.orchestra.run.smithi087.stdout: librbd1-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.300 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:03:12.300 INFO:teuthology.orchestra.run.smithi087.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-base-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-radosgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-selinux-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: fmt-8.1.1-5.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: libarrow-9.0.0-10.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: libarrow-doc-9.0.0-10.el9.noarch 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: libcephfs2-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: libradosstriper1-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.301 INFO:teuthology.orchestra.run.smithi087.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: librgw2-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: parquet-libs-9.0.0-10.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: python3-ceph-argparse-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: python3-ceph-common-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cephfs-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rados-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: re2-1:20211101-3.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-12T11:03:12.302 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:12.303 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:03:12.473 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 1/1 2024-04-12T11:03:12.473 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:12.473 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:03:12.473 INFO:teuthology.orchestra.run.smithi029.stdout: cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:12.473 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:12.474 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:03:12.664 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-test 2024-04-12T11:03:12.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-immutable-object-cache 2024-04-12T11:03:13.258 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:00:30 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:03:13.310 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:03 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:03:13.543 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:03:13.543 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:03:13.543 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:03:13.543 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.543 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:03:13.543 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.543 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.544 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.545 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.546 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:03:13.547 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.548 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.549 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:13.550 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.551 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.615 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:03:13.615 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:03:13.615 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:03:13.616 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.617 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.618 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.619 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.620 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.621 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.622 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.622 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.622 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:13.622 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.622 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.622 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.622 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.622 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:13.622 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.622 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.623 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.623 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.623 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:13.623 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.623 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.623 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.623 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.623 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:13.623 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.623 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.624 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2024-04-12T11:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-test x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 40 M 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: libcephsqlite x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 166 k 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: oniguruma x86_64 6.9.6-1.el9.5 baseos 218 k 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout:Install 6 Packages 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 41 M 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 164 M 2024-04-12T11:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:03:13.688 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:03:13.689 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:13.689 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2024-04-12T11:03:13.689 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:13.689 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:03:13.689 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-immutable-object-cache x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 143 k 2024-04-12T11:03:13.689 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:13.690 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:03:13.690 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:13.690 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-04-12T11:03:13.690 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:13.690 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 143 k 2024-04-12T11:03:13.690 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 395 k 2024-04-12T11:03:13.690 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:03:13.908 INFO:teuthology.orchestra.run.smithi029.stdout:ceph-immutable-object-cache-18.2.2-1189.gc96c8d 653 kB/s | 143 kB 00:00 2024-04-12T11:03:13.909 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:03:13.909 INFO:teuthology.orchestra.run.smithi029.stdout:Total 649 kB/s | 143 kB 00:00 2024-04-12T11:03:13.909 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:03:13.918 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:03:13.918 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:03:13.973 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:03:13.973 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:03:14.124 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:03:14.185 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-immutable-object-cache-2:18.2.2-1189.gc96c8d47. 1/1 2024-04-12T11:03:14.210 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.2-1189.gc96c8d47. 1/1 2024-04-12T11:03:14.210 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:14.210 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-12T11:03:14.210 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:14.335 INFO:teuthology.orchestra.run.smithi087.stdout:(1/6): jq-1.6-16.el9.x86_64.rpm 1.2 MB/s | 187 kB 00:00 2024-04-12T11:03:14.402 INFO:teuthology.orchestra.run.smithi087.stdout:(2/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 3.2 MB/s | 218 kB 00:00 2024-04-12T11:03:14.536 INFO:teuthology.orchestra.run.smithi087.stdout:(3/6): libcephsqlite-18.2.2-1189.gc96c8d47.el9. 464 kB/s | 166 kB 00:00 2024-04-12T11:03:14.644 INFO:teuthology.orchestra.run.smithi087.stdout:(4/6): socat-1.7.4.1-5.el9.x86_64.rpm 1.2 MB/s | 305 kB 00:00 2024-04-12T11:03:14.719 INFO:teuthology.orchestra.run.smithi087.stdout:(5/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 347 kB/s | 64 kB 00:00 2024-04-12T11:03:15.239 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-immutable-object-cache-2:18.2.2-1189.gc96c8d47. 1/1 2024-04-12T11:03:15.239 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:15.239 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:03:15.239 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-immutable-object-cache-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:15.239 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:15.239 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:03:15.537 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr 2024-04-12T11:03:16.143 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:06 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:03:16.331 INFO:teuthology.orchestra.run.smithi087.stdout:(6/6): ceph-test-18.2.2-1189.gc96c8d47.el9.x86_ 18 MB/s | 40 MB 00:02 2024-04-12T11:03:16.470 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:03:16.470 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:03:16.470 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:03:16.470 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.470 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.471 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:03:16.472 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.472 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:03:16.472 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.472 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:03:16.472 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.472 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:03:16.472 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.472 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:03:16.472 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.472 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:03:16.473 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.473 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:03:16.473 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.473 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:03:16.473 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.473 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:03:16.473 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.473 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:03:16.474 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.474 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:03:16.474 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.474 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:03:16.474 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.474 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:03:16.474 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.475 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:03:16.475 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.475 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:03:16.475 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.475 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:03:16.475 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.475 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:03:16.475 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.475 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:03:16.476 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.477 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.478 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.479 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.480 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.481 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.481 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:16.481 INFO:teuthology.orchestra.run.smithi029.stdout:Package ceph-mgr-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:03:16.482 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:03:16.482 INFO:teuthology.orchestra.run.smithi087.stdout:Total 15 MB/s | 41 MB 00:02 2024-04-12T11:03:16.482 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:03:16.482 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:03:16.482 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:03:16.516 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:03:16.517 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-04-12T11:03:16.517 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:03:16.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-dashboard 2024-04-12T11:03:16.741 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:03:16.741 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:03:17.195 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:07 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:03:17.268 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:03:17.346 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-12T11:03:17.392 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-12T11:03:17.427 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-12T11:03:17.469 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-12T11:03:17.497 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libcephsqlite-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/6 2024-04-12T11:03:17.498 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:03:17.498 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.499 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:03:17.500 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.501 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:03:17.502 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:03:17.503 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.504 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.505 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.506 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.507 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.507 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.507 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:17.507 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.507 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.507 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.507 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:17.571 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:03:17.573 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:17.573 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-04-12T11:03:17.573 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:17.573 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:03:17.573 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-dashboard noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 3.5 M 2024-04-12T11:03:17.573 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-04-12T11:03:17.573 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-grafana-dashboards noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 24 k 2024-04-12T11:03:17.573 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-prometheus-alerts noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 15 k 2024-04-12T11:03:17.573 INFO:teuthology.orchestra.run.smithi029.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-12T11:03:17.573 INFO:teuthology.orchestra.run.smithi029.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-12T11:03:17.574 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:17.574 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:03:17.574 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:17.574 INFO:teuthology.orchestra.run.smithi029.stdout:Install 5 Packages 2024-04-12T11:03:17.574 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:17.574 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 3.8 M 2024-04-12T11:03:17.574 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 87 M 2024-04-12T11:03:17.574 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:03:18.188 INFO:teuthology.orchestra.run.smithi029.stdout:(1/5): ceph-prometheus-alerts-18.2.2-1189.gc96c 55 kB/s | 15 kB 00:00 2024-04-12T11:03:18.213 INFO:teuthology.orchestra.run.smithi029.stdout:(2/5): ceph-grafana-dashboards-18.2.2-1189.gc96 83 kB/s | 24 kB 00:00 2024-04-12T11:03:18.397 INFO:teuthology.orchestra.run.smithi029.stdout:(3/5): ceph-mgr-dashboard-18.2.2-1189.gc96c8d47 7.4 MB/s | 3.5 MB 00:00 2024-04-12T11:03:18.539 INFO:teuthology.orchestra.run.smithi029.stdout:(4/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 88 kB/s | 31 kB 00:00 2024-04-12T11:03:18.656 INFO:teuthology.orchestra.run.smithi029.stdout:(5/5): python3-routes-2.5.1-5.el9.noarch.rpm 426 kB/s | 188 kB 00:00 2024-04-12T11:03:18.659 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:03:18.660 INFO:teuthology.orchestra.run.smithi029.stdout:Total 3.5 MB/s | 3.8 MB 00:01 2024-04-12T11:03:18.660 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:03:18.668 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:03:18.668 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:03:18.740 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:03:18.740 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:03:18.993 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:03:19.081 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-04-12T11:03:19.122 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-04-12T11:03:19.160 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-prometheus-alerts-2:18.2.2-1189.gc96c8d47.el9.n 3/5 2024-04-12T11:03:20.269 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-grafana-dashboards-2:18.2.2-1189.gc96c8d47.el9. 4/5 2024-04-12T11:03:20.315 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-dashboard-2:18.2.2-1189.gc96c8d47.el9.noarc 5/5 2024-04-12T11:03:21.236 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: libcephsqlite-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/6 2024-04-12T11:03:21.314 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-test-2:18.2.2-1189.gc96c8d47.el9.x86_64 6/6 2024-04-12T11:03:22.035 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.2-1189.gc96c8d47.el9.noarc 5/5 2024-04-12T11:03:22.036 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-grafana-dashboards-2:18.2.2-1189.gc96c8d47.el9. 1/5 2024-04-12T11:03:22.036 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-dashboard-2:18.2.2-1189.gc96c8d47.el9.noarc 2/5 2024-04-12T11:03:22.036 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-prometheus-alerts-2:18.2.2-1189.gc96c8d47.el9.n 3/5 2024-04-12T11:03:22.036 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-04-12T11:03:22.357 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-04-12T11:03:22.358 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:22.358 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:03:22.358 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-grafana-dashboards-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:22.358 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-dashboard-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:22.358 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-prometheus-alerts-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:22.358 INFO:teuthology.orchestra.run.smithi029.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-12T11:03:22.358 INFO:teuthology.orchestra.run.smithi029.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-12T11:03:22.358 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:22.358 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:03:22.634 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-12T11:03:23.240 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:13 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:03:23.355 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-test-2:18.2.2-1189.gc96c8d47.el9.x86_64 6/6 2024-04-12T11:03:23.355 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-test-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/6 2024-04-12T11:03:23.355 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libcephsqlite-2:18.2.2-1189.gc96c8d47.el9.x86_64 2/6 2024-04-12T11:03:23.355 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : jq-1.6-16.el9.x86_64 3/6 2024-04-12T11:03:23.356 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 4/6 2024-04-12T11:03:23.356 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 5/6 2024-04-12T11:03:23.543 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:03:23.544 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:03:23.545 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.546 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.547 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.548 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.549 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.550 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.551 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.552 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:23.620 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:03:23.621 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================= 2024-04-12T11:03:23.621 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-04-12T11:03:23.621 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================= 2024-04-12T11:03:23.621 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:03:23.621 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 7.4 M 2024-04-12T11:03:23.621 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-04-12T11:03:23.621 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================= 2024-04-12T11:03:23.622 INFO:teuthology.orchestra.run.smithi029.stdout:Install 12 Packages 2024-04-12T11:03:23.623 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:23.623 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 38 M 2024-04-12T11:03:23.623 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 209 M 2024-04-12T11:03:23.623 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:03:23.780 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 6/6 2024-04-12T11:03:23.781 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:23.781 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:03:23.781 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-test-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:23.781 INFO:teuthology.orchestra.run.smithi087.stdout: jq-1.6-16.el9.x86_64 2024-04-12T11:03:23.781 INFO:teuthology.orchestra.run.smithi087.stdout: libcephsqlite-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:23.781 INFO:teuthology.orchestra.run.smithi087.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-12T11:03:23.781 INFO:teuthology.orchestra.run.smithi087.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-12T11:03:23.781 INFO:teuthology.orchestra.run.smithi087.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-12T11:03:23.781 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:23.781 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:03:24.072 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph 2024-04-12T11:03:24.464 INFO:teuthology.orchestra.run.smithi029.stdout:(1/12): ceph-mgr-diskprediction-local-18.2.2-11 11 MB/s | 7.4 MB 00:00 2024-04-12T11:03:24.670 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:00:41 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:03:24.732 INFO:teuthology.orchestra.run.smithi029.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 123 kB/s | 33 kB 00:00 2024-04-12T11:03:24.799 INFO:teuthology.orchestra.run.smithi029.stdout:(3/12): libquadmath-11.4.1-3.el9.x86_64.rpm 196 kB/s | 195 kB 00:00 2024-04-12T11:03:24.958 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:03:24.959 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.960 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.961 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.962 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.963 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:03:24.964 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.965 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.966 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.967 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.967 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.967 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:24.967 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.967 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.967 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:24.967 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:25.034 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:03:25.036 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:25.036 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: ceph x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 6.5 k 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mds x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 2.1 M 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 1.5 M 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-modules-core noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 246 k 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mon x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 4.7 M 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-osd x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 17 M 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-12T11:03:25.037 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-collections 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-12T11:03:25.038 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout:Install 33 Packages 2024-04-12T11:03:25.039 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:25.040 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 29 M 2024-04-12T11:03:25.040 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 104 M 2024-04-12T11:03:25.040 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:03:25.066 INFO:teuthology.orchestra.run.smithi029.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 68 kB/s | 18 kB 00:00 2024-04-12T11:03:25.250 INFO:teuthology.orchestra.run.smithi029.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 192 kB/s | 35 kB 00:00 2024-04-12T11:03:25.409 INFO:teuthology.orchestra.run.smithi029.stdout:(6/12): libgfortran-11.4.1-3.el9.x86_64.rpm 500 kB/s | 803 kB 00:01 2024-04-12T11:03:25.759 INFO:teuthology.orchestra.run.smithi029.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.9 MB/s | 3.0 MB 00:01 2024-04-12T11:03:25.955 INFO:teuthology.orchestra.run.smithi087.stdout:(1/33): ceph-18.2.2-1189.gc96c8d47.el9.x86_64.r 28 kB/s | 6.5 kB 00:00 2024-04-12T11:03:26.002 INFO:teuthology.orchestra.run.smithi029.stdout:(8/12): python3-devel-3.9.17-2.el9.x86_64.rpm 414 kB/s | 245 kB 00:00 2024-04-12T11:03:26.122 INFO:teuthology.orchestra.run.smithi087.stdout:(2/33): ceph-mds-18.2.2-1189.gc96c8d47.el9.x86_ 5.2 MB/s | 2.1 MB 00:00 2024-04-12T11:03:26.203 INFO:teuthology.orchestra.run.smithi029.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 1.2 MB/s | 250 kB 00:00 2024-04-12T11:03:26.255 INFO:teuthology.orchestra.run.smithi087.stdout:(3/33): ceph-mgr-18.2.2-1189.gc96c8d47.el9.x86_ 2.7 MB/s | 1.5 MB 00:00 2024-04-12T11:03:26.356 INFO:teuthology.orchestra.run.smithi087.stdout:(4/33): ceph-mgr-modules-core-18.2.2-1189.gc96c 2.4 MB/s | 246 kB 00:00 2024-04-12T11:03:26.531 INFO:teuthology.orchestra.run.smithi087.stdout:(5/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.4 MB/s | 253 kB 00:00 2024-04-12T11:03:26.579 INFO:teuthology.orchestra.run.smithi029.stdout:(10/12): openblas-openmp-0.3.21-2.el9.x86_64.rp 3.6 MB/s | 4.7 MB 00:01 2024-04-12T11:03:26.798 INFO:teuthology.orchestra.run.smithi087.stdout:(6/33): python3-cryptography-36.0.1-4.el9.x86_6 4.6 MB/s | 1.2 MB 00:00 2024-04-12T11:03:26.858 INFO:teuthology.orchestra.run.smithi087.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 1.8 MB/s | 106 kB 00:00 2024-04-12T11:03:27.040 INFO:teuthology.orchestra.run.smithi087.stdout:(8/33): ceph-osd-18.2.2-1189.gc96c8d47.el9.x86_ 18 MB/s | 17 MB 00:00 2024-04-12T11:03:27.107 INFO:teuthology.orchestra.run.smithi087.stdout:(9/33): ceph-mon-18.2.2-1189.gc96c8d47.el9.x86_ 4.1 MB/s | 4.7 MB 00:01 2024-04-12T11:03:27.133 INFO:teuthology.orchestra.run.smithi087.stdout:(10/33): python3-pycparser-2.20-6.el9.noarch.rp 490 kB/s | 135 kB 00:00 2024-04-12T11:03:27.250 INFO:teuthology.orchestra.run.smithi087.stdout:(11/33): python3-urllib3-1.26.5-5.el9.noarch.rp 1.8 MB/s | 215 kB 00:00 2024-04-12T11:03:27.254 INFO:teuthology.orchestra.run.smithi029.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 3.4 MB/s | 5.1 MB 00:01 2024-04-12T11:03:27.266 INFO:teuthology.orchestra.run.smithi087.stdout:(12/33): python3-requests-2.25.1-8.el9.noarch.r 554 kB/s | 125 kB 00:00 2024-04-12T11:03:27.392 INFO:teuthology.orchestra.run.smithi087.stdout:(13/33): python3-mako-1.1.4-6.el9.noarch.rpm 666 kB/s | 172 kB 00:00 2024-04-12T11:03:27.416 INFO:teuthology.orchestra.run.smithi087.stdout:(14/33): python3-markupsafe-1.1.1-12.el9.x86_64 208 kB/s | 35 kB 00:00 2024-04-12T11:03:27.458 INFO:teuthology.orchestra.run.smithi087.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 621 kB/s | 42 kB 00:00 2024-04-12T11:03:27.484 INFO:teuthology.orchestra.run.smithi087.stdout:(16/33): python3-pytz-2021.1-5.el9.noarch.rpm 234 kB/s | 51 kB 00:00 2024-04-12T11:03:27.817 INFO:teuthology.orchestra.run.smithi087.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 108 kB/s | 43 kB 00:00 2024-04-12T11:03:27.926 INFO:teuthology.orchestra.run.smithi087.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 99 kB/s | 11 kB 00:00 2024-04-12T11:03:28.001 INFO:teuthology.orchestra.run.smithi087.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 317 kB/s | 172 kB 00:00 2024-04-12T11:03:28.034 INFO:teuthology.orchestra.run.smithi087.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 163 kB/s | 18 kB 00:00 2024-04-12T11:03:28.093 INFO:teuthology.orchestra.run.smithi087.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 252 kB/s | 23 kB 00:00 2024-04-12T11:03:28.118 INFO:teuthology.orchestra.run.smithi087.stdout:(22/33): python3-cherrypy-18.6.1-2.el9.noarch.r 565 kB/s | 358 kB 00:00 2024-04-12T11:03:28.152 INFO:teuthology.orchestra.run.smithi087.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 166 kB/s | 19 kB 00:00 2024-04-12T11:03:28.194 INFO:teuthology.orchestra.run.smithi087.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 196 kB/s | 20 kB 00:00 2024-04-12T11:03:28.227 INFO:teuthology.orchestra.run.smithi087.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 426 kB/s | 46 kB 00:00 2024-04-12T11:03:28.328 INFO:teuthology.orchestra.run.smithi087.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 164 kB/s | 16 kB 00:00 2024-04-12T11:03:28.353 INFO:teuthology.orchestra.run.smithi087.stdout:(27/33): python3-more-itertools-8.12.0-2.el9.no 393 kB/s | 79 kB 00:00 2024-04-12T11:03:28.378 INFO:teuthology.orchestra.run.smithi087.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.4 MB/s | 272 kB 00:00 2024-04-12T11:03:28.437 INFO:teuthology.orchestra.run.smithi087.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 826 kB/s | 90 kB 00:00 2024-04-12T11:03:28.470 INFO:teuthology.orchestra.run.smithi087.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 305 kB/s | 36 kB 00:00 2024-04-12T11:03:28.504 INFO:teuthology.orchestra.run.smithi087.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.8 MB/s | 230 kB 00:00 2024-04-12T11:03:28.579 INFO:teuthology.orchestra.run.smithi087.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 184 kB/s | 20 kB 00:00 2024-04-12T11:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.5 MB/s | 427 kB 00:00 2024-04-12T11:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout:Total 8.2 MB/s | 29 MB 00:03 2024-04-12T11:03:28.733 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:03:28.770 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:03:28.770 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:03:29.121 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:03:29.122 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:03:30.451 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:03:30.532 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-12T11:03:30.565 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-12T11:03:30.597 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-12T11:03:30.688 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-12T11:03:30.759 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-12T11:03:30.800 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-12T11:03:30.835 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-12T11:03:30.865 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-12T11:03:30.899 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-12T11:03:30.933 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-12T11:03:30.973 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-12T11:03:31.008 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-12T11:03:31.042 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-12T11:03:31.080 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-12T11:03:31.152 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-12T11:03:31.290 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-12T11:03:31.372 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-12T11:03:31.433 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-12T11:03:31.473 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-12T11:03:31.520 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-12T11:03:31.584 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-12T11:03:31.769 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-12T11:03:31.825 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-12T11:03:31.911 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-12T11:03:32.002 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-12T11:03:32.041 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-12T11:03:32.159 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-12T11:03:32.309 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-modules-core-2:18.2.2-1189.gc96c8d47.el9. 28/33 2024-04-12T11:03:32.335 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-2:18.2.2-1189.gc96c8d47.el9.x86_64 29/33 2024-04-12T11:03:32.360 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mgr-2:18.2.2-1189.gc96c8d47.el9.x86_64 29/33 2024-04-12T11:03:32.360 INFO:teuthology.orchestra.run.smithi087.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:32.360 INFO:teuthology.orchestra.run.smithi087.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-12T11:03:32.360 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-12T11:03:32.360 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-12T11:03:32.360 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:33.769 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-osd-2:18.2.2-1189.gc96c8d47.el9.x86_64 30/33 2024-04-12T11:03:33.796 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-osd-2:18.2.2-1189.gc96c8d47.el9.x86_64 30/33 2024-04-12T11:03:33.796 INFO:teuthology.orchestra.run.smithi087.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:33.796 INFO:teuthology.orchestra.run.smithi087.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-12T11:03:33.797 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-12T11:03:33.797 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-12T11:03:33.797 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:34.236 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mon-2:18.2.2-1189.gc96c8d47.el9.x86_64 31/33 2024-04-12T11:03:34.260 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mon-2:18.2.2-1189.gc96c8d47.el9.x86_64 31/33 2024-04-12T11:03:34.260 INFO:teuthology.orchestra.run.smithi087.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:34.260 INFO:teuthology.orchestra.run.smithi087.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-12T11:03:34.260 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-12T11:03:34.261 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-12T11:03:34.261 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:34.454 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mds-2:18.2.2-1189.gc96c8d47.el9.x86_64 32/33 2024-04-12T11:03:34.477 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mds-2:18.2.2-1189.gc96c8d47.el9.x86_64 32/33 2024-04-12T11:03:34.477 INFO:teuthology.orchestra.run.smithi087.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:34.478 INFO:teuthology.orchestra.run.smithi087.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-12T11:03:34.478 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-12T11:03:34.478 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-12T11:03:34.478 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:34.543 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-2:18.2.2-1189.gc96c8d47.el9.x86_64 33/33 2024-04-12T11:03:36.223 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-2:18.2.2-1189.gc96c8d47.el9.x86_64 33/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mds-2:18.2.2-1189.gc96c8d47.el9.x86_64 2/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-2:18.2.2-1189.gc96c8d47.el9.x86_64 3/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mon-2:18.2.2-1189.gc96c8d47.el9.x86_64 4/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-osd-2:18.2.2-1189.gc96c8d47.el9.x86_64 5/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-modules-core-2:18.2.2-1189.gc96c8d47.el9. 6/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-12T11:03:36.224 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-12T11:03:36.225 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-12T11:03:36.226 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-12T11:03:36.226 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-12T11:03:36.226 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-12T11:03:36.226 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-12T11:03:37.120 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-12T11:03:37.120 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:37.120 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:03:37.120 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:37.120 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mds-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:37.120 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:37.120 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-modules-core-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:37.120 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mon-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:37.120 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-osd-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:37.120 INFO:teuthology.orchestra.run.smithi087.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-12T11:03:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: python3-ply-3.11-14.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:37.122 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:03:37.435 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-base 2024-04-12T11:03:38.030 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:00:55 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:03:38.317 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:03:38.317 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:03:38.317 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:03:38.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.317 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.318 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.319 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:03:38.320 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:03:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:03:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.326 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:38.326 INFO:teuthology.orchestra.run.smithi087.stdout:Package ceph-base-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:03:38.384 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:03:38.385 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2024-04-12T11:03:38.386 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:03:38.455 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install cephadm 2024-04-12T11:03:39.045 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:00:56 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:03:39.330 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:03:39.330 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.331 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:03:39.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.333 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:03:39.334 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.335 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.336 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.337 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.337 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.337 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.337 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.337 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:39.337 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.337 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.338 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.339 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.340 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:39.340 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.340 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.340 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.340 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:39.398 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout: cephadm noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 224 k 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 224 k 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 216 k 2024-04-12T11:03:39.399 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:03:39.660 INFO:teuthology.orchestra.run.smithi087.stdout:cephadm-18.2.2-1189.gc96c8d47.el9.noarch.rpm 862 kB/s | 224 kB 00:00 2024-04-12T11:03:39.660 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:03:39.660 INFO:teuthology.orchestra.run.smithi087.stdout:Total 858 kB/s | 224 kB 00:00 2024-04-12T11:03:39.660 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:03:39.665 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:03:39.665 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:03:39.668 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:03:39.668 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:03:39.737 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:03:40.151 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 1/1 2024-04-12T11:03:40.236 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 1/1 2024-04-12T11:03:40.624 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 1/1 2024-04-12T11:03:40.909 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 1/1 2024-04-12T11:03:40.909 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:40.909 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:03:40.909 INFO:teuthology.orchestra.run.smithi087.stdout: cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:40.909 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:40.909 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:03:41.099 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-immutable-object-cache 2024-04-12T11:03:41.704 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:00:58 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:03:41.998 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:03:41.998 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:03:41.998 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:03:41.998 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:41.999 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:03:42.000 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:03:42.001 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.002 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.003 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:42.004 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.005 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:42.006 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.007 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.007 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.007 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:42.066 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:03:42.066 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:42.066 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repo Size 2024-04-12T11:03:42.066 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:42.066 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:03:42.066 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-immutable-object-cache x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 143 k 2024-04-12T11:03:42.066 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:42.067 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:03:42.067 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:42.067 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2024-04-12T11:03:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:42.067 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 143 k 2024-04-12T11:03:42.068 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 395 k 2024-04-12T11:03:42.068 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:03:42.287 INFO:teuthology.orchestra.run.smithi087.stdout:ceph-immutable-object-cache-18.2.2-1189.gc96c8d 650 kB/s | 143 kB 00:00 2024-04-12T11:03:42.287 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:03:42.288 INFO:teuthology.orchestra.run.smithi087.stdout:Total 646 kB/s | 143 kB 00:00 2024-04-12T11:03:42.288 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:03:42.295 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:03:42.298 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:03:42.348 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:03:42.348 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:03:42.488 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:03:42.555 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-immutable-object-cache-2:18.2.2-1189.gc96c8d47. 1/1 2024-04-12T11:03:42.579 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.2-1189.gc96c8d47. 1/1 2024-04-12T11:03:42.579 INFO:teuthology.orchestra.run.smithi087.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:03:42.580 INFO:teuthology.orchestra.run.smithi087.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-12T11:03:42.580 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:43.536 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-immutable-object-cache-2:18.2.2-1189.gc96c8d47. 1/1 2024-04-12T11:03:43.536 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:43.536 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:03:43.536 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-immutable-object-cache-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:03:43.536 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:43.536 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:03:43.771 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-mgr 2024-04-12T11:03:44.371 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:01 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:03:44.403 INFO:teuthology.orchestra.run.smithi029.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 914 kB/s | 16 MB 00:18 2024-04-12T11:03:44.404 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:03:44.405 INFO:teuthology.orchestra.run.smithi029.stdout:Total 1.8 MB/s | 38 MB 00:20 2024-04-12T11:03:44.658 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:03:44.658 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:03:44.658 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:03:44.658 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.658 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:03:44.658 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.658 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.659 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:03:44.660 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:03:44.661 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.662 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.663 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.664 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.665 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.666 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:44.667 INFO:teuthology.orchestra.run.smithi087.stdout:Package ceph-mgr-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:03:44.727 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:03:44.728 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2024-04-12T11:03:44.728 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:03:44.766 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:03:44.795 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:03:44.795 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:03:44.809 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-mgr-dashboard 2024-04-12T11:03:45.333 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:03:45.333 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:03:45.394 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:02 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:03:45.679 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:03:45.679 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:03:45.679 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:03:45.680 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:03:45.681 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:03:45.682 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.683 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.684 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.685 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:45.686 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.687 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:45.688 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.688 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.688 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.688 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.688 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:45.688 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.688 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.688 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.688 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:45.747 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:03:45.747 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-dashboard noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 3.5 M 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-grafana-dashboards noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 24 k 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-prometheus-alerts noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 15 k 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:03:45.748 INFO:teuthology.orchestra.run.smithi087.stdout:Install 5 Packages 2024-04-12T11:03:45.749 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:45.749 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 3.8 M 2024-04-12T11:03:45.749 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 87 M 2024-04-12T11:03:45.749 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:03:46.119 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:03:46.157 INFO:teuthology.orchestra.run.smithi087.stdout:(1/5): ceph-grafana-dashboards-18.2.2-1189.gc96 98 kB/s | 24 kB 00:00 2024-04-12T11:03:46.174 INFO:teuthology.orchestra.run.smithi087.stdout:(2/5): ceph-prometheus-alerts-18.2.2-1189.gc96c 56 kB/s | 15 kB 00:00 2024-04-12T11:03:46.182 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-12T11:03:46.236 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-12T11:03:46.342 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-12T11:03:46.389 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-12T11:03:46.482 INFO:teuthology.orchestra.run.smithi087.stdout:(3/5): ceph-mgr-dashboard-18.2.2-1189.gc96c8d47 6.2 MB/s | 3.5 MB 00:00 2024-04-12T11:03:46.516 INFO:teuthology.orchestra.run.smithi087.stdout:(4/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 86 kB/s | 31 kB 00:00 2024-04-12T11:03:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-12T11:03:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-12T11:03:46.715 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-12T11:03:46.750 INFO:teuthology.orchestra.run.smithi087.stdout:(5/5): python3-routes-2.5.1-5.el9.noarch.rpm 328 kB/s | 188 kB 00:00 2024-04-12T11:03:46.752 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:03:46.752 INFO:teuthology.orchestra.run.smithi087.stdout:Total 3.8 MB/s | 3.8 MB 00:01 2024-04-12T11:03:46.752 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:03:46.759 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:03:46.759 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:03:46.834 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:03:46.835 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:03:47.088 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:03:47.167 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-04-12T11:03:47.197 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-04-12T11:03:47.224 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-prometheus-alerts-2:18.2.2-1189.gc96c8d47.el9.n 3/5 2024-04-12T11:03:47.410 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-12T11:03:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-12T11:03:48.275 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-grafana-dashboards-2:18.2.2-1189.gc96c8d47.el9. 4/5 2024-04-12T11:03:48.311 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-dashboard-2:18.2.2-1189.gc96c8d47.el9.noarc 5/5 2024-04-12T11:03:48.825 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-12T11:03:49.431 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.2-1189.gc96c8d47.el9.noarc 5/5 2024-04-12T11:03:49.431 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-grafana-dashboards-2:18.2.2-1189.gc96c8d47.el9. 1/5 2024-04-12T11:03:49.432 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-dashboard-2:18.2.2-1189.gc96c8d47.el9.noarc 2/5 2024-04-12T11:03:49.432 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-prometheus-alerts-2:18.2.2-1189.gc96c8d47.el9.n 3/5 2024-04-12T11:03:49.432 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-04-12T11:03:49.749 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-04-12T11:03:49.749 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:49.749 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:03:49.749 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-grafana-dashboards-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:49.749 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-dashboard-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:49.749 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-prometheus-alerts-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:49.749 INFO:teuthology.orchestra.run.smithi087.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-12T11:03:49.749 INFO:teuthology.orchestra.run.smithi087.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-12T11:03:49.749 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:49.749 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:03:49.932 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-12T11:03:49.962 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.2-1189.gc96c8 12/12 2024-04-12T11:03:49.981 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-12T11:03:50.577 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:07 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:03:50.872 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:03:50.872 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:03:50.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:03:50.872 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:03:50.872 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:03:50.872 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:03:50.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.874 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:03:50.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:03:50.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.877 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.878 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.879 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.880 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.881 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.881 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:50.881 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.881 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.881 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.881 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:50.941 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:03:50.942 INFO:teuthology.orchestra.run.smithi087.stdout:======================================================================================= 2024-04-12T11:03:50.942 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2024-04-12T11:03:50.942 INFO:teuthology.orchestra.run.smithi087.stdout:======================================================================================= 2024-04-12T11:03:50.942 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:03:50.942 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 7.4 M 2024-04-12T11:03:50.942 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2024-04-12T11:03:50.942 INFO:teuthology.orchestra.run.smithi087.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-12T11:03:50.942 INFO:teuthology.orchestra.run.smithi087.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout:======================================================================================= 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout:Install 12 Packages 2024-04-12T11:03:50.943 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:03:50.944 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 38 M 2024-04-12T11:03:50.944 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 209 M 2024-04-12T11:03:50.944 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:03:51.356 INFO:teuthology.orchestra.run.smithi087.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.1 MB/s | 195 kB 00:00 2024-04-12T11:03:51.423 INFO:teuthology.orchestra.run.smithi087.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 3.3 MB/s | 803 kB 00:00 2024-04-12T11:03:51.632 INFO:teuthology.orchestra.run.smithi087.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 119 kB/s | 33 kB 00:00 2024-04-12T11:03:51.749 INFO:teuthology.orchestra.run.smithi087.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 156 kB/s | 18 kB 00:00 2024-04-12T11:03:51.832 INFO:teuthology.orchestra.run.smithi087.stdout:(5/12): ceph-mgr-diskprediction-local-18.2.2-11 11 MB/s | 7.4 MB 00:00 2024-04-12T11:03:51.883 INFO:teuthology.orchestra.run.smithi087.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 263 kB/s | 35 kB 00:00 2024-04-12T11:03:52.159 INFO:teuthology.orchestra.run.smithi087.stdout:(7/12): python3-devel-3.9.17-2.el9.x86_64.rpm 892 kB/s | 245 kB 00:00 2024-04-12T11:03:52.575 INFO:teuthology.orchestra.run.smithi087.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.6 MB/s | 3.0 MB 00:01 2024-04-12T11:03:52.695 INFO:teuthology.orchestra.run.smithi087.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.1 MB/s | 250 kB 00:00 2024-04-12T11:03:52.775 INFO:teuthology.orchestra.run.smithi087.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 8.3 MB/s | 5.1 MB 00:00 2024-04-12T11:03:53.001 INFO:teuthology.orchestra.run.smithi087.stdout:(11/12): openblas-openmp-0.3.21-2.el9.x86_64.rp 4.1 MB/s | 4.7 MB 00:01 2024-04-12T11:03:54.158 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.2-1189.gc96c8 12/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.2-1189.gc96c8 1/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 9/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-12T11:03:54.159 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-diskprediction-local-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-12T11:03:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-12T11:03:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-12T11:03:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-12T11:03:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-12T11:03:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-12T11:03:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:54.665 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:03:55.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-rook 2024-04-12T11:03:55.760 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:45 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:03:56.063 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:03:56.063 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:03:56.064 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:03:56.065 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:03:56.066 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:03:56.067 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:03:56.068 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.069 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.070 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.071 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.072 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:03:56.072 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.072 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.072 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.072 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:03:56.136 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-rook noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 49 k 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-12T11:03:56.138 INFO:teuthology.orchestra.run.smithi029.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout:Install 13 Packages 2024-04-12T11:03:56.139 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:03:56.140 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 2.2 M 2024-04-12T11:03:56.140 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 26 M 2024-04-12T11:03:56.140 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:03:56.720 INFO:teuthology.orchestra.run.smithi029.stdout:(1/13): ceph-mgr-rook-18.2.2-1189.gc96c8d47.el9 242 kB/s | 49 kB 00:00 2024-04-12T11:03:56.778 INFO:teuthology.orchestra.run.smithi029.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 98 kB/s | 26 kB 00:00 2024-04-12T11:03:56.796 INFO:teuthology.orchestra.run.smithi029.stdout:(3/13): python3-jsonpointer-2.0-4.el9.noarch.rp 69 kB/s | 19 kB 00:00 2024-04-12T11:03:57.129 INFO:teuthology.orchestra.run.smithi029.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 452 kB/s | 159 kB 00:00 2024-04-12T11:03:57.230 INFO:teuthology.orchestra.run.smithi029.stdout:(5/13): python3-requests-oauthlib-1.3.0-12.el9. 536 kB/s | 54 kB 00:00 2024-04-12T11:03:57.265 INFO:teuthology.orchestra.run.smithi029.stdout:(6/13): python3-pyasn1-modules-0.4.8-6.el9.noar 594 kB/s | 279 kB 00:00 2024-04-12T11:03:57.374 INFO:teuthology.orchestra.run.smithi029.stdout:(7/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 339 kB/s | 222 kB 00:00 2024-04-12T11:03:57.409 INFO:teuthology.orchestra.run.smithi029.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 181 kB/s | 32 kB 00:00 2024-04-12T11:03:57.450 INFO:teuthology.orchestra.run.smithi029.stdout:(9/13): python3-certifi-2021.10.8-2.el9.noarch. 81 kB/s | 15 kB 00:00 2024-04-12T11:03:57.501 INFO:teuthology.orchestra.run.smithi029.stdout:(10/13): python3-google-auth-2.28.2-1.el9.noarc 1.6 MB/s | 203 kB 00:00 2024-04-12T11:03:57.560 INFO:teuthology.orchestra.run.smithi029.stdout:(11/13): python3-kubernetes-26.1.0-2.el9.noarch 6.8 MB/s | 1.0 MB 00:00 2024-04-12T11:03:57.585 INFO:teuthology.orchestra.run.smithi029.stdout:(12/13): python3-rsa-4.9-2.el9.noarch.rpm 440 kB/s | 59 kB 00:00 2024-04-12T11:03:57.610 INFO:teuthology.orchestra.run.smithi029.stdout:(13/13): python3-websocket-client-1.2.3-2.el9.n 828 kB/s | 90 kB 00:00 2024-04-12T11:03:57.613 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:03:57.613 INFO:teuthology.orchestra.run.smithi029.stdout:Total 1.5 MB/s | 2.2 MB 00:01 2024-04-12T11:03:57.684 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:03:57.700 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:03:57.700 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:03:57.803 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:03:57.803 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:03:58.329 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:03:58.491 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-12T11:03:58.548 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-12T11:03:58.606 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-12T11:03:58.643 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-12T11:03:58.686 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-12T11:03:58.784 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-12T11:03:58.902 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-12T11:03:58.958 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-12T11:03:59.720 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-12T11:03:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-12T11:03:59.818 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-12T11:03:59.868 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-12T11:03:59.897 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-rook-2:18.2.2-1189.gc96c8d47.el9.noarch 13/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.2-1189.gc96c8d47.el9.noarch 13/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-rook-2:18.2.2-1189.gc96c8d47.el9.noarch 1/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-12T11:04:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-12T11:04:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-12T11:04:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-rook-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-12T11:04:01.371 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-12T11:04:01.372 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-12T11:04:01.372 INFO:teuthology.orchestra.run.smithi029.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-12T11:04:01.372 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-12T11:04:01.372 INFO:teuthology.orchestra.run.smithi029.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-12T11:04:01.372 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:01.372 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:01.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-cephadm 2024-04-12T11:04:02.336 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:52 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:02.644 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:02.644 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:02.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:02.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.647 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.648 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:02.649 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.650 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:02.651 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.652 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.653 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.653 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:02.721 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:02.721 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:02.721 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-cephadm noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 138 k 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:02.722 INFO:teuthology.orchestra.run.smithi029.stdout:Install 6 Packages 2024-04-12T11:04:02.723 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:02.723 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 7.0 M 2024-04-12T11:04:02.723 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 33 M 2024-04-12T11:04:02.723 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:04:03.304 INFO:teuthology.orchestra.run.smithi029.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 950 kB/s | 248 kB 00:00 2024-04-12T11:04:03.445 INFO:teuthology.orchestra.run.smithi029.stdout:(2/6): ceph-mgr-cephadm-18.2.2-1189.gc96c8d47.e 342 kB/s | 138 kB 00:00 2024-04-12T11:04:03.639 INFO:teuthology.orchestra.run.smithi029.stdout:(3/6): python3-babel-2.9.1-2.el9.noarch.rpm 10 MB/s | 6.0 MB 00:00 2024-04-12T11:04:03.832 INFO:teuthology.orchestra.run.smithi029.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 150 kB/s | 58 kB 00:00 2024-04-12T11:04:03.873 INFO:teuthology.orchestra.run.smithi029.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 962 kB/s | 548 kB 00:00 2024-04-12T11:04:03.957 INFO:teuthology.orchestra.run.smithi029.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 161 kB/s | 51 kB 00:00 2024-04-12T11:04:03.960 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:03.960 INFO:teuthology.orchestra.run.smithi029.stdout:Total 5.6 MB/s | 7.0 MB 00:01 2024-04-12T11:04:04.033 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:04:04.045 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:04:04.045 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:04:04.151 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:04:04.151 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:04:04.457 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:04:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-12T11:04:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-12T11:04:05.240 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-12T11:04:05.345 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-12T11:04:05.415 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-12T11:04:05.452 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 6/6 2024-04-12T11:04:06.549 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 6/6 2024-04-12T11:04:06.549 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 1/6 2024-04-12T11:04:06.549 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-12T11:04:06.550 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-12T11:04:06.550 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-12T11:04:06.550 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-12T11:04:06.890 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-12T11:04:06.891 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:06.891 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:04:06.891 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:04:06.891 INFO:teuthology.orchestra.run.smithi029.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-12T11:04:06.891 INFO:teuthology.orchestra.run.smithi029.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-12T11:04:06.891 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-12T11:04:06.891 INFO:teuthology.orchestra.run.smithi029.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-12T11:04:06.891 INFO:teuthology.orchestra.run.smithi029.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-12T11:04:06.891 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:06.891 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:07.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-fuse 2024-04-12T11:04:07.641 INFO:teuthology.orchestra.run.smithi087.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 1.1 MB/s | 16 MB 00:14 2024-04-12T11:04:07.642 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:07.642 INFO:teuthology.orchestra.run.smithi087.stdout:Total 2.3 MB/s | 38 MB 00:16 2024-04-12T11:04:07.838 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:57 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:07.991 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:04:08.018 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:04:08.018 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.140 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.141 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.142 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.143 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:08.144 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:08.145 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.146 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.147 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.148 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.148 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:08.148 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.148 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.148 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.148 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:08.212 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:08.212 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:08.212 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-04-12T11:04:08.213 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:08.213 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:04:08.213 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-fuse x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 848 k 2024-04-12T11:04:08.213 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-04-12T11:04:08.213 INFO:teuthology.orchestra.run.smithi029.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-12T11:04:08.213 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:08.213 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:04:08.213 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:08.213 INFO:teuthology.orchestra.run.smithi029.stdout:Install 2 Packages 2024-04-12T11:04:08.213 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:08.214 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 928 k 2024-04-12T11:04:08.214 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 2.7 M 2024-04-12T11:04:08.214 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:04:08.429 INFO:teuthology.orchestra.run.smithi029.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 572 kB/s | 80 kB 00:00 2024-04-12T11:04:08.508 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:04:08.509 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:04:08.671 INFO:teuthology.orchestra.run.smithi029.stdout:(2/2): ceph-fuse-18.2.2-1189.gc96c8d47.el9.x86_ 2.2 MB/s | 848 kB 00:00 2024-04-12T11:04:08.672 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:08.672 INFO:teuthology.orchestra.run.smithi029.stdout:Total 2.0 MB/s | 928 kB 00:00 2024-04-12T11:04:08.684 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:04:08.707 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:04:08.707 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:04:08.787 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:04:08.787 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:04:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:04:09.116 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-12T11:04:09.191 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 2/2 2024-04-12T11:04:09.261 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:04:09.325 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-12T11:04:09.363 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-12T11:04:09.469 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-12T11:04:09.506 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-12T11:04:09.689 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-12T11:04:09.713 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-12T11:04:09.805 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-12T11:04:09.952 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 2/2 2024-04-12T11:04:09.952 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/2 2024-04-12T11:04:10.282 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-12T11:04:10.282 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:10.282 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:04:10.282 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-12T11:04:10.282 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:10.282 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-12T11:04:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-12T11:04:10.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install librados-devel 2024-04-12T11:04:11.183 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:01 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:11.490 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:11.490 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:11.490 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:11.490 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:11.491 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:11.492 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.493 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.494 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.495 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.496 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.497 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.498 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:11.560 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:11.560 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:11.561 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2024-04-12T11:04:11.561 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:11.561 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:04:11.561 INFO:teuthology.orchestra.run.smithi029.stdout: librados-devel x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 127 k 2024-04-12T11:04:11.561 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:11.561 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:04:11.561 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:11.561 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-04-12T11:04:11.561 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:11.562 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 127 k 2024-04-12T11:04:11.562 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 456 k 2024-04-12T11:04:11.562 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:04:11.764 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-12T11:04:11.817 INFO:teuthology.orchestra.run.smithi029.stdout:librados-devel-18.2.2-1189.gc96c8d47.el9.x86_64 495 kB/s | 127 kB 00:00 2024-04-12T11:04:11.818 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:11.818 INFO:teuthology.orchestra.run.smithi029.stdout:Total 493 kB/s | 127 kB 00:00 2024-04-12T11:04:11.818 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:04:11.824 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:04:11.824 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:04:11.886 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:04:11.886 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:04:12.034 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:04:12.135 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librados-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:12.617 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librados-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:12.799 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-12T11:04:12.828 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.2-1189.gc96c8 12/12 2024-04-12T11:04:12.940 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librados-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:12.940 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:12.940 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:04:12.940 INFO:teuthology.orchestra.run.smithi029.stdout: librados-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:04:12.940 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:12.940 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:13.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install libcephfs2 2024-04-12T11:04:13.884 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:03 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:14.187 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.188 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.189 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.190 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.191 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.192 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.193 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.194 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:14.195 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.196 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:14.197 INFO:teuthology.orchestra.run.smithi029.stdout:Package libcephfs2-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:14.260 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:14.261 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-04-12T11:04:14.261 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:14.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install libcephfs-devel 2024-04-12T11:04:14.954 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:04 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:15.248 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.2-1189.gc96c8 12/12 2024-04-12T11:04:15.248 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.2-1189.gc96c8 1/12 2024-04-12T11:04:15.248 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-12T11:04:15.248 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-12T11:04:15.248 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-12T11:04:15.248 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-12T11:04:15.249 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-12T11:04:15.249 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-12T11:04:15.249 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-12T11:04:15.249 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 9/12 2024-04-12T11:04:15.249 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-12T11:04:15.249 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-12T11:04:15.262 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:15.262 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:15.262 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:15.262 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.262 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:15.262 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.262 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.263 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:15.264 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.265 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:15.266 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.267 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.268 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:15.269 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.270 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:15.337 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs-devel x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 31 k 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:15.338 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 31 k 2024-04-12T11:04:15.339 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 139 k 2024-04-12T11:04:15.339 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:04:15.512 INFO:teuthology.orchestra.run.smithi029.stdout:libcephfs-devel-18.2.2-1189.gc96c8d47.el9.x86_6 180 kB/s | 31 kB 00:00 2024-04-12T11:04:15.513 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:15.513 INFO:teuthology.orchestra.run.smithi029.stdout:Total 178 kB/s | 31 kB 00:00 2024-04-12T11:04:15.513 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:04:15.516 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:04:15.517 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:04:15.546 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:04:15.547 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:04:15.645 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:04:15.682 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-diskprediction-local-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-12T11:04:15.683 INFO:teuthology.orchestra.run.smithi087.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-12T11:04:15.684 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:15.684 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:15.764 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libcephfs-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:16.047 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-mgr-rook 2024-04-12T11:04:16.167 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libcephfs-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libcephfs-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:16.464 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:04:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:04:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:16.464 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:16.653 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:33 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:16.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install librados2 2024-04-12T11:04:16.943 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:16.943 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:16.943 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:16.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.943 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:16.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.945 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:16.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.947 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:16.948 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.949 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.950 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.951 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.952 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:16.952 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.952 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.952 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:16.952 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.012 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-rook noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 49 k 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-12T11:04:17.014 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout:Install 13 Packages 2024-04-12T11:04:17.015 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:17.016 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 2.2 M 2024-04-12T11:04:17.016 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 26 M 2024-04-12T11:04:17.016 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:04:17.320 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:07 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:17.428 INFO:teuthology.orchestra.run.smithi087.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 208 kB/s | 26 kB 00:00 2024-04-12T11:04:17.446 INFO:teuthology.orchestra.run.smithi087.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 137 kB/s | 19 kB 00:00 2024-04-12T11:04:17.470 INFO:teuthology.orchestra.run.smithi087.stdout:(3/13): ceph-mgr-rook-18.2.2-1189.gc96c8d47.el9 298 kB/s | 49 kB 00:00 2024-04-12T11:04:17.529 INFO:teuthology.orchestra.run.smithi087.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.2 MB/s | 222 kB 00:00 2024-04-12T11:04:17.555 INFO:teuthology.orchestra.run.smithi087.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.4 MB/s | 159 kB 00:00 2024-04-12T11:04:17.572 INFO:teuthology.orchestra.run.smithi087.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.3 MB/s | 54 kB 00:00 2024-04-12T11:04:17.625 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:17.625 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:17.625 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:17.626 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.626 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:17.626 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.626 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:17.626 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.627 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:17.627 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.627 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:17.627 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.627 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:17.628 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.628 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:17.628 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.628 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:17.628 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.629 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:17.629 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.629 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:17.629 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.629 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:17.630 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.630 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:17.630 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.630 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:17.630 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.631 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:17.631 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.631 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:17.631 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.631 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:17.632 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.632 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:17.632 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.632 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:17.632 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.632 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:17.632 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.633 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:17.633 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.633 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:17.633 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.633 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:17.633 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.633 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:17.634 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.634 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:17.634 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.634 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:17.634 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.634 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:17.634 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.635 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:17.635 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.635 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:17.635 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.635 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:17.636 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.636 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:17.636 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.636 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:17.636 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.636 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:17.637 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.637 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:17.637 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.637 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:17.637 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.638 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:17.638 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.638 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:17.638 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.638 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:17.639 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.639 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.639 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.639 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:17.639 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.640 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.640 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.640 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.640 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:17.640 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.640 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.641 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.641 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.641 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.641 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.641 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:17.641 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.642 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.642 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.642 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.642 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:17.642 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.643 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.643 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.643 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.643 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:17.643 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.643 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.644 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.644 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.644 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:17.644 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.644 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.645 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:17.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.645 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.646 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:17.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.646 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:17.647 INFO:teuthology.orchestra.run.smithi029.stdout:Package librados2-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:17.655 INFO:teuthology.orchestra.run.smithi087.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.5 MB/s | 279 kB 00:00 2024-04-12T11:04:17.699 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:17.700 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-04-12T11:04:17.700 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:17.739 INFO:teuthology.orchestra.run.smithi087.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 90 kB/s | 15 kB 00:00 2024-04-12T11:04:17.756 INFO:teuthology.orchestra.run.smithi087.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 160 kB/s | 32 kB 00:00 2024-04-12T11:04:17.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install librbd1 2024-04-12T11:04:18.239 INFO:teuthology.orchestra.run.smithi087.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 122 kB/s | 59 kB 00:00 2024-04-12T11:04:18.436 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:08 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:18.437 INFO:teuthology.orchestra.run.smithi087.stdout:(11/13): python3-google-auth-2.28.2-1.el9.noarc 333 kB/s | 203 kB 00:00 2024-04-12T11:04:18.437 INFO:teuthology.orchestra.run.smithi087.stdout:(12/13): python3-kubernetes-26.1.0-2.el9.noarch 1.7 MB/s | 1.0 MB 00:00 2024-04-12T11:04:18.437 INFO:teuthology.orchestra.run.smithi087.stdout:(13/13): python3-websocket-client-1.2.3-2.el9.n 671 kB/s | 90 kB 00:00 2024-04-12T11:04:18.437 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:18.437 INFO:teuthology.orchestra.run.smithi087.stdout:Total 1.6 MB/s | 2.2 MB 00:01 2024-04-12T11:04:18.439 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:04:18.454 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:04:18.459 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:04:18.550 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:04:18.550 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:04:18.706 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:18.706 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:18.706 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:18.706 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.706 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:18.707 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.708 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.709 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.710 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:18.711 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.712 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.713 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.714 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:18.715 INFO:teuthology.orchestra.run.smithi029.stdout:Package librbd1-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:18.779 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:18.780 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-04-12T11:04:18.780 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:18.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-rados 2024-04-12T11:04:19.090 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:04:19.267 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-12T11:04:19.334 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-12T11:04:19.390 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-12T11:04:19.425 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-12T11:04:19.475 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:09 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:19.476 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-12T11:04:19.572 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-12T11:04:19.679 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-12T11:04:19.728 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-12T11:04:19.777 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.778 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.779 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:19.780 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:19.781 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:19.782 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:19.783 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.784 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.785 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.786 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:19.786 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.786 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.786 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.786 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:19.786 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-rados-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:19.851 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:19.852 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-04-12T11:04:19.852 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:19.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-rgw 2024-04-12T11:04:20.481 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-12T11:04:20.539 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-12T11:04:20.541 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:10 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:20.573 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-12T11:04:20.622 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-12T11:04:20.651 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-rook-2:18.2.2-1189.gc96c8d47.el9.noarch 13/13 2024-04-12T11:04:20.843 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:20.844 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:20.844 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:20.844 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.844 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:20.844 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.845 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:20.845 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.845 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:20.845 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.846 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:20.846 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.846 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:20.846 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.846 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:20.846 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.847 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:20.847 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.847 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:20.848 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.848 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:20.848 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.849 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:20.849 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.849 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:20.850 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.850 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:20.850 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.850 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:20.851 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.851 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:20.851 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.851 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:20.851 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.851 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:20.851 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.851 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:20.851 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.852 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:20.852 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.852 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:20.852 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.852 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:20.852 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.852 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:20.852 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.852 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:20.853 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.853 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:20.853 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.853 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:20.853 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.853 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:20.853 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.853 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:20.853 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.854 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.855 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:20.856 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.857 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:20.858 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.858 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.858 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.858 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.858 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:20.858 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.858 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.858 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.858 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:20.859 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-rgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:20.919 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:20.920 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-04-12T11:04:20.921 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:21.002 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-cephfs 2024-04-12T11:04:21.447 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.2-1189.gc96c8d47.el9.noarch 13/13 2024-04-12T11:04:21.447 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-rook-2:18.2.2-1189.gc96c8d47.el9.noarch 1/13 2024-04-12T11:04:21.447 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-12T11:04:21.447 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-12T11:04:21.447 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-12T11:04:21.447 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-12T11:04:21.447 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-12T11:04:21.447 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-12T11:04:21.448 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-12T11:04:21.448 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-12T11:04:21.448 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-12T11:04:21.448 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-12T11:04:21.448 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-12T11:04:21.632 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:11 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:21.820 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-rook-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-12T11:04:21.821 INFO:teuthology.orchestra.run.smithi087.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-12T11:04:21.822 INFO:teuthology.orchestra.run.smithi087.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-12T11:04:21.822 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:21.822 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:21.936 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:21.936 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:21.936 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:21.936 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.936 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:21.936 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.937 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:21.937 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.937 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:21.937 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.937 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:21.937 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.937 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:21.937 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.937 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:21.937 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:21.938 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.939 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:21.940 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.940 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:21.940 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.940 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:21.940 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.940 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:21.940 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.940 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:21.940 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.940 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.941 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:21.942 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.942 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:21.942 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.942 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:21.942 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.942 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:21.942 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.942 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:21.942 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.943 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.944 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.945 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:21.946 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.946 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.946 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.946 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.946 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:21.946 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.946 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.946 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.946 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:21.947 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-cephfs-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:22.012 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:22.012 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-04-12T11:04:22.012 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:22.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-rbd 2024-04-12T11:04:22.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-mgr-cephadm 2024-04-12T11:04:22.690 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:39 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:22.704 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:12 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:22.979 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:22.979 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:22.979 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:22.979 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.979 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:22.979 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.979 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:22.979 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.979 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:22.979 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.980 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:22.981 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:22.982 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.983 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:22.983 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.983 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:22.983 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.983 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:22.983 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.983 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:22.983 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.984 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:22.984 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.984 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:22.984 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.984 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:22.984 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.984 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:22.984 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.984 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:22.984 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.985 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:22.985 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.985 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:22.985 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.985 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:22.985 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.985 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.986 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.987 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:22.988 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.989 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.989 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:22.989 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.005 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:23.005 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.006 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.007 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.008 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.009 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:23.010 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:23.011 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:23.012 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.013 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.014 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:23.014 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.014 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.014 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.014 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.014 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:23.014 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.014 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.014 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.014 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.015 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:23.015 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.015 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.015 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.015 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:23.015 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-rbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:23.048 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:23.049 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:23.049 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2024-04-12T11:04:23.049 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:23.049 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:04:23.049 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-cephadm noarch 2:18.2.2-1189.gc96c8d47.el9 ceph-noarch 138 k 2024-04-12T11:04:23.049 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2024-04-12T11:04:23.049 INFO:teuthology.orchestra.run.smithi087.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-12T11:04:23.050 INFO:teuthology.orchestra.run.smithi087.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-12T11:04:23.050 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-12T11:04:23.050 INFO:teuthology.orchestra.run.smithi087.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-12T11:04:23.050 INFO:teuthology.orchestra.run.smithi087.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-12T11:04:23.050 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:23.050 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:04:23.050 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:23.050 INFO:teuthology.orchestra.run.smithi087.stdout:Install 6 Packages 2024-04-12T11:04:23.050 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:23.051 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 7.0 M 2024-04-12T11:04:23.051 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 33 M 2024-04-12T11:04:23.051 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:04:23.077 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:23.078 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-04-12T11:04:23.078 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:23.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install rbd-fuse 2024-04-12T11:04:23.524 INFO:teuthology.orchestra.run.smithi087.stdout:(1/6): ceph-mgr-cephadm-18.2.2-1189.gc96c8d47.e 666 kB/s | 138 kB 00:00 2024-04-12T11:04:23.776 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:13 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:23.982 INFO:teuthology.orchestra.run.smithi087.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.2 MB/s | 548 kB 00:00 2024-04-12T11:04:24.049 INFO:teuthology.orchestra.run.smithi087.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 867 kB/s | 58 kB 00:00 2024-04-12T11:04:24.080 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:24.080 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:24.080 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:24.080 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.080 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:24.080 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.080 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.081 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.082 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.083 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.084 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:24.085 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.086 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.087 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.088 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:24.116 INFO:teuthology.orchestra.run.smithi087.stdout:(4/6): python3-typing-extensions-4.4.0-2.el9.no 764 kB/s | 51 kB 00:00 2024-04-12T11:04:24.153 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:24.153 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:24.153 INFO:teuthology.orchestra.run.smithi029.stdout: Package Architecture Version Repository Size 2024-04-12T11:04:24.153 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:24.153 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:04:24.153 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-fuse x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 87 k 2024-04-12T11:04:24.153 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:24.153 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:04:24.154 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:24.154 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-04-12T11:04:24.154 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:24.154 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 87 k 2024-04-12T11:04:24.154 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 227 k 2024-04-12T11:04:24.154 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:04:24.233 INFO:teuthology.orchestra.run.smithi087.stdout:(5/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 270 kB/s | 248 kB 00:00 2024-04-12T11:04:24.487 INFO:teuthology.orchestra.run.smithi029.stdout:rbd-fuse-18.2.2-1189.gc96c8d47.el9.x86_64.rpm 261 kB/s | 87 kB 00:00 2024-04-12T11:04:24.488 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:24.488 INFO:teuthology.orchestra.run.smithi029.stdout:Total 259 kB/s | 87 kB 00:00 2024-04-12T11:04:24.488 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:04:24.496 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:04:24.496 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:04:24.557 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:04:24.558 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:04:24.702 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:04:24.814 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : rbd-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:25.297 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: rbd-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:25.623 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : rbd-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:25.623 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:25.623 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:04:25.623 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:04:25.623 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:25.623 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:25.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install rbd-mirror 2024-04-12T11:04:26.571 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:16 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.877 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:26.878 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.879 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:26.880 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.881 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.882 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.883 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:26.884 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.885 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:26.950 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:26.951 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:26.951 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2024-04-12T11:04:26.951 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:26.951 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:04:26.951 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-mirror x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 3.0 M 2024-04-12T11:04:26.951 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:26.951 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:04:26.951 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:26.951 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-04-12T11:04:26.951 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:26.952 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 3.0 M 2024-04-12T11:04:26.952 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 12 M 2024-04-12T11:04:26.952 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:04:27.513 INFO:teuthology.orchestra.run.smithi029.stdout:rbd-mirror-18.2.2-1189.gc96c8d47.el9.x86_64.rpm 5.3 MB/s | 3.0 MB 00:00 2024-04-12T11:04:27.514 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:27.514 INFO:teuthology.orchestra.run.smithi029.stdout:Total 5.3 MB/s | 3.0 MB 00:00 2024-04-12T11:04:27.514 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:04:27.524 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:04:27.524 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:04:27.603 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:04:27.603 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:04:28.031 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:04:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : rbd-mirror-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: rbd-mirror-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:28.123 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:04:28.123 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-12T11:04:28.123 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-12T11:04:28.123 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-12T11:04:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : rbd-mirror-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:29.437 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:04:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-mirror-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:04:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:29.438 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:29.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install rbd-nbd 2024-04-12T11:04:30.359 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:20 ago on Fri 12 Apr 2024 11:02:10 AM UTC. 2024-04-12T11:04:30.660 INFO:teuthology.orchestra.run.smithi029.stderr:Modular dependency problems: 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 1: conflicting requests 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 2: conflicting requests 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 3: conflicting requests 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 4: conflicting requests 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 5: conflicting requests 2024-04-12T11:04:30.661 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 6: conflicting requests 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 7: conflicting requests 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 8: conflicting requests 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 9: conflicting requests 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 10: conflicting requests 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 11: conflicting requests 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 12: conflicting requests 2024-04-12T11:04:30.662 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 13: conflicting requests 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 14: conflicting requests 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 15: conflicting requests 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 16: conflicting requests 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 17: conflicting requests 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 18: conflicting requests 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 19: conflicting requests 2024-04-12T11:04:30.663 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 20: conflicting requests 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 21: conflicting requests 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 22: conflicting requests 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 23: conflicting requests 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 24: conflicting requests 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 25: conflicting requests 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 26: conflicting requests 2024-04-12T11:04:30.664 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 27: conflicting requests 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 28: conflicting requests 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 29: conflicting requests 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 30: conflicting requests 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 31: conflicting requests 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 32: conflicting requests 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 33: conflicting requests 2024-04-12T11:04:30.665 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 34: conflicting requests 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 35: conflicting requests 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 36: conflicting requests 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 37: conflicting requests 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.666 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 39: conflicting requests 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:30.667 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.668 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.669 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:30.669 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.669 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.669 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.669 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.669 INFO:teuthology.orchestra.run.smithi029.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:30.669 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.669 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.669 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.669 INFO:teuthology.orchestra.run.smithi029.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:30.733 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-04-12T11:04:30.734 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:30.734 INFO:teuthology.orchestra.run.smithi029.stdout: Package Architecture Version Repository Size 2024-04-12T11:04:30.734 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:30.734 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-04-12T11:04:30.734 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-nbd x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 172 k 2024-04-12T11:04:30.735 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:30.735 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-04-12T11:04:30.735 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-04-12T11:04:30.735 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-04-12T11:04:30.735 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:30.735 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 172 k 2024-04-12T11:04:30.735 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 494 k 2024-04-12T11:04:30.735 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-04-12T11:04:31.089 INFO:teuthology.orchestra.run.smithi029.stdout:rbd-nbd-18.2.2-1189.gc96c8d47.el9.x86_64.rpm 486 kB/s | 172 kB 00:00 2024-04-12T11:04:31.090 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:31.090 INFO:teuthology.orchestra.run.smithi029.stdout:Total 484 kB/s | 172 kB 00:00 2024-04-12T11:04:31.090 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-04-12T11:04:31.099 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-04-12T11:04:31.099 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-04-12T11:04:31.161 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-04-12T11:04:31.161 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-04-12T11:04:31.313 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-04-12T11:04:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : rbd-nbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:31.441 INFO:teuthology.orchestra.run.smithi087.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 752 kB/s | 6.0 MB 00:08 2024-04-12T11:04:31.443 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:31.443 INFO:teuthology.orchestra.run.smithi087.stdout:Total 852 kB/s | 7.0 MB 00:08 2024-04-12T11:04:31.516 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:04:31.529 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:04:31.529 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:04:31.631 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:04:31.632 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:04:31.884 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: rbd-nbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:31.951 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:04:32.120 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-12T11:04:32.166 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-12T11:04:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : rbd-nbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:32.221 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-04-12T11:04:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-nbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:04:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:04:32.221 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-04-12T11:04:32.524 DEBUG:teuthology.parallel:result is None 2024-04-12T11:04:32.673 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-12T11:04:32.760 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-12T11:04:32.822 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-12T11:04:32.846 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-mgr-cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 6/6 2024-04-12T11:04:33.640 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 6/6 2024-04-12T11:04:33.640 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-mgr-cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 1/6 2024-04-12T11:04:33.640 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-12T11:04:33.640 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-12T11:04:33.641 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-12T11:04:33.641 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-12T11:04:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-12T11:04:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:33.989 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:04:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-mgr-cephadm-2:18.2.2-1189.gc96c8d47.el9.noarch 2024-04-12T11:04:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-12T11:04:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-12T11:04:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-12T11:04:33.989 INFO:teuthology.orchestra.run.smithi087.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-12T11:04:33.990 INFO:teuthology.orchestra.run.smithi087.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-12T11:04:33.990 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:33.990 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:34.231 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install ceph-fuse 2024-04-12T11:04:34.834 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:51 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:35.124 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:35.124 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:35.124 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:35.124 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.124 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:35.125 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:35.126 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:35.127 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.128 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:35.129 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.130 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:35.131 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.131 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:35.131 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.132 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:35.132 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.132 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:35.132 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.132 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:35.133 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.133 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:35.133 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.133 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:35.133 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.133 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:35.134 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.134 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:35.134 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.134 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:35.135 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.136 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.137 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:35.194 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:35.194 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:35.194 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2024-04-12T11:04:35.195 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:35.195 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:04:35.195 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-fuse x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 848 k 2024-04-12T11:04:35.195 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2024-04-12T11:04:35.195 INFO:teuthology.orchestra.run.smithi087.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-12T11:04:35.195 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:35.195 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:04:35.195 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:35.195 INFO:teuthology.orchestra.run.smithi087.stdout:Install 2 Packages 2024-04-12T11:04:35.195 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:35.196 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 928 k 2024-04-12T11:04:35.196 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 2.7 M 2024-04-12T11:04:35.196 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:04:35.408 INFO:teuthology.orchestra.run.smithi087.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 786 kB/s | 80 kB 00:00 2024-04-12T11:04:35.608 INFO:teuthology.orchestra.run.smithi087.stdout:(2/2): ceph-fuse-18.2.2-1189.gc96c8d47.el9.x86_ 2.7 MB/s | 848 kB 00:00 2024-04-12T11:04:35.609 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:35.609 INFO:teuthology.orchestra.run.smithi087.stdout:Total 2.2 MB/s | 928 kB 00:00 2024-04-12T11:04:35.620 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:04:35.641 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:04:35.641 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:04:35.717 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:04:35.718 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:04:35.897 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:04:36.022 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-12T11:04:36.098 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : ceph-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 2/2 2024-04-12T11:04:36.802 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: ceph-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 2/2 2024-04-12T11:04:36.802 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : ceph-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/2 2024-04-12T11:04:37.137 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-12T11:04:37.138 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:37.138 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:04:37.138 INFO:teuthology.orchestra.run.smithi087.stdout: ceph-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-12T11:04:37.138 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:37.138 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:37.418 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install librados-devel 2024-04-12T11:04:38.026 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:55 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:38.321 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:38.321 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.322 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.323 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.324 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:38.325 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.326 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:38.326 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.326 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:38.326 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.326 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:38.326 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.327 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:38.328 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.329 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.330 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.331 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.332 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:38.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.332 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:38.393 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:38.393 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:38.393 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repo Size 2024-04-12T11:04:38.393 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:38.393 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:04:38.393 INFO:teuthology.orchestra.run.smithi087.stdout: librados-devel x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 127 k 2024-04-12T11:04:38.393 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:38.394 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:04:38.394 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:38.394 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2024-04-12T11:04:38.394 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:38.394 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 127 k 2024-04-12T11:04:38.394 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 456 k 2024-04-12T11:04:38.394 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:04:38.622 INFO:teuthology.orchestra.run.smithi087.stdout:librados-devel-18.2.2-1189.gc96c8d47.el9.x86_64 558 kB/s | 127 kB 00:00 2024-04-12T11:04:38.623 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:38.623 INFO:teuthology.orchestra.run.smithi087.stdout:Total 554 kB/s | 127 kB 00:00 2024-04-12T11:04:38.623 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:04:38.628 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:04:38.628 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:04:38.687 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:04:38.687 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:04:38.839 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:04:38.938 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : librados-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:39.375 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: librados-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:39.721 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : librados-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:39.721 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:39.721 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:04:39.721 INFO:teuthology.orchestra.run.smithi087.stdout: librados-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:04:39.721 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:39.721 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:40.002 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install libcephfs2 2024-04-12T11:04:40.612 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:57 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:40.899 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:40.899 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:40.900 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:40.901 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:40.902 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.903 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.904 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:40.905 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.906 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.907 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.908 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.908 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.908 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:40.908 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.908 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.908 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.908 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:40.908 INFO:teuthology.orchestra.run.smithi087.stdout:Package libcephfs2-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:40.967 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:40.968 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2024-04-12T11:04:40.968 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:41.052 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install libcephfs-devel 2024-04-12T11:04:41.650 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:01:58 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:41.937 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:41.937 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.938 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:41.939 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.940 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.941 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.942 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.943 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.944 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.945 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.946 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:41.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:41.946 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:42.006 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:42.007 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:42.007 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repo Size 2024-04-12T11:04:42.007 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:42.007 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:04:42.007 INFO:teuthology.orchestra.run.smithi087.stdout: libcephfs-devel x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 31 k 2024-04-12T11:04:42.007 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:42.008 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:04:42.008 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:42.008 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2024-04-12T11:04:42.008 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:42.008 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 31 k 2024-04-12T11:04:42.008 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 139 k 2024-04-12T11:04:42.008 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:04:42.175 INFO:teuthology.orchestra.run.smithi087.stdout:libcephfs-devel-18.2.2-1189.gc96c8d47.el9.x86_6 186 kB/s | 31 kB 00:00 2024-04-12T11:04:42.176 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:42.176 INFO:teuthology.orchestra.run.smithi087.stdout:Total 184 kB/s | 31 kB 00:00 2024-04-12T11:04:42.176 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:04:42.179 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:04:42.179 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:04:42.207 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:04:42.207 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:04:42.292 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:04:42.390 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : libcephfs-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:42.776 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: libcephfs-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:43.060 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : libcephfs-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:43.060 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:43.061 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:04:43.061 INFO:teuthology.orchestra.run.smithi087.stdout: libcephfs-devel-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:04:43.061 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:43.061 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:43.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install librados2 2024-04-12T11:04:43.893 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:00 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:44.187 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:44.187 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:44.187 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:44.187 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.187 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:44.187 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.187 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:44.187 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.187 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:44.188 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.189 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:44.190 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.191 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.192 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.193 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.194 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.195 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.195 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:44.195 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.195 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.195 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.195 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:44.196 INFO:teuthology.orchestra.run.smithi087.stdout:Package librados2-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:44.256 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:44.257 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2024-04-12T11:04:44.257 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:44.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install librbd1 2024-04-12T11:04:44.936 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:01 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:45.223 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:45.224 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:45.225 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.226 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:45.227 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.228 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.229 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.230 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.231 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:45.232 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.232 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.232 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.232 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.232 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:45.232 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.232 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.232 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.232 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:45.232 INFO:teuthology.orchestra.run.smithi087.stdout:Package librbd1-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:45.291 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:45.292 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2024-04-12T11:04:45.293 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:45.375 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install python3-rados 2024-04-12T11:04:45.970 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:02 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:46.258 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:46.259 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.260 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:46.260 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.260 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:46.260 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.260 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:46.260 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.260 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:46.260 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.260 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:46.260 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:46.261 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.262 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:46.262 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.262 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:46.262 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.262 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:46.262 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.262 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:46.262 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.263 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:46.263 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.263 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:46.263 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.263 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:46.264 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.264 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:46.264 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.264 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:46.264 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.265 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:46.265 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.265 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:46.265 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.265 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:46.265 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.265 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:46.265 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.265 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:46.265 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.266 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.267 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.268 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.269 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.270 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:46.270 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.270 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.270 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.270 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:46.270 INFO:teuthology.orchestra.run.smithi087.stdout:Package python3-rados-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:46.328 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:46.329 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2024-04-12T11:04:46.329 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:46.405 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install python3-rgw 2024-04-12T11:04:46.997 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:03 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:47.282 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:47.282 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:47.282 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:47.282 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.282 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:47.282 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.283 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.284 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:47.285 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.285 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:47.285 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.285 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:47.285 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.285 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:47.285 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.285 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:47.285 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.286 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:47.286 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.286 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:47.286 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.286 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:47.286 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.286 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:47.286 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.286 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:47.286 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.287 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:47.287 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.287 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:47.287 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.287 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:47.287 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.287 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:47.287 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.287 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:47.288 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.288 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:47.288 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.288 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:47.288 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.288 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:47.288 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.288 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:47.289 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.289 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:47.289 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.289 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:47.290 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.290 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:47.291 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.291 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:47.291 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.291 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:47.291 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.292 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.292 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.292 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:47.292 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.292 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.292 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.293 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.294 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.295 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:47.295 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.295 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.295 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.295 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.295 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:47.295 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.295 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.295 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.295 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:47.296 INFO:teuthology.orchestra.run.smithi087.stdout:Package python3-rgw-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:47.349 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:47.350 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2024-04-12T11:04:47.350 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:47.421 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install python3-cephfs 2024-04-12T11:04:48.021 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:05 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.309 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:48.310 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.311 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:48.312 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.313 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:48.313 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.313 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:48.313 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.313 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:48.313 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.313 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:48.314 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.315 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:48.316 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.317 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.318 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.318 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:48.318 INFO:teuthology.orchestra.run.smithi087.stdout:Package python3-cephfs-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:48.378 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:48.379 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2024-04-12T11:04:48.379 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:48.465 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install python3-rbd 2024-04-12T11:04:49.060 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:06 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:49.348 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:49.348 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:49.348 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.349 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.350 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.351 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:49.352 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:49.353 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.354 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:49.355 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.356 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:49.357 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.357 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.357 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.357 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:49.357 INFO:teuthology.orchestra.run.smithi087.stdout:Package python3-rbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 is already installed. 2024-04-12T11:04:49.416 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:49.417 INFO:teuthology.orchestra.run.smithi087.stdout:Nothing to do. 2024-04-12T11:04:49.417 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:49.496 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install rbd-fuse 2024-04-12T11:04:50.168 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:07 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:50.456 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:50.456 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:50.456 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:50.456 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.456 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:50.456 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.456 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:50.456 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:50.457 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.458 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:50.459 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.460 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.461 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.462 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:50.463 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.464 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:50.525 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:50.526 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:50.526 INFO:teuthology.orchestra.run.smithi087.stdout: Package Architecture Version Repository Size 2024-04-12T11:04:50.526 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:50.526 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:04:50.526 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-fuse x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 87 k 2024-04-12T11:04:50.526 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:50.526 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:04:50.526 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:50.527 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2024-04-12T11:04:50.527 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:50.527 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 87 k 2024-04-12T11:04:50.527 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 227 k 2024-04-12T11:04:50.527 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:04:50.737 INFO:teuthology.orchestra.run.smithi087.stdout:rbd-fuse-18.2.2-1189.gc96c8d47.el9.x86_64.rpm 414 kB/s | 87 kB 00:00 2024-04-12T11:04:50.737 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:50.737 INFO:teuthology.orchestra.run.smithi087.stdout:Total 411 kB/s | 87 kB 00:00 2024-04-12T11:04:50.737 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:04:50.744 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:04:50.744 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:04:50.804 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:04:50.804 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:04:50.947 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:04:51.043 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : rbd-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:51.450 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: rbd-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:51.762 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : rbd-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:51.762 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:51.762 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:04:51.763 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-fuse-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:04:51.763 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:51.763 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:51.968 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install rbd-mirror 2024-04-12T11:04:52.579 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:09 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:52.868 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:52.868 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.869 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:52.870 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.871 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.872 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.873 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.874 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.875 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.876 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:52.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.877 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:52.936 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:52.937 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:52.937 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repo Size 2024-04-12T11:04:52.937 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:52.937 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:04:52.937 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-mirror x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 3.0 M 2024-04-12T11:04:52.937 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:52.937 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:04:52.937 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:52.937 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2024-04-12T11:04:52.937 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:52.938 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 3.0 M 2024-04-12T11:04:52.938 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 12 M 2024-04-12T11:04:52.938 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:04:53.464 INFO:teuthology.orchestra.run.smithi087.stdout:rbd-mirror-18.2.2-1189.gc96c8d47.el9.x86_64.rpm 5.7 MB/s | 3.0 MB 00:00 2024-04-12T11:04:53.464 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:53.465 INFO:teuthology.orchestra.run.smithi087.stdout:Total 5.7 MB/s | 3.0 MB 00:00 2024-04-12T11:04:53.465 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:04:53.474 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:04:53.474 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:04:53.550 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:04:53.551 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:04:53.962 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:04:54.023 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : rbd-mirror-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:54.048 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: rbd-mirror-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:54.048 INFO:teuthology.orchestra.run.smithi087.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-12T11:04:54.048 INFO:teuthology.orchestra.run.smithi087.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-12T11:04:54.048 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-12T11:04:54.048 INFO:teuthology.orchestra.run.smithi087.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-12T11:04:54.048 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:55.233 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : rbd-mirror-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:55.233 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:55.233 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:04:55.233 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-mirror-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:04:55.233 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:55.233 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:55.524 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum -y install rbd-nbd 2024-04-12T11:04:56.138 INFO:teuthology.orchestra.run.smithi087.stdout:Last metadata expiration check: 0:02:13 ago on Fri 12 Apr 2024 11:02:43 AM UTC. 2024-04-12T11:04:56.427 INFO:teuthology.orchestra.run.smithi087.stderr:Modular dependency problems: 2024-04-12T11:04:56.427 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T11:04:56.428 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 1: conflicting requests 2024-04-12T11:04:56.428 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.428 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 2: conflicting requests 2024-04-12T11:04:56.428 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.428 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 3: conflicting requests 2024-04-12T11:04:56.428 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.428 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 4: conflicting requests 2024-04-12T11:04:56.428 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.428 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 5: conflicting requests 2024-04-12T11:04:56.428 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.429 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 6: conflicting requests 2024-04-12T11:04:56.429 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.429 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 7: conflicting requests 2024-04-12T11:04:56.429 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.429 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 8: conflicting requests 2024-04-12T11:04:56.429 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.429 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 9: conflicting requests 2024-04-12T11:04:56.429 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.429 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 10: conflicting requests 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 11: conflicting requests 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 12: conflicting requests 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 13: conflicting requests 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 14: conflicting requests 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 15: conflicting requests 2024-04-12T11:04:56.430 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 16: conflicting requests 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 17: conflicting requests 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 18: conflicting requests 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 19: conflicting requests 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 20: conflicting requests 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 21: conflicting requests 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 22: conflicting requests 2024-04-12T11:04:56.431 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 23: conflicting requests 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 24: conflicting requests 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 25: conflicting requests 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 26: conflicting requests 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 27: conflicting requests 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 28: conflicting requests 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.432 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 29: conflicting requests 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 30: conflicting requests 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 31: conflicting requests 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 32: conflicting requests 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 33: conflicting requests 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 34: conflicting requests 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 35: conflicting requests 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.433 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 36: conflicting requests 2024-04-12T11:04:56.434 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.434 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 37: conflicting requests 2024-04-12T11:04:56.434 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.434 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.434 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.434 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T11:04:56.434 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.434 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 39: conflicting requests 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.435 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.436 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.437 INFO:teuthology.orchestra.run.smithi087.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T11:04:56.497 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-12T11:04:56.498 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:56.498 INFO:teuthology.orchestra.run.smithi087.stdout: Package Architecture Version Repository Size 2024-04-12T11:04:56.498 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:56.499 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-12T11:04:56.499 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-nbd x86_64 2:18.2.2-1189.gc96c8d47.el9 ceph 172 k 2024-04-12T11:04:56.499 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:56.499 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-12T11:04:56.499 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-12T11:04:56.499 INFO:teuthology.orchestra.run.smithi087.stdout:Install 1 Package 2024-04-12T11:04:56.499 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:56.499 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 172 k 2024-04-12T11:04:56.499 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 494 k 2024-04-12T11:04:56.499 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-12T11:04:56.725 INFO:teuthology.orchestra.run.smithi087.stdout:rbd-nbd-18.2.2-1189.gc96c8d47.el9.x86_64.rpm 761 kB/s | 172 kB 00:00 2024-04-12T11:04:56.726 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:04:56.726 INFO:teuthology.orchestra.run.smithi087.stdout:Total 756 kB/s | 172 kB 00:00 2024-04-12T11:04:56.726 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-12T11:04:56.734 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-12T11:04:56.734 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-12T11:04:56.794 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-12T11:04:56.794 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-12T11:04:56.947 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-12T11:04:57.049 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : rbd-nbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:57.443 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: rbd-nbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:57.742 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : rbd-nbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 1/1 2024-04-12T11:04:57.742 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:57.742 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-12T11:04:57.743 INFO:teuthology.orchestra.run.smithi087.stdout: rbd-nbd-2:18.2.2-1189.gc96c8d47.el9.x86_64 2024-04-12T11:04:57.743 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:04:57.743 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-12T11:04:57.981 DEBUG:teuthology.parallel:result is None 2024-04-12T11:04:57.981 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T11:04:57.982 INFO:teuthology.packaging:ref: None 2024-04-12T11:04:57.982 INFO:teuthology.packaging:tag: None 2024-04-12T11:04:57.982 INFO:teuthology.packaging:branch: reef 2024-04-12T11:04:57.982 INFO:teuthology.packaging:sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:04:57.982 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-04-12T11:04:58.356 DEBUG:teuthology.orchestra.run.smithi029:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-12T11:04:58.380 INFO:teuthology.orchestra.run.smithi029.stdout:18.2.2-1189.gc96c8d47.el9 2024-04-12T11:04:58.380 INFO:teuthology.packaging:The installed version of ceph is 18.2.2-1189.gc96c8d47.el9 2024-04-12T11:04:58.380 INFO:teuthology.task.install:The correct ceph version 18.2.2-1189.gc96c8d47 is installed. 2024-04-12T11:04:58.382 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T11:04:58.382 INFO:teuthology.packaging:ref: None 2024-04-12T11:04:58.382 INFO:teuthology.packaging:tag: None 2024-04-12T11:04:58.382 INFO:teuthology.packaging:branch: reef 2024-04-12T11:04:58.383 INFO:teuthology.packaging:sha1: 2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:04:58.383 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-04-12T11:04:58.734 DEBUG:teuthology.orchestra.run.smithi087:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-12T11:04:58.756 INFO:teuthology.orchestra.run.smithi087.stdout:18.2.2-1189.gc96c8d47.el9 2024-04-12T11:04:58.756 INFO:teuthology.packaging:The installed version of ceph is 18.2.2-1189.gc96c8d47.el9 2024-04-12T11:04:58.756 INFO:teuthology.task.install:The correct ceph version 18.2.2-1189.gc96c8d47 is installed. 2024-04-12T11:04:58.759 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-12T11:04:58.759 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:04:58.759 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-12T11:04:58.791 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:04:58.791 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-12T11:04:58.824 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-12T11:04:58.824 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:04:58.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/usr/bin/daemon-helper 2024-04-12T11:04:58.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-12T11:04:58.966 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:04:58.966 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/usr/bin/daemon-helper 2024-04-12T11:04:58.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-12T11:04:59.061 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-12T11:04:59.061 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:04:59.061 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-12T11:04:59.091 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-12T11:04:59.159 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:04:59.159 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-12T11:04:59.186 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-12T11:04:59.254 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-12T11:04:59.255 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:04:59.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/usr/bin/stdin-killer 2024-04-12T11:04:59.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-12T11:04:59.349 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:04:59.349 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/usr/bin/stdin-killer 2024-04-12T11:04:59.376 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-12T11:04:59.445 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-12T11:04:59.580 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95'} 2024-04-12T11:04:59.581 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-04-12T11:04:59.581 INFO:tasks.cephadm:Cluster fsid is 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:04:59.581 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-12T11:04:59.582 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.29', 'mon.c': '[v2:172.21.15.29:3301,v1:172.21.15.29:6790]', 'mon.b': '172.21.15.87'} 2024-04-12T11:04:59.582 INFO:tasks.cephadm:First mon is mon.a on smithi029 2024-04-12T11:04:59.582 INFO:tasks.cephadm:First mgr is y 2024-04-12T11:04:59.582 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-12T11:04:59.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo hostname $(hostname -s) 2024-04-12T11:04:59.611 DEBUG:teuthology.orchestra.run.smithi087:> sudo hostname $(hostname -s) 2024-04-12T11:04:59.640 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-04-12T11:04:59.640 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:04:59.784 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/78504/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.0.0-2874-g2cee1857', 'node_name': '172.21.5.35+adami05', 'job_name': 'ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.0.0-2874.g2cee1857'}, 'url': 'https://4.chacra.ceph.com/r/ceph/wip-rf-comb1/2cee1857f8068f7ca29db3a37937b8e9f919ba95/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-04-12 06:47:39.691808', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'wip-rf-comb1', 'chacra_url': 'https://4.chacra.ceph.com/repos/ceph/wip-rf-comb1/2cee1857f8068f7ca29db3a37937b8e9f919ba95/centos/9/flavors/default/', 'archs': ['source', 'x86_64'], 'distro': 'centos'}] 2024-04-12T11:05:00.140 INFO:tasks.util.chacra:got chacra host 2.chacra.ceph.com, ref reef, sha1 c96c8d478b644b6433700141c5f69a1a68f8121b from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&ref=reef 2024-04-12T11:05:00.143 INFO:tasks.cephadm:Discovered cachra url: https://2.chacra.ceph.com/binaries/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/x86_64/flavors/default/cephadm 2024-04-12T11:05:00.144 DEBUG:teuthology.orchestra.run.smithi029:> curl --silent -L https://2.chacra.ceph.com/binaries/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-12T11:05:00.572 INFO:teuthology.orchestra.run.smithi029.stdout:-rw-r--r--. 1 ubuntu ubuntu 215335 Apr 12 11:05 /home/ubuntu/cephtest/cephadm 2024-04-12T11:05:00.572 DEBUG:teuthology.orchestra.run.smithi087:> curl --silent -L https://2.chacra.ceph.com/binaries/ceph/reef/c96c8d478b644b6433700141c5f69a1a68f8121b/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-12T11:05:00.991 INFO:teuthology.orchestra.run.smithi087.stdout:-rw-r--r--. 1 ubuntu ubuntu 215335 Apr 12 11:05 /home/ubuntu/cephtest/cephadm 2024-04-12T11:05:00.992 DEBUG:teuthology.orchestra.run.smithi029:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-12T11:05:01.008 DEBUG:teuthology.orchestra.run.smithi087:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-12T11:05:01.041 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-04-12T11:05:01.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-04-12T11:05:01.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-04-12T11:05:01.927 INFO:teuthology.orchestra.run.smithi087.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-04-12T11:05:01.993 INFO:teuthology.orchestra.run.smithi029.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-04-12T11:05:48.218 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:05:48.219 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph_version": "ceph version 18.2.2-1187-gd540ebac (d540ebaca6b131a1dd560e7f69e024b133bbaa42) reef (stable)", 2024-04-12T11:05:48.219 INFO:teuthology.orchestra.run.smithi029.stdout: "image_id": "bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28", 2024-04-12T11:05:48.219 INFO:teuthology.orchestra.run.smithi029.stdout: "repo_digests": [ 2024-04-12T11:05:48.219 INFO:teuthology.orchestra.run.smithi029.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3" 2024-04-12T11:05:48.219 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:05:48.219 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:05:57.250 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:05:57.251 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph_version": "ceph version 18.2.2-1187-gd540ebac (d540ebaca6b131a1dd560e7f69e024b133bbaa42) reef (stable)", 2024-04-12T11:05:57.251 INFO:teuthology.orchestra.run.smithi087.stdout: "image_id": "bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28", 2024-04-12T11:05:57.251 INFO:teuthology.orchestra.run.smithi087.stdout: "repo_digests": [ 2024-04-12T11:05:57.251 INFO:teuthology.orchestra.run.smithi087.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3" 2024-04-12T11:05:57.251 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:05:57.251 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:05:57.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo mkdir -p /etc/ceph 2024-04-12T11:05:57.311 DEBUG:teuthology.orchestra.run.smithi087:> sudo mkdir -p /etc/ceph 2024-04-12T11:05:57.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod 777 /etc/ceph 2024-04-12T11:05:57.383 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod 777 /etc/ceph 2024-04-12T11:05:57.414 INFO:tasks.cephadm:Writing seed config... 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-12T11:05:57.416 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-12T11:05:57.417 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:05:57.417 DEBUG:teuthology.orchestra.run.smithi029:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-12T11:05:57.441 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 80efffc6-f8bc-11ee-bc8c-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-12T11:05:57.441 DEBUG:teuthology.orchestra.run.smithi029:mon.a> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.a.service 2024-04-12T11:05:57.483 DEBUG:teuthology.orchestra.run.smithi029:mgr.y> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.y.service 2024-04-12T11:05:57.527 INFO:tasks.cephadm:Bootstrapping... 2024-04-12T11:05:57.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.29 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-12T11:05:57.694 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-04-12T11:05:57.695 INFO:teuthology.orchestra.run.smithi029.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', '80efffc6-f8bc-11ee-bc8c-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.29', '--skip-admin-label'] 2024-04-12T11:05:57.719 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-12T11:05:57.720 INFO:teuthology.orchestra.run.smithi029.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-12T11:05:57.720 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying podman|docker is present... 2024-04-12T11:05:57.742 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-12T11:05:57.742 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying lvm2 is present... 2024-04-12T11:05:57.742 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying time synchronization is in place... 2024-04-12T11:05:57.749 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-12T11:05:57.749 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-12T11:05:57.755 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-12T11:05:57.755 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-04-12T11:05:57.761 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout enabled 2024-04-12T11:05:57.767 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout active 2024-04-12T11:05:57.767 INFO:teuthology.orchestra.run.smithi029.stdout:Unit chronyd.service is enabled and running 2024-04-12T11:05:57.767 INFO:teuthology.orchestra.run.smithi029.stdout:Repeating the final host check... 2024-04-12T11:05:57.791 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-12T11:05:57.791 INFO:teuthology.orchestra.run.smithi029.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-12T11:05:57.791 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl is present 2024-04-12T11:05:57.791 INFO:teuthology.orchestra.run.smithi029.stdout:lvcreate is present 2024-04-12T11:05:57.797 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-12T11:05:57.797 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-12T11:05:57.802 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-12T11:05:57.802 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-04-12T11:05:57.808 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout enabled 2024-04-12T11:05:57.815 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout active 2024-04-12T11:05:57.815 INFO:teuthology.orchestra.run.smithi029.stdout:Unit chronyd.service is enabled and running 2024-04-12T11:05:57.815 INFO:teuthology.orchestra.run.smithi029.stdout:Host looks OK 2024-04-12T11:05:57.815 INFO:teuthology.orchestra.run.smithi029.stdout:Cluster fsid: 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:05:57.815 INFO:teuthology.orchestra.run.smithi029.stdout:Acquiring lock 140134238409680 on /run/cephadm/80efffc6-f8bc-11ee-bc8c-c7b262605968.lock 2024-04-12T11:05:57.815 INFO:teuthology.orchestra.run.smithi029.stdout:Lock 140134238409680 acquired on /run/cephadm/80efffc6-f8bc-11ee-bc8c-c7b262605968.lock 2024-04-12T11:05:57.815 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying IP 172.21.15.29 port 3300 ... 2024-04-12T11:05:57.815 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying IP 172.21.15.29 port 6789 ... 2024-04-12T11:05:57.816 INFO:teuthology.orchestra.run.smithi029.stdout:Base mon IP(s) is [172.21.15.29:3300, 172.21.15.29:6789], mon addrv is [v2:172.21.15.29:3300,v1:172.21.15.29:6789] 2024-04-12T11:05:57.818 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.29 metric 100 2024-04-12T11:05:57.818 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.29 metric 100 2024-04-12T11:05:57.820 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-12T11:05:57.821 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-04-12T11:05:57.821 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-04-12T11:05:57.823 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-12T11:05:57.823 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-12T11:05:57.823 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-12T11:05:57.823 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-04-12T11:05:57.823 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:125a/64 scope link noprefixroute 2024-04-12T11:05:57.823 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-12T11:05:57.824 INFO:teuthology.orchestra.run.smithi029.stdout:Mon IP `172.21.15.29` is in CIDR network `172.21.0.0/20` 2024-04-12T11:05:57.824 INFO:teuthology.orchestra.run.smithi029.stdout:Mon IP `172.21.15.29` is in CIDR network `172.21.0.0/20` 2024-04-12T11:05:57.824 INFO:teuthology.orchestra.run.smithi029.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-12T11:05:57.824 INFO:teuthology.orchestra.run.smithi029.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-12T11:05:57.825 INFO:teuthology.orchestra.run.smithi029.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-04-12T11:05:58.616 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 2024-04-12T11:05:58.616 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:reef... 2024-04-12T11:05:58.616 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Getting image source signatures 2024-04-12T11:05:58.616 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Copying blob sha256:9ae191255112f837f5ddf23e5c5ab1101aa18a0a5cef4402020e5a45c4dfa044 2024-04-12T11:05:58.616 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Copying blob sha256:77070339c81bcb5c53bbbf3f31a47f7187e5393cb0e096452852a31192d97570 2024-04-12T11:05:58.616 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Copying config sha256:bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 2024-04-12T11:05:58.616 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-12T11:05:59.553 INFO:teuthology.orchestra.run.smithi029.stdout:ceph: stdout ceph version 18.2.2-1187-gd540ebac (d540ebaca6b131a1dd560e7f69e024b133bbaa42) reef (stable) 2024-04-12T11:05:59.554 INFO:teuthology.orchestra.run.smithi029.stdout:Ceph version: ceph version 18.2.2-1187-gd540ebac (d540ebaca6b131a1dd560e7f69e024b133bbaa42) reef (stable) 2024-04-12T11:05:59.554 INFO:teuthology.orchestra.run.smithi029.stdout:Extracting ceph user uid/gid from container image... 2024-04-12T11:06:00.425 INFO:teuthology.orchestra.run.smithi029.stdout:stat: stdout 167 167 2024-04-12T11:06:00.425 INFO:teuthology.orchestra.run.smithi029.stdout:Creating initial keys... 2024-04-12T11:06:01.338 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph-authtool: stdout AQCYFRlmp5XwKxAAVpBF/urq+IuP9gIVe/xt4g== 2024-04-12T11:06:02.309 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph-authtool: stdout AQCZFRlmk1dKKhAAekUlfeGA3x5nMlGe/dpu1g== 2024-04-12T11:06:03.222 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph-authtool: stdout AQCaFRlmyUaOJRAANJE5Un7vlx3CtV51iZKwUw== 2024-04-12T11:06:03.223 INFO:teuthology.orchestra.run.smithi029.stdout:Creating initial monmap... 2024-04-12T11:06:04.128 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-12T11:06:04.128 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-04-12T11:06:04.128 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:06:04.128 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-12T11:06:04.128 INFO:teuthology.orchestra.run.smithi029.stdout:monmaptool for a [v2:172.21.15.29:3300,v1:172.21.15.29:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-12T11:06:04.128 INFO:teuthology.orchestra.run.smithi029.stdout:setting min_mon_release = pacific 2024-04-12T11:06:04.129 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: set fsid to 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:06:04.129 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-12T11:06:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:06:04.129 INFO:teuthology.orchestra.run.smithi029.stdout:Creating mon... 2024-04-12T11:06:05.025 INFO:teuthology.orchestra.run.smithi029.stdout:create mon.a on 2024-04-12T11:06:05.236 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-12T11:06:05.371 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-12T11:06:05.542 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968.target → /etc/systemd/system/ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968.target. 2024-04-12T11:06:05.542 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968.target → /etc/systemd/system/ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968.target. 2024-04-12T11:06:05.772 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.a 2024-04-12T11:06:05.772 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to reset failed state of unit ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.a.service: Unit ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.a.service not loaded. 2024-04-12T11:06:05.936 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968.target.wants/ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.a.service → /etc/systemd/system/ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@.service. 2024-04-12T11:06:06.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:06 smithi029 ceph-mon[28965]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-12T11:06:06.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:06 smithi029 bash[28945]: 3d36b85f22eee72a1fa1b8dc0c5a24fc1fd81ca84b0f2199a9da4f531d34625b 2024-04-12T11:06:06.458 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-12T11:06:06.458 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2024-04-12T11:06:06.466 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-12T11:06:06.466 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-04-12T11:06:06.467 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2024-04-12T11:06:06.467 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to enable service . firewalld.service is not available 2024-04-12T11:06:06.467 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mon to start... 2024-04-12T11:06:06.467 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mon... 2024-04-12T11:06:06.705 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:06 smithi029 systemd[1]: Started Ceph mon.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:06:07.590 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:07 smithi029 ceph-mon[28965]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-12T11:06:07.594 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:07 smithi029 ceph-mon[28965]: monmap e1: 1 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T11:06:07.594 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:07 smithi029 ceph-mon[28965]: fsmap 2024-04-12T11:06:07.594 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:07 smithi029 ceph-mon[28965]: osdmap e1: 0 total, 0 up, 0 in 2024-04-12T11:06:07.594 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:07 smithi029 ceph-mon[28965]: mgrmap e1: no daemons active 2024-04-12T11:06:07.594 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:07 smithi029 ceph-mon[28965]: from='client.? 172.21.15.29:0/2319454433' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-12T11:06:07.596 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout cluster: 2024-04-12T11:06:07.596 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout id: 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:06:07.596 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-12T11:06:07.596 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:07.596 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout services: 2024-04-12T11:06:07.596 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.589921s) 2024-04-12T11:06:07.596 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-12T11:06:07.597 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-12T11:06:07.597 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:07.597 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout data: 2024-04-12T11:06:07.597 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-12T11:06:07.597 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-12T11:06:07.597 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-12T11:06:07.597 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout pgs: 2024-04-12T11:06:07.597 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:07.597 INFO:teuthology.orchestra.run.smithi029.stdout:mon is available 2024-04-12T11:06:07.597 INFO:teuthology.orchestra.run.smithi029.stdout:Assimilating anything we can from ceph.conf... 2024-04-12T11:06:08.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:08 smithi029 ceph-mon[28965]: from='client.? 172.21.15.29:0/1373584501' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-12T11:06:08.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:08 smithi029 ceph-mon[28965]: from='client.? 172.21.15.29:0/1373584501' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-12T11:06:08.659 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:08.659 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [global] 2024-04-12T11:06:08.659 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout fsid = 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:06:08.659 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.29:3300,v1:172.21.15.29:6789] 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [osd] 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-12T11:06:08.660 INFO:teuthology.orchestra.run.smithi029.stdout:Generating new minimal ceph.conf... 2024-04-12T11:06:09.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:09 smithi029 ceph-mon[28965]: from='client.? 172.21.15.29:0/1356433593' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:06:09.688 INFO:teuthology.orchestra.run.smithi029.stdout:Restarting the monitor... 2024-04-12T11:06:09.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:09 smithi029 systemd[1]: Stopping Ceph mon.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:06:10.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:09 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[28960]: 2024-04-12T11:06:09.948+0000 7f17a4f80700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:06:10.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:09 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[28960]: 2024-04-12T11:06:09.948+0000 7f17a4f80700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-12T11:06:10.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:10 smithi029 podman[29226]: 2024-04-12 11:06:10.10322345 +0000 UTC m=+0.250819100 container died 3d36b85f22eee72a1fa1b8dc0c5a24fc1fd81ca84b0f2199a9da4f531d34625b (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, RELEASE=reef-d540eba, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS) 2024-04-12T11:06:10.619 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:10 smithi029 podman[29226]: 2024-04-12 11:06:10.433997397 +0000 UTC m=+0.581593045 container cleanup 3d36b85f22eee72a1fa1b8dc0c5a24fc1fd81ca84b0f2199a9da4f531d34625b (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=reef-d540eba, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-12T11:06:10.619 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:10 smithi029 bash[29226]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a 2024-04-12T11:06:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:10 smithi029 podman[29240]: 2024-04-12 11:06:10.619351555 +0000 UTC m=+0.515323025 container remove 3d36b85f22eee72a1fa1b8dc0c5a24fc1fd81ca84b0f2199a9da4f531d34625b (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, org.label-schema.schema-version=1.0, RELEASE=reef-d540eba, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403) 2024-04-12T11:06:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:10 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-12T11:06:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:10 smithi029 systemd[1]: Stopped Ceph mon.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:06:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:10 smithi029 systemd[1]: Starting Ceph mon.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:06:11.297 INFO:teuthology.orchestra.run.smithi029.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-12T11:06:11.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 podman[29329]: 2024-04-12 11:06:11.062989138 +0000 UTC m=+0.107746850 container create 1cf52b0b92d9eba320e090e01a740447595c03c3cdef85d9effe11eb945eeb7b (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=reef-d540eba, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T11:06:11.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 podman[29329]: 2024-04-12 11:06:10.972684888 +0000 UTC m=+0.017442561 image pull bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 quay.ceph.io/ceph-ci/ceph:reef 2024-04-12T11:06:11.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 podman[29329]: 2024-04-12 11:06:11.21829209 +0000 UTC m=+0.263049763 container init 1cf52b0b92d9eba320e090e01a740447595c03c3cdef85d9effe11eb945eeb7b (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=reef-d540eba, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True) 2024-04-12T11:06:11.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 podman[29329]: 2024-04-12 11:06:11.220977377 +0000 UTC m=+0.265735048 container start 1cf52b0b92d9eba320e090e01a740447595c03c3cdef85d9effe11eb945eeb7b (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, RELEASE=reef-d540eba, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-12T11:06:11.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T11:06:11.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: ceph version 18.2.2-1187-gd540ebac (d540ebaca6b131a1dd560e7f69e024b133bbaa42) reef (stable), process ceph-mon, pid 2 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: pidfile_write: ignore empty --pid-file 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: load: jerasure load: lrc 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: RocksDB version: 7.9.2 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Git sha 0 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Compile date 2024-04-08 23:21:21 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: DB SUMMARY 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: DB Session ID: 3BD8TJ281I6UWKMJCLLE 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: CURRENT file: CURRENT 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84552 ; 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.error_if_exists: 0 2024-04-12T11:06:11.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.create_if_missing: 0 2024-04-12T11:06:11.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.paranoid_checks: 1 2024-04-12T11:06:11.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T11:06:11.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T11:06:11.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T11:06:11.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.env: 0x557553bc27a0 2024-04-12T11:06:11.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T11:06:11.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.info_log: 0x557556d17500 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.statistics: (nil) 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.use_fsync: 0 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_log_file_size: 0 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.allow_fallocate: 1 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.use_direct_reads: 0 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T11:06:11.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.db_log_dir: 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.wal_dir: 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.write_buffer_manager: 0x557555f8c140 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T11:06:11.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.unordered_write: 0 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.row_cache: None 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.wal_filter: None 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.two_write_queues: 0 2024-04-12T11:06:11.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.wal_compression: 0 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.atomic_flush: 0 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.log_readahead_size: 0 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T11:06:11.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_background_jobs: 2 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_background_compactions: -1 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_subcompactions: 1 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_open_files: -1 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T11:06:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_background_flushes: -1 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Compression algorithms supported: 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: kZSTD supported: 0 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: kXpressCompression supported: 0 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: kZlibCompression supported: 1 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: kSnappyCompression supported: 1 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: kLZ4Compression supported: 1 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: kBZip2Compression supported: 0 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T11:06:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.merge_operator: 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_filter: None 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557556d17600) 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks: 1 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_top_level_index_and_filter: 1 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_type: 0 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_index_type: 0 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_shortening: 1 2024-04-12T11:06:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: checksum: 4 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: no_block_cache: 0 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache: 0x55755602d1f0 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_name: BinnedLRUCache 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_options: 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: capacity : 536870912 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: num_shard_bits : 4 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: strict_capacity_limit : 0 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: high_pri_pool_ratio: 0.000 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_compressed: (nil) 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: persistent_cache: (nil) 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size: 4096 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size_deviation: 10 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_restart_interval: 16 2024-04-12T11:06:11.423 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_block_restart_interval: 1 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: metadata_block_size: 4096 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: partition_filters: 0 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: use_delta_encoding: 1 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: filter_policy: bloomfilter 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: whole_key_filtering: 1 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: verify_compression: 0 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: read_amp_bytes_per_bit: 0 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: format_version: 5 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_index_compression: 1 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_align: 0 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_auto_readahead_size: 262144 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: prepopulate_block_cache: 0 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: initial_auto_readahead_size: 8192 2024-04-12T11:06:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compression: NoCompression 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.num_levels: 7 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T11:06:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T11:06:11.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T11:06:11.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T11:06:11.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.table_properties_collectors: 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.inplace_update_support: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.bloom_locality: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.max_successive_merges: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.ttl: 2592000 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T11:06:11.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.enable_blob_files: false 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.min_blob_size: 0 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f2a020de-3b3d-46d1-b09f-6540083d5ac9 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712919971249111, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-04-12T11:06:11.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712919971250026, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81076, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79252, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73453, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712919971, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f2a020de-3b3d-46d1-b09f-6540083d5ac9", "db_session_id": "3BD8TJ281I6UWKMJCLLE", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712919971250109, "job": 1, "event": "recovery_finished"} 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5575560cc000 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: DB pointer 0x5575560ba000 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** DB Stats ** 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T11:06:11.431 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: L0 2/0 81.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 128.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: Sum 2/0 81.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 128.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 128.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 128.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:06:11.432 INFO:journalctl@ceph.mon.a.smithi029.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative compaction: 0.00 GB write, 26.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval compaction: 0.00 GB write, 26.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: Block cache BinnedLRUCache@0x55755602d1f0#2 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 7e-06 secs_since: 0 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: starting mon.a rank 0 at public addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] at bind addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:06:11.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: mon.a@-1(???) e1 preinit fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: mon.a@-1(???).mds e1 new map 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: mon.a@-1(???).mds e1 print_map 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout: e1 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout: legacy client fscid: -1 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout: No filesystems configured 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 bash[29329]: 1cf52b0b92d9eba320e090e01a740447595c03c3cdef85d9effe11eb945eeb7b 2024-04-12T11:06:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 systemd[1]: Started Ceph mon.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:06:11.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-12T11:06:11.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: monmap e1: 1 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T11:06:11.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: fsmap 2024-04-12T11:06:11.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: osdmap e1: 0 total, 0 up, 0 in 2024-04-12T11:06:11.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:11 smithi029 ceph-mon[29349]: mgrmap e1: no daemons active 2024-04-12T11:06:12.396 INFO:teuthology.orchestra.run.smithi029.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-12T11:06:12.427 INFO:teuthology.orchestra.run.smithi029.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-12T11:06:12.427 INFO:teuthology.orchestra.run.smithi029.stdout:Creating mgr... 2024-04-12T11:06:12.427 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying port 0.0.0.0:9283 ... 2024-04-12T11:06:12.428 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying port 0.0.0.0:8765 ... 2024-04-12T11:06:12.703 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.y 2024-04-12T11:06:12.703 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to reset failed state of unit ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.y.service: Unit ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.y.service not loaded. 2024-04-12T11:06:12.852 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968.target.wants/ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@.service. 2024-04-12T11:06:13.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:12 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1534703375' entity='client.admin' 2024-04-12T11:06:13.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:13 smithi029 podman[29566]: 2024-04-12 11:06:13.255537136 +0000 UTC m=+0.279604418 container start 314601fec666f567978eb64350aaa551e54d3e23c9abbd0f1c595750425cc7db (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_CLEAN=True, RELEASE=reef-d540eba, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-12T11:06:14.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:13.886+0000 7f8acee10200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:06:14.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:13.983+0000 7f8acee10200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:06:14.770 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-12T11:06:14.770 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2024-04-12T11:06:14.777 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-12T11:06:14.777 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-04-12T11:06:14.777 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2024-04-12T11:06:14.777 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to enable service . firewalld.service is not available 2024-04-12T11:06:14.785 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-12T11:06:14.785 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2024-04-12T11:06:14.793 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-12T11:06:14.793 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-04-12T11:06:14.793 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2024-04-12T11:06:14.793 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-04-12T11:06:14.793 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr to start... 2024-04-12T11:06:14.793 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr... 2024-04-12T11:06:14.919 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:14.657+0000 7f8acee10200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:06:14.919 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:14.751+0000 7f8acee10200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:06:14.919 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:14 smithi029 bash[29566]: 314601fec666f567978eb64350aaa551e54d3e23c9abbd0f1c595750425cc7db 2024-04-12T11:06:14.919 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:14 smithi029 systemd[1]: Started Ceph mgr.y for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:06:14.919 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:14.901+0000 7f8acee10200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:06:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:15.001+0000 7f8acee10200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:06:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:15.088+0000 7f8acee10200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:06:15.661 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:15.359+0000 7f8acee10200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:06:15.661 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:15.473+0000 7f8acee10200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:06:15.661 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:15.562+0000 7f8acee10200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:06:15.661 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:15.660+0000 7f8acee10200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:06:16.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:15.840+0000 7f8acee10200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:06:17.169 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:16 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:16.854+0000 7f8acee10200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:06:17.169 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:17 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:16.993+0000 7f8acee10200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:06:17.428 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:17 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:17.333+0000 7f8acee10200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:06:17.428 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:17 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:17.426+0000 7f8acee10200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:06:17.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:17 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:17.634+0000 7f8acee10200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:06:18.143 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:17 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:17.832+0000 7f8acee10200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:06:18.409 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:18 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:18.141+0000 7f8acee10200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:06:18.409 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:18 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:18.408+0000 7f8acee10200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:06:18.766 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:18 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:18.655+0000 7f8acee10200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:06:19.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:19 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:19.510+0000 7f8acee10200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:06:20.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:20 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3932248740' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T11:06:20.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:20 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:20.100+0000 7f8acee10200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:06:20.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:20 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:20.203+0000 7f8acee10200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:06:20.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:20 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:20.430+0000 7f8acee10200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:06:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: Activating manager daemon y 2024-04-12T11:06:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: mgrmap e2: y(active, starting, since 0.00304645s) 2024-04-12T11:06:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: from='mgr.14100 172.21.15.29:0/3061642589' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:06:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: from='mgr.14100 172.21.15.29:0/3061642589' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:06:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: from='mgr.14100 172.21.15.29:0/3061642589' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:06:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: from='mgr.14100 172.21.15.29:0/3061642589' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:06:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: from='mgr.14100 172.21.15.29:0/3061642589' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:06:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: Manager daemon y is now available 2024-04-12T11:06:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: from='mgr.14100 172.21.15.29:0/3061642589' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:06:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: from='mgr.14100 172.21.15.29:0/3061642589' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:06:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: from='mgr.14100 172.21.15.29:0/3061642589' entity='mgr.y' 2024-04-12T11:06:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: from='mgr.14100 172.21.15.29:0/3061642589' entity='mgr.y' 2024-04-12T11:06:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:21 smithi029 ceph-mon[29349]: from='mgr.14100 172.21.15.29:0/3061642589' entity='mgr.y' 2024-04-12T11:06:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:22 smithi029 ceph-mon[29349]: mgrmap e3: y(active, since 1.00694s) 2024-04-12T11:06:23.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:23 smithi029 ceph-mon[29349]: mgrmap e4: y(active, since 2s) 2024-04-12T11:06:30.014 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:30.014 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-04-12T11:06:30.014 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsid": "80efffc6-f8bc-11ee-bc8c-c7b262605968", 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "health": { 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 0 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "a" 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2024-04-12T11:06:30.015 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-12T11:06:30.016 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-12T11:06:30.017 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "restful" 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modified": "2024-04-12T11:06:06.411040+0000", 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-12T11:06:30.018 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-04-12T11:06:30.019 INFO:teuthology.orchestra.run.smithi029.stdout:mgr not available, waiting (1/15)... 2024-04-12T11:06:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:33 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3186971648' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T11:06:34.021 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:34.023 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-04-12T11:06:34.023 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsid": "80efffc6-f8bc-11ee-bc8c-c7b262605968", 2024-04-12T11:06:34.023 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "health": { 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 0 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "a" 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_age": 22, 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-12T11:06:34.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T11:06:34.025 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-04-12T11:06:34.025 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-12T11:06:34.025 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:34.025 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-12T11:06:34.025 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T11:06:34.025 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-12T11:06:34.025 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-12T11:06:34.025 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-12T11:06:34.028 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-12T11:06:34.029 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "restful" 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-04-12T11:06:34.030 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T11:06:34.031 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:34.031 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-12T11:06:34.031 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-12T11:06:34.031 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modified": "2024-04-12T11:06:06.411040+0000", 2024-04-12T11:06:34.031 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-12T11:06:34.031 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-04-12T11:06:34.031 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-12T11:06:34.031 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-04-12T11:06:34.031 INFO:teuthology.orchestra.run.smithi029.stdout:mgr is available 2024-04-12T11:06:36.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:36 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3416276049' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-12T11:06:36.805 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:36.809 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [global] 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout fsid = 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [osd] 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-12T11:06:36.810 INFO:teuthology.orchestra.run.smithi029.stdout:Enabling cephadm module... 2024-04-12T11:06:37.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:37 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3741544594' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-12T11:06:37.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:37 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ignoring --setuser ceph since I am not root 2024-04-12T11:06:37.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:37 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ignoring --setgroup ceph since I am not root 2024-04-12T11:06:38.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:38 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:38.121+0000 7f964cdd9200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:06:38.505 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:38 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:38.217+0000 7f964cdd9200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:06:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:38 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3741544594' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-12T11:06:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:38 smithi029 ceph-mon[29349]: mgrmap e5: y(active, since 17s) 2024-04-12T11:06:39.122 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:38 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:38.884+0000 7f964cdd9200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:06:39.122 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:38 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:38.978+0000 7f964cdd9200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:06:39.123 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:39.121+0000 7f964cdd9200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:06:39.431 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:39.219+0000 7f964cdd9200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:06:39.431 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:39.303+0000 7f964cdd9200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:06:39.434 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-04-12T11:06:39.434 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-12T11:06:39.434 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-12T11:06:39.434 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-12T11:06:39.434 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-12T11:06:39.434 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-04-12T11:06:39.434 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for the mgr to restart... 2024-04-12T11:06:39.434 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr epoch 5... 2024-04-12T11:06:39.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:39 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2431248345' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-12T11:06:39.683 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:39.577+0000 7f964cdd9200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:06:40.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:39.690+0000 7f964cdd9200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:06:40.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:39.778+0000 7f964cdd9200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:06:40.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:39.877+0000 7f964cdd9200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:06:40.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:40 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:40.056+0000 7f964cdd9200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:06:41.513 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:41 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:41.040+0000 7f964cdd9200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:06:41.514 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:41 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:41.175+0000 7f964cdd9200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:06:41.514 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:41 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:41.512+0000 7f964cdd9200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:06:41.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:41 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:41.605+0000 7f964cdd9200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:06:42.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:41 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:41.805+0000 7f964cdd9200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:06:42.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:41 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:41.995+0000 7f964cdd9200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:06:42.547 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:42 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:42.284+0000 7f964cdd9200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:06:42.547 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:42 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:42.546+0000 7f964cdd9200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:06:43.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:42 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:42.787+0000 7f964cdd9200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:06:44.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:43 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:43.609+0000 7f964cdd9200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:06:44.507 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:44.187+0000 7f964cdd9200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:06:44.508 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:44.286+0000 7f964cdd9200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:06:44.508 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:06:44.506+0000 7f964cdd9200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:06:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: Active manager daemon y restarted 2024-04-12T11:06:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: Activating manager daemon y 2024-04-12T11:06:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: osdmap e2: 0 total, 0 up, 0 in 2024-04-12T11:06:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: mgrmap e6: y(active, starting, since 0.00292741s) 2024-04-12T11:06:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:06:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:06:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:06:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:06:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:06:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: Manager daemon y is now available 2024-04-12T11:06:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:06:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:06:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:06:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:44 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:06:45.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:45 smithi029 ceph-mon[29349]: Found migration_current of "None". Setting to last migration. 2024-04-12T11:06:45.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:45 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:45.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:45 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:45.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:45 smithi029 ceph-mon[29349]: mgrmap e7: y(active, since 1.0048s) 2024-04-12T11:06:46.109 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-04-12T11:06:46.109 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-12T11:06:46.109 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-12T11:06:46.109 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-04-12T11:06:46.110 INFO:teuthology.orchestra.run.smithi029.stdout:mgr epoch 5 is available 2024-04-12T11:06:46.110 INFO:teuthology.orchestra.run.smithi029.stdout:Setting orchestrator backend to cephadm... 2024-04-12T11:06:46.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:46 smithi029 ceph-mon[29349]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-12T11:06:46.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:46 smithi029 ceph-mon[29349]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-12T11:06:46.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:46 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:06:47.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:47 smithi029 ceph-mon[29349]: [12/Apr/2024:11:06:45] ENGINE Bus STARTING 2024-04-12T11:06:47.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:47 smithi029 ceph-mon[29349]: [12/Apr/2024:11:06:45] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:06:47.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:47 smithi029 ceph-mon[29349]: [12/Apr/2024:11:06:46] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:06:47.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:47 smithi029 ceph-mon[29349]: [12/Apr/2024:11:06:46] ENGINE Bus STARTED 2024-04-12T11:06:47.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:47 smithi029 ceph-mon[29349]: mgrmap e8: y(active, since 2s) 2024-04-12T11:06:47.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:47 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:47.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:47 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:06:48.454 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-12T11:06:48.454 INFO:teuthology.orchestra.run.smithi029.stdout:Generating ssh key... 2024-04-12T11:06:48.848 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:48 smithi029 ceph-mon[29349]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:06:49.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: Generating public/private rsa key pair. 2024-04-12T11:06:49.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: Your identification has been saved in /tmp/tmp8n16tj9c/key. 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: Your public key has been saved in /tmp/tmp8n16tj9c/key.pub. 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: The key fingerprint is: 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: SHA256:TJWbFwsyRsHcf0Iid2ralSyeLGvM1WqJ8X2vy/7xi/0 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: The key's randomart image is: 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: +---[RSA 3072]----+ 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: | +oo.. | 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: | B.* + | 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: | ..= @ + | 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: | o * O . | 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: | S* * o | 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: | + * . | 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: | o B + . | 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: | * = ..+ o| 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: | . . +=BE| 2024-04-12T11:06:49.519 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: +----[SHA256]-----+ 2024-04-12T11:06:49.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-mon[29349]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:06:49.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:49.882 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:49 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:50.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:50 smithi029 ceph-mon[29349]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:06:50.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:50 smithi029 ceph-mon[29349]: Generating ssh key... 2024-04-12T11:06:51.059 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCao6urvESmtKlXzAMP/Bw6wKEuqm7rrBisAaScLSnsRXAb9tad46XD0r+n3N4iREYkttzlSbgqkD0+lWjShM8vAVmuDWx54F6wV1JD0+dos9rqsPy6ZLvCwe+n/bhWR1jKSRylnz61939juCL1gThZfy68LovWRHKGQhxLsObtjOdf0D3/qeu9KklBHepV9+7JYglk7JbJA/Ra/6vzSu1id8pqLUz/WMGpAgRdSgyy7B26zARk2IVUM5rUgWZ6cedpc/Pk7nP+xLsuwW7qF9fWOr32Tj9Gjzy7C8gh1YwEZjmnuPvbxgY4gmagk5skgKFBymXhVGnLwluypLYtP4HKx10FfMvERKS7qGlT2OLNsQQr28ByGzPXyzdRgxKpKL+rlyy95UdC2V+hgR+zXqSNfV/Km1fRabVbo22GNty40FWpM5bMETbujkA3IUg2zHOl735jREnc2oOMc/YUM5XlnswdSr5lLCE6/EQHsOjSNzO3c2nbc7AHKfEksHMmsMc= ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:06:51.059 INFO:teuthology.orchestra.run.smithi029.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-12T11:06:51.059 INFO:teuthology.orchestra.run.smithi029.stdout:Adding key to root@localhost authorized_keys... 2024-04-12T11:06:51.060 INFO:teuthology.orchestra.run.smithi029.stdout:Adding host smithi029... 2024-04-12T11:06:51.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:51 smithi029 ceph-mon[29349]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:06:52.912 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:52 smithi029 ceph-mon[29349]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi029", "addr": "172.21.15.29", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:06:52.912 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:52 smithi029 ceph-mon[29349]: Deploying cephadm binary to smithi029 2024-04-12T11:06:54.904 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Added host 'smithi029' with addr '172.21.15.29' 2024-04-12T11:06:54.904 INFO:teuthology.orchestra.run.smithi029.stdout:Deploying unmanaged mon service... 2024-04-12T11:06:55.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:55 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:55.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:55 smithi029 ceph-mon[29349]: Added host smithi029 2024-04-12T11:06:55.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:55 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:06:56.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:56 smithi029 ceph-mon[29349]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:06:56.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:56 smithi029 ceph-mon[29349]: Saving service mon spec with placement count:5 2024-04-12T11:06:56.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:56 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:56.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:56 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:56.900 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-12T11:06:56.900 INFO:teuthology.orchestra.run.smithi029.stdout:Deploying unmanaged mgr service... 2024-04-12T11:06:58.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:57 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:58.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:57 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:06:58.521 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-12T11:06:58.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:58 smithi029 ceph-mon[29349]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:06:58.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:06:58 smithi029 ceph-mon[29349]: Saving service mgr spec with placement count:2 2024-04-12T11:07:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:00 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:07:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:00 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/504047944' entity='client.admin' 2024-04-12T11:07:00.475 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:00 smithi029 ceph-mon[29349]: from='mgr.14116 172.21.15.29:0/1983621841' entity='mgr.y' 2024-04-12T11:07:01.635 INFO:teuthology.orchestra.run.smithi029.stdout:Enabling the dashboard module... 2024-04-12T11:07:01.750 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:01 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1076718796' entity='client.admin' 2024-04-12T11:07:02.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:02 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2748339895' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-12T11:07:02.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:02 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ignoring --setuser ceph since I am not root 2024-04-12T11:07:02.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:02 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ignoring --setgroup ceph since I am not root 2024-04-12T11:07:03.280 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:03.033+0000 7efe2d7d9200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:07:03.280 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:03.130+0000 7efe2d7d9200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:07:03.730 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:03 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2748339895' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-12T11:07:03.731 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:03 smithi029 ceph-mon[29349]: mgrmap e9: y(active, since 17s) 2024-04-12T11:07:04.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:03.793+0000 7efe2d7d9200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:07:04.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:03.886+0000 7efe2d7d9200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:07:04.318 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:04.031+0000 7efe2d7d9200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:07:04.318 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:04.128+0000 7efe2d7d9200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:07:04.319 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:04.212+0000 7efe2d7d9200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:07:04.321 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-04-12T11:07:04.322 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-12T11:07:04.322 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-12T11:07:04.322 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-12T11:07:04.322 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-12T11:07:04.322 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-04-12T11:07:04.322 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for the mgr to restart... 2024-04-12T11:07:04.322 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr epoch 9... 2024-04-12T11:07:04.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:04 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3775457544' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-12T11:07:04.574 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:04.487+0000 7efe2d7d9200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:07:04.964 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:04.600+0000 7efe2d7d9200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:07:04.964 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:04.688+0000 7efe2d7d9200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:07:04.964 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:04.786+0000 7efe2d7d9200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:07:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:04.964+0000 7efe2d7d9200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:07:06.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:05 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:05.942+0000 7efe2d7d9200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:07:06.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:06.076+0000 7efe2d7d9200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:07:06.703 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:06.412+0000 7efe2d7d9200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:07:06.703 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:06.503+0000 7efe2d7d9200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:07:06.703 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:06.702+0000 7efe2d7d9200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:07:07.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:06.890+0000 7efe2d7d9200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:07:07.440 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:07.180+0000 7efe2d7d9200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:07:07.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:07.439+0000 7efe2d7d9200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:07:07.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:07.679+0000 7efe2d7d9200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:07:08.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:08.500+0000 7efe2d7d9200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:07:09.397 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:09.076+0000 7efe2d7d9200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:07:09.397 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:09.175+0000 7efe2d7d9200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:07:09.397 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:07:09.396+0000 7efe2d7d9200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:07:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: Active manager daemon y restarted 2024-04-12T11:07:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: Activating manager daemon y 2024-04-12T11:07:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: osdmap e3: 0 total, 0 up, 0 in 2024-04-12T11:07:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: mgrmap e10: y(active, starting, since 0.00313883s) 2024-04-12T11:07:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:07:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:07:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:07:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:07:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:07:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: Manager daemon y is now available 2024-04-12T11:07:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:07:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:07:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:07:10.960 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-04-12T11:07:10.960 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-12T11:07:10.961 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-12T11:07:10.961 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-04-12T11:07:10.961 INFO:teuthology.orchestra.run.smithi029.stdout:mgr epoch 9 is available 2024-04-12T11:07:10.961 INFO:teuthology.orchestra.run.smithi029.stdout:Generating a dashboard self-signed certificate... 2024-04-12T11:07:11.755 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:11 smithi029 ceph-mon[29349]: [12/Apr/2024:11:07:09] ENGINE Bus STARTING 2024-04-12T11:07:11.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:11 smithi029 ceph-mon[29349]: [12/Apr/2024:11:07:10] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:07:11.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:11 smithi029 ceph-mon[29349]: [12/Apr/2024:11:07:10] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:07:11.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:11 smithi029 ceph-mon[29349]: [12/Apr/2024:11:07:10] ENGINE Bus STARTED 2024-04-12T11:07:11.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:11 smithi029 ceph-mon[29349]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-12T11:07:11.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:11 smithi029 ceph-mon[29349]: mgrmap e11: y(active, since 1.0058s) 2024-04-12T11:07:11.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:11 smithi029 ceph-mon[29349]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-12T11:07:11.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:11 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:12.384 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-12T11:07:12.384 INFO:teuthology.orchestra.run.smithi029.stdout:Creating initial admin user... 2024-04-12T11:07:12.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:12 smithi029 ceph-mon[29349]: from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:07:12.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:12 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:12.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:12 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:12.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:12 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:12.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:12 smithi029 ceph-mon[29349]: mgrmap e12: y(active, since 2s) 2024-04-12T11:07:14.538 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$YaPkVriax48ztIikDFbb1.noe0k18wduAJM4JCXxmxfI1WmnE4OXW", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712920033, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-12T11:07:14.538 INFO:teuthology.orchestra.run.smithi029.stdout:Fetching dashboard port number... 2024-04-12T11:07:15.248 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:14 smithi029 ceph-mon[29349]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:07:15.248 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:14 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:15.786 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 8443 2024-04-12T11:07:15.794 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-12T11:07:15.795 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2024-04-12T11:07:15.802 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-12T11:07:15.802 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-04-12T11:07:15.802 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2024-04-12T11:07:15.802 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-12T11:07:15.803 INFO:teuthology.orchestra.run.smithi029.stdout:Ceph Dashboard is now available at: 2024-04-12T11:07:15.803 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:15.803 INFO:teuthology.orchestra.run.smithi029.stdout: URL: https://smithi029.front.sepia.ceph.com:8443/ 2024-04-12T11:07:15.803 INFO:teuthology.orchestra.run.smithi029.stdout: User: admin 2024-04-12T11:07:15.804 INFO:teuthology.orchestra.run.smithi029.stdout: Password: dc7jo6rfro 2024-04-12T11:07:15.804 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:15.804 INFO:teuthology.orchestra.run.smithi029.stdout:Saving cluster configuration to /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config directory 2024-04-12T11:07:15.804 INFO:teuthology.orchestra.run.smithi029.stdout:Enabling autotune for osd_memory_target 2024-04-12T11:07:16.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:15 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1257856898' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-12T11:07:17.254 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:16 smithi029 ceph-mon[29349]: mgrmap e13: y(active, since 6s) 2024-04-12T11:07:18.265 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-12T11:07:18.265 INFO:teuthology.orchestra.run.smithi029.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-12T11:07:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout:Or, if you are only running a single cluster on this host: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: ceph telemetry on 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout:For more information see: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:18.266 INFO:teuthology.orchestra.run.smithi029.stdout:Bootstrap complete. 2024-04-12T11:07:18.310 INFO:tasks.cephadm:Fetching config... 2024-04-12T11:07:18.310 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:07:18.310 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-12T11:07:18.327 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-12T11:07:18.327 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:07:18.327 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-12T11:07:18.381 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-12T11:07:18.381 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:07:18.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-12T11:07:18.449 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-12T11:07:18.449 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:07:18.449 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-12T11:07:18.503 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-12T11:07:18.503 DEBUG:teuthology.orchestra.run.smithi029:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCao6urvESmtKlXzAMP/Bw6wKEuqm7rrBisAaScLSnsRXAb9tad46XD0r+n3N4iREYkttzlSbgqkD0+lWjShM8vAVmuDWx54F6wV1JD0+dos9rqsPy6ZLvCwe+n/bhWR1jKSRylnz61939juCL1gThZfy68LovWRHKGQhxLsObtjOdf0D3/qeu9KklBHepV9+7JYglk7JbJA/Ra/6vzSu1id8pqLUz/WMGpAgRdSgyy7B26zARk2IVUM5rUgWZ6cedpc/Pk7nP+xLsuwW7qF9fWOr32Tj9Gjzy7C8gh1YwEZjmnuPvbxgY4gmagk5skgKFBymXhVGnLwluypLYtP4HKx10FfMvERKS7qGlT2OLNsQQr28ByGzPXyzdRgxKpKL+rlyy95UdC2V+hgR+zXqSNfV/Km1fRabVbo22GNty40FWpM5bMETbujkA3IUg2zHOl735jREnc2oOMc/YUM5XlnswdSr5lLCE6/EQHsOjSNzO3c2nbc7AHKfEksHMmsMc= ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-12T11:07:18.584 INFO:teuthology.orchestra.run.smithi029.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCao6urvESmtKlXzAMP/Bw6wKEuqm7rrBisAaScLSnsRXAb9tad46XD0r+n3N4iREYkttzlSbgqkD0+lWjShM8vAVmuDWx54F6wV1JD0+dos9rqsPy6ZLvCwe+n/bhWR1jKSRylnz61939juCL1gThZfy68LovWRHKGQhxLsObtjOdf0D3/qeu9KklBHepV9+7JYglk7JbJA/Ra/6vzSu1id8pqLUz/WMGpAgRdSgyy7B26zARk2IVUM5rUgWZ6cedpc/Pk7nP+xLsuwW7qF9fWOr32Tj9Gjzy7C8gh1YwEZjmnuPvbxgY4gmagk5skgKFBymXhVGnLwluypLYtP4HKx10FfMvERKS7qGlT2OLNsQQr28ByGzPXyzdRgxKpKL+rlyy95UdC2V+hgR+zXqSNfV/Km1fRabVbo22GNty40FWpM5bMETbujkA3IUg2zHOl735jREnc2oOMc/YUM5XlnswdSr5lLCE6/EQHsOjSNzO3c2nbc7AHKfEksHMmsMc= ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:07:18.599 DEBUG:teuthology.orchestra.run.smithi087:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCao6urvESmtKlXzAMP/Bw6wKEuqm7rrBisAaScLSnsRXAb9tad46XD0r+n3N4iREYkttzlSbgqkD0+lWjShM8vAVmuDWx54F6wV1JD0+dos9rqsPy6ZLvCwe+n/bhWR1jKSRylnz61939juCL1gThZfy68LovWRHKGQhxLsObtjOdf0D3/qeu9KklBHepV9+7JYglk7JbJA/Ra/6vzSu1id8pqLUz/WMGpAgRdSgyy7B26zARk2IVUM5rUgWZ6cedpc/Pk7nP+xLsuwW7qF9fWOr32Tj9Gjzy7C8gh1YwEZjmnuPvbxgY4gmagk5skgKFBymXhVGnLwluypLYtP4HKx10FfMvERKS7qGlT2OLNsQQr28ByGzPXyzdRgxKpKL+rlyy95UdC2V+hgR+zXqSNfV/Km1fRabVbo22GNty40FWpM5bMETbujkA3IUg2zHOl735jREnc2oOMc/YUM5XlnswdSr5lLCE6/EQHsOjSNzO3c2nbc7AHKfEksHMmsMc= ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-12T11:07:18.644 INFO:teuthology.orchestra.run.smithi087.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCao6urvESmtKlXzAMP/Bw6wKEuqm7rrBisAaScLSnsRXAb9tad46XD0r+n3N4iREYkttzlSbgqkD0+lWjShM8vAVmuDWx54F6wV1JD0+dos9rqsPy6ZLvCwe+n/bhWR1jKSRylnz61939juCL1gThZfy68LovWRHKGQhxLsObtjOdf0D3/qeu9KklBHepV9+7JYglk7JbJA/Ra/6vzSu1id8pqLUz/WMGpAgRdSgyy7B26zARk2IVUM5rUgWZ6cedpc/Pk7nP+xLsuwW7qF9fWOr32Tj9Gjzy7C8gh1YwEZjmnuPvbxgY4gmagk5skgKFBymXhVGnLwluypLYtP4HKx10FfMvERKS7qGlT2OLNsQQr28ByGzPXyzdRgxKpKL+rlyy95UdC2V+hgR+zXqSNfV/Km1fRabVbo22GNty40FWpM5bMETbujkA3IUg2zHOl735jREnc2oOMc/YUM5XlnswdSr5lLCE6/EQHsOjSNzO3c2nbc7AHKfEksHMmsMc= ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:07:18.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-12T11:07:18.878 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:07:18.989 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:18 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/987686609' entity='client.admin' 2024-04-12T11:07:21.811 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-12T11:07:21.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-12T11:07:22.095 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:07:22.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:22 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1255844855' entity='client.admin' 2024-04-12T11:07:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:24 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:24 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:24 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi029", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:07:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:24 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:24 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:07:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:24 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:25.243 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi087 2024-04-12T11:07:25.243 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:07:25.243 DEBUG:teuthology.orchestra.run.smithi087:> dd of=/etc/ceph/ceph.conf 2024-04-12T11:07:25.261 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:07:25.261 DEBUG:teuthology.orchestra.run.smithi087:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:07:25.319 INFO:tasks.cephadm:Adding host smithi087 to orchestrator... 2024-04-12T11:07:25.319 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch host add smithi087 2024-04-12T11:07:25.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:25 smithi029 ceph-mon[29349]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:07:25.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:25 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:25.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:25 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:07:25.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:25 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:25.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:25 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:07:25.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:25 smithi029 ceph-mon[29349]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:07:25.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:25 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:25.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:25 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:25.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:25 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:25.520 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:07:26.544 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:26 smithi029 ceph-mon[29349]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:26.544 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:26 smithi029 ceph-mon[29349]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:07:26.544 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:26 smithi029 ceph-mon[29349]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:07:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:27 smithi029 ceph-mon[29349]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi087", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:07:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:28 smithi029 ceph-mon[29349]: Deploying cephadm binary to smithi087 2024-04-12T11:07:29.963 INFO:teuthology.orchestra.run.smithi029.stdout:Added host 'smithi087' with addr '172.21.15.87' 2024-04-12T11:07:30.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:30 smithi029 ceph-mon[29349]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:30.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:30 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:30.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:30 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:07:30.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:30 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:30.596 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch host ls --format=json 2024-04-12T11:07:30.797 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:07:31.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:31 smithi029 ceph-mon[29349]: Added host smithi087 2024-04-12T11:07:31.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:31 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:32.504 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:07:32.504 INFO:teuthology.orchestra.run.smithi029.stdout:[{"addr": "172.21.15.29", "hostname": "smithi029", "labels": [], "status": ""}, {"addr": "172.21.15.87", "hostname": "smithi087", "labels": [], "status": ""}] 2024-04-12T11:07:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:32 smithi029 ceph-mon[29349]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:32 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:33.061 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-12T11:07:33.061 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd crush tunables default 2024-04-12T11:07:33.263 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:07:33.711 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:33 smithi029 ceph-mon[29349]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:07:34.558 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:34 smithi029 ceph-mon[29349]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:35.466 INFO:teuthology.orchestra.run.smithi029.stderr:adjusted tunables profile to default 2024-04-12T11:07:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi087", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:07:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:07:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:07:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/253285884' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-12T11:07:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:35 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:36.117 INFO:tasks.cephadm:Adding mon.a on smithi029 2024-04-12T11:07:36.117 INFO:tasks.cephadm:Adding mon.c on smithi029 2024-04-12T11:07:36.117 INFO:tasks.cephadm:Adding mon.b on smithi087 2024-04-12T11:07:36.117 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch apply mon '3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi087:172.21.15.87=b' 2024-04-12T11:07:36.313 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:36 smithi029 ceph-mon[29349]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:36 smithi029 ceph-mon[29349]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:07:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:36 smithi029 ceph-mon[29349]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:07:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:36 smithi029 ceph-mon[29349]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:36 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/253285884' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-12T11:07:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:36 smithi029 ceph-mon[29349]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T11:07:37.846 INFO:teuthology.orchestra.run.smithi087.stdout:Scheduled mon update... 2024-04-12T11:07:38.408 DEBUG:teuthology.orchestra.run.smithi029:mon.c> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.c.service 2024-04-12T11:07:38.411 DEBUG:teuthology.orchestra.run.smithi087:mon.b> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.b.service 2024-04-12T11:07:38.414 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-12T11:07:38.414 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph mon dump -f json 2024-04-12T11:07:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:38 smithi029 ceph-mon[29349]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:38 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:38 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:07:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:38 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:38 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:07:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:38 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:38 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:07:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:38 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:39 smithi029 ceph-mon[29349]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi087:172.21.15.87=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:07:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:39 smithi029 ceph-mon[29349]: Saving service mon spec with placement smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi087:172.21.15.87=b;count:3 2024-04-12T11:07:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:39 smithi029 ceph-mon[29349]: Deploying daemon mon.b on smithi087 2024-04-12T11:07:40.218 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:40.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:40 smithi029 ceph-mon[29349]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:42.531 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:07:42.531 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":1,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","modified":"2024-04-12T11:06:03.531247Z","created":"2024-04-12T11:06:03.531247Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3300","nonce":0},{"type":"v1","addr":"172.21.15.29:6789","nonce":0}]},"addr":"172.21.15.29:6789/0","public_addr":"172.21.15.29:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T11:07:42.533 INFO:teuthology.orchestra.run.smithi087.stderr:dumped monmap epoch 1 2024-04-12T11:07:42.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:42 smithi029 ceph-mon[29349]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:44.258 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-12T11:07:44.259 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph mon dump -f json 2024-04-12T11:07:44.468 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:07:47.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:07:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: mon.a calling monitor election 2024-04-12T11:07:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: mon.b calling monitor election 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: monmap e2: 2 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: fsmap 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: mgrmap e13: y(active, since 38s) 2024-04-12T11:07:48.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: overall HEALTH_OK 2024-04-12T11:07:48.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:48.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:48.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:48.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:07:48.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:48.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:48 smithi029 ceph-mon[29349]: Deploying daemon mon.c on smithi029 2024-04-12T11:07:48.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:48 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:48 smithi087 ceph-mon[32917]: Deploying daemon mon.c on smithi029 2024-04-12T11:07:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:48 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:49.294 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:07:49.294 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":2,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","modified":"2024-04-12T11:07:42.571618Z","created":"2024-04-12T11:06:03.531247Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3300","nonce":0},{"type":"v1","addr":"172.21.15.29:6789","nonce":0}]},"addr":"172.21.15.29:6789/0","public_addr":"172.21.15.29:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:3300","nonce":0},{"type":"v1","addr":"172.21.15.87:6789","nonce":0}]},"addr":"172.21.15.87:6789/0","public_addr":"172.21.15.87:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-12T11:07:49.296 INFO:teuthology.orchestra.run.smithi087.stderr:dumped monmap epoch 2 2024-04-12T11:07:49.842 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:49 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/2496156649' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T11:07:49.842 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:49 smithi087 ceph-mon[32917]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:49.973 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:49 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/2496156649' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T11:07:49.973 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:49 smithi029 ceph-mon[29349]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:50.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 systemd[1]: Starting Ceph mon.c for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:07:50.844 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-12T11:07:50.844 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph mon dump -f json 2024-04-12T11:07:50.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 podman[39981]: 2024-04-12 11:07:50.531474297 +0000 UTC m=+0.017579909 image pull bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3 2024-04-12T11:07:50.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 podman[39981]: 2024-04-12 11:07:50.669216955 +0000 UTC m=+0.155322553 container create 903a61eba1ba470c97553c87f35a543e56032a5838a7ac748beadbde0977b75e (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , RELEASE=reef-d540eba, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T11:07:50.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 podman[39981]: 2024-04-12 11:07:50.865201116 +0000 UTC m=+0.351306725 container init 903a61eba1ba470c97553c87f35a543e56032a5838a7ac748beadbde0977b75e (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, CEPH_POINT_RELEASE=, RELEASE=reef-d540eba, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True) 2024-04-12T11:07:50.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 podman[39981]: 2024-04-12 11:07:50.868338662 +0000 UTC m=+0.354444260 container start 903a61eba1ba470c97553c87f35a543e56032a5838a7ac748beadbde0977b75e (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-d540eba, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403) 2024-04-12T11:07:50.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T11:07:50.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: ceph version 18.2.2-1187-gd540ebac (d540ebaca6b131a1dd560e7f69e024b133bbaa42) reef (stable), process ceph-mon, pid 2 2024-04-12T11:07:50.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: pidfile_write: ignore empty --pid-file 2024-04-12T11:07:51.057 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:07:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: load: jerasure load: lrc 2024-04-12T11:07:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: RocksDB version: 7.9.2 2024-04-12T11:07:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Git sha 0 2024-04-12T11:07:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Compile date 2024-04-08 23:21:21 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: DB SUMMARY 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: DB Session ID: MFM0D36A88XTM6BNJKYC 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: CURRENT file: CURRENT 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.error_if_exists: 0 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.create_if_missing: 0 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.paranoid_checks: 1 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T11:07:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.env: 0x55c2341ee7a0 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.info_log: 0x55c2372c9500 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.statistics: (nil) 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.use_fsync: 0 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_log_file_size: 0 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.allow_fallocate: 1 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T11:07:51.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T11:07:51.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.use_direct_reads: 0 2024-04-12T11:07:51.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T11:07:51.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T11:07:51.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.db_log_dir: 2024-04-12T11:07:51.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.wal_dir: 2024-04-12T11:07:51.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T11:07:51.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T11:07:51.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T11:07:51.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.write_buffer_manager: 0x55c23653e140 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T11:07:51.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.unordered_write: 0 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.row_cache: None 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.wal_filter: None 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.two_write_queues: 0 2024-04-12T11:07:51.274 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.wal_compression: 0 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.atomic_flush: 0 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.log_readahead_size: 0 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T11:07:51.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_background_jobs: 2 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_background_compactions: -1 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_subcompactions: 1 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_open_files: -1 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T11:07:51.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_background_flushes: -1 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Compression algorithms supported: 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: kZSTD supported: 0 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: kXpressCompression supported: 0 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: kZlibCompression supported: 1 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: kSnappyCompression supported: 1 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: kLZ4Compression supported: 1 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: kBZip2Compression supported: 0 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T11:07:51.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.merge_operator: 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_filter: None 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c2372c9600) 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks: 1 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_top_level_index_and_filter: 1 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_type: 0 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_index_type: 0 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_shortening: 1 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T11:07:51.278 INFO:journalctl@ceph.mon.c.smithi029.stdout: checksum: 4 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: no_block_cache: 0 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache: 0x55c2365df1f0 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_name: BinnedLRUCache 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_options: 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: capacity : 536870912 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: num_shard_bits : 4 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: strict_capacity_limit : 0 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: high_pri_pool_ratio: 0.000 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_compressed: (nil) 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: persistent_cache: (nil) 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size: 4096 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size_deviation: 10 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_restart_interval: 16 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_block_restart_interval: 1 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: metadata_block_size: 4096 2024-04-12T11:07:51.279 INFO:journalctl@ceph.mon.c.smithi029.stdout: partition_filters: 0 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: use_delta_encoding: 1 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: filter_policy: bloomfilter 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: whole_key_filtering: 1 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: verify_compression: 0 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: read_amp_bytes_per_bit: 0 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: format_version: 5 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_index_compression: 1 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_align: 0 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: max_auto_readahead_size: 262144 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: prepopulate_block_cache: 0 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: initial_auto_readahead_size: 8192 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compression: NoCompression 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T11:07:51.280 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.num_levels: 7 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T11:07:51.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T11:07:51.282 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T11:07:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.table_properties_collectors: 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.inplace_update_support: 0 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.bloom_locality: 0 2024-04-12T11:07:51.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.max_successive_merges: 0 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.ttl: 2592000 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.enable_blob_files: false 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.min_blob_size: 0 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T11:07:51.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: afee1398-990a-44be-958f-7bc352c06785 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920070897813, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920070898220, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1747, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712920070, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "afee1398-990a-44be-958f-7bc352c06785", "db_session_id": "MFM0D36A88XTM6BNJKYC", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920070898299, "job": 1, "event": "recovery_finished"} 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T11:07:51.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c23667e000 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: DB pointer 0x55c23666c000 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** DB Stats ** 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T11:07:51.287 INFO:journalctl@ceph.mon.c.smithi029.stdout: L0 1/0 1.71 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: Sum 1/0 1.71 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T11:07:51.288 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative compaction: 0.00 GB write, 0.70 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval compaction: 0.00 GB write, 0.70 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout: Block cache BinnedLRUCache@0x55c2365df1f0#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: using public_addrv [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: starting mon.c rank -1 at public addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] at bind addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(???) e0 preinit fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:07:51.289 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).mds e1 new map 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout: e1 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout: legacy client fscid: -1 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout: No filesystems configured 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-12T11:07:51.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi087", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/253285884' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/253285884' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:51.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi087:172.21.15.87=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: Saving service mon spec with placement smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi087:172.21.15.87=b;count:3 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: Deploying daemon mon.b on smithi087 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.a calling monitor election 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.b calling monitor election 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:51.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: monmap e2: 2 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: fsmap 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mgrmap e13: y(active, since 38s) 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: overall HEALTH_OK 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: Deploying daemon mon.c on smithi029 2024-04-12T11:07:51.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:51.295 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/2496156649' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T11:07:51.295 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:51.295 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 ceph-mon[39999]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-12T11:07:51.295 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 bash[39981]: 903a61eba1ba470c97553c87f35a543e56032a5838a7ac748beadbde0977b75e 2024-04-12T11:07:51.295 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:50 smithi029 systemd[1]: Started Ceph mon.c for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:07:56.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: mon.a calling monitor election 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: mon.b calling monitor election 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: mon.c calling monitor election 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: fsmap 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: mgrmap e13: y(active, since 46s) 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: overall HEALTH_OK 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: mon.a calling monitor election 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: mon.b calling monitor election 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: mon.c calling monitor election 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:07:56.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T11:07:56.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: fsmap 2024-04-12T11:07:56.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T11:07:56.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: mgrmap e13: y(active, since 46s) 2024-04-12T11:07:56.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: overall HEALTH_OK 2024-04-12T11:07:56.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.408 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-12T11:07:56.408 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":3,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","modified":"2024-04-12T11:07:50.917802Z","created":"2024-04-12T11:06:03.531247Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3300","nonce":0},{"type":"v1","addr":"172.21.15.29:6789","nonce":0}]},"addr":"172.21.15.29:6789/0","public_addr":"172.21.15.29:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:3300","nonce":0},{"type":"v1","addr":"172.21.15.87:6789","nonce":0}]},"addr":"172.21.15.87:6789/0","public_addr":"172.21.15.87:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3301","nonce":0},{"type":"v1","addr":"172.21.15.29:6790","nonce":0}]},"addr":"172.21.15.29:6790/0","public_addr":"172.21.15.29:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-12T11:07:56.408 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:07:56.408 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: mon.a calling monitor election 2024-04-12T11:07:56.408 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:07:56.408 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.408 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: mon.b calling monitor election 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: mon.c calling monitor election 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: fsmap 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T11:07:56.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: mgrmap e13: y(active, since 46s) 2024-04-12T11:07:56.410 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: overall HEALTH_OK 2024-04-12T11:07:56.410 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.410 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.410 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.410 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:56.410 INFO:teuthology.orchestra.run.smithi087.stderr:dumped monmap epoch 3 2024-04-12T11:07:56.971 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-12T11:07:56.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph config generate-minimal-conf 2024-04-12T11:07:57.187 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:07:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:56 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:07:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:56 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/2414529778' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T11:07:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:56 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:57.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:56 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:07:57.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:56 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/2414529778' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T11:07:57.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:56 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:57.277 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:56 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:07:57.277 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:56 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/2414529778' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T11:07:57.277 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:56 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:07:58.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:58.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:58.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:07:58.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:07:58.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:07:58.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:07:58.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:58 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:58.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:07:58.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:07:58.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:07:58.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:58 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:59.049 INFO:teuthology.orchestra.run.smithi029.stdout:# minimal ceph.conf for 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:07:59.049 INFO:teuthology.orchestra.run.smithi029.stdout:[global] 2024-04-12T11:07:59.049 INFO:teuthology.orchestra.run.smithi029.stdout: fsid = 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:07:59.049 INFO:teuthology.orchestra.run.smithi029.stdout: mon_host = [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] 2024-04-12T11:07:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[29349]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:07:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[29349]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:07:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[29349]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[29349]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[29349]: Reconfiguring mon.a (unknown last config time)... 2024-04-12T11:07:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[29349]: Reconfiguring daemon mon.a on smithi029 2024-04-12T11:07:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2955174456' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:59.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[39999]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:07:59.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:07:59.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:59.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[39999]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:59.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[39999]: Reconfiguring mon.a (unknown last config time)... 2024-04-12T11:07:59.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[39999]: Reconfiguring daemon mon.a on smithi029 2024-04-12T11:07:59.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:07:59 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2955174456' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:07:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:59 smithi087 ceph-mon[32917]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:07:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:59 smithi087 ceph-mon[32917]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:07:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:59 smithi087 ceph-mon[32917]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:59 smithi087 ceph-mon[32917]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:07:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:59 smithi087 ceph-mon[32917]: Reconfiguring mon.a (unknown last config time)... 2024-04-12T11:07:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:59 smithi087 ceph-mon[32917]: Reconfiguring daemon mon.a on smithi029 2024-04-12T11:07:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:07:59 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2955174456' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:00.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[29349]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:00.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:00.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:00.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:00.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:08:00.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:08:00.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:00.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[39999]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:00.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:00.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:00.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:00.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:08:00.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:08:00.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:00.556 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-12T11:08:00.556 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:08:00.556 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/ceph/ceph.conf 2024-04-12T11:08:00.586 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:08:00.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:08:00.653 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:08:00.653 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/ceph/ceph.conf 2024-04-12T11:08:00.688 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:08:00.688 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:08:00.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:00 smithi087 ceph-mon[32917]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:08:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:08:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:00.747 INFO:tasks.cephadm:Adding mgr.y on smithi029 2024-04-12T11:08:00.747 INFO:tasks.cephadm:Adding mgr.x on smithi087 2024-04-12T11:08:00.747 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch apply mgr '2;smithi029=y;smithi087=x' 2024-04-12T11:08:00.984 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:08:01.525 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:01 smithi029 ceph-mon[29349]: Reconfiguring mon.c (monmap changed)... 2024-04-12T11:08:01.525 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:01 smithi029 ceph-mon[29349]: Reconfiguring daemon mon.c on smithi029 2024-04-12T11:08:01.525 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:01 smithi029 ceph-mon[39999]: Reconfiguring mon.c (monmap changed)... 2024-04-12T11:08:01.525 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:01 smithi029 ceph-mon[39999]: Reconfiguring daemon mon.c on smithi029 2024-04-12T11:08:01.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:01 smithi087 ceph-mon[32917]: Reconfiguring mon.c (monmap changed)... 2024-04-12T11:08:01.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:01 smithi087 ceph-mon[32917]: Reconfiguring daemon mon.c on smithi029 2024-04-12T11:08:02.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:02 smithi087 ceph-mon[32917]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:02.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:02 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:02.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:02 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:02.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:02 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:08:02.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:02 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:08:02.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:02 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:02.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[39999]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:02.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:02.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:02.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:08:02.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:08:02.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:02.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[29349]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:02.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:02.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:02.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:08:02.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:08:02.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:02 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:03.319 INFO:teuthology.orchestra.run.smithi087.stdout:Scheduled mgr update... 2024-04-12T11:08:03.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:03 smithi087 ceph-mon[32917]: Reconfiguring mon.b (monmap changed)... 2024-04-12T11:08:03.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:03 smithi087 ceph-mon[32917]: Reconfiguring daemon mon.b on smithi087 2024-04-12T11:08:03.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:03 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:03 smithi029 ceph-mon[29349]: Reconfiguring mon.b (monmap changed)... 2024-04-12T11:08:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:03 smithi029 ceph-mon[29349]: Reconfiguring daemon mon.b on smithi087 2024-04-12T11:08:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:03 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:03 smithi029 ceph-mon[39999]: Reconfiguring mon.b (monmap changed)... 2024-04-12T11:08:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:03 smithi029 ceph-mon[39999]: Reconfiguring daemon mon.b on smithi087 2024-04-12T11:08:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:03 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:04.027 DEBUG:teuthology.orchestra.run.smithi087:mgr.x> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.x.service 2024-04-12T11:08:04.029 INFO:tasks.cephadm:Deploying OSDs... 2024-04-12T11:08:04.030 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:08:04.030 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/scratch_devs of=/dev/stdout 2024-04-12T11:08:04.048 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-12T11:08:04.048 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_1 2024-04-12T11:08:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-12T11:08:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T11:08:04.106 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-12T11:08:04.106 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T11:08:04.106 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T11:08:04.106 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2024-04-12 11:07:50.351311575 +0000 2024-04-12T11:08:04.106 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2024-04-12 11:01:01.430011338 +0000 2024-04-12T11:08:04.106 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2024-04-12 11:01:01.430011338 +0000 2024-04-12T11:08:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: 2024-04-12 11:01:01.430011338 +0000 2024-04-12T11:08:04.106 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-12T11:08:04.175 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2024-04-12T11:08:04.175 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2024-04-12T11:08:04.175 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 0.000123503 s, 4.1 MB/s 2024-04-12T11:08:04.176 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-12T11:08:04.233 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_2 2024-04-12T11:08:04.290 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-12T11:08:04.290 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T11:08:04.290 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-04-12T11:08:04.290 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T11:08:04.290 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T11:08:04.290 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2024-04-12 11:07:50.351311575 +0000 2024-04-12T11:08:04.290 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2024-04-12 11:01:01.772005730 +0000 2024-04-12T11:08:04.291 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2024-04-12 11:01:01.772005730 +0000 2024-04-12T11:08:04.291 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: 2024-04-12 11:01:01.772005730 +0000 2024-04-12T11:08:04.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-12T11:08:04.357 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2024-04-12T11:08:04.357 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2024-04-12T11:08:04.358 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 0.000104445 s, 4.9 MB/s 2024-04-12T11:08:04.359 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-12T11:08:04.415 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_3 2024-04-12T11:08:04.471 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-12T11:08:04.471 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T11:08:04.471 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-04-12T11:08:04.472 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T11:08:04.472 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T11:08:04.472 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2024-04-12 11:07:50.352311559 +0000 2024-04-12T11:08:04.472 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2024-04-12 11:01:02.100000352 +0000 2024-04-12T11:08:04.472 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2024-04-12 11:01:02.100000352 +0000 2024-04-12T11:08:04.472 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: 2024-04-12 11:01:02.100000352 +0000 2024-04-12T11:08:04.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-12T11:08:04.539 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2024-04-12T11:08:04.539 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2024-04-12T11:08:04.539 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 0.00011606 s, 4.4 MB/s 2024-04-12T11:08:04.540 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-12T11:08:04.597 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_4 2024-04-12T11:08:04.635 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi029=y;smithi087=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:08:04.635 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: Saving service mgr spec with placement smithi029=y;smithi087=x;count:2 2024-04-12T11:08:04.635 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:04.636 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:04.636 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:04.636 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:04.636 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:04.636 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:04.636 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:04.636 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:08:04.636 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-12T11:08:04.636 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:08:04.636 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-12T11:08:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T11:08:04.652 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-12T11:08:04.652 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T11:08:04.652 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T11:08:04.652 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2024-04-12 11:07:50.352311559 +0000 2024-04-12T11:08:04.652 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2024-04-12 11:01:02.434994859 +0000 2024-04-12T11:08:04.652 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2024-04-12 11:01:02.434994859 +0000 2024-04-12T11:08:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: 2024-04-12 11:01:02.434994859 +0000 2024-04-12T11:08:04.653 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-12T11:08:04.719 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2024-04-12T11:08:04.719 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2024-04-12T11:08:04.719 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 9.3526e-05 s, 5.5 MB/s 2024-04-12T11:08:04.720 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-12T11:08:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi029=y;smithi087=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:08:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: Saving service mgr spec with placement smithi029=y;smithi087=x;count:2 2024-04-12T11:08:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:08:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-12T11:08:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:08:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi029=y;smithi087=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:08:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: Saving service mgr spec with placement smithi029=y;smithi087=x;count:2 2024-04-12T11:08:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:08:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-12T11:08:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:08:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:04.776 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:08:04.776 DEBUG:teuthology.orchestra.run.smithi087:> dd if=/scratch_devs of=/dev/stdout 2024-04-12T11:08:04.793 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-12T11:08:04.793 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_1 2024-04-12T11:08:04.847 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-12T11:08:04.847 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T11:08:04.848 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-12T11:08:04.848 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T11:08:04.848 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T11:08:04.848 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2024-04-12 11:07:41.534050422 +0000 2024-04-12T11:08:04.848 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2024-04-12 11:00:43.765365138 +0000 2024-04-12T11:08:04.848 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2024-04-12 11:00:43.765365138 +0000 2024-04-12T11:08:04.848 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: 2024-04-12 11:00:43.765365138 +0000 2024-04-12T11:08:04.848 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-12T11:08:04.914 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2024-04-12T11:08:04.915 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2024-04-12T11:08:04.915 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000258068 s, 2.0 MB/s 2024-04-12T11:08:04.916 DEBUG:teuthology.orchestra.run.smithi087:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-12T11:08:04.971 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_2 2024-04-12T11:08:05.026 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-12T11:08:05.026 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T11:08:05.026 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-04-12T11:08:05.026 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T11:08:05.026 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T11:08:05.026 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2024-04-12 11:07:41.534050422 +0000 2024-04-12T11:08:05.027 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2024-04-12 11:00:44.107357234 +0000 2024-04-12T11:08:05.027 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2024-04-12 11:00:44.107357234 +0000 2024-04-12T11:08:05.027 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: 2024-04-12 11:00:44.107357234 +0000 2024-04-12T11:08:05.027 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-12T11:08:05.091 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2024-04-12T11:08:05.091 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2024-04-12T11:08:05.091 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000197215 s, 2.6 MB/s 2024-04-12T11:08:05.092 DEBUG:teuthology.orchestra.run.smithi087:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-12T11:08:05.148 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_3 2024-04-12T11:08:05.202 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-12T11:08:05.202 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T11:08:05.202 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-12T11:08:05.202 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T11:08:05.202 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T11:08:05.202 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2024-04-12 11:07:41.535050400 +0000 2024-04-12T11:08:05.202 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2024-04-12 11:00:44.454349213 +0000 2024-04-12T11:08:05.202 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2024-04-12 11:00:44.454349213 +0000 2024-04-12T11:08:05.203 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: 2024-04-12 11:00:44.454349213 +0000 2024-04-12T11:08:05.203 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-12T11:08:05.267 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2024-04-12T11:08:05.267 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2024-04-12T11:08:05.267 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.00016768 s, 3.1 MB/s 2024-04-12T11:08:05.268 DEBUG:teuthology.orchestra.run.smithi087:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-12T11:08:05.323 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_4 2024-04-12T11:08:05.377 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-12T11:08:05.378 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T11:08:05.378 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-04-12T11:08:05.378 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T11:08:05.378 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T11:08:05.378 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2024-04-12 11:07:41.535050400 +0000 2024-04-12T11:08:05.378 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2024-04-12 11:00:44.795341331 +0000 2024-04-12T11:08:05.378 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2024-04-12 11:00:44.795341331 +0000 2024-04-12T11:08:05.378 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: 2024-04-12 11:00:44.795341331 +0000 2024-04-12T11:08:05.378 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-12T11:08:05.571 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:05 smithi087 ceph-mon[32917]: Deploying daemon mgr.x on smithi087 2024-04-12T11:08:05.572 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2024-04-12T11:08:05.573 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2024-04-12T11:08:05.573 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000200057 s, 2.6 MB/s 2024-04-12T11:08:05.574 DEBUG:teuthology.orchestra.run.smithi087:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-12T11:08:05.589 INFO:tasks.cephadm:Deploying osd.0 on smithi029 with /dev/vg_nvme/lv_4... 2024-04-12T11:08:05.589 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-12T11:08:05.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:05 smithi029 ceph-mon[29349]: Deploying daemon mgr.x on smithi087 2024-04-12T11:08:05.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:05 smithi029 ceph-mon[39999]: Deploying daemon mgr.x on smithi087 2024-04-12T11:08:05.800 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:08:05.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:05 smithi087 systemd[1]: Starting Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:08:06.261 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:05 smithi087 podman[34777]: 2024-04-12 11:08:05.887300909 +0000 UTC m=+0.017836987 image pull bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3 2024-04-12T11:08:06.261 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:05 smithi087 podman[34777]: 2024-04-12 11:08:05.99133387 +0000 UTC m=+0.121869935 container create 6117969a131454edf79c97d1d333181f327a53f4faf670858971fc1dad4a0448 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240403, RELEASE=reef-d540eba, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T11:08:06.261 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:06 smithi087 podman[34777]: 2024-04-12 11:08:06.185446088 +0000 UTC m=+0.315982160 container init 6117969a131454edf79c97d1d333181f327a53f4faf670858971fc1dad4a0448 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, ceph=True, RELEASE=reef-d540eba, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403) 2024-04-12T11:08:06.261 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:06 smithi087 podman[34777]: 2024-04-12 11:08:06.188117727 +0000 UTC m=+0.318653790 container start 6117969a131454edf79c97d1d333181f327a53f4faf670858971fc1dad4a0448 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=reef-d540eba, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-04-12T11:08:06.261 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:06 smithi087 ceph-mgr[34796]: -- 172.21.15.87:0/1014182669 <== mon.1 v2:172.21.15.87:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560acebd81a0 con 0x560acebaec00 2024-04-12T11:08:06.261 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:06 smithi087 bash[34777]: 6117969a131454edf79c97d1d333181f327a53f4faf670858971fc1dad4a0448 2024-04-12T11:08:06.513 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:06 smithi087 systemd[1]: Started Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:08:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[29349]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:06.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[39999]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:06.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:06 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:06.867 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:06.760+0000 7f18de7e5200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:08:06.867 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:06.865+0000 7f18de7e5200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:08:06.867 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:06 smithi087 ceph-mon[32917]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:06.867 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:06 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.867 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:06 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.867 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:06 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.867 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:06 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:06.867 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:06 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:07.785 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-12T11:08:07.785 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-12T11:08:07.785 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2024-04-12T11:08:07.785 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2024-04-12T11:08:07.785 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0085719 s, 1.2 GB/s 2024-04-12T11:08:07.785 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2024-04-12T11:08:07.862 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:07 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:07.527+0000 7f18de7e5200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:08:07.862 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:07 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:07.626+0000 7f18de7e5200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:08:07.862 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:07 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:07.766+0000 7f18de7e5200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:08:07.862 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:07 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:07.860+0000 7f18de7e5200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:08:08.201 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:07 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:07.942+0000 7f18de7e5200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:08:08.201 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:08.200+0000 7f18de7e5200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:08:08.422 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi029:vg_nvme/lv_4 2024-04-12T11:08:08.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:08.308+0000 7f18de7e5200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:08:08.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:08.392+0000 7f18de7e5200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:08:08.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[39999]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:08.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:08.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[29349]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:08.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:08.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:08:08.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:08:08.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:08.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:08.484+0000 7f18de7e5200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:08:08.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:08.660+0000 7f18de7e5200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:08:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-mon[32917]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:08:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:08:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:09.522 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:08:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:09 smithi029 ceph-mon[39999]: Reconfiguring mgr.y (unknown last config time)... 2024-04-12T11:08:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:09 smithi029 ceph-mon[39999]: Reconfiguring daemon mgr.y on smithi029 2024-04-12T11:08:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:09 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:09 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:09 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:09 smithi029 ceph-mon[29349]: Reconfiguring mgr.y (unknown last config time)... 2024-04-12T11:08:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:09 smithi029 ceph-mon[29349]: Reconfiguring daemon mgr.y on smithi029 2024-04-12T11:08:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:09 smithi087 ceph-mon[32917]: Reconfiguring mgr.y (unknown last config time)... 2024-04-12T11:08:09.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:09 smithi087 ceph-mon[32917]: Reconfiguring daemon mgr.y on smithi029 2024-04-12T11:08:09.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:09 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:09.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:09 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:09.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:09 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:09.970 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:09 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:09.624+0000 7f18de7e5200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:08:09.970 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:09 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:09.753+0000 7f18de7e5200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:08:10.360 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:10.082+0000 7f18de7e5200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:08:10.360 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:10.170+0000 7f18de7e5200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:08:10.360 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:10.359+0000 7f18de7e5200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:08:10.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:10 smithi029 ceph-mon[29349]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:10.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:10 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:10.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:10 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:10.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:10 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:10.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:10 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:10.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:10 smithi029 ceph-mon[39999]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:10.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:10 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:10.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:10 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:10.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:10 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:10.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:10 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:10.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:10.538+0000 7f18de7e5200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:08:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:10 smithi087 ceph-mon[32917]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:10 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:10 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:10 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:10 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:11.074 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:10.817+0000 7f18de7e5200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:08:11.075 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:11 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:11.073+0000 7f18de7e5200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:08:11.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:11 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:11.305+0000 7f18de7e5200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:08:11.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:11 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:08:11.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:11 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:08:11.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:11 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:11.805 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:11 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:08:11.805 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:11 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:08:11.805 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:11 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:11 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:08:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:11 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:08:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:11 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:12.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:12.099+0000 7f18de7e5200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:08:12.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:12 smithi029 ceph-mon[29349]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:08:12.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:12 smithi029 ceph-mon[29349]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:12.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:12 smithi029 ceph-mon[39999]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:08:12.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:12 smithi029 ceph-mon[39999]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:12 smithi087 ceph-mon[32917]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:08:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:12 smithi087 ceph-mon[32917]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:12.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:12.656+0000 7f18de7e5200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:08:12.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:12.753+0000 7f18de7e5200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:08:12.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:08:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:08:12.965+0000 7f18de7e5200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:08:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:13 smithi087 ceph-mon[32917]: Standby manager daemon x started 2024-04-12T11:08:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:13 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:08:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:13 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:08:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:13 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:08:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:13 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:08:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:13 smithi029 ceph-mon[39999]: Standby manager daemon x started 2024-04-12T11:08:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:13 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:08:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:13 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:08:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:13 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:08:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:13 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:08:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:13 smithi029 ceph-mon[29349]: Standby manager daemon x started 2024-04-12T11:08:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:13 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:08:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:13 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:08:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:13 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:08:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:13 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1565292153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:08:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:14 smithi087 ceph-mon[32917]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:14 smithi087 ceph-mon[32917]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-12T11:08:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:14 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:08:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:14 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3600699334' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c3c7583-fa1d-4a1d-95ea-77be7e31737e"}]: dispatch 2024-04-12T11:08:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:14 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3600699334' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4c3c7583-fa1d-4a1d-95ea-77be7e31737e"}]': finished 2024-04-12T11:08:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:14 smithi087 ceph-mon[32917]: osdmap e5: 1 total, 0 up, 1 in 2024-04-12T11:08:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:14 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:14 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3626838453' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:08:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[29349]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[29349]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-12T11:08:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:08:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3600699334' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c3c7583-fa1d-4a1d-95ea-77be7e31737e"}]: dispatch 2024-04-12T11:08:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3600699334' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4c3c7583-fa1d-4a1d-95ea-77be7e31737e"}]': finished 2024-04-12T11:08:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[29349]: osdmap e5: 1 total, 0 up, 1 in 2024-04-12T11:08:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3626838453' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:08:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[39999]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[39999]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-12T11:08:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:08:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3600699334' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c3c7583-fa1d-4a1d-95ea-77be7e31737e"}]: dispatch 2024-04-12T11:08:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3600699334' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4c3c7583-fa1d-4a1d-95ea-77be7e31737e"}]': finished 2024-04-12T11:08:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[39999]: osdmap e5: 1 total, 0 up, 1 in 2024-04-12T11:08:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:14 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3626838453' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:08:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:15 smithi087 ceph-mon[32917]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:15 smithi029 ceph-mon[29349]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:15 smithi029 ceph-mon[39999]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:18.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:18 smithi029 ceph-mon[39999]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:18.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:18 smithi029 ceph-mon[29349]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:18.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:18 smithi087 ceph-mon[32917]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:20.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:20 smithi029 ceph-mon[39999]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:20.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:20 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T11:08:20.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:20 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:20.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:20 smithi029 ceph-mon[29349]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:20.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:20 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T11:08:20.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:20 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:20 smithi087 ceph-mon[32917]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:20 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T11:08:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:20 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:21 smithi029 ceph-mon[29349]: Deploying daemon osd.0 on smithi029 2024-04-12T11:08:21.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:21 smithi029 ceph-mon[39999]: Deploying daemon osd.0 on smithi029 2024-04-12T11:08:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:21 smithi087 ceph-mon[32917]: Deploying daemon osd.0 on smithi029 2024-04-12T11:08:22.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:22 smithi029 ceph-mon[29349]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:22.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:22 smithi029 ceph-mon[39999]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:22 smithi087 ceph-mon[32917]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:24.596 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:24 smithi029 ceph-mon[29349]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:24.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:24 smithi029 ceph-mon[39999]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:24 smithi087 ceph-mon[32917]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:26.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:26 smithi087 ceph-mon[32917]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:26 smithi029 ceph-mon[29349]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:26 smithi029 ceph-mon[39999]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:26.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:26.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:26.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:28.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:28 smithi087 ceph-mon[32917]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:28.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:28 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:28.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:28 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:28.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:28 smithi029 ceph-mon[29349]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:28.761 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:28 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:28.761 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:28 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:28.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:28 smithi029 ceph-mon[39999]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:28.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:28 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:28.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:28 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:28.834 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 0 on host 'smithi029' 2024-04-12T11:08:29.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:29.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:29.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:29.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[29349]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T11:08:29.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:29.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:29.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:29.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:29.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:29.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:29.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[39999]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T11:08:29.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:29.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:29.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:29.441 DEBUG:teuthology.orchestra.run.smithi029:osd.0> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.0.service 2024-04-12T11:08:29.443 INFO:tasks.cephadm:Deploying osd.1 on smithi029 with /dev/vg_nvme/lv_3... 2024-04-12T11:08:29.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-12T11:08:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:29 smithi087 ceph-mon[32917]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T11:08:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:30.183 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:08:30.547 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-mon[29349]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:30.547 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-mon[29349]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T11:08:30.547 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-mon[29349]: osdmap e6: 1 total, 0 up, 1 in 2024-04-12T11:08:30.547 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:30.547 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-mon[29349]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:08:30.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-mon[39999]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:30.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-mon[39999]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T11:08:30.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-mon[39999]: osdmap e6: 1 total, 0 up, 1 in 2024-04-12T11:08:30.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:30.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-mon[39999]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:08:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:30 smithi087 ceph-mon[32917]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:30 smithi087 ceph-mon[32917]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T11:08:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:30 smithi087 ceph-mon[32917]: osdmap e6: 1 total, 0 up, 1 in 2024-04-12T11:08:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:30 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:30 smithi087 ceph-mon[32917]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:08:31.104 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[43889]: 2024-04-12T11:08:30.802+0000 7f316320a700 -1 osd.0 0 waiting for initial osdmap 2024-04-12T11:08:31.104 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:08:30 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[43889]: 2024-04-12T11:08:30.805+0000 7f315a7f6700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:08:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[29349]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:08:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[29349]: osdmap e7: 1 total, 0 up, 1 in 2024-04-12T11:08:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[29349]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' 2024-04-12T11:08:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[29349]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:31.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[39999]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:08:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[39999]: osdmap e7: 1 total, 0 up, 1 in 2024-04-12T11:08:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[39999]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' 2024-04-12T11:08:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:31 smithi029 ceph-mon[39999]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:31 smithi087 ceph-mon[32917]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:08:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:31 smithi087 ceph-mon[32917]: osdmap e7: 1 total, 0 up, 1 in 2024-04-12T11:08:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:31 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:31 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:31 smithi087 ceph-mon[32917]: from='osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597]' entity='osd.0' 2024-04-12T11:08:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:31 smithi087 ceph-mon[32917]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T11:08:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[29349]: purged_snaps scrub starts 2024-04-12T11:08:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[29349]: purged_snaps scrub ok 2024-04-12T11:08:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:32.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[29349]: osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597] boot 2024-04-12T11:08:32.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[29349]: osdmap e8: 1 total, 1 up, 1 in 2024-04-12T11:08:32.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:32.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[39999]: purged_snaps scrub starts 2024-04-12T11:08:32.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[39999]: purged_snaps scrub ok 2024-04-12T11:08:32.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:32.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[39999]: osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597] boot 2024-04-12T11:08:32.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[39999]: osdmap e8: 1 total, 1 up, 1 in 2024-04-12T11:08:32.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:32 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:32.948 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-12T11:08:32.948 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-12T11:08:32.948 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2024-04-12T11:08:32.948 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2024-04-12T11:08:32.948 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00967423 s, 1.1 GB/s 2024-04-12T11:08:32.948 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2024-04-12T11:08:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:32 smithi087 ceph-mon[32917]: purged_snaps scrub starts 2024-04-12T11:08:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:32 smithi087 ceph-mon[32917]: purged_snaps scrub ok 2024-04-12T11:08:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:32 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:32 smithi087 ceph-mon[32917]: osd.0 [v2:172.21.15.29:6802/2299659597,v1:172.21.15.29:6803/2299659597] boot 2024-04-12T11:08:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:32 smithi087 ceph-mon[32917]: osdmap e8: 1 total, 1 up, 1 in 2024-04-12T11:08:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:32 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:08:34.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:33 smithi087 ceph-mon[32917]: osdmap e9: 1 total, 1 up, 1 in 2024-04-12T11:08:34.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:33 smithi087 ceph-mon[32917]: pgmap v41: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:34.231 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:33 smithi029 ceph-mon[29349]: osdmap e9: 1 total, 1 up, 1 in 2024-04-12T11:08:34.232 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:33 smithi029 ceph-mon[29349]: pgmap v41: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:34.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:33 smithi029 ceph-mon[39999]: osdmap e9: 1 total, 1 up, 1 in 2024-04-12T11:08:34.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:33 smithi029 ceph-mon[39999]: pgmap v41: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:34.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi029:vg_nvme/lv_3 2024-04-12T11:08:34.468 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:08:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:34 smithi087 ceph-mon[32917]: Detected new or changed devices on smithi029 2024-04-12T11:08:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:34 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:34 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:34 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:08:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:34 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi029 to 16069M 2024-04-12T11:08:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:34 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:34 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:34 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:34 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.261 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[39999]: Detected new or changed devices on smithi029 2024-04-12T11:08:35.262 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.262 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.262 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:08:35.262 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi029 to 16069M 2024-04-12T11:08:35.262 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.262 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:35.262 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:35.262 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.262 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[29349]: Detected new or changed devices on smithi029 2024-04-12T11:08:35.262 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:08:35.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi029 to 16069M 2024-04-12T11:08:35.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:35.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:35.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:08:35.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:34 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:36.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:35 smithi087 ceph-mon[32917]: pgmap v42: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:36.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:35 smithi029 ceph-mon[29349]: pgmap v42: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:36.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:35 smithi029 ceph-mon[39999]: pgmap v42: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:36.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:36 smithi029 ceph-mon[39999]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:08:36.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:36 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:08:36.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:36 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:08:36.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:36 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:36.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:36 smithi029 ceph-mon[29349]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:08:36.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:36 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:08:36.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:36 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:08:36.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:36 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:36 smithi087 ceph-mon[32917]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:08:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:36 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:08:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:36 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:08:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:36 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:37 smithi087 ceph-mon[32917]: pgmap v43: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:38.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:37 smithi029 ceph-mon[29349]: pgmap v43: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:38.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:37 smithi029 ceph-mon[39999]: pgmap v43: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:38 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/800433816' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58da0c82-e2ec-4067-82d5-05c379001ae5"}]: dispatch 2024-04-12T11:08:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:38 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58da0c82-e2ec-4067-82d5-05c379001ae5"}]: dispatch 2024-04-12T11:08:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:38 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58da0c82-e2ec-4067-82d5-05c379001ae5"}]': finished 2024-04-12T11:08:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:38 smithi087 ceph-mon[32917]: osdmap e10: 2 total, 1 up, 2 in 2024-04-12T11:08:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:38 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:38 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/800433816' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58da0c82-e2ec-4067-82d5-05c379001ae5"}]: dispatch 2024-04-12T11:08:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:38 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58da0c82-e2ec-4067-82d5-05c379001ae5"}]: dispatch 2024-04-12T11:08:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:38 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58da0c82-e2ec-4067-82d5-05c379001ae5"}]': finished 2024-04-12T11:08:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:38 smithi029 ceph-mon[29349]: osdmap e10: 2 total, 1 up, 2 in 2024-04-12T11:08:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:38 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:38 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/800433816' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58da0c82-e2ec-4067-82d5-05c379001ae5"}]: dispatch 2024-04-12T11:08:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:38 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58da0c82-e2ec-4067-82d5-05c379001ae5"}]: dispatch 2024-04-12T11:08:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:38 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58da0c82-e2ec-4067-82d5-05c379001ae5"}]': finished 2024-04-12T11:08:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:38 smithi029 ceph-mon[39999]: osdmap e10: 2 total, 1 up, 2 in 2024-04-12T11:08:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:38 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:40.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:39 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1662921154' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:08:40.219 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:39 smithi087 ceph-mon[32917]: pgmap v45: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:39 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1662921154' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:08:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:39 smithi029 ceph-mon[29349]: pgmap v45: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:39 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1662921154' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:08:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:39 smithi029 ceph-mon[39999]: pgmap v45: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:42.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:42 smithi029 ceph-mon[29349]: pgmap v46: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:42.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:42 smithi029 ceph-mon[39999]: pgmap v46: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:42 smithi087 ceph-mon[32917]: pgmap v46: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:44.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:44 smithi029 ceph-mon[29349]: pgmap v47: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:44.766 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:44 smithi029 ceph-mon[39999]: pgmap v47: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:44 smithi087 ceph-mon[32917]: pgmap v47: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:45 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T11:08:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:45 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:45 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T11:08:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:45 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:45.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:45 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T11:08:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:45 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:08:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:46 smithi029 ceph-mon[29349]: Deploying daemon osd.1 on smithi029 2024-04-12T11:08:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:46 smithi029 ceph-mon[29349]: pgmap v48: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:46.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:46 smithi029 ceph-mon[39999]: Deploying daemon osd.1 on smithi029 2024-04-12T11:08:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:46 smithi029 ceph-mon[39999]: pgmap v48: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:46 smithi087 ceph-mon[32917]: Deploying daemon osd.1 on smithi029 2024-04-12T11:08:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:46 smithi087 ceph-mon[32917]: pgmap v48: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:48.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:48 smithi029 ceph-mon[29349]: pgmap v49: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:48.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:48 smithi029 ceph-mon[39999]: pgmap v49: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:48.769 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:48 smithi087 ceph-mon[32917]: pgmap v49: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:50.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:50 smithi029 ceph-mon[39999]: pgmap v50: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:50.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:50.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:50.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:50.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:50 smithi029 ceph-mon[29349]: pgmap v50: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:50.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:50 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:50.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:50 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:50.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:50 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:50 smithi087 ceph-mon[32917]: pgmap v50: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:50 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:50 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:50 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:08:52.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:52 smithi029 ceph-mon[29349]: pgmap v51: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:52 smithi029 ceph-mon[39999]: pgmap v51: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:52 smithi087 ceph-mon[32917]: pgmap v51: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:53.485 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 1 on host 'smithi029' 2024-04-12T11:08:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:53 smithi029 ceph-mon[39999]: from='osd.1 [v2:172.21.15.29:6810/3193349615,v1:172.21.15.29:6811/3193349615]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T11:08:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:53 smithi029 ceph-mon[39999]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T11:08:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:53 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:53 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:53 smithi029 ceph-mon[29349]: from='osd.1 [v2:172.21.15.29:6810/3193349615,v1:172.21.15.29:6811/3193349615]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T11:08:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:53 smithi029 ceph-mon[29349]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T11:08:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:53 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:53 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:53.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:53 smithi087 ceph-mon[32917]: from='osd.1 [v2:172.21.15.29:6810/3193349615,v1:172.21.15.29:6811/3193349615]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T11:08:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:53 smithi087 ceph-mon[32917]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T11:08:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:53 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:53 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:54.082 DEBUG:teuthology.orchestra.run.smithi029:osd.1> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.1.service 2024-04-12T11:08:54.084 INFO:tasks.cephadm:Deploying osd.2 on smithi029 with /dev/vg_nvme/lv_2... 2024-04-12T11:08:54.084 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-12T11:08:54.332 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:08:54.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[39999]: pgmap v52: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:54.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[39999]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T11:08:54.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[39999]: osdmap e11: 2 total, 1 up, 2 in 2024-04-12T11:08:54.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:54.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[39999]: from='osd.1 [v2:172.21.15.29:6810/3193349615,v1:172.21.15.29:6811/3193349615]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:08:54.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[39999]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:08:54.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:54.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:54.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[29349]: pgmap v52: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:54.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[29349]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T11:08:54.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[29349]: osdmap e11: 2 total, 1 up, 2 in 2024-04-12T11:08:54.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:54.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[29349]: from='osd.1 [v2:172.21.15.29:6810/3193349615,v1:172.21.15.29:6811/3193349615]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:08:54.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[29349]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:08:54.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:54.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:54.764 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[50016]: 2024-04-12T11:08:54.496+0000 7f743071a700 -1 osd.1 0 waiting for initial osdmap 2024-04-12T11:08:54.764 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:08:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[50016]: 2024-04-12T11:08:54.499+0000 7f742c50f700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:08:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:54 smithi087 ceph-mon[32917]: pgmap v52: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:54 smithi087 ceph-mon[32917]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T11:08:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:54 smithi087 ceph-mon[32917]: osdmap e11: 2 total, 1 up, 2 in 2024-04-12T11:08:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:54 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:54 smithi087 ceph-mon[32917]: from='osd.1 [v2:172.21.15.29:6810/3193349615,v1:172.21.15.29:6811/3193349615]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:08:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:54 smithi087 ceph-mon[32917]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:08:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:54 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:54 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:08:55.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:55 smithi029 ceph-mon[39999]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:08:55.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:55 smithi029 ceph-mon[39999]: osdmap e12: 2 total, 1 up, 2 in 2024-04-12T11:08:55.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:55.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:55.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:55 smithi029 ceph-mon[29349]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:08:55.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:55 smithi029 ceph-mon[29349]: osdmap e12: 2 total, 1 up, 2 in 2024-04-12T11:08:55.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:55.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:55 smithi087 ceph-mon[32917]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:08:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:55 smithi087 ceph-mon[32917]: osdmap e12: 2 total, 1 up, 2 in 2024-04-12T11:08:55.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:55.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:56.425 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-12T11:08:56.426 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-12T11:08:56.426 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2024-04-12T11:08:56.426 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2024-04-12T11:08:56.426 INFO:teuthology.orchestra.run.smithi029.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00944832 s, 1.1 GB/s 2024-04-12T11:08:56.426 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2024-04-12T11:08:56.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[29349]: purged_snaps scrub starts 2024-04-12T11:08:56.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[29349]: purged_snaps scrub ok 2024-04-12T11:08:56.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[29349]: pgmap v55: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:56.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[29349]: osd.1 [v2:172.21.15.29:6810/3193349615,v1:172.21.15.29:6811/3193349615] boot 2024-04-12T11:08:56.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[29349]: osdmap e13: 2 total, 2 up, 2 in 2024-04-12T11:08:56.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:56.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[39999]: purged_snaps scrub starts 2024-04-12T11:08:56.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[39999]: purged_snaps scrub ok 2024-04-12T11:08:56.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[39999]: pgmap v55: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:56.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[39999]: osd.1 [v2:172.21.15.29:6810/3193349615,v1:172.21.15.29:6811/3193349615] boot 2024-04-12T11:08:56.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[39999]: osdmap e13: 2 total, 2 up, 2 in 2024-04-12T11:08:56.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:56 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:56 smithi087 ceph-mon[32917]: purged_snaps scrub starts 2024-04-12T11:08:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:56 smithi087 ceph-mon[32917]: purged_snaps scrub ok 2024-04-12T11:08:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:56 smithi087 ceph-mon[32917]: pgmap v55: 0 pgs: ; 0 B data, 30 MiB used, 89 GiB / 89 GiB avail 2024-04-12T11:08:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:56 smithi087 ceph-mon[32917]: osd.1 [v2:172.21.15.29:6810/3193349615,v1:172.21.15.29:6811/3193349615] boot 2024-04-12T11:08:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:56 smithi087 ceph-mon[32917]: osdmap e13: 2 total, 2 up, 2 in 2024-04-12T11:08:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:56 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:08:57.077 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi029:vg_nvme/lv_2 2024-04-12T11:08:57.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:57 smithi029 ceph-mon[29349]: osdmap e14: 2 total, 2 up, 2 in 2024-04-12T11:08:57.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:57 smithi029 ceph-mon[39999]: osdmap e14: 2 total, 2 up, 2 in 2024-04-12T11:08:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:57 smithi087 ceph-mon[32917]: osdmap e14: 2 total, 2 up, 2 in 2024-04-12T11:08:58.000 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:08:58.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:08:58 smithi029 ceph-mon[29349]: pgmap v58: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:08:58.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:08:58 smithi029 ceph-mon[39999]: pgmap v58: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:08:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:08:58 smithi087 ceph-mon[32917]: pgmap v58: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: Detected new or changed devices on smithi029 2024-04-12T11:09:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi029 to 8034M 2024-04-12T11:09:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: pgmap v59: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: Detected new or changed devices on smithi029 2024-04-12T11:09:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi029 to 8034M 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: pgmap v59: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: Detected new or changed devices on smithi029 2024-04-12T11:09:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi029 to 8034M 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: pgmap v59: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:00.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:00.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:01.383 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:01 smithi029 ceph-mon[39999]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:09:01.383 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:01 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:09:01.383 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:01 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:09:01.384 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:01 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:01.384 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:01 smithi029 ceph-mon[29349]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:09:01.384 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:01 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:09:01.384 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:01 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:09:01.384 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:01 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:01.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:01 smithi087 ceph-mon[32917]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:09:01.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:01 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:09:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:01 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:09:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:01 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:02.318 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:02 smithi029 ceph-mon[29349]: pgmap v60: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:02.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:02 smithi087 ceph-mon[32917]: pgmap v60: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:02.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:02 smithi029 ceph-mon[39999]: pgmap v60: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:03 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3378343154' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "53b6c920-2e05-4905-be45-0f33131d358d"}]: dispatch 2024-04-12T11:09:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:03 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3378343154' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "53b6c920-2e05-4905-be45-0f33131d358d"}]': finished 2024-04-12T11:09:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:03 smithi087 ceph-mon[32917]: osdmap e15: 3 total, 2 up, 3 in 2024-04-12T11:09:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:03 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:03 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3378343154' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "53b6c920-2e05-4905-be45-0f33131d358d"}]: dispatch 2024-04-12T11:09:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:03 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3378343154' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "53b6c920-2e05-4905-be45-0f33131d358d"}]': finished 2024-04-12T11:09:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:03 smithi029 ceph-mon[29349]: osdmap e15: 3 total, 2 up, 3 in 2024-04-12T11:09:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:03 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:03 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3378343154' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "53b6c920-2e05-4905-be45-0f33131d358d"}]: dispatch 2024-04-12T11:09:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:03 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3378343154' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "53b6c920-2e05-4905-be45-0f33131d358d"}]': finished 2024-04-12T11:09:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:03 smithi029 ceph-mon[39999]: osdmap e15: 3 total, 2 up, 3 in 2024-04-12T11:09:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:03 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:04.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:04 smithi087 ceph-mon[32917]: pgmap v62: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:04 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2438180807' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:09:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:04 smithi029 ceph-mon[29349]: pgmap v62: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:04 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2438180807' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:09:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:04 smithi029 ceph-mon[39999]: pgmap v62: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:04 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2438180807' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:09:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:06 smithi029 ceph-mon[29349]: pgmap v63: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:06.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:06 smithi029 ceph-mon[39999]: pgmap v63: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:06 smithi087 ceph-mon[32917]: pgmap v63: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:08 smithi029 ceph-mon[29349]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:08 smithi029 ceph-mon[39999]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:08 smithi087 ceph-mon[32917]: pgmap v64: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T11:09:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:09 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T11:09:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:09 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:09 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T11:09:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:09 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:10 smithi029 ceph-mon[39999]: Deploying daemon osd.2 on smithi029 2024-04-12T11:09:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:10 smithi029 ceph-mon[39999]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:10 smithi029 ceph-mon[29349]: Deploying daemon osd.2 on smithi029 2024-04-12T11:09:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:10 smithi029 ceph-mon[29349]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:10 smithi087 ceph-mon[32917]: Deploying daemon osd.2 on smithi029 2024-04-12T11:09:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:10 smithi087 ceph-mon[32917]: pgmap v65: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:12.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:12 smithi029 ceph-mon[29349]: pgmap v66: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:12.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:12 smithi029 ceph-mon[39999]: pgmap v66: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:12 smithi087 ceph-mon[32917]: pgmap v66: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:14 smithi029 ceph-mon[29349]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:14 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:14 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:14.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:14 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:14 smithi029 ceph-mon[39999]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:14 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:14 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:14 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:14 smithi087 ceph-mon[32917]: pgmap v67: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:14 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:14 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:14 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:16.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:16 smithi029 ceph-mon[29349]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:16.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:16 smithi029 ceph-mon[39999]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:16 smithi087 ceph-mon[32917]: pgmap v68: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:17.718 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 2 on host 'smithi029' 2024-04-12T11:09:18.320 DEBUG:teuthology.orchestra.run.smithi029:osd.2> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.2.service 2024-04-12T11:09:18.323 INFO:tasks.cephadm:Deploying osd.3 on smithi029 with /dev/vg_nvme/lv_1... 2024-04-12T11:09:18.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-12T11:09:18.594 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[39999]: pgmap v69: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:18.594 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[39999]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T11:09:18.594 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.594 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.594 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.595 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[29349]: pgmap v69: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:18.595 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[29349]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T11:09:18.595 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.595 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.595 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.595 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:18 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.769 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:18 smithi087 ceph-mon[32917]: pgmap v69: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:18.769 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:18 smithi087 ceph-mon[32917]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T11:09:18.769 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:18 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.769 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:18 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.769 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:18 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:18.769 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:18 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[39999]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T11:09:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[39999]: osdmap e16: 3 total, 2 up, 3 in 2024-04-12T11:09:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[39999]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:09:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[39999]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:09:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[39999]: osdmap e17: 3 total, 2 up, 3 in 2024-04-12T11:09:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:19.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[29349]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T11:09:19.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[29349]: osdmap e16: 3 total, 2 up, 3 in 2024-04-12T11:09:19.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:19.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[29349]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:09:19.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[29349]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:09:19.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[29349]: osdmap e17: 3 total, 2 up, 3 in 2024-04-12T11:09:19.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:19.715 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[55643]: 2024-04-12T11:09:19.497+0000 7feca26e2700 -1 osd.2 0 waiting for initial osdmap 2024-04-12T11:09:19.715 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:09:19 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[55643]: 2024-04-12T11:09:19.502+0000 7fec9e4d7700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:09:19.908 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:09:19.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:19 smithi087 ceph-mon[32917]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T11:09:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:19 smithi087 ceph-mon[32917]: osdmap e16: 3 total, 2 up, 3 in 2024-04-12T11:09:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:19 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:19 smithi087 ceph-mon[32917]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:09:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:19 smithi087 ceph-mon[32917]: from='osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:09:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:19 smithi087 ceph-mon[32917]: osdmap e17: 3 total, 2 up, 3 in 2024-04-12T11:09:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:19 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:20.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:20 smithi029 ceph-mon[39999]: pgmap v71: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:20.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:20 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:20.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:20 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:20.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:20 smithi029 ceph-mon[29349]: pgmap v71: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:20.553 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:20 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:20.553 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:20 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:20 smithi087 ceph-mon[32917]: pgmap v71: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:20 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:20 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:21.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: purged_snaps scrub starts 2024-04-12T11:09:21.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: purged_snaps scrub ok 2024-04-12T11:09:21.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732] boot 2024-04-12T11:09:21.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: osdmap e18: 3 total, 3 up, 3 in 2024-04-12T11:09:21.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:21.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:21.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:21.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:21.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:21.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: purged_snaps scrub starts 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: purged_snaps scrub ok 2024-04-12T11:09:21.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732] boot 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: osdmap e18: 3 total, 3 up, 3 in 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:21.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:22.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: purged_snaps scrub starts 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: purged_snaps scrub ok 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: osd.2 [v2:172.21.15.29:6818/1946236732,v1:172.21.15.29:6819/1946236732] boot 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: osdmap e18: 3 total, 3 up, 3 in 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:22.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:22.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:22.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:22.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:22.437 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-12T11:09:22.437 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-12T11:09:22.437 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2024-04-12T11:09:22.437 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2024-04-12T11:09:22.437 INFO:teuthology.orchestra.run.smithi029.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00942559 s, 1.1 GB/s 2024-04-12T11:09:22.437 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2024-04-12T11:09:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[29349]: Detected new or changed devices on smithi029 2024-04-12T11:09:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi029 to 5356M 2024-04-12T11:09:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[29349]: pgmap v74: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:09:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-12T11:09:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[29349]: osdmap e19: 3 total, 3 up, 3 in 2024-04-12T11:09:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:09:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[39999]: Detected new or changed devices on smithi029 2024-04-12T11:09:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi029 to 5356M 2024-04-12T11:09:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[39999]: pgmap v74: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:09:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-12T11:09:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[39999]: osdmap e19: 3 total, 3 up, 3 in 2024-04-12T11:09:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:09:23.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi029:vg_nvme/lv_1 2024-04-12T11:09:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:22 smithi087 ceph-mon[32917]: Detected new or changed devices on smithi029 2024-04-12T11:09:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:22 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi029 to 5356M 2024-04-12T11:09:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:22 smithi087 ceph-mon[32917]: pgmap v74: 0 pgs: ; 0 B data, 57 MiB used, 179 GiB / 179 GiB avail 2024-04-12T11:09:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:09:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-12T11:09:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:22 smithi087 ceph-mon[32917]: osdmap e19: 3 total, 3 up, 3 in 2024-04-12T11:09:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:09:23.268 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:09:23.374 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59304]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T11:09:23.374 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59304]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:23.374 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59304]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:23.374 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59304]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:23.696 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59363]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T11:09:23.697 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59363]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:23.697 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59363]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:23.697 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59363]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:23.999 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:23 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:09:23.999 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:23 smithi029 ceph-mon[29349]: osdmap e20: 3 total, 3 up, 3 in 2024-04-12T11:09:23.999 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:23 smithi029 ceph-mon[29349]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:23.999 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:23 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:24.000 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:23 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:09:24.000 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:23 smithi029 ceph-mon[39999]: osdmap e20: 3 total, 3 up, 3 in 2024-04-12T11:09:24.000 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:23 smithi029 ceph-mon[39999]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:24.000 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:23 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:24.000 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59385]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T11:09:24.001 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59385]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:24.001 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59385]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:24.001 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59385]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:24.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:23 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:09:24.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:23 smithi087 ceph-mon[32917]: osdmap e20: 3 total, 3 up, 3 in 2024-04-12T11:09:24.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:23 smithi087 ceph-mon[32917]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:24.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:23 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:24.259 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:09:23 smithi029 sudo[59390]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T11:09:24.259 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59390]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:24.259 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59390]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:24.259 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59390]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:24.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59393]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T11:09:24.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59393]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:24.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59393]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:24.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59393]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:24.794 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59417]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T11:09:24.794 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59417]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:24.794 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59417]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:24.794 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59417]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:25.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59427]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T11:09:25.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59427]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:25.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59427]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:25.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:24 smithi029 sudo[59427]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:25.325 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:25 smithi087 ceph-mon[32917]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-12T11:09:25.326 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:25 smithi087 ceph-mon[32917]: osdmap e21: 3 total, 3 up, 3 in 2024-04-12T11:09:25.326 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:25 smithi087 sudo[35493]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T11:09:25.326 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:25 smithi087 sudo[35493]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:25.326 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:25 smithi087 sudo[35493]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:25.326 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:25 smithi087 sudo[35493]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:25.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:25 smithi029 ceph-mon[39999]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-12T11:09:25.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:25 smithi029 ceph-mon[39999]: osdmap e21: 3 total, 3 up, 3 in 2024-04-12T11:09:25.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:25 smithi029 sudo[59450]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T11:09:25.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:25 smithi029 sudo[59450]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:25.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:25 smithi029 sudo[59450]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:25.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:25 smithi029 sudo[59450]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:25.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:25 smithi029 ceph-mon[29349]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-12T11:09:25.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:25 smithi029 ceph-mon[29349]: osdmap e21: 3 total, 3 up, 3 in 2024-04-12T11:09:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:25 smithi087 sudo[35496]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T11:09:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:25 smithi087 sudo[35496]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:25 smithi087 sudo[35496]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:25 smithi087 sudo[35496]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:26.105 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:25 smithi029 sudo[59590]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T11:09:26.106 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:25 smithi029 sudo[59590]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:26.106 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:25 smithi029 sudo[59590]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:26.106 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:25 smithi029 sudo[59590]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:09:26.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:09:26.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:09:26.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:26.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:26.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T11:09:26.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:26.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:09:26.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:09:26.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:09:26.469 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 sudo[59593]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 sudo[59593]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 sudo[59593]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 sudo[59593]: pam_unix(sudo:session): session closed for user root 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:09:26.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:09:26.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:09:26.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:09:27.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:27 smithi087 ceph-mon[32917]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T11:09:27.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:27 smithi087 ceph-mon[32917]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T11:09:27.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:27 smithi029 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T11:09:27.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:27 smithi029 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T11:09:27.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:27 smithi029 ceph-mon[39999]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T11:09:27.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:27 smithi029 ceph-mon[39999]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T11:09:28.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:28 smithi087 ceph-mon[32917]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 85 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:28 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1423711941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c33079c-a506-4d4f-97bc-11df1245e5ba"}]: dispatch 2024-04-12T11:09:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:28 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c33079c-a506-4d4f-97bc-11df1245e5ba"}]: dispatch 2024-04-12T11:09:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:28 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5c33079c-a506-4d4f-97bc-11df1245e5ba"}]': finished 2024-04-12T11:09:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:28 smithi087 ceph-mon[32917]: osdmap e22: 4 total, 3 up, 4 in 2024-04-12T11:09:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:28 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:28.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[29349]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 85 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:28.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1423711941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c33079c-a506-4d4f-97bc-11df1245e5ba"}]: dispatch 2024-04-12T11:09:28.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c33079c-a506-4d4f-97bc-11df1245e5ba"}]: dispatch 2024-04-12T11:09:28.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5c33079c-a506-4d4f-97bc-11df1245e5ba"}]': finished 2024-04-12T11:09:28.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[29349]: osdmap e22: 4 total, 3 up, 4 in 2024-04-12T11:09:28.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:28.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[39999]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 85 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:28.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1423711941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c33079c-a506-4d4f-97bc-11df1245e5ba"}]: dispatch 2024-04-12T11:09:28.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c33079c-a506-4d4f-97bc-11df1245e5ba"}]: dispatch 2024-04-12T11:09:28.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5c33079c-a506-4d4f-97bc-11df1245e5ba"}]': finished 2024-04-12T11:09:28.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[39999]: osdmap e22: 4 total, 3 up, 4 in 2024-04-12T11:09:28.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:28 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:29.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:29 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1185370963' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:09:29.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:29 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1185370963' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:09:29.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:29 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1185370963' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:09:30.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:30 smithi029 ceph-mon[29349]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:30.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:30 smithi029 ceph-mon[39999]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:30.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:30 smithi087 ceph-mon[32917]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:32 smithi029 ceph-mon[29349]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:32 smithi029 ceph-mon[39999]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:32 smithi087 ceph-mon[32917]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:34.738 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:34 smithi029 ceph-mon[29349]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:34.738 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:34 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T11:09:34.738 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:34 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:34.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:34 smithi029 ceph-mon[39999]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:34.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:34 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T11:09:34.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:34 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:34 smithi087 ceph-mon[32917]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:34 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T11:09:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:34 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:35.747 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:35 smithi029 ceph-mon[29349]: Deploying daemon osd.3 on smithi029 2024-04-12T11:09:35.747 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:35 smithi029 ceph-mon[39999]: Deploying daemon osd.3 on smithi029 2024-04-12T11:09:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:35 smithi087 ceph-mon[32917]: Deploying daemon osd.3 on smithi029 2024-04-12T11:09:36.751 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:36 smithi029 ceph-mon[39999]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:36.752 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:36 smithi029 ceph-mon[29349]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:36 smithi087 ceph-mon[32917]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:38.755 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:38 smithi029 ceph-mon[29349]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:38.756 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:38 smithi029 ceph-mon[39999]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:38 smithi087 ceph-mon[32917]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:40.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:40 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:40.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:40 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:40.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:40 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:40.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:40 smithi029 ceph-mon[29349]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:40.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:40 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:40.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:40 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:40.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:40 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:40.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:40 smithi029 ceph-mon[39999]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:40.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:40 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:40.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:40 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:40.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:40 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:40.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:40 smithi087 ceph-mon[32917]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:42.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:42 smithi029 ceph-mon[29349]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:42.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:42 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:42.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:42 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:42.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:42 smithi029 ceph-mon[29349]: from='osd.3 [v2:172.21.15.29:6826/4104584972,v1:172.21.15.29:6827/4104584972]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:09:42.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:42 smithi029 ceph-mon[29349]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:09:42.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:42 smithi029 ceph-mon[39999]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:42.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:42 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:42.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:42 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:42.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:42 smithi029 ceph-mon[39999]: from='osd.3 [v2:172.21.15.29:6826/4104584972,v1:172.21.15.29:6827/4104584972]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:09:42.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:42 smithi029 ceph-mon[39999]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:09:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:42 smithi087 ceph-mon[32917]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:42 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:42 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:42 smithi087 ceph-mon[32917]: from='osd.3 [v2:172.21.15.29:6826/4104584972,v1:172.21.15.29:6827/4104584972]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:09:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:42 smithi087 ceph-mon[32917]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:09:43.079 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 3 on host 'smithi029' 2024-04-12T11:09:43.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:43.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:43.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:43.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:43.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:43.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:43.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T11:09:43.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: osdmap e23: 4 total, 3 up, 4 in 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: from='osd.3 [v2:172.21.15.29:6826/4104584972,v1:172.21.15.29:6827/4104584972]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[39999]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: osdmap e23: 4 total, 3 up, 4 in 2024-04-12T11:09:43.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:43.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: from='osd.3 [v2:172.21.15.29:6826/4104584972,v1:172.21.15.29:6827/4104584972]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:09:43.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:43 smithi029 ceph-mon[29349]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:09:43.715 DEBUG:teuthology.orchestra.run.smithi029:osd.3> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.3.service 2024-04-12T11:09:43.718 INFO:tasks.cephadm:Deploying osd.4 on smithi087 with /dev/vg_nvme/lv_4... 2024-04-12T11:09:43.718 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: osdmap e23: 4 total, 3 up, 4 in 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: from='osd.3 [v2:172.21.15.29:6826/4104584972,v1:172.21.15.29:6827/4104584972]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:09:43.748 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:43 smithi087 ceph-mon[32917]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:09:43.925 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:09:44.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[29349]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:44.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[29349]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:09:44.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[29349]: osdmap e24: 4 total, 3 up, 4 in 2024-04-12T11:09:44.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:44.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:44.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[29349]: from='osd.3 ' entity='osd.3' 2024-04-12T11:09:44.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[39999]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:44.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[39999]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:09:44.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[39999]: osdmap e24: 4 total, 3 up, 4 in 2024-04-12T11:09:44.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:44.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:44.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-mon[39999]: from='osd.3 ' entity='osd.3' 2024-04-12T11:09:44.519 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[61361]: 2024-04-12T11:09:44.390+0000 7fbf0087a700 -1 osd.3 0 waiting for initial osdmap 2024-04-12T11:09:44.519 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:09:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[61361]: 2024-04-12T11:09:44.394+0000 7fbef7e66700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:09:44.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:44 smithi087 ceph-mon[32917]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:44.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:44 smithi087 ceph-mon[32917]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-04-12T11:09:44.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:44 smithi087 ceph-mon[32917]: osdmap e24: 4 total, 3 up, 4 in 2024-04-12T11:09:44.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:44 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:44.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:44 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:44.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:44 smithi087 ceph-mon[32917]: from='osd.3 ' entity='osd.3' 2024-04-12T11:09:45.722 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:45 smithi087 ceph-mon[32917]: purged_snaps scrub starts 2024-04-12T11:09:45.723 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:45 smithi087 ceph-mon[32917]: purged_snaps scrub ok 2024-04-12T11:09:45.723 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:45 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:45.723 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:45 smithi087 ceph-mon[32917]: osd.3 [v2:172.21.15.29:6826/4104584972,v1:172.21.15.29:6827/4104584972] boot 2024-04-12T11:09:45.723 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:45 smithi087 ceph-mon[32917]: osdmap e25: 4 total, 4 up, 4 in 2024-04-12T11:09:45.723 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:45 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:45.723 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-12T11:09:45.723 INFO:teuthology.orchestra.run.smithi087.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-12T11:09:45.723 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10+0 records in 2024-04-12T11:09:45.723 INFO:teuthology.orchestra.run.smithi087.stderr:10+0 records out 2024-04-12T11:09:45.723 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0132943 s, 789 MB/s 2024-04-12T11:09:45.724 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping successful for: 2024-04-12T11:09:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[29349]: purged_snaps scrub starts 2024-04-12T11:09:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[29349]: purged_snaps scrub ok 2024-04-12T11:09:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[29349]: osd.3 [v2:172.21.15.29:6826/4104584972,v1:172.21.15.29:6827/4104584972] boot 2024-04-12T11:09:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[29349]: osdmap e25: 4 total, 4 up, 4 in 2024-04-12T11:09:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[39999]: purged_snaps scrub starts 2024-04-12T11:09:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[39999]: purged_snaps scrub ok 2024-04-12T11:09:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[39999]: osd.3 [v2:172.21.15.29:6826/4104584972,v1:172.21.15.29:6827/4104584972] boot 2024-04-12T11:09:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[39999]: osdmap e25: 4 total, 4 up, 4 in 2024-04-12T11:09:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:45 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:09:46.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi087:vg_nvme/lv_4 2024-04-12T11:09:46.507 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:09:46.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:46 smithi029 ceph-mon[39999]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:46.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:46 smithi029 ceph-mon[29349]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:46.628 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:46 smithi087 ceph-mon[32917]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-04-12T11:09:47.552 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: osdmap e26: 4 total, 4 up, 4 in 2024-04-12T11:09:47.552 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.552 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.552 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.552 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.552 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.553 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.553 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.553 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:47.553 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:47.553 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:47 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: osdmap e26: 4 total, 4 up, 4 in 2024-04-12T11:09:47.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: osdmap e26: 4 total, 4 up, 4 in 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:47.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:09:47.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:47 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:09:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[29349]: Detected new or changed devices on smithi029 2024-04-12T11:09:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi029 to 4017M 2024-04-12T11:09:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[29349]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:09:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:09:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[39999]: Detected new or changed devices on smithi029 2024-04-12T11:09:48.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi029 to 4017M 2024-04-12T11:09:48.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[39999]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:48.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:09:48.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:09:48.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:48 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:48.816 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:48 smithi087 ceph-mon[32917]: Detected new or changed devices on smithi029 2024-04-12T11:09:48.816 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:48 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi029 to 4017M 2024-04-12T11:09:48.816 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:48 smithi087 ceph-mon[32917]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:48.816 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:48 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:09:48.816 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:48 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:09:48.816 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:48 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:49.610 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:49 smithi087 ceph-mon[32917]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:09:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:49 smithi029 ceph-mon[29349]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:09:49.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:49 smithi029 ceph-mon[39999]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:09:50.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:50 smithi029 ceph-mon[29349]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:50.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:50 smithi029 ceph-mon[39999]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:50 smithi087 ceph-mon[32917]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/526139368' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f776683c-bddc-4a6f-a85d-a43639afadee"}]: dispatch 2024-04-12T11:09:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f776683c-bddc-4a6f-a85d-a43639afadee"}]: dispatch 2024-04-12T11:09:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f776683c-bddc-4a6f-a85d-a43639afadee"}]': finished 2024-04-12T11:09:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[29349]: osdmap e27: 5 total, 4 up, 5 in 2024-04-12T11:09:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:09:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/1044921002' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:09:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/526139368' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f776683c-bddc-4a6f-a85d-a43639afadee"}]: dispatch 2024-04-12T11:09:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f776683c-bddc-4a6f-a85d-a43639afadee"}]: dispatch 2024-04-12T11:09:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f776683c-bddc-4a6f-a85d-a43639afadee"}]': finished 2024-04-12T11:09:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[39999]: osdmap e27: 5 total, 4 up, 5 in 2024-04-12T11:09:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:09:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:51 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/1044921002' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:09:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:51 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/526139368' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f776683c-bddc-4a6f-a85d-a43639afadee"}]: dispatch 2024-04-12T11:09:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:51 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f776683c-bddc-4a6f-a85d-a43639afadee"}]: dispatch 2024-04-12T11:09:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:51 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f776683c-bddc-4a6f-a85d-a43639afadee"}]': finished 2024-04-12T11:09:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:51 smithi087 ceph-mon[32917]: osdmap e27: 5 total, 4 up, 5 in 2024-04-12T11:09:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:51 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:09:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:51 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/1044921002' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:09:52.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:52 smithi029 ceph-mon[29349]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:52 smithi029 ceph-mon[39999]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:52 smithi087 ceph-mon[32917]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:54.699 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:54 smithi087 ceph-mon[32917]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:54.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:54 smithi029 ceph-mon[29349]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:54.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:54 smithi029 ceph-mon[39999]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:56.543 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:56 smithi087 ceph-mon[32917]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:56 smithi029 ceph-mon[29349]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:56 smithi029 ceph-mon[39999]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:57 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T11:09:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:57 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:57 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T11:09:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:57 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:57 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T11:09:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:57 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:09:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:58 smithi087 ceph-mon[32917]: Deploying daemon osd.4 on smithi087 2024-04-12T11:09:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:09:58 smithi087 ceph-mon[32917]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:58 smithi029 ceph-mon[29349]: Deploying daemon osd.4 on smithi087 2024-04-12T11:09:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:09:58 smithi029 ceph-mon[29349]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:09:59.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:58 smithi029 ceph-mon[39999]: Deploying daemon osd.4 on smithi087 2024-04-12T11:09:59.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:09:58 smithi029 ceph-mon[39999]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:00.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:00 smithi087 ceph-mon[32917]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:00 smithi087 ceph-mon[32917]: overall HEALTH_OK 2024-04-12T11:10:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:00 smithi029 ceph-mon[39999]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:00 smithi029 ceph-mon[39999]: overall HEALTH_OK 2024-04-12T11:10:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:00 smithi029 ceph-mon[29349]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:00 smithi029 ceph-mon[29349]: overall HEALTH_OK 2024-04-12T11:10:02.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:02 smithi087 ceph-mon[32917]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:02.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:02 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:02.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:02 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:02.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:02 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:03.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:02 smithi029 ceph-mon[29349]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:03.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:02 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:03.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:02 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:03.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:02 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:03.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:02 smithi029 ceph-mon[39999]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:03.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:02 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:03.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:02 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:03.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:02 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:04.858 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:04 smithi087 ceph-mon[32917]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:04.859 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:04.859 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:04.859 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:04.859 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:04.859 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:04 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:04.899 INFO:teuthology.orchestra.run.smithi087.stdout:Created osd(s) 4 on host 'smithi087' 2024-04-12T11:10:05.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[29349]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:05.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:05.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:05.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:05.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:05.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[39999]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:05.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:05.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:05.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:05.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:05.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:04 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:05.475 DEBUG:teuthology.orchestra.run.smithi087:osd.4> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.4.service 2024-04-12T11:10:05.477 INFO:tasks.cephadm:Deploying osd.5 on smithi087 with /dev/vg_nvme/lv_3... 2024-04-12T11:10:05.477 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-12T11:10:05.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:05 smithi087 ceph-mon[32917]: from='osd.4 [v2:172.21.15.87:6800/2970504758,v1:172.21.15.87:6801/2970504758]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:10:05.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:05 smithi087 ceph-mon[32917]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:10:05.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:05 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:05.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:05 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:05 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:05 smithi029 ceph-mon[39999]: from='osd.4 [v2:172.21.15.87:6800/2970504758,v1:172.21.15.87:6801/2970504758]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:10:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:05 smithi029 ceph-mon[39999]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:10:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:05 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:05 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:05 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:05 smithi029 ceph-mon[29349]: from='osd.4 [v2:172.21.15.87:6800/2970504758,v1:172.21.15.87:6801/2970504758]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:10:06.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:05 smithi029 ceph-mon[29349]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:10:06.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:05 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:06.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:05 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:06.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:05 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:06.799 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:06 smithi087 ceph-mon[32917]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:06.799 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:06 smithi087 ceph-mon[32917]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T11:10:06.799 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:06 smithi087 ceph-mon[32917]: osdmap e28: 5 total, 4 up, 5 in 2024-04-12T11:10:06.799 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:06 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:06.799 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:06 smithi087 ceph-mon[32917]: from='osd.4 [v2:172.21.15.87:6800/2970504758,v1:172.21.15.87:6801/2970504758]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:06.800 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:06 smithi087 ceph-mon[32917]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[29349]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[29349]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T11:10:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[29349]: osdmap e28: 5 total, 4 up, 5 in 2024-04-12T11:10:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[29349]: from='osd.4 [v2:172.21.15.87:6800/2970504758,v1:172.21.15.87:6801/2970504758]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[29349]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:07.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[39999]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:07.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[39999]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T11:10:07.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[39999]: osdmap e28: 5 total, 4 up, 5 in 2024-04-12T11:10:07.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:07.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[39999]: from='osd.4 [v2:172.21.15.87:6800/2970504758,v1:172.21.15.87:6801/2970504758]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:07.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:06 smithi029 ceph-mon[39999]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:07.055 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:10:07.182 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:10:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[37843]: 2024-04-12T11:10:06.978+0000 7fd60c176700 -1 osd.4 0 waiting for initial osdmap 2024-04-12T11:10:07.182 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:10:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[37843]: 2024-04-12T11:10:06.984+0000 7fd603f63700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:10:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:07 smithi087 ceph-mon[32917]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:10:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:07 smithi087 ceph-mon[32917]: osdmap e29: 5 total, 4 up, 5 in 2024-04-12T11:10:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:07 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:07.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:07 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:07.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:07 smithi087 ceph-mon[32917]: from='osd.4 ' entity='osd.4' 2024-04-12T11:10:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:07 smithi029 ceph-mon[29349]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:10:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:07 smithi029 ceph-mon[29349]: osdmap e29: 5 total, 4 up, 5 in 2024-04-12T11:10:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:07 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:07 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:07 smithi029 ceph-mon[29349]: from='osd.4 ' entity='osd.4' 2024-04-12T11:10:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:07 smithi029 ceph-mon[39999]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:10:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:07 smithi029 ceph-mon[39999]: osdmap e29: 5 total, 4 up, 5 in 2024-04-12T11:10:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:07 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:07 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:07 smithi029 ceph-mon[39999]: from='osd.4 ' entity='osd.4' 2024-04-12T11:10:08.657 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: purged_snaps scrub starts 2024-04-12T11:10:08.657 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: purged_snaps scrub ok 2024-04-12T11:10:08.657 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: osd.4 [v2:172.21.15.87:6800/2970504758,v1:172.21.15.87:6801/2970504758] boot 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: osdmap e30: 5 total, 5 up, 5 in 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: Detected new or changed devices on smithi087 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi087 to 17088M 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:08.658 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:08 smithi087 ceph-mon[32917]: osdmap e31: 5 total, 5 up, 5 in 2024-04-12T11:10:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: purged_snaps scrub starts 2024-04-12T11:10:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: purged_snaps scrub ok 2024-04-12T11:10:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: osd.4 [v2:172.21.15.87:6800/2970504758,v1:172.21.15.87:6801/2970504758] boot 2024-04-12T11:10:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: osdmap e30: 5 total, 5 up, 5 in 2024-04-12T11:10:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: Detected new or changed devices on smithi087 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi087 to 17088M 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[29349]: osdmap e31: 5 total, 5 up, 5 in 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: purged_snaps scrub starts 2024-04-12T11:10:09.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: purged_snaps scrub ok 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 116 MiB used, 357 GiB / 358 GiB avail 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: osd.4 [v2:172.21.15.87:6800/2970504758,v1:172.21.15.87:6801/2970504758] boot 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: osdmap e30: 5 total, 5 up, 5 in 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: Detected new or changed devices on smithi087 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi087 to 17088M 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:09.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:08 smithi029 ceph-mon[39999]: osdmap e31: 5 total, 5 up, 5 in 2024-04-12T11:10:09.453 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-12T11:10:09.453 INFO:teuthology.orchestra.run.smithi087.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-12T11:10:09.453 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10+0 records in 2024-04-12T11:10:09.453 INFO:teuthology.orchestra.run.smithi087.stderr:10+0 records out 2024-04-12T11:10:09.453 INFO:teuthology.orchestra.run.smithi087.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.01253 s, 837 MB/s 2024-04-12T11:10:09.453 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping successful for: 2024-04-12T11:10:09.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:09 smithi087 ceph-mon[32917]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 431 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:09 smithi087 ceph-mon[32917]: osdmap e32: 5 total, 5 up, 5 in 2024-04-12T11:10:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:09 smithi029 ceph-mon[29349]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 431 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:09 smithi029 ceph-mon[29349]: osdmap e32: 5 total, 5 up, 5 in 2024-04-12T11:10:10.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:09 smithi029 ceph-mon[39999]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 431 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:10.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:09 smithi029 ceph-mon[39999]: osdmap e32: 5 total, 5 up, 5 in 2024-04-12T11:10:10.120 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi087:vg_nvme/lv_3 2024-04-12T11:10:10.318 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:10:11.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:11 smithi087 ceph-mon[32917]: osdmap e33: 5 total, 5 up, 5 in 2024-04-12T11:10:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:11 smithi087 ceph-mon[32917]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 431 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:11 smithi029 ceph-mon[29349]: osdmap e33: 5 total, 5 up, 5 in 2024-04-12T11:10:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:11 smithi029 ceph-mon[29349]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 431 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:11 smithi029 ceph-mon[39999]: osdmap e33: 5 total, 5 up, 5 in 2024-04-12T11:10:12.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:11 smithi029 ceph-mon[39999]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 431 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:12.654 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:12 smithi087 ceph-mon[32917]: osdmap e34: 5 total, 5 up, 5 in 2024-04-12T11:10:12.654 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:12 smithi087 ceph-mon[32917]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:10:12.654 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:12 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:10:12.654 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:12 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:10:12.654 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:12 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:12 smithi029 ceph-mon[39999]: osdmap e34: 5 total, 5 up, 5 in 2024-04-12T11:10:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:12 smithi029 ceph-mon[39999]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:10:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:12 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:10:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:12 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:10:13.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:12 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:13.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:12 smithi029 ceph-mon[29349]: osdmap e34: 5 total, 5 up, 5 in 2024-04-12T11:10:13.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:12 smithi029 ceph-mon[29349]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:10:13.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:12 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:10:13.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:12 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:10:13.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:12 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:13.661 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:13 smithi087 ceph-mon[32917]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 144 MiB used, 447 GiB / 447 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-12T11:10:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:13 smithi029 ceph-mon[39999]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 144 MiB used, 447 GiB / 447 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-12T11:10:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:13 smithi029 ceph-mon[29349]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 144 MiB used, 447 GiB / 447 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-12T11:10:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:14 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/1571032466' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca436dff-e509-439d-a17b-0f4efe4d4693"}]: dispatch 2024-04-12T11:10:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:14 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca436dff-e509-439d-a17b-0f4efe4d4693"}]: dispatch 2024-04-12T11:10:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:14 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca436dff-e509-439d-a17b-0f4efe4d4693"}]': finished 2024-04-12T11:10:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:14 smithi087 ceph-mon[32917]: osdmap e35: 6 total, 5 up, 6 in 2024-04-12T11:10:14.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:14 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:14 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/1571032466' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca436dff-e509-439d-a17b-0f4efe4d4693"}]: dispatch 2024-04-12T11:10:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:14 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca436dff-e509-439d-a17b-0f4efe4d4693"}]: dispatch 2024-04-12T11:10:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:14 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca436dff-e509-439d-a17b-0f4efe4d4693"}]': finished 2024-04-12T11:10:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:14 smithi029 ceph-mon[39999]: osdmap e35: 6 total, 5 up, 6 in 2024-04-12T11:10:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:14 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:15.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:14 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/1571032466' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca436dff-e509-439d-a17b-0f4efe4d4693"}]: dispatch 2024-04-12T11:10:15.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:14 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca436dff-e509-439d-a17b-0f4efe4d4693"}]: dispatch 2024-04-12T11:10:15.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:14 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca436dff-e509-439d-a17b-0f4efe4d4693"}]': finished 2024-04-12T11:10:15.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:14 smithi029 ceph-mon[29349]: osdmap e35: 6 total, 5 up, 6 in 2024-04-12T11:10:15.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:14 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:15 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/693179431' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:10:15.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:15 smithi087 ceph-mon[32917]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 144 MiB used, 447 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-12T11:10:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:15 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/693179431' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:10:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:15 smithi029 ceph-mon[29349]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 144 MiB used, 447 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-12T11:10:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:15 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/693179431' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:10:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:15 smithi029 ceph-mon[39999]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 144 MiB used, 447 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-12T11:10:18.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:18 smithi029 ceph-mon[29349]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-04-12T11:10:18.737 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:18 smithi029 ceph-mon[39999]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-04-12T11:10:18.937 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:18 smithi087 ceph-mon[32917]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-04-12T11:10:20.588 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:20 smithi087 ceph-mon[32917]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T11:10:20.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:20 smithi029 ceph-mon[29349]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T11:10:20.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:20 smithi029 ceph-mon[39999]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T11:10:21.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T11:10:21.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:21 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T11:10:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:21 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T11:10:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:21 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:22 smithi029 ceph-mon[29349]: Deploying daemon osd.5 on smithi087 2024-04-12T11:10:22.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:22 smithi029 ceph-mon[29349]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:22.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:22 smithi029 ceph-mon[39999]: Deploying daemon osd.5 on smithi087 2024-04-12T11:10:22.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:22 smithi029 ceph-mon[39999]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:22 smithi087 ceph-mon[32917]: Deploying daemon osd.5 on smithi087 2024-04-12T11:10:22.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:22 smithi087 ceph-mon[32917]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:24 smithi029 ceph-mon[29349]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:24 smithi029 ceph-mon[39999]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:24.833 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:24 smithi087 ceph-mon[32917]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:26.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:26 smithi087 ceph-mon[32917]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:26.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:26.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:26 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:26 smithi029 ceph-mon[29349]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:26 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:26 smithi029 ceph-mon[39999]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:26 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[29349]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[39999]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:28 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:28.805 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:28 smithi087 ceph-mon[32917]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:28.805 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:28 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:28.805 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:28 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:28.805 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:28 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:28.805 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:28 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:28.805 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:28 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:29.206 INFO:teuthology.orchestra.run.smithi087.stdout:Created osd(s) 5 on host 'smithi087' 2024-04-12T11:10:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:29 smithi029 ceph-mon[29349]: from='osd.5 [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:10:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:29 smithi029 ceph-mon[29349]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:10:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:29 smithi029 ceph-mon[39999]: from='osd.5 [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:10:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:29 smithi029 ceph-mon[39999]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:10:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:29.782 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:29 smithi087 ceph-mon[32917]: from='osd.5 [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:10:29.782 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:29 smithi087 ceph-mon[32917]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:10:29.782 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:29.782 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:29.782 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:29.803 DEBUG:teuthology.orchestra.run.smithi087:osd.5> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.5.service 2024-04-12T11:10:29.805 INFO:tasks.cephadm:Deploying osd.6 on smithi087 with /dev/vg_nvme/lv_2... 2024-04-12T11:10:29.805 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-12T11:10:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:30 smithi087 ceph-mon[32917]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:30 smithi087 ceph-mon[32917]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T11:10:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:30 smithi087 ceph-mon[32917]: osdmap e36: 6 total, 5 up, 6 in 2024-04-12T11:10:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:30 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:30 smithi087 ceph-mon[32917]: from='osd.5 [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:30 smithi087 ceph-mon[32917]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:30.717 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:10:30 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[43892]: 2024-04-12T11:10:30.514+0000 7f1653be2700 -1 osd.5 0 waiting for initial osdmap 2024-04-12T11:10:30.717 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:10:30 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[43892]: 2024-04-12T11:10:30.519+0000 7f164f9d7700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:10:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[29349]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[29349]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T11:10:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[29349]: osdmap e36: 6 total, 5 up, 6 in 2024-04-12T11:10:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[29349]: from='osd.5 [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[29349]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[39999]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[39999]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T11:10:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[39999]: osdmap e36: 6 total, 5 up, 6 in 2024-04-12T11:10:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[39999]: from='osd.5 [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:30 smithi029 ceph-mon[39999]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:31.364 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:10:31.614 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:31 smithi087 ceph-mon[32917]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:10:31.614 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:31 smithi087 ceph-mon[32917]: osdmap e37: 6 total, 5 up, 6 in 2024-04-12T11:10:31.614 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:31 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:31.614 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:31 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:31.614 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:31 smithi087 ceph-mon[32917]: osd.5 [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185] boot 2024-04-12T11:10:31.614 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:31 smithi087 ceph-mon[32917]: osdmap e38: 6 total, 6 up, 6 in 2024-04-12T11:10:31.614 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:31 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[29349]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:10:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[29349]: osdmap e37: 6 total, 5 up, 6 in 2024-04-12T11:10:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[29349]: osd.5 [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185] boot 2024-04-12T11:10:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[29349]: osdmap e38: 6 total, 6 up, 6 in 2024-04-12T11:10:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[39999]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:10:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[39999]: osdmap e37: 6 total, 5 up, 6 in 2024-04-12T11:10:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[39999]: osd.5 [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185] boot 2024-04-12T11:10:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[39999]: osdmap e38: 6 total, 6 up, 6 in 2024-04-12T11:10:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:31 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:10:32.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:32 smithi087 ceph-mon[32917]: purged_snaps scrub starts 2024-04-12T11:10:32.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:32 smithi087 ceph-mon[32917]: purged_snaps scrub ok 2024-04-12T11:10:32.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:32 smithi087 ceph-mon[32917]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:32 smithi087 ceph-mon[32917]: osdmap e39: 6 total, 6 up, 6 in 2024-04-12T11:10:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:32 smithi029 ceph-mon[29349]: purged_snaps scrub starts 2024-04-12T11:10:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:32 smithi029 ceph-mon[29349]: purged_snaps scrub ok 2024-04-12T11:10:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:32 smithi029 ceph-mon[29349]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:32 smithi029 ceph-mon[29349]: osdmap e39: 6 total, 6 up, 6 in 2024-04-12T11:10:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:32 smithi029 ceph-mon[39999]: purged_snaps scrub starts 2024-04-12T11:10:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:32 smithi029 ceph-mon[39999]: purged_snaps scrub ok 2024-04-12T11:10:32.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:32 smithi029 ceph-mon[39999]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-12T11:10:32.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:32 smithi029 ceph-mon[39999]: osdmap e39: 6 total, 6 up, 6 in 2024-04-12T11:10:33.759 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-12T11:10:33.759 INFO:teuthology.orchestra.run.smithi087.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-12T11:10:33.759 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10+0 records in 2024-04-12T11:10:33.759 INFO:teuthology.orchestra.run.smithi087.stderr:10+0 records out 2024-04-12T11:10:33.759 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0134488 s, 780 MB/s 2024-04-12T11:10:33.759 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping successful for: 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: Detected new or changed devices on smithi087 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi087 to 8544M 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: osdmap e40: 6 total, 6 up, 6 in 2024-04-12T11:10:33.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:33 smithi087 ceph-mon[32917]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:34.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: Detected new or changed devices on smithi087 2024-04-12T11:10:34.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:34.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi087 to 8544M 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: osdmap e40: 6 total, 6 up, 6 in 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[29349]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: Detected new or changed devices on smithi087 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:34.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:34.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi087 to 8544M 2024-04-12T11:10:34.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:34.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:34.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:34.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:34.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: osdmap e40: 6 total, 6 up, 6 in 2024-04-12T11:10:34.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:33 smithi029 ceph-mon[39999]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:34.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi087:vg_nvme/lv_2 2024-04-12T11:10:34.572 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:10:34.693 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:34 smithi087 ceph-mon[32917]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:10:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:34 smithi029 ceph-mon[29349]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:10:35.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:34 smithi029 ceph-mon[39999]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:10:35.911 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:35 smithi087 ceph-mon[32917]: osdmap e41: 6 total, 6 up, 6 in 2024-04-12T11:10:35.911 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:35 smithi087 ceph-mon[32917]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:35 smithi029 ceph-mon[29349]: osdmap e41: 6 total, 6 up, 6 in 2024-04-12T11:10:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:35 smithi029 ceph-mon[29349]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:35 smithi029 ceph-mon[39999]: osdmap e41: 6 total, 6 up, 6 in 2024-04-12T11:10:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:35 smithi029 ceph-mon[39999]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:36.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:36 smithi087 ceph-mon[32917]: osdmap e42: 6 total, 6 up, 6 in 2024-04-12T11:10:36.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:36 smithi087 ceph-mon[32917]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:10:36.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:36 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:10:36.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:36 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:10:36.897 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:36 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:36 smithi029 ceph-mon[29349]: osdmap e42: 6 total, 6 up, 6 in 2024-04-12T11:10:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:36 smithi029 ceph-mon[29349]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:10:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:36 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:10:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:36 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:10:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:36 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:36 smithi029 ceph-mon[39999]: osdmap e42: 6 total, 6 up, 6 in 2024-04-12T11:10:37.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:36 smithi029 ceph-mon[39999]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:10:37.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:36 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:10:37.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:36 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:10:37.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:36 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:37.873 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:37 smithi087 ceph-mon[32917]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:37 smithi029 ceph-mon[29349]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:37 smithi029 ceph-mon[39999]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:38 smithi087 ceph-mon[32917]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-12T11:10:38.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:38 smithi087 ceph-mon[32917]: Cluster is now healthy 2024-04-12T11:10:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:38 smithi029 ceph-mon[29349]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-12T11:10:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:38 smithi029 ceph-mon[29349]: Cluster is now healthy 2024-04-12T11:10:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:38 smithi029 ceph-mon[39999]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-12T11:10:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:38 smithi029 ceph-mon[39999]: Cluster is now healthy 2024-04-12T11:10:39.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:39 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/3623256947' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3daf2021-0642-495d-803d-ad11f134a765"}]: dispatch 2024-04-12T11:10:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:39 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3daf2021-0642-495d-803d-ad11f134a765"}]: dispatch 2024-04-12T11:10:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:39 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3daf2021-0642-495d-803d-ad11f134a765"}]': finished 2024-04-12T11:10:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:39 smithi087 ceph-mon[32917]: osdmap e43: 7 total, 6 up, 7 in 2024-04-12T11:10:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:39 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:39 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/834949984' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:10:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:39 smithi087 ceph-mon[32917]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T11:10:40.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/3623256947' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3daf2021-0642-495d-803d-ad11f134a765"}]: dispatch 2024-04-12T11:10:40.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3daf2021-0642-495d-803d-ad11f134a765"}]: dispatch 2024-04-12T11:10:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3daf2021-0642-495d-803d-ad11f134a765"}]': finished 2024-04-12T11:10:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[29349]: osdmap e43: 7 total, 6 up, 7 in 2024-04-12T11:10:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/834949984' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:10:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[29349]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T11:10:40.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/3623256947' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3daf2021-0642-495d-803d-ad11f134a765"}]: dispatch 2024-04-12T11:10:40.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3daf2021-0642-495d-803d-ad11f134a765"}]: dispatch 2024-04-12T11:10:40.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3daf2021-0642-495d-803d-ad11f134a765"}]': finished 2024-04-12T11:10:40.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[39999]: osdmap e43: 7 total, 6 up, 7 in 2024-04-12T11:10:40.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:40.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/834949984' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:10:40.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:39 smithi029 ceph-mon[39999]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T11:10:42.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:42 smithi087 ceph-mon[32917]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-04-12T11:10:42.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:42 smithi029 ceph-mon[29349]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-04-12T11:10:42.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:42 smithi029 ceph-mon[39999]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-04-12T11:10:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:44 smithi029 ceph-mon[29349]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T11:10:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:44 smithi029 ceph-mon[39999]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T11:10:44.787 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:44 smithi087 ceph-mon[32917]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-12T11:10:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:45 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T11:10:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:45 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:45 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T11:10:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:45 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:45.808 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:45 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T11:10:45.808 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:45 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:46 smithi029 ceph-mon[29349]: Deploying daemon osd.6 on smithi087 2024-04-12T11:10:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:46 smithi029 ceph-mon[29349]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-12T11:10:46.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:46 smithi029 ceph-mon[39999]: Deploying daemon osd.6 on smithi087 2024-04-12T11:10:46.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:46 smithi029 ceph-mon[39999]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-12T11:10:46.816 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:46 smithi087 ceph-mon[32917]: Deploying daemon osd.6 on smithi087 2024-04-12T11:10:46.816 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:46 smithi087 ceph-mon[32917]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-12T11:10:48.765 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:48 smithi087 ceph-mon[32917]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:48 smithi029 ceph-mon[29349]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:48 smithi029 ceph-mon[39999]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:50.685 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:50 smithi087 ceph-mon[32917]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:50.685 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:50 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:50.685 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:50 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:50.686 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:50 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:50.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:50 smithi029 ceph-mon[29349]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:50.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:50 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:50.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:50 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:50.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:50 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:50.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:50 smithi029 ceph-mon[39999]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:50.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:50.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:50.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:52.735 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:52 smithi087 ceph-mon[32917]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:52.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:52 smithi029 ceph-mon[29349]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:52 smithi029 ceph-mon[39999]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:53.836 INFO:teuthology.orchestra.run.smithi087.stdout:Created osd(s) 6 on host 'smithi087' 2024-04-12T11:10:54.184 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:53 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.184 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:53 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.184 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:53 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:54.184 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:53 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:54.184 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:53 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.184 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:53 smithi087 ceph-mon[32917]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T11:10:54.184 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:53 smithi087 ceph-mon[32917]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:54.184 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:53 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.184 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:53 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.185 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:53 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[29349]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[29349]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:54.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:54.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[39999]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T11:10:54.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[39999]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:54.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:54.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:53 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:10:54.456 DEBUG:teuthology.orchestra.run.smithi087:osd.6> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.6.service 2024-04-12T11:10:54.458 INFO:tasks.cephadm:Deploying osd.7 on smithi087 with /dev/vg_nvme/lv_1... 2024-04-12T11:10:54.458 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-12T11:10:55.121 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:10:55.242 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:54 smithi087 ceph-mon[32917]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T11:10:55.243 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:54 smithi087 ceph-mon[32917]: osdmap e44: 7 total, 6 up, 7 in 2024-04-12T11:10:55.243 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:54 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:55.243 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:54 smithi087 ceph-mon[32917]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:55.243 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 11:10:54 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T11:10:54.994+0000 7f562fe62700 -1 osd.6 0 waiting for initial osdmap 2024-04-12T11:10:55.243 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 11:10:55 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T11:10:55.000+0000 7f562bc57700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:10:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:54 smithi029 ceph-mon[29349]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T11:10:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:54 smithi029 ceph-mon[29349]: osdmap e44: 7 total, 6 up, 7 in 2024-04-12T11:10:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:54 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:54 smithi029 ceph-mon[29349]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:54 smithi029 ceph-mon[39999]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T11:10:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:54 smithi029 ceph-mon[39999]: osdmap e44: 7 total, 6 up, 7 in 2024-04-12T11:10:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:54 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:55.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:54 smithi029 ceph-mon[39999]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:10:56.070 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:55 smithi087 ceph-mon[32917]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:10:56.070 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:55 smithi087 ceph-mon[32917]: osdmap e45: 7 total, 6 up, 7 in 2024-04-12T11:10:56.070 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:56.070 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:55 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:56.070 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:55 smithi087 ceph-mon[32917]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:56.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:55 smithi029 ceph-mon[29349]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:10:56.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:55 smithi029 ceph-mon[29349]: osdmap e45: 7 total, 6 up, 7 in 2024-04-12T11:10:56.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:56.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:55 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:56.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:55 smithi029 ceph-mon[29349]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:56.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:55 smithi029 ceph-mon[39999]: from='osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:10:56.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:55 smithi029 ceph-mon[39999]: osdmap e45: 7 total, 6 up, 7 in 2024-04-12T11:10:56.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:56.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:55 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:56.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:55 smithi029 ceph-mon[39999]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-12T11:10:57.265 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-12T11:10:57.266 INFO:teuthology.orchestra.run.smithi087.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-12T11:10:57.266 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10+0 records in 2024-04-12T11:10:57.266 INFO:teuthology.orchestra.run.smithi087.stderr:10+0 records out 2024-04-12T11:10:57.266 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012516 s, 838 MB/s 2024-04-12T11:10:57.266 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping successful for: 2024-04-12T11:10:57.266 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:56 smithi087 ceph-mon[32917]: purged_snaps scrub starts 2024-04-12T11:10:57.266 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:56 smithi087 ceph-mon[32917]: purged_snaps scrub ok 2024-04-12T11:10:57.266 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:56 smithi087 ceph-mon[32917]: osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062] boot 2024-04-12T11:10:57.266 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:56 smithi087 ceph-mon[32917]: osdmap e46: 7 total, 7 up, 7 in 2024-04-12T11:10:57.267 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:56 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:57.267 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:56 smithi087 ceph-mon[32917]: osdmap e47: 7 total, 7 up, 7 in 2024-04-12T11:10:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[29349]: purged_snaps scrub starts 2024-04-12T11:10:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[29349]: purged_snaps scrub ok 2024-04-12T11:10:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[29349]: osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062] boot 2024-04-12T11:10:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[29349]: osdmap e46: 7 total, 7 up, 7 in 2024-04-12T11:10:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[29349]: osdmap e47: 7 total, 7 up, 7 in 2024-04-12T11:10:57.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[39999]: purged_snaps scrub starts 2024-04-12T11:10:57.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[39999]: purged_snaps scrub ok 2024-04-12T11:10:57.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[39999]: osd.6 [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062] boot 2024-04-12T11:10:57.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[39999]: osdmap e46: 7 total, 7 up, 7 in 2024-04-12T11:10:57.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:10:57.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:56 smithi029 ceph-mon[39999]: osdmap e47: 7 total, 7 up, 7 in 2024-04-12T11:10:57.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch daemon add osd smithi087:vg_nvme/lv_1 2024-04-12T11:10:58.520 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:58 smithi087 ceph-mon[32917]: osdmap e48: 7 total, 7 up, 7 in 2024-04-12T11:10:58.520 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:58 smithi087 ceph-mon[32917]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:10:58.568 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:10:58.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:58 smithi029 ceph-mon[29349]: osdmap e48: 7 total, 7 up, 7 in 2024-04-12T11:10:58.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:58 smithi029 ceph-mon[29349]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:10:58.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:58 smithi029 ceph-mon[39999]: osdmap e48: 7 total, 7 up, 7 in 2024-04-12T11:10:58.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:58 smithi029 ceph-mon[39999]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:10:59.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:59 smithi087 ceph-mon[32917]: osdmap e49: 7 total, 7 up, 7 in 2024-04-12T11:10:59.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:59 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:59 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:59 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:59.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:59 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:59.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:59 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:59.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:59 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:59 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:59.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:59 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:59.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:10:59 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[29349]: osdmap e49: 7 total, 7 up, 7 in 2024-04-12T11:10:59.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:59.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:59.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[39999]: osdmap e49: 7 total, 7 up, 7 in 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:10:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:10:59.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:10:59.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:10:59 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:00.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:00 smithi087 ceph-mon[32917]: Detected new or changed devices on smithi087 2024-04-12T11:11:00.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:00 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi087 to 5696M 2024-04-12T11:11:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:00 smithi087 ceph-mon[32917]: osdmap e50: 7 total, 7 up, 7 in 2024-04-12T11:11:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:00 smithi087 ceph-mon[32917]: pgmap v155: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail; 167 KiB/s, 0 objects/s recovering 2024-04-12T11:11:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:11:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:11:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:00 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[29349]: Detected new or changed devices on smithi087 2024-04-12T11:11:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi087 to 5696M 2024-04-12T11:11:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[29349]: osdmap e50: 7 total, 7 up, 7 in 2024-04-12T11:11:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[29349]: pgmap v155: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail; 167 KiB/s, 0 objects/s recovering 2024-04-12T11:11:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:11:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:11:00.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:00.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[39999]: Detected new or changed devices on smithi087 2024-04-12T11:11:00.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi087 to 5696M 2024-04-12T11:11:00.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[39999]: osdmap e50: 7 total, 7 up, 7 in 2024-04-12T11:11:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[39999]: pgmap v155: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail; 167 KiB/s, 0 objects/s recovering 2024-04-12T11:11:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T11:11:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T11:11:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:00 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:01.662 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:01 smithi087 ceph-mon[32917]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:01.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:01 smithi029 ceph-mon[29349]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:01.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:01 smithi029 ceph-mon[39999]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:02.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:02 smithi087 ceph-mon[32917]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-12T11:11:02.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:02 smithi029 ceph-mon[29349]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-12T11:11:02.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:02 smithi029 ceph-mon[39999]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-12T11:11:03.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:03 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/2761675014' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e"}]: dispatch 2024-04-12T11:11:03.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:03 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e"}]: dispatch 2024-04-12T11:11:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:03 smithi087 ceph-mon[32917]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e"}]': finished 2024-04-12T11:11:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:03 smithi087 ceph-mon[32917]: osdmap e51: 8 total, 7 up, 8 in 2024-04-12T11:11:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:03 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:03 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/2761675014' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e"}]: dispatch 2024-04-12T11:11:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:03 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e"}]: dispatch 2024-04-12T11:11:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:03 smithi029 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e"}]': finished 2024-04-12T11:11:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:03 smithi029 ceph-mon[29349]: osdmap e51: 8 total, 7 up, 8 in 2024-04-12T11:11:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:03 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:03 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/2761675014' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e"}]: dispatch 2024-04-12T11:11:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:03 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e"}]: dispatch 2024-04-12T11:11:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:03 smithi029 ceph-mon[39999]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e"}]': finished 2024-04-12T11:11:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:03 smithi029 ceph-mon[39999]: osdmap e51: 8 total, 7 up, 8 in 2024-04-12T11:11:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:03 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:04.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:04 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/3463711684' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:11:04.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:04 smithi087 ceph-mon[32917]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T11:11:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:04 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/3463711684' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:11:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:04 smithi029 ceph-mon[29349]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T11:11:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:04 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/3463711684' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T11:11:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:04 smithi029 ceph-mon[39999]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T11:11:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:06 smithi029 ceph-mon[29349]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:06.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:06 smithi029 ceph-mon[39999]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:06.774 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:06 smithi087 ceph-mon[32917]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:08 smithi029 ceph-mon[29349]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:08 smithi029 ceph-mon[39999]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:08.953 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:08 smithi087 ceph-mon[32917]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:09.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:09 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T11:11:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:09 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T11:11:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:09 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:09 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T11:11:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:09 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:10.699 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:10 smithi087 ceph-mon[32917]: Deploying daemon osd.7 on smithi087 2024-04-12T11:11:10.699 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:10 smithi087 ceph-mon[32917]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:10 smithi029 ceph-mon[29349]: Deploying daemon osd.7 on smithi087 2024-04-12T11:11:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:10 smithi029 ceph-mon[29349]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:10 smithi029 ceph-mon[39999]: Deploying daemon osd.7 on smithi087 2024-04-12T11:11:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:10 smithi029 ceph-mon[39999]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:12.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:12 smithi029 ceph-mon[29349]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:12.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:12 smithi029 ceph-mon[39999]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:12.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:12 smithi087 ceph-mon[32917]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:14 smithi029 ceph-mon[29349]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:14 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:14 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:14 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:14.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:14 smithi029 ceph-mon[39999]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:14 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:14 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:14 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:14.850 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:14 smithi087 ceph-mon[32917]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:14.850 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:14 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:14.850 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:14 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:14.850 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:14 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:16 smithi029 ceph-mon[29349]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:16.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:16 smithi029 ceph-mon[39999]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:16.791 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:16 smithi087 ceph-mon[32917]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:17.781 INFO:teuthology.orchestra.run.smithi087.stdout:Created osd(s) 7 on host 'smithi087' 2024-04-12T11:11:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: from='osd.7 [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T11:11:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T11:11:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:11:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:17 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:18.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:18.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:18.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:18.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: from='osd.7 [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T11:11:18.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: from='osd.7 [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T11:11:18.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:18.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:11:18.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:18.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:18.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:18.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:17 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:18.344 DEBUG:teuthology.orchestra.run.smithi087:osd.7> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.7.service 2024-04-12T11:11:18.346 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-12T11:11:18.346 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd stat -f json 2024-04-12T11:11:18.566 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:11:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:18 smithi087 ceph-mon[32917]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T11:11:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:18 smithi087 ceph-mon[32917]: osdmap e52: 8 total, 7 up, 8 in 2024-04-12T11:11:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:18 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:18 smithi087 ceph-mon[32917]: from='osd.7 [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:11:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:18 smithi087 ceph-mon[32917]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:11:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:18 smithi029 ceph-mon[29349]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T11:11:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:18 smithi029 ceph-mon[29349]: osdmap e52: 8 total, 7 up, 8 in 2024-04-12T11:11:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:18 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:18 smithi029 ceph-mon[29349]: from='osd.7 [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:11:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:18 smithi029 ceph-mon[29349]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:11:19.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:18 smithi029 ceph-mon[39999]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T11:11:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:18 smithi029 ceph-mon[39999]: osdmap e52: 8 total, 7 up, 8 in 2024-04-12T11:11:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:18 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:18 smithi029 ceph-mon[39999]: from='osd.7 [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:11:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:18 smithi029 ceph-mon[39999]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:11:19.497 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 11:11:19 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T11:11:19.400+0000 7f2afb068700 -1 osd.7 0 waiting for initial osdmap 2024-04-12T11:11:19.497 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 11:11:19 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T11:11:19.407+0000 7f2af2e55700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:11:19.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[29349]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:11:19.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[29349]: osdmap e53: 8 total, 7 up, 8 in 2024-04-12T11:11:19.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:19.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:19.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[29349]: from='osd.7 ' entity='osd.7' 2024-04-12T11:11:19.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[29349]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:19.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[39999]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:11:19.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[39999]: osdmap e53: 8 total, 7 up, 8 in 2024-04-12T11:11:19.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:19.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:19.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[39999]: from='osd.7 ' entity='osd.7' 2024-04-12T11:11:19.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:19 smithi029 ceph-mon[39999]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:20.069 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:19 smithi087 ceph-mon[32917]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-12T11:11:20.069 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:19 smithi087 ceph-mon[32917]: osdmap e53: 8 total, 7 up, 8 in 2024-04-12T11:11:20.069 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:19 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:20.069 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:19 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:20.070 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:19 smithi087 ceph-mon[32917]: from='osd.7 ' entity='osd.7' 2024-04-12T11:11:20.070 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:19 smithi087 ceph-mon[32917]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-12T11:11:20.435 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:11:21.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[39999]: purged_snaps scrub starts 2024-04-12T11:11:21.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[39999]: purged_snaps scrub ok 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[39999]: osd.7 [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515] boot 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[39999]: osdmap e54: 8 total, 8 up, 8 in 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1473124589' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[39999]: osdmap e55: 8 total, 8 up, 8 in 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[29349]: purged_snaps scrub starts 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[29349]: purged_snaps scrub ok 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[29349]: osd.7 [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515] boot 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[29349]: osdmap e54: 8 total, 8 up, 8 in 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1473124589' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-12T11:11:21.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:20 smithi029 ceph-mon[29349]: osdmap e55: 8 total, 8 up, 8 in 2024-04-12T11:11:21.104 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":54,"num_osds":8,"num_up_osds":8,"osd_up_since":1712920279,"num_in_osds":8,"osd_in_since":1712920262,"num_remapped_pgs":1} 2024-04-12T11:11:21.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T11:11:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:20 smithi087 ceph-mon[32917]: purged_snaps scrub starts 2024-04-12T11:11:21.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:20 smithi087 ceph-mon[32917]: purged_snaps scrub ok 2024-04-12T11:11:21.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:20 smithi087 ceph-mon[32917]: osd.7 [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515] boot 2024-04-12T11:11:21.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:20 smithi087 ceph-mon[32917]: osdmap e54: 8 total, 8 up, 8 in 2024-04-12T11:11:21.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:20 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:21.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:20 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1473124589' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-12T11:11:21.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:20 smithi087 ceph-mon[32917]: osdmap e55: 8 total, 8 up, 8 in 2024-04-12T11:11:21.313 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:11:22.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: Detected new or changed devices on smithi087 2024-04-12T11:11:22.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi087 to 4272M 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[29349]: osdmap e56: 8 total, 8 up, 8 in 2024-04-12T11:11:22.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: Detected new or changed devices on smithi087 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi087 to 4272M 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:22.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:22 smithi029 ceph-mon[39999]: osdmap e56: 8 total, 8 up, 8 in 2024-04-12T11:11:22.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: Detected new or changed devices on smithi087 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi087 to 4272M 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:22 smithi087 ceph-mon[32917]: osdmap e56: 8 total, 8 up, 8 in 2024-04-12T11:11:23.147 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:11:23.147 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":56,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T11:11:21.952975+0000","last_up_change":"2024-04-12T11:11:19.950253+0000","last_in_change":"2024-04-12T11:11:02.730106+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6803","nonce":2299659597}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6805","nonce":2299659597}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6809","nonce":2299659597}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6807","nonce":2299659597}]},"public_addr":"172.21.15.29:6803/2299659597","cluster_addr":"172.21.15.29:6805/2299659597","heartbeat_back_addr":"172.21.15.29:6809/2299659597","heartbeat_front_addr":"172.21.15.29:6807/2299659597","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6811","nonce":3193349615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6813","nonce":3193349615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6817","nonce":3193349615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6815","nonce":3193349615}]},"public_addr":"172.21.15.29:6811/3193349615","cluster_addr":"172.21.15.29:6813/3193349615","heartbeat_back_addr":"172.21.15.29:6817/3193349615","heartbeat_front_addr":"172.21.15.29:6815/3193349615","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6819","nonce":1946236732}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6821","nonce":1946236732}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6825","nonce":1946236732}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6823","nonce":1946236732}]},"public_addr":"172.21.15.29:6819/1946236732","cluster_addr":"172.21.15.29:6821/1946236732","heartbeat_back_addr":"172.21.15.29:6825/1946236732","heartbeat_front_addr":"172.21.15.29:6823/1946236732","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6827","nonce":4104584972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6829","nonce":4104584972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6833","nonce":4104584972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6831","nonce":4104584972}]},"public_addr":"172.21.15.29:6827/4104584972","cluster_addr":"172.21.15.29:6829/4104584972","heartbeat_back_addr":"172.21.15.29:6833/4104584972","heartbeat_front_addr":"172.21.15.29:6831/4104584972","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6801","nonce":2970504758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6803","nonce":2970504758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6807","nonce":2970504758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6805","nonce":2970504758}]},"public_addr":"172.21.15.87:6801/2970504758","cluster_addr":"172.21.15.87:6803/2970504758","heartbeat_back_addr":"172.21.15.87:6807/2970504758","heartbeat_front_addr":"172.21.15.87:6805/2970504758","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6809","nonce":3642701185}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6811","nonce":3642701185}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6815","nonce":3642701185}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6813","nonce":3642701185}]},"public_addr":"172.21.15.87:6809/3642701185","cluster_addr":"172.21.15.87:6811/3642701185","heartbeat_back_addr":"172.21.15.87:6815/3642701185","heartbeat_front_addr":"172.21.15.87:6813/3642701185","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T11:11:23.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:23 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3345752632' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:11:23.731 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:23 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3345752632' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:11:23.731 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:23 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3345752632' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:11:23.757 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-12T11:09:21.719063+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-12T11:11:23.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-12T11:11:23.965 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:11:24.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:24 smithi087 ceph-mon[32917]: osdmap e57: 8 total, 8 up, 8 in 2024-04-12T11:11:24.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:24 smithi087 ceph-mon[32917]: pgmap v174: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2024-04-12T11:11:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:24 smithi029 ceph-mon[39999]: osdmap e57: 8 total, 8 up, 8 in 2024-04-12T11:11:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:24 smithi029 ceph-mon[39999]: pgmap v174: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2024-04-12T11:11:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:24 smithi029 ceph-mon[29349]: osdmap e57: 8 total, 8 up, 8 in 2024-04-12T11:11:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:24 smithi029 ceph-mon[29349]: pgmap v174: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2024-04-12T11:11:25.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:25 smithi029 ceph-mon[29349]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:11:25.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:25 smithi029 ceph-mon[29349]: osdmap e58: 8 total, 8 up, 8 in 2024-04-12T11:11:25.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:25 smithi029 ceph-mon[39999]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:11:25.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:25 smithi029 ceph-mon[39999]: osdmap e58: 8 total, 8 up, 8 in 2024-04-12T11:11:25.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:25 smithi087 ceph-mon[32917]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:11:25.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:25 smithi087 ceph-mon[32917]: osdmap e58: 8 total, 8 up, 8 in 2024-04-12T11:11:25.894 INFO:teuthology.orchestra.run.smithi029.stdout:pg_num: 1 2024-04-12T11:11:26.490 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:26 smithi029 ceph-mon[29349]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-04-12T11:11:26.491 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:26 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2865360538' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-12T11:11:26.491 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:26 smithi029 ceph-mon[39999]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-04-12T11:11:26.491 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:26 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2865360538' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-12T11:11:26.514 INFO:tasks.cephadm:Adding prometheus.a on smithi087 2024-04-12T11:11:26.514 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch apply prometheus '1;smithi087=a' 2024-04-12T11:11:26.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:26 smithi087 ceph-mon[32917]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-04-12T11:11:26.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:26 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2865360538' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-12T11:11:26.735 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:11:27.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:27 smithi087 ceph-mon[32917]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-12T11:11:27.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:27 smithi087 ceph-mon[32917]: Cluster is now healthy 2024-04-12T11:11:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:27 smithi029 ceph-mon[29349]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-12T11:11:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:27 smithi029 ceph-mon[29349]: Cluster is now healthy 2024-04-12T11:11:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:27 smithi029 ceph-mon[39999]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-12T11:11:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:27 smithi029 ceph-mon[39999]: Cluster is now healthy 2024-04-12T11:11:28.433 INFO:teuthology.orchestra.run.smithi087.stdout:Scheduled prometheus update... 2024-04-12T11:11:28.446 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:28 smithi087 ceph-mon[32917]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T11:11:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:28 smithi029 ceph-mon[29349]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T11:11:28.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:28 smithi029 ceph-mon[39999]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-12T11:11:29.065 DEBUG:teuthology.orchestra.run.smithi087:prometheus.a> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@prometheus.a.service 2024-04-12T11:11:29.067 INFO:tasks.cephadm:Adding node-exporter.a on smithi029 2024-04-12T11:11:29.067 INFO:tasks.cephadm:Adding node-exporter.b on smithi087 2024-04-12T11:11:29.067 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch apply node-exporter '2;smithi029=a;smithi087=b' 2024-04-12T11:11:29.328 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:11:29.696 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:29 smithi087 ceph-mon[32917]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi087=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:29.696 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:29 smithi087 ceph-mon[32917]: Saving service prometheus spec with placement smithi087=a;count:1 2024-04-12T11:11:29.696 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:29.696 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:29.696 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:29.696 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:11:29.696 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:29 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[29349]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi087=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[29349]: Saving service prometheus spec with placement smithi087=a;count:1 2024-04-12T11:11:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:11:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[39999]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi087=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[39999]: Saving service prometheus spec with placement smithi087=a;count:1 2024-04-12T11:11:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:11:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:11:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:29 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:30.698 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:30 smithi087 ceph-mon[32917]: Deploying daemon prometheus.a on smithi087 2024-04-12T11:11:30.698 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:30 smithi087 ceph-mon[32917]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-04-12T11:11:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:30 smithi029 ceph-mon[29349]: Deploying daemon prometheus.a on smithi087 2024-04-12T11:11:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:30 smithi029 ceph-mon[29349]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-04-12T11:11:30.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:30 smithi029 ceph-mon[39999]: Deploying daemon prometheus.a on smithi087 2024-04-12T11:11:30.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:30 smithi029 ceph-mon[39999]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-04-12T11:11:31.794 INFO:teuthology.orchestra.run.smithi087.stdout:Scheduled node-exporter update... 2024-04-12T11:11:32.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:32 smithi087 ceph-mon[32917]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-12T11:11:32.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:32 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:32 smithi029 ceph-mon[39999]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-12T11:11:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:32 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:32.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:32 smithi029 ceph-mon[29349]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-12T11:11:32.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:32 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:32.788 DEBUG:teuthology.orchestra.run.smithi029:node-exporter.a> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@node-exporter.a.service 2024-04-12T11:11:32.790 DEBUG:teuthology.orchestra.run.smithi087:node-exporter.b> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@node-exporter.b.service 2024-04-12T11:11:32.792 INFO:tasks.cephadm:Adding alertmanager.a on smithi029 2024-04-12T11:11:32.793 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch apply alertmanager '1;smithi029=a' 2024-04-12T11:11:33.034 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:11:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:33 smithi029 ceph-mon[39999]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi029=a;smithi087=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:33 smithi029 ceph-mon[39999]: Saving service node-exporter spec with placement smithi029=a;smithi087=b;count:2 2024-04-12T11:11:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:33 smithi029 ceph-mon[29349]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi029=a;smithi087=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:33 smithi029 ceph-mon[29349]: Saving service node-exporter spec with placement smithi029=a;smithi087=b;count:2 2024-04-12T11:11:33.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:33 smithi087 ceph-mon[32917]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi029=a;smithi087=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:33.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:33 smithi087 ceph-mon[32917]: Saving service node-exporter spec with placement smithi029=a;smithi087=b;count:2 2024-04-12T11:11:34.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:34 smithi029 ceph-mon[29349]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T11:11:34.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:34 smithi029 ceph-mon[39999]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T11:11:34.851 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:34 smithi087 ceph-mon[32917]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T11:11:35.331 INFO:teuthology.orchestra.run.smithi087.stdout:Scheduled alertmanager update... 2024-04-12T11:11:35.961 DEBUG:teuthology.orchestra.run.smithi029:alertmanager.a> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@alertmanager.a.service 2024-04-12T11:11:35.964 INFO:tasks.cephadm:Adding grafana.a on smithi087 2024-04-12T11:11:35.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph orch apply grafana '1;smithi087=a' 2024-04-12T11:11:36.168 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:11:36.544 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:36 smithi087 ceph-mon[32917]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi029=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:36.545 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:36 smithi087 ceph-mon[32917]: Saving service alertmanager spec with placement smithi029=a;count:1 2024-04-12T11:11:36.545 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:36 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:36.545 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:36 smithi087 ceph-mon[32917]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T11:11:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:36 smithi029 ceph-mon[29349]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi029=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:36 smithi029 ceph-mon[29349]: Saving service alertmanager spec with placement smithi029=a;count:1 2024-04-12T11:11:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:36 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:36 smithi029 ceph-mon[29349]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T11:11:36.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:36 smithi029 ceph-mon[39999]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi029=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:36.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:36 smithi029 ceph-mon[39999]: Saving service alertmanager spec with placement smithi029=a;count:1 2024-04-12T11:11:36.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:36 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:36.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:36 smithi029 ceph-mon[39999]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T11:11:38.077 INFO:teuthology.orchestra.run.smithi087.stdout:Scheduled grafana update... 2024-04-12T11:11:38.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:38 smithi087 ceph-mon[32917]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:38.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:38 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:38.762 DEBUG:teuthology.orchestra.run.smithi087:grafana.a> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@grafana.a.service 2024-04-12T11:11:38.765 INFO:tasks.cephadm:Setting up client nodes... 2024-04-12T11:11:38.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-12T11:11:38.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:38 smithi029 ceph-mon[29349]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:38.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:38 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:38.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:38 smithi029 ceph-mon[39999]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:38.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:38 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:38.981 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:11:39.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:39 smithi029 ceph-mon[39999]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi087=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:39.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:39 smithi029 ceph-mon[39999]: Saving service grafana spec with placement smithi087=a;count:1 2024-04-12T11:11:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:39 smithi029 ceph-mon[29349]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi087=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:39 smithi029 ceph-mon[29349]: Saving service grafana spec with placement smithi087=a;count:1 2024-04-12T11:11:39.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:39 smithi087 ceph-mon[32917]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi087=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:11:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:39 smithi087 ceph-mon[32917]: Saving service grafana spec with placement smithi087=a;count:1 2024-04-12T11:11:40.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:40 smithi029 ceph-mon[39999]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:40.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:40 smithi029 ceph-mon[29349]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:40.892 INFO:teuthology.orchestra.run.smithi029.stdout:[client.0] 2024-04-12T11:11:40.892 INFO:teuthology.orchestra.run.smithi029.stdout: key = AQDsFhlmGUcINRAAimPAW4+UgVHBQIgnp2Pntg== 2024-04-12T11:11:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:40 smithi087 ceph-mon[32917]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:41.572 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-04-12T11:11:41.572 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-12T11:11:41.572 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-12T11:11:41.616 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-12T11:11:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:41 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2713752518' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T11:11:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:41 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2713752518' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T11:11:41.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:41 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2713752518' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T11:11:41.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:41 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2713752518' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T11:11:41.828 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:11:41.950 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:41 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2713752518' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T11:11:41.950 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:41 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2713752518' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T11:11:42.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:42 smithi029 ceph-mon[29349]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:42.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:42 smithi029 ceph-mon[39999]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:42 smithi087 ceph-mon[32917]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:43.987 INFO:teuthology.orchestra.run.smithi087.stdout:[client.1] 2024-04-12T11:11:43.987 INFO:teuthology.orchestra.run.smithi087.stdout: key = AQDvFhlmoW2gOhAA4FwjRvGr6oLwyOUvLsALjQ== 2024-04-12T11:11:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:44 smithi029 ceph-mon[39999]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:44 smithi029 ceph-mon[39999]: from='client.? 172.21.15.87:0/1664599867' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T11:11:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:44 smithi029 ceph-mon[39999]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T11:11:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:44 smithi029 ceph-mon[39999]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T11:11:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:44 smithi029 ceph-mon[29349]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:44 smithi029 ceph-mon[29349]: from='client.? 172.21.15.87:0/1664599867' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T11:11:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:44 smithi029 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T11:11:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:44 smithi029 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T11:11:44.841 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:44 smithi087 ceph-mon[32917]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:44.841 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:44 smithi087 ceph-mon[32917]: from='client.? 172.21.15.87:0/1664599867' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T11:11:44.841 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:44 smithi087 ceph-mon[32917]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T11:11:44.841 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:44 smithi087 ceph-mon[32917]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T11:11:45.432 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-12T11:11:45.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-12T11:11:45.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-12T11:11:45.478 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-12T11:11:45.478 DEBUG:teuthology.orchestra.run.smithi029:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-12T11:11:45.779 INFO:teuthology.misc.health.smithi029.stdout: 2024-04-12T11:11:45.780 INFO:teuthology.misc.health.smithi029.stdout:{"epoch":58,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T11:11:24.379497+0000","last_up_change":"2024-04-12T11:11:19.950253+0000","last_in_change":"2024-04-12T11:11:02.730106+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6803","nonce":2299659597}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6805","nonce":2299659597}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6809","nonce":2299659597}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6807","nonce":2299659597}]},"public_addr":"172.21.15.29:6803/2299659597","cluster_addr":"172.21.15.29:6805/2299659597","heartbeat_back_addr":"172.21.15.29:6809/2299659597","heartbeat_front_addr":"172.21.15.29:6807/2299659597","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6811","nonce":3193349615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6813","nonce":3193349615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6817","nonce":3193349615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6815","nonce":3193349615}]},"public_addr":"172.21.15.29:6811/3193349615","cluster_addr":"172.21.15.29:6813/3193349615","heartbeat_back_addr":"172.21.15.29:6817/3193349615","heartbeat_front_addr":"172.21.15.29:6815/3193349615","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6819","nonce":1946236732}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6821","nonce":1946236732}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6825","nonce":1946236732}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6823","nonce":1946236732}]},"public_addr":"172.21.15.29:6819/1946236732","cluster_addr":"172.21.15.29:6821/1946236732","heartbeat_back_addr":"172.21.15.29:6825/1946236732","heartbeat_front_addr":"172.21.15.29:6823/1946236732","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6827","nonce":4104584972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6829","nonce":4104584972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6833","nonce":4104584972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6831","nonce":4104584972}]},"public_addr":"172.21.15.29:6827/4104584972","cluster_addr":"172.21.15.29:6829/4104584972","heartbeat_back_addr":"172.21.15.29:6833/4104584972","heartbeat_front_addr":"172.21.15.29:6831/4104584972","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6801","nonce":2970504758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6803","nonce":2970504758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6807","nonce":2970504758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6805","nonce":2970504758}]},"public_addr":"172.21.15.87:6801/2970504758","cluster_addr":"172.21.15.87:6803/2970504758","heartbeat_back_addr":"172.21.15.87:6807/2970504758","heartbeat_front_addr":"172.21.15.87:6805/2970504758","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6809","nonce":3642701185}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6811","nonce":3642701185}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6815","nonce":3642701185}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6813","nonce":3642701185}]},"public_addr":"172.21.15.87:6809/3642701185","cluster_addr":"172.21.15.87:6811/3642701185","heartbeat_back_addr":"172.21.15.87:6815/3642701185","heartbeat_front_addr":"172.21.15.87:6813/3642701185","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T11:11:45.791 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-12T11:11:45.791 INFO:tasks.cephadm:Creating RBD pool 2024-04-12T11:11:45.791 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-12T11:11:45.990 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:11:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:46 smithi029 ceph-mon[29349]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:46 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1951598991' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:11:46.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:46 smithi029 ceph-mon[39999]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:46.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:46 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1951598991' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:11:46.854 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:46 smithi087 ceph-mon[32917]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:46.854 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:46 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1951598991' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:11:48.525 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'rbd' created 2024-04-12T11:11:48.803 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:48 smithi087 ceph-mon[32917]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:48.803 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:48 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3277600422' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-12T11:11:48.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:48 smithi029 ceph-mon[39999]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:48.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:48 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3277600422' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-12T11:11:48.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:48 smithi029 ceph-mon[29349]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:48.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:48 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3277600422' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-12T11:11:49.262 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-12T11:11:49.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 systemd[1]: Starting Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:11:49.473 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:11:49.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:49 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3277600422' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-12T11:11:49.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:49 smithi029 ceph-mon[29349]: osdmap e59: 8 total, 8 up, 8 in 2024-04-12T11:11:49.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:49 smithi029 ceph-mon[29349]: osdmap e60: 8 total, 8 up, 8 in 2024-04-12T11:11:49.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:49 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3277600422' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-12T11:11:49.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:49 smithi029 ceph-mon[39999]: osdmap e59: 8 total, 8 up, 8 in 2024-04-12T11:11:49.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:49 smithi029 ceph-mon[39999]: osdmap e60: 8 total, 8 up, 8 in 2024-04-12T11:11:49.833 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3277600422' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-12T11:11:49.833 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-mon[32917]: osdmap e59: 8 total, 8 up, 8 in 2024-04-12T11:11:49.833 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-mon[32917]: osdmap e60: 8 total, 8 up, 8 in 2024-04-12T11:11:49.833 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 podman[61011]: 2024-04-12 11:11:49.486727601 +0000 UTC m=+0.018640890 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-12T11:11:49.834 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 podman[61011]: 2024-04-12 11:11:49.614772506 +0000 UTC m=+0.146685788 container create 545830a9d26da01d894bc15988ee94c5fba8f5624d5f2564fccc488c42c22216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:11:49.834 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 podman[61011]: 2024-04-12 11:11:49.833554766 +0000 UTC m=+0.365468049 container init 545830a9d26da01d894bc15988ee94c5fba8f5624d5f2564fccc488c42c22216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:11:50.217 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 podman[61011]: 2024-04-12 11:11:49.83612562 +0000 UTC m=+0.368038903 container start 545830a9d26da01d894bc15988ee94c5fba8f5624d5f2564fccc488c42c22216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:11:50.217 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.859Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-12T11:11:50.217 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.859Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-12T11:11:50.217 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.859Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi087 (none))" 2024-04-12T11:11:50.217 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.859Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-12T11:11:50.217 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.859Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-12T11:11:50.217 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.860Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.861Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.861Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.861Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.863Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.863Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.099µs 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.863Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.863Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.864Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.907µs wal_replay_duration=273.48µs wbl_replay_duration=141ns total_replay_duration=390.711µs 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.864Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.864Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.864Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-12T11:11:50.218 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.881Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.261812ms db_storage=705ns remote_storage=995ns web_handler=341ns query_engine=547ns scrape=400.392µs scrape_sd=75.778µs notify=870ns notify_sd=2.603µs rules=15.594346ms tracing=7.145µs 2024-04-12T11:11:50.219 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.881Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-12T11:11:50.219 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:11:49.881Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-12T11:11:50.219 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 bash[61011]: 545830a9d26da01d894bc15988ee94c5fba8f5624d5f2564fccc488c42c22216 2024-04-12T11:11:50.219 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:11:49 smithi087 systemd[1]: Started Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:11:50.795 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[39999]: pgmap v189: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:50.795 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[39999]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:11:50.795 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:50.796 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:50.796 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:50.796 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-12T11:11:50.796 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[39999]: osdmap e61: 8 total, 8 up, 8 in 2024-04-12T11:11:50.796 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[29349]: pgmap v189: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:50.796 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[29349]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:11:50.796 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:50.796 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:50.796 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:50.796 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-12T11:11:50.797 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:50 smithi029 ceph-mon[29349]: osdmap e61: 8 total, 8 up, 8 in 2024-04-12T11:11:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:50 smithi087 ceph-mon[32917]: pgmap v189: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:11:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:50 smithi087 ceph-mon[32917]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:11:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:50 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:50 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:50 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' 2024-04-12T11:11:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:50 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-12T11:11:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:50 smithi087 ceph-mon[32917]: osdmap e61: 8 total, 8 up, 8 in 2024-04-12T11:11:51.260 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: ignoring --setuser ceph since I am not root 2024-04-12T11:11:51.260 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: ignoring --setgroup ceph since I am not root 2024-04-12T11:11:51.260 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:51 smithi087 ceph-mgr[34796]: -- 172.21.15.87:0/4172063747 <== mon.1 v2:172.21.15.87:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d5d5b781a0 con 0x55d5d5b4e800 2024-04-12T11:11:51.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:51 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ignoring --setuser ceph since I am not root 2024-04-12T11:11:51.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:51 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ignoring --setgroup ceph since I am not root 2024-04-12T11:11:51.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:51.612+0000 7fa19ddff200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:11:51.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:51.709+0000 7fa19ddff200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:11:52.017 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:51 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:51.622+0000 7f9e9fcc3200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:11:52.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:51 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:51.718+0000 7f9e9fcc3200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:11:52.043 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-12T11:11:52.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-mon[39999]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-12T11:11:52.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-mon[39999]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-12T11:11:52.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/414296459' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:11:52.290 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-mon[29349]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-12T11:11:52.290 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-mon[29349]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-12T11:11:52.290 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/414296459' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:11:52.361 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:52 smithi087 ceph-mon[32917]: from='mgr.14148 172.21.15.29:0/4249332101' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-12T11:11:52.361 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:52 smithi087 ceph-mon[32917]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-12T11:11:52.361 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:52 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/414296459' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:11:52.619 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:52.381+0000 7f9e9fcc3200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:11:52.619 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:52.474+0000 7f9e9fcc3200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:11:52.678 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-12T11:11:52.679 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-12T11:11:52.679 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph mgr dump --format=json 2024-04-12T11:11:52.686 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:52.360+0000 7fa19ddff200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:11:52.686 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:52.450+0000 7fa19ddff200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:11:52.686 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:52.591+0000 7fa19ddff200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:11:52.686 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:52.685+0000 7fa19ddff200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:11:52.886 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:11:52.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:52.767+0000 7fa19ddff200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:11:53.008 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:52.617+0000 7f9e9fcc3200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:11:53.009 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:52.718+0000 7f9e9fcc3200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:11:53.009 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:52.805+0000 7f9e9fcc3200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:11:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:53 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/414296459' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:11:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:53 smithi029 ceph-mon[39999]: osdmap e62: 8 total, 8 up, 8 in 2024-04-12T11:11:53.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:53 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/414296459' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:11:53.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:53 smithi029 ceph-mon[29349]: osdmap e62: 8 total, 8 up, 8 in 2024-04-12T11:11:53.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:53 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:53.083+0000 7f9e9fcc3200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:11:53.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:53 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:53.198+0000 7f9e9fcc3200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:11:53.318 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:53 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/414296459' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:11:53.319 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:53 smithi087 ceph-mon[32917]: osdmap e62: 8 total, 8 up, 8 in 2024-04-12T11:11:53.319 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:53.029+0000 7fa19ddff200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:11:53.319 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:53.139+0000 7fa19ddff200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:11:53.319 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:53.223+0000 7fa19ddff200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:11:53.319 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:53.317+0000 7fa19ddff200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:11:53.566 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:53 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:53.289+0000 7f9e9fcc3200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:11:53.566 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:53 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:53.386+0000 7f9e9fcc3200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:11:53.566 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:53 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:53.565+0000 7f9e9fcc3200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:11:53.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:53.491+0000 7fa19ddff200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:11:54.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:54 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:54.441+0000 7fa19ddff200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:11:54.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:54 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:54.570+0000 7fa19ddff200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:11:54.748 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:11:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:54 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1626373603' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T11:11:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:54 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1626373603' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T11:11:55.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:54.569+0000 7f9e9fcc3200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:11:55.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:54.704+0000 7f9e9fcc3200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:11:55.183 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:54 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:54.901+0000 7fa19ddff200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:11:55.184 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:54 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:54.990+0000 7fa19ddff200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:11:55.184 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:54 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1626373603' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T11:11:55.345 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:55 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:55.050+0000 7f9e9fcc3200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:11:55.345 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:55 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:55.144+0000 7f9e9fcc3200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:11:55.345 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:55 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:55.344+0000 7f9e9fcc3200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:11:55.444 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":16,"flags":0,"active_gid":14148,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":1422272459},{"type":"v1","addr":"172.21.15.29:6801","nonce":1422272459}]},"active_addr":"172.21.15.29:6801/1422272459","active_change":"2024-04-12T11:07:09.398579+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24106,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.29:8443/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":3,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":2946468060}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":364858296}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":659258741}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3254157714}]}]} 2024-04-12T11:11:55.448 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-12T11:11:55.448 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-12T11:11:55.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T11:11:55.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:55 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:55.182+0000 7fa19ddff200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:11:55.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:55 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:55.365+0000 7fa19ddff200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:11:55.659 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:11:55.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:55 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:55.540+0000 7f9e9fcc3200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:11:55.905 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:55 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:55.647+0000 7fa19ddff200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:11:56.111 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:55 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:55.845+0000 7f9e9fcc3200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:11:56.111 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:56 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:56.109+0000 7f9e9fcc3200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:11:56.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:55 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:55.904+0000 7fa19ddff200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:11:56.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:56.135+0000 7fa19ddff200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:11:56.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:56 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:56.351+0000 7f9e9fcc3200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:11:57.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:56.925+0000 7fa19ddff200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:11:57.511 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:11:57.511 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":62,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T11:11:52.032791+0000","last_up_change":"2024-04-12T11:11:19.950253+0000","last_in_change":"2024-04-12T11:11:02.730106+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T11:11:48.103219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6803","nonce":2299659597}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6805","nonce":2299659597}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6809","nonce":2299659597}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6807","nonce":2299659597}]},"public_addr":"172.21.15.29:6803/2299659597","cluster_addr":"172.21.15.29:6805/2299659597","heartbeat_back_addr":"172.21.15.29:6809/2299659597","heartbeat_front_addr":"172.21.15.29:6807/2299659597","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6811","nonce":3193349615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6813","nonce":3193349615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6817","nonce":3193349615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6815","nonce":3193349615}]},"public_addr":"172.21.15.29:6811/3193349615","cluster_addr":"172.21.15.29:6813/3193349615","heartbeat_back_addr":"172.21.15.29:6817/3193349615","heartbeat_front_addr":"172.21.15.29:6815/3193349615","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6819","nonce":1946236732}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6821","nonce":1946236732}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6825","nonce":1946236732}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6823","nonce":1946236732}]},"public_addr":"172.21.15.29:6819/1946236732","cluster_addr":"172.21.15.29:6821/1946236732","heartbeat_back_addr":"172.21.15.29:6825/1946236732","heartbeat_front_addr":"172.21.15.29:6823/1946236732","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6827","nonce":4104584972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6829","nonce":4104584972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6833","nonce":4104584972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6831","nonce":4104584972}]},"public_addr":"172.21.15.29:6827/4104584972","cluster_addr":"172.21.15.29:6829/4104584972","heartbeat_back_addr":"172.21.15.29:6833/4104584972","heartbeat_front_addr":"172.21.15.29:6831/4104584972","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6801","nonce":2970504758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6803","nonce":2970504758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6807","nonce":2970504758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6805","nonce":2970504758}]},"public_addr":"172.21.15.87:6801/2970504758","cluster_addr":"172.21.15.87:6803/2970504758","heartbeat_back_addr":"172.21.15.87:6807/2970504758","heartbeat_front_addr":"172.21.15.87:6805/2970504758","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6809","nonce":3642701185}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6811","nonce":3642701185}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6815","nonce":3642701185}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6813","nonce":3642701185}]},"public_addr":"172.21.15.87:6809/3642701185","cluster_addr":"172.21.15.87:6811/3642701185","heartbeat_back_addr":"172.21.15.87:6815/3642701185","heartbeat_front_addr":"172.21.15.87:6813/3642701185","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T11:11:57.512 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:57 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:57.187+0000 7f9e9fcc3200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:11:57.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:57 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/694035377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:11:57.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:57 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/694035377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:11:57.798 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:57 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/694035377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:11:57.799 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:57.487+0000 7fa19ddff200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:11:57.799 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:57.585+0000 7fa19ddff200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:11:58.056 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:57 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:57.778+0000 7f9e9fcc3200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:11:58.056 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:57 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:57.880+0000 7f9e9fcc3200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:11:58.077 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-12T11:11:58.078 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T11:11:58.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:57.797+0000 7fa19ddff200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:11:58.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: [12/Apr/2024:11:11:57] ENGINE Bus STARTING 2024-04-12T11:11:58.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: CherryPy Checker: 2024-04-12T11:11:58.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: The Application mounted at '' has an empty config. 2024-04-12T11:11:58.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: 2024-04-12T11:11:58.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: [12/Apr/2024:11:11:57] ENGINE Serving on http://:::9283 2024-04-12T11:11:58.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:11:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[34792]: [12/Apr/2024:11:11:57] ENGINE Bus STARTED 2024-04-12T11:11:58.310 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:58.105+0000 7f9e9fcc3200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:11:58.310 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:11:58] ENGINE Bus STARTING 2024-04-12T11:11:58.333 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:11:58.563 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: CherryPy Checker: 2024-04-12T11:11:58.563 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: The Application mounted at '' has an empty config. 2024-04-12T11:11:58.563 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:11:58.563 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:11:58] ENGINE Serving on http://:::9283 2024-04-12T11:11:58.563 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:11:58] ENGINE Bus STARTED 2024-04-12T11:11:58.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: Standby manager daemon x restarted 2024-04-12T11:11:58.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: Standby manager daemon x started 2024-04-12T11:11:58.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:11:58.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:11:58.815 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: Standby manager daemon x restarted 2024-04-12T11:11:58.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: Standby manager daemon x started 2024-04-12T11:11:58.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:11:58.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:11:58.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:11:58.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:11:58.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: Active manager daemon y restarted 2024-04-12T11:11:58.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: Activating manager daemon y 2024-04-12T11:11:58.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: osdmap e63: 8 total, 8 up, 8 in 2024-04-12T11:11:58.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: mgrmap e17: y(active, starting, since 0.00756386s), standbys: x 2024-04-12T11:11:58.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:11:58.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:11:58.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:58.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:11:58.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:11:58.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:11:58.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: Manager daemon y is now available 2024-04-12T11:11:58.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:58.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:11:58.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:11:58.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:11:58.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:11:58.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:11:58.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:11:58.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:11:58.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: Active manager daemon y restarted 2024-04-12T11:11:58.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: Activating manager daemon y 2024-04-12T11:11:58.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: osdmap e63: 8 total, 8 up, 8 in 2024-04-12T11:11:58.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: mgrmap e17: y(active, starting, since 0.00756386s), standbys: x 2024-04-12T11:11:58.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:11:58.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:11:58.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:58.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:11:58.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:11:58.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:11:58.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: Manager daemon y is now available 2024-04-12T11:11:58.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:58.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:11:58.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:11:58.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:11:58.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:11:58.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:11:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: Standby manager daemon x restarted 2024-04-12T11:11:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: Standby manager daemon x started 2024-04-12T11:11:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:11:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:11:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:11:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1605740749' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:11:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: Active manager daemon y restarted 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: Activating manager daemon y 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: osdmap e63: 8 total, 8 up, 8 in 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: mgrmap e17: y(active, starting, since 0.00756386s), standbys: x 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:11:58.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: Manager daemon y is now available 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:11:58.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:11:58.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:11:58.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:11:58.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:11:59.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:59 smithi029 ceph-mon[39999]: [12/Apr/2024:11:11:58] ENGINE Bus STARTING 2024-04-12T11:11:59.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:59 smithi029 ceph-mon[39999]: [12/Apr/2024:11:11:58] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:11:59.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:59 smithi029 ceph-mon[39999]: [12/Apr/2024:11:11:58] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:11:59.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:59 smithi029 ceph-mon[39999]: [12/Apr/2024:11:11:58] ENGINE Bus STARTED 2024-04-12T11:11:59.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:11:59 smithi029 ceph-mon[39999]: mgrmap e18: y(active, since 1.01206s), standbys: x 2024-04-12T11:11:59.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:59 smithi029 ceph-mon[29349]: [12/Apr/2024:11:11:58] ENGINE Bus STARTING 2024-04-12T11:11:59.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:59 smithi029 ceph-mon[29349]: [12/Apr/2024:11:11:58] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:11:59.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:59 smithi029 ceph-mon[29349]: [12/Apr/2024:11:11:58] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:11:59.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:59 smithi029 ceph-mon[29349]: [12/Apr/2024:11:11:58] ENGINE Bus STARTED 2024-04-12T11:11:59.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:11:59 smithi029 ceph-mon[29349]: mgrmap e18: y(active, since 1.01206s), standbys: x 2024-04-12T11:11:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:59 smithi087 ceph-mon[32917]: [12/Apr/2024:11:11:58] ENGINE Bus STARTING 2024-04-12T11:11:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:59 smithi087 ceph-mon[32917]: [12/Apr/2024:11:11:58] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:11:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:59 smithi087 ceph-mon[32917]: [12/Apr/2024:11:11:58] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:11:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:59 smithi087 ceph-mon[32917]: [12/Apr/2024:11:11:58] ENGINE Bus STARTED 2024-04-12T11:11:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:11:59 smithi087 ceph-mon[32917]: mgrmap e18: y(active, since 1.01206s), standbys: x 2024-04-12T11:12:00.252 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:12:00.252 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":63,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T11:11:58.108291+0000","last_up_change":"2024-04-12T11:11:19.950253+0000","last_in_change":"2024-04-12T11:11:02.730106+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T11:11:48.103219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6803","nonce":2299659597}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6805","nonce":2299659597}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6809","nonce":2299659597}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6807","nonce":2299659597}]},"public_addr":"172.21.15.29:6803/2299659597","cluster_addr":"172.21.15.29:6805/2299659597","heartbeat_back_addr":"172.21.15.29:6809/2299659597","heartbeat_front_addr":"172.21.15.29:6807/2299659597","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6811","nonce":3193349615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6813","nonce":3193349615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6817","nonce":3193349615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6815","nonce":3193349615}]},"public_addr":"172.21.15.29:6811/3193349615","cluster_addr":"172.21.15.29:6813/3193349615","heartbeat_back_addr":"172.21.15.29:6817/3193349615","heartbeat_front_addr":"172.21.15.29:6815/3193349615","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6819","nonce":1946236732}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6821","nonce":1946236732}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6825","nonce":1946236732}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6823","nonce":1946236732}]},"public_addr":"172.21.15.29:6819/1946236732","cluster_addr":"172.21.15.29:6821/1946236732","heartbeat_back_addr":"172.21.15.29:6825/1946236732","heartbeat_front_addr":"172.21.15.29:6823/1946236732","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6827","nonce":4104584972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6829","nonce":4104584972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6833","nonce":4104584972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6831","nonce":4104584972}]},"public_addr":"172.21.15.29:6827/4104584972","cluster_addr":"172.21.15.29:6829/4104584972","heartbeat_back_addr":"172.21.15.29:6833/4104584972","heartbeat_front_addr":"172.21.15.29:6831/4104584972","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6801","nonce":2970504758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6803","nonce":2970504758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6807","nonce":2970504758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6805","nonce":2970504758}]},"public_addr":"172.21.15.87:6801/2970504758","cluster_addr":"172.21.15.87:6803/2970504758","heartbeat_back_addr":"172.21.15.87:6807/2970504758","heartbeat_front_addr":"172.21.15.87:6805/2970504758","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6809","nonce":3642701185}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6811","nonce":3642701185}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6815","nonce":3642701185}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6813","nonce":3642701185}]},"public_addr":"172.21.15.87:6809/3642701185","cluster_addr":"172.21.15.87:6811/3642701185","heartbeat_back_addr":"172.21.15.87:6815/3642701185","heartbeat_front_addr":"172.21.15.87:6813/3642701185","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2946468060":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6800/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/364858296":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/3254157714":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/659258741":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T11:12:00.963 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[39999]: pgmap v3: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:00.963 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[39999]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:12:00.963 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[39999]: Cluster is now healthy 2024-04-12T11:12:00.963 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1332838670' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:12:00.963 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:00.963 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:00.963 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[29349]: pgmap v3: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:00.964 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[29349]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:12:00.964 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[29349]: Cluster is now healthy 2024-04-12T11:12:00.964 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1332838670' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:12:00.964 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:00.964 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:00 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:00 smithi087 ceph-mon[32917]: pgmap v3: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:00 smithi087 ceph-mon[32917]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:12:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:00 smithi087 ceph-mon[32917]: Cluster is now healthy 2024-04-12T11:12:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:00 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1332838670' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:12:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:00 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:00 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:01.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-12T11:12:01.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-12T11:12:01.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-12T11:12:01.349 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-12T11:12:01.349 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-12T11:12:01.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-12T11:12:01.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-12T11:12:01.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-12T11:12:01.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[29349]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:01.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[29349]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-12T11:12:01.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:01.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:01.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:01.740 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:01.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[39999]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:01.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[39999]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-12T11:12:01.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:01.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:01.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:01 smithi087 ceph-mon[32917]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:01 smithi087 ceph-mon[32917]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-12T11:12:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:12:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:02.528 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:02.528 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:02.528 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:02.529 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:02.529 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:02.532 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:02.915 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:02 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi087 to 4016M 2024-04-12T11:12:02.915 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:02 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:02.915 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:02 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:02.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:02 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi087 to 4016M 2024-04-12T11:12:02.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:02 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:02.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:02 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:02 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi087 to 4016M 2024-04-12T11:12:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:02 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:02 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:03.186 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:12:03] "GET /metrics HTTP/1.1" 200 26009 "" "Prometheus/2.43.0" 2024-04-12T11:12:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[39999]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[39999]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-12T11:12:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:12:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:12:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[39999]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:12:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:12:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[29349]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[29349]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-12T11:12:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:12:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:12:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[29349]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:12:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:03 smithi029 ceph-mon[29349]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:12:03.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:03 smithi087 ceph-mon[32917]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:03.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:03 smithi087 ceph-mon[32917]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-12T11:12:03.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:03.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:12:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:12:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:03 smithi087 ceph-mon[32917]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:12:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:03 smithi087 ceph-mon[32917]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:12:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:12:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:12:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:12:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:12:04.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:12:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:12:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:04 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:12:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:12:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:12:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:12:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:12:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:12:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:04 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:05.754 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:05 smithi029 ceph-mon[39999]: pgmap v6: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:05.754 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:05 smithi029 ceph-mon[39999]: Deploying daemon alertmanager.a on smithi029 2024-04-12T11:12:05.754 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:05 smithi029 ceph-mon[29349]: pgmap v6: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:05.754 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:05 smithi029 ceph-mon[29349]: Deploying daemon alertmanager.a on smithi029 2024-04-12T11:12:05.864 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:05 smithi087 ceph-mon[32917]: pgmap v6: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:05 smithi087 ceph-mon[32917]: Deploying daemon alertmanager.a on smithi029 2024-04-12T11:12:07.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:07 smithi029 ceph-mon[29349]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-12T11:12:07.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:07 smithi029 ceph-mon[39999]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-12T11:12:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:07 smithi087 ceph-mon[32917]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-12T11:12:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:09 smithi087 ceph-mon[32917]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-12T11:12:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:09 smithi029 ceph-mon[29349]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-12T11:12:10.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:09 smithi029 ceph-mon[39999]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-12T11:12:11.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:11 smithi029 ceph-mon[39999]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-12T11:12:11.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:11 smithi029 ceph-mon[29349]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-12T11:12:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:11 smithi087 ceph-mon[32917]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-12T11:12:13.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:12:13] "GET /metrics HTTP/1.1" 200 26009 "" "Prometheus/2.43.0" 2024-04-12T11:12:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:13 smithi087 ceph-mon[32917]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:12:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:13 smithi029 ceph-mon[29349]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:12:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:13 smithi029 ceph-mon[39999]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:12:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:14.754 INFO:teuthology.orchestra.run.smithi029.stdout:34359738413 2024-04-12T11:12:14.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-12T11:12:15.324 INFO:teuthology.orchestra.run.smithi029.stdout:77309411363 2024-04-12T11:12:15.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-12T11:12:15.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:15 smithi029 ceph-mon[39999]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:12:15.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:15 smithi029 ceph-mon[29349]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:12:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:15 smithi087 ceph-mon[32917]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:12:17.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:17 smithi029 ceph-mon[39999]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:12:17.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:17 smithi029 ceph-mon[29349]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:12:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:17 smithi087 ceph-mon[32917]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:12:18.975 INFO:teuthology.orchestra.run.smithi029.stdout:107374182432 2024-04-12T11:12:18.975 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-12T11:12:19.481 INFO:teuthology.orchestra.run.smithi029.stdout:163208757270 2024-04-12T11:12:19.482 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-12T11:12:19.727 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:19 smithi029 ceph-mon[39999]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:19.728 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:19 smithi029 ceph-mon[29349]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:19.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:19 smithi087 ceph-mon[32917]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:20.133 INFO:teuthology.orchestra.run.smithi029.stdout:55834574889 2024-04-12T11:12:20.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-12T11:12:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:21 smithi029 ceph-mon[29349]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:21 smithi029 ceph-mon[39999]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:21.831 INFO:teuthology.orchestra.run.smithi029.stdout:128849018908 2024-04-12T11:12:21.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-12T11:12:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:21 smithi087 ceph-mon[32917]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:22.297 INFO:teuthology.orchestra.run.smithi029.stdout:197568495635 2024-04-12T11:12:22.297 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-12T11:12:23.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:12:23] "GET /metrics HTTP/1.1" 200 28558 "" "Prometheus/2.43.0" 2024-04-12T11:12:23.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:23 smithi087 ceph-mon[32917]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:24.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:23 smithi029 ceph-mon[29349]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:24.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:23 smithi029 ceph-mon[39999]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:24.268 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:24.269 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:24.837 INFO:teuthology.orchestra.run.smithi029.stdout:231928233998 2024-04-12T11:12:24.838 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-12T11:12:24.861 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:24.861 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:25.523 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:25 smithi029 ceph-mon[29349]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:25 smithi029 ceph-mon[39999]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:25 smithi087 ceph-mon[32917]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:27.510 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:27.513 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:27 smithi087 ceph-mon[32917]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:28.004 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:27 smithi029 ceph-mon[39999]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:28.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:27 smithi029 ceph-mon[29349]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:28.652 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:28 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:28.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:28 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:28 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:29.518 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:12:29 smithi029 systemd[1]: Started Ceph alertmanager.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:12:29.906 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[29349]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:29.907 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.907 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.907 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.907 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.907 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.907 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.907 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T11:12:29.907 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[39999]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:29.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T11:12:29.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.909 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:12:29] ENGINE Bus STOPPING 2024-04-12T11:12:29.909 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:12:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T11:12:29.909 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:12:29] ENGINE Bus STOPPED 2024-04-12T11:12:29.909 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:12:29] ENGINE Bus STARTING 2024-04-12T11:12:29.909 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:12:29] ENGINE Serving on http://:::9283 2024-04-12T11:12:29.909 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:12:29] ENGINE Bus STARTED 2024-04-12T11:12:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:29 smithi087 ceph-mon[32917]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T11:12:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:30 smithi087 ceph-mon[32917]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-12T11:12:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:30 smithi087 ceph-mon[32917]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T11:12:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:30 smithi087 ceph-mon[32917]: Deploying daemon grafana.a on smithi087 2024-04-12T11:12:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:30 smithi029 ceph-mon[29349]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-12T11:12:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:30 smithi029 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T11:12:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:30 smithi029 ceph-mon[29349]: Deploying daemon grafana.a on smithi087 2024-04-12T11:12:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:30 smithi029 ceph-mon[39999]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-12T11:12:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:30 smithi029 ceph-mon[39999]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T11:12:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:30 smithi029 ceph-mon[39999]: Deploying daemon grafana.a on smithi087 2024-04-12T11:12:31.427 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:12:31 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[70909]: ts=2024-04-12T11:12:31.259Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000616015s 2024-04-12T11:12:31.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:31 smithi029 ceph-mon[29349]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:31.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:31 smithi029 ceph-mon[39999]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:31 smithi087 ceph-mon[32917]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:33.140 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:12:33] "GET /metrics HTTP/1.1" 200 28558 "" "Prometheus/2.43.0" 2024-04-12T11:12:33.207 INFO:teuthology.orchestra.run.smithi029.stdout:34359738418 2024-04-12T11:12:33.366 INFO:teuthology.orchestra.run.smithi029.stdout:231928234000 2024-04-12T11:12:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:33 smithi029 ceph-mon[29349]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:33 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:33 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2598699258' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T11:12:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:33 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2021945587' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T11:12:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:33 smithi029 ceph-mon[39999]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:33 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:33 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2598699258' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T11:12:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:33 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2021945587' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T11:12:33.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:33 smithi087 ceph-mon[32917]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:33 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:33 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2598699258' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T11:12:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:33 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2021945587' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T11:12:34.131 INFO:teuthology.orchestra.run.smithi029.stdout:77309411367 2024-04-12T11:12:34.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:34 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2297526840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T11:12:34.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:34 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2297526840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T11:12:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:34 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2297526840' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T11:12:35.358 INFO:teuthology.orchestra.run.smithi029.stdout:197568495638 2024-04-12T11:12:35.648 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233998 got 231928234000 for osd.7 2024-04-12T11:12:35.648 DEBUG:teuthology.parallel:result is None 2024-04-12T11:12:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:35 smithi087 ceph-mon[32917]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:35 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3001199408' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T11:12:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:35 smithi029 ceph-mon[29349]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:35 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3001199408' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T11:12:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:35 smithi029 ceph-mon[39999]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:35 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3001199408' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T11:12:36.069 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411363 got 77309411367 for osd.2 2024-04-12T11:12:36.069 DEBUG:teuthology.parallel:result is None 2024-04-12T11:12:36.283 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:37.958 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:37 smithi029 ceph-mon[29349]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:37.958 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:37 smithi029 ceph-mon[39999]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:37 smithi087 ceph-mon[32917]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:39.205 INFO:teuthology.orchestra.run.smithi029.stdout:55834574893 2024-04-12T11:12:39.468 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:12:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[70909]: ts=2024-04-12T11:12:39.262Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004340583s 2024-04-12T11:12:39.488 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738413 got 34359738418 for osd.0 2024-04-12T11:12:39.488 DEBUG:teuthology.parallel:result is None 2024-04-12T11:12:39.722 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:39 smithi029 ceph-mon[39999]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:39.722 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:39 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2041944874' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T11:12:39.722 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:39 smithi029 ceph-mon[29349]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:39.722 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:39 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2041944874' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T11:12:39.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:39 smithi087 ceph-mon[32917]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:39.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:39 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2041944874' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T11:12:40.425 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495635 got 197568495638 for osd.6 2024-04-12T11:12:40.425 DEBUG:teuthology.parallel:result is None 2024-04-12T11:12:41.336 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574893 for osd.1 2024-04-12T11:12:41.336 DEBUG:teuthology.parallel:result is None 2024-04-12T11:12:41.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:41 smithi029 ceph-mon[39999]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:41.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:41 smithi029 ceph-mon[29349]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:41.936 INFO:teuthology.orchestra.run.smithi029.stdout:163208757275 2024-04-12T11:12:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:41 smithi087 ceph-mon[32917]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:42.152 INFO:teuthology.orchestra.run.smithi029.stdout:107374182437 2024-04-12T11:12:42.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:42 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1381713450' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T11:12:42.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:42 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2959080625' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T11:12:42.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:42 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1381713450' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T11:12:42.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:42 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2959080625' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T11:12:42.788 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182432 got 107374182437 for osd.3 2024-04-12T11:12:42.788 DEBUG:teuthology.parallel:result is None 2024-04-12T11:12:42.920 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:42 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1381713450' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T11:12:42.920 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:42 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2959080625' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T11:12:43.370 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:43 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:12:43] "GET /metrics HTTP/1.1" 200 28558 "" "Prometheus/2.43.0" 2024-04-12T11:12:43.516 INFO:teuthology.orchestra.run.smithi029.stdout:128849018912 2024-04-12T11:12:43.573 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757270 got 163208757275 for osd.5 2024-04-12T11:12:43.573 DEBUG:teuthology.parallel:result is None 2024-04-12T11:12:43.900 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:43 smithi087 ceph-mon[32917]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:43.900 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:43 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:43.900 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:43 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3762425860' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T11:12:44.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:43 smithi029 ceph-mon[39999]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:44.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:43 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:44.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:43 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3762425860' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T11:12:44.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:43 smithi029 ceph-mon[29349]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:44.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:43 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:44.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:43 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3762425860' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T11:12:44.176 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018912 for osd.4 2024-04-12T11:12:44.177 DEBUG:teuthology.parallel:result is None 2024-04-12T11:12:44.177 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-12T11:12:44.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:12:44.388 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:45.466 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 systemd[1]: Starting Ceph grafana.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:12:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:45 smithi029 ceph-mon[29349]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:45 smithi029 ceph-mon[39999]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:45.777 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-mon[32917]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:45.778 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 podman[62861]: 2024-04-12 11:12:45.509052281 +0000 UTC m=+0.101769180 container create d4ba4d587fdb414b40bb283109da73d4b744eab8f3bb18236cb4ec45dad4f472 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, vcs-type=git, io.openshift.expose-services=, vendor=Red Hat, Inc., distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, description=Ceph Grafana Container) 2024-04-12T11:12:45.778 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 podman[62861]: 2024-04-12 11:12:45.424554902 +0000 UTC m=+0.017271765 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-12T11:12:45.778 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 podman[62861]: 2024-04-12 11:12:45.777960079 +0000 UTC m=+0.370676932 container init d4ba4d587fdb414b40bb283109da73d4b744eab8f3bb18236cb4ec45dad4f472 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a, vendor=Red Hat, Inc., name=ubi8, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, io.openshift.expose-services=, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5) 2024-04-12T11:12:46.030 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 podman[62861]: 2024-04-12 11:12:45.78078953 +0000 UTC m=+0.373506385 container start d4ba4d587fdb414b40bb283109da73d4b744eab8f3bb18236cb4ec45dad4f472 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, vcs-type=git, name=ubi8, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, version=8.5, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, release=239.1651231664) 2024-04-12T11:12:46.030 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 bash[62861]: d4ba4d587fdb414b40bb283109da73d4b744eab8f3bb18236cb4ec45dad4f472 2024-04-12T11:12:46.030 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 systemd[1]: Started Ceph grafana.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:12:46.030 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929683126Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-12T11:12:46.030 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929865402Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-12T11:12:46.030 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929884267Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-12T11:12:46.030 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929891377Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-12T11:12:46.030 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929897786Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929908261Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929914143Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929920073Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929926267Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929932082Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929937765Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929943159Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929948959Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=settings t=2024-04-12T11:12:45.929954685Z level=info msg="App mode production" 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=sqlstore t=2024-04-12T11:12:45.929999068Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=sqlstore t=2024-04-12T11:12:45.930014788Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.93082668Z level=info msg="Starting DB migrations" 2024-04-12T11:12:46.031 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.930897982Z level=info msg="Executing migration" id="create migration_log table" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.931613696Z level=info msg="Executing migration" id="create user table" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.932121972Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.932692171Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.933137161Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.933760207Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.934177249Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.934915784Z level=info msg="Executing migration" id="create user table v2" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.935380089Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.93580227Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.936688362Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.937496013Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.9383536Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-12T11:12:46.032 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.939250806Z level=info msg="Executing migration" id="Update user table charset" 2024-04-12T11:12:46.033 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.940125887Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-12T11:12:46.033 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.940993313Z level=info msg="Executing migration" id="Add missing user data" 2024-04-12T11:12:46.033 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.941898803Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-12T11:12:46.033 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.942777939Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-12T11:12:46.033 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.943637606Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.944511157Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.945655409Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.946548541Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.947412674Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.948273052Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.949104534Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.949988655Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.950858309Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.951711836Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.952708986Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.953457492Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-12T11:12:46.034 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.953874063Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.956776183Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.959493459Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.96248955Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.965413432Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.968201724Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.968655554Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.971391205Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.971770797Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.974264836Z level=info msg="Executing migration" id="create star table" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.974660683Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.977424327Z level=info msg="Executing migration" id="create org table v1" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.980198064Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.982952385Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-12T11:12:46.035 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.985977579Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.988685937Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.991487107Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.994489889Z level=info msg="Executing migration" id="Update org table charset" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:45.997320266Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.000270714Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.002873359Z level=info msg="Executing migration" id="create dashboard table" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.005824252Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.008304341Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.011053531Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.013682638Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.016190693Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.018788116Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.021515946Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-12T11:12:46.036 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.024335497Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-12T11:12:46.037 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.027062177Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-12T11:12:46.037 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.029644642Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-12T11:12:46.220 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:12:46.222 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:12:46.282 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.032301274Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-12T11:12:46.282 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.034906188Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-12T11:12:46.282 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.03757596Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-12T11:12:46.282 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.040310996Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-12T11:12:46.282 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.042891297Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-12T11:12:46.282 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.04367305Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.04448626Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.045381143Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.046392405Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.047183185Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.048099635Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.04886763Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.049857904Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.05067368Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.051444142Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.052413815Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-12T11:12:46.283 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.053150693Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.054151815Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.054891155Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.055793956Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.05630948Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.056694857Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.05761502Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.0585623Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.059317073Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.060330242Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.061089515Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.061968822Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-12T11:12:46.284 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.062855639Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-12T11:12:46.285 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.06382677Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-12T11:12:46.285 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.064613997Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-12T11:12:46.285 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.06553976Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-12T11:12:46.285 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.066328026Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-12T11:12:46.285 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.067357464Z level=info msg="Executing migration" id="create data_source table" 2024-04-12T11:12:46.285 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.068048234Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-12T11:12:46.285 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.069053507Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-12T11:12:46.285 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.069821834Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-12T11:12:46.285 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.070673455Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-12T11:12:46.287 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.071554549Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-12T11:12:46.287 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.07275135Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-12T11:12:46.287 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.075820491Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-12T11:12:46.287 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.078665702Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-12T11:12:46.287 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.081626425Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.084597352Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.087524618Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.090518032Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.093318383Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.09627125Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.099158006Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.102146118Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.105122074Z level=info msg="Executing migration" id="Add uid column" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.107963729Z level=info msg="Executing migration" id="Update uid value" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.11088514Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.113621662Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.116542955Z level=info msg="Executing migration" id="create api_key table" 2024-04-12T11:12:46.288 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.119441522Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.122438778Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.125232697Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.128001084Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.130681679Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.133458048Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.13601923Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.137072778Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.139848117Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.142605112Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.145321305Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-12T11:12:46.289 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.145781515Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.148404648Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.148887417Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.151053677Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.151643434Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.15378207Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.15412311Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.154719946Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.155315989Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.155792923Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.156153186Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-12T11:12:46.290 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.156584921Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.158881428Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.159357721Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.159805956Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.160095681Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.160414433Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.161017027Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.161591875Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.161918327Z level=info msg="Executing migration" id="create quota table v1" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.162321885Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.162771949Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-12T11:12:46.291 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.163059638Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-12T11:12:46.292 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.163489843Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-12T11:12:46.292 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.163923644Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-12T11:12:46.292 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.164520722Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-12T11:12:46.292 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.164835502Z level=info msg="Executing migration" id="create session table" 2024-04-12T11:12:46.292 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.165272709Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-12T11:12:46.292 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.165624934Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-12T11:12:46.292 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.1659297Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-12T11:12:46.292 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.16634984Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-12T11:12:46.292 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.166751598Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-12T11:12:46.292 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.167028736Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.167312205Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.16763297Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.167934855Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.168341946Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.168660199Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.169263259Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.16959766Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.170197177Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.170827561Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.171118568Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-12T11:12:46.293 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.171577239Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.172042414Z level=info msg="Executing migration" id="create alert table v1" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.174911775Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.177561338Z level=info msg="Executing migration" id="add index alert state" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.178010777Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.178452592Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.178848793Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.17931859Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.179738019Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.181111784Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.181525463Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.181969819Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-12T11:12:46.294 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.182278479Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.182660401Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.183039498Z level=info msg="Executing migration" id="Add column is_default" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.183727167Z level=info msg="Executing migration" id="Add column frequency" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.184368001Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.185028856Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.18567526Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.186112803Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.186431587Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.186744102Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.187169671Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.187634921Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.188007701Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.18843188Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-12T11:12:46.295 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.188861717Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.189552618Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.190198538Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.190553648Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.190982369Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.191414598Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.192079735Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.192408851Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.192846818Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.193278901Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.193613143Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-12T11:12:46.296 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.194291855Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.196882914Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.197338905Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.197778492Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.198280259Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.201050499Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.201471493Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.202152625Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.203163956Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.2040016Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.204824272Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-12T11:12:46.297 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.205291751Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.206086667Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.207683885Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.208474351Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.209049716Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.209743756Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.210536503Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.211313159Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.21199063Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.212867857Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.213798801Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-12T11:12:46.298 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.214437522Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-12T11:12:46.299 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.214778114Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-12T11:12:46.299 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.21562098Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-12T11:12:46.299 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.216526612Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-12T11:12:46.299 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.217378517Z level=info msg="Executing migration" id="Move region to single row" 2024-04-12T11:12:46.299 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.217773481Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-12T11:12:46.299 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.218616776Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-12T11:12:46.299 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.219048545Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-12T11:12:46.299 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.219544947Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-12T11:12:46.299 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.220446634Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-12T11:12:46.300 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.22099994Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-12T11:12:46.300 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.221848785Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-12T11:12:46.300 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.222731312Z level=info msg="Executing migration" id="create test_data table" 2024-04-12T11:12:46.300 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.223129473Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-12T11:12:46.300 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.22358102Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-12T11:12:46.300 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.224603858Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-12T11:12:46.300 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.22530799Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-12T11:12:46.300 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.22618369Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-12T11:12:46.300 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.226545064Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-12T11:12:46.300 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.227473036Z level=info msg="Executing migration" id="create team table" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.228126388Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.228594677Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.231552845Z level=info msg="Executing migration" id="create team member table" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.234326227Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.237265115Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.240032536Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.242718508Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.243589659Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.246332847Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.249355495Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.252021717Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-12T11:12:46.301 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.252536787Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.255447118Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.256070188Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.259045924Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.259503707Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.262323823Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.265074796Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.267883626Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.270624806Z level=info msg="Executing migration" id="create tag table" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.273423784Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.276085118Z level=info msg="Executing migration" id="create login attempt table" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.278866034Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.281785341Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-12T11:12:46.302 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.284451193Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-12T11:12:46.532 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.287467748Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-12T11:12:46.532 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.290211791Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-12T11:12:46.532 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.291006137Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-12T11:12:46.532 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.291864285Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-12T11:12:46.532 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.292746316Z level=info msg="Executing migration" id="create user auth table" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.293622235Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.294489681Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.295331748Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.296443438Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.297544204Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.298693303Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.299826163Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.300678409Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.301523837Z level=info msg="Executing migration" id="create server_lock table" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.30241003Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.303285107Z level=info msg="Executing migration" id="create user auth token table" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.304115727Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-12T11:12:46.533 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.30499265Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-12T11:12:46.534 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.305966833Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-12T11:12:46.535 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.306727988Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-12T11:12:46.535 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.307642105Z level=info msg="Executing migration" id="create cache_data table" 2024-04-12T11:12:46.535 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.308652535Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-12T11:12:46.535 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.30940164Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-12T11:12:46.535 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.31030816Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-12T11:12:46.535 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.311314208Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-12T11:12:46.535 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.312014099Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-12T11:12:46.535 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.312977062Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-12T11:12:46.535 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.313501458Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-12T11:12:46.535 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.313965328Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.314987492Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.315737923Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.316540368Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.317533727Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.318308533Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.319310306Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.320189366Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.321122746Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.324143765Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.326859165Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.329907847Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.332961729Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.33582969Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.338625083Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-12T11:12:46.536 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.341459785Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.34435728Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.34743423Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.350287475Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.353223977Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.356029674Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.359423194Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.362815142Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.365700314Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.368444944Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.371325199Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.374002146Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.37665106Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-12T11:12:46.537 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.379595663Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.382265303Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.38500926Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.387669095Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.390440405Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.393089092Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.395742042Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.39846766Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.40108068Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.40384624Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.406446862Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.409124119Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.41160803Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-12T11:12:46.538 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.414166461Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.416915774Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.419578834Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.420436265Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.421157402Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.422057157Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.422932484Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.423812602Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.424738976Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.425602385Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.426541474Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.427300368Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.428190353Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.429062632Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-12T11:12:46.539 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.429993311Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.430827464Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.431747204Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.432600699Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.433526283Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.434527827Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.435537263Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.436330167Z level=info msg="Executing migration" id="create alert_image table" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.437176634Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.438175883Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.438958869Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.439812499Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.440689714Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-12T11:12:46.540 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.44077519Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.441527581Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.442441586Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.443312818Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.443440072Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.443551861Z level=info msg="alerts found to migrate" alerts=0 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.444203867Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.445154397Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.44604203Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.446796795Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.447680234Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.448573656Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.449396874Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.450314055Z level=info msg="Executing migration" id="create data_keys table" 2024-04-12T11:12:46.541 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.453426348Z level=info msg="Executing migration" id="create secrets table" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.456365935Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.460444353Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.46340811Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.466342673Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.470767858Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.47498124Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.477885959Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.480860146Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.483867616Z level=info msg="Executing migration" id="create permission table" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.486637279Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.489353757Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.492314302Z level=info msg="Executing migration" id="create role table" 2024-04-12T11:12:46.542 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.495222687Z level=info msg="Executing migration" id="add column display_name" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.498050957Z level=info msg="Executing migration" id="add column group_name" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.500899597Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.50394478Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.506766846Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.509630401Z level=info msg="Executing migration" id="create team role table" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.512366287Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.515101286Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.517973377Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.520681501Z level=info msg="Executing migration" id="create user role table" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.523332129Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.523851242Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.526616273Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-12T11:12:46.543 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.529418933Z level=info msg="Executing migration" id="create builtin role table" 2024-04-12T11:12:46.544 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.532145916Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-12T11:12:46.544 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.534755238Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-12T11:12:46.867 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":27,"stamp":"2024-04-12T11:12:46.127592+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":231028,"kb_used_data":6684,"kb_used_omap":12,"kb_used_meta":224243,"kb_avail":749730188,"statfs":{"total":767960285184,"available":767723712512,"internally_reserved":0,"allocated":6844416,"data_stored":3876320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":229625401},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001676"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-12T11:11:52.968666+0000","last_change":"2024-04-12T11:11:49.523873+0000","last_active":"2024-04-12T11:11:52.968666+0000","last_peered":"2024-04-12T11:11:52.968666+0000","last_clean":"2024-04-12T11:11:52.968666+0000","last_became_active":"2024-04-12T11:11:49.523570+0000","last_became_peered":"2024-04-12T11:11:49.523570+0000","last_unstale":"2024-04-12T11:11:52.968666+0000","last_undegraded":"2024-04-12T11:11:52.968666+0000","last_fullsized":"2024-04-12T11:11:52.968666+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:09:00.491446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":13,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:11:58.112697+0000","last_change":"2024-04-12T11:11:49.882400+0000","last_active":"2024-04-12T11:11:58.112697+0000","last_peered":"2024-04-12T11:11:58.112697+0000","last_clean":"2024-04-12T11:11:58.112697+0000","last_became_active":"2024-04-12T11:11:49.882073+0000","last_became_peered":"2024-04-12T11:11:49.882073+0000","last_unstale":"2024-04-12T11:11:58.112697+0000","last_undegraded":"2024-04-12T11:11:58.112697+0000","last_fullsized":"2024-04-12T11:11:58.112697+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:38:46.841744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-12T11:11:52.968901+0000","last_change":"2024-04-12T11:11:49.882324+0000","last_active":"2024-04-12T11:11:52.968901+0000","last_peered":"2024-04-12T11:11:52.968901+0000","last_clean":"2024-04-12T11:11:52.968901+0000","last_became_active":"2024-04-12T11:11:49.881975+0000","last_became_peered":"2024-04-12T11:11:49.881975+0000","last_unstale":"2024-04-12T11:11:52.968901+0000","last_undegraded":"2024-04-12T11:11:52.968901+0000","last_fullsized":"2024-04-12T11:11:52.968901+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:10.140250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-12T11:11:52.968858+0000","last_change":"2024-04-12T11:11:49.524657+0000","last_active":"2024-04-12T11:11:52.968858+0000","last_peered":"2024-04-12T11:11:52.968858+0000","last_clean":"2024-04-12T11:11:52.968858+0000","last_became_active":"2024-04-12T11:11:49.524379+0000","last_became_peered":"2024-04-12T11:11:49.524379+0000","last_unstale":"2024-04-12T11:11:52.968858+0000","last_undegraded":"2024-04-12T11:11:52.968858+0000","last_fullsized":"2024-04-12T11:11:52.968858+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:30:52.444470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":258,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:12:00.186224+0000","last_change":"2024-04-12T11:11:24.390399+0000","last_active":"2024-04-12T11:12:00.186224+0000","last_peered":"2024-04-12T11:12:00.186224+0000","last_clean":"2024-04-12T11:12:00.186224+0000","last_became_active":"2024-04-12T11:11:24.389943+0000","last_became_peered":"2024-04-12T11:11:24.389943+0000","last_unstale":"2024-04-12T11:12:00.186224+0000","last_undegraded":"2024-04-12T11:12:00.186224+0000","last_fullsized":"2024-04-12T11:12:00.186224+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:06:44.231202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:11:58.113204+0000","last_change":"2024-04-12T11:11:49.524628+0000","last_active":"2024-04-12T11:11:58.113204+0000","last_peered":"2024-04-12T11:11:58.113204+0000","last_clean":"2024-04-12T11:11:58.113204+0000","last_became_active":"2024-04-12T11:11:49.524226+0000","last_became_peered":"2024-04-12T11:11:49.524226+0000","last_unstale":"2024-04-12T11:11:58.113204+0000","last_undegraded":"2024-04-12T11:11:58.113204+0000","last_fullsized":"2024-04-12T11:11:58.113204+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:43:26.148427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":13,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:11:58.111945+0000","last_change":"2024-04-12T11:11:49.523432+0000","last_active":"2024-04-12T11:11:58.111945+0000","last_peered":"2024-04-12T11:11:58.111945+0000","last_clean":"2024-04-12T11:11:58.111945+0000","last_became_active":"2024-04-12T11:11:49.523059+0000","last_became_peered":"2024-04-12T11:11:49.523059+0000","last_unstale":"2024-04-12T11:11:58.111945+0000","last_undegraded":"2024-04-12T11:11:58.111945+0000","last_fullsized":"2024-04-12T11:11:58.111945+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:55:25.692863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:11:58.113179+0000","last_change":"2024-04-12T11:11:49.523947+0000","last_active":"2024-04-12T11:11:58.113179+0000","last_peered":"2024-04-12T11:11:58.113179+0000","last_clean":"2024-04-12T11:11:58.113179+0000","last_became_active":"2024-04-12T11:11:49.523650+0000","last_became_peered":"2024-04-12T11:11:49.523650+0000","last_unstale":"2024-04-12T11:11:58.113179+0000","last_undegraded":"2024-04-12T11:11:58.113179+0000","last_fullsized":"2024-04-12T11:11:58.113179+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:05:22.528981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":13,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:11:58.112622+0000","last_change":"2024-04-12T11:11:49.524259+0000","last_active":"2024-04-12T11:11:58.112622+0000","last_peered":"2024-04-12T11:11:58.112622+0000","last_clean":"2024-04-12T11:11:58.112622+0000","last_became_active":"2024-04-12T11:11:49.523641+0000","last_became_peered":"2024-04-12T11:11:49.523641+0000","last_unstale":"2024-04-12T11:11:58.112622+0000","last_undegraded":"2024-04-12T11:11:58.112622+0000","last_fullsized":"2024-04-12T11:11:58.112622+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:28:22.480630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28216,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716936,"statfs":{"total":95995035648,"available":95966142464,"internally_reserved":0,"allocated":1224704,"data_stored":853341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.4780000000000002,"5min":2.4780000000000002,"15min":2.4780000000000002},"last":0.754}]},{"osd":1,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":0.70399999999999996}]},{"osd":2,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.111,"15min":1.111},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":15.228999999999999,"5min":15.228999999999999,"15min":15.228999999999999},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.157,"5min":1.157,"15min":1.157},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":15.249000000000001,"5min":15.249000000000001,"15min":15.249000000000001},"last":0.71699999999999997}]},{"osd":3,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.1539999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.66700000000000004}]},{"osd":4,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.56299999999999994}]},{"osd":5,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.59499999999999997}]},{"osd":6,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.76800000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495640,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28208,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716944,"statfs":{"total":95995035648,"available":95966150656,"internally_reserved":0,"allocated":1220608,"data_stored":853118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.30399999999999999}]},{"osd":1,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.31900000000000001}]},{"osd":2,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.35399999999999998}]},{"osd":3,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.40300000000000002}]},{"osd":4,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.44},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.28799999999999998}]},{"osd":5,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.45000000000000001}]},{"osd":7,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.41799999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27640,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717512,"statfs":{"total":95995035648,"available":95966732288,"internally_reserved":0,"allocated":634880,"data_stored":263304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.35199999999999998,"15min":0.312},"min":{"1min":0.245,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.5710000000000002,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.38800000000000001,"15min":0.34000000000000002},"min":{"1min":0.32300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":2.4020000000000001,"5min":2.4020000000000001,"15min":2.4020000000000001},"last":0.71799999999999997}]},{"osd":2,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.41499999999999998,"15min":0.38700000000000001},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.885,"5min":1.885,"15min":1.885},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.39400000000000002,"15min":0.374},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.218},"max":{"1min":2.0350000000000001,"5min":2.0350000000000001,"15min":2.0350000000000001},"last":0.47799999999999998}]},{"osd":3,"last update":"Fri Apr 12 11:11:48 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41099999999999998,"15min":0.40999999999999998},"min":{"1min":0.33000000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.54100000000000004,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.46600000000000003,"15min":0.46800000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.53700000000000003}]},{"osd":4,"last update":"Fri Apr 12 11:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.58799999999999997,"15min":0.57199999999999995},"min":{"1min":0.46600000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.7470000000000001,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.56699999999999995,"15min":0.55300000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":0.61099999999999999}]},{"osd":5,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.59799999999999998,"15min":0.58999999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.833,"5min":1.833,"15min":1.833},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.58499999999999996,"15min":0.57699999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":2.012,"5min":2.012,"15min":2.012},"last":0.621}]},{"osd":6,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69299999999999995}]},{"osd":7,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.859,"5min":1.859,"15min":1.859},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.6559999999999999,"5min":1.6559999999999999,"15min":1.6559999999999999},"last":0.64800000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32312,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712840,"statfs":{"total":95995035648,"available":95961948160,"internally_reserved":0,"allocated":1228800,"data_stored":853672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 11:12:00 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.34100000000000003,"15min":0.31900000000000001},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.50900000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.316},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.34799999999999998,"15min":0.32600000000000001},"min":{"1min":0.246,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.629,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.30499999999999999}]},{"osd":2,"last update":"Fri Apr 12 11:12:26 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.40000000000000002,"15min":0.38600000000000001},"min":{"1min":0.30399999999999999,"5min":0.219,"15min":0.219},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.39300000000000002,"15min":0.372},"min":{"1min":0.26000000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.25600000000000001}]},{"osd":3,"last update":"Fri Apr 12 11:11:50 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.438,"15min":0.436},"min":{"1min":0.32000000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42899999999999999,"15min":0.43099999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.36299999999999999}]},{"osd":4,"last update":"Fri Apr 12 11:12:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.58899999999999997,"15min":0.58499999999999996},"min":{"1min":0.38400000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.448,"5min":1.448,"15min":1.448},"last":0.47799999999999998}]},{"osd":5,"last update":"Fri Apr 12 11:12:37 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.59899999999999998,"15min":0.59099999999999997},"min":{"1min":0.40000000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.4770000000000001,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.89200000000000002,"15min":0.94599999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70199999999999996,"5min":7.9340000000000002,"15min":7.9340000000000002},"last":0.58499999999999996}]},{"osd":6,"last update":"Fri Apr 12 11:11:57 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.55000000000000004}]},{"osd":7,"last update":"Fri Apr 12 11:12:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.506,"5min":1.506,"15min":1.506},"last":0.59699999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27640,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717512,"statfs":{"total":95995035648,"available":95966732288,"internally_reserved":0,"allocated":634880,"data_stored":263304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.374},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.374},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.379,"15min":0.36499999999999999},"min":{"1min":0.29199999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.54600000000000004,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.251}]},{"osd":1,"last update":"Fri Apr 12 11:12:23 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.40699999999999997,"15min":0.38900000000000001},"min":{"1min":0.33300000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.59799999999999998,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.41399999999999998,"15min":0.40500000000000003},"min":{"1min":0.27500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.32500000000000001}]},{"osd":3,"last update":"Fri Apr 12 11:11:52 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.47999999999999998,"15min":0.47799999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.65200000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.46300000000000002,"15min":0.45900000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.64700000000000002,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.29699999999999999}]},{"osd":4,"last update":"Fri Apr 12 11:12:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.101,"15min":1.1910000000000001},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79100000000000004,"5min":13.686999999999999,"15min":13.686999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.1399999999999999,"15min":1.2270000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86599999999999999,"5min":13.721,"15min":13.721},"last":0.53000000000000003}]},{"osd":5,"last update":"Fri Apr 12 11:12:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.63300000000000001,"15min":0.63700000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.624}]},{"osd":6,"last update":"Fri Apr 12 11:11:59 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.48299999999999998}]},{"osd":7,"last update":"Fri Apr 12 11:12:23 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.44}]}]},{"osd":3,"up_from":25,"seq":107374182438,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31736,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713416,"statfs":{"total":95995035648,"available":95962537984,"internally_reserved":0,"allocated":634880,"data_stored":263304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31848904},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:11:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.67800000000000005,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.39400000000000002,"15min":0.39000000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.5}]},{"osd":1,"last update":"Fri Apr 12 11:11:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.437,"15min":0.439},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.42499999999999999,"15min":0.42699999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.48499999999999999}]},{"osd":2,"last update":"Fri Apr 12 11:11:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.45900000000000002,"15min":0.45800000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.46000000000000002,"15min":0.45800000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.622}]},{"osd":4,"last update":"Fri Apr 12 11:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.56999999999999995,"15min":0.57399999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.72799999999999998,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.292,"5min":0.72299999999999998,"15min":0.628},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":11.705,"5min":11.705,"15min":11.705},"last":0.60299999999999998}]},{"osd":5,"last update":"Fri Apr 12 11:12:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.58799999999999997,"15min":0.59699999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70799999999999996,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.173,"5min":0.71899999999999997,"15min":0.64300000000000002},"min":{"1min":0.44,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":0.64000000000000001}]},{"osd":6,"last update":"Fri Apr 12 11:11:58 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.78700000000000003}]},{"osd":7,"last update":"Fri Apr 12 11:12:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.83599999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27640,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717512,"statfs":{"total":95995035648,"available":95966732288,"internally_reserved":0,"allocated":634880,"data_stored":263304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.58299999999999996,"15min":0.59299999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.79500000000000004,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.623},{"interface":"front","average":{"1min":0.495,"5min":0.54200000000000004,"15min":0.55000000000000004},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.52700000000000002}]},{"osd":1,"last update":"Fri Apr 12 11:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.55800000000000005,"15min":0.56599999999999995},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71099999999999997,"5min":0.752,"15min":0.752},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.78900000000000003,"15min":0.83099999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73199999999999998,"5min":5.5590000000000002,"15min":5.5590000000000002},"last":0.54000000000000004}]},{"osd":2,"last update":"Fri Apr 12 11:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.74299999999999999,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.79200000000000004,"15min":0.83399999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76800000000000002,"5min":5.5419999999999998,"15min":5.5419999999999998},"last":0.59099999999999997}]},{"osd":3,"last update":"Fri Apr 12 11:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.56100000000000005,"15min":0.56599999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76300000000000001,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.504,"5min":0.57599999999999996,"15min":0.58799999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.61299999999999999,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.505}]},{"osd":5,"last update":"Fri Apr 12 11:12:37 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.40999999999999998,"15min":0.40300000000000002},"min":{"1min":0.311,"5min":0.308,"15min":0.308},"max":{"1min":0.58799999999999997,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.39000000000000001,"15min":0.38500000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.443}]},{"osd":6,"last update":"Fri Apr 12 11:11:57 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.59799999999999998}]},{"osd":7,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.49099999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27636,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717516,"statfs":{"total":95995035648,"available":95966736384,"internally_reserved":0,"allocated":630784,"data_stored":262973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.57899999999999996,"15min":0.58299999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.69799999999999995,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.53800000000000003,"15min":0.54200000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.60999999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.63}]},{"osd":1,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.55000000000000004,"15min":0.55600000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.67400000000000004,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.69899999999999995,"5min":0.873,"15min":0.873},"last":0.58599999999999997}]},{"osd":2,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.55600000000000005,"15min":0.56599999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.64300000000000002,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.55800000000000005,"15min":0.56599999999999995},"min":{"1min":0.41699999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.64600000000000002,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.68999999999999995}]},{"osd":3,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.57799999999999996,"15min":0.58999999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.65100000000000002,"5min":1.016,"15min":1.016},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.58399999999999996,"15min":0.59099999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.71899999999999997,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.55900000000000005}]},{"osd":4,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.40200000000000002,"15min":0.40000000000000002},"min":{"1min":0.29799999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.499},{"interface":"front","average":{"1min":0.442,"5min":0.41899999999999998,"15min":0.41499999999999998},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.48299999999999998}]},{"osd":6,"last update":"Fri Apr 12 11:11:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.64100000000000001}]},{"osd":7,"last update":"Fri Apr 12 11:12:20 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.53000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:12:46.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:12:46.959 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.537470399Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-12T11:12:46.959 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.540206222Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-12T11:12:46.959 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.540754831Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-12T11:12:46.959 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.543557506Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-12T11:12:46.959 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.543997479Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-12T11:12:46.959 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.544424156Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-12T11:12:46.959 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.5447971Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-12T11:12:46.959 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.545252714Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.546197657Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.548876835Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.549320201Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.549606957Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.549978092Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.550418913Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.550777069Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.551047765Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.551727038Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.552449297Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-12T11:12:46.960 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.553062654Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.554044562Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.554349752Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.554756193Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.555174128Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.555611226Z level=info msg="Executing migration" id="add correlation config column" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.556570135Z level=info msg="Executing migration" id="create entity_events table" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.556983686Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.559618898Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.559697481Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.5599734Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.560048144Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.560288594Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-12T11:12:46.961 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.560681925Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-12T11:12:46.962 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.563022708Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-12T11:12:46.962 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.563700066Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-12T11:12:46.962 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.564130135Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-12T11:12:46.962 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.564978919Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-12T11:12:46.962 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.565899473Z level=info msg="Executing migration" id="Drop public config table" 2024-04-12T11:12:46.962 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.566643957Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.567591493Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.568341004Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.56932254Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.570018611Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.572986767Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.574011793Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.575021687Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.575968956Z level=info msg="Executing migration" id="add share column" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.576993136Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.577929788Z level=info msg="Executing migration" id="create file table" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.578576193Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.579428048Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-12T11:12:46.963 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.580276535Z level=info msg="Executing migration" id="create file_meta table" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.580696704Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.581769045Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.582423352Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.583419089Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.584263774Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.585006823Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.586067735Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.586940086Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.587800695Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.588678507Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.589616934Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.590486827Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.59170019Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-12T11:12:46.964 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.592793297Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.593782953Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.603584318Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.604450407Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.607429652Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.611025331Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.613935145Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.616861316Z level=info msg="Executing migration" id="create folder table" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.619694872Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.62248584Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.625401572Z level=info msg="Executing migration" id="Update folder title length" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.628074419Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=migrator t=2024-04-12T11:12:46.631025767Z level=info msg="migrations completed" performed=485 skipped=0 duration=700.144964ms 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=sqlstore t=2024-04-12T11:12:46.631549129Z level=info msg="Created default organization" 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=plugin.loader t=2024-04-12T11:12:46.687552954Z level=info msg="Plugin registered" pluginID=input 2024-04-12T11:12:46.965 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=plugin.loader t=2024-04-12T11:12:46.714219597Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-12T11:12:46.966 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=plugin.loader t=2024-04-12T11:12:46.714236495Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-12T11:12:46.966 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=secrets t=2024-04-12T11:12:46.714287222Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-12T11:12:46.966 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=query_data t=2024-04-12T11:12:46.715962904Z level=info msg="Query Service initialization" 2024-04-12T11:12:46.966 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=live.push_http t=2024-04-12T11:12:46.718411784Z level=info msg="Live Push Gateway initialization" 2024-04-12T11:12:47.091 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:46 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:46 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:46 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:46 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:46 smithi029 ceph-mon[29349]: Deploying daemon node-exporter.a on smithi029 2024-04-12T11:12:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:46 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:46 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:46 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:46 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:46 smithi029 ceph-mon[39999]: Deploying daemon node-exporter.a on smithi029 2024-04-12T11:12:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:46 smithi087 ceph-mon[32917]: Deploying daemon node-exporter.a on smithi029 2024-04-12T11:12:47.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=infra.usagestats.collector t=2024-04-12T11:12:47.426707082Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-12T11:12:47.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=provisioning.datasources t=2024-04-12T11:12:47.479990295Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-12T11:12:47.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=provisioning.datasources t=2024-04-12T11:12:47.511580922Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-12T11:12:47.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=provisioning.alerting t=2024-04-12T11:12:47.533106421Z level=info msg="starting to provision alerting" 2024-04-12T11:12:47.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=provisioning.alerting t=2024-04-12T11:12:47.53312636Z level=info msg="finished to provision alerting" 2024-04-12T11:12:47.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=grafanaStorageLogger t=2024-04-12T11:12:47.533279445Z level=info msg="storage starting" 2024-04-12T11:12:47.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=ngalert.state.manager t=2024-04-12T11:12:47.533415667Z level=info msg="Warming state cache for startup" 2024-04-12T11:12:47.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=http.server t=2024-04-12T11:12:47.535091766Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-12T11:12:47.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=sqlstore.transactions t=2024-04-12T11:12:47.543597376Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-12T11:12:47.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=ngalert.state.manager t=2024-04-12T11:12:47.5436979Z level=info msg="State cache has been initialized" states=0 duration=10.27704ms 2024-04-12T11:12:47.718 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=ticker t=2024-04-12T11:12:47.543780903Z level=info msg=starting first_tick=2024-04-12T11:12:50Z 2024-04-12T11:12:47.718 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=ngalert.multiorg.alertmanager t=2024-04-12T11:12:47.543786269Z level=info msg="starting MultiOrg Alertmanager" 2024-04-12T11:12:47.718 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=sqlstore.transactions t=2024-04-12T11:12:47.591262126Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-12T11:12:47.718 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=sqlstore.transactions t=2024-04-12T11:12:47.621977831Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-12T11:12:47.718 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:12:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=sqlstore.transactions t=2024-04-12T11:12:47.633373848Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-12T11:12:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:48 smithi029 ceph-mon[29349]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:48 smithi029 ceph-mon[29349]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:12:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:48 smithi029 ceph-mon[39999]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:48.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:48 smithi029 ceph-mon[39999]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:12:48.394 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:48 smithi087 ceph-mon[32917]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:48.395 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:48 smithi087 ceph-mon[32917]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:12:49.055 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:12:49.056 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:12:49.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:49 smithi087 ceph-mon[32917]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:49.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:49 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:49.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:49 smithi087 ceph-mon[32917]: from='client.24518 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:12:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:49 smithi029 ceph-mon[29349]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:49 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:49 smithi029 ceph-mon[29349]: from='client.24518 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:12:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:49 smithi029 ceph-mon[39999]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:49 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:49 smithi029 ceph-mon[39999]: from='client.24518 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:12:49.519 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:49 smithi029 bash[72023]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-12T11:12:49.885 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":28,"stamp":"2024-04-12T11:12:48.128007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":231028,"kb_used_data":6684,"kb_used_omap":12,"kb_used_meta":224243,"kb_avail":749730188,"statfs":{"total":767960285184,"available":767723712512,"internally_reserved":0,"allocated":6844416,"data_stored":3876320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":229625401},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001808"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-12T11:11:52.968666+0000","last_change":"2024-04-12T11:11:49.523873+0000","last_active":"2024-04-12T11:11:52.968666+0000","last_peered":"2024-04-12T11:11:52.968666+0000","last_clean":"2024-04-12T11:11:52.968666+0000","last_became_active":"2024-04-12T11:11:49.523570+0000","last_became_peered":"2024-04-12T11:11:49.523570+0000","last_unstale":"2024-04-12T11:11:52.968666+0000","last_undegraded":"2024-04-12T11:11:52.968666+0000","last_fullsized":"2024-04-12T11:11:52.968666+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:09:00.491446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":13,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:11:58.112697+0000","last_change":"2024-04-12T11:11:49.882400+0000","last_active":"2024-04-12T11:11:58.112697+0000","last_peered":"2024-04-12T11:11:58.112697+0000","last_clean":"2024-04-12T11:11:58.112697+0000","last_became_active":"2024-04-12T11:11:49.882073+0000","last_became_peered":"2024-04-12T11:11:49.882073+0000","last_unstale":"2024-04-12T11:11:58.112697+0000","last_undegraded":"2024-04-12T11:11:58.112697+0000","last_fullsized":"2024-04-12T11:11:58.112697+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:38:46.841744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-12T11:11:52.968901+0000","last_change":"2024-04-12T11:11:49.882324+0000","last_active":"2024-04-12T11:11:52.968901+0000","last_peered":"2024-04-12T11:11:52.968901+0000","last_clean":"2024-04-12T11:11:52.968901+0000","last_became_active":"2024-04-12T11:11:49.881975+0000","last_became_peered":"2024-04-12T11:11:49.881975+0000","last_unstale":"2024-04-12T11:11:52.968901+0000","last_undegraded":"2024-04-12T11:11:52.968901+0000","last_fullsized":"2024-04-12T11:11:52.968901+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:10.140250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":13,"reported_epoch":62,"state":"active+clean","last_fresh":"2024-04-12T11:11:52.968858+0000","last_change":"2024-04-12T11:11:49.524657+0000","last_active":"2024-04-12T11:11:52.968858+0000","last_peered":"2024-04-12T11:11:52.968858+0000","last_clean":"2024-04-12T11:11:52.968858+0000","last_became_active":"2024-04-12T11:11:49.524379+0000","last_became_peered":"2024-04-12T11:11:49.524379+0000","last_unstale":"2024-04-12T11:11:52.968858+0000","last_undegraded":"2024-04-12T11:11:52.968858+0000","last_fullsized":"2024-04-12T11:11:52.968858+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:30:52.444470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":258,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:12:00.186224+0000","last_change":"2024-04-12T11:11:24.390399+0000","last_active":"2024-04-12T11:12:00.186224+0000","last_peered":"2024-04-12T11:12:00.186224+0000","last_clean":"2024-04-12T11:12:00.186224+0000","last_became_active":"2024-04-12T11:11:24.389943+0000","last_became_peered":"2024-04-12T11:11:24.389943+0000","last_unstale":"2024-04-12T11:12:00.186224+0000","last_undegraded":"2024-04-12T11:12:00.186224+0000","last_fullsized":"2024-04-12T11:12:00.186224+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:06:44.231202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:11:58.113204+0000","last_change":"2024-04-12T11:11:49.524628+0000","last_active":"2024-04-12T11:11:58.113204+0000","last_peered":"2024-04-12T11:11:58.113204+0000","last_clean":"2024-04-12T11:11:58.113204+0000","last_became_active":"2024-04-12T11:11:49.524226+0000","last_became_peered":"2024-04-12T11:11:49.524226+0000","last_unstale":"2024-04-12T11:11:58.113204+0000","last_undegraded":"2024-04-12T11:11:58.113204+0000","last_fullsized":"2024-04-12T11:11:58.113204+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:43:26.148427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":13,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:11:58.111945+0000","last_change":"2024-04-12T11:11:49.523432+0000","last_active":"2024-04-12T11:11:58.111945+0000","last_peered":"2024-04-12T11:11:58.111945+0000","last_clean":"2024-04-12T11:11:58.111945+0000","last_became_active":"2024-04-12T11:11:49.523059+0000","last_became_peered":"2024-04-12T11:11:49.523059+0000","last_unstale":"2024-04-12T11:11:58.111945+0000","last_undegraded":"2024-04-12T11:11:58.111945+0000","last_fullsized":"2024-04-12T11:11:58.111945+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:55:25.692863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:11:58.113179+0000","last_change":"2024-04-12T11:11:49.523947+0000","last_active":"2024-04-12T11:11:58.113179+0000","last_peered":"2024-04-12T11:11:58.113179+0000","last_clean":"2024-04-12T11:11:58.113179+0000","last_became_active":"2024-04-12T11:11:49.523650+0000","last_became_peered":"2024-04-12T11:11:49.523650+0000","last_unstale":"2024-04-12T11:11:58.113179+0000","last_undegraded":"2024-04-12T11:11:58.113179+0000","last_fullsized":"2024-04-12T11:11:58.113179+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:05:22.528981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":13,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-12T11:11:58.112622+0000","last_change":"2024-04-12T11:11:49.524259+0000","last_active":"2024-04-12T11:11:58.112622+0000","last_peered":"2024-04-12T11:11:58.112622+0000","last_clean":"2024-04-12T11:11:58.112622+0000","last_became_active":"2024-04-12T11:11:49.523641+0000","last_became_peered":"2024-04-12T11:11:49.523641+0000","last_unstale":"2024-04-12T11:11:58.112622+0000","last_undegraded":"2024-04-12T11:11:58.112622+0000","last_fullsized":"2024-04-12T11:11:58.112622+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:28:22.480630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28216,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716936,"statfs":{"total":95995035648,"available":95966142464,"internally_reserved":0,"allocated":1224704,"data_stored":853341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.4780000000000002,"5min":2.4780000000000002,"15min":2.4780000000000002},"last":0.754}]},{"osd":1,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":0.70399999999999996}]},{"osd":2,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.111,"15min":1.111},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":15.228999999999999,"5min":15.228999999999999,"15min":15.228999999999999},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.157,"5min":1.157,"15min":1.157},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":15.249000000000001,"5min":15.249000000000001,"15min":15.249000000000001},"last":0.71699999999999997}]},{"osd":3,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.1539999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.66700000000000004}]},{"osd":4,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.56299999999999994}]},{"osd":5,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.59499999999999997}]},{"osd":6,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.76800000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495640,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28208,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716944,"statfs":{"total":95995035648,"available":95966150656,"internally_reserved":0,"allocated":1220608,"data_stored":853118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.30399999999999999}]},{"osd":1,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.31900000000000001}]},{"osd":2,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.35399999999999998}]},{"osd":3,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.40300000000000002}]},{"osd":4,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.44},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.28799999999999998}]},{"osd":5,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.45000000000000001}]},{"osd":7,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.41799999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27640,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717512,"statfs":{"total":95995035648,"available":95966732288,"internally_reserved":0,"allocated":634880,"data_stored":263304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.35199999999999998,"15min":0.312},"min":{"1min":0.245,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.5710000000000002,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.38800000000000001,"15min":0.34000000000000002},"min":{"1min":0.32300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":2.4020000000000001,"5min":2.4020000000000001,"15min":2.4020000000000001},"last":0.71799999999999997}]},{"osd":2,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.41499999999999998,"15min":0.38700000000000001},"min":{"1min":0.28599999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.885,"5min":1.885,"15min":1.885},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.39400000000000002,"15min":0.374},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.218},"max":{"1min":2.0350000000000001,"5min":2.0350000000000001,"15min":2.0350000000000001},"last":0.47799999999999998}]},{"osd":3,"last update":"Fri Apr 12 11:11:48 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41099999999999998,"15min":0.40999999999999998},"min":{"1min":0.33000000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.54100000000000004,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.46600000000000003,"15min":0.46800000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.53700000000000003}]},{"osd":4,"last update":"Fri Apr 12 11:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.58799999999999997,"15min":0.57199999999999995},"min":{"1min":0.46600000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.7470000000000001,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.56699999999999995,"15min":0.55300000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":0.61099999999999999}]},{"osd":5,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.59799999999999998,"15min":0.58999999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.833,"5min":1.833,"15min":1.833},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.58499999999999996,"15min":0.57699999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":2.012,"5min":2.012,"15min":2.012},"last":0.621}]},{"osd":6,"last update":"Fri Apr 12 11:11:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69299999999999995}]},{"osd":7,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.859,"5min":1.859,"15min":1.859},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.6559999999999999,"5min":1.6559999999999999,"15min":1.6559999999999999},"last":0.64800000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32312,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712840,"statfs":{"total":95995035648,"available":95961948160,"internally_reserved":0,"allocated":1228800,"data_stored":853672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 11:12:00 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.34100000000000003,"15min":0.31900000000000001},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.50900000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.316},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.34799999999999998,"15min":0.32600000000000001},"min":{"1min":0.246,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.629,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.30499999999999999}]},{"osd":2,"last update":"Fri Apr 12 11:12:26 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.40000000000000002,"15min":0.38600000000000001},"min":{"1min":0.30399999999999999,"5min":0.219,"15min":0.219},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.39300000000000002,"15min":0.372},"min":{"1min":0.26000000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.25600000000000001}]},{"osd":3,"last update":"Fri Apr 12 11:11:50 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.438,"15min":0.436},"min":{"1min":0.32000000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42899999999999999,"15min":0.43099999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.36299999999999999}]},{"osd":4,"last update":"Fri Apr 12 11:12:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.58899999999999997,"15min":0.58499999999999996},"min":{"1min":0.38400000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.448,"5min":1.448,"15min":1.448},"last":0.47799999999999998}]},{"osd":5,"last update":"Fri Apr 12 11:12:37 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.59899999999999998,"15min":0.59099999999999997},"min":{"1min":0.40000000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.4770000000000001,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.89200000000000002,"15min":0.94599999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.70199999999999996,"5min":7.9340000000000002,"15min":7.9340000000000002},"last":0.58499999999999996}]},{"osd":6,"last update":"Fri Apr 12 11:11:57 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.55000000000000004}]},{"osd":7,"last update":"Fri Apr 12 11:12:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.506,"5min":1.506,"15min":1.506},"last":0.59699999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27640,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717512,"statfs":{"total":95995035648,"available":95966732288,"internally_reserved":0,"allocated":634880,"data_stored":263304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.38500000000000001,"15min":0.374},"min":{"1min":0.23599999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.374},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.379,"15min":0.36499999999999999},"min":{"1min":0.29199999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.54600000000000004,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.251}]},{"osd":1,"last update":"Fri Apr 12 11:12:23 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.40699999999999997,"15min":0.38900000000000001},"min":{"1min":0.33300000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.59799999999999998,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.41399999999999998,"15min":0.40500000000000003},"min":{"1min":0.27500000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.32500000000000001}]},{"osd":3,"last update":"Fri Apr 12 11:11:52 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.47999999999999998,"15min":0.47799999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.65200000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.46300000000000002,"15min":0.45900000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.64700000000000002,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.29699999999999999}]},{"osd":4,"last update":"Fri Apr 12 11:12:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.101,"15min":1.1910000000000001},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79100000000000004,"5min":13.686999999999999,"15min":13.686999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.1399999999999999,"15min":1.2270000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86599999999999999,"5min":13.721,"15min":13.721},"last":0.53000000000000003}]},{"osd":5,"last update":"Fri Apr 12 11:12:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.63300000000000001,"15min":0.63700000000000001},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.624}]},{"osd":6,"last update":"Fri Apr 12 11:11:59 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.48299999999999998}]},{"osd":7,"last update":"Fri Apr 12 11:12:23 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.44}]}]},{"osd":3,"up_from":25,"seq":107374182438,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31736,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713416,"statfs":{"total":95995035648,"available":95962537984,"internally_reserved":0,"allocated":634880,"data_stored":263304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31848904},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:11:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.67800000000000005,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.39400000000000002,"15min":0.39000000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.5}]},{"osd":1,"last update":"Fri Apr 12 11:11:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.437,"15min":0.439},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.42499999999999999,"15min":0.42699999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.48499999999999999}]},{"osd":2,"last update":"Fri Apr 12 11:11:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.45900000000000002,"15min":0.45800000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.46000000000000002,"15min":0.45800000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.622}]},{"osd":4,"last update":"Fri Apr 12 11:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.56999999999999995,"15min":0.57399999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.72799999999999998,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.292,"5min":0.72299999999999998,"15min":0.628},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":11.705,"5min":11.705,"15min":11.705},"last":0.60299999999999998}]},{"osd":5,"last update":"Fri Apr 12 11:12:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.58799999999999997,"15min":0.59699999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.70799999999999996,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.173,"5min":0.71899999999999997,"15min":0.64300000000000002},"min":{"1min":0.44,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":0.64000000000000001}]},{"osd":6,"last update":"Fri Apr 12 11:11:58 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.78700000000000003}]},{"osd":7,"last update":"Fri Apr 12 11:12:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.83599999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27640,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717512,"statfs":{"total":95995035648,"available":95966732288,"internally_reserved":0,"allocated":634880,"data_stored":263304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.58299999999999996,"15min":0.59299999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.79500000000000004,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.623},{"interface":"front","average":{"1min":0.495,"5min":0.54200000000000004,"15min":0.55000000000000004},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.52700000000000002}]},{"osd":1,"last update":"Fri Apr 12 11:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.55800000000000005,"15min":0.56599999999999995},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71099999999999997,"5min":0.752,"15min":0.752},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.78900000000000003,"15min":0.83099999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73199999999999998,"5min":5.5590000000000002,"15min":5.5590000000000002},"last":0.54000000000000004}]},{"osd":2,"last update":"Fri Apr 12 11:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.74299999999999999,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.79200000000000004,"15min":0.83399999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76800000000000002,"5min":5.5419999999999998,"15min":5.5419999999999998},"last":0.59099999999999997}]},{"osd":3,"last update":"Fri Apr 12 11:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.56100000000000005,"15min":0.56599999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76300000000000001,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.504,"5min":0.57599999999999996,"15min":0.58799999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.61299999999999999,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.505}]},{"osd":5,"last update":"Fri Apr 12 11:12:37 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.40999999999999998,"15min":0.40300000000000002},"min":{"1min":0.311,"5min":0.308,"15min":0.308},"max":{"1min":0.58799999999999997,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.39000000000000001,"15min":0.38500000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.443}]},{"osd":6,"last update":"Fri Apr 12 11:11:57 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.59799999999999998}]},{"osd":7,"last update":"Fri Apr 12 11:12:22 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.49099999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27636,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717516,"statfs":{"total":95995035648,"available":95966736384,"internally_reserved":0,"allocated":630784,"data_stored":262973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.57899999999999996,"15min":0.58299999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.69799999999999995,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.53800000000000003,"15min":0.54200000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.60999999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.63}]},{"osd":1,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.55000000000000004,"15min":0.55600000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.67400000000000004,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.55200000000000005,"15min":0.55600000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.69899999999999995,"5min":0.873,"15min":0.873},"last":0.58599999999999997}]},{"osd":2,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.55600000000000005,"15min":0.56599999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.64300000000000002,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.55800000000000005,"15min":0.56599999999999995},"min":{"1min":0.41699999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.64600000000000002,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.68999999999999995}]},{"osd":3,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.57799999999999996,"15min":0.58999999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.65100000000000002,"5min":1.016,"15min":1.016},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.58399999999999996,"15min":0.59099999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.71899999999999997,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.55900000000000005}]},{"osd":4,"last update":"Fri Apr 12 11:12:33 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.40200000000000002,"15min":0.40000000000000002},"min":{"1min":0.29799999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.499},{"interface":"front","average":{"1min":0.442,"5min":0.41899999999999998,"15min":0.41499999999999998},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.48299999999999998}]},{"osd":6,"last update":"Fri Apr 12 11:11:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.64100000000000001}]},{"osd":7,"last update":"Fri Apr 12 11:12:20 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.53000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:12:49.887 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-12T11:12:49.887 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-12T11:12:49.887 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-12T11:12:49.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T11:12:50.222 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:51.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:51 smithi029 ceph-mon[29349]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:51.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:51 smithi029 ceph-mon[39999]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:51.432 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:51 smithi029 bash[72023]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-12T11:12:51.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:51 smithi087 ceph-mon[32917]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:52.018 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:51 smithi029 bash[72023]: Writing manifest to image destination 2024-04-12T11:12:52.349 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 podman[72023]: 2024-04-12 11:12:52.00965777 +0000 UTC m=+3.380960191 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-12T11:12:52.349 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 podman[72023]: 2024-04-12 11:12:52.226384381 +0000 UTC m=+3.597686847 container create 32124106e91232b059a65c0e575644bb1158c18157b23e3e78ec63a78abb91ee (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T11:12:52.559 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 podman[72023]: 2024-04-12 11:12:52.550074895 +0000 UTC m=+3.921377304 container init 32124106e91232b059a65c0e575644bb1158c18157b23e3e78ec63a78abb91ee (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T11:12:52.559 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 podman[72023]: 2024-04-12 11:12:52.553049652 +0000 UTC m=+3.924352068 container start 32124106e91232b059a65c0e575644bb1158c18157b23e3e78ec63a78abb91ee (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T11:12:52.769 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.557Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-12T11:12:52.769 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.557Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-12T11:12:52.769 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.557Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-12T11:12:52.769 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.557Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-12T11:12:52.769 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-12T11:12:52.769 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-12T11:12:52.769 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-12T11:12:52.769 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=arp 2024-04-12T11:12:52.769 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=edac 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-12T11:12:52.770 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.558Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=os 2024-04-12T11:12:52.771 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=stat 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=time 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-12T11:12:52.772 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=uname 2024-04-12T11:12:52.773 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-12T11:12:52.773 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-12T11:12:52.773 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-12T11:12:52.773 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-12T11:12:52.773 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a[72238]: ts=2024-04-12T11:12:52.559Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-12T11:12:52.773 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 bash[72023]: 32124106e91232b059a65c0e575644bb1158c18157b23e3e78ec63a78abb91ee 2024-04-12T11:12:52.773 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 11:12:52 smithi029 systemd[1]: Started Ceph node-exporter.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:12:53.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:12:53] "GET /metrics HTTP/1.1" 200 28555 "" "Prometheus/2.43.0" 2024-04-12T11:12:53.280 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:12:53.280 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[39999]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[39999]: Deploying daemon node-exporter.b on smithi087 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3929554546' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[29349]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:53.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[29349]: Deploying daemon node-exporter.b on smithi087 2024-04-12T11:12:53.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:53 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3929554546' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T11:12:53.947 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-12T11:12:53.948 INFO:tasks.cephadm:Setup complete, yielding 2024-04-12T11:12:53.948 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T11:12:53.959 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi029.front.sepia.ceph.com 2024-04-12T11:12:53.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-04-12T11:12:53.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:53 smithi087 ceph-mon[32917]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:53 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:53 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:53 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:53 smithi087 ceph-mon[32917]: Deploying daemon node-exporter.b on smithi087 2024-04-12T11:12:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:53 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3929554546' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T11:12:53.967 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:53 smithi087 bash[63148]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-12T11:12:54.173 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:54.466 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:54 smithi087 bash[63148]: Getting image source signatures 2024-04-12T11:12:54.466 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:54 smithi087 bash[63148]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-12T11:12:54.467 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:54 smithi087 bash[63148]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-12T11:12:54.467 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:54 smithi087 bash[63148]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-12T11:12:55.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:55 smithi029 ceph-mon[29349]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:55.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:55 smithi029 ceph-mon[39999]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:55 smithi087 ceph-mon[32917]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:55.967 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:55 smithi087 bash[63148]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-12T11:12:55.967 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:55 smithi087 bash[63148]: Writing manifest to image destination 2024-04-12T11:12:56.361 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 podman[63148]: 2024-04-12 11:12:56.073993351 +0000 UTC m=+2.387576152 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-12T11:12:56.361 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 podman[63148]: 2024-04-12 11:12:56.172742478 +0000 UTC m=+2.486325323 container create 78910b2d30c688a6fe964f8df008770efccb0b0e369df975994ff3cc5e26ef8a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T11:12:56.361 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 podman[63148]: 2024-04-12 11:12:56.35839953 +0000 UTC m=+2.671982341 container init 78910b2d30c688a6fe964f8df008770efccb0b0e369df975994ff3cc5e26ef8a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T11:12:56.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 podman[63148]: 2024-04-12 11:12:56.361384216 +0000 UTC m=+2.674967021 container start 78910b2d30c688a6fe964f8df008770efccb0b0e369df975994ff3cc5e26ef8a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T11:12:56.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.365Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-12T11:12:56.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.365Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-12T11:12:56.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-12T11:12:56.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-12T11:12:56.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-12T11:12:56.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-12T11:12:56.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-12T11:12:56.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=node_exporter.go:117 level=info collector=arp 2024-04-12T11:12:56.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-12T11:12:56.718 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-12T11:12:56.718 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-12T11:12:56.718 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-12T11:12:56.718 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-12T11:12:56.718 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-12T11:12:56.718 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.366Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-12T11:12:56.718 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=edac 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-12T11:12:56.719 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=os 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-12T11:12:56.720 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=stat 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=time 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=uname 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b[63199]: ts=2024-04-12T11:12:56.367Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 bash[63148]: 78910b2d30c688a6fe964f8df008770efccb0b0e369df975994ff3cc5e26ef8a 2024-04-12T11:12:56.721 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 11:12:56 smithi087 systemd[1]: Started Ceph node-exporter.b for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:12:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-12T11:12:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:12:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-12T11:12:57.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:56 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:12:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-12T11:12:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:57.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:56 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:12:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-mon[29349]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-mon[29349]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:12:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-12T11:12:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-mon[29349]: osdmap e64: 8 total, 8 up, 8 in 2024-04-12T11:12:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-12T11:12:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[29345]: 2024-04-12T11:12:57.762+0000 7f1ec63b5700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T11:12:58.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-mon[39999]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:58.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-mon[39999]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:12:58.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-12T11:12:58.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-mon[39999]: osdmap e64: 8 total, 8 up, 8 in 2024-04-12T11:12:58.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:57 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-12T11:12:58.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:57 smithi087 ceph-mon[32917]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:12:58.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:57 smithi087 ceph-mon[32917]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:12:58.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:57 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-12T11:12:58.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:57 smithi087 ceph-mon[32917]: osdmap e64: 8 total, 8 up, 8 in 2024-04-12T11:12:58.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:57 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-12T11:12:58.509 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-12T11:12:58.737 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:12:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: osdmap e65: 8 total, 8 up, 8 in 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: osdmap e66: 8 total, 8 up, 8 in 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: fsmap foo:0 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: Saving service mds.foo spec with placement count:2 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: osdmap e65: 8 total, 8 up, 8 in 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: osdmap e66: 8 total, 8 up, 8 in 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: fsmap foo:0 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: Saving service mds.foo spec with placement count:2 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: osdmap e65: 8 total, 8 up, 8 in 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: osdmap e66: 8 total, 8 up, 8 in 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: fsmap foo:0 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: Saving service mds.foo spec with placement count:2 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:12:59.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:12:59.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:00.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:59 smithi087 ceph-mon[32917]: pgmap v36: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:00.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:12:59 smithi087 ceph-mon[32917]: osdmap e67: 8 total, 8 up, 8 in 2024-04-12T11:13:00.241 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:59 smithi029 ceph-mon[39999]: pgmap v36: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:00.242 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:12:59 smithi029 ceph-mon[39999]: osdmap e67: 8 total, 8 up, 8 in 2024-04-12T11:13:00.242 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:59 smithi029 ceph-mon[29349]: pgmap v36: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:00.242 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:12:59 smithi029 ceph-mon[29349]: osdmap e67: 8 total, 8 up, 8 in 2024-04-12T11:13:01.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:00 smithi029 ceph-mon[39999]: osdmap e68: 8 total, 8 up, 8 in 2024-04-12T11:13:01.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:00 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:01.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:00 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:01.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:00 smithi029 ceph-mon[29349]: osdmap e68: 8 total, 8 up, 8 in 2024-04-12T11:13:01.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:00 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:01.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:00 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:01.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:00 smithi087 ceph-mon[32917]: osdmap e68: 8 total, 8 up, 8 in 2024-04-12T11:13:01.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:00 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:01.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:00 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:01 smithi029 ceph-mon[29349]: pgmap v39: 73 pgs: 12 creating+activating, 61 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:01 smithi029 ceph-mon[39999]: pgmap v39: 73 pgs: 12 creating+activating, 61 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:02.207 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-12T11:13:02.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:01 smithi087 ceph-mon[32917]: pgmap v39: 73 pgs: 12 creating+activating, 61 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:02.549 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:03.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:13:03] "GET /metrics HTTP/1.1" 200 33352 "" "Prometheus/2.43.0" 2024-04-12T11:13:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[29349]: pgmap v40: 73 pgs: 12 creating+activating, 61 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:13:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.scbuux", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.scbuux", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[29349]: Deploying daemon mds.foo.smithi029.scbuux on smithi029 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[39999]: pgmap v40: 73 pgs: 12 creating+activating, 61 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.scbuux", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:13:03.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.scbuux", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T11:13:03.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:03.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:03 smithi029 ceph-mon[39999]: Deploying daemon mds.foo.smithi029.scbuux on smithi029 2024-04-12T11:13:03.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:03 smithi087 ceph-mon[32917]: pgmap v40: 73 pgs: 12 creating+activating, 61 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:13:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.scbuux", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:13:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.scbuux", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T11:13:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:03 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:03 smithi087 ceph-mon[32917]: Deploying daemon mds.foo.smithi029.scbuux on smithi029 2024-04-12T11:13:05.449 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-12T11:13:05.458 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-12T11:13:05.458 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-12T11:13:05.459 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph mgr dump --format=json 2024-04-12T11:13:05.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:05 smithi029 ceph-mon[29349]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:05.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:05 smithi029 ceph-mon[39999]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:05 smithi087 ceph-mon[32917]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:06.003 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:07.758 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:07 smithi029 ceph-mon[39999]: pgmap v42: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:07.758 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:07 smithi029 ceph-mon[29349]: pgmap v42: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:13:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:07 smithi087 ceph-mon[32917]: pgmap v42: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2759402837' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T11:13:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi087.mbflzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:13:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi087.mbflzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T11:13:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2759402837' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T11:13:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi087.mbflzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:13:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi087.mbflzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T11:13:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:08 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:08.831 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":20,"flags":0,"active_gid":14472,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":3931254419},{"type":"v1","addr":"172.21.15.29:6801","nonce":3931254419}]},"active_addr":"172.21.15.29:6801/3931254419","active_change":"2024-04-12T11:11:58.108411+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":14475,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":63,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3128860153}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":4202867789}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":2463058069}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":1221327863}]}]} 2024-04-12T11:13:08.835 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-12T11:13:08.835 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-12T11:13:08.835 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T11:13:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:08 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2759402837' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T11:13:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:08 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:08 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:08 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:08 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi087.mbflzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:13:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:08 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi087.mbflzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T11:13:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:08 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:09.047 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: Deploying daemon mds.foo.smithi087.mbflzu on smithi087 2024-04-12T11:13:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: mds.? [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] up:boot 2024-04-12T11:13:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: daemon mds.foo.smithi029.scbuux assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-12T11:13:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T11:13:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-12T11:13:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: Cluster is now healthy 2024-04-12T11:13:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: fsmap foo:0 1 up:standby 2024-04-12T11:13:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:13:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: fsmap foo:1 {0=foo.smithi029.scbuux=up:creating} 2024-04-12T11:13:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[39999]: daemon mds.foo.smithi029.scbuux is now active in filesystem foo as rank 0 2024-04-12T11:13:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: Deploying daemon mds.foo.smithi087.mbflzu on smithi087 2024-04-12T11:13:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: mds.? [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] up:boot 2024-04-12T11:13:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: daemon mds.foo.smithi029.scbuux assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-12T11:13:09.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T11:13:09.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-12T11:13:09.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: Cluster is now healthy 2024-04-12T11:13:09.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: fsmap foo:0 1 up:standby 2024-04-12T11:13:09.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:13:09.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi029.scbuux=up:creating} 2024-04-12T11:13:09.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:09 smithi029 ceph-mon[29349]: daemon mds.foo.smithi029.scbuux is now active in filesystem foo as rank 0 2024-04-12T11:13:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: pgmap v43: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: Deploying daemon mds.foo.smithi087.mbflzu on smithi087 2024-04-12T11:13:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: mds.? [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] up:boot 2024-04-12T11:13:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: daemon mds.foo.smithi029.scbuux assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-12T11:13:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T11:13:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-12T11:13:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: Cluster is now healthy 2024-04-12T11:13:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: fsmap foo:0 1 up:standby 2024-04-12T11:13:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:13:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: fsmap foo:1 {0=foo.smithi029.scbuux=up:creating} 2024-04-12T11:13:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:09 smithi087 ceph-mon[32917]: daemon mds.foo.smithi029.scbuux is now active in filesystem foo as rank 0 2024-04-12T11:13:10.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:10 smithi087 ceph-mon[32917]: mds.? [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] up:active 2024-04-12T11:13:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:10 smithi087 ceph-mon[32917]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 2024-04-12T11:13:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:10 smithi029 ceph-mon[39999]: mds.? [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] up:active 2024-04-12T11:13:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:10 smithi029 ceph-mon[39999]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 2024-04-12T11:13:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:10 smithi029 ceph-mon[29349]: mds.? [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] up:active 2024-04-12T11:13:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:10 smithi029 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 2024-04-12T11:13:10.958 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:13:10.958 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":68,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T11:12:59.774514+0000","last_up_change":"2024-04-12T11:11:19.950253+0000","last_in_change":"2024-04-12T11:11:02.730106+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T11:11:48.103219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T11:12:56.493343+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T11:12:57.001342+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6803","nonce":2299659597}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6805","nonce":2299659597}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6809","nonce":2299659597}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6807","nonce":2299659597}]},"public_addr":"172.21.15.29:6803/2299659597","cluster_addr":"172.21.15.29:6805/2299659597","heartbeat_back_addr":"172.21.15.29:6809/2299659597","heartbeat_front_addr":"172.21.15.29:6807/2299659597","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6811","nonce":3193349615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6813","nonce":3193349615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6817","nonce":3193349615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6815","nonce":3193349615}]},"public_addr":"172.21.15.29:6811/3193349615","cluster_addr":"172.21.15.29:6813/3193349615","heartbeat_back_addr":"172.21.15.29:6817/3193349615","heartbeat_front_addr":"172.21.15.29:6815/3193349615","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6819","nonce":1946236732}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6821","nonce":1946236732}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6825","nonce":1946236732}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6823","nonce":1946236732}]},"public_addr":"172.21.15.29:6819/1946236732","cluster_addr":"172.21.15.29:6821/1946236732","heartbeat_back_addr":"172.21.15.29:6825/1946236732","heartbeat_front_addr":"172.21.15.29:6823/1946236732","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6827","nonce":4104584972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6829","nonce":4104584972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6833","nonce":4104584972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6831","nonce":4104584972}]},"public_addr":"172.21.15.29:6827/4104584972","cluster_addr":"172.21.15.29:6829/4104584972","heartbeat_back_addr":"172.21.15.29:6833/4104584972","heartbeat_front_addr":"172.21.15.29:6831/4104584972","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6801","nonce":2970504758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6803","nonce":2970504758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6807","nonce":2970504758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6805","nonce":2970504758}]},"public_addr":"172.21.15.87:6801/2970504758","cluster_addr":"172.21.15.87:6803/2970504758","heartbeat_back_addr":"172.21.15.87:6807/2970504758","heartbeat_front_addr":"172.21.15.87:6805/2970504758","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6809","nonce":3642701185}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6811","nonce":3642701185}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6815","nonce":3642701185}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6813","nonce":3642701185}]},"public_addr":"172.21.15.87:6809/3642701185","cluster_addr":"172.21.15.87:6811/3642701185","heartbeat_back_addr":"172.21.15.87:6815/3642701185","heartbeat_front_addr":"172.21.15.87:6813/3642701185","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2946468060":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6800/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/364858296":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/3254157714":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/659258741":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T11:13:11.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:11 smithi087 ceph-mon[32917]: pgmap v44: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-04-12T11:13:11.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:11 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3451779408' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:13:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:11 smithi029 ceph-mon[29349]: pgmap v44: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-04-12T11:13:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:11 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3451779408' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:13:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:11 smithi029 ceph-mon[39999]: pgmap v44: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-04-12T11:13:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:11 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3451779408' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:13:12.053 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-12T11:13:12.053 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T11:13:12.266 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:13.442 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:13:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:13:13] "GET /metrics HTTP/1.1" 200 33352 "" "Prometheus/2.43.0" 2024-04-12T11:13:13.755 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:13 smithi029 ceph-mon[39999]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:13.755 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:13.755 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:13.755 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:13 smithi029 ceph-mon[29349]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:13.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:13.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:13 smithi087 ceph-mon[32917]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:14.202 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:13:14.202 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":68,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T11:12:59.774514+0000","last_up_change":"2024-04-12T11:11:19.950253+0000","last_in_change":"2024-04-12T11:11:02.730106+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T11:11:48.103219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T11:12:56.493343+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T11:12:57.001342+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6803","nonce":2299659597}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6805","nonce":2299659597}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6809","nonce":2299659597}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":2299659597},{"type":"v1","addr":"172.21.15.29:6807","nonce":2299659597}]},"public_addr":"172.21.15.29:6803/2299659597","cluster_addr":"172.21.15.29:6805/2299659597","heartbeat_back_addr":"172.21.15.29:6809/2299659597","heartbeat_front_addr":"172.21.15.29:6807/2299659597","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6811","nonce":3193349615}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6813","nonce":3193349615}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6817","nonce":3193349615}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3193349615},{"type":"v1","addr":"172.21.15.29:6815","nonce":3193349615}]},"public_addr":"172.21.15.29:6811/3193349615","cluster_addr":"172.21.15.29:6813/3193349615","heartbeat_back_addr":"172.21.15.29:6817/3193349615","heartbeat_front_addr":"172.21.15.29:6815/3193349615","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6819","nonce":1946236732}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6821","nonce":1946236732}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6825","nonce":1946236732}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1946236732},{"type":"v1","addr":"172.21.15.29:6823","nonce":1946236732}]},"public_addr":"172.21.15.29:6819/1946236732","cluster_addr":"172.21.15.29:6821/1946236732","heartbeat_back_addr":"172.21.15.29:6825/1946236732","heartbeat_front_addr":"172.21.15.29:6823/1946236732","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6827","nonce":4104584972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6829","nonce":4104584972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6833","nonce":4104584972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":4104584972},{"type":"v1","addr":"172.21.15.29:6831","nonce":4104584972}]},"public_addr":"172.21.15.29:6827/4104584972","cluster_addr":"172.21.15.29:6829/4104584972","heartbeat_back_addr":"172.21.15.29:6833/4104584972","heartbeat_front_addr":"172.21.15.29:6831/4104584972","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6801","nonce":2970504758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6803","nonce":2970504758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6807","nonce":2970504758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":2970504758},{"type":"v1","addr":"172.21.15.87:6805","nonce":2970504758}]},"public_addr":"172.21.15.87:6801/2970504758","cluster_addr":"172.21.15.87:6803/2970504758","heartbeat_back_addr":"172.21.15.87:6807/2970504758","heartbeat_front_addr":"172.21.15.87:6805/2970504758","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6809","nonce":3642701185}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6811","nonce":3642701185}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6815","nonce":3642701185}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6813","nonce":3642701185}]},"public_addr":"172.21.15.87:6809/3642701185","cluster_addr":"172.21.15.87:6811/3642701185","heartbeat_back_addr":"172.21.15.87:6815/3642701185","heartbeat_front_addr":"172.21.15.87:6813/3642701185","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2946468060":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6800/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/364858296":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/3254157714":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/659258741":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T11:13:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:14 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2363389649' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:13:14.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:14 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2363389649' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:13:14.794 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:14 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2363389649' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:13:14.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-12T11:13:14.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-12T11:13:14.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-12T11:13:14.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-12T11:13:14.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-12T11:13:14.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-12T11:13:14.818 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-12T11:13:14.818 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-12T11:13:15.275 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:15.275 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:15.276 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:15.277 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:15.277 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:15.282 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:15.287 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:15.312 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:15.736 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:15 smithi029 ceph-mon[39999]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:15.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:15 smithi029 ceph-mon[29349]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:15 smithi087 ceph-mon[32917]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:16.605 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:16 smithi087 ceph-mon[32917]: mds.? [v2:172.21.15.87:6832/2304329976,v1:172.21.15.87:6833/2304329976] up:boot 2024-04-12T11:13:16.605 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:16 smithi087 ceph-mon[32917]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:16.605 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:16 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:13:16.605 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:16 smithi087 ceph-mon[32917]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:16.605 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:16 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.605 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:16 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.605 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:16 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.605 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:16 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.605 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:16 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.606 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:16 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:13:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[29349]: mds.? [v2:172.21.15.87:6832/2304329976,v1:172.21.15.87:6833/2304329976] up:boot 2024-04-12T11:13:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:13:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[39999]: mds.? [v2:172.21.15.87:6832/2304329976,v1:172.21.15.87:6833/2304329976] up:boot 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[39999]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[39999]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:16 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:13:17.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:17 smithi029 ceph-mon[39999]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:17.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:17 smithi029 ceph-mon[39999]: mds.? [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] up:active 2024-04-12T11:13:17.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:17 smithi029 ceph-mon[39999]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:17.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:17 smithi029 ceph-mon[29349]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:17.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:17 smithi029 ceph-mon[29349]: mds.? [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] up:active 2024-04-12T11:13:17.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:17 smithi029 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:17 smithi087 ceph-mon[32917]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:17 smithi087 ceph-mon[32917]: mds.? [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] up:active 2024-04-12T11:13:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:17 smithi087 ceph-mon[32917]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:19.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:19 smithi087 ceph-mon[32917]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:19.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:19 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:19 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:19 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:19.485 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:19 smithi029 ceph-mon[39999]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:19.485 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:19 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:19.485 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:19 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:19.485 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:19 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:19.485 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:19 smithi029 ceph-mon[29349]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:19.485 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:19 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:19.485 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:19 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:19.486 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:19 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:21.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:21 smithi087 ceph-mon[32917]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:21 smithi087 ceph-mon[32917]: mds.? [v2:172.21.15.87:6832/2304329976,v1:172.21.15.87:6833/2304329976] up:standby 2024-04-12T11:13:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:21 smithi087 ceph-mon[32917]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:21 smithi029 ceph-mon[29349]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:21 smithi029 ceph-mon[29349]: mds.? [v2:172.21.15.87:6832/2304329976,v1:172.21.15.87:6833/2304329976] up:standby 2024-04-12T11:13:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:21 smithi029 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:21.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:21 smithi029 ceph-mon[39999]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-04-12T11:13:21.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:21 smithi029 ceph-mon[39999]: mds.? [v2:172.21.15.87:6832/2304329976,v1:172.21.15.87:6833/2304329976] up:standby 2024-04-12T11:13:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:21 smithi029 ceph-mon[39999]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:13:23.416 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:23 smithi029 ceph-mon[39999]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:13:23.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:23 smithi029 ceph-mon[29349]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:13:23.417 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:13:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:13:23] "GET /metrics HTTP/1.1" 200 33643 "" "Prometheus/2.43.0" 2024-04-12T11:13:23.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:23 smithi087 ceph-mon[32917]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:13:25.463 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:25 smithi029 ceph-mon[39999]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:13:25.463 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:25 smithi029 ceph-mon[29349]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:13:25.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:25 smithi087 ceph-mon[32917]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:13:26.155 INFO:teuthology.orchestra.run.smithi029.stdout:231928234011 2024-04-12T11:13:26.155 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-12T11:13:27.011 INFO:teuthology.orchestra.run.smithi029.stdout:77309411379 2024-04-12T11:13:27.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-12T11:13:27.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:27 smithi029 ceph-mon[29349]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:27.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:27 smithi029 ceph-mon[39999]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:27.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:27 smithi087 ceph-mon[32917]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:28.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:28 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:28.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:28 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:28.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:28 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:29.359 INFO:teuthology.orchestra.run.smithi029.stdout:197568495649 2024-04-12T11:13:29.359 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-12T11:13:29.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:29 smithi029 ceph-mon[39999]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:29.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:29 smithi029 ceph-mon[29349]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:29.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:29 smithi087 ceph-mon[32917]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:30.397 INFO:teuthology.orchestra.run.smithi029.stdout:163208757285 2024-04-12T11:13:30.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-12T11:13:31.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:31 smithi029 ceph-mon[29349]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:31.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:31 smithi029 ceph-mon[39999]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:31.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:31 smithi087 ceph-mon[32917]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:31.875 INFO:teuthology.orchestra.run.smithi029.stdout:107374182448 2024-04-12T11:13:31.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-12T11:13:32.441 INFO:teuthology.orchestra.run.smithi029.stdout:34359738430 2024-04-12T11:13:32.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-12T11:13:33.121 INFO:teuthology.orchestra.run.smithi029.stdout:128849018923 2024-04-12T11:13:33.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-12T11:13:33.334 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:33 smithi029 ceph-mon[39999]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:33.334 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:33 smithi029 ceph-mon[29349]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:33.335 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:13:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:13:33] "GET /metrics HTTP/1.1" 200 33892 "" "Prometheus/2.43.0" 2024-04-12T11:13:33.487 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:33.487 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:33.490 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:33.490 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:33.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:33 smithi087 ceph-mon[32917]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:34.244 INFO:teuthology.orchestra.run.smithi029.stdout:55834574905 2024-04-12T11:13:34.245 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-12T11:13:35.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:35 smithi029 ceph-mon[29349]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:35.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:35 smithi029 ceph-mon[39999]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:35.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:35 smithi087 ceph-mon[32917]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:36.220 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:37.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:37 smithi029 ceph-mon[29349]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:37.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:37 smithi029 ceph-mon[39999]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:37.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:37 smithi087 ceph-mon[32917]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:37.987 INFO:teuthology.orchestra.run.smithi029.stdout:77309411381 2024-04-12T11:13:38.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:38 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/286418355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T11:13:38.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:38 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/286418355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T11:13:38.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:38 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/286418355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T11:13:39.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:39 smithi029 ceph-mon[29349]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:39.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:39 smithi029 ceph-mon[39999]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:39.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:39 smithi087 ceph-mon[32917]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:40.766 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411379 got 77309411381 for osd.2 2024-04-12T11:13:40.766 DEBUG:teuthology.parallel:result is None 2024-04-12T11:13:41.193 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:41.194 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:41.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:41 smithi029 ceph-mon[29349]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:41.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:41 smithi029 ceph-mon[39999]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:41.422 INFO:teuthology.orchestra.run.smithi029.stdout:231928234015 2024-04-12T11:13:41.631 INFO:teuthology.orchestra.run.smithi029.stdout:197568495652 2024-04-12T11:13:41.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:41 smithi087 ceph-mon[32917]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:42.377 INFO:teuthology.orchestra.run.smithi029.stdout:163208757288 2024-04-12T11:13:42.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:42 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/704866386' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T11:13:42.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:42 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1110279817' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T11:13:42.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:42 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/704866386' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T11:13:42.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:42 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1110279817' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T11:13:42.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:42 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/704866386' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T11:13:42.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:42 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1110279817' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T11:13:43.240 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:13:43 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:13:43] "GET /metrics HTTP/1.1" 200 33892 "" "Prometheus/2.43.0" 2024-04-12T11:13:43.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:43 smithi029 ceph-mon[29349]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:43.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:43 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1616605461' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T11:13:43.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:43 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:43.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:43 smithi029 ceph-mon[39999]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:43.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:43 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1616605461' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T11:13:43.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:43 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:43.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:43 smithi087 ceph-mon[32917]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:43 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1616605461' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T11:13:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:43 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:44.399 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234011 got 231928234015 for osd.7 2024-04-12T11:13:44.399 DEBUG:teuthology.parallel:result is None 2024-04-12T11:13:45.126 INFO:teuthology.orchestra.run.smithi029.stdout:107374182451 2024-04-12T11:13:45.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:45 smithi029 ceph-mon[29349]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:45.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:45 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2206641486' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T11:13:45.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:45 smithi029 ceph-mon[39999]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:45.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:45 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2206641486' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T11:13:45.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:45 smithi087 ceph-mon[32917]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:45.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:45 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2206641486' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T11:13:47.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:47 smithi029 ceph-mon[29349]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:47.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:47 smithi029 ceph-mon[39999]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:47.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:47 smithi087 ceph-mon[32917]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:49 smithi029 ceph-mon[29349]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:49.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:49 smithi029 ceph-mon[39999]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:49.518 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:49 smithi087 ceph-mon[32917]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:49.986 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182448 got 107374182451 for osd.3 2024-04-12T11:13:49.986 DEBUG:teuthology.parallel:result is None 2024-04-12T11:13:50.645 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757285 got 163208757288 for osd.5 2024-04-12T11:13:50.646 DEBUG:teuthology.parallel:result is None 2024-04-12T11:13:50.647 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:13:51.043 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495649 got 197568495652 for osd.6 2024-04-12T11:13:51.044 DEBUG:teuthology.parallel:result is None 2024-04-12T11:13:51.454 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:51 smithi029 ceph-mon[39999]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:51.455 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:51 smithi029 ceph-mon[29349]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:51.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:51 smithi087 ceph-mon[32917]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:52.646 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:52.646 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:52.646 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[39999]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[39999]: Reconfiguring daemon alertmanager.a on smithi029 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[29349]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-12T11:13:52.647 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:52 smithi029 ceph-mon[29349]: Reconfiguring daemon alertmanager.a on smithi029 2024-04-12T11:13:52.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:52 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:52 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:52 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:13:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:52 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:13:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:52 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:13:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:52 smithi087 ceph-mon[32917]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-12T11:13:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:52 smithi087 ceph-mon[32917]: Reconfiguring daemon alertmanager.a on smithi029 2024-04-12T11:13:53.220 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:13:53 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:13:53] "GET /metrics HTTP/1.1" 200 33888 "" "Prometheus/2.43.0" 2024-04-12T11:13:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:53 smithi029 ceph-mon[39999]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:54.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:53 smithi029 ceph-mon[29349]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:54.081 INFO:teuthology.orchestra.run.smithi029.stdout:34359738435 2024-04-12T11:13:54.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:53 smithi087 ceph-mon[32917]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:54 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2549612850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T11:13:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:54 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2549612850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T11:13:55.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:54 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2549612850' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T11:13:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:55 smithi029 ceph-mon[29349]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:55 smithi029 ceph-mon[39999]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:56.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:55 smithi087 ceph-mon[32917]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:56.843 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738430 got 34359738435 for osd.0 2024-04-12T11:13:56.843 DEBUG:teuthology.parallel:result is None 2024-04-12T11:13:57.518 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:13:57 smithi029 systemd[1]: Stopping Ceph alertmanager.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:13:58.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:57 smithi029 ceph-mon[29349]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:58.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:57 smithi029 ceph-mon[39999]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:58.372 INFO:teuthology.orchestra.run.smithi029.stdout:128849018928 2024-04-12T11:13:58.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:57 smithi087 ceph-mon[32917]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:13:59.107 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018923 got 128849018928 for osd.4 2024-04-12T11:13:59.107 DEBUG:teuthology.parallel:result is None 2024-04-12T11:13:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-04-12T11:13:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-12T11:13:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-12T11:13:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:58 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1222812832' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T11:13:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-04-12T11:13:59.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-12T11:13:59.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-12T11:13:59.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:59.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:58 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1222812832' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T11:13:59.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-04-12T11:13:59.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-12T11:13:59.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-12T11:13:59.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:13:59.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:58 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1222812832' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T11:13:59.720 INFO:teuthology.orchestra.run.smithi029.stdout:55834574910 2024-04-12T11:13:59.720 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[70909]: ts=2024-04-12T11:13:59.599Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-12T11:13:59.720 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:13:59 smithi029 podman[77320]: 2024-04-12 11:13:59.62005013 +0000 UTC m=+0.446249859 container died f28dbb25cb2d6a0e42493e453b49326706b9ecde18eb6ba19a4679ec408a5918 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:00.268 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:00 smithi029 podman[77320]: 2024-04-12 11:14:00.004202574 +0000 UTC m=+0.830402299 container cleanup f28dbb25cb2d6a0e42493e453b49326706b9ecde18eb6ba19a4679ec408a5918 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:00.268 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:00 smithi029 bash[77320]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a 2024-04-12T11:14:00.268 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:00 smithi029 podman[77357]: 2024-04-12 11:14:00.240132901 +0000 UTC m=+0.616334554 container remove f28dbb25cb2d6a0e42493e453b49326706b9ecde18eb6ba19a4679ec408a5918 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[29349]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[29349]: osdmap e69: 8 total, 8 up, 8 in 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/139734746' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[39999]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[39999]: osdmap e69: 8 total, 8 up, 8 in 2024-04-12T11:14:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:13:59 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/139734746' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T11:14:00.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:59 smithi087 ceph-mon[32917]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:00.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:59 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2024-04-12T11:14:00.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:59 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-12T11:14:00.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:59 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-12T11:14:00.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:59 smithi087 ceph-mon[32917]: osdmap e69: 8 total, 8 up, 8 in 2024-04-12T11:14:00.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:13:59 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/139734746' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T11:14:00.638 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:00 smithi029 podman[77357]: 2024-04-12 11:14:00.341472677 +0000 UTC m=+0.717674328 volume remove e6aff1502f4d3563d343caf035596b5eda5134c26606d44495a4c5cbd25429ff 2024-04-12T11:14:00.962 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574905 got 55834574910 for osd.1 2024-04-12T11:14:00.962 DEBUG:teuthology.parallel:result is None 2024-04-12T11:14:00.962 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-12T11:14:00.962 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:14:01.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:00 smithi029 ceph-mon[29349]: osdmap e70: 8 total, 8 up, 8 in 2024-04-12T11:14:01.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:00 smithi029 ceph-mon[39999]: osdmap e70: 8 total, 8 up, 8 in 2024-04-12T11:14:01.198 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:00 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-12T11:14:01.198 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:00 smithi029 systemd[1]: Stopped Ceph alertmanager.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:14:01.198 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:00 smithi029 systemd[1]: Starting Ceph alertmanager.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:14:01.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:00 smithi087 ceph-mon[32917]: osdmap e70: 8 total, 8 up, 8 in 2024-04-12T11:14:01.482 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 podman[77484]: 2024-04-12 11:14:01.101629245 +0000 UTC m=+0.018847714 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-12T11:14:01.482 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 podman[77484]: 2024-04-12 11:14:01.204630513 +0000 UTC m=+0.121849023 volume create ed109fb9dd2ad9e389d5fe825664a413b61df8d12f1a530d4487886def9dbfdd 2024-04-12T11:14:01.482 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 podman[77484]: 2024-04-12 11:14:01.313967734 +0000 UTC m=+0.231186204 container create 5f36b9c37cec222056a033e8e274cf62192e7cb3a466cd8cf261d35bdbc305c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:01.817 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:01.938 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 podman[77484]: 2024-04-12 11:14:01.593405817 +0000 UTC m=+0.510624293 container init 5f36b9c37cec222056a033e8e274cf62192e7cb3a466cd8cf261d35bdbc305c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:01.938 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 podman[77484]: 2024-04-12 11:14:01.596093948 +0000 UTC m=+0.513312417 container start 5f36b9c37cec222056a033e8e274cf62192e7cb3a466cd8cf261d35bdbc305c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:01.938 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T11:14:01.607Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-12T11:14:01.938 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T11:14:01.607Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-12T11:14:01.939 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T11:14:01.609Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.29 port=9094 2024-04-12T11:14:01.939 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T11:14:01.610Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-12T11:14:01.939 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T11:14:01.633Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-12T11:14:01.939 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T11:14:01.633Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-12T11:14:01.939 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T11:14:01.636Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-12T11:14:01.939 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T11:14:01.636Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-12T11:14:01.939 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 bash[77484]: 5f36b9c37cec222056a033e8e274cf62192e7cb3a466cd8cf261d35bdbc305c2 2024-04-12T11:14:01.939 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:01 smithi029 systemd[1]: Started Ceph alertmanager.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:14:02.261 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-mon[39999]: pgmap v71: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:02.261 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:02.261 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:02.261 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-mon[29349]: pgmap v71: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:02.261 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:02.261 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:02.436 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:01 smithi087 ceph-mon[32917]: pgmap v71: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:02.436 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:02.437 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:03 smithi087 ceph-mon[32917]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T11:14:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:03 smithi087 ceph-mon[32917]: Reconfiguring daemon prometheus.a on smithi087 2024-04-12T11:14:03.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:03 smithi029 ceph-mon[39999]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T11:14:03.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:03 smithi029 ceph-mon[39999]: Reconfiguring daemon prometheus.a on smithi087 2024-04-12T11:14:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:03 smithi029 ceph-mon[29349]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T11:14:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:03 smithi029 ceph-mon[29349]: Reconfiguring daemon prometheus.a on smithi087 2024-04-12T11:14:03.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:14:03] "GET /metrics HTTP/1.1" 200 33893 "" "Prometheus/2.43.0" 2024-04-12T11:14:03.729 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:14:03.731 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:14:04.004 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T11:14:03.610Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000363014s 2024-04-12T11:14:04.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:04 smithi087 ceph-mon[32917]: pgmap v72: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:04.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:04 smithi087 ceph-mon[32917]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-04-12T11:14:04.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:04 smithi029 ceph-mon[29349]: pgmap v72: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:04.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:04 smithi029 ceph-mon[29349]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-04-12T11:14:04.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:04 smithi029 ceph-mon[39999]: pgmap v72: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:04.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:04 smithi029 ceph-mon[39999]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-04-12T11:14:04.436 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":72,"stamp":"2024-04-12T11:14:02.143401+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592730,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":231596,"kb_used_data":7272,"kb_used_omap":12,"kb_used_meta":224243,"kb_avail":749729620,"statfs":{"total":767960285184,"available":767723130880,"internally_reserved":0,"allocated":7446528,"data_stored":4181648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":229625401},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001522"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169003+0000","last_change":"2024-04-12T11:12:58.787924+0000","last_active":"2024-04-12T11:13:00.169003+0000","last_peered":"2024-04-12T11:13:00.169003+0000","last_clean":"2024-04-12T11:13:00.169003+0000","last_became_active":"2024-04-12T11:12:58.787657+0000","last_became_peered":"2024-04-12T11:12:58.787657+0000","last_unstale":"2024-04-12T11:13:00.169003+0000","last_undegraded":"2024-04-12T11:13:00.169003+0000","last_fullsized":"2024-04-12T11:13:00.169003+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:19:10.702420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.536970+0000","last_change":"2024-04-12T11:12:57.768925+0000","last_active":"2024-04-12T11:13:08.536970+0000","last_peered":"2024-04-12T11:13:08.536970+0000","last_clean":"2024-04-12T11:13:08.536970+0000","last_became_active":"2024-04-12T11:12:57.768245+0000","last_became_peered":"2024-04-12T11:12:57.768245+0000","last_unstale":"2024-04-12T11:13:08.536970+0000","last_undegraded":"2024-04-12T11:13:08.536970+0000","last_fullsized":"2024-04-12T11:13:08.536970+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:40:15.574078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169688+0000","last_change":"2024-04-12T11:12:59.271181+0000","last_active":"2024-04-12T11:13:00.169688+0000","last_peered":"2024-04-12T11:13:00.169688+0000","last_clean":"2024-04-12T11:13:00.169688+0000","last_became_active":"2024-04-12T11:12:59.270640+0000","last_became_peered":"2024-04-12T11:12:59.270640+0000","last_unstale":"2024-04-12T11:13:00.169688+0000","last_undegraded":"2024-04-12T11:13:00.169688+0000","last_fullsized":"2024-04-12T11:13:00.169688+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:55:12.317879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.539332+0000","last_change":"2024-04-12T11:12:57.768614+0000","last_active":"2024-04-12T11:13:08.539332+0000","last_peered":"2024-04-12T11:13:08.539332+0000","last_clean":"2024-04-12T11:13:08.539332+0000","last_became_active":"2024-04-12T11:12:57.768323+0000","last_became_peered":"2024-04-12T11:12:57.768323+0000","last_unstale":"2024-04-12T11:13:08.539332+0000","last_undegraded":"2024-04-12T11:13:08.539332+0000","last_fullsized":"2024-04-12T11:13:08.539332+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:41.849038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.170005+0000","last_change":"2024-04-12T11:12:58.785574+0000","last_active":"2024-04-12T11:13:00.170005+0000","last_peered":"2024-04-12T11:13:00.170005+0000","last_clean":"2024-04-12T11:13:00.170005+0000","last_became_active":"2024-04-12T11:12:58.785281+0000","last_became_peered":"2024-04-12T11:12:58.785281+0000","last_unstale":"2024-04-12T11:13:00.170005+0000","last_undegraded":"2024-04-12T11:13:00.170005+0000","last_fullsized":"2024-04-12T11:13:00.170005+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:48:49.798531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.539624+0000","last_change":"2024-04-12T11:12:57.760870+0000","last_active":"2024-04-12T11:13:08.539624+0000","last_peered":"2024-04-12T11:13:08.539624+0000","last_clean":"2024-04-12T11:13:08.539624+0000","last_became_active":"2024-04-12T11:12:57.760587+0000","last_became_peered":"2024-04-12T11:12:57.760587+0000","last_unstale":"2024-04-12T11:13:08.539624+0000","last_undegraded":"2024-04-12T11:13:08.539624+0000","last_fullsized":"2024-04-12T11:13:08.539624+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:00:14.733542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2024-04-12T11:13:58.988375+0000","last_change":"2024-04-12T11:13:58.988521+0000","last_active":"2024-04-12T11:13:58.988375+0000","last_peered":"2024-04-12T11:13:00.170576+0000","last_clean":"2024-04-12T11:13:00.170576+0000","last_became_active":"2024-04-12T11:12:58.787713+0000","last_became_peered":"2024-04-12T11:12:58.787713+0000","last_unstale":"2024-04-12T11:13:58.988375+0000","last_undegraded":"2024-04-12T11:13:58.988375+0000","last_fullsized":"2024-04-12T11:13:58.988375+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:14:24.242715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,3],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.536853+0000","last_change":"2024-04-12T11:12:57.760302+0000","last_active":"2024-04-12T11:13:08.536853+0000","last_peered":"2024-04-12T11:13:08.536853+0000","last_clean":"2024-04-12T11:13:08.536853+0000","last_became_active":"2024-04-12T11:12:57.760005+0000","last_became_peered":"2024-04-12T11:12:57.760005+0000","last_unstale":"2024-04-12T11:13:08.536853+0000","last_undegraded":"2024-04-12T11:13:08.536853+0000","last_fullsized":"2024-04-12T11:13:08.536853+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:03:33.678493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.780448+0000","last_change":"2024-04-12T11:12:57.780520+0000","last_active":"2024-04-12T11:12:59.780448+0000","last_peered":"2024-04-12T11:12:59.780448+0000","last_clean":"2024-04-12T11:12:59.780448+0000","last_became_active":"2024-04-12T11:12:57.780227+0000","last_became_peered":"2024-04-12T11:12:57.780227+0000","last_unstale":"2024-04-12T11:12:59.780448+0000","last_undegraded":"2024-04-12T11:12:59.780448+0000","last_fullsized":"2024-04-12T11:12:59.780448+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:20:26.510304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.170150+0000","last_change":"2024-04-12T11:12:57.765674+0000","last_active":"2024-04-12T11:13:00.170150+0000","last_peered":"2024-04-12T11:13:00.170150+0000","last_clean":"2024-04-12T11:13:00.170150+0000","last_became_active":"2024-04-12T11:12:57.765384+0000","last_became_peered":"2024-04-12T11:12:57.765384+0000","last_unstale":"2024-04-12T11:13:00.170150+0000","last_undegraded":"2024-04-12T11:13:00.170150+0000","last_fullsized":"2024-04-12T11:13:00.170150+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:14:21.536217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169361+0000","last_change":"2024-04-12T11:12:58.786647+0000","last_active":"2024-04-12T11:13:00.169361+0000","last_peered":"2024-04-12T11:13:00.169361+0000","last_clean":"2024-04-12T11:13:00.169361+0000","last_became_active":"2024-04-12T11:12:58.786350+0000","last_became_peered":"2024-04-12T11:12:58.786350+0000","last_unstale":"2024-04-12T11:13:00.169361+0000","last_undegraded":"2024-04-12T11:13:00.169361+0000","last_fullsized":"2024-04-12T11:13:00.169361+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:44:23.801172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.170051+0000","last_change":"2024-04-12T11:12:57.763606+0000","last_active":"2024-04-12T11:13:00.170051+0000","last_peered":"2024-04-12T11:13:00.170051+0000","last_clean":"2024-04-12T11:13:00.170051+0000","last_became_active":"2024-04-12T11:12:57.763299+0000","last_became_peered":"2024-04-12T11:12:57.763299+0000","last_unstale":"2024-04-12T11:13:00.170051+0000","last_undegraded":"2024-04-12T11:13:00.170051+0000","last_fullsized":"2024-04-12T11:13:00.170051+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:24:22.305524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.988485+0000","last_change":"2024-04-12T11:11:49.523873+0000","last_active":"2024-04-12T11:13:58.988485+0000","last_peered":"2024-04-12T11:13:58.988485+0000","last_clean":"2024-04-12T11:13:58.988485+0000","last_became_active":"2024-04-12T11:11:49.523570+0000","last_became_peered":"2024-04-12T11:11:49.523570+0000","last_unstale":"2024-04-12T11:13:58.988485+0000","last_undegraded":"2024-04-12T11:13:58.988485+0000","last_fullsized":"2024-04-12T11:13:58.988485+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:09:00.491446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169401+0000","last_change":"2024-04-12T11:12:59.163787+0000","last_active":"2024-04-12T11:13:00.169401+0000","last_peered":"2024-04-12T11:13:00.169401+0000","last_clean":"2024-04-12T11:13:00.169401+0000","last_became_active":"2024-04-12T11:12:59.163457+0000","last_became_peered":"2024-04-12T11:12:59.163457+0000","last_unstale":"2024-04-12T11:13:00.169401+0000","last_undegraded":"2024-04-12T11:13:00.169401+0000","last_fullsized":"2024-04-12T11:13:00.169401+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:21.459548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.782272+0000","last_change":"2024-04-12T11:11:49.882400+0000","last_active":"2024-04-12T11:12:59.782272+0000","last_peered":"2024-04-12T11:12:59.782272+0000","last_clean":"2024-04-12T11:12:59.782272+0000","last_became_active":"2024-04-12T11:11:49.882073+0000","last_became_peered":"2024-04-12T11:11:49.882073+0000","last_unstale":"2024-04-12T11:12:59.782272+0000","last_undegraded":"2024-04-12T11:12:59.782272+0000","last_fullsized":"2024-04-12T11:12:59.782272+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:38:46.841744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.538345+0000","last_change":"2024-04-12T11:12:57.770877+0000","last_active":"2024-04-12T11:13:08.538345+0000","last_peered":"2024-04-12T11:13:08.538345+0000","last_clean":"2024-04-12T11:13:08.538345+0000","last_became_active":"2024-04-12T11:12:57.770217+0000","last_became_peered":"2024-04-12T11:12:57.770217+0000","last_unstale":"2024-04-12T11:13:08.538345+0000","last_undegraded":"2024-04-12T11:13:08.538345+0000","last_fullsized":"2024-04-12T11:13:08.538345+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:25:07.457621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.164504+0000","last_change":"2024-04-12T11:12:59.163973+0000","last_active":"2024-04-12T11:12:59.164504+0000","last_peered":"2024-04-12T11:12:59.164504+0000","last_clean":"2024-04-12T11:12:59.164504+0000","last_became_active":"2024-04-12T11:12:59.163648+0000","last_became_peered":"2024-04-12T11:12:59.163648+0000","last_unstale":"2024-04-12T11:12:59.164504+0000","last_undegraded":"2024-04-12T11:12:59.164504+0000","last_fullsized":"2024-04-12T11:12:59.164504+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:26:50.345253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.781778+0000","last_change":"2024-04-12T11:12:57.756170+0000","last_active":"2024-04-12T11:12:59.781778+0000","last_peered":"2024-04-12T11:12:59.781778+0000","last_clean":"2024-04-12T11:12:59.781778+0000","last_became_active":"2024-04-12T11:12:57.755885+0000","last_became_peered":"2024-04-12T11:12:57.755885+0000","last_unstale":"2024-04-12T11:12:59.781778+0000","last_undegraded":"2024-04-12T11:12:59.781778+0000","last_fullsized":"2024-04-12T11:12:59.781778+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:08:27.183396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.781143+0000","last_change":"2024-04-12T11:11:49.882324+0000","last_active":"2024-04-12T11:12:59.781143+0000","last_peered":"2024-04-12T11:12:59.781143+0000","last_clean":"2024-04-12T11:12:59.781143+0000","last_became_active":"2024-04-12T11:11:49.881975+0000","last_became_peered":"2024-04-12T11:11:49.881975+0000","last_unstale":"2024-04-12T11:12:59.781143+0000","last_undegraded":"2024-04-12T11:12:59.781143+0000","last_fullsized":"2024-04-12T11:12:59.781143+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:10.140250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.271232+0000","last_change":"2024-04-12T11:12:59.270863+0000","last_active":"2024-04-12T11:12:59.271232+0000","last_peered":"2024-04-12T11:12:59.271232+0000","last_clean":"2024-04-12T11:12:59.271232+0000","last_became_active":"2024-04-12T11:12:59.270555+0000","last_became_peered":"2024-04-12T11:12:59.270555+0000","last_unstale":"2024-04-12T11:12:59.271232+0000","last_undegraded":"2024-04-12T11:12:59.271232+0000","last_fullsized":"2024-04-12T11:12:59.271232+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:28.109904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.125133+0000","last_change":"2024-04-12T11:12:57.767163+0000","last_active":"2024-04-12T11:12:59.125133+0000","last_peered":"2024-04-12T11:12:59.125133+0000","last_clean":"2024-04-12T11:12:59.125133+0000","last_became_active":"2024-04-12T11:12:57.766842+0000","last_became_peered":"2024-04-12T11:12:57.766842+0000","last_unstale":"2024-04-12T11:12:59.125133+0000","last_undegraded":"2024-04-12T11:12:59.125133+0000","last_fullsized":"2024-04-12T11:12:59.125133+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:50.322014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.780880+0000","last_change":"2024-04-12T11:11:49.524657+0000","last_active":"2024-04-12T11:12:59.780880+0000","last_peered":"2024-04-12T11:12:59.780880+0000","last_clean":"2024-04-12T11:12:59.780880+0000","last_became_active":"2024-04-12T11:11:49.524379+0000","last_became_peered":"2024-04-12T11:11:49.524379+0000","last_unstale":"2024-04-12T11:12:59.780880+0000","last_undegraded":"2024-04-12T11:12:59.780880+0000","last_fullsized":"2024-04-12T11:12:59.780880+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:30:52.444470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169703+0000","last_change":"2024-04-12T11:12:58.173490+0000","last_active":"2024-04-12T11:13:00.169703+0000","last_peered":"2024-04-12T11:13:00.169703+0000","last_clean":"2024-04-12T11:13:00.169703+0000","last_became_active":"2024-04-12T11:12:58.173170+0000","last_became_peered":"2024-04-12T11:12:58.173170+0000","last_unstale":"2024-04-12T11:13:00.169703+0000","last_undegraded":"2024-04-12T11:13:00.169703+0000","last_fullsized":"2024-04-12T11:13:00.169703+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:29.680553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169693+0000","last_change":"2024-04-12T11:12:57.766887+0000","last_active":"2024-04-12T11:13:00.169693+0000","last_peered":"2024-04-12T11:13:00.169693+0000","last_clean":"2024-04-12T11:13:00.169693+0000","last_became_active":"2024-04-12T11:12:57.766494+0000","last_became_peered":"2024-04-12T11:12:57.766494+0000","last_unstale":"2024-04-12T11:13:00.169693+0000","last_undegraded":"2024-04-12T11:13:00.169693+0000","last_fullsized":"2024-04-12T11:13:00.169693+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:47:47.566185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":264,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.780787+0000","last_change":"2024-04-12T11:11:24.390399+0000","last_active":"2024-04-12T11:12:59.780787+0000","last_peered":"2024-04-12T11:12:59.780787+0000","last_clean":"2024-04-12T11:12:59.780787+0000","last_became_active":"2024-04-12T11:11:24.389943+0000","last_became_peered":"2024-04-12T11:11:24.389943+0000","last_unstale":"2024-04-12T11:12:59.780787+0000","last_undegraded":"2024-04-12T11:12:59.780787+0000","last_fullsized":"2024-04-12T11:12:59.780787+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:06:44.231202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169532+0000","last_change":"2024-04-12T11:11:49.524628+0000","last_active":"2024-04-12T11:13:00.169532+0000","last_peered":"2024-04-12T11:13:00.169532+0000","last_clean":"2024-04-12T11:13:00.169532+0000","last_became_active":"2024-04-12T11:11:49.524226+0000","last_became_peered":"2024-04-12T11:11:49.524226+0000","last_unstale":"2024-04-12T11:13:00.169532+0000","last_undegraded":"2024-04-12T11:13:00.169532+0000","last_fullsized":"2024-04-12T11:13:00.169532+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:43:26.148427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.782403+0000","last_change":"2024-04-12T11:12:57.973543+0000","last_active":"2024-04-12T11:12:59.782403+0000","last_peered":"2024-04-12T11:12:59.782403+0000","last_clean":"2024-04-12T11:12:59.782403+0000","last_became_active":"2024-04-12T11:12:57.973234+0000","last_became_peered":"2024-04-12T11:12:57.973234+0000","last_unstale":"2024-04-12T11:12:59.782403+0000","last_undegraded":"2024-04-12T11:12:59.782403+0000","last_fullsized":"2024-04-12T11:12:59.782403+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:42:39.697895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.781212+0000","last_change":"2024-04-12T11:12:57.759232+0000","last_active":"2024-04-12T11:12:59.781212+0000","last_peered":"2024-04-12T11:12:59.781212+0000","last_clean":"2024-04-12T11:12:59.781212+0000","last_became_active":"2024-04-12T11:12:57.758748+0000","last_became_peered":"2024-04-12T11:12:57.758748+0000","last_unstale":"2024-04-12T11:12:59.781212+0000","last_undegraded":"2024-04-12T11:12:59.781212+0000","last_fullsized":"2024-04-12T11:12:59.781212+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:21:31.374347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.780322+0000","last_change":"2024-04-12T11:11:49.523432+0000","last_active":"2024-04-12T11:12:59.780322+0000","last_peered":"2024-04-12T11:12:59.780322+0000","last_clean":"2024-04-12T11:12:59.780322+0000","last_became_active":"2024-04-12T11:11:49.523059+0000","last_became_peered":"2024-04-12T11:11:49.523059+0000","last_unstale":"2024-04-12T11:12:59.780322+0000","last_undegraded":"2024-04-12T11:12:59.780322+0000","last_fullsized":"2024-04-12T11:12:59.780322+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:55:25.692863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.781732+0000","last_change":"2024-04-12T11:12:57.780685+0000","last_active":"2024-04-12T11:12:59.781732+0000","last_peered":"2024-04-12T11:12:59.781732+0000","last_clean":"2024-04-12T11:12:59.781732+0000","last_became_active":"2024-04-12T11:12:57.780396+0000","last_became_peered":"2024-04-12T11:12:57.780396+0000","last_unstale":"2024-04-12T11:12:59.781732+0000","last_undegraded":"2024-04-12T11:12:59.781732+0000","last_fullsized":"2024-04-12T11:12:59.781732+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:41:06.039444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.170441+0000","last_change":"2024-04-12T11:12:57.760336+0000","last_active":"2024-04-12T11:13:00.170441+0000","last_peered":"2024-04-12T11:13:00.170441+0000","last_clean":"2024-04-12T11:13:00.170441+0000","last_became_active":"2024-04-12T11:12:57.760032+0000","last_became_peered":"2024-04-12T11:12:57.760032+0000","last_unstale":"2024-04-12T11:13:00.170441+0000","last_undegraded":"2024-04-12T11:13:00.170441+0000","last_fullsized":"2024-04-12T11:13:00.170441+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:11:54.530608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169811+0000","last_change":"2024-04-12T11:11:49.523947+0000","last_active":"2024-04-12T11:13:00.169811+0000","last_peered":"2024-04-12T11:13:00.169811+0000","last_clean":"2024-04-12T11:13:00.169811+0000","last_became_active":"2024-04-12T11:11:49.523650+0000","last_became_peered":"2024-04-12T11:11:49.523650+0000","last_unstale":"2024-04-12T11:13:00.169811+0000","last_undegraded":"2024-04-12T11:13:00.169811+0000","last_fullsized":"2024-04-12T11:13:00.169811+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:05:22.528981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.781457+0000","last_change":"2024-04-12T11:12:57.990458+0000","last_active":"2024-04-12T11:12:59.781457+0000","last_peered":"2024-04-12T11:12:59.781457+0000","last_clean":"2024-04-12T11:12:59.781457+0000","last_became_active":"2024-04-12T11:12:57.990148+0000","last_became_peered":"2024-04-12T11:12:57.990148+0000","last_unstale":"2024-04-12T11:12:59.781457+0000","last_undegraded":"2024-04-12T11:12:59.781457+0000","last_fullsized":"2024-04-12T11:12:59.781457+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:13:35.647039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169719+0000","last_change":"2024-04-12T11:12:57.768636+0000","last_active":"2024-04-12T11:13:00.169719+0000","last_peered":"2024-04-12T11:13:00.169719+0000","last_clean":"2024-04-12T11:13:00.169719+0000","last_became_active":"2024-04-12T11:12:57.768359+0000","last_became_peered":"2024-04-12T11:12:57.768359+0000","last_unstale":"2024-04-12T11:13:00.169719+0000","last_undegraded":"2024-04-12T11:13:00.169719+0000","last_fullsized":"2024-04-12T11:13:00.169719+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:00:41.208948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.781688+0000","last_change":"2024-04-12T11:11:49.524259+0000","last_active":"2024-04-12T11:12:59.781688+0000","last_peered":"2024-04-12T11:12:59.781688+0000","last_clean":"2024-04-12T11:12:59.781688+0000","last_became_active":"2024-04-12T11:11:49.523641+0000","last_became_peered":"2024-04-12T11:11:49.523641+0000","last_unstale":"2024-04-12T11:12:59.781688+0000","last_undegraded":"2024-04-12T11:12:59.781688+0000","last_fullsized":"2024-04-12T11:12:59.781688+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:28:22.480630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.782403+0000","last_change":"2024-04-12T11:12:57.990460+0000","last_active":"2024-04-12T11:12:59.782403+0000","last_peered":"2024-04-12T11:12:59.782403+0000","last_clean":"2024-04-12T11:12:59.782403+0000","last_became_active":"2024-04-12T11:12:57.990156+0000","last_became_peered":"2024-04-12T11:12:57.990156+0000","last_unstale":"2024-04-12T11:12:59.782403+0000","last_undegraded":"2024-04-12T11:12:59.782403+0000","last_fullsized":"2024-04-12T11:12:59.782403+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:26:24.526812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169398+0000","last_change":"2024-04-12T11:12:57.769233+0000","last_active":"2024-04-12T11:13:00.169398+0000","last_peered":"2024-04-12T11:13:00.169398+0000","last_clean":"2024-04-12T11:13:00.169398+0000","last_became_active":"2024-04-12T11:12:57.768383+0000","last_became_peered":"2024-04-12T11:12:57.768383+0000","last_unstale":"2024-04-12T11:13:00.169398+0000","last_undegraded":"2024-04-12T11:13:00.169398+0000","last_fullsized":"2024-04-12T11:13:00.169398+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:06:42.355317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169866+0000","last_change":"2024-04-12T11:12:58.788638+0000","last_active":"2024-04-12T11:13:00.169866+0000","last_peered":"2024-04-12T11:13:00.169866+0000","last_clean":"2024-04-12T11:13:00.169866+0000","last_became_active":"2024-04-12T11:12:58.788338+0000","last_became_peered":"2024-04-12T11:12:58.788338+0000","last_unstale":"2024-04-12T11:13:00.169866+0000","last_undegraded":"2024-04-12T11:13:00.169866+0000","last_fullsized":"2024-04-12T11:13:00.169866+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:30.910258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169270+0000","last_change":"2024-04-12T11:12:57.766578+0000","last_active":"2024-04-12T11:13:00.169270+0000","last_peered":"2024-04-12T11:13:00.169270+0000","last_clean":"2024-04-12T11:13:00.169270+0000","last_became_active":"2024-04-12T11:12:57.766305+0000","last_became_peered":"2024-04-12T11:12:57.766305+0000","last_unstale":"2024-04-12T11:13:00.169270+0000","last_undegraded":"2024-04-12T11:13:00.169270+0000","last_fullsized":"2024-04-12T11:13:00.169270+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:46:14.183201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.170459+0000","last_change":"2024-04-12T11:12:59.271140+0000","last_active":"2024-04-12T11:13:00.170459+0000","last_peered":"2024-04-12T11:13:00.170459+0000","last_clean":"2024-04-12T11:13:00.170459+0000","last_became_active":"2024-04-12T11:12:59.270866+0000","last_became_peered":"2024-04-12T11:12:59.270866+0000","last_unstale":"2024-04-12T11:13:00.170459+0000","last_undegraded":"2024-04-12T11:13:00.170459+0000","last_fullsized":"2024-04-12T11:13:00.170459+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:23:02.348448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169802+0000","last_change":"2024-04-12T11:12:57.760710+0000","last_active":"2024-04-12T11:13:00.169802+0000","last_peered":"2024-04-12T11:13:00.169802+0000","last_clean":"2024-04-12T11:13:00.169802+0000","last_became_active":"2024-04-12T11:12:57.760454+0000","last_became_peered":"2024-04-12T11:12:57.760454+0000","last_unstale":"2024-04-12T11:13:00.169802+0000","last_undegraded":"2024-04-12T11:13:00.169802+0000","last_fullsized":"2024-04-12T11:13:00.169802+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:03:14.723304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169778+0000","last_change":"2024-04-12T11:12:58.786811+0000","last_active":"2024-04-12T11:13:00.169778+0000","last_peered":"2024-04-12T11:13:00.169778+0000","last_clean":"2024-04-12T11:13:00.169778+0000","last_became_active":"2024-04-12T11:12:58.786423+0000","last_became_peered":"2024-04-12T11:12:58.786423+0000","last_unstale":"2024-04-12T11:13:00.169778+0000","last_undegraded":"2024-04-12T11:13:00.169778+0000","last_fullsized":"2024-04-12T11:13:00.169778+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:26:51.148882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.537682+0000","last_change":"2024-04-12T11:12:57.769995+0000","last_active":"2024-04-12T11:13:08.537682+0000","last_peered":"2024-04-12T11:13:08.537682+0000","last_clean":"2024-04-12T11:13:08.537682+0000","last_became_active":"2024-04-12T11:12:57.769701+0000","last_became_peered":"2024-04-12T11:12:57.769701+0000","last_unstale":"2024-04-12T11:13:08.537682+0000","last_undegraded":"2024-04-12T11:13:08.537682+0000","last_fullsized":"2024-04-12T11:13:08.537682+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:30:52.071458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.125189+0000","last_change":"2024-04-12T11:12:58.785119+0000","last_active":"2024-04-12T11:12:59.125189+0000","last_peered":"2024-04-12T11:12:59.125189+0000","last_clean":"2024-04-12T11:12:59.125189+0000","last_became_active":"2024-04-12T11:12:58.784825+0000","last_became_peered":"2024-04-12T11:12:58.784825+0000","last_unstale":"2024-04-12T11:12:59.125189+0000","last_undegraded":"2024-04-12T11:12:59.125189+0000","last_fullsized":"2024-04-12T11:12:59.125189+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:20:37.720179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.170065+0000","last_change":"2024-04-12T11:12:57.765020+0000","last_active":"2024-04-12T11:13:00.170065+0000","last_peered":"2024-04-12T11:13:00.170065+0000","last_clean":"2024-04-12T11:13:00.170065+0000","last_became_active":"2024-04-12T11:12:57.764740+0000","last_became_peered":"2024-04-12T11:12:57.764740+0000","last_unstale":"2024-04-12T11:13:00.170065+0000","last_undegraded":"2024-04-12T11:13:00.170065+0000","last_fullsized":"2024-04-12T11:13:00.170065+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:31:18.778202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169886+0000","last_change":"2024-04-12T11:12:58.173247+0000","last_active":"2024-04-12T11:13:00.169886+0000","last_peered":"2024-04-12T11:13:00.169886+0000","last_clean":"2024-04-12T11:13:00.169886+0000","last_became_active":"2024-04-12T11:12:58.172953+0000","last_became_peered":"2024-04-12T11:12:58.172953+0000","last_unstale":"2024-04-12T11:13:00.169886+0000","last_undegraded":"2024-04-12T11:13:00.169886+0000","last_fullsized":"2024-04-12T11:13:00.169886+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:26:06.124062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.781243+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:12:59.781243+0000","last_peered":"2024-04-12T11:12:59.781243+0000","last_clean":"2024-04-12T11:12:59.781243+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:12:59.781243+0000","last_undegraded":"2024-04-12T11:12:59.781243+0000","last_fullsized":"2024-04-12T11:12:59.781243+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2024-04-12T11:13:58.988231+0000","last_change":"2024-04-12T11:13:58.988893+0000","last_active":"2024-04-12T11:13:58.988231+0000","last_peered":"2024-04-12T11:13:00.169180+0000","last_clean":"2024-04-12T11:13:00.169180+0000","last_became_active":"2024-04-12T11:12:58.787481+0000","last_became_peered":"2024-04-12T11:12:58.787481+0000","last_unstale":"2024-04-12T11:13:58.988231+0000","last_undegraded":"2024-04-12T11:13:58.988231+0000","last_fullsized":"2024-04-12T11:13:58.988231+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:19:52.940708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,3,5],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.540961+0000","last_change":"2024-04-12T11:12:57.772960+0000","last_active":"2024-04-12T11:13:08.540961+0000","last_peered":"2024-04-12T11:13:08.540961+0000","last_clean":"2024-04-12T11:13:08.540961+0000","last_became_active":"2024-04-12T11:12:57.772624+0000","last_became_peered":"2024-04-12T11:12:57.772624+0000","last_unstale":"2024-04-12T11:13:08.540961+0000","last_undegraded":"2024-04-12T11:13:08.540961+0000","last_fullsized":"2024-04-12T11:13:08.540961+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:38:00.390687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169209+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:13:00.169209+0000","last_peered":"2024-04-12T11:13:00.169209+0000","last_clean":"2024-04-12T11:13:00.169209+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:13:00.169209+0000","last_undegraded":"2024-04-12T11:13:00.169209+0000","last_fullsized":"2024-04-12T11:13:00.169209+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.781202+0000","last_change":"2024-04-12T11:12:57.764312+0000","last_active":"2024-04-12T11:12:59.781202+0000","last_peered":"2024-04-12T11:12:59.781202+0000","last_clean":"2024-04-12T11:12:59.781202+0000","last_became_active":"2024-04-12T11:12:57.763625+0000","last_became_peered":"2024-04-12T11:12:57.763625+0000","last_unstale":"2024-04-12T11:12:59.781202+0000","last_undegraded":"2024-04-12T11:12:59.781202+0000","last_fullsized":"2024-04-12T11:12:59.781202+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:18:28.157847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.170017+0000","last_change":"2024-04-12T11:12:59.164479+0000","last_active":"2024-04-12T11:13:00.170017+0000","last_peered":"2024-04-12T11:13:00.170017+0000","last_clean":"2024-04-12T11:13:00.170017+0000","last_became_active":"2024-04-12T11:12:59.164163+0000","last_became_peered":"2024-04-12T11:12:59.164163+0000","last_unstale":"2024-04-12T11:13:00.170017+0000","last_undegraded":"2024-04-12T11:13:00.170017+0000","last_fullsized":"2024-04-12T11:13:00.170017+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:29:41.965765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":22,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.541447+0000","last_change":"2024-04-12T11:12:57.757955+0000","last_active":"2024-04-12T11:13:08.541447+0000","last_peered":"2024-04-12T11:13:08.541447+0000","last_clean":"2024-04-12T11:13:08.541447+0000","last_became_active":"2024-04-12T11:12:57.757667+0000","last_became_peered":"2024-04-12T11:12:57.757667+0000","last_unstale":"2024-04-12T11:13:08.541447+0000","last_undegraded":"2024-04-12T11:13:08.541447+0000","last_fullsized":"2024-04-12T11:13:08.541447+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:03:59.584101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.165648+0000","last_change":"2024-04-12T11:12:59.164965+0000","last_active":"2024-04-12T11:12:59.165648+0000","last_peered":"2024-04-12T11:12:59.165648+0000","last_clean":"2024-04-12T11:12:59.165648+0000","last_became_active":"2024-04-12T11:12:59.164602+0000","last_became_peered":"2024-04-12T11:12:59.164602+0000","last_unstale":"2024-04-12T11:12:59.165648+0000","last_undegraded":"2024-04-12T11:12:59.165648+0000","last_fullsized":"2024-04-12T11:12:59.165648+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:16.279172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.162994+0000","last_change":"2024-04-12T11:12:57.765194+0000","last_active":"2024-04-12T11:12:59.162994+0000","last_peered":"2024-04-12T11:12:59.162994+0000","last_clean":"2024-04-12T11:12:59.162994+0000","last_became_active":"2024-04-12T11:12:57.764910+0000","last_became_peered":"2024-04-12T11:12:57.764910+0000","last_unstale":"2024-04-12T11:12:59.162994+0000","last_undegraded":"2024-04-12T11:12:59.162994+0000","last_fullsized":"2024-04-12T11:12:59.162994+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:51:58.266747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169644+0000","last_change":"2024-04-12T11:12:59.163926+0000","last_active":"2024-04-12T11:13:00.169644+0000","last_peered":"2024-04-12T11:13:00.169644+0000","last_clean":"2024-04-12T11:13:00.169644+0000","last_became_active":"2024-04-12T11:12:59.163561+0000","last_became_peered":"2024-04-12T11:12:59.163561+0000","last_unstale":"2024-04-12T11:13:00.169644+0000","last_undegraded":"2024-04-12T11:13:00.169644+0000","last_fullsized":"2024-04-12T11:13:00.169644+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:07:16.980521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.125169+0000","last_change":"2024-04-12T11:12:57.767434+0000","last_active":"2024-04-12T11:12:59.125169+0000","last_peered":"2024-04-12T11:12:59.125169+0000","last_clean":"2024-04-12T11:12:59.125169+0000","last_became_active":"2024-04-12T11:12:57.767138+0000","last_became_peered":"2024-04-12T11:12:57.767138+0000","last_unstale":"2024-04-12T11:12:59.125169+0000","last_undegraded":"2024-04-12T11:12:59.125169+0000","last_fullsized":"2024-04-12T11:12:59.125169+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:55:01.265880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.170016+0000","last_change":"2024-04-12T11:12:59.163760+0000","last_active":"2024-04-12T11:13:00.170016+0000","last_peered":"2024-04-12T11:13:00.170016+0000","last_clean":"2024-04-12T11:13:00.170016+0000","last_became_active":"2024-04-12T11:12:59.163409+0000","last_became_peered":"2024-04-12T11:12:59.163409+0000","last_unstale":"2024-04-12T11:13:00.170016+0000","last_undegraded":"2024-04-12T11:13:00.170016+0000","last_fullsized":"2024-04-12T11:13:00.170016+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:19.432190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.541115+0000","last_change":"2024-04-12T11:12:57.761557+0000","last_active":"2024-04-12T11:13:08.541115+0000","last_peered":"2024-04-12T11:13:08.541115+0000","last_clean":"2024-04-12T11:13:08.541115+0000","last_became_active":"2024-04-12T11:12:57.761289+0000","last_became_peered":"2024-04-12T11:12:57.761289+0000","last_unstale":"2024-04-12T11:13:08.541115+0000","last_undegraded":"2024-04-12T11:13:08.541115+0000","last_fullsized":"2024-04-12T11:13:08.541115+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:35:25.156169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.170553+0000","last_change":"2024-04-12T11:12:58.788035+0000","last_active":"2024-04-12T11:13:00.170553+0000","last_peered":"2024-04-12T11:13:00.170553+0000","last_clean":"2024-04-12T11:13:00.170553+0000","last_became_active":"2024-04-12T11:12:58.787757+0000","last_became_peered":"2024-04-12T11:12:58.787757+0000","last_unstale":"2024-04-12T11:13:00.170553+0000","last_undegraded":"2024-04-12T11:13:00.170553+0000","last_fullsized":"2024-04-12T11:13:00.170553+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:13:06.142205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.542082+0000","last_change":"2024-04-12T11:12:57.762165+0000","last_active":"2024-04-12T11:13:08.542082+0000","last_peered":"2024-04-12T11:13:08.542082+0000","last_clean":"2024-04-12T11:13:08.542082+0000","last_became_active":"2024-04-12T11:12:57.761892+0000","last_became_peered":"2024-04-12T11:12:57.761892+0000","last_unstale":"2024-04-12T11:13:08.542082+0000","last_undegraded":"2024-04-12T11:13:08.542082+0000","last_fullsized":"2024-04-12T11:13:08.542082+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:16:06.935058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.781410+0000","last_change":"2024-04-12T11:12:57.781767+0000","last_active":"2024-04-12T11:12:59.781410+0000","last_peered":"2024-04-12T11:12:59.781410+0000","last_clean":"2024-04-12T11:12:59.781410+0000","last_became_active":"2024-04-12T11:12:57.781458+0000","last_became_peered":"2024-04-12T11:12:57.781458+0000","last_unstale":"2024-04-12T11:12:59.781410+0000","last_undegraded":"2024-04-12T11:12:59.781410+0000","last_fullsized":"2024-04-12T11:12:59.781410+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:17:45.096856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.540738+0000","last_change":"2024-04-12T11:12:57.770617+0000","last_active":"2024-04-12T11:13:08.540738+0000","last_peered":"2024-04-12T11:13:08.540738+0000","last_clean":"2024-04-12T11:13:08.540738+0000","last_became_active":"2024-04-12T11:12:57.770304+0000","last_became_peered":"2024-04-12T11:12:57.770304+0000","last_unstale":"2024-04-12T11:13:08.540738+0000","last_undegraded":"2024-04-12T11:13:08.540738+0000","last_fullsized":"2024-04-12T11:13:08.540738+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:23:12.952288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169686+0000","last_change":"2024-04-12T11:12:59.164937+0000","last_active":"2024-04-12T11:13:00.169686+0000","last_peered":"2024-04-12T11:13:00.169686+0000","last_clean":"2024-04-12T11:13:00.169686+0000","last_became_active":"2024-04-12T11:12:59.164385+0000","last_became_peered":"2024-04-12T11:12:59.164385+0000","last_unstale":"2024-04-12T11:13:00.169686+0000","last_undegraded":"2024-04-12T11:13:00.169686+0000","last_fullsized":"2024-04-12T11:13:00.169686+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:43:00.186857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169499+0000","last_change":"2024-04-12T11:12:57.759915+0000","last_active":"2024-04-12T11:13:00.169499+0000","last_peered":"2024-04-12T11:13:00.169499+0000","last_clean":"2024-04-12T11:13:00.169499+0000","last_became_active":"2024-04-12T11:12:57.759614+0000","last_became_peered":"2024-04-12T11:12:57.759614+0000","last_unstale":"2024-04-12T11:13:00.169499+0000","last_undegraded":"2024-04-12T11:13:00.169499+0000","last_fullsized":"2024-04-12T11:13:00.169499+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:54.241243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2024-04-12T11:13:58.989453+0000","last_change":"2024-04-12T11:13:58.989630+0000","last_active":"2024-04-12T11:13:58.989453+0000","last_peered":"2024-04-12T11:13:00.169509+0000","last_clean":"2024-04-12T11:13:00.169509+0000","last_became_active":"2024-04-12T11:12:58.784168+0000","last_became_peered":"2024-04-12T11:12:58.784168+0000","last_unstale":"2024-04-12T11:13:58.989453+0000","last_undegraded":"2024-04-12T11:13:58.989453+0000","last_fullsized":"2024-04-12T11:13:58.989453+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:50:42.356319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.125509+0000","last_change":"2024-04-12T11:12:57.769092+0000","last_active":"2024-04-12T11:12:59.125509+0000","last_peered":"2024-04-12T11:12:59.125509+0000","last_clean":"2024-04-12T11:12:59.125509+0000","last_became_active":"2024-04-12T11:12:57.768652+0000","last_became_peered":"2024-04-12T11:12:57.768652+0000","last_unstale":"2024-04-12T11:12:59.125509+0000","last_undegraded":"2024-04-12T11:12:59.125509+0000","last_fullsized":"2024-04-12T11:12:59.125509+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:23:40.531470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169643+0000","last_change":"2024-04-12T11:12:57.830539+0000","last_active":"2024-04-12T11:13:00.169643+0000","last_peered":"2024-04-12T11:13:00.169643+0000","last_clean":"2024-04-12T11:13:00.169643+0000","last_became_active":"2024-04-12T11:12:57.830211+0000","last_became_peered":"2024-04-12T11:12:57.830211+0000","last_unstale":"2024-04-12T11:13:00.169643+0000","last_undegraded":"2024-04-12T11:13:00.169643+0000","last_fullsized":"2024-04-12T11:13:00.169643+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:29:42.745036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.169507+0000","last_change":"2024-04-12T11:12:57.770057+0000","last_active":"2024-04-12T11:13:00.169507+0000","last_peered":"2024-04-12T11:13:00.169507+0000","last_clean":"2024-04-12T11:13:00.169507+0000","last_became_active":"2024-04-12T11:12:57.769438+0000","last_became_peered":"2024-04-12T11:12:57.769438+0000","last_unstale":"2024-04-12T11:13:00.169507+0000","last_undegraded":"2024-04-12T11:13:00.169507+0000","last_fullsized":"2024-04-12T11:13:00.169507+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:20:55.956601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.164294+0000","last_change":"2024-04-12T11:12:59.163978+0000","last_active":"2024-04-12T11:12:59.164294+0000","last_peered":"2024-04-12T11:12:59.164294+0000","last_clean":"2024-04-12T11:12:59.164294+0000","last_became_active":"2024-04-12T11:12:59.163643+0000","last_became_peered":"2024-04-12T11:12:59.163643+0000","last_unstale":"2024-04-12T11:12:59.164294+0000","last_undegraded":"2024-04-12T11:12:59.164294+0000","last_fullsized":"2024-04-12T11:12:59.164294+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:22:02.655822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:12:59.781283+0000","last_change":"2024-04-12T11:12:57.765860+0000","last_active":"2024-04-12T11:12:59.781283+0000","last_peered":"2024-04-12T11:12:59.781283+0000","last_clean":"2024-04-12T11:12:59.781283+0000","last_became_active":"2024-04-12T11:12:57.765539+0000","last_became_peered":"2024-04-12T11:12:57.765539+0000","last_unstale":"2024-04-12T11:12:59.781283+0000","last_undegraded":"2024-04-12T11:12:59.781283+0000","last_fullsized":"2024-04-12T11:12:59.781283+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:54:43.603517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:00.170256+0000","last_change":"2024-04-12T11:12:58.786291+0000","last_active":"2024-04-12T11:13:00.170256+0000","last_peered":"2024-04-12T11:13:00.170256+0000","last_clean":"2024-04-12T11:13:00.170256+0000","last_became_active":"2024-04-12T11:12:58.785657+0000","last_became_peered":"2024-04-12T11:12:58.785657+0000","last_unstale":"2024-04-12T11:13:00.170256+0000","last_undegraded":"2024-04-12T11:13:00.170256+0000","last_fullsized":"2024-04-12T11:13:00.170256+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:28:04.443928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2024-04-12T11:13:08.537293+0000","last_change":"2024-04-12T11:12:57.757459+0000","last_active":"2024-04-12T11:13:08.537293+0000","last_peered":"2024-04-12T11:13:08.537293+0000","last_clean":"2024-04-12T11:13:08.537293+0000","last_became_active":"2024-04-12T11:12:57.757149+0000","last_became_peered":"2024-04-12T11:12:57.757149+0000","last_unstale":"2024-04-12T11:13:08.537293+0000","last_undegraded":"2024-04-12T11:13:08.537293+0000","last_fullsized":"2024-04-12T11:13:08.537293+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:54:52.213208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234019,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28300,"kb_used_data":1284,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716852,"statfs":{"total":95995035648,"available":95966056448,"internally_reserved":0,"allocated":1314816,"data_stored":891515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.50900000000000001,"15min":0.503},"min":{"1min":0.39800000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.624},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56599999999999995,"15min":0.56999999999999995},"min":{"1min":0.42499999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.754,"5min":2.4780000000000002,"15min":2.4780000000000002},"last":0.58499999999999996}]},{"osd":1,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.41699999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.78300000000000003,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70399999999999996,"5min":1.712,"15min":1.712},"last":0.54300000000000004}]},{"osd":2,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.99099999999999999,"15min":1.071},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.72999999999999998,"5min":15.228999999999999,"15min":15.228999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.038,"15min":1.117},"min":{"1min":0.39900000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.73899999999999999,"5min":15.249000000000001,"15min":15.249000000000001},"last":0.60099999999999998}]},{"osd":3,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.53000000000000003,"15min":0.53100000000000003},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.67900000000000005,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56299999999999994,"15min":0.56200000000000006},"min":{"1min":0.36899999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78600000000000003,"5min":1.044,"15min":1.044},"last":0.56999999999999995}]},{"osd":4,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44,"15min":0.439},"min":{"1min":0.315,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.66600000000000004,"5min":0.752,"15min":0.752},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.626,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.36799999999999999}]},{"osd":5,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81499999999999995,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.52500000000000002,"15min":0.52900000000000003},"min":{"1min":0.34899999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71799999999999997,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.439}]},{"osd":6,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.52400000000000002,"15min":0.52500000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.77300000000000002,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.52300000000000002,"15min":0.52600000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.76800000000000002,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.41199999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495656,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28284,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716868,"statfs":{"total":95995035648,"available":95966072832,"internally_reserved":0,"allocated":1298432,"data_stored":891146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.48199999999999998,"15min":0.47099999999999997},"min":{"1min":0.36699999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.52700000000000002,"15min":0.52100000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.752,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.59899999999999998}]},{"osd":1,"last update":"Fri Apr 12 11:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52900000000000003,"15min":0.52400000000000002},"min":{"1min":0.33300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.74299999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.497,"15min":0.48699999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.59099999999999997}]},{"osd":2,"last update":"Fri Apr 12 11:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.51900000000000002,"15min":0.50900000000000001},"min":{"1min":0.38300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70499999999999996,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.51900000000000002,"15min":0.51300000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.70899999999999996,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.56499999999999995}]},{"osd":3,"last update":"Fri Apr 12 11:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.55500000000000005,"15min":0.54900000000000004},"min":{"1min":0.374,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.76200000000000001,"5min":1.03,"15min":1.03},"last":0.55400000000000005}]},{"osd":4,"last update":"Fri Apr 12 11:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.38400000000000001,"15min":0.372},"min":{"1min":0.29099999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.40699999999999997,"15min":0.39700000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.36099999999999999}]},{"osd":5,"last update":"Fri Apr 12 11:12:56 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.41999999999999998,"15min":0.40699999999999997},"min":{"1min":0.33500000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.435,"15min":0.42299999999999999},"min":{"1min":0.39100000000000001,"5min":0.316,"15min":0.316},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.45500000000000002}]},{"osd":7,"last update":"Fri Apr 12 11:13:24 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.48299999999999998,"15min":0.47799999999999998},"min":{"1min":0.34100000000000003,"5min":0.316,"15min":0.316},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.52800000000000002,"15min":0.52700000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.50700000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574911,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27696,"kb_used_data":680,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717456,"statfs":{"total":95995035648,"available":95966674944,"internally_reserved":0,"allocated":696320,"data_stored":299698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:05 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36699999999999999,"15min":0.317},"min":{"1min":0.26900000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54900000000000004,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":0.25},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.40699999999999997,"15min":0.34699999999999998},"min":{"1min":0.26000000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.81599999999999995,"5min":2.4020000000000001,"15min":2.4020000000000001},"last":0.45400000000000001}]},{"osd":2,"last update":"Fri Apr 12 11:13:25 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.42699999999999999,"15min":0.39100000000000001},"min":{"1min":0.254,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.69699999999999995,"5min":1.885,"15min":1.885},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.39500000000000002,"15min":0.375},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.50600000000000001,"5min":2.0350000000000001,"15min":2.0350000000000001},"last":0.33900000000000002}]},{"osd":3,"last update":"Fri Apr 12 11:13:54 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.41899999999999998,"15min":0.41299999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.69099999999999995,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.46300000000000002,"15min":0.46700000000000003},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.55900000000000005,"5min":1.8069999999999999,"15min":1.8069999999999999},"last":0.48099999999999998}]},{"osd":4,"last update":"Fri Apr 12 11:13:14 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.58399999999999996,"15min":0.57099999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.80500000000000005,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.56299999999999994,"15min":0.55200000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.83399999999999996,"5min":1.589,"15min":1.589},"last":0.50800000000000001}]},{"osd":5,"last update":"Fri Apr 12 11:13:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59999999999999998,"15min":0.58999999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77900000000000003,"5min":1.833,"15min":1.833},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.57499999999999996,"15min":0.57399999999999995},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67300000000000004,"5min":2.012,"15min":2.012},"last":0.43099999999999999}]},{"osd":6,"last update":"Fri Apr 12 11:12:59 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.56200000000000006,"15min":0.55700000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.9850000000000001,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.64600000000000002,"15min":0.64400000000000002},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":1.7809999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":0.70799999999999996}]},{"osd":7,"last update":"Fri Apr 12 11:13:25 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.69999999999999996,"15min":0.71199999999999997},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82699999999999996,"5min":1.859,"15min":1.859},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.66700000000000004,"15min":0.68300000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89000000000000001,"5min":1.6559999999999999,"15min":1.6559999999999999},"last":0.46400000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32380,"kb_used_data":1268,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712772,"statfs":{"total":95995035648,"available":95961878528,"internally_reserved":0,"allocated":1298432,"data_stored":890670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 11:13:00 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.36099999999999999,"15min":0.32500000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.316},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.36199999999999999,"15min":0.33100000000000002},"min":{"1min":0.24199999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.60699999999999998,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.24199999999999999}]},{"osd":2,"last update":"Fri Apr 12 11:13:27 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.40000000000000002,"15min":0.38600000000000001},"min":{"1min":0.247,"5min":0.219,"15min":0.219},"max":{"1min":0.626,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.38800000000000001,"15min":0.37},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.50700000000000001,"5min":1.393,"15min":1.393},"last":0.34399999999999997}]},{"osd":3,"last update":"Fri Apr 12 11:13:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.44600000000000001,"15min":0.438},"min":{"1min":0.29399999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.60699999999999998,"5min":1.532,"15min":1.532},"last":0.372},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.42899999999999999,"15min":0.43099999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56499999999999995,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.377}]},{"osd":4,"last update":"Fri Apr 12 11:13:12 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.57599999999999996,"15min":0.58299999999999996},"min":{"1min":0.378,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.84299999999999997,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.39400000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.73199999999999998,"5min":1.448,"15min":1.448},"last":0.52400000000000002}]},{"osd":5,"last update":"Fri Apr 12 11:13:40 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.59799999999999998,"15min":0.58999999999999997},"min":{"1min":0.39500000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.87,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":0.501},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.81399999999999995,"15min":0.92000000000000004},"min":{"1min":0.40100000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78400000000000003,"5min":7.9340000000000002,"15min":7.9340000000000002},"last":0.53400000000000003}]},{"osd":6,"last update":"Fri Apr 12 11:13:58 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.62,"15min":0.62},"min":{"1min":0.42499999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.64600000000000002,"15min":0.64000000000000001},"min":{"1min":0.46600000000000003,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.91000000000000003,"5min":1.425,"15min":1.425},"last":0.64900000000000002}]},{"osd":7,"last update":"Fri Apr 12 11:13:21 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.59799999999999998,"15min":0.60399999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.67100000000000004,"15min":0.68100000000000005},"min":{"1min":0.47699999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.85599999999999998,"5min":1.506,"15min":1.506},"last":0.57799999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411386,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27720,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717432,"statfs":{"total":95995035648,"available":95966650368,"internally_reserved":0,"allocated":716800,"data_stored":307404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:26 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38800000000000001,"15min":0.375},"min":{"1min":0.23499999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.64900000000000002,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38500000000000001,"15min":0.36699999999999999},"min":{"1min":0.251,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.60699999999999998,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.42999999999999999}]},{"osd":1,"last update":"Fri Apr 12 11:13:26 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.40600000000000003,"15min":0.38900000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.53700000000000003,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.41499999999999998,"15min":0.40500000000000003},"min":{"1min":0.26000000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.52100000000000002}]},{"osd":3,"last update":"Fri Apr 12 11:13:56 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.45700000000000002,"15min":0.47099999999999997},"min":{"1min":0.29499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.57699999999999996,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.45200000000000001,"15min":0.45500000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.70399999999999996,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.56399999999999995}]},{"osd":4,"last update":"Fri Apr 12 11:13:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.96399999999999997,"15min":1.145},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.81000000000000005,"5min":13.686999999999999,"15min":13.686999999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.002,"15min":1.181},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.81599999999999995,"5min":13.721,"15min":13.721},"last":0.88500000000000001}]},{"osd":5,"last update":"Fri Apr 12 11:13:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.39000000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.83899999999999997,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.745},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.621,"15min":0.63300000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76200000000000001,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.65100000000000002}]},{"osd":6,"last update":"Fri Apr 12 11:13:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.629,"15min":0.63500000000000001},"min":{"1min":0.42099999999999999,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83899999999999997,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.61699999999999999,"15min":0.61899999999999999},"min":{"1min":0.441,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.78000000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.84499999999999997}]},{"osd":7,"last update":"Fri Apr 12 11:13:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.65800000000000003,"15min":0.67000000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.79600000000000004,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.94599999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.66600000000000004,"15min":0.67800000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.86099999999999999,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.85799999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182454,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31796,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713356,"statfs":{"total":95995035648,"available":95962476544,"internally_reserved":0,"allocated":700416,"data_stored":299788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31848904},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:50 2024","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.38,"15min":0.39700000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.47999999999999998,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.38400000000000001,"15min":0.38700000000000001},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.59799999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.30499999999999999}]},{"osd":1,"last update":"Fri Apr 12 11:13:50 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.40600000000000003,"15min":0.42899999999999999},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.52700000000000002,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.40200000000000002,"15min":0.41899999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.51100000000000001,"5min":0.624,"15min":0.624},"last":0.27400000000000002}]},{"osd":2,"last update":"Fri Apr 12 11:13:50 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.435,"15min":0.45000000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.55600000000000005,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.375,"5min":0.42999999999999999,"15min":0.44800000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.57799999999999996,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.34499999999999997}]},{"osd":4,"last update":"Fri Apr 12 11:13:13 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55900000000000005,"15min":0.56999999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.71799999999999997,"15min":0.626},"min":{"1min":0.38500000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.74099999999999999,"5min":11.705,"15min":11.705},"last":0.57899999999999996}]},{"osd":5,"last update":"Fri Apr 12 11:13:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.57799999999999996,"15min":0.59399999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.502},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.71399999999999997,"15min":0.64200000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.88800000000000001,"5min":11.68,"15min":11.68},"last":0.61099999999999999}]},{"osd":6,"last update":"Fri Apr 12 11:13:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59699999999999998,"15min":0.59799999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.627,"5min":0.64200000000000002,"15min":0.64400000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.56499999999999995}]},{"osd":7,"last update":"Fri Apr 12 11:13:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.59999999999999998,"15min":0.60399999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.53400000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018929,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27720,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717432,"statfs":{"total":95995035648,"available":95966650368,"internally_reserved":0,"allocated":720896,"data_stored":301408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.59199999999999997},"min":{"1min":0.35699999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.79800000000000004,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54300000000000004,"15min":0.55000000000000004},"min":{"1min":0.35099999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56299999999999994}]},{"osd":1,"last update":"Fri Apr 12 11:13:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55300000000000005,"15min":0.56499999999999995},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.73699999999999999,"15min":0.81399999999999995},"min":{"1min":0.371,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82599999999999996,"5min":5.5590000000000002,"15min":5.5590000000000002},"last":0.64900000000000002}]},{"osd":2,"last update":"Fri Apr 12 11:13:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56100000000000005,"15min":0.56899999999999995},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.76900000000000002,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.73699999999999999,"15min":0.81599999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.80900000000000005,"5min":5.5419999999999998,"15min":5.5419999999999998},"last":0.70099999999999996}]},{"osd":3,"last update":"Fri Apr 12 11:13:13 2024","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.67400000000000004,"15min":0.60399999999999998},"min":{"1min":0.34599999999999997,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":10.676,"5min":10.676,"15min":10.676},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.56699999999999995,"15min":0.58499999999999996},"min":{"1min":0.39500000000000002,"5min":0.373,"15min":0.373},"max":{"1min":0.73199999999999998,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.63600000000000001}]},{"osd":5,"last update":"Fri Apr 12 11:13:40 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.41599999999999998,"15min":0.40500000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.57799999999999996,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.40699999999999997,"15min":0.39100000000000001},"min":{"1min":0.30399999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.33800000000000002}]},{"osd":6,"last update":"Fri Apr 12 11:13:00 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.40999999999999998,"15min":0.40200000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.45500000000000002,"15min":0.44700000000000001},"min":{"1min":0.34899999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.60299999999999998,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.434}]},{"osd":7,"last update":"Fri Apr 12 11:13:24 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.73699999999999999,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.52200000000000002,"15min":0.52300000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67600000000000005,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.52700000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757292,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27700,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717452,"statfs":{"total":95995035648,"available":95966670848,"internally_reserved":0,"allocated":700416,"data_stored":300019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.56100000000000005,"15min":0.57699999999999996},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.65300000000000002,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.498,"5min":0.52900000000000003,"15min":0.53900000000000003},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.63,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.72799999999999998}]},{"osd":1,"last update":"Fri Apr 12 11:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.53600000000000003,"15min":0.55100000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.68400000000000005,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.65500000000000003,"5min":0.873,"15min":0.873},"last":0.68500000000000005}]},{"osd":2,"last update":"Fri Apr 12 11:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.54700000000000004,"15min":0.56299999999999994},"min":{"1min":0.36199999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.65800000000000003,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55300000000000005,"15min":0.56499999999999995},"min":{"1min":0.41699999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.69899999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.621}]},{"osd":3,"last update":"Fri Apr 12 11:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.56399999999999995,"15min":0.58499999999999996},"min":{"1min":0.39900000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.64500000000000002,"5min":1.016,"15min":1.016},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.57499999999999996,"15min":0.58799999999999997},"min":{"1min":0.42699999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.67200000000000004,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.60599999999999998}]},{"osd":4,"last update":"Fri Apr 12 11:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.40799999999999997,"15min":0.40200000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.65900000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.42599999999999999,"15min":0.41799999999999998},"min":{"1min":0.27300000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.44900000000000001}]},{"osd":6,"last update":"Fri Apr 12 11:13:00 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.29699999999999999,"5min":0.246,"15min":0.246},"max":{"1min":0.67700000000000005,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.433},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.51300000000000001,"15min":0.51700000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.67600000000000005,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.51200000000000001}]},{"osd":7,"last update":"Fri Apr 12 11:13:20 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.48699999999999999,"15min":0.48499999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67200000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65600000000000003,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.47299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:14:04.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:14:04.656 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:05 smithi029 ceph-mon[39999]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:14:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:05 smithi029 ceph-mon[29349]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:14:05.399 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 systemd[1]: Stopping Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:14:05.399 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-mon[32917]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:14:05.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.398Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-12T11:14:05.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.398Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-12T11:14:05.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.398Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-12T11:14:05.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.398Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-12T11:14:05.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.398Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-12T11:14:05.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.398Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-12T11:14:05.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.398Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-12T11:14:05.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.398Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-12T11:14:05.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.399Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-12T11:14:05.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.399Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-12T11:14:05.675 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.399Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-12T11:14:05.675 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[61026]: ts=2024-04-12T11:14:05.399Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-12T11:14:05.675 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 podman[64925]: 2024-04-12 11:14:05.408738283 +0000 UTC m=+0.112063170 container died 545830a9d26da01d894bc15988ee94c5fba8f5624d5f2564fccc488c42c22216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:05.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 podman[64925]: 2024-04-12 11:14:05.804721489 +0000 UTC m=+0.508046376 container cleanup 545830a9d26da01d894bc15988ee94c5fba8f5624d5f2564fccc488c42c22216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:05.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:05 smithi087 bash[64925]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a 2024-04-12T11:14:06.261 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-mon[39999]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:06.261 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-mon[39999]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-04-12T11:14:06.261 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-mon[39999]: Cluster is now healthy 2024-04-12T11:14:06.261 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-mon[29349]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:06.262 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-mon[29349]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-04-12T11:14:06.262 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-mon[29349]: Cluster is now healthy 2024-04-12T11:14:06.423 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-mon[32917]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:06.424 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-mon[32917]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-04-12T11:14:06.424 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-mon[32917]: Cluster is now healthy 2024-04-12T11:14:06.424 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 podman[64939]: 2024-04-12 11:14:06.07495519 +0000 UTC m=+0.665805423 container remove 545830a9d26da01d894bc15988ee94c5fba8f5624d5f2564fccc488c42c22216 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:06.424 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-12T11:14:06.424 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 systemd[1]: Stopped Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:14:06.424 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@prometheus.a.service: Consumed 1.175s CPU time. 2024-04-12T11:14:06.424 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 systemd[1]: Starting Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:14:06.579 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:14:06.581 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:14:06.679 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 podman[65037]: 2024-04-12 11:14:06.327939315 +0000 UTC m=+0.019076012 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-12T11:14:06.679 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 podman[65037]: 2024-04-12 11:14:06.459802821 +0000 UTC m=+0.150939550 container create 06b93cbb7d0ce332957f7651c0e0c2420d7623e754a17415c085da0f09473b7d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:06.679 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 podman[65037]: 2024-04-12 11:14:06.653549903 +0000 UTC m=+0.344686587 container init 06b93cbb7d0ce332957f7651c0e0c2420d7623e754a17415c085da0f09473b7d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:06.679 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 podman[65037]: 2024-04-12 11:14:06.655980043 +0000 UTC m=+0.347116739 container start 06b93cbb7d0ce332957f7651c0e0c2420d7623e754a17415c085da0f09473b7d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:14:06.916 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:06] ENGINE Bus STOPPING 2024-04-12T11:14:06.916 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T11:14:06.916 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:06] ENGINE Bus STOPPED 2024-04-12T11:14:06.916 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:06] ENGINE Bus STARTING 2024-04-12T11:14:06.916 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:06] ENGINE Serving on http://:::9283 2024-04-12T11:14:06.916 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:06] ENGINE Bus STARTED 2024-04-12T11:14:06.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.679Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-12T11:14:06.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.679Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-12T11:14:06.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.679Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi087 (none))" 2024-04-12T11:14:06.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.679Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-12T11:14:06.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.679Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.681Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.682Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.682Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.682Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.683Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.683Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.524µs 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.683Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.686Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.687Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.687Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=16.872µs wal_replay_duration=3.046462ms wbl_replay_duration=114ns total_replay_duration=3.082104ms 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.687Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.687Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-12T11:14:06.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.687Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-12T11:14:06.969 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.703Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.285572ms db_storage=829ns remote_storage=922ns web_handler=356ns query_engine=592ns scrape=410.741µs scrape_sd=105.1µs notify=16.125µs notify_sd=18.05µs rules=14.52801ms tracing=5.124µs 2024-04-12T11:14:06.969 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.703Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-12T11:14:06.969 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:14:06.703Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-12T11:14:06.969 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 bash[65037]: 06b93cbb7d0ce332957f7651c0e0c2420d7623e754a17415c085da0f09473b7d 2024-04-12T11:14:06.969 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:14:06 smithi087 systemd[1]: Started Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:14:07.270 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:06] ENGINE Bus STOPPING 2024-04-12T11:14:07.270 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T11:14:07.270 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:06] ENGINE Bus STOPPED 2024-04-12T11:14:07.270 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:06] ENGINE Bus STARTING 2024-04-12T11:14:07.270 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:07] ENGINE Serving on http://:::9283 2024-04-12T11:14:07.270 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:07] ENGINE Bus STARTED 2024-04-12T11:14:07.271 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:07] ENGINE Bus STOPPING 2024-04-12T11:14:07.271 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T11:14:07.271 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:07] ENGINE Bus STOPPED 2024-04-12T11:14:07.271 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:07] ENGINE Bus STARTING 2024-04-12T11:14:07.271 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:07] ENGINE Serving on http://:::9283 2024-04-12T11:14:07.271 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:07] ENGINE Bus STARTED 2024-04-12T11:14:07.387 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":74,"stamp":"2024-04-12T11:14:06.144347+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592730,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":231808,"kb_used_data":7452,"kb_used_omap":12,"kb_used_meta":224243,"kb_avail":749729408,"statfs":{"total":767960285184,"available":767722913792,"internally_reserved":0,"allocated":7630848,"data_stored":4287430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":229625401},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001630"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987184+0000","last_change":"2024-04-12T11:12:58.787924+0000","last_active":"2024-04-12T11:13:58.987184+0000","last_peered":"2024-04-12T11:13:58.987184+0000","last_clean":"2024-04-12T11:13:58.987184+0000","last_became_active":"2024-04-12T11:12:58.787657+0000","last_became_peered":"2024-04-12T11:12:58.787657+0000","last_unstale":"2024-04-12T11:13:58.987184+0000","last_undegraded":"2024-04-12T11:13:58.987184+0000","last_fullsized":"2024-04-12T11:13:58.987184+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:19:10.702420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.751268+0000","last_change":"2024-04-12T11:12:57.768925+0000","last_active":"2024-04-12T11:13:59.751268+0000","last_peered":"2024-04-12T11:13:59.751268+0000","last_clean":"2024-04-12T11:13:59.751268+0000","last_became_active":"2024-04-12T11:12:57.768245+0000","last_became_peered":"2024-04-12T11:12:57.768245+0000","last_unstale":"2024-04-12T11:13:59.751268+0000","last_undegraded":"2024-04-12T11:13:59.751268+0000","last_fullsized":"2024-04-12T11:13:59.751268+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:40:15.574078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.989250+0000","last_change":"2024-04-12T11:12:59.271181+0000","last_active":"2024-04-12T11:13:58.989250+0000","last_peered":"2024-04-12T11:13:58.989250+0000","last_clean":"2024-04-12T11:13:58.989250+0000","last_became_active":"2024-04-12T11:12:59.270640+0000","last_became_peered":"2024-04-12T11:12:59.270640+0000","last_unstale":"2024-04-12T11:13:58.989250+0000","last_undegraded":"2024-04-12T11:13:58.989250+0000","last_fullsized":"2024-04-12T11:13:58.989250+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:55:12.317879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.989448+0000","last_change":"2024-04-12T11:12:57.768614+0000","last_active":"2024-04-12T11:13:58.989448+0000","last_peered":"2024-04-12T11:13:58.989448+0000","last_clean":"2024-04-12T11:13:58.989448+0000","last_became_active":"2024-04-12T11:12:57.768323+0000","last_became_peered":"2024-04-12T11:12:57.768323+0000","last_unstale":"2024-04-12T11:13:58.989448+0000","last_undegraded":"2024-04-12T11:13:58.989448+0000","last_fullsized":"2024-04-12T11:13:58.989448+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:41.849038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987349+0000","last_change":"2024-04-12T11:12:58.785574+0000","last_active":"2024-04-12T11:13:58.987349+0000","last_peered":"2024-04-12T11:13:58.987349+0000","last_clean":"2024-04-12T11:13:58.987349+0000","last_became_active":"2024-04-12T11:12:58.785281+0000","last_became_peered":"2024-04-12T11:12:58.785281+0000","last_unstale":"2024-04-12T11:13:58.987349+0000","last_undegraded":"2024-04-12T11:13:58.987349+0000","last_fullsized":"2024-04-12T11:13:58.987349+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:48:49.798531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.991044+0000","last_change":"2024-04-12T11:12:57.760870+0000","last_active":"2024-04-12T11:13:58.991044+0000","last_peered":"2024-04-12T11:13:58.991044+0000","last_clean":"2024-04-12T11:13:58.991044+0000","last_became_active":"2024-04-12T11:12:57.760587+0000","last_became_peered":"2024-04-12T11:12:57.760587+0000","last_unstale":"2024-04-12T11:13:58.991044+0000","last_undegraded":"2024-04-12T11:13:58.991044+0000","last_fullsized":"2024-04-12T11:13:58.991044+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:00:14.733542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T11:14:00.033447+0000","last_change":"2024-04-12T11:14:00.033447+0000","last_active":"2024-04-12T11:14:00.033447+0000","last_peered":"2024-04-12T11:14:00.033447+0000","last_clean":"2024-04-12T11:14:00.033447+0000","last_became_active":"2024-04-12T11:14:00.033077+0000","last_became_peered":"2024-04-12T11:14:00.033077+0000","last_unstale":"2024-04-12T11:14:00.033447+0000","last_undegraded":"2024-04-12T11:14:00.033447+0000","last_fullsized":"2024-04-12T11:14:00.033447+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:20:42.578781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.990798+0000","last_change":"2024-04-12T11:12:57.760302+0000","last_active":"2024-04-12T11:13:58.990798+0000","last_peered":"2024-04-12T11:13:58.990798+0000","last_clean":"2024-04-12T11:13:58.990798+0000","last_became_active":"2024-04-12T11:12:57.760005+0000","last_became_peered":"2024-04-12T11:12:57.760005+0000","last_unstale":"2024-04-12T11:13:58.990798+0000","last_undegraded":"2024-04-12T11:13:58.990798+0000","last_fullsized":"2024-04-12T11:13:58.990798+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:03:33.678493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.990399+0000","last_change":"2024-04-12T11:12:57.780520+0000","last_active":"2024-04-12T11:13:58.990399+0000","last_peered":"2024-04-12T11:13:58.990399+0000","last_clean":"2024-04-12T11:13:58.990399+0000","last_became_active":"2024-04-12T11:12:57.780227+0000","last_became_peered":"2024-04-12T11:12:57.780227+0000","last_unstale":"2024-04-12T11:13:58.990399+0000","last_undegraded":"2024-04-12T11:13:58.990399+0000","last_fullsized":"2024-04-12T11:13:58.990399+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:20:26.510304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.751259+0000","last_change":"2024-04-12T11:12:57.765674+0000","last_active":"2024-04-12T11:13:59.751259+0000","last_peered":"2024-04-12T11:13:59.751259+0000","last_clean":"2024-04-12T11:13:59.751259+0000","last_became_active":"2024-04-12T11:12:57.765384+0000","last_became_peered":"2024-04-12T11:12:57.765384+0000","last_unstale":"2024-04-12T11:13:59.751259+0000","last_undegraded":"2024-04-12T11:13:59.751259+0000","last_fullsized":"2024-04-12T11:13:59.751259+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:14:21.536217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987963+0000","last_change":"2024-04-12T11:12:58.786647+0000","last_active":"2024-04-12T11:13:58.987963+0000","last_peered":"2024-04-12T11:13:58.987963+0000","last_clean":"2024-04-12T11:13:58.987963+0000","last_became_active":"2024-04-12T11:12:58.786350+0000","last_became_peered":"2024-04-12T11:12:58.786350+0000","last_unstale":"2024-04-12T11:13:58.987963+0000","last_undegraded":"2024-04-12T11:13:58.987963+0000","last_fullsized":"2024-04-12T11:13:58.987963+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:44:23.801172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.751148+0000","last_change":"2024-04-12T11:12:57.763606+0000","last_active":"2024-04-12T11:13:59.751148+0000","last_peered":"2024-04-12T11:13:59.751148+0000","last_clean":"2024-04-12T11:13:59.751148+0000","last_became_active":"2024-04-12T11:12:57.763299+0000","last_became_peered":"2024-04-12T11:12:57.763299+0000","last_unstale":"2024-04-12T11:13:59.751148+0000","last_undegraded":"2024-04-12T11:13:59.751148+0000","last_fullsized":"2024-04-12T11:13:59.751148+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:24:22.305524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.132477+0000","last_change":"2024-04-12T11:11:49.523873+0000","last_active":"2024-04-12T11:13:59.132477+0000","last_peered":"2024-04-12T11:13:59.132477+0000","last_clean":"2024-04-12T11:13:59.132477+0000","last_became_active":"2024-04-12T11:11:49.523570+0000","last_became_peered":"2024-04-12T11:11:49.523570+0000","last_unstale":"2024-04-12T11:13:59.132477+0000","last_undegraded":"2024-04-12T11:13:59.132477+0000","last_fullsized":"2024-04-12T11:13:59.132477+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:09:00.491446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.989953+0000","last_change":"2024-04-12T11:12:59.163787+0000","last_active":"2024-04-12T11:13:58.989953+0000","last_peered":"2024-04-12T11:13:58.989953+0000","last_clean":"2024-04-12T11:13:58.989953+0000","last_became_active":"2024-04-12T11:12:59.163457+0000","last_became_peered":"2024-04-12T11:12:59.163457+0000","last_unstale":"2024-04-12T11:13:58.989953+0000","last_undegraded":"2024-04-12T11:13:58.989953+0000","last_fullsized":"2024-04-12T11:13:58.989953+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:21.459548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987968+0000","last_change":"2024-04-12T11:11:49.882400+0000","last_active":"2024-04-12T11:13:58.987968+0000","last_peered":"2024-04-12T11:13:58.987968+0000","last_clean":"2024-04-12T11:13:58.987968+0000","last_became_active":"2024-04-12T11:11:49.882073+0000","last_became_peered":"2024-04-12T11:11:49.882073+0000","last_unstale":"2024-04-12T11:13:58.987968+0000","last_undegraded":"2024-04-12T11:13:58.987968+0000","last_fullsized":"2024-04-12T11:13:58.987968+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:38:46.841744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.989651+0000","last_change":"2024-04-12T11:12:57.770877+0000","last_active":"2024-04-12T11:13:58.989651+0000","last_peered":"2024-04-12T11:13:58.989651+0000","last_clean":"2024-04-12T11:13:58.989651+0000","last_became_active":"2024-04-12T11:12:57.770217+0000","last_became_peered":"2024-04-12T11:12:57.770217+0000","last_unstale":"2024-04-12T11:13:58.989651+0000","last_undegraded":"2024-04-12T11:13:58.989651+0000","last_fullsized":"2024-04-12T11:13:58.989651+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:25:07.457621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.751050+0000","last_change":"2024-04-12T11:12:59.163973+0000","last_active":"2024-04-12T11:13:59.751050+0000","last_peered":"2024-04-12T11:13:59.751050+0000","last_clean":"2024-04-12T11:13:59.751050+0000","last_became_active":"2024-04-12T11:12:59.163648+0000","last_became_peered":"2024-04-12T11:12:59.163648+0000","last_unstale":"2024-04-12T11:13:59.751050+0000","last_undegraded":"2024-04-12T11:13:59.751050+0000","last_fullsized":"2024-04-12T11:13:59.751050+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:26:50.345253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.988256+0000","last_change":"2024-04-12T11:12:57.756170+0000","last_active":"2024-04-12T11:13:58.988256+0000","last_peered":"2024-04-12T11:13:58.988256+0000","last_clean":"2024-04-12T11:13:58.988256+0000","last_became_active":"2024-04-12T11:12:57.755885+0000","last_became_peered":"2024-04-12T11:12:57.755885+0000","last_unstale":"2024-04-12T11:13:58.988256+0000","last_undegraded":"2024-04-12T11:13:58.988256+0000","last_fullsized":"2024-04-12T11:13:58.988256+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:08:27.183396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.276967+0000","last_change":"2024-04-12T11:11:49.882324+0000","last_active":"2024-04-12T11:13:59.276967+0000","last_peered":"2024-04-12T11:13:59.276967+0000","last_clean":"2024-04-12T11:13:59.276967+0000","last_became_active":"2024-04-12T11:11:49.881975+0000","last_became_peered":"2024-04-12T11:11:49.881975+0000","last_unstale":"2024-04-12T11:13:59.276967+0000","last_undegraded":"2024-04-12T11:13:59.276967+0000","last_fullsized":"2024-04-12T11:13:59.276967+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:10.140250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.751268+0000","last_change":"2024-04-12T11:12:59.270863+0000","last_active":"2024-04-12T11:13:59.751268+0000","last_peered":"2024-04-12T11:13:59.751268+0000","last_clean":"2024-04-12T11:13:59.751268+0000","last_became_active":"2024-04-12T11:12:59.270555+0000","last_became_peered":"2024-04-12T11:12:59.270555+0000","last_unstale":"2024-04-12T11:13:59.751268+0000","last_undegraded":"2024-04-12T11:13:59.751268+0000","last_fullsized":"2024-04-12T11:13:59.751268+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:28.109904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.751115+0000","last_change":"2024-04-12T11:12:57.767163+0000","last_active":"2024-04-12T11:13:59.751115+0000","last_peered":"2024-04-12T11:13:59.751115+0000","last_clean":"2024-04-12T11:13:59.751115+0000","last_became_active":"2024-04-12T11:12:57.766842+0000","last_became_peered":"2024-04-12T11:12:57.766842+0000","last_unstale":"2024-04-12T11:13:59.751115+0000","last_undegraded":"2024-04-12T11:13:59.751115+0000","last_fullsized":"2024-04-12T11:13:59.751115+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:50.322014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.276726+0000","last_change":"2024-04-12T11:11:49.524657+0000","last_active":"2024-04-12T11:13:59.276726+0000","last_peered":"2024-04-12T11:13:59.276726+0000","last_clean":"2024-04-12T11:13:59.276726+0000","last_became_active":"2024-04-12T11:11:49.524379+0000","last_became_peered":"2024-04-12T11:11:49.524379+0000","last_unstale":"2024-04-12T11:13:59.276726+0000","last_undegraded":"2024-04-12T11:13:59.276726+0000","last_fullsized":"2024-04-12T11:13:59.276726+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:30:52.444470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.988410+0000","last_change":"2024-04-12T11:12:58.173490+0000","last_active":"2024-04-12T11:13:58.988410+0000","last_peered":"2024-04-12T11:13:58.988410+0000","last_clean":"2024-04-12T11:13:58.988410+0000","last_became_active":"2024-04-12T11:12:58.173170+0000","last_became_peered":"2024-04-12T11:12:58.173170+0000","last_unstale":"2024-04-12T11:13:58.988410+0000","last_undegraded":"2024-04-12T11:13:58.988410+0000","last_fullsized":"2024-04-12T11:13:58.988410+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:29.680553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.989133+0000","last_change":"2024-04-12T11:12:57.766887+0000","last_active":"2024-04-12T11:13:58.989133+0000","last_peered":"2024-04-12T11:13:58.989133+0000","last_clean":"2024-04-12T11:13:58.989133+0000","last_became_active":"2024-04-12T11:12:57.766494+0000","last_became_peered":"2024-04-12T11:12:57.766494+0000","last_unstale":"2024-04-12T11:13:58.989133+0000","last_undegraded":"2024-04-12T11:13:58.989133+0000","last_fullsized":"2024-04-12T11:13:58.989133+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:47:47.566185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"63'128","reported_seq":266,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.276642+0000","last_change":"2024-04-12T11:11:24.390399+0000","last_active":"2024-04-12T11:13:59.276642+0000","last_peered":"2024-04-12T11:13:59.276642+0000","last_clean":"2024-04-12T11:13:59.276642+0000","last_became_active":"2024-04-12T11:11:24.389943+0000","last_became_peered":"2024-04-12T11:11:24.389943+0000","last_unstale":"2024-04-12T11:13:59.276642+0000","last_undegraded":"2024-04-12T11:13:59.276642+0000","last_fullsized":"2024-04-12T11:13:59.276642+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:06:44.231202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T11:14:00.503115+0000","last_change":"2024-04-12T11:11:49.524628+0000","last_active":"2024-04-12T11:14:00.503115+0000","last_peered":"2024-04-12T11:14:00.503115+0000","last_clean":"2024-04-12T11:14:00.503115+0000","last_became_active":"2024-04-12T11:11:49.524226+0000","last_became_peered":"2024-04-12T11:11:49.524226+0000","last_unstale":"2024-04-12T11:14:00.503115+0000","last_undegraded":"2024-04-12T11:14:00.503115+0000","last_fullsized":"2024-04-12T11:14:00.503115+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:43:26.148427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.988167+0000","last_change":"2024-04-12T11:12:57.973543+0000","last_active":"2024-04-12T11:13:58.988167+0000","last_peered":"2024-04-12T11:13:58.988167+0000","last_clean":"2024-04-12T11:13:58.988167+0000","last_became_active":"2024-04-12T11:12:57.973234+0000","last_became_peered":"2024-04-12T11:12:57.973234+0000","last_unstale":"2024-04-12T11:13:58.988167+0000","last_undegraded":"2024-04-12T11:13:58.988167+0000","last_fullsized":"2024-04-12T11:13:58.988167+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:42:39.697895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987077+0000","last_change":"2024-04-12T11:12:57.759232+0000","last_active":"2024-04-12T11:13:58.987077+0000","last_peered":"2024-04-12T11:13:58.987077+0000","last_clean":"2024-04-12T11:13:58.987077+0000","last_became_active":"2024-04-12T11:12:57.758748+0000","last_became_peered":"2024-04-12T11:12:57.758748+0000","last_unstale":"2024-04-12T11:13:58.987077+0000","last_undegraded":"2024-04-12T11:13:58.987077+0000","last_fullsized":"2024-04-12T11:13:58.987077+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:21:31.374347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":21,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T11:14:00.032018+0000","last_change":"2024-04-12T11:11:49.523432+0000","last_active":"2024-04-12T11:14:00.032018+0000","last_peered":"2024-04-12T11:14:00.032018+0000","last_clean":"2024-04-12T11:14:00.032018+0000","last_became_active":"2024-04-12T11:11:49.523059+0000","last_became_peered":"2024-04-12T11:11:49.523059+0000","last_unstale":"2024-04-12T11:14:00.032018+0000","last_undegraded":"2024-04-12T11:14:00.032018+0000","last_fullsized":"2024-04-12T11:14:00.032018+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:55:25.692863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987776+0000","last_change":"2024-04-12T11:12:57.780685+0000","last_active":"2024-04-12T11:13:58.987776+0000","last_peered":"2024-04-12T11:13:58.987776+0000","last_clean":"2024-04-12T11:13:58.987776+0000","last_became_active":"2024-04-12T11:12:57.780396+0000","last_became_peered":"2024-04-12T11:12:57.780396+0000","last_unstale":"2024-04-12T11:13:58.987776+0000","last_undegraded":"2024-04-12T11:13:58.987776+0000","last_fullsized":"2024-04-12T11:13:58.987776+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:41:06.039444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987470+0000","last_change":"2024-04-12T11:12:57.760336+0000","last_active":"2024-04-12T11:13:58.987470+0000","last_peered":"2024-04-12T11:13:58.987470+0000","last_clean":"2024-04-12T11:13:58.987470+0000","last_became_active":"2024-04-12T11:12:57.760032+0000","last_became_peered":"2024-04-12T11:12:57.760032+0000","last_unstale":"2024-04-12T11:13:58.987470+0000","last_undegraded":"2024-04-12T11:13:58.987470+0000","last_fullsized":"2024-04-12T11:13:58.987470+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:11:54.530608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T11:14:00.503441+0000","last_change":"2024-04-12T11:11:49.523947+0000","last_active":"2024-04-12T11:14:00.503441+0000","last_peered":"2024-04-12T11:14:00.503441+0000","last_clean":"2024-04-12T11:14:00.503441+0000","last_became_active":"2024-04-12T11:11:49.523650+0000","last_became_peered":"2024-04-12T11:11:49.523650+0000","last_unstale":"2024-04-12T11:14:00.503441+0000","last_undegraded":"2024-04-12T11:14:00.503441+0000","last_fullsized":"2024-04-12T11:14:00.503441+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:05:22.528981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987283+0000","last_change":"2024-04-12T11:12:57.990458+0000","last_active":"2024-04-12T11:13:58.987283+0000","last_peered":"2024-04-12T11:13:58.987283+0000","last_clean":"2024-04-12T11:13:58.987283+0000","last_became_active":"2024-04-12T11:12:57.990148+0000","last_became_peered":"2024-04-12T11:12:57.990148+0000","last_unstale":"2024-04-12T11:13:58.987283+0000","last_undegraded":"2024-04-12T11:13:58.987283+0000","last_fullsized":"2024-04-12T11:13:58.987283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:13:35.647039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.990730+0000","last_change":"2024-04-12T11:12:57.768636+0000","last_active":"2024-04-12T11:13:58.990730+0000","last_peered":"2024-04-12T11:13:58.990730+0000","last_clean":"2024-04-12T11:13:58.990730+0000","last_became_active":"2024-04-12T11:12:57.768359+0000","last_became_peered":"2024-04-12T11:12:57.768359+0000","last_unstale":"2024-04-12T11:13:58.990730+0000","last_undegraded":"2024-04-12T11:13:58.990730+0000","last_fullsized":"2024-04-12T11:13:58.990730+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:00:41.208948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.989400+0000","last_change":"2024-04-12T11:11:49.524259+0000","last_active":"2024-04-12T11:13:59.989400+0000","last_peered":"2024-04-12T11:13:59.989400+0000","last_clean":"2024-04-12T11:13:59.989400+0000","last_became_active":"2024-04-12T11:11:49.523641+0000","last_became_peered":"2024-04-12T11:11:49.523641+0000","last_unstale":"2024-04-12T11:13:59.989400+0000","last_undegraded":"2024-04-12T11:13:59.989400+0000","last_fullsized":"2024-04-12T11:13:59.989400+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:28:22.480630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.988242+0000","last_change":"2024-04-12T11:12:57.990460+0000","last_active":"2024-04-12T11:13:58.988242+0000","last_peered":"2024-04-12T11:13:58.988242+0000","last_clean":"2024-04-12T11:13:58.988242+0000","last_became_active":"2024-04-12T11:12:57.990156+0000","last_became_peered":"2024-04-12T11:12:57.990156+0000","last_unstale":"2024-04-12T11:13:58.988242+0000","last_undegraded":"2024-04-12T11:13:58.988242+0000","last_fullsized":"2024-04-12T11:13:58.988242+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:26:24.526812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.990045+0000","last_change":"2024-04-12T11:12:57.769233+0000","last_active":"2024-04-12T11:13:58.990045+0000","last_peered":"2024-04-12T11:13:58.990045+0000","last_clean":"2024-04-12T11:13:58.990045+0000","last_became_active":"2024-04-12T11:12:57.768383+0000","last_became_peered":"2024-04-12T11:12:57.768383+0000","last_unstale":"2024-04-12T11:13:58.990045+0000","last_undegraded":"2024-04-12T11:13:58.990045+0000","last_fullsized":"2024-04-12T11:13:58.990045+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:06:42.355317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987734+0000","last_change":"2024-04-12T11:12:58.788638+0000","last_active":"2024-04-12T11:13:58.987734+0000","last_peered":"2024-04-12T11:13:58.987734+0000","last_clean":"2024-04-12T11:13:58.987734+0000","last_became_active":"2024-04-12T11:12:58.788338+0000","last_became_peered":"2024-04-12T11:12:58.788338+0000","last_unstale":"2024-04-12T11:13:58.987734+0000","last_undegraded":"2024-04-12T11:13:58.987734+0000","last_fullsized":"2024-04-12T11:13:58.987734+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:30.910258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987879+0000","last_change":"2024-04-12T11:12:57.766578+0000","last_active":"2024-04-12T11:13:58.987879+0000","last_peered":"2024-04-12T11:13:58.987879+0000","last_clean":"2024-04-12T11:13:58.987879+0000","last_became_active":"2024-04-12T11:12:57.766305+0000","last_became_peered":"2024-04-12T11:12:57.766305+0000","last_unstale":"2024-04-12T11:13:58.987879+0000","last_undegraded":"2024-04-12T11:13:58.987879+0000","last_fullsized":"2024-04-12T11:13:58.987879+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:46:14.183201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.988174+0000","last_change":"2024-04-12T11:12:59.271140+0000","last_active":"2024-04-12T11:13:58.988174+0000","last_peered":"2024-04-12T11:13:58.988174+0000","last_clean":"2024-04-12T11:13:58.988174+0000","last_became_active":"2024-04-12T11:12:59.270866+0000","last_became_peered":"2024-04-12T11:12:59.270866+0000","last_unstale":"2024-04-12T11:13:58.988174+0000","last_undegraded":"2024-04-12T11:13:58.988174+0000","last_fullsized":"2024-04-12T11:13:58.988174+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:23:02.348448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987110+0000","last_change":"2024-04-12T11:12:57.760710+0000","last_active":"2024-04-12T11:13:58.987110+0000","last_peered":"2024-04-12T11:13:58.987110+0000","last_clean":"2024-04-12T11:13:58.987110+0000","last_became_active":"2024-04-12T11:12:57.760454+0000","last_became_peered":"2024-04-12T11:12:57.760454+0000","last_unstale":"2024-04-12T11:13:58.987110+0000","last_undegraded":"2024-04-12T11:13:58.987110+0000","last_fullsized":"2024-04-12T11:13:58.987110+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:03:14.723304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987706+0000","last_change":"2024-04-12T11:12:58.786811+0000","last_active":"2024-04-12T11:13:58.987706+0000","last_peered":"2024-04-12T11:13:58.987706+0000","last_clean":"2024-04-12T11:13:58.987706+0000","last_became_active":"2024-04-12T11:12:58.786423+0000","last_became_peered":"2024-04-12T11:12:58.786423+0000","last_unstale":"2024-04-12T11:13:58.987706+0000","last_undegraded":"2024-04-12T11:13:58.987706+0000","last_fullsized":"2024-04-12T11:13:58.987706+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:26:51.148882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.990115+0000","last_change":"2024-04-12T11:12:57.769995+0000","last_active":"2024-04-12T11:13:58.990115+0000","last_peered":"2024-04-12T11:13:58.990115+0000","last_clean":"2024-04-12T11:13:58.990115+0000","last_became_active":"2024-04-12T11:12:57.769701+0000","last_became_peered":"2024-04-12T11:12:57.769701+0000","last_unstale":"2024-04-12T11:13:58.990115+0000","last_undegraded":"2024-04-12T11:13:58.990115+0000","last_fullsized":"2024-04-12T11:13:58.990115+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:30:52.071458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.750954+0000","last_change":"2024-04-12T11:12:58.785119+0000","last_active":"2024-04-12T11:13:59.750954+0000","last_peered":"2024-04-12T11:13:59.750954+0000","last_clean":"2024-04-12T11:13:59.750954+0000","last_became_active":"2024-04-12T11:12:58.784825+0000","last_became_peered":"2024-04-12T11:12:58.784825+0000","last_unstale":"2024-04-12T11:13:59.750954+0000","last_undegraded":"2024-04-12T11:13:59.750954+0000","last_fullsized":"2024-04-12T11:13:59.750954+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:20:37.720179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.990702+0000","last_change":"2024-04-12T11:12:57.765020+0000","last_active":"2024-04-12T11:13:58.990702+0000","last_peered":"2024-04-12T11:13:58.990702+0000","last_clean":"2024-04-12T11:13:58.990702+0000","last_became_active":"2024-04-12T11:12:57.764740+0000","last_became_peered":"2024-04-12T11:12:57.764740+0000","last_unstale":"2024-04-12T11:13:58.990702+0000","last_undegraded":"2024-04-12T11:13:58.990702+0000","last_fullsized":"2024-04-12T11:13:58.990702+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:31:18.778202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987370+0000","last_change":"2024-04-12T11:12:58.173247+0000","last_active":"2024-04-12T11:13:58.987370+0000","last_peered":"2024-04-12T11:13:58.987370+0000","last_clean":"2024-04-12T11:13:58.987370+0000","last_became_active":"2024-04-12T11:12:58.172953+0000","last_became_peered":"2024-04-12T11:12:58.172953+0000","last_unstale":"2024-04-12T11:13:58.987370+0000","last_undegraded":"2024-04-12T11:13:58.987370+0000","last_fullsized":"2024-04-12T11:13:58.987370+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:26:06.124062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.277065+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:13:59.277065+0000","last_peered":"2024-04-12T11:13:59.277065+0000","last_clean":"2024-04-12T11:13:59.277065+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:13:59.277065+0000","last_undegraded":"2024-04-12T11:13:59.277065+0000","last_fullsized":"2024-04-12T11:13:59.277065+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T11:14:00.504120+0000","last_change":"2024-04-12T11:14:00.504120+0000","last_active":"2024-04-12T11:14:00.504120+0000","last_peered":"2024-04-12T11:14:00.504120+0000","last_clean":"2024-04-12T11:14:00.504120+0000","last_became_active":"2024-04-12T11:14:00.503698+0000","last_became_peered":"2024-04-12T11:14:00.503698+0000","last_unstale":"2024-04-12T11:14:00.504120+0000","last_undegraded":"2024-04-12T11:14:00.504120+0000","last_fullsized":"2024-04-12T11:14:00.504120+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:07.616807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.276971+0000","last_change":"2024-04-12T11:12:57.772960+0000","last_active":"2024-04-12T11:13:59.276971+0000","last_peered":"2024-04-12T11:13:59.276971+0000","last_clean":"2024-04-12T11:13:59.276971+0000","last_became_active":"2024-04-12T11:12:57.772624+0000","last_became_peered":"2024-04-12T11:12:57.772624+0000","last_unstale":"2024-04-12T11:13:59.276971+0000","last_undegraded":"2024-04-12T11:13:59.276971+0000","last_fullsized":"2024-04-12T11:13:59.276971+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:38:00.390687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.990275+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:13:58.990275+0000","last_peered":"2024-04-12T11:13:58.990275+0000","last_clean":"2024-04-12T11:13:58.990275+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:13:58.990275+0000","last_undegraded":"2024-04-12T11:13:58.990275+0000","last_fullsized":"2024-04-12T11:13:58.990275+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.277125+0000","last_change":"2024-04-12T11:12:57.764312+0000","last_active":"2024-04-12T11:13:59.277125+0000","last_peered":"2024-04-12T11:13:59.277125+0000","last_clean":"2024-04-12T11:13:59.277125+0000","last_became_active":"2024-04-12T11:12:57.763625+0000","last_became_peered":"2024-04-12T11:12:57.763625+0000","last_unstale":"2024-04-12T11:13:59.277125+0000","last_undegraded":"2024-04-12T11:13:59.277125+0000","last_fullsized":"2024-04-12T11:13:59.277125+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:18:28.157847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.989572+0000","last_change":"2024-04-12T11:12:59.164479+0000","last_active":"2024-04-12T11:13:58.989572+0000","last_peered":"2024-04-12T11:13:58.989572+0000","last_clean":"2024-04-12T11:13:58.989572+0000","last_became_active":"2024-04-12T11:12:59.164163+0000","last_became_peered":"2024-04-12T11:12:59.164163+0000","last_unstale":"2024-04-12T11:13:58.989572+0000","last_undegraded":"2024-04-12T11:13:58.989572+0000","last_fullsized":"2024-04-12T11:13:58.989572+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:29:41.965765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":23,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987919+0000","last_change":"2024-04-12T11:12:57.757955+0000","last_active":"2024-04-12T11:13:58.987919+0000","last_peered":"2024-04-12T11:13:58.987919+0000","last_clean":"2024-04-12T11:13:58.987919+0000","last_became_active":"2024-04-12T11:12:57.757667+0000","last_became_peered":"2024-04-12T11:12:57.757667+0000","last_unstale":"2024-04-12T11:13:58.987919+0000","last_undegraded":"2024-04-12T11:13:58.987919+0000","last_fullsized":"2024-04-12T11:13:58.987919+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:03:59.584101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.751001+0000","last_change":"2024-04-12T11:12:59.164965+0000","last_active":"2024-04-12T11:13:59.751001+0000","last_peered":"2024-04-12T11:13:59.751001+0000","last_clean":"2024-04-12T11:13:59.751001+0000","last_became_active":"2024-04-12T11:12:59.164602+0000","last_became_peered":"2024-04-12T11:12:59.164602+0000","last_unstale":"2024-04-12T11:13:59.751001+0000","last_undegraded":"2024-04-12T11:13:59.751001+0000","last_fullsized":"2024-04-12T11:13:59.751001+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:16.279172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.276618+0000","last_change":"2024-04-12T11:12:57.765194+0000","last_active":"2024-04-12T11:13:59.276618+0000","last_peered":"2024-04-12T11:13:59.276618+0000","last_clean":"2024-04-12T11:13:59.276618+0000","last_became_active":"2024-04-12T11:12:57.764910+0000","last_became_peered":"2024-04-12T11:12:57.764910+0000","last_unstale":"2024-04-12T11:13:59.276618+0000","last_undegraded":"2024-04-12T11:13:59.276618+0000","last_fullsized":"2024-04-12T11:13:59.276618+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:51:58.266747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.990660+0000","last_change":"2024-04-12T11:12:59.163926+0000","last_active":"2024-04-12T11:13:58.990660+0000","last_peered":"2024-04-12T11:13:58.990660+0000","last_clean":"2024-04-12T11:13:58.990660+0000","last_became_active":"2024-04-12T11:12:59.163561+0000","last_became_peered":"2024-04-12T11:12:59.163561+0000","last_unstale":"2024-04-12T11:13:58.990660+0000","last_undegraded":"2024-04-12T11:13:58.990660+0000","last_fullsized":"2024-04-12T11:13:58.990660+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:07:16.980521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.750917+0000","last_change":"2024-04-12T11:12:57.767434+0000","last_active":"2024-04-12T11:13:59.750917+0000","last_peered":"2024-04-12T11:13:59.750917+0000","last_clean":"2024-04-12T11:13:59.750917+0000","last_became_active":"2024-04-12T11:12:57.767138+0000","last_became_peered":"2024-04-12T11:12:57.767138+0000","last_unstale":"2024-04-12T11:13:59.750917+0000","last_undegraded":"2024-04-12T11:13:59.750917+0000","last_fullsized":"2024-04-12T11:13:59.750917+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:55:01.265880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.989089+0000","last_change":"2024-04-12T11:12:59.163760+0000","last_active":"2024-04-12T11:13:58.989089+0000","last_peered":"2024-04-12T11:13:58.989089+0000","last_clean":"2024-04-12T11:13:58.989089+0000","last_became_active":"2024-04-12T11:12:59.163409+0000","last_became_peered":"2024-04-12T11:12:59.163409+0000","last_unstale":"2024-04-12T11:13:58.989089+0000","last_undegraded":"2024-04-12T11:13:58.989089+0000","last_fullsized":"2024-04-12T11:13:58.989089+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:19.432190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.276839+0000","last_change":"2024-04-12T11:12:57.761557+0000","last_active":"2024-04-12T11:13:59.276839+0000","last_peered":"2024-04-12T11:13:59.276839+0000","last_clean":"2024-04-12T11:13:59.276839+0000","last_became_active":"2024-04-12T11:12:57.761289+0000","last_became_peered":"2024-04-12T11:12:57.761289+0000","last_unstale":"2024-04-12T11:13:59.276839+0000","last_undegraded":"2024-04-12T11:13:59.276839+0000","last_fullsized":"2024-04-12T11:13:59.276839+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:35:25.156169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987566+0000","last_change":"2024-04-12T11:12:58.788035+0000","last_active":"2024-04-12T11:13:58.987566+0000","last_peered":"2024-04-12T11:13:58.987566+0000","last_clean":"2024-04-12T11:13:58.987566+0000","last_became_active":"2024-04-12T11:12:58.787757+0000","last_became_peered":"2024-04-12T11:12:58.787757+0000","last_unstale":"2024-04-12T11:13:58.987566+0000","last_undegraded":"2024-04-12T11:13:58.987566+0000","last_fullsized":"2024-04-12T11:13:58.987566+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:13:06.142205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987716+0000","last_change":"2024-04-12T11:12:57.762165+0000","last_active":"2024-04-12T11:13:58.987716+0000","last_peered":"2024-04-12T11:13:58.987716+0000","last_clean":"2024-04-12T11:13:58.987716+0000","last_became_active":"2024-04-12T11:12:57.761892+0000","last_became_peered":"2024-04-12T11:12:57.761892+0000","last_unstale":"2024-04-12T11:13:58.987716+0000","last_undegraded":"2024-04-12T11:13:58.987716+0000","last_fullsized":"2024-04-12T11:13:58.987716+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:16:06.935058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987127+0000","last_change":"2024-04-12T11:12:57.781767+0000","last_active":"2024-04-12T11:13:58.987127+0000","last_peered":"2024-04-12T11:13:58.987127+0000","last_clean":"2024-04-12T11:13:58.987127+0000","last_became_active":"2024-04-12T11:12:57.781458+0000","last_became_peered":"2024-04-12T11:12:57.781458+0000","last_unstale":"2024-04-12T11:13:58.987127+0000","last_undegraded":"2024-04-12T11:13:58.987127+0000","last_fullsized":"2024-04-12T11:13:58.987127+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:17:45.096856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.276974+0000","last_change":"2024-04-12T11:12:57.770617+0000","last_active":"2024-04-12T11:13:59.276974+0000","last_peered":"2024-04-12T11:13:59.276974+0000","last_clean":"2024-04-12T11:13:59.276974+0000","last_became_active":"2024-04-12T11:12:57.770304+0000","last_became_peered":"2024-04-12T11:12:57.770304+0000","last_unstale":"2024-04-12T11:13:59.276974+0000","last_undegraded":"2024-04-12T11:13:59.276974+0000","last_fullsized":"2024-04-12T11:13:59.276974+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:23:12.952288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.989346+0000","last_change":"2024-04-12T11:12:59.164937+0000","last_active":"2024-04-12T11:13:58.989346+0000","last_peered":"2024-04-12T11:13:58.989346+0000","last_clean":"2024-04-12T11:13:58.989346+0000","last_became_active":"2024-04-12T11:12:59.164385+0000","last_became_peered":"2024-04-12T11:12:59.164385+0000","last_unstale":"2024-04-12T11:13:58.989346+0000","last_undegraded":"2024-04-12T11:13:58.989346+0000","last_fullsized":"2024-04-12T11:13:58.989346+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:43:00.186857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.991036+0000","last_change":"2024-04-12T11:12:57.759915+0000","last_active":"2024-04-12T11:13:58.991036+0000","last_peered":"2024-04-12T11:13:58.991036+0000","last_clean":"2024-04-12T11:13:58.991036+0000","last_became_active":"2024-04-12T11:12:57.759614+0000","last_became_peered":"2024-04-12T11:12:57.759614+0000","last_unstale":"2024-04-12T11:13:58.991036+0000","last_undegraded":"2024-04-12T11:13:58.991036+0000","last_fullsized":"2024-04-12T11:13:58.991036+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:54.241243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-12T11:14:00.033086+0000","last_change":"2024-04-12T11:14:00.033086+0000","last_active":"2024-04-12T11:14:00.033086+0000","last_peered":"2024-04-12T11:14:00.033086+0000","last_clean":"2024-04-12T11:14:00.033086+0000","last_became_active":"2024-04-12T11:14:00.032739+0000","last_became_peered":"2024-04-12T11:14:00.032739+0000","last_unstale":"2024-04-12T11:14:00.033086+0000","last_undegraded":"2024-04-12T11:14:00.033086+0000","last_fullsized":"2024-04-12T11:14:00.033086+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:02:29.336826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.751271+0000","last_change":"2024-04-12T11:12:57.769092+0000","last_active":"2024-04-12T11:13:59.751271+0000","last_peered":"2024-04-12T11:13:59.751271+0000","last_clean":"2024-04-12T11:13:59.751271+0000","last_became_active":"2024-04-12T11:12:57.768652+0000","last_became_peered":"2024-04-12T11:12:57.768652+0000","last_unstale":"2024-04-12T11:13:59.751271+0000","last_undegraded":"2024-04-12T11:13:59.751271+0000","last_fullsized":"2024-04-12T11:13:59.751271+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:23:40.531470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.988434+0000","last_change":"2024-04-12T11:12:57.830539+0000","last_active":"2024-04-12T11:13:58.988434+0000","last_peered":"2024-04-12T11:13:58.988434+0000","last_clean":"2024-04-12T11:13:58.988434+0000","last_became_active":"2024-04-12T11:12:57.830211+0000","last_became_peered":"2024-04-12T11:12:57.830211+0000","last_unstale":"2024-04-12T11:13:58.988434+0000","last_undegraded":"2024-04-12T11:13:58.988434+0000","last_fullsized":"2024-04-12T11:13:58.988434+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:29:42.745036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.990183+0000","last_change":"2024-04-12T11:12:57.770057+0000","last_active":"2024-04-12T11:13:58.990183+0000","last_peered":"2024-04-12T11:13:58.990183+0000","last_clean":"2024-04-12T11:13:58.990183+0000","last_became_active":"2024-04-12T11:12:57.769438+0000","last_became_peered":"2024-04-12T11:12:57.769438+0000","last_unstale":"2024-04-12T11:13:58.990183+0000","last_undegraded":"2024-04-12T11:13:58.990183+0000","last_fullsized":"2024-04-12T11:13:58.990183+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:20:55.956601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:59.751023+0000","last_change":"2024-04-12T11:12:59.163978+0000","last_active":"2024-04-12T11:13:59.751023+0000","last_peered":"2024-04-12T11:13:59.751023+0000","last_clean":"2024-04-12T11:13:59.751023+0000","last_became_active":"2024-04-12T11:12:59.163643+0000","last_became_peered":"2024-04-12T11:12:59.163643+0000","last_unstale":"2024-04-12T11:13:59.751023+0000","last_undegraded":"2024-04-12T11:13:59.751023+0000","last_fullsized":"2024-04-12T11:13:59.751023+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:22:02.655822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.986998+0000","last_change":"2024-04-12T11:12:57.765860+0000","last_active":"2024-04-12T11:13:58.986998+0000","last_peered":"2024-04-12T11:13:58.986998+0000","last_clean":"2024-04-12T11:13:58.986998+0000","last_became_active":"2024-04-12T11:12:57.765539+0000","last_became_peered":"2024-04-12T11:12:57.765539+0000","last_unstale":"2024-04-12T11:13:58.986998+0000","last_undegraded":"2024-04-12T11:13:58.986998+0000","last_fullsized":"2024-04-12T11:13:58.986998+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:54:43.603517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.989324+0000","last_change":"2024-04-12T11:12:58.786291+0000","last_active":"2024-04-12T11:13:58.989324+0000","last_peered":"2024-04-12T11:13:58.989324+0000","last_clean":"2024-04-12T11:13:58.989324+0000","last_became_active":"2024-04-12T11:12:58.785657+0000","last_became_peered":"2024-04-12T11:12:58.785657+0000","last_unstale":"2024-04-12T11:13:58.989324+0000","last_undegraded":"2024-04-12T11:13:58.989324+0000","last_fullsized":"2024-04-12T11:13:58.989324+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:28:04.443928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-12T11:13:58.987572+0000","last_change":"2024-04-12T11:12:57.757459+0000","last_active":"2024-04-12T11:13:58.987572+0000","last_peered":"2024-04-12T11:13:58.987572+0000","last_clean":"2024-04-12T11:13:58.987572+0000","last_became_active":"2024-04-12T11:12:57.757149+0000","last_became_peered":"2024-04-12T11:12:57.757149+0000","last_unstale":"2024-04-12T11:13:58.987572+0000","last_undegraded":"2024-04-12T11:13:58.987572+0000","last_fullsized":"2024-04-12T11:13:58.987572+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:54:52.213208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234020,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28332,"kb_used_data":1308,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716820,"statfs":{"total":95995035648,"available":95966023680,"internally_reserved":0,"allocated":1339392,"data_stored":905624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.50900000000000001,"15min":0.503},"min":{"1min":0.39800000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.627},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56599999999999995,"15min":0.56999999999999995},"min":{"1min":0.42499999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.754,"5min":2.4780000000000002,"15min":2.4780000000000002},"last":0.63700000000000001}]},{"osd":1,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.41699999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.78300000000000003,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70399999999999996,"5min":1.712,"15min":1.712},"last":0.77500000000000002}]},{"osd":2,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.99099999999999999,"15min":1.071},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.72999999999999998,"5min":15.228999999999999,"15min":15.228999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.038,"15min":1.117},"min":{"1min":0.39900000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.73899999999999999,"5min":15.249000000000001,"15min":15.249000000000001},"last":0.67800000000000005}]},{"osd":3,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.53000000000000003,"15min":0.53100000000000003},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.67900000000000005,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.754},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56299999999999994,"15min":0.56200000000000006},"min":{"1min":0.36899999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78600000000000003,"5min":1.044,"15min":1.044},"last":0.65400000000000003}]},{"osd":4,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44,"15min":0.439},"min":{"1min":0.315,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.66600000000000004,"5min":0.752,"15min":0.752},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.433,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.626,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.54700000000000004}]},{"osd":5,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81499999999999995,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.52500000000000002,"15min":0.52900000000000003},"min":{"1min":0.34899999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71799999999999997,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.69999999999999996}]},{"osd":6,"last update":"Fri Apr 12 11:13:23 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.52400000000000002,"15min":0.52500000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.77300000000000002,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.52300000000000002,"15min":0.52600000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.76800000000000002,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.61399999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495657,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28312,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716840,"statfs":{"total":95995035648,"available":95966044160,"internally_reserved":0,"allocated":1323008,"data_stored":905255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:59 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.49399999999999999,"15min":0.47499999999999998},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52900000000000003,"15min":0.52200000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.79200000000000004,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.56799999999999995}]},{"osd":1,"last update":"Fri Apr 12 11:13:59 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.53800000000000003,"15min":0.52700000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.873,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.50800000000000001,"15min":0.48999999999999999},"min":{"1min":0.33000000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.746,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.59199999999999997}]},{"osd":2,"last update":"Fri Apr 12 11:13:59 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.53000000000000003,"15min":0.51300000000000001},"min":{"1min":0.35099999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.84099999999999997,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52400000000000002,"15min":0.51500000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.76000000000000001,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.628}]},{"osd":3,"last update":"Fri Apr 12 11:13:59 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.52500000000000002,"15min":0.51400000000000001},"min":{"1min":0.38100000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.56299999999999994,"15min":0.55100000000000005},"min":{"1min":0.38500000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.93700000000000006,"5min":1.03,"15min":1.03},"last":0.64300000000000002}]},{"osd":4,"last update":"Fri Apr 12 11:13:59 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.40500000000000003,"15min":0.379},"min":{"1min":0.28699999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.42999999999999999,"15min":0.40400000000000003},"min":{"1min":0.30199999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.153,"5min":1.153,"15min":1.153},"last":0.55200000000000005}]},{"osd":5,"last update":"Fri Apr 12 11:13:59 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.439,"15min":0.41299999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.65500000000000003,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.505},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.46000000000000002,"15min":0.43099999999999999},"min":{"1min":0.32700000000000001,"5min":0.316,"15min":0.316},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.61099999999999999}]},{"osd":7,"last update":"Fri Apr 12 11:13:24 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.48299999999999998,"15min":0.47799999999999998},"min":{"1min":0.34100000000000003,"5min":0.316,"15min":0.316},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.52800000000000002,"15min":0.52700000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.69399999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574912,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27720,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717432,"statfs":{"total":95995035648,"available":95966650368,"internally_reserved":0,"allocated":720896,"data_stored":313807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:05 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36699999999999999,"15min":0.317},"min":{"1min":0.26900000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54900000000000004,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":0.439},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.40699999999999997,"15min":0.34699999999999998},"min":{"1min":0.26000000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.81599999999999995,"5min":2.4020000000000001,"15min":2.4020000000000001},"last":0.53800000000000003}]},{"osd":2,"last update":"Fri Apr 12 11:13:25 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.42699999999999999,"15min":0.39100000000000001},"min":{"1min":0.254,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.69699999999999995,"5min":1.885,"15min":1.885},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.371,"5min":0.39500000000000002,"15min":0.375},"min":{"1min":0.26500000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.50600000000000001,"5min":2.0350000000000001,"15min":2.0350000000000001},"last":0.49399999999999999}]},{"osd":3,"last update":"Fri Apr 12 11:13:54 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.41899999999999998,"15min":0.41299999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.69099999999999995,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.46300000000000002,"15min":0.46700000000000003},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.55900000000000005,"5min":1.8069999999999999,"15min":1.8069999999999999},"last":0.51300000000000001}]},{"osd":4,"last update":"Fri Apr 12 11:13:14 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.58399999999999996,"15min":0.57099999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.80500000000000005,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.621},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.56299999999999994,"15min":0.55200000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.83399999999999996,"5min":1.589,"15min":1.589},"last":0.64000000000000001}]},{"osd":5,"last update":"Fri Apr 12 11:13:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59999999999999998,"15min":0.58999999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77900000000000003,"5min":1.833,"15min":1.833},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.57499999999999996,"15min":0.57399999999999995},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67300000000000004,"5min":2.012,"15min":2.012},"last":0.65400000000000003}]},{"osd":6,"last update":"Fri Apr 12 11:14:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56299999999999994,"15min":0.55700000000000005},"min":{"1min":0.371,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.747,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.64200000000000002,"15min":0.64300000000000002},"min":{"1min":0.50900000000000001,"5min":0.438,"15min":0.438},"max":{"1min":0.83599999999999997,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":0.72999999999999998}]},{"osd":7,"last update":"Fri Apr 12 11:13:25 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.69999999999999996,"15min":0.71199999999999997},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82699999999999996,"5min":1.859,"15min":1.859},"last":0.75},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.66700000000000004,"15min":0.68300000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89000000000000001,"5min":1.6559999999999999,"15min":1.6559999999999999},"last":0.68000000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32408,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712744,"statfs":{"total":95995035648,"available":95961849856,"internally_reserved":0,"allocated":1323008,"data_stored":904779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 11:14:01 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.371,"15min":0.32900000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.52000000000000002,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.375,"15min":0.33500000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.59099999999999997,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.42299999999999999}]},{"osd":2,"last update":"Fri Apr 12 11:13:27 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.40000000000000002,"15min":0.38600000000000001},"min":{"1min":0.247,"5min":0.219,"15min":0.219},"max":{"1min":0.626,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.372},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.38800000000000001,"15min":0.37},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.50700000000000001,"5min":1.393,"15min":1.393},"last":0.33100000000000002}]},{"osd":3,"last update":"Fri Apr 12 11:13:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.44600000000000001,"15min":0.438},"min":{"1min":0.29399999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.60699999999999998,"5min":1.532,"15min":1.532},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.42899999999999999,"15min":0.43099999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56499999999999995,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.40400000000000003}]},{"osd":4,"last update":"Fri Apr 12 11:13:12 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.57599999999999996,"15min":0.58299999999999996},"min":{"1min":0.378,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.84299999999999997,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.39400000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.73199999999999998,"5min":1.448,"15min":1.448},"last":0.48999999999999999}]},{"osd":5,"last update":"Fri Apr 12 11:13:40 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.59799999999999998,"15min":0.58999999999999997},"min":{"1min":0.39500000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.87,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.81399999999999995,"15min":0.92000000000000004},"min":{"1min":0.40100000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78400000000000003,"5min":7.9340000000000002,"15min":7.9340000000000002},"last":0.46600000000000003}]},{"osd":6,"last update":"Fri Apr 12 11:13:58 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.62,"15min":0.62},"min":{"1min":0.42499999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.504},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.64600000000000002,"15min":0.64000000000000001},"min":{"1min":0.46600000000000003,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.91000000000000003,"5min":1.425,"15min":1.425},"last":0.434}]},{"osd":7,"last update":"Fri Apr 12 11:13:21 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.59799999999999998,"15min":0.60399999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.67100000000000004,"15min":0.68100000000000005},"min":{"1min":0.47699999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.85599999999999998,"5min":1.506,"15min":1.506},"last":0.52300000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411387,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27736,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717416,"statfs":{"total":95995035648,"available":95966633984,"internally_reserved":0,"allocated":729088,"data_stored":314423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:26 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38800000000000001,"15min":0.375},"min":{"1min":0.23499999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.64900000000000002,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38500000000000001,"15min":0.36699999999999999},"min":{"1min":0.251,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.60699999999999998,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.34799999999999998}]},{"osd":1,"last update":"Fri Apr 12 11:13:26 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.40600000000000003,"15min":0.38900000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.53700000000000003,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.41499999999999998,"15min":0.40500000000000003},"min":{"1min":0.26000000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.31900000000000001}]},{"osd":3,"last update":"Fri Apr 12 11:13:56 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.45700000000000002,"15min":0.47099999999999997},"min":{"1min":0.29499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.57699999999999996,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.45200000000000001,"15min":0.45500000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.70399999999999996,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.41199999999999998}]},{"osd":4,"last update":"Fri Apr 12 11:13:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.96399999999999997,"15min":1.145},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.81000000000000005,"5min":13.686999999999999,"15min":13.686999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.002,"15min":1.181},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.81599999999999995,"5min":13.721,"15min":13.721},"last":0.60999999999999999}]},{"osd":5,"last update":"Fri Apr 12 11:13:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.39000000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.83899999999999997,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.621,"15min":0.63300000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76200000000000001,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.622}]},{"osd":6,"last update":"Fri Apr 12 11:13:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.629,"15min":0.63500000000000001},"min":{"1min":0.42099999999999999,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83899999999999997,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.61699999999999999,"15min":0.61899999999999999},"min":{"1min":0.441,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.78000000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.72699999999999998}]},{"osd":7,"last update":"Fri Apr 12 11:13:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.65800000000000003,"15min":0.67000000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.79600000000000004,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.66600000000000004,"15min":0.67800000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.86099999999999999,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.76200000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182455,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31820,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713332,"statfs":{"total":95995035648,"available":95962451968,"internally_reserved":0,"allocated":724992,"data_stored":313897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":31848904},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:50 2024","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.38,"15min":0.39700000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.47999999999999998,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.38400000000000001,"15min":0.38700000000000001},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.59799999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.30499999999999999}]},{"osd":1,"last update":"Fri Apr 12 11:13:50 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.40600000000000003,"15min":0.42899999999999999},"min":{"1min":0.25900000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.52700000000000002,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.40200000000000002,"15min":0.41899999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.51100000000000001,"5min":0.624,"15min":0.624},"last":0.27400000000000002}]},{"osd":2,"last update":"Fri Apr 12 11:13:50 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.435,"15min":0.45000000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.55600000000000005,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.375,"5min":0.42999999999999999,"15min":0.44800000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.57799999999999996,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.34499999999999997}]},{"osd":4,"last update":"Fri Apr 12 11:13:13 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.55900000000000005,"15min":0.56999999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.71799999999999997,"15min":0.626},"min":{"1min":0.38500000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.74099999999999999,"5min":11.705,"15min":11.705},"last":0.57899999999999996}]},{"osd":5,"last update":"Fri Apr 12 11:13:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.57799999999999996,"15min":0.59399999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.502},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.71399999999999997,"15min":0.64200000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.88800000000000001,"5min":11.68,"15min":11.68},"last":0.61099999999999999}]},{"osd":6,"last update":"Fri Apr 12 11:13:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59699999999999998,"15min":0.59799999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.627,"5min":0.64200000000000002,"15min":0.64400000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.56499999999999995}]},{"osd":7,"last update":"Fri Apr 12 11:13:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.59999999999999998,"15min":0.60399999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.53400000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018930,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":745472,"data_stored":315517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.59199999999999997},"min":{"1min":0.35699999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.79800000000000004,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54300000000000004,"15min":0.55000000000000004},"min":{"1min":0.35099999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.45900000000000002}]},{"osd":1,"last update":"Fri Apr 12 11:13:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55300000000000005,"15min":0.56499999999999995},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.73699999999999999,"15min":0.81399999999999995},"min":{"1min":0.371,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82599999999999996,"5min":5.5590000000000002,"15min":5.5590000000000002},"last":0.72699999999999998}]},{"osd":2,"last update":"Fri Apr 12 11:13:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56100000000000005,"15min":0.56899999999999995},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.76900000000000002,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.73699999999999999,"15min":0.81599999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.80900000000000005,"5min":5.5419999999999998,"15min":5.5419999999999998},"last":0.57899999999999996}]},{"osd":3,"last update":"Fri Apr 12 11:13:13 2024","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.67400000000000004,"15min":0.60399999999999998},"min":{"1min":0.34599999999999997,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":10.676,"5min":10.676,"15min":10.676},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.56699999999999995,"15min":0.58499999999999996},"min":{"1min":0.39500000000000002,"5min":0.373,"15min":0.373},"max":{"1min":0.73199999999999998,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.53500000000000003}]},{"osd":5,"last update":"Fri Apr 12 11:13:40 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.41599999999999998,"15min":0.40500000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.57799999999999996,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.40699999999999997,"15min":0.39100000000000001},"min":{"1min":0.30399999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.68100000000000005}]},{"osd":6,"last update":"Fri Apr 12 11:14:01 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.42699999999999999,"15min":0.40699999999999997},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.47199999999999998,"15min":0.45200000000000001},"min":{"1min":0.40100000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.77600000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.77600000000000002}]},{"osd":7,"last update":"Fri Apr 12 11:13:24 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.73699999999999999,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.52200000000000002,"15min":0.52300000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67600000000000005,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.71499999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757293,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27728,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717424,"statfs":{"total":95995035648,"available":95966642176,"internally_reserved":0,"allocated":724992,"data_stored":314128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 11:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.56100000000000005,"15min":0.57699999999999996},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.65300000000000002,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.498,"5min":0.52900000000000003,"15min":0.53900000000000003},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.63,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.50600000000000001}]},{"osd":1,"last update":"Fri Apr 12 11:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.53600000000000003,"15min":0.55100000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.68400000000000005,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.65500000000000003,"5min":0.873,"15min":0.873},"last":0.46899999999999997}]},{"osd":2,"last update":"Fri Apr 12 11:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.54700000000000004,"15min":0.56299999999999994},"min":{"1min":0.36199999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.65800000000000003,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55300000000000005,"15min":0.56499999999999995},"min":{"1min":0.41699999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.69899999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.48099999999999998}]},{"osd":3,"last update":"Fri Apr 12 11:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.56399999999999995,"15min":0.58499999999999996},"min":{"1min":0.39900000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.64500000000000002,"5min":1.016,"15min":1.016},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.57499999999999996,"15min":0.58799999999999997},"min":{"1min":0.42699999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.67200000000000004,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.52200000000000002}]},{"osd":4,"last update":"Fri Apr 12 11:13:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.40799999999999997,"15min":0.40200000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.65900000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.312},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.42599999999999999,"15min":0.41799999999999998},"min":{"1min":0.27300000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.39100000000000001}]},{"osd":6,"last update":"Fri Apr 12 11:14:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.46200000000000002,"15min":0.45900000000000002},"min":{"1min":0.34999999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.61499999999999999,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.52400000000000002,"15min":0.52000000000000002},"min":{"1min":0.438,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.69799999999999995,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.438}]},{"osd":7,"last update":"Fri Apr 12 11:13:20 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.48699999999999999,"15min":0.48499999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67200000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65600000000000003,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.41299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:14:07.391 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-12T11:14:07.391 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-12T11:14:07.391 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-12T11:14:07.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T11:14:07.618 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:14:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T11:14:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T11:14:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi087.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi087.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi087.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi087.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi087.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T11:14:08.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi087.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T11:14:08.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T11:14:08.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T11:14:08.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi087.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T11:14:08.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi087.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T11:14:08.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:07 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:14:08.200 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:08.200 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:14:08.200 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi087.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi087.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi087.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T11:14:08.201 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi087.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T11:14:08.202 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:08.202 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:07 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:14:10.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:09 smithi029 ceph-mon[39999]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:09 smithi029 ceph-mon[29349]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:10.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:09 smithi087 ceph-mon[32917]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:10.292 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:14:10.292 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-12T11:14:11.200 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-12T11:14:11.200 INFO:teuthology.run_tasks:Running task print... 2024-04-12T11:14:11.210 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-12T11:14:11.210 INFO:teuthology.run_tasks:Running task parallel... 2024-04-12T11:14:11.223 INFO:teuthology.task.parallel:starting parallel... 2024-04-12T11:14:11.223 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-12T11:14:11.224 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-04-12T11:14:11.225 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2024-04-12T11:14:11.227 INFO:tasks.rados:Beginning rados... 2024-04-12T11:14:11.227 INFO:tasks.rados:config is {'clients': ['client.0'], 'objects': 50, 'op_weights': {'delete': 50, 'read': 100, 'rollback': 50, 'snap_create': 50, 'snap_remove': 50, 'write': 100}, 'ops': 4000, 'write_append_excl': False} 2024-04-12T11:14:11.227 INFO:tasks.rados:overrides is {'admin_socket': {'branch': 'wip-rf-comb1'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'wip-rf-comb1', 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95'}} 2024-04-12T11:14:11.227 INFO:tasks.rados:config is {'clients': ['client.0'], 'objects': 50, 'op_weights': {'delete': 50, 'read': 100, 'rollback': 50, 'snap_create': 50, 'snap_remove': 50, 'write': 100}, 'ops': 4000, 'write_append_excl': False} 2024-04-12T11:14:11.228 INFO:tasks.rados:joining rados 2024-04-12T11:14:11.228 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-12T11:14:11.228 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-12T11:14:11.228 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi029.front.sepia.ceph.com 2024-04-12T11:14:11.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-12T11:14:11.229 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2024-04-12T11:14:11.229 INFO:tasks.rados:starting run 0 out of 1 2024-04-12T11:14:11.229 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-04-12T11:14:11.229 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-04-12T11:14:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:11 smithi029 ceph-mon[39999]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:11 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:11 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:11 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2756657214' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T11:14:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:11 smithi029 ceph-mon[29349]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:11 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:11 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:11 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2756657214' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T11:14:11.653 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:11.656 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:11.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:11 smithi087 ceph-mon[32917]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:11 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:11 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:11 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2756657214' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T11:14:11.990 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 11:14:11 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T11:14:11.612Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001844965s 2024-04-12T11:14:13.361 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:14:13] "GET /metrics HTTP/1.1" 200 33893 "" "Prometheus/2.43.0" 2024-04-12T11:14:13.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[39999]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:13.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:13.639 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[29349]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:13.639 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:13 smithi087 ceph-mon[32917]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:14:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:14:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:14:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:14:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:14:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:14:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:15.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:15 smithi029 ceph-mon[39999]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:15.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:15 smithi029 ceph-mon[29349]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:15 smithi087 ceph-mon[32917]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:17.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:17 smithi087 ceph-mon[32917]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:17.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:17 smithi029 ceph-mon[39999]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:17.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:17 smithi029 ceph-mon[29349]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:19.496 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_0' created 2024-04-12T11:14:19.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:19 smithi087 ceph-mon[32917]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:19.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:19 smithi029 ceph-mon[29349]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:19.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:19 smithi029 ceph-mon[39999]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:20.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-04-12T11:14:20.640 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:20.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:20 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/918294328' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-12T11:14:20.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:20 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/918294328' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-12T11:14:20.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:20 smithi087 ceph-mon[32917]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T11:14:20.761 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:20 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/918294328' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-12T11:14:20.761 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:20 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/918294328' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-12T11:14:20.761 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:20 smithi029 ceph-mon[29349]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T11:14:20.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:20 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/918294328' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-12T11:14:20.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:20 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/918294328' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-12T11:14:20.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:20 smithi029 ceph-mon[39999]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T11:14:21.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:20 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:20] ENGINE Bus STOPPING 2024-04-12T11:14:21.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:20 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T11:14:21.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:20 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:20] ENGINE Bus STOPPED 2024-04-12T11:14:21.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:20 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:20] ENGINE Bus STARTING 2024-04-12T11:14:21.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:21] ENGINE Serving on http://:::9283 2024-04-12T11:14:21.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:14:21] ENGINE Bus STARTED 2024-04-12T11:14:21.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-12T11:14:21.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:21 smithi087 ceph-mon[32917]: pgmap v82: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:21.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:21 smithi087 ceph-mon[32917]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:14:21.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:21 smithi087 ceph-mon[32917]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T11:14:21.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:21 smithi087 ceph-mon[32917]: from='client.? ' entity='client.admin' 2024-04-12T11:14:21.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:21 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:14:21.727 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-mon[39999]: pgmap v82: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:21.727 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-mon[39999]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:14:21.727 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-mon[39999]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T11:14:21.727 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-mon[39999]: from='client.? ' entity='client.admin' 2024-04-12T11:14:21.727 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:14:21.728 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-mon[29349]: pgmap v82: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:21.728 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-mon[29349]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:14:21.728 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-mon[29349]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T11:14:21.728 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-mon[29349]: from='client.? ' entity='client.admin' 2024-04-12T11:14:21.728 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:21 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:14:22.588 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:22.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:22 smithi087 ceph-mon[32917]: osdmap e73: 8 total, 8 up, 8 in 2024-04-12T11:14:22.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:22 smithi029 ceph-mon[29349]: osdmap e73: 8 total, 8 up, 8 in 2024-04-12T11:14:22.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:22 smithi029 ceph-mon[39999]: osdmap e73: 8 total, 8 up, 8 in 2024-04-12T11:14:23.359 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:14:23] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.43.0" 2024-04-12T11:14:23.637 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-04-12T11:14:23.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:23 smithi029 ceph-mon[39999]: pgmap v85: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:23.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:23 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/673427504' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:14:23.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:23 smithi029 ceph-mon[39999]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:14:23.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:23 smithi029 ceph-mon[29349]: pgmap v85: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:23.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:23 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/673427504' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:14:23.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:23 smithi029 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:14:23.754 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:23 smithi087 ceph-mon[32917]: pgmap v85: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:23.754 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:23 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/673427504' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:14:23.754 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:23 smithi087 ceph-mon[32917]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:14:24.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:24 smithi029 ceph-mon[39999]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:14:24.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:24 smithi029 ceph-mon[39999]: osdmap e74: 8 total, 8 up, 8 in 2024-04-12T11:14:24.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:24 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:24.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:24 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:24.708 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:24 smithi029 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:14:24.708 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:24 smithi029 ceph-mon[29349]: osdmap e74: 8 total, 8 up, 8 in 2024-04-12T11:14:24.709 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:24 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:24.709 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:24 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:24 smithi087 ceph-mon[32917]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:14:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:24 smithi087 ceph-mon[32917]: osdmap e74: 8 total, 8 up, 8 in 2024-04-12T11:14:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:24 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:24 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:25.449 DEBUG:teuthology.orchestra.run.smithi029:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 100 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --pool unique_pool_0 2024-04-12T11:14:25.477 INFO:tasks.rados.rados.0.smithi029.stdout:adding op weight read -> 100 2024-04-12T11:14:25.477 INFO:tasks.rados.rados.0.smithi029.stdout:adding op weight write -> 100 2024-04-12T11:14:25.477 INFO:tasks.rados.rados.0.smithi029.stdout:adding op weight delete -> 50 2024-04-12T11:14:25.477 INFO:tasks.rados.rados.0.smithi029.stdout:adding op weight snap_create -> 50 2024-04-12T11:14:25.477 INFO:tasks.rados.rados.0.smithi029.stdout:adding op weight snap_remove -> 50 2024-04-12T11:14:25.477 INFO:tasks.rados.rados.0.smithi029.stdout:adding op weight rollback -> 50 2024-04-12T11:14:25.477 INFO:tasks.rados.rados.0.smithi029.stdout:ceph version 18.2.2-1189-gc96c8d47 (c96c8d478b644b6433700141c5f69a1a68f8121b) reef (stable) 2024-04-12T11:14:25.477 INFO:tasks.rados.rados.0.smithi029.stdout:Configuration: 2024-04-12T11:14:25.477 INFO:tasks.rados.rados.0.smithi029.stdout: Number of operations: 4000 2024-04-12T11:14:25.477 INFO:tasks.rados.rados.0.smithi029.stdout: Number of objects: 50 2024-04-12T11:14:25.478 INFO:tasks.rados.rados.0.smithi029.stdout: Max in flight operations: 16 2024-04-12T11:14:25.478 INFO:tasks.rados.rados.0.smithi029.stdout: Object size (in bytes): 4000000 2024-04-12T11:14:25.478 INFO:tasks.rados.rados.0.smithi029.stdout: Write stride min: 400000 2024-04-12T11:14:25.478 INFO:tasks.rados.rados.0.smithi029.stdout: Write stride max: 800000 2024-04-12T11:14:25.514 INFO:tasks.rados.rados.0.smithi029.stdout:1: write initial oid 1 2024-04-12T11:14:25.514 INFO:tasks.rados.rados.0.smithi029.stdout:1: seq_num 0 ranges {416807=475249,1542129=474678} 2024-04-12T11:14:25.519 INFO:tasks.rados.rados.0.smithi029.stdout:1: writing smithi02979277-1 from 416807 to 892056 tid 1 2024-04-12T11:14:25.525 INFO:tasks.rados.rados.0.smithi029.stdout:1: writing smithi02979277-1 from 1542129 to 2016807 tid 2 2024-04-12T11:14:25.525 INFO:tasks.rados.rados.0.smithi029.stdout:2: write initial oid 2 2024-04-12T11:14:25.526 INFO:tasks.rados.rados.0.smithi029.stdout:2: seq_num 1 ranges {416807=475249,1542129=474678} 2024-04-12T11:14:25.530 INFO:tasks.rados.rados.0.smithi029.stdout:2: writing smithi02979277-2 from 416807 to 892056 tid 1 2024-04-12T11:14:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:2: writing smithi02979277-2 from 1542129 to 2016807 tid 2 2024-04-12T11:14:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:3: write initial oid 3 2024-04-12T11:14:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:3: seq_num 2 ranges {433614=550498,1600611=433003} 2024-04-12T11:14:25.542 INFO:tasks.rados.rados.0.smithi029.stdout:3: writing smithi02979277-3 from 433614 to 984112 tid 1 2024-04-12T11:14:25.548 INFO:tasks.rados.rados.0.smithi029.stdout:3: writing smithi02979277-3 from 1600611 to 2033614 tid 2 2024-04-12T11:14:25.548 INFO:tasks.rados.rados.0.smithi029.stdout:4: write initial oid 4 2024-04-12T11:14:25.548 INFO:tasks.rados.rados.0.smithi029.stdout:4: seq_num 3 ranges {450421=625747,1659093=391328} 2024-04-12T11:14:25.554 INFO:tasks.rados.rados.0.smithi029.stdout:4: writing smithi02979277-4 from 450421 to 1076168 tid 1 2024-04-12T11:14:25.559 INFO:tasks.rados.rados.0.smithi029.stdout:4: writing smithi02979277-4 from 1659093 to 2050421 tid 2 2024-04-12T11:14:25.560 INFO:tasks.rados.rados.0.smithi029.stdout:5: write initial oid 5 2024-04-12T11:14:25.560 INFO:tasks.rados.rados.0.smithi029.stdout:5: seq_num 4 ranges {467228=700996,1717575=349653} 2024-04-12T11:14:25.566 INFO:tasks.rados.rados.0.smithi029.stdout:5: writing smithi02979277-5 from 467228 to 1168224 tid 1 2024-04-12T11:14:25.571 INFO:tasks.rados.rados.0.smithi029.stdout:5: writing smithi02979277-5 from 1717575 to 2067228 tid 2 2024-04-12T11:14:25.571 INFO:tasks.rados.rados.0.smithi029.stdout:6: write initial oid 6 2024-04-12T11:14:25.571 INFO:tasks.rados.rados.0.smithi029.stdout:6: seq_num 5 ranges {484035=776245,2059704=24331} 2024-04-12T11:14:25.577 INFO:tasks.rados.rados.0.smithi029.stdout:6: writing smithi02979277-6 from 484035 to 1260280 tid 1 2024-04-12T11:14:25.582 INFO:tasks.rados.rados.0.smithi029.stdout:6: writing smithi02979277-6 from 2059704 to 2084035 tid 2 2024-04-12T11:14:25.582 INFO:tasks.rados.rados.0.smithi029.stdout:7: write initial oid 7 2024-04-12T11:14:25.582 INFO:tasks.rados.rados.0.smithi029.stdout:7: seq_num 6 ranges {500842=451494,1718186=382656} 2024-04-12T11:14:25.587 INFO:tasks.rados.rados.0.smithi029.stdout:7: writing smithi02979277-7 from 500842 to 952336 tid 1 2024-04-12T11:14:25.594 INFO:tasks.rados.rados.0.smithi029.stdout:7: writing smithi02979277-7 from 1718186 to 2100842 tid 2 2024-04-12T11:14:25.594 INFO:tasks.rados.rados.0.smithi029.stdout:8: write initial oid 8 2024-04-12T11:14:25.594 INFO:tasks.rados.rados.0.smithi029.stdout:8: seq_num 7 ranges {517649=526743,1776668=340981} 2024-04-12T11:14:25.599 INFO:tasks.rados.rados.0.smithi029.stdout:8: writing smithi02979277-8 from 517649 to 1044392 tid 1 2024-04-12T11:14:25.605 INFO:tasks.rados.rados.0.smithi029.stdout:8: writing smithi02979277-8 from 1776668 to 2117649 tid 2 2024-04-12T11:14:25.605 INFO:tasks.rados.rados.0.smithi029.stdout:9: write initial oid 9 2024-04-12T11:14:25.606 INFO:tasks.rados.rados.0.smithi029.stdout:9: seq_num 8 ranges {534456=718345,1951503=182953} 2024-04-12T11:14:25.612 INFO:tasks.rados.rados.0.smithi029.stdout:9: writing smithi02979277-9 from 534456 to 1252801 tid 1 2024-04-12T11:14:25.617 INFO:tasks.rados.rados.0.smithi029.stdout:9: writing smithi02979277-9 from 1951503 to 2134456 tid 2 2024-04-12T11:14:25.617 INFO:tasks.rados.rados.0.smithi029.stdout:10: write initial oid 10 2024-04-12T11:14:25.617 INFO:tasks.rados.rados.0.smithi029.stdout:10: seq_num 9 ranges {551263=793594,1893632=257631} 2024-04-12T11:14:25.624 INFO:tasks.rados.rados.0.smithi029.stdout:10: writing smithi02979277-10 from 551263 to 1344857 tid 1 2024-04-12T11:14:25.629 INFO:tasks.rados.rados.0.smithi029.stdout:10: writing smithi02979277-10 from 1893632 to 2151263 tid 2 2024-04-12T11:14:25.629 INFO:tasks.rados.rados.0.smithi029.stdout:11: write initial oid 11 2024-04-12T11:14:25.629 INFO:tasks.rados.rados.0.smithi029.stdout:11: seq_num 10 ranges {568070=468843,1552114=615956} 2024-04-12T11:14:25.634 INFO:tasks.rados.rados.0.smithi029.stdout:11: writing smithi02979277-11 from 568070 to 1036913 tid 1 2024-04-12T11:14:25.641 INFO:tasks.rados.rados.0.smithi029.stdout:11: writing smithi02979277-11 from 1552114 to 2168070 tid 2 2024-04-12T11:14:25.641 INFO:tasks.rados.rados.0.smithi029.stdout:12: write initial oid 12 2024-04-12T11:14:25.641 INFO:tasks.rados.rados.0.smithi029.stdout:12: seq_num 11 ranges {584877=544092,1610596=562003} 2024-04-12T11:14:25.647 INFO:tasks.rados.rados.0.smithi029.stdout:12: writing smithi02979277-12 from 584877 to 1128969 tid 1 2024-04-12T11:14:25.653 INFO:tasks.rados.rados.0.smithi029.stdout:12: writing smithi02979277-12 from 1610596 to 2172599 tid 2 2024-04-12T11:14:25.653 INFO:tasks.rados.rados.0.smithi029.stdout:13: write initial oid 13 2024-04-12T11:14:25.653 INFO:tasks.rados.rados.0.smithi029.stdout:13: seq_num 12 ranges {601684=619341,1669078=532606} 2024-04-12T11:14:25.661 INFO:tasks.rados.rados.0.smithi029.stdout:13: writing smithi02979277-13 from 601684 to 1221025 tid 1 2024-04-12T11:14:25.665 INFO:tasks.rados.rados.0.smithi029.stdout:13: writing smithi02979277-13 from 1669078 to 2201684 tid 2 2024-04-12T11:14:25.665 INFO:tasks.rados.rados.0.smithi029.stdout:14: write initial oid 14 2024-04-12T11:14:25.665 INFO:tasks.rados.rados.0.smithi029.stdout:14: seq_num 13 ranges {618491=694590,2011207=207284} 2024-04-12T11:14:25.672 INFO:tasks.rados.rados.0.smithi029.stdout:14: writing smithi02979277-14 from 618491 to 1313081 tid 1 2024-04-12T11:14:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:14: writing smithi02979277-14 from 2011207 to 2218491 tid 2 2024-04-12T11:14:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:15: write initial oid 15 2024-04-12T11:14:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:15: seq_num 14 ranges {635298=769839,2069689=165609} 2024-04-12T11:14:25.685 INFO:tasks.rados.rados.0.smithi029.stdout:15: writing smithi02979277-15 from 635298 to 1405137 tid 1 2024-04-12T11:14:25.689 INFO:tasks.rados.rados.0.smithi029.stdout:15: writing smithi02979277-15 from 2069689 to 2235298 tid 2 2024-04-12T11:14:25.689 INFO:tasks.rados.rados.0.smithi029.stdout:16: write initial oid 16 2024-04-12T11:14:25.689 INFO:tasks.rados.rados.0.smithi029.stdout:16: seq_num 15 ranges {652105=445088,1728171=452988} 2024-04-12T11:14:25.695 INFO:tasks.rados.rados.0.smithi029.stdout:16: writing smithi02979277-16 from 652105 to 1097193 tid 1 2024-04-12T11:14:25.701 INFO:tasks.rados.rados.0.smithi029.stdout:16: writing smithi02979277-16 from 1728171 to 2181159 tid 2 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:1: finishing write tid 1 to smithi02979277-1 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:1: finishing write tid 2 to smithi02979277-1 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:1: finishing write tid 3 to smithi02979277-1 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:1: finishing write tid 4 to smithi02979277-1 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:1: oid 1 updating version 0 to 1 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:1: oid 1 updating version 1 to 2 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:1: oid 1 updating version 2 to 3 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 3 (ObjNum 0 snap 0 seq_num 0) dirty exists 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:2: finishing write tid 1 to smithi02979277-2 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:2: finishing write tid 2 to smithi02979277-2 2024-04-12T11:14:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:1: done (15 left) 2024-04-12T11:14:25.703 INFO:tasks.rados.rados.0.smithi029.stdout:17: write initial oid 17 2024-04-12T11:14:25.703 INFO:tasks.rados.rados.0.smithi029.stdout:17: seq_num 16 ranges {668912=636690,1903006=365906} 2024-04-12T11:14:25.708 INFO:tasks.rados.rados.0.smithi029.stdout:17: writing smithi02979277-17 from 668912 to 1305602 tid 1 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout:17: writing smithi02979277-17 from 1903006 to 2268912 tid 2 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout:2: finishing write tid 3 to smithi02979277-2 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout:2: finishing write tid 4 to smithi02979277-2 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout:2: oid 2 updating version 0 to 1 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout:2: oid 2 updating version 1 to 2 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout:2: oid 2 updating version 2 to 3 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 3 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout:3: finishing write tid 1 to smithi02979277-3 2024-04-12T11:14:25.715 INFO:tasks.rados.rados.0.smithi029.stdout:3: finishing write tid 2 to smithi02979277-3 2024-04-12T11:14:25.716 INFO:tasks.rados.rados.0.smithi029.stdout:2: done (15 left) 2024-04-12T11:14:25.716 INFO:tasks.rados.rados.0.smithi029.stdout:18: write initial oid 18 2024-04-12T11:14:25.716 INFO:tasks.rados.rados.0.smithi029.stdout:18: seq_num 17 ranges {685719=711939,1845135=440584} 2024-04-12T11:14:25.722 INFO:tasks.rados.rados.0.smithi029.stdout:18: writing smithi02979277-18 from 685719 to 1397658 tid 1 2024-04-12T11:14:25.727 INFO:tasks.rados.rados.0.smithi029.stdout:18: writing smithi02979277-18 from 1845135 to 2285719 tid 2 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:3: finishing write tid 3 to smithi02979277-3 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:3: finishing write tid 4 to smithi02979277-3 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:3: oid 3 updating version 0 to 1 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:3: oid 3 updating version 1 to 2 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:3: oid 3 updating version 2 to 3 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 3 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:4: finishing write tid 1 to smithi02979277-4 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:4: finishing write tid 2 to smithi02979277-4 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:3: done (15 left) 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:19: write initial oid 19 2024-04-12T11:14:25.728 INFO:tasks.rados.rados.0.smithi029.stdout:19: seq_num 18 ranges {702526=787188,1903617=398909} 2024-04-12T11:14:25.735 INFO:tasks.rados.rados.0.smithi029.stdout:19: writing smithi02979277-19 from 702526 to 1489714 tid 1 2024-04-12T11:14:25.740 INFO:tasks.rados.rados.0.smithi029.stdout:19: writing smithi02979277-19 from 1903617 to 2302526 tid 2 2024-04-12T11:14:25.740 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.740 INFO:tasks.rados.rados.0.smithi029.stdout:4: finishing write tid 3 to smithi02979277-4 2024-04-12T11:14:25.740 INFO:tasks.rados.rados.0.smithi029.stdout:4: finishing write tid 4 to smithi02979277-4 2024-04-12T11:14:25.740 INFO:tasks.rados.rados.0.smithi029.stdout:4: oid 4 updating version 0 to 2 2024-04-12T11:14:25.740 INFO:tasks.rados.rados.0.smithi029.stdout:4: oid 4 version 2 is already newer than 1 2024-04-12T11:14:25.740 INFO:tasks.rados.rados.0.smithi029.stdout:4: oid 4 updating version 2 to 3 2024-04-12T11:14:25.740 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 3 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-04-12T11:14:25.740 INFO:tasks.rados.rados.0.smithi029.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2024-04-12T11:14:25.740 INFO:tasks.rados.rados.0.smithi029.stdout:5: finishing write tid 1 to smithi02979277-5 2024-04-12T11:14:25.741 INFO:tasks.rados.rados.0.smithi029.stdout:4: done (15 left) 2024-04-12T11:14:25.741 INFO:tasks.rados.rados.0.smithi029.stdout:20: write initial oid 20 2024-04-12T11:14:25.741 INFO:tasks.rados.rados.0.smithi029.stdout:20: seq_num 19 ranges {719333=462437,1962099=357234} 2024-04-12T11:14:25.746 INFO:tasks.rados.rados.0.smithi029.stdout:20: writing smithi02979277-20 from 719333 to 1181770 tid 1 2024-04-12T11:14:25.753 INFO:tasks.rados.rados.0.smithi029.stdout:20: writing smithi02979277-20 from 1962099 to 2319333 tid 2 2024-04-12T11:14:25.753 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.753 INFO:tasks.rados.rados.0.smithi029.stdout:5: finishing write tid 2 to smithi02979277-5 2024-04-12T11:14:25.753 INFO:tasks.rados.rados.0.smithi029.stdout:5: finishing write tid 3 to smithi02979277-5 2024-04-12T11:14:25.753 INFO:tasks.rados.rados.0.smithi029.stdout:5: finishing write tid 4 to smithi02979277-5 2024-04-12T11:14:25.753 INFO:tasks.rados.rados.0.smithi029.stdout:5: oid 5 updating version 0 to 4 2024-04-12T11:14:25.753 INFO:tasks.rados.rados.0.smithi029.stdout:5: oid 5 updating version 4 to 5 2024-04-12T11:14:25.753 INFO:tasks.rados.rados.0.smithi029.stdout:5: oid 5 updating version 5 to 6 2024-04-12T11:14:25.753 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 6 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-04-12T11:14:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2024-04-12T11:14:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:6: finishing write tid 1 to smithi02979277-6 2024-04-12T11:14:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:5: done (15 left) 2024-04-12T11:14:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:21: write initial oid 21 2024-04-12T11:14:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:21: seq_num 20 ranges {736140=537686,2020581=315559} 2024-04-12T11:14:25.760 INFO:tasks.rados.rados.0.smithi029.stdout:21: writing smithi02979277-21 from 736140 to 1273826 tid 1 2024-04-12T11:14:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:21: writing smithi02979277-21 from 2020581 to 2336140 tid 2 2024-04-12T11:14:25.766 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:6: finishing write tid 2 to smithi02979277-6 2024-04-12T11:14:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:6: finishing write tid 3 to smithi02979277-6 2024-04-12T11:14:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:6: finishing write tid 4 to smithi02979277-6 2024-04-12T11:14:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:6: oid 6 updating version 0 to 2 2024-04-12T11:14:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:6: oid 6 version 2 is already newer than 1 2024-04-12T11:14:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:6: oid 6 updating version 2 to 3 2024-04-12T11:14:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 3 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-04-12T11:14:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2024-04-12T11:14:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:7: finishing write tid 1 to smithi02979277-7 2024-04-12T11:14:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:6: done (15 left) 2024-04-12T11:14:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:22: write initial oid 22 2024-04-12T11:14:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:22: seq_num 21 ranges {752947=612935,1962710=390237} 2024-04-12T11:14:25.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:25 smithi029 ceph-mon[39999]: pgmap v87: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:25.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:25 smithi029 ceph-mon[39999]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:14:25.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:25 smithi029 ceph-mon[39999]: Cluster is now healthy 2024-04-12T11:14:25.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:25 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2526178583' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-04-12T11:14:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:25 smithi029 ceph-mon[29349]: pgmap v87: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:25 smithi029 ceph-mon[29349]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:14:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:25 smithi029 ceph-mon[29349]: Cluster is now healthy 2024-04-12T11:14:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:25 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2526178583' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-04-12T11:14:25.773 INFO:tasks.rados.rados.0.smithi029.stdout:22: writing smithi02979277-22 from 752947 to 1365882 tid 1 2024-04-12T11:14:25.779 INFO:tasks.rados.rados.0.smithi029.stdout:22: writing smithi02979277-22 from 1962710 to 2352947 tid 2 2024-04-12T11:14:25.779 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.779 INFO:tasks.rados.rados.0.smithi029.stdout:7: finishing write tid 2 to smithi02979277-7 2024-04-12T11:14:25.779 INFO:tasks.rados.rados.0.smithi029.stdout:7: finishing write tid 3 to smithi02979277-7 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:7: finishing write tid 4 to smithi02979277-7 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:7: oid 7 updating version 0 to 2 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:7: oid 7 updating version 2 to 3 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:7: oid 7 version 3 is already newer than 1 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 3 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:8: finishing write tid 1 to smithi02979277-8 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:8: finishing write tid 2 to smithi02979277-8 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:7: done (15 left) 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:23: write initial oid 23 2024-04-12T11:14:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:23: seq_num 22 ranges {769754=688184,2021192=348562} 2024-04-12T11:14:25.786 INFO:tasks.rados.rados.0.smithi029.stdout:23: writing smithi02979277-23 from 769754 to 1457938 tid 1 2024-04-12T11:14:25.792 INFO:tasks.rados.rados.0.smithi029.stdout:23: writing smithi02979277-23 from 2021192 to 2369754 tid 2 2024-04-12T11:14:25.792 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.792 INFO:tasks.rados.rados.0.smithi029.stdout:8: finishing write tid 3 to smithi02979277-8 2024-04-12T11:14:25.792 INFO:tasks.rados.rados.0.smithi029.stdout:8: finishing write tid 4 to smithi02979277-8 2024-04-12T11:14:25.792 INFO:tasks.rados.rados.0.smithi029.stdout:8: oid 8 updating version 0 to 1 2024-04-12T11:14:25.793 INFO:tasks.rados.rados.0.smithi029.stdout:8: oid 8 updating version 1 to 3 2024-04-12T11:14:25.793 INFO:tasks.rados.rados.0.smithi029.stdout:8: oid 8 version 3 is already newer than 2 2024-04-12T11:14:25.793 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 3 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-04-12T11:14:25.793 INFO:tasks.rados.rados.0.smithi029.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2024-04-12T11:14:25.793 INFO:tasks.rados.rados.0.smithi029.stdout:9: finishing write tid 1 to smithi02979277-9 2024-04-12T11:14:25.793 INFO:tasks.rados.rados.0.smithi029.stdout:9: finishing write tid 2 to smithi02979277-9 2024-04-12T11:14:25.793 INFO:tasks.rados.rados.0.smithi029.stdout:9: finishing write tid 3 to smithi02979277-9 2024-04-12T11:14:25.793 INFO:tasks.rados.rados.0.smithi029.stdout:8: done (15 left) 2024-04-12T11:14:25.793 INFO:tasks.rados.rados.0.smithi029.stdout:24: write initial oid 24 2024-04-12T11:14:25.793 INFO:tasks.rados.rados.0.smithi029.stdout:24: seq_num 23 ranges {786561=479786,1796027=467664} 2024-04-12T11:14:25.798 INFO:tasks.rados.rados.0.smithi029.stdout:24: writing smithi02979277-24 from 786561 to 1266347 tid 1 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:24: writing smithi02979277-24 from 1796027 to 2263691 tid 2 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:9: finishing write tid 4 to smithi02979277-9 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:9: oid 9 updating version 0 to 1 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:9: oid 9 updating version 1 to 3 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:9: oid 9 version 3 is already newer than 2 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 3 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:10: finishing write tid 1 to smithi02979277-10 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:10: finishing write tid 2 to smithi02979277-10 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:9: done (15 left) 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:25: write initial oid 25 2024-04-12T11:14:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:25: seq_num 24 ranges {403368=555035,1454509=727675} 2024-04-12T11:14:25.809 INFO:tasks.rados.rados.0.smithi029.stdout:25: writing smithi02979277-25 from 403368 to 958403 tid 1 2024-04-12T11:14:25.818 INFO:tasks.rados.rados.0.smithi029.stdout:25: writing smithi02979277-25 from 1454509 to 2182184 tid 2 2024-04-12T11:14:25.818 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.818 INFO:tasks.rados.rados.0.smithi029.stdout:10: finishing write tid 3 to smithi02979277-10 2024-04-12T11:14:25.818 INFO:tasks.rados.rados.0.smithi029.stdout:10: finishing write tid 4 to smithi02979277-10 2024-04-12T11:14:25.818 INFO:tasks.rados.rados.0.smithi029.stdout:10: oid 10 updating version 0 to 1 2024-04-12T11:14:25.818 INFO:tasks.rados.rados.0.smithi029.stdout:10: oid 10 updating version 1 to 2 2024-04-12T11:14:25.818 INFO:tasks.rados.rados.0.smithi029.stdout:10: oid 10 updating version 2 to 3 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 3 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:11: finishing write tid 1 to smithi02979277-11 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:11: finishing write tid 2 to smithi02979277-11 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:11: finishing write tid 3 to smithi02979277-11 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:11: finishing write tid 4 to smithi02979277-11 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:11: oid 11 updating version 0 to 4 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:11: oid 11 updating version 4 to 5 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:11: oid 11 updating version 5 to 6 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 6 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:12: finishing write tid 1 to smithi02979277-12 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:12: finishing write tid 2 to smithi02979277-12 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:12: finishing write tid 3 to smithi02979277-12 2024-04-12T11:14:25.819 INFO:tasks.rados.rados.0.smithi029.stdout:12: finishing write tid 4 to smithi02979277-12 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:12: oid 12 updating version 0 to 4 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:12: oid 12 updating version 4 to 5 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:12: oid 12 updating version 5 to 6 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 6 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:13: finishing write tid 1 to smithi02979277-13 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:13: finishing write tid 2 to smithi02979277-13 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:13: finishing write tid 3 to smithi02979277-13 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:13: finishing write tid 4 to smithi02979277-13 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:13: oid 13 updating version 0 to 1 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:13: oid 13 updating version 1 to 2 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:13: oid 13 updating version 2 to 3 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 3 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-04-12T11:14:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:14: finishing write tid 1 to smithi02979277-14 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:14: finishing write tid 2 to smithi02979277-14 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:14: finishing write tid 3 to smithi02979277-14 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:14: finishing write tid 4 to smithi02979277-14 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:14: oid 14 updating version 0 to 8 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:14: oid 14 version 8 is already newer than 7 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:14: oid 14 updating version 8 to 9 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 9 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:15: finishing write tid 1 to smithi02979277-15 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:15: finishing write tid 2 to smithi02979277-15 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:15: finishing write tid 3 to smithi02979277-15 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:15: finishing write tid 4 to smithi02979277-15 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:15: oid 15 updating version 0 to 3 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:15: oid 15 version 3 is already newer than 2 2024-04-12T11:14:25.821 INFO:tasks.rados.rados.0.smithi029.stdout:15: oid 15 version 3 is already newer than 1 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 3 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:16: finishing write tid 1 to smithi02979277-16 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:10: done (15 left) 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:11: done (14 left) 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:12: done (13 left) 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:13: done (12 left) 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:14: done (11 left) 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:15: done (10 left) 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:26: write initial oid 26 2024-04-12T11:14:25.822 INFO:tasks.rados.rados.0.smithi029.stdout:26: seq_num 25 ranges {420175=630284,1796638=471333} 2024-04-12T11:14:25.823 INFO:tasks.rados.rados.0.smithi029.stdout:26: writing smithi02979277-26 from 420175 to 1050459 tid 1 2024-04-12T11:14:25.829 INFO:tasks.rados.rados.0.smithi029.stdout:26: writing smithi02979277-26 from 1796638 to 2267971 tid 2 2024-04-12T11:14:25.829 INFO:tasks.rados.rados.0.smithi029.stdout:27: write initial oid 27 2024-04-12T11:14:25.829 INFO:tasks.rados.rados.0.smithi029.stdout:27: seq_num 26 ranges {436982=705533,1855120=581862} 2024-04-12T11:14:25.835 INFO:tasks.rados.rados.0.smithi029.stdout:27: writing smithi02979277-27 from 436982 to 1142515 tid 1 2024-04-12T11:14:25.843 INFO:tasks.rados.rados.0.smithi029.stdout:27: writing smithi02979277-27 from 1855120 to 2436982 tid 2 2024-04-12T11:14:25.843 INFO:tasks.rados.rados.0.smithi029.stdout:28: write initial oid 28 2024-04-12T11:14:25.843 INFO:tasks.rados.rados.0.smithi029.stdout:28: seq_num 27 ranges {453789=780782,1913602=475002} 2024-04-12T11:14:25.850 INFO:tasks.rados.rados.0.smithi029.stdout:28: writing smithi02979277-28 from 453789 to 1234571 tid 1 2024-04-12T11:14:25.857 INFO:tasks.rados.rados.0.smithi029.stdout:28: writing smithi02979277-28 from 1913602 to 2388604 tid 2 2024-04-12T11:14:25.857 INFO:tasks.rados.rados.0.smithi029.stdout:29: write initial oid 29 2024-04-12T11:14:25.857 INFO:tasks.rados.rados.0.smithi029.stdout:29: seq_num 28 ranges {470596=456031,1572084=618660} 2024-04-12T11:14:25.862 INFO:tasks.rados.rados.0.smithi029.stdout:29: writing smithi02979277-29 from 470596 to 926627 tid 1 2024-04-12T11:14:25.869 INFO:tasks.rados.rados.0.smithi029.stdout:29: writing smithi02979277-29 from 1572084 to 2190744 tid 2 2024-04-12T11:14:25.869 INFO:tasks.rados.rados.0.smithi029.stdout:30: write initial oid 30 2024-04-12T11:14:25.869 INFO:tasks.rados.rados.0.smithi029.stdout:30: seq_num 29 ranges {487403=531280,1514213=478671} 2024-04-12T11:14:25.874 INFO:tasks.rados.rados.0.smithi029.stdout:30: writing smithi02979277-30 from 487403 to 1018683 tid 1 2024-04-12T11:14:25.880 INFO:tasks.rados.rados.0.smithi029.stdout:30: writing smithi02979277-30 from 1514213 to 1992884 tid 2 2024-04-12T11:14:25.880 INFO:tasks.rados.rados.0.smithi029.stdout:31: write initial oid 31 2024-04-12T11:14:25.880 INFO:tasks.rados.rados.0.smithi029.stdout:31: seq_num 30 ranges {504210=606529,1572695=622329} 2024-04-12T11:14:25.886 INFO:tasks.rados.rados.0.smithi029.stdout:31: writing smithi02979277-31 from 504210 to 1110739 tid 1 2024-04-12T11:14:25.896 INFO:tasks.rados.rados.0.smithi029.stdout:31: writing smithi02979277-31 from 1572695 to 2195024 tid 2 2024-04-12T11:14:25.896 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:25.896 INFO:tasks.rados.rados.0.smithi029.stdout:16: finishing write tid 2 to smithi02979277-16 2024-04-12T11:14:25.896 INFO:tasks.rados.rados.0.smithi029.stdout:16: finishing write tid 3 to smithi02979277-16 2024-04-12T11:14:25.896 INFO:tasks.rados.rados.0.smithi029.stdout:16: finishing write tid 4 to smithi02979277-16 2024-04-12T11:14:25.896 INFO:tasks.rados.rados.0.smithi029.stdout:16: oid 16 updating version 0 to 10 2024-04-12T11:14:25.896 INFO:tasks.rados.rados.0.smithi029.stdout:16: oid 16 updating version 10 to 11 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:16: oid 16 updating version 11 to 12 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 12 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:17: finishing write tid 1 to smithi02979277-17 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:17: finishing write tid 2 to smithi02979277-17 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:17: finishing write tid 3 to smithi02979277-17 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:17: finishing write tid 4 to smithi02979277-17 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:17: oid 17 updating version 0 to 5 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:17: oid 17 updating version 5 to 6 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:17: oid 17 version 6 is already newer than 4 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 6 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2024-04-12T11:14:25.897 INFO:tasks.rados.rados.0.smithi029.stdout:18: finishing write tid 1 to smithi02979277-18 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:18: finishing write tid 2 to smithi02979277-18 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:18: finishing write tid 3 to smithi02979277-18 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:18: finishing write tid 4 to smithi02979277-18 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:18: oid 18 updating version 0 to 8 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:18: oid 18 updating version 8 to 9 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:18: oid 18 version 9 is already newer than 7 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 9 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:19: finishing write tid 1 to smithi02979277-19 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:19: finishing write tid 2 to smithi02979277-19 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:19: finishing write tid 3 to smithi02979277-19 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:19: finishing write tid 4 to smithi02979277-19 2024-04-12T11:14:25.898 INFO:tasks.rados.rados.0.smithi029.stdout:19: oid 19 updating version 0 to 14 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:19: oid 19 version 14 is already newer than 13 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:19: oid 19 updating version 14 to 15 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 15 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:20: finishing write tid 1 to smithi02979277-20 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:20: finishing write tid 2 to smithi02979277-20 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:20: finishing write tid 3 to smithi02979277-20 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:20: finishing write tid 4 to smithi02979277-20 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:20: oid 20 updating version 0 to 3 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:20: oid 20 version 3 is already newer than 1 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:20: oid 20 version 3 is already newer than 2 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 3 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-04-12T11:14:25.899 INFO:tasks.rados.rados.0.smithi029.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:21: finishing write tid 1 to smithi02979277-21 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:21: finishing write tid 2 to smithi02979277-21 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:21: finishing write tid 3 to smithi02979277-21 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:21: finishing write tid 4 to smithi02979277-21 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:21: oid 21 updating version 0 to 5 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:21: oid 21 updating version 5 to 6 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:21: oid 21 version 6 is already newer than 4 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 6 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:22: finishing write tid 1 to smithi02979277-22 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:22: finishing write tid 2 to smithi02979277-22 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:22: finishing write tid 3 to smithi02979277-22 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:22: finishing write tid 4 to smithi02979277-22 2024-04-12T11:14:25.900 INFO:tasks.rados.rados.0.smithi029.stdout:22: oid 22 updating version 0 to 3 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:22: oid 22 version 3 is already newer than 2 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:22: oid 22 version 3 is already newer than 1 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 3 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:23: finishing write tid 1 to smithi02979277-23 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:23: finishing write tid 2 to smithi02979277-23 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:23: finishing write tid 3 to smithi02979277-23 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:23: finishing write tid 4 to smithi02979277-23 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:23: oid 23 updating version 0 to 6 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:23: oid 23 version 6 is already newer than 4 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:23: oid 23 version 6 is already newer than 5 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 6 (ObjNum 22 snap 0 seq_num 22) dirty exists 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:24: finishing write tid 1 to smithi02979277-24 2024-04-12T11:14:25.901 INFO:tasks.rados.rados.0.smithi029.stdout:24: finishing write tid 2 to smithi02979277-24 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:24: finishing write tid 3 to smithi02979277-24 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:24: finishing write tid 4 to smithi02979277-24 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:24: oid 24 updating version 0 to 9 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:24: oid 24 version 9 is already newer than 8 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:24: oid 24 version 9 is already newer than 7 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 9 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:25: finishing write tid 1 to smithi02979277-25 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:25: finishing write tid 2 to smithi02979277-25 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:25: finishing write tid 3 to smithi02979277-25 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:25: finishing write tid 4 to smithi02979277-25 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:25: oid 25 updating version 0 to 11 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:25: oid 25 version 11 is already newer than 10 2024-04-12T11:14:25.902 INFO:tasks.rados.rados.0.smithi029.stdout:25: oid 25 updating version 11 to 12 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 12 (ObjNum 24 snap 0 seq_num 24) dirty exists 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:26: finishing write tid 1 to smithi02979277-26 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:26: finishing write tid 2 to smithi02979277-26 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:26: finishing write tid 3 to smithi02979277-26 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:26: finishing write tid 4 to smithi02979277-26 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:26: oid 26 updating version 0 to 12 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:26: oid 26 version 12 is already newer than 10 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:26: oid 26 version 12 is already newer than 11 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 12 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2024-04-12T11:14:25.903 INFO:tasks.rados.rados.0.smithi029.stdout:27: finishing write tid 1 to smithi02979277-27 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:27: finishing write tid 2 to smithi02979277-27 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:27: finishing write tid 3 to smithi02979277-27 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:27: finishing write tid 4 to smithi02979277-27 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:27: oid 27 updating version 0 to 5 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:27: oid 27 version 5 is already newer than 4 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:27: oid 27 updating version 5 to 6 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 6 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:28: finishing write tid 1 to smithi02979277-28 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:28: finishing write tid 2 to smithi02979277-28 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:28: finishing write tid 3 to smithi02979277-28 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:28: finishing write tid 4 to smithi02979277-28 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:28: oid 28 updating version 0 to 5 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:28: oid 28 version 5 is already newer than 4 2024-04-12T11:14:25.904 INFO:tasks.rados.rados.0.smithi029.stdout:28: oid 28 updating version 5 to 6 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 6 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:29: finishing write tid 1 to smithi02979277-29 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:29: finishing write tid 2 to smithi02979277-29 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:29: finishing write tid 3 to smithi02979277-29 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:29: finishing write tid 4 to smithi02979277-29 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:29: oid 29 updating version 0 to 3 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:29: oid 29 version 3 is already newer than 2 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:29: oid 29 version 3 is already newer than 1 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 3 (ObjNum 28 snap 0 seq_num 28) dirty exists 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:30: finishing write tid 1 to smithi02979277-30 2024-04-12T11:14:25.905 INFO:tasks.rados.rados.0.smithi029.stdout:30: finishing write tid 2 to smithi02979277-30 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:30: finishing write tid 3 to smithi02979277-30 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:30: finishing write tid 4 to smithi02979277-30 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:30: oid 30 updating version 0 to 8 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:30: oid 30 updating version 8 to 9 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:30: oid 30 version 9 is already newer than 7 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 9 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:16: done (15 left) 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:17: done (14 left) 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:18: done (13 left) 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:19: done (12 left) 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:20: done (11 left) 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:21: done (10 left) 2024-04-12T11:14:25.906 INFO:tasks.rados.rados.0.smithi029.stdout:22: done (9 left) 2024-04-12T11:14:25.907 INFO:tasks.rados.rados.0.smithi029.stdout:23: done (8 left) 2024-04-12T11:14:25.907 INFO:tasks.rados.rados.0.smithi029.stdout:24: done (7 left) 2024-04-12T11:14:25.907 INFO:tasks.rados.rados.0.smithi029.stdout:25: done (6 left) 2024-04-12T11:14:25.907 INFO:tasks.rados.rados.0.smithi029.stdout:26: done (5 left) 2024-04-12T11:14:25.907 INFO:tasks.rados.rados.0.smithi029.stdout:27: done (4 left) 2024-04-12T11:14:25.907 INFO:tasks.rados.rados.0.smithi029.stdout:28: done (3 left) 2024-04-12T11:14:25.907 INFO:tasks.rados.rados.0.smithi029.stdout:29: done (2 left) 2024-04-12T11:14:25.907 INFO:tasks.rados.rados.0.smithi029.stdout:30: done (1 left) 2024-04-12T11:14:25.907 INFO:tasks.rados.rados.0.smithi029.stdout:32: write initial oid 32 2024-04-12T11:14:25.907 INFO:tasks.rados.rados.0.smithi029.stdout:32: seq_num 31 ranges {521017=798131,1747530=482340} 2024-04-12T11:14:25.908 INFO:tasks.rados.rados.0.smithi029.stdout:32: writing smithi02979277-32 from 521017 to 1319148 tid 1 2024-04-12T11:14:25.908 INFO:tasks.rados.rados.0.smithi029.stdout:32: writing smithi02979277-32 from 1747530 to 2229870 tid 2 2024-04-12T11:14:25.908 INFO:tasks.rados.rados.0.smithi029.stdout:33: write initial oid 33 2024-04-12T11:14:25.908 INFO:tasks.rados.rados.0.smithi029.stdout:33: seq_num 32 ranges {537824=473380,1806012=625998} 2024-04-12T11:14:25.910 INFO:tasks.rados.rados.0.smithi029.stdout:33: writing smithi02979277-33 from 537824 to 1011204 tid 1 2024-04-12T11:14:25.919 INFO:tasks.rados.rados.0.smithi029.stdout:33: writing smithi02979277-33 from 1806012 to 2432010 tid 2 2024-04-12T11:14:25.919 INFO:tasks.rados.rados.0.smithi029.stdout:34: write initial oid 34 2024-04-12T11:14:25.919 INFO:tasks.rados.rados.0.smithi029.stdout:34: seq_num 33 ranges {554631=548629,1748141=486009} 2024-04-12T11:14:25.924 INFO:tasks.rados.rados.0.smithi029.stdout:34: writing smithi02979277-34 from 554631 to 1103260 tid 1 2024-04-12T11:14:25.931 INFO:tasks.rados.rados.0.smithi029.stdout:34: writing smithi02979277-34 from 1748141 to 2234150 tid 2 2024-04-12T11:14:25.931 INFO:tasks.rados.rados.0.smithi029.stdout:35: write initial oid 35 2024-04-12T11:14:25.931 INFO:tasks.rados.rados.0.smithi029.stdout:35: seq_num 34 ranges {571438=623878,1806623=629667} 2024-04-12T11:14:25.937 INFO:tasks.rados.rados.0.smithi029.stdout:35: writing smithi02979277-35 from 571438 to 1195316 tid 1 2024-04-12T11:14:25.944 INFO:tasks.rados.rados.0.smithi029.stdout:35: writing smithi02979277-35 from 1806623 to 2436290 tid 2 2024-04-12T11:14:25.944 INFO:tasks.rados.rados.0.smithi029.stdout:36: write initial oid 36 2024-04-12T11:14:25.944 INFO:tasks.rados.rados.0.smithi029.stdout:36: seq_num 35 ranges {588245=699127,1865105=489678} 2024-04-12T11:14:25.951 INFO:tasks.rados.rados.0.smithi029.stdout:36: writing smithi02979277-36 from 588245 to 1287372 tid 1 2024-04-12T11:14:25.957 INFO:tasks.rados.rados.0.smithi029.stdout:36: writing smithi02979277-36 from 1865105 to 2354783 tid 2 2024-04-12T11:14:25.957 INFO:tasks.rados.rados.0.smithi029.stdout:37: write initial oid 37 2024-04-12T11:14:25.957 INFO:tasks.rados.rados.0.smithi029.stdout:37: seq_num 36 ranges {605052=774376,1923587=633336} 2024-04-12T11:14:25.964 INFO:tasks.rados.rados.0.smithi029.stdout:37: writing smithi02979277-37 from 605052 to 1379428 tid 1 2024-04-12T11:14:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:25 smithi087 ceph-mon[32917]: pgmap v87: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:25 smithi087 ceph-mon[32917]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:14:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:25 smithi087 ceph-mon[32917]: Cluster is now healthy 2024-04-12T11:14:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:25 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2526178583' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-04-12T11:14:25.971 INFO:tasks.rados.rados.0.smithi029.stdout:37: writing smithi02979277-37 from 1923587 to 2556923 tid 2 2024-04-12T11:14:25.971 INFO:tasks.rados.rados.0.smithi029.stdout:38: write initial oid 38 2024-04-12T11:14:25.971 INFO:tasks.rados.rados.0.smithi029.stdout:31: finishing write tid 1 to smithi02979277-31 2024-04-12T11:14:25.971 INFO:tasks.rados.rados.0.smithi029.stdout:38: seq_num 37 ranges {621859=449625,1865716=756143} 2024-04-12T11:14:25.977 INFO:tasks.rados.rados.0.smithi029.stdout:38: writing smithi02979277-38 from 621859 to 1071484 tid 1 2024-04-12T11:14:25.986 INFO:tasks.rados.rados.0.smithi029.stdout:38: writing smithi02979277-38 from 1865716 to 2621859 tid 2 2024-04-12T11:14:25.986 INFO:tasks.rados.rados.0.smithi029.stdout:39: write initial oid 39 2024-04-12T11:14:25.986 INFO:tasks.rados.rados.0.smithi029.stdout:39: seq_num 38 ranges {638666=524874,1924198=637005} 2024-04-12T11:14:25.992 INFO:tasks.rados.rados.0.smithi029.stdout:39: writing smithi02979277-39 from 638666 to 1163540 tid 1 2024-04-12T11:14:26.000 INFO:tasks.rados.rados.0.smithi029.stdout:39: writing smithi02979277-39 from 1924198 to 2561203 tid 2 2024-04-12T11:14:26.000 INFO:tasks.rados.rados.0.smithi029.stdout:40: write initial oid 40 2024-04-12T11:14:26.000 INFO:tasks.rados.rados.0.smithi029.stdout:40: seq_num 39 ranges {655473=716476,2099033=556440} 2024-04-12T11:14:26.008 INFO:tasks.rados.rados.0.smithi029.stdout:40: writing smithi02979277-40 from 655473 to 1371949 tid 1 2024-04-12T11:14:26.014 INFO:tasks.rados.rados.0.smithi029.stdout:40: writing smithi02979277-40 from 2099033 to 2655473 tid 2 2024-04-12T11:14:26.015 INFO:tasks.rados.rados.0.smithi029.stdout:41: write initial oid 41 2024-04-12T11:14:26.015 INFO:tasks.rados.rados.0.smithi029.stdout:41: seq_num 40 ranges {672280=791725,2157515=514765} 2024-04-12T11:14:26.022 INFO:tasks.rados.rados.0.smithi029.stdout:41: writing smithi02979277-41 from 672280 to 1464005 tid 1 2024-04-12T11:14:26.029 INFO:tasks.rados.rados.0.smithi029.stdout:41: writing smithi02979277-41 from 2157515 to 2672280 tid 2 2024-04-12T11:14:26.029 INFO:tasks.rados.rados.0.smithi029.stdout:42: write initial oid 42 2024-04-12T11:14:26.029 INFO:tasks.rados.rados.0.smithi029.stdout:42: seq_num 41 ranges {689087=466974,1699644=784332} 2024-04-12T11:14:26.035 INFO:tasks.rados.rados.0.smithi029.stdout:42: writing smithi02979277-42 from 689087 to 1156061 tid 1 2024-04-12T11:14:26.043 INFO:tasks.rados.rados.0.smithi029.stdout:42: writing smithi02979277-42 from 1699644 to 2483976 tid 2 2024-04-12T11:14:26.043 INFO:tasks.rados.rados.0.smithi029.stdout:43: write initial oid 43 2024-04-12T11:14:26.043 INFO:tasks.rados.rados.0.smithi029.stdout:43: seq_num 42 ranges {705894=542223,1758126=644343} 2024-04-12T11:14:26.049 INFO:tasks.rados.rados.0.smithi029.stdout:43: writing smithi02979277-43 from 705894 to 1248117 tid 1 2024-04-12T11:14:26.056 INFO:tasks.rados.rados.0.smithi029.stdout:43: writing smithi02979277-43 from 1758126 to 2402469 tid 2 2024-04-12T11:14:26.056 INFO:tasks.rados.rados.0.smithi029.stdout:44: write initial oid 44 2024-04-12T11:14:26.056 INFO:tasks.rados.rados.0.smithi029.stdout:44: seq_num 43 ranges {722701=617472,1816608=788001} 2024-04-12T11:14:26.063 INFO:tasks.rados.rados.0.smithi029.stdout:44: writing smithi02979277-44 from 722701 to 1340173 tid 1 2024-04-12T11:14:26.070 INFO:tasks.rados.rados.0.smithi029.stdout:44: writing smithi02979277-44 from 1816608 to 2604609 tid 2 2024-04-12T11:14:26.070 INFO:tasks.rados.rados.0.smithi029.stdout:45: write initial oid 45 2024-04-12T11:14:26.070 INFO:tasks.rados.rados.0.smithi029.stdout:45: seq_num 44 ranges {739508=692721,1875090=648012} 2024-04-12T11:14:26.078 INFO:tasks.rados.rados.0.smithi029.stdout:45: writing smithi02979277-45 from 739508 to 1432229 tid 1 2024-04-12T11:14:26.084 INFO:tasks.rados.rados.0.smithi029.stdout:45: writing smithi02979277-45 from 1875090 to 2523102 tid 2 2024-04-12T11:14:26.084 INFO:tasks.rados.rados.0.smithi029.stdout:46: write initial oid 46 2024-04-12T11:14:26.084 INFO:tasks.rados.rados.0.smithi029.stdout:46: seq_num 45 ranges {756315=767970,1933572=791670} 2024-04-12T11:14:26.093 INFO:tasks.rados.rados.0.smithi029.stdout:46: writing smithi02979277-46 from 756315 to 1524285 tid 1 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout:46: writing smithi02979277-46 from 1933572 to 2725242 tid 2 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout:31: finishing write tid 2 to smithi02979277-31 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout:31: finishing write tid 3 to smithi02979277-31 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout:31: finishing write tid 4 to smithi02979277-31 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout:31: oid 31 updating version 0 to 8 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout:31: oid 31 version 8 is already newer than 7 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout:31: oid 31 updating version 8 to 9 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 9 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2024-04-12T11:14:26.100 INFO:tasks.rados.rados.0.smithi029.stdout:32: finishing write tid 1 to smithi02979277-32 2024-04-12T11:14:26.101 INFO:tasks.rados.rados.0.smithi029.stdout:32: finishing write tid 2 to smithi02979277-32 2024-04-12T11:14:26.101 INFO:tasks.rados.rados.0.smithi029.stdout:31: done (15 left) 2024-04-12T11:14:26.101 INFO:tasks.rados.rados.0.smithi029.stdout:47: write initial oid 47 2024-04-12T11:14:26.101 INFO:tasks.rados.rados.0.smithi029.stdout:47: seq_num 46 ranges {773122=559572,1992054=651681} 2024-04-12T11:14:26.107 INFO:tasks.rados.rados.0.smithi029.stdout:47: writing smithi02979277-47 from 773122 to 1332694 tid 1 2024-04-12T11:14:26.115 INFO:tasks.rados.rados.0.smithi029.stdout:47: writing smithi02979277-47 from 1992054 to 2643735 tid 2 2024-04-12T11:14:26.115 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:26.115 INFO:tasks.rados.rados.0.smithi029.stdout:32: finishing write tid 3 to smithi02979277-32 2024-04-12T11:14:26.115 INFO:tasks.rados.rados.0.smithi029.stdout:32: finishing write tid 4 to smithi02979277-32 2024-04-12T11:14:26.115 INFO:tasks.rados.rados.0.smithi029.stdout:32: oid 32 updating version 0 to 5 2024-04-12T11:14:26.115 INFO:tasks.rados.rados.0.smithi029.stdout:32: oid 32 version 5 is already newer than 4 2024-04-12T11:14:26.115 INFO:tasks.rados.rados.0.smithi029.stdout:32: oid 32 updating version 5 to 6 2024-04-12T11:14:26.115 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 6 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:33: finishing write tid 1 to smithi02979277-33 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:33: finishing write tid 2 to smithi02979277-33 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:33: finishing write tid 3 to smithi02979277-33 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:33: finishing write tid 4 to smithi02979277-33 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:33: oid 33 updating version 0 to 9 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:33: oid 33 version 9 is already newer than 7 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:33: oid 33 version 9 is already newer than 8 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 9 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2024-04-12T11:14:26.116 INFO:tasks.rados.rados.0.smithi029.stdout:34: finishing write tid 1 to smithi02979277-34 2024-04-12T11:14:26.117 INFO:tasks.rados.rados.0.smithi029.stdout:34: finishing write tid 2 to smithi02979277-34 2024-04-12T11:14:26.117 INFO:tasks.rados.rados.0.smithi029.stdout:32: done (15 left) 2024-04-12T11:14:26.117 INFO:tasks.rados.rados.0.smithi029.stdout:33: done (14 left) 2024-04-12T11:14:26.117 INFO:tasks.rados.rados.0.smithi029.stdout:48: write initial oid 48 2024-04-12T11:14:26.117 INFO:tasks.rados.rados.0.smithi029.stdout:48: seq_num 47 ranges {789929=634821,2050536=739393} 2024-04-12T11:14:26.122 INFO:tasks.rados.rados.0.smithi029.stdout:48: writing smithi02979277-48 from 789929 to 1424750 tid 1 2024-04-12T11:14:26.130 INFO:tasks.rados.rados.0.smithi029.stdout:48: writing smithi02979277-48 from 2050536 to 2789929 tid 2 2024-04-12T11:14:26.130 INFO:tasks.rados.rados.0.smithi029.stdout:49: write initial oid 49 2024-04-12T11:14:26.130 INFO:tasks.rados.rados.0.smithi029.stdout:49: seq_num 48 ranges {406736=710070,1709018=655350} 2024-04-12T11:14:26.136 INFO:tasks.rados.rados.0.smithi029.stdout:49: writing smithi02979277-49 from 406736 to 1116806 tid 1 2024-04-12T11:14:26.145 INFO:tasks.rados.rados.0.smithi029.stdout:49: writing smithi02979277-49 from 1709018 to 2364368 tid 2 2024-04-12T11:14:26.145 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 16 2024-04-12T11:14:26.145 INFO:tasks.rados.rados.0.smithi029.stdout:34: finishing write tid 3 to smithi02979277-34 2024-04-12T11:14:26.145 INFO:tasks.rados.rados.0.smithi029.stdout:34: finishing write tid 4 to smithi02979277-34 2024-04-12T11:14:26.145 INFO:tasks.rados.rados.0.smithi029.stdout:34: oid 34 updating version 0 to 9 2024-04-12T11:14:26.145 INFO:tasks.rados.rados.0.smithi029.stdout:34: oid 34 version 9 is already newer than 8 2024-04-12T11:14:26.145 INFO:tasks.rados.rados.0.smithi029.stdout:34: oid 34 version 9 is already newer than 7 2024-04-12T11:14:26.145 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 9 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:35: finishing write tid 1 to smithi02979277-35 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:35: finishing write tid 2 to smithi02979277-35 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:35: finishing write tid 3 to smithi02979277-35 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:35: finishing write tid 4 to smithi02979277-35 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:35: oid 35 updating version 0 to 14 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:35: oid 35 version 14 is already newer than 13 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:35: oid 35 updating version 14 to 15 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 15 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:36: finishing write tid 1 to smithi02979277-36 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:36: finishing write tid 2 to smithi02979277-36 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:36: finishing write tid 3 to smithi02979277-36 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:34: done (15 left) 2024-04-12T11:14:26.146 INFO:tasks.rados.rados.0.smithi029.stdout:35: done (14 left) 2024-04-12T11:14:26.147 INFO:tasks.rados.rados.0.smithi029.stdout:50: write initial oid 50 2024-04-12T11:14:26.147 INFO:tasks.rados.rados.0.smithi029.stdout:50: seq_num 49 ranges {423543=785319,1767500=799008} 2024-04-12T11:14:26.152 INFO:tasks.rados.rados.0.smithi029.stdout:50: writing smithi02979277-50 from 423543 to 1208862 tid 1 2024-04-12T11:14:26.160 INFO:tasks.rados.rados.0.smithi029.stdout:50: writing smithi02979277-50 from 1767500 to 2566508 tid 2 2024-04-12T11:14:26.160 INFO:tasks.rados.rados.0.smithi029.stdout:51: write oid 2 current snap is 0 2024-04-12T11:14:26.160 INFO:tasks.rados.rados.0.smithi029.stdout:36: finishing write tid 4 to smithi02979277-36 2024-04-12T11:14:26.160 INFO:tasks.rados.rados.0.smithi029.stdout:36: oid 36 updating version 0 to 6 2024-04-12T11:14:26.160 INFO:tasks.rados.rados.0.smithi029.stdout:36: oid 36 version 6 is already newer than 4 2024-04-12T11:14:26.160 INFO:tasks.rados.rados.0.smithi029.stdout:36: oid 36 version 6 is already newer than 5 2024-04-12T11:14:26.160 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 6 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-04-12T11:14:26.160 INFO:tasks.rados.rados.0.smithi029.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2024-04-12T11:14:26.161 INFO:tasks.rados.rados.0.smithi029.stdout:37: finishing write tid 1 to smithi02979277-37 2024-04-12T11:14:26.161 INFO:tasks.rados.rados.0.smithi029.stdout:51: seq_num 50 ranges {440350=460568,1309629=542666,2323973=516377} 2024-04-12T11:14:26.165 INFO:tasks.rados.rados.0.smithi029.stdout:51: writing smithi02979277-2 from 440350 to 900918 tid 1 2024-04-12T11:14:26.170 INFO:tasks.rados.rados.0.smithi029.stdout:51: writing smithi02979277-2 from 1309629 to 1852295 tid 2 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:51: writing smithi02979277-2 from 2323973 to 2840350 tid 3 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:36: done (15 left) 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:52: rollback oid 14 current snap is 0 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:52: done (15 left) 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:53: write oid 19 current snap is 0 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:37: finishing write tid 2 to smithi02979277-37 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:37: finishing write tid 3 to smithi02979277-37 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:37: finishing write tid 4 to smithi02979277-37 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:37: oid 37 updating version 0 to 4 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:37: oid 37 updating version 4 to 5 2024-04-12T11:14:26.177 INFO:tasks.rados.rados.0.smithi029.stdout:37: oid 37 updating version 5 to 6 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 6 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:38: finishing write tid 1 to smithi02979277-38 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:38: finishing write tid 2 to smithi02979277-38 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:38: finishing write tid 3 to smithi02979277-38 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:38: finishing write tid 4 to smithi02979277-38 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:38: oid 38 updating version 0 to 5 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:38: oid 38 updating version 5 to 6 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:38: oid 38 version 6 is already newer than 4 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 6 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:39: finishing write tid 1 to smithi02979277-39 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:39: finishing write tid 2 to smithi02979277-39 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:39: finishing write tid 3 to smithi02979277-39 2024-04-12T11:14:26.178 INFO:tasks.rados.rados.0.smithi029.stdout:53: seq_num 51 ranges {457157=535817,1768111=402677} 2024-04-12T11:14:26.182 INFO:tasks.rados.rados.0.smithi029.stdout:53: writing smithi02979277-19 from 457157 to 992974 tid 1 2024-04-12T11:14:26.189 INFO:tasks.rados.rados.0.smithi029.stdout:53: writing smithi02979277-19 from 1768111 to 2170788 tid 2 2024-04-12T11:14:26.189 INFO:tasks.rados.rados.0.smithi029.stdout:37: done (15 left) 2024-04-12T11:14:26.190 INFO:tasks.rados.rados.0.smithi029.stdout:38: done (14 left) 2024-04-12T11:14:26.190 INFO:tasks.rados.rados.0.smithi029.stdout:54: write oid 22 current snap is 0 2024-04-12T11:14:26.190 INFO:tasks.rados.rados.0.smithi029.stdout:39: finishing write tid 4 to smithi02979277-39 2024-04-12T11:14:26.190 INFO:tasks.rados.rados.0.smithi029.stdout:39: oid 39 updating version 0 to 2 2024-04-12T11:14:26.190 INFO:tasks.rados.rados.0.smithi029.stdout:39: oid 39 updating version 2 to 3 2024-04-12T11:14:26.190 INFO:tasks.rados.rados.0.smithi029.stdout:39: oid 39 version 3 is already newer than 1 2024-04-12T11:14:26.190 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 3 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-04-12T11:14:26.190 INFO:tasks.rados.rados.0.smithi029.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2024-04-12T11:14:26.190 INFO:tasks.rados.rados.0.smithi029.stdout:40: finishing write tid 1 to smithi02979277-40 2024-04-12T11:14:26.190 INFO:tasks.rados.rados.0.smithi029.stdout:54: seq_num 52 ranges {473964=611066,1826593=546335,2778819=95145} 2024-04-12T11:14:26.196 INFO:tasks.rados.rados.0.smithi029.stdout:54: writing smithi02979277-22 from 473964 to 1085030 tid 1 2024-04-12T11:14:26.203 INFO:tasks.rados.rados.0.smithi029.stdout:54: writing smithi02979277-22 from 1826593 to 2372928 tid 2 2024-04-12T11:14:26.208 INFO:tasks.rados.rados.0.smithi029.stdout:54: writing smithi02979277-22 from 2778819 to 2873964 tid 3 2024-04-12T11:14:26.208 INFO:tasks.rados.rados.0.smithi029.stdout:39: done (14 left) 2024-04-12T11:14:26.208 INFO:tasks.rados.rados.0.smithi029.stdout:55: delete oid 10 current snap is 0 2024-04-12T11:14:26.208 INFO:tasks.rados.rados.0.smithi029.stdout:40: finishing write tid 2 to smithi02979277-40 2024-04-12T11:14:26.208 INFO:tasks.rados.rados.0.smithi029.stdout:40: finishing write tid 3 to smithi02979277-40 2024-04-12T11:14:26.208 INFO:tasks.rados.rados.0.smithi029.stdout:40: finishing write tid 4 to smithi02979277-40 2024-04-12T11:14:26.208 INFO:tasks.rados.rados.0.smithi029.stdout:40: oid 40 updating version 0 to 17 2024-04-12T11:14:26.208 INFO:tasks.rados.rados.0.smithi029.stdout:40: oid 40 version 17 is already newer than 16 2024-04-12T11:14:26.208 INFO:tasks.rados.rados.0.smithi029.stdout:40: oid 40 updating version 17 to 18 2024-04-12T11:14:26.208 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 18 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:41: finishing write tid 1 to smithi02979277-41 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:41: finishing write tid 2 to smithi02979277-41 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:41: finishing write tid 3 to smithi02979277-41 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:41: finishing write tid 4 to smithi02979277-41 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:41: oid 41 updating version 0 to 7 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:41: oid 41 updating version 7 to 9 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:41: oid 41 version 9 is already newer than 8 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 9 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:42: finishing write tid 1 to smithi02979277-42 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:42: finishing write tid 2 to smithi02979277-42 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:42: finishing write tid 3 to smithi02979277-42 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:42: finishing write tid 4 to smithi02979277-42 2024-04-12T11:14:26.209 INFO:tasks.rados.rados.0.smithi029.stdout:42: oid 42 updating version 0 to 2 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:42: oid 42 updating version 2 to 3 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:42: oid 42 version 3 is already newer than 1 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 3 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:43: finishing write tid 1 to smithi02979277-43 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:43: finishing write tid 2 to smithi02979277-43 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:43: finishing write tid 3 to smithi02979277-43 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:43: finishing write tid 4 to smithi02979277-43 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:43: oid 43 updating version 0 to 12 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:43: oid 43 version 12 is already newer than 11 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:43: oid 43 version 12 is already newer than 10 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 12 (ObjNum 42 snap 0 seq_num 42) dirty exists 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:44: finishing write tid 1 to smithi02979277-44 2024-04-12T11:14:26.210 INFO:tasks.rados.rados.0.smithi029.stdout:44: finishing write tid 2 to smithi02979277-44 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:44: finishing write tid 3 to smithi02979277-44 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:44: finishing write tid 4 to smithi02979277-44 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:44: oid 44 updating version 0 to 12 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:44: oid 44 version 12 is already newer than 10 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:44: oid 44 version 12 is already newer than 11 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 12 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:45: finishing write tid 1 to smithi02979277-45 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:45: finishing write tid 2 to smithi02979277-45 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:45: finishing write tid 3 to smithi02979277-45 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:45: finishing write tid 4 to smithi02979277-45 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:45: oid 45 updating version 0 to 8 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:45: oid 45 version 8 is already newer than 7 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:45: oid 45 updating version 8 to 9 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 9 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-04-12T11:14:26.211 INFO:tasks.rados.rados.0.smithi029.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:46: finishing write tid 1 to smithi02979277-46 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:46: finishing write tid 2 to smithi02979277-46 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:46: finishing write tid 3 to smithi02979277-46 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:46: finishing write tid 4 to smithi02979277-46 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:46: oid 46 updating version 0 to 9 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:46: oid 46 version 9 is already newer than 8 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:46: oid 46 version 9 is already newer than 7 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 9 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:47: finishing write tid 1 to smithi02979277-47 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:47: finishing write tid 2 to smithi02979277-47 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:47: finishing write tid 3 to smithi02979277-47 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:47: finishing write tid 4 to smithi02979277-47 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:47: oid 47 updating version 0 to 11 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:47: oid 47 updating version 11 to 12 2024-04-12T11:14:26.212 INFO:tasks.rados.rados.0.smithi029.stdout:47: oid 47 version 12 is already newer than 10 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 12 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:48: finishing write tid 1 to smithi02979277-48 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:48: finishing write tid 2 to smithi02979277-48 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:48: finishing write tid 3 to smithi02979277-48 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:48: finishing write tid 4 to smithi02979277-48 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:48: oid 48 updating version 0 to 12 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:48: oid 48 version 12 is already newer than 10 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:48: oid 48 version 12 is already newer than 11 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 12 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:49: finishing write tid 1 to smithi02979277-49 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:49: finishing write tid 2 to smithi02979277-49 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:49: finishing write tid 3 to smithi02979277-49 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:49: finishing write tid 4 to smithi02979277-49 2024-04-12T11:14:26.213 INFO:tasks.rados.rados.0.smithi029.stdout:49: oid 49 updating version 0 to 8 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:49: oid 49 updating version 8 to 9 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:49: oid 49 version 9 is already newer than 7 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 9 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:50: finishing write tid 1 to smithi02979277-50 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:50: finishing write tid 2 to smithi02979277-50 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:50: finishing write tid 3 to smithi02979277-50 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:50: finishing write tid 4 to smithi02979277-50 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:50: oid 50 updating version 0 to 11 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:50: oid 50 updating version 11 to 12 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:50: oid 50 version 12 is already newer than 10 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 12 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-04-12T11:14:26.214 INFO:tasks.rados.rados.0.smithi029.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:51: finishing write tid 1 to smithi02979277-2 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:51: finishing write tid 2 to smithi02979277-2 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:51: finishing write tid 3 to smithi02979277-2 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:51: finishing write tid 4 to smithi02979277-2 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:51: finishing write tid 5 to smithi02979277-2 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:51: oid 2 updating version 0 to 4 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:51: oid 2 updating version 4 to 6 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:51: oid 2 updating version 6 to 7 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:51: oid 2 version 7 is already newer than 5 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 7 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:51: left oid 2 (ObjNum 50 snap 0 seq_num 50) 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:53: finishing write tid 1 to smithi02979277-19 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:53: finishing write tid 2 to smithi02979277-19 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:53: finishing write tid 3 to smithi02979277-19 2024-04-12T11:14:26.215 INFO:tasks.rados.rados.0.smithi029.stdout:53: finishing write tid 4 to smithi02979277-19 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:53: oid 19 updating version 0 to 16 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:53: oid 19 updating version 16 to 18 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:53: oid 19 version 18 is already newer than 17 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 18 (ObjNum 51 snap 0 seq_num 51) dirty exists 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:53: left oid 19 (ObjNum 51 snap 0 seq_num 51) 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:54: finishing write tid 1 to smithi02979277-22 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:54: finishing write tid 2 to smithi02979277-22 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:40: done (14 left) 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:41: done (13 left) 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:42: done (12 left) 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:43: done (11 left) 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:44: done (10 left) 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:45: done (9 left) 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:46: done (8 left) 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:47: done (7 left) 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:48: done (6 left) 2024-04-12T11:14:26.216 INFO:tasks.rados.rados.0.smithi029.stdout:49: done (5 left) 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:50: done (4 left) 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:51: done (3 left) 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:53: done (2 left) 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:55: done (1 left) 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:56: read oid 15 snap -1 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:56: expect (ObjNum 14 snap 0 seq_num 14) 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:57: delete oid 13 current snap is 0 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:54: finishing write tid 3 to smithi02979277-22 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:54: finishing write tid 4 to smithi02979277-22 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:54: finishing write tid 5 to smithi02979277-22 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:54: oid 22 updating version 0 to 15 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:54: oid 22 updating version 15 to 16 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:54: oid 22 version 16 is already newer than 14 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:54: oid 22 version 16 is already newer than 13 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 16 (ObjNum 52 snap 0 seq_num 52) dirty exists 2024-04-12T11:14:26.217 INFO:tasks.rados.rados.0.smithi029.stdout:54: left oid 22 (ObjNum 52 snap 0 seq_num 52) 2024-04-12T11:14:26.218 INFO:tasks.rados.rados.0.smithi029.stdout:54: done (2 left) 2024-04-12T11:14:26.218 INFO:tasks.rados.rados.0.smithi029.stdout:57: done (1 left) 2024-04-12T11:14:26.218 INFO:tasks.rados.rados.0.smithi029.stdout:58: read oid 3 snap -1 2024-04-12T11:14:26.218 INFO:tasks.rados.rados.0.smithi029.stdout:58: expect (ObjNum 2 snap 0 seq_num 2) 2024-04-12T11:14:26.219 INFO:tasks.rados.rados.0.smithi029.stdout:59: read oid 28 snap -1 2024-04-12T11:14:26.219 INFO:tasks.rados.rados.0.smithi029.stdout:59: expect (ObjNum 27 snap 0 seq_num 27) 2024-04-12T11:14:26.219 INFO:tasks.rados.rados.0.smithi029.stdout:60: read oid 34 snap -1 2024-04-12T11:14:26.219 INFO:tasks.rados.rados.0.smithi029.stdout:60: expect (ObjNum 33 snap 0 seq_num 33) 2024-04-12T11:14:26.219 INFO:tasks.rados.rados.0.smithi029.stdout:61: delete oid 2 current snap is 0 2024-04-12T11:14:26.219 INFO:tasks.rados.rados.0.smithi029.stdout:56: expect (ObjNum 14 snap 0 seq_num 14) 2024-04-12T11:14:26.307 INFO:tasks.rados.rados.0.smithi029.stdout:58: expect (ObjNum 2 snap 0 seq_num 2) 2024-04-12T11:14:26.384 INFO:tasks.rados.rados.0.smithi029.stdout:59: expect (ObjNum 27 snap 0 seq_num 27) 2024-04-12T11:14:26.470 INFO:tasks.rados.rados.0.smithi029.stdout:60: expect (ObjNum 33 snap 0 seq_num 33) 2024-04-12T11:14:26.558 INFO:tasks.rados.rados.0.smithi029.stdout:56: done (4 left) 2024-04-12T11:14:26.558 INFO:tasks.rados.rados.0.smithi029.stdout:58: done (3 left) 2024-04-12T11:14:26.558 INFO:tasks.rados.rados.0.smithi029.stdout:59: done (2 left) 2024-04-12T11:14:26.558 INFO:tasks.rados.rados.0.smithi029.stdout:60: done (1 left) 2024-04-12T11:14:26.558 INFO:tasks.rados.rados.0.smithi029.stdout:61: done (0 left) 2024-04-12T11:14:26.558 INFO:tasks.rados.rados.0.smithi029.stdout:62: write oid 29 current snap is 0 2024-04-12T11:14:26.558 INFO:tasks.rados.rados.0.smithi029.stdout:62: seq_num 56 ranges {541192=628415,1660521=553673,2888511=52681} 2024-04-12T11:14:26.564 INFO:tasks.rados.rados.0.smithi029.stdout:62: writing smithi02979277-29 from 541192 to 1169607 tid 1 2024-04-12T11:14:26.569 INFO:tasks.rados.rados.0.smithi029.stdout:62: writing smithi02979277-29 from 1660521 to 2214194 tid 2 2024-04-12T11:14:26.574 INFO:tasks.rados.rados.0.smithi029.stdout:62: writing smithi02979277-29 from 2888511 to 2941192 tid 3 2024-04-12T11:14:26.574 INFO:tasks.rados.rados.0.smithi029.stdout:63: write oid 48 current snap is 0 2024-04-12T11:14:26.574 INFO:tasks.rados.rados.0.smithi029.stdout:63: seq_num 57 ranges {557999=703664,1719003=413684,2632287=325712} 2024-04-12T11:14:26.580 INFO:tasks.rados.rados.0.smithi029.stdout:63: writing smithi02979277-48 from 557999 to 1261663 tid 1 2024-04-12T11:14:26.585 INFO:tasks.rados.rados.0.smithi029.stdout:63: writing smithi02979277-48 from 1719003 to 2132687 tid 2 2024-04-12T11:14:26.590 INFO:tasks.rados.rados.0.smithi029.stdout:63: writing smithi02979277-48 from 2632287 to 2957999 tid 3 2024-04-12T11:14:26.590 INFO:tasks.rados.rados.0.smithi029.stdout:64: write oid 22 current snap is 0 2024-04-12T11:14:26.590 INFO:tasks.rados.rados.0.smithi029.stdout:64: seq_num 58 ranges {574806=778913,2061132=557342} 2024-04-12T11:14:26.597 INFO:tasks.rados.rados.0.smithi029.stdout:64: writing smithi02979277-22 from 574806 to 1353719 tid 1 2024-04-12T11:14:26.605 INFO:tasks.rados.rados.0.smithi029.stdout:64: writing smithi02979277-22 from 2061132 to 2618474 tid 2 2024-04-12T11:14:26.605 INFO:tasks.rados.rados.0.smithi029.stdout:65: rollback oid 7 current snap is 0 2024-04-12T11:14:26.605 INFO:tasks.rados.rados.0.smithi029.stdout:65: done (3 left) 2024-04-12T11:14:26.606 INFO:tasks.rados.rados.0.smithi029.stdout:66: read oid 24 snap -1 2024-04-12T11:14:26.606 INFO:tasks.rados.rados.0.smithi029.stdout:66: expect (ObjNum 23 snap 0 seq_num 23) 2024-04-12T11:14:26.606 INFO:tasks.rados.rados.0.smithi029.stdout:67: write oid 12 current snap is 0 2024-04-12T11:14:26.606 INFO:tasks.rados.rados.0.smithi029.stdout:62: finishing write tid 1 to smithi02979277-29 2024-04-12T11:14:26.606 INFO:tasks.rados.rados.0.smithi029.stdout:62: finishing write tid 2 to smithi02979277-29 2024-04-12T11:14:26.606 INFO:tasks.rados.rados.0.smithi029.stdout:62: finishing write tid 3 to smithi02979277-29 2024-04-12T11:14:26.606 INFO:tasks.rados.rados.0.smithi029.stdout:67: seq_num 59 ranges {591613=454162,1719614=417353,2570780=420833} 2024-04-12T11:14:26.611 INFO:tasks.rados.rados.0.smithi029.stdout:67: writing smithi02979277-12 from 591613 to 1045775 tid 1 2024-04-12T11:14:26.617 INFO:tasks.rados.rados.0.smithi029.stdout:67: writing smithi02979277-12 from 1719614 to 2136967 tid 2 2024-04-12T11:14:26.623 INFO:tasks.rados.rados.0.smithi029.stdout:67: writing smithi02979277-12 from 2570780 to 2991613 tid 3 2024-04-12T11:14:26.623 INFO:tasks.rados.rados.0.smithi029.stdout:68: write oid 25 current snap is 0 2024-04-12T11:14:26.623 INFO:tasks.rados.rados.0.smithi029.stdout:62: finishing write tid 4 to smithi02979277-29 2024-04-12T11:14:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:62: finishing write tid 5 to smithi02979277-29 2024-04-12T11:14:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:62: oid 29 updating version 0 to 16 2024-04-12T11:14:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:62: oid 29 version 16 is already newer than 14 2024-04-12T11:14:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:62: oid 29 version 16 is already newer than 13 2024-04-12T11:14:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:62: oid 29 version 16 is already newer than 15 2024-04-12T11:14:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 16 (ObjNum 56 snap 0 seq_num 56) dirty exists 2024-04-12T11:14:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:62: left oid 29 (ObjNum 56 snap 0 seq_num 56) 2024-04-12T11:14:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:63: finishing write tid 1 to smithi02979277-48 2024-04-12T11:14:26.625 INFO:tasks.rados.rados.0.smithi029.stdout:63: finishing write tid 2 to smithi02979277-48 2024-04-12T11:14:26.625 INFO:tasks.rados.rados.0.smithi029.stdout:63: finishing write tid 3 to smithi02979277-48 2024-04-12T11:14:26.625 INFO:tasks.rados.rados.0.smithi029.stdout:63: finishing write tid 4 to smithi02979277-48 2024-04-12T11:14:26.625 INFO:tasks.rados.rados.0.smithi029.stdout:68: seq_num 60 ranges {608420=529411,1778096=561011,2881850=126570} 2024-04-12T11:14:26.628 INFO:tasks.rados.rados.0.smithi029.stdout:68: writing smithi02979277-25 from 608420 to 1137831 tid 1 2024-04-12T11:14:26.634 INFO:tasks.rados.rados.0.smithi029.stdout:68: writing smithi02979277-25 from 1778096 to 2339107 tid 2 2024-04-12T11:14:26.638 INFO:tasks.rados.rados.0.smithi029.stdout:68: writing smithi02979277-25 from 2881850 to 3008420 tid 3 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:62: done (5 left) 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:69: write oid 28 current snap is 0 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:63: finishing write tid 5 to smithi02979277-48 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:63: oid 48 updating version 0 to 15 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:63: oid 48 updating version 15 to 16 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:63: oid 48 version 16 is already newer than 14 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:63: oid 48 updating version 16 to 17 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 17 (ObjNum 57 snap 0 seq_num 57) dirty exists 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:63: left oid 48 (ObjNum 57 snap 0 seq_num 57) 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:64: finishing write tid 1 to smithi02979277-22 2024-04-12T11:14:26.639 INFO:tasks.rados.rados.0.smithi029.stdout:69: seq_num 61 ranges {625227=721013,1952931=704669} 2024-04-12T11:14:26.646 INFO:tasks.rados.rados.0.smithi029.stdout:69: writing smithi02979277-28 from 625227 to 1346240 tid 1 2024-04-12T11:14:26.653 INFO:tasks.rados.rados.0.smithi029.stdout:69: writing smithi02979277-28 from 1952931 to 2657600 tid 2 2024-04-12T11:14:26.653 INFO:tasks.rados.rados.0.smithi029.stdout:63: done (5 left) 2024-04-12T11:14:26.653 INFO:tasks.rados.rados.0.smithi029.stdout:70: write oid 23 current snap is 0 2024-04-12T11:14:26.654 INFO:tasks.rados.rados.0.smithi029.stdout:70: seq_num 62 ranges {642034=796262,1895060=564680} 2024-04-12T11:14:26.661 INFO:tasks.rados.rados.0.smithi029.stdout:70: writing smithi02979277-23 from 642034 to 1438296 tid 1 2024-04-12T11:14:26.667 INFO:tasks.rados.rados.0.smithi029.stdout:70: writing smithi02979277-23 from 1895060 to 2459740 tid 2 2024-04-12T11:14:26.667 INFO:tasks.rados.rados.0.smithi029.stdout:71: snap_create 2024-04-12T11:14:26.667 INFO:tasks.rados.rados.0.smithi029.stdout:66: expect (ObjNum 23 snap 0 seq_num 23) 2024-04-12T11:14:26.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:26 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2526178583' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-04-12T11:14:26.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:26 smithi029 ceph-mon[39999]: osdmap e75: 8 total, 8 up, 8 in 2024-04-12T11:14:26.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:26 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2526178583' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-04-12T11:14:26.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:26 smithi029 ceph-mon[29349]: osdmap e75: 8 total, 8 up, 8 in 2024-04-12T11:14:26.748 INFO:tasks.rados.rados.0.smithi029.stdout:64: finishing write tid 2 to smithi02979277-22 2024-04-12T11:14:26.748 INFO:tasks.rados.rados.0.smithi029.stdout:64: finishing write tid 3 to smithi02979277-22 2024-04-12T11:14:26.748 INFO:tasks.rados.rados.0.smithi029.stdout:64: finishing write tid 4 to smithi02979277-22 2024-04-12T11:14:26.748 INFO:tasks.rados.rados.0.smithi029.stdout:64: oid 22 updating version 0 to 17 2024-04-12T11:14:26.748 INFO:tasks.rados.rados.0.smithi029.stdout:64: oid 22 updating version 17 to 18 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:64: oid 22 updating version 18 to 19 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 19 (ObjNum 58 snap 0 seq_num 58) dirty exists 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:64: left oid 22 (ObjNum 58 snap 0 seq_num 58) 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:67: finishing write tid 1 to smithi02979277-12 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:67: finishing write tid 2 to smithi02979277-12 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:67: finishing write tid 3 to smithi02979277-12 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:67: finishing write tid 4 to smithi02979277-12 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:67: finishing write tid 5 to smithi02979277-12 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:67: oid 12 updating version 0 to 13 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:67: oid 12 updating version 13 to 15 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:67: oid 12 version 15 is already newer than 14 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:67: oid 12 updating version 15 to 16 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 16 (ObjNum 59 snap 0 seq_num 59) dirty exists 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:67: left oid 12 (ObjNum 59 snap 0 seq_num 59) 2024-04-12T11:14:26.749 INFO:tasks.rados.rados.0.smithi029.stdout:68: finishing write tid 1 to smithi02979277-25 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:68: finishing write tid 2 to smithi02979277-25 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:68: finishing write tid 3 to smithi02979277-25 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:68: finishing write tid 4 to smithi02979277-25 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:68: finishing write tid 5 to smithi02979277-25 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:68: oid 25 updating version 0 to 15 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:68: oid 25 updating version 15 to 16 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:68: oid 25 updating version 16 to 17 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:68: oid 25 version 17 is already newer than 14 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 17 (ObjNum 60 snap 0 seq_num 60) dirty exists 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:68: left oid 25 (ObjNum 60 snap 0 seq_num 60) 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:69: finishing write tid 1 to smithi02979277-28 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:69: finishing write tid 2 to smithi02979277-28 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:69: finishing write tid 3 to smithi02979277-28 2024-04-12T11:14:26.750 INFO:tasks.rados.rados.0.smithi029.stdout:69: finishing write tid 4 to smithi02979277-28 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:69: oid 28 updating version 0 to 20 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:69: oid 28 version 20 is already newer than 18 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:69: oid 28 version 20 is already newer than 19 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 20 (ObjNum 61 snap 0 seq_num 61) dirty exists 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:69: left oid 28 (ObjNum 61 snap 0 seq_num 61) 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:70: finishing write tid 1 to smithi02979277-23 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:70: finishing write tid 2 to smithi02979277-23 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:70: finishing write tid 3 to smithi02979277-23 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:70: finishing write tid 4 to smithi02979277-23 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:70: oid 23 updating version 0 to 20 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:70: oid 23 updating version 20 to 21 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:70: oid 23 updating version 21 to 22 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 22 (ObjNum 62 snap 0 seq_num 62) dirty exists 2024-04-12T11:14:26.751 INFO:tasks.rados.rados.0.smithi029.stdout:70: left oid 23 (ObjNum 62 snap 0 seq_num 62) 2024-04-12T11:14:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:26 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2526178583' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-04-12T11:14:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:26 smithi087 ceph-mon[32917]: osdmap e75: 8 total, 8 up, 8 in 2024-04-12T11:14:27.474 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-12T11:14:27.526 INFO:tasks.rados.rados.0.smithi029.stdout:64: done (6 left) 2024-04-12T11:14:27.526 INFO:tasks.rados.rados.0.smithi029.stdout:66: done (5 left) 2024-04-12T11:14:27.526 INFO:tasks.rados.rados.0.smithi029.stdout:67: done (4 left) 2024-04-12T11:14:27.526 INFO:tasks.rados.rados.0.smithi029.stdout:68: done (3 left) 2024-04-12T11:14:27.526 INFO:tasks.rados.rados.0.smithi029.stdout:69: done (2 left) 2024-04-12T11:14:27.526 INFO:tasks.rados.rados.0.smithi029.stdout:70: done (1 left) 2024-04-12T11:14:27.526 INFO:tasks.rados.rados.0.smithi029.stdout:71: done (0 left) 2024-04-12T11:14:27.526 INFO:tasks.rados.rados.0.smithi029.stdout:72: write oid 50 current snap is 1 2024-04-12T11:14:27.526 INFO:tasks.rados.rados.0.smithi029.stdout:72: seq_num 63 ranges {658841=471511,1553542=708338,2964119=94722} 2024-04-12T11:14:27.532 INFO:tasks.rados.rados.0.smithi029.stdout:72: writing smithi02979277-50 from 658841 to 1130352 tid 1 2024-04-12T11:14:27.538 INFO:tasks.rados.rados.0.smithi029.stdout:72: writing smithi02979277-50 from 1553542 to 2261880 tid 2 2024-04-12T11:14:27.543 INFO:tasks.rados.rados.0.smithi029.stdout:72: writing smithi02979277-50 from 2964119 to 3058841 tid 3 2024-04-12T11:14:27.543 INFO:tasks.rados.rados.0.smithi029.stdout:73: rollback oid 38 current snap is 1 2024-04-12T11:14:27.543 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 38 to 0 2024-04-12T11:14:27.543 INFO:tasks.rados.rados.0.smithi029.stdout:72: finishing write tid 1 to smithi02979277-50 2024-04-12T11:14:27.543 INFO:tasks.rados.rados.0.smithi029.stdout:72: finishing write tid 2 to smithi02979277-50 2024-04-12T11:14:27.543 INFO:tasks.rados.rados.0.smithi029.stdout:74: write oid 33 current snap is 1 2024-04-12T11:14:27.543 INFO:tasks.rados.rados.0.smithi029.stdout:74: seq_num 64 ranges {675648=546760,2012024=568349} 2024-04-12T11:14:27.550 INFO:tasks.rados.rados.0.smithi029.stdout:74: writing smithi02979277-33 from 675648 to 1222408 tid 1 2024-04-12T11:14:27.558 INFO:tasks.rados.rados.0.smithi029.stdout:74: writing smithi02979277-33 from 2012024 to 2580373 tid 2 2024-04-12T11:14:27.558 INFO:tasks.rados.rados.0.smithi029.stdout:75: rollback oid 37 current snap is 1 2024-04-12T11:14:27.558 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 0 2024-04-12T11:14:27.558 INFO:tasks.rados.rados.0.smithi029.stdout:72: finishing write tid 3 to smithi02979277-50 2024-04-12T11:14:27.558 INFO:tasks.rados.rados.0.smithi029.stdout:72: finishing write tid 4 to smithi02979277-50 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:72: finishing write tid 5 to smithi02979277-50 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:72: oid 50 updating version 0 to 27 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:72: oid 50 version 27 is already newer than 26 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:72: oid 50 version 27 is already newer than 25 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:72: oid 50 version 27 is already newer than 24 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 27 (ObjNum 63 snap 1 seq_num 63) dirty exists 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:72: left oid 50 (ObjNum 63 snap 1 seq_num 63) 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:73: finishing rollback tid 0 to smithi02979277-38 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:73: finishing rollback tid 1 to smithi02979277-38 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:72: done (3 left) 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:76: snap_create 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:73: finishing rollback tid 2 to smithi02979277-38 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 13 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-04-12T11:14:27.559 INFO:tasks.rados.rados.0.smithi029.stdout:74: finishing write tid 1 to smithi02979277-33 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:74: finishing write tid 2 to smithi02979277-33 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:74: finishing write tid 3 to smithi02979277-33 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:74: finishing write tid 4 to smithi02979277-33 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:74: oid 33 updating version 0 to 29 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:74: oid 33 updating version 29 to 30 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:74: oid 33 updating version 30 to 31 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 31 (ObjNum 64 snap 1 seq_num 64) dirty exists 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:74: left oid 33 (ObjNum 64 snap 1 seq_num 64) 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:75: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:75: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:14:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:75: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:14:27.562 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 13 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-04-12T11:14:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:27 smithi029 ceph-mon[39999]: pgmap v89: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:27 smithi029 ceph-mon[29349]: pgmap v89: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:27.820 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:27 smithi087 ceph-mon[32917]: pgmap v89: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:28.531 INFO:tasks.rados.rados.0.smithi029.stdout:73: done (3 left) 2024-04-12T11:14:28.531 INFO:tasks.rados.rados.0.smithi029.stdout:74: done (2 left) 2024-04-12T11:14:28.531 INFO:tasks.rados.rados.0.smithi029.stdout:75: done (1 left) 2024-04-12T11:14:28.531 INFO:tasks.rados.rados.0.smithi029.stdout:76: done (0 left) 2024-04-12T11:14:28.531 INFO:tasks.rados.rados.0.smithi029.stdout:77: read oid 33 snap -1 2024-04-12T11:14:28.532 INFO:tasks.rados.rados.0.smithi029.stdout:77: expect (ObjNum 64 snap 1 seq_num 64) 2024-04-12T11:14:28.532 INFO:tasks.rados.rados.0.smithi029.stdout:78: snap_create 2024-04-12T11:14:28.540 INFO:tasks.rados.rados.0.smithi029.stdout:77: expect (ObjNum 64 snap 1 seq_num 64) 2024-04-12T11:14:28.621 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:28 smithi029 ceph-mon[39999]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T11:14:28.621 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:28 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:28.621 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:28 smithi029 ceph-mon[29349]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T11:14:28.621 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:28 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:28 smithi087 ceph-mon[32917]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T11:14:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:28 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:29.531 INFO:tasks.rados.rados.0.smithi029.stdout:77: done (1 left) 2024-04-12T11:14:29.532 INFO:tasks.rados.rados.0.smithi029.stdout:78: done (0 left) 2024-04-12T11:14:29.532 INFO:tasks.rados.rados.0.smithi029.stdout:79: rollback oid 9 current snap is 3 2024-04-12T11:14:29.532 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 9 to 0 2024-04-12T11:14:29.532 INFO:tasks.rados.rados.0.smithi029.stdout:80: snap_create 2024-04-12T11:14:29.533 INFO:tasks.rados.rados.0.smithi029.stdout:79: finishing rollback tid 0 to smithi02979277-9 2024-04-12T11:14:29.535 INFO:tasks.rados.rados.0.smithi029.stdout:79: finishing rollback tid 1 to smithi02979277-9 2024-04-12T11:14:29.535 INFO:tasks.rados.rados.0.smithi029.stdout:79: finishing rollback tid 2 to smithi02979277-9 2024-04-12T11:14:29.535 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 17 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-04-12T11:14:29.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[29349]: pgmap v91: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:29.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[29349]: osdmap e77: 8 total, 8 up, 8 in 2024-04-12T11:14:29.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:29.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:29.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:14:29.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:14:29.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:29.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[39999]: pgmap v91: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:29.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[39999]: osdmap e77: 8 total, 8 up, 8 in 2024-04-12T11:14:29.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:29.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:29.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:14:29.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:14:29.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:29 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:29 smithi087 ceph-mon[32917]: pgmap v91: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:29 smithi087 ceph-mon[32917]: osdmap e77: 8 total, 8 up, 8 in 2024-04-12T11:14:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:14:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:14:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:29 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:30.339 INFO:teuthology.orchestra.run.smithi029.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:14:30.533 INFO:tasks.rados.rados.0.smithi029.stdout:79: done (1 left) 2024-04-12T11:14:30.534 INFO:tasks.rados.rados.0.smithi029.stdout:80: done (0 left) 2024-04-12T11:14:30.534 INFO:tasks.rados.rados.0.smithi029.stdout:81: read oid 26 snap -1 2024-04-12T11:14:30.534 INFO:tasks.rados.rados.0.smithi029.stdout:81: expect (ObjNum 25 snap 0 seq_num 25) 2024-04-12T11:14:30.534 INFO:tasks.rados.rados.0.smithi029.stdout:82: read oid 36 snap -1 2024-04-12T11:14:30.534 INFO:tasks.rados.rados.0.smithi029.stdout:82: expect (ObjNum 35 snap 0 seq_num 35) 2024-04-12T11:14:30.534 INFO:tasks.rados.rados.0.smithi029.stdout:83: snap_create 2024-04-12T11:14:30.540 INFO:tasks.rados.rados.0.smithi029.stdout:81: expect (ObjNum 25 snap 0 seq_num 25) 2024-04-12T11:14:30.625 INFO:tasks.rados.rados.0.smithi029.stdout:82: expect (ObjNum 35 snap 0 seq_num 35) 2024-04-12T11:14:30.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[39999]: osdmap e78: 8 total, 8 up, 8 in 2024-04-12T11:14:30.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:30.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:14:30.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:14:30.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:14:30.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:30.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[29349]: osdmap e78: 8 total, 8 up, 8 in 2024-04-12T11:14:30.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:30.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:14:30.658 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:14:30.658 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:14:30.658 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:30 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:30 smithi087 ceph-mon[32917]: osdmap e78: 8 total, 8 up, 8 in 2024-04-12T11:14:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:30 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:30 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:14:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:30 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:14:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:30 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:14:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:30 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:14:31.063 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch ps' 2024-04-12T11:14:31.324 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:31.538 INFO:tasks.rados.rados.0.smithi029.stdout:81: done (2 left) 2024-04-12T11:14:31.539 INFO:tasks.rados.rados.0.smithi029.stdout:82: done (1 left) 2024-04-12T11:14:31.539 INFO:tasks.rados.rados.0.smithi029.stdout:83: done (0 left) 2024-04-12T11:14:31.539 INFO:tasks.rados.rados.0.smithi029.stdout:84: delete oid 7 current snap is 5 2024-04-12T11:14:31.542 INFO:tasks.rados.rados.0.smithi029.stdout:84: done (0 left) 2024-04-12T11:14:31.542 INFO:tasks.rados.rados.0.smithi029.stdout:85: write oid 48 current snap is 5 2024-04-12T11:14:31.542 INFO:tasks.rados.rados.0.smithi029.stdout:85: seq_num 66 ranges {709262=697258,2012635=572018,3046388=62874} 2024-04-12T11:14:31.550 INFO:tasks.rados.rados.0.smithi029.stdout:85: writing smithi02979277-48 from 709262 to 1406520 tid 1 2024-04-12T11:14:31.556 INFO:tasks.rados.rados.0.smithi029.stdout:85: writing smithi02979277-48 from 2012635 to 2584653 tid 2 2024-04-12T11:14:31.560 INFO:tasks.rados.rados.0.smithi029.stdout:85: writing smithi02979277-48 from 3046388 to 3109262 tid 3 2024-04-12T11:14:31.560 INFO:tasks.rados.rados.0.smithi029.stdout:86: snap_create 2024-04-12T11:14:31.560 INFO:tasks.rados.rados.0.smithi029.stdout:85: finishing write tid 1 to smithi02979277-48 2024-04-12T11:14:31.560 INFO:tasks.rados.rados.0.smithi029.stdout:85: finishing write tid 2 to smithi02979277-48 2024-04-12T11:14:31.561 INFO:tasks.rados.rados.0.smithi029.stdout:85: finishing write tid 3 to smithi02979277-48 2024-04-12T11:14:31.561 INFO:tasks.rados.rados.0.smithi029.stdout:85: finishing write tid 4 to smithi02979277-48 2024-04-12T11:14:31.561 INFO:tasks.rados.rados.0.smithi029.stdout:85: finishing write tid 5 to smithi02979277-48 2024-04-12T11:14:31.561 INFO:tasks.rados.rados.0.smithi029.stdout:85: oid 48 updating version 0 to 22 2024-04-12T11:14:31.561 INFO:tasks.rados.rados.0.smithi029.stdout:85: oid 48 updating version 22 to 25 2024-04-12T11:14:31.561 INFO:tasks.rados.rados.0.smithi029.stdout:85: oid 48 version 25 is already newer than 23 2024-04-12T11:14:31.561 INFO:tasks.rados.rados.0.smithi029.stdout:85: oid 48 version 25 is already newer than 24 2024-04-12T11:14:31.561 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 25 (ObjNum 66 snap 5 seq_num 66) dirty exists 2024-04-12T11:14:31.561 INFO:tasks.rados.rados.0.smithi029.stdout:85: left oid 48 (ObjNum 66 snap 5 seq_num 66) 2024-04-12T11:14:31.726 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:31 smithi029 ceph-mon[39999]: pgmap v94: 89 pgs: 89 active+clean; 117 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 15 MiB/s wr, 91 op/s 2024-04-12T11:14:31.726 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:31 smithi029 ceph-mon[39999]: from='client.14823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:2cee1857f8068f7ca29db3a37937b8e9f919ba95", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:14:31.726 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:31 smithi029 ceph-mon[39999]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:14:31.726 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:31 smithi029 ceph-mon[39999]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:14:31.726 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:31 smithi029 ceph-mon[39999]: osdmap e79: 8 total, 8 up, 8 in 2024-04-12T11:14:31.726 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:31 smithi029 ceph-mon[29349]: pgmap v94: 89 pgs: 89 active+clean; 117 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 15 MiB/s wr, 91 op/s 2024-04-12T11:14:31.726 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:31 smithi029 ceph-mon[29349]: from='client.14823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:2cee1857f8068f7ca29db3a37937b8e9f919ba95", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:14:31.726 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:31 smithi029 ceph-mon[29349]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:14:31.726 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:31 smithi029 ceph-mon[29349]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:14:31.726 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:31 smithi029 ceph-mon[29349]: osdmap e79: 8 total, 8 up, 8 in 2024-04-12T11:14:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:31 smithi087 ceph-mon[32917]: pgmap v94: 89 pgs: 89 active+clean; 117 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 15 MiB/s wr, 91 op/s 2024-04-12T11:14:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:31 smithi087 ceph-mon[32917]: from='client.14823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:2cee1857f8068f7ca29db3a37937b8e9f919ba95", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:14:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:31 smithi087 ceph-mon[32917]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:14:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:31 smithi087 ceph-mon[32917]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:2cee1857f8068f7ca29db3a37937b8e9f919ba95 2024-04-12T11:14:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:31 smithi087 ceph-mon[32917]: osdmap e79: 8 total, 8 up, 8 in 2024-04-12T11:14:32.548 INFO:tasks.rados.rados.0.smithi029.stdout:85: done (1 left) 2024-04-12T11:14:32.548 INFO:tasks.rados.rados.0.smithi029.stdout:86: done (0 left) 2024-04-12T11:14:32.548 INFO:tasks.rados.rados.0.smithi029.stdout:87: rollback oid 10 current snap is 6 2024-04-12T11:14:32.548 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 10 to 2 2024-04-12T11:14:32.548 INFO:tasks.rados.rados.0.smithi029.stdout:88: snap_create 2024-04-12T11:14:32.549 INFO:tasks.rados.rados.0.smithi029.stdout:87: finishing rollback tid 1 to smithi02979277-10 2024-04-12T11:14:32.549 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 0 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-04-12T11:14:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:32 smithi029 ceph-mon[39999]: osdmap e80: 8 total, 8 up, 8 in 2024-04-12T11:14:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:32 smithi029 ceph-mon[29349]: osdmap e80: 8 total, 8 up, 8 in 2024-04-12T11:14:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:32 smithi087 ceph-mon[32917]: osdmap e80: 8 total, 8 up, 8 in 2024-04-12T11:14:33.098 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:14:33] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.43.0" 2024-04-12T11:14:33.472 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (32s) 4s ago 2m 23.6M - 0.25.0 c8568f914cd2 5f36b9c37cec 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi087 *:3000 running (108s) 9s ago 107s 86.8M - 9.4.7 954c08fa6188 d4ba4d587fdb 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.scbuux smithi029 running (85s) 4s ago 85s 13.1M - 18.2.2-1187-gd540ebac bb2c5eac26a4 a659ce0bccaf 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi087.mbflzu smithi087 running (78s) 9s ago 77s 15.0M - 18.2.2-1187-gd540ebac bb2c5eac26a4 64bafd1e6a4c 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi087 *:8443,8765 running (6m) 9s ago 6m 448M - 18.2.2-1187-gd540ebac bb2c5eac26a4 6117969a1314 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:9283,8765 running (8m) 4s ago 8m 495M - 18.2.2-1187-gd540ebac bb2c5eac26a4 314601fec666 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (8m) 4s ago 8m 46.2M 2048M 18.2.2-1187-gd540ebac bb2c5eac26a4 1cf52b0b92d9 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi087 running (6m) 9s ago 6m 37.5M 2048M 18.2.2-1187-gd540ebac bb2c5eac26a4 adfeefa5d84f 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (6m) 4s ago 6m 37.7M 2048M 18.2.2-1187-gd540ebac bb2c5eac26a4 903a61eba1ba 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (101s) 4s ago 100s 14.9M - 1.5.0 0da6a335fe13 32124106e912 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi087 *:9100 running (97s) 9s ago 97s 7096k - 1.5.0 0da6a335fe13 78910b2d30c6 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (6m) 4s ago 6m 50.7M 4017M 18.2.2-1187-gd540ebac bb2c5eac26a4 f65c324b0084 2024-04-12T11:14:33.473 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (5m) 4s ago 5m 44.7M 4017M 18.2.2-1187-gd540ebac bb2c5eac26a4 bb552e04f578 2024-04-12T11:14:33.474 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (5m) 4s ago 5m 42.5M 4017M 18.2.2-1187-gd540ebac bb2c5eac26a4 dbb7abb8911b 2024-04-12T11:14:33.474 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (4m) 4s ago 4m 48.6M 4017M 18.2.2-1187-gd540ebac bb2c5eac26a4 a9b5c72d15da 2024-04-12T11:14:33.474 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi087 running (4m) 9s ago 4m 49.8M 4016M 18.2.2-1187-gd540ebac bb2c5eac26a4 ab3a8e864afd 2024-04-12T11:14:33.474 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi087 running (4m) 9s ago 4m 44.5M 4016M 18.2.2-1187-gd540ebac bb2c5eac26a4 573e2f90e4da 2024-04-12T11:14:33.474 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi087 running (3m) 9s ago 3m 41.0M 4016M 18.2.2-1187-gd540ebac bb2c5eac26a4 2b67d3f477dd 2024-04-12T11:14:33.474 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi087 running (3m) 9s ago 3m 52.6M 4016M 18.2.2-1187-gd540ebac bb2c5eac26a4 9c484b9f0790 2024-04-12T11:14:33.474 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi087 *:9095 running (27s) 9s ago 2m 29.0M - 2.43.0 a07b618ecd1d 06b93cbb7d0c 2024-04-12T11:14:33.549 INFO:tasks.rados.rados.0.smithi029.stdout:87: done (1 left) 2024-04-12T11:14:33.550 INFO:tasks.rados.rados.0.smithi029.stdout:88: done (0 left) 2024-04-12T11:14:33.550 INFO:tasks.rados.rados.0.smithi029.stdout:89: delete oid 23 current snap is 7 2024-04-12T11:14:33.557 INFO:tasks.rados.rados.0.smithi029.stdout:89: done (0 left) 2024-04-12T11:14:33.557 INFO:tasks.rados.rados.0.smithi029.stdout:90: read oid 27 snap 3 2024-04-12T11:14:33.557 INFO:tasks.rados.rados.0.smithi029.stdout:90: expect (ObjNum 26 snap 0 seq_num 26) 2024-04-12T11:14:33.557 INFO:tasks.rados.rados.0.smithi029.stdout:91: write oid 12 current snap is 7 2024-04-12T11:14:33.557 INFO:tasks.rados.rados.0.smithi029.stdout:91: seq_num 68 ranges {742876=447756,1729599=575687,3101234=41642} 2024-04-12T11:14:33.564 INFO:tasks.rados.rados.0.smithi029.stdout:91: writing smithi02979277-12 from 742876 to 1190632 tid 1 2024-04-12T11:14:33.572 INFO:tasks.rados.rados.0.smithi029.stdout:91: writing smithi02979277-12 from 1729599 to 2305286 tid 2 2024-04-12T11:14:33.577 INFO:tasks.rados.rados.0.smithi029.stdout:91: writing smithi02979277-12 from 3101234 to 3142876 tid 3 2024-04-12T11:14:33.577 INFO:tasks.rados.rados.0.smithi029.stdout:92: snap_create 2024-04-12T11:14:33.577 INFO:tasks.rados.rados.0.smithi029.stdout:90: expect (ObjNum 26 snap 0 seq_num 26) 2024-04-12T11:14:33.685 INFO:tasks.rados.rados.0.smithi029.stdout:91: finishing write tid 1 to smithi02979277-12 2024-04-12T11:14:33.685 INFO:tasks.rados.rados.0.smithi029.stdout:91: finishing write tid 2 to smithi02979277-12 2024-04-12T11:14:33.685 INFO:tasks.rados.rados.0.smithi029.stdout:91: finishing write tid 3 to smithi02979277-12 2024-04-12T11:14:33.685 INFO:tasks.rados.rados.0.smithi029.stdout:91: finishing write tid 4 to smithi02979277-12 2024-04-12T11:14:33.685 INFO:tasks.rados.rados.0.smithi029.stdout:91: finishing write tid 5 to smithi02979277-12 2024-04-12T11:14:33.685 INFO:tasks.rados.rados.0.smithi029.stdout:91: oid 12 updating version 0 to 22 2024-04-12T11:14:33.685 INFO:tasks.rados.rados.0.smithi029.stdout:91: oid 12 version 22 is already newer than 21 2024-04-12T11:14:33.685 INFO:tasks.rados.rados.0.smithi029.stdout:91: oid 12 version 22 is already newer than 20 2024-04-12T11:14:33.685 INFO:tasks.rados.rados.0.smithi029.stdout:91: oid 12 updating version 22 to 23 2024-04-12T11:14:33.685 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 23 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-04-12T11:14:33.686 INFO:tasks.rados.rados.0.smithi029.stdout:91: left oid 12 (ObjNum 68 snap 7 seq_num 68) 2024-04-12T11:14:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:33 smithi029 ceph-mon[29349]: pgmap v97: 89 pgs: 89 active+clean; 117 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 17 MiB/s wr, 106 op/s 2024-04-12T11:14:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:33 smithi029 ceph-mon[29349]: osdmap e81: 8 total, 8 up, 8 in 2024-04-12T11:14:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:33 smithi029 ceph-mon[39999]: pgmap v97: 89 pgs: 89 active+clean; 117 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 17 MiB/s wr, 106 op/s 2024-04-12T11:14:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:33 smithi029 ceph-mon[39999]: osdmap e81: 8 total, 8 up, 8 in 2024-04-12T11:14:33.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:33 smithi087 ceph-mon[32917]: pgmap v97: 89 pgs: 89 active+clean; 117 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 17 MiB/s wr, 106 op/s 2024-04-12T11:14:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:33 smithi087 ceph-mon[32917]: osdmap e81: 8 total, 8 up, 8 in 2024-04-12T11:14:34.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-12T11:14:34.560 INFO:tasks.rados.rados.0.smithi029.stdout:90: done (2 left) 2024-04-12T11:14:34.561 INFO:tasks.rados.rados.0.smithi029.stdout:91: done (1 left) 2024-04-12T11:14:34.561 INFO:tasks.rados.rados.0.smithi029.stdout:92: done (0 left) 2024-04-12T11:14:34.561 INFO:tasks.rados.rados.0.smithi029.stdout:93: snap_remove snap 7 2024-04-12T11:14:34.577 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:34.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:34 smithi029 ceph-mon[39999]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:14:34.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:34 smithi029 ceph-mon[39999]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T11:14:34.702 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:34 smithi029 ceph-mon[29349]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:14:34.702 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:34 smithi029 ceph-mon[29349]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T11:14:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:34 smithi087 ceph-mon[32917]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:14:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:34 smithi087 ceph-mon[32917]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T11:14:35.560 INFO:tasks.rados.rados.0.smithi029.stdout:93: done (0 left) 2024-04-12T11:14:35.560 INFO:tasks.rados.rados.0.smithi029.stdout:94: write oid 15 current snap is 8 2024-04-12T11:14:35.560 INFO:tasks.rados.rados.0.smithi029.stdout:94: seq_num 69 ranges {759683=639358,1904434=719345,3128657=31026} 2024-04-12T11:14:35.568 INFO:tasks.rados.rados.0.smithi029.stdout:94: writing smithi02979277-15 from 759683 to 1399041 tid 1 2024-04-12T11:14:35.575 INFO:tasks.rados.rados.0.smithi029.stdout:94: writing smithi02979277-15 from 1904434 to 2623779 tid 2 2024-04-12T11:14:35.578 INFO:tasks.rados.rados.0.smithi029.stdout:94: writing smithi02979277-15 from 3128657 to 3159683 tid 3 2024-04-12T11:14:35.578 INFO:tasks.rados.rados.0.smithi029.stdout:95: snap_create 2024-04-12T11:14:35.578 INFO:tasks.rados.rados.0.smithi029.stdout:94: finishing write tid 1 to smithi02979277-15 2024-04-12T11:14:35.578 INFO:tasks.rados.rados.0.smithi029.stdout:94: finishing write tid 2 to smithi02979277-15 2024-04-12T11:14:35.580 INFO:tasks.rados.rados.0.smithi029.stdout:94: finishing write tid 3 to smithi02979277-15 2024-04-12T11:14:35.580 INFO:tasks.rados.rados.0.smithi029.stdout:94: finishing write tid 4 to smithi02979277-15 2024-04-12T11:14:35.580 INFO:tasks.rados.rados.0.smithi029.stdout:94: finishing write tid 5 to smithi02979277-15 2024-04-12T11:14:35.580 INFO:tasks.rados.rados.0.smithi029.stdout:94: oid 15 updating version 0 to 11 2024-04-12T11:14:35.580 INFO:tasks.rados.rados.0.smithi029.stdout:94: oid 15 updating version 11 to 13 2024-04-12T11:14:35.580 INFO:tasks.rados.rados.0.smithi029.stdout:94: oid 15 version 13 is already newer than 12 2024-04-12T11:14:35.580 INFO:tasks.rados.rados.0.smithi029.stdout:94: oid 15 updating version 13 to 14 2024-04-12T11:14:35.580 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 14 (ObjNum 69 snap 8 seq_num 69) dirty exists 2024-04-12T11:14:35.580 INFO:tasks.rados.rados.0.smithi029.stdout:94: left oid 15 (ObjNum 69 snap 8 seq_num 69) 2024-04-12T11:14:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:35 smithi029 ceph-mon[29349]: pgmap v100: 89 pgs: 89 active+clean; 119 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 586 KiB/s wr, 10 op/s 2024-04-12T11:14:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:35 smithi029 ceph-mon[29349]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T11:14:35.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:35 smithi029 ceph-mon[39999]: pgmap v100: 89 pgs: 89 active+clean; 119 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 586 KiB/s wr, 10 op/s 2024-04-12T11:14:35.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:35 smithi029 ceph-mon[39999]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T11:14:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:35 smithi087 ceph-mon[32917]: pgmap v100: 89 pgs: 89 active+clean; 119 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 586 KiB/s wr, 10 op/s 2024-04-12T11:14:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:35 smithi087 ceph-mon[32917]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T11:14:36.570 INFO:tasks.rados.rados.0.smithi029.stdout:94: done (1 left) 2024-04-12T11:14:36.570 INFO:tasks.rados.rados.0.smithi029.stdout:95: done (0 left) 2024-04-12T11:14:36.570 INFO:tasks.rados.rados.0.smithi029.stdout:96: rollback oid 42 current snap is 9 2024-04-12T11:14:36.570 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 5 2024-04-12T11:14:36.570 INFO:tasks.rados.rados.0.smithi029.stdout:97: write oid 3 current snap is 9 2024-04-12T11:14:36.571 INFO:tasks.rados.rados.0.smithi029.stdout:97: seq_num 70 ranges {776490=714607,2246563=579356} 2024-04-12T11:14:36.579 INFO:tasks.rados.rados.0.smithi029.stdout:97: writing smithi02979277-3 from 776490 to 1491097 tid 1 2024-04-12T11:14:36.587 INFO:tasks.rados.rados.0.smithi029.stdout:97: writing smithi02979277-3 from 2246563 to 2825919 tid 2 2024-04-12T11:14:36.587 INFO:tasks.rados.rados.0.smithi029.stdout:98: delete oid 4 current snap is 9 2024-04-12T11:14:36.587 INFO:tasks.rados.rados.0.smithi029.stdout:96: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:14:36.587 INFO:tasks.rados.rados.0.smithi029.stdout:96: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:14:36.587 INFO:tasks.rados.rados.0.smithi029.stdout:96: finishing rollback tid 2 to smithi02979277-42 2024-04-12T11:14:36.587 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 7 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-04-12T11:14:36.587 INFO:tasks.rados.rados.0.smithi029.stdout:97: finishing write tid 1 to smithi02979277-3 2024-04-12T11:14:36.589 INFO:tasks.rados.rados.0.smithi029.stdout:96: done (2 left) 2024-04-12T11:14:36.589 INFO:tasks.rados.rados.0.smithi029.stdout:98: done (1 left) 2024-04-12T11:14:36.589 INFO:tasks.rados.rados.0.smithi029.stdout:99: read oid 46 snap -1 2024-04-12T11:14:36.589 INFO:tasks.rados.rados.0.smithi029.stdout:99: expect (ObjNum 45 snap 0 seq_num 45) 2024-04-12T11:14:36.589 INFO:tasks.rados.rados.0.smithi029.stdout:100: write oid 36 current snap is 9 2024-04-12T11:14:36.589 INFO:tasks.rados.rados.0.smithi029.stdout:100: seq_num 72 ranges {410104=465105,1563527=583025,2810926=399178} 2024-04-12T11:14:36.591 INFO:teuthology.orchestra.run.smithi029.stdout:wait for minority of mons to upgrade 2024-04-12T11:14:36.594 INFO:tasks.rados.rados.0.smithi029.stdout:100: writing smithi02979277-36 from 410104 to 875209 tid 1 2024-04-12T11:14:36.602 INFO:tasks.rados.rados.0.smithi029.stdout:100: writing smithi02979277-36 from 1563527 to 2146552 tid 2 2024-04-12T11:14:36.609 INFO:tasks.rados.rados.0.smithi029.stdout:100: writing smithi02979277-36 from 2810926 to 3210104 tid 3 2024-04-12T11:14:36.609 INFO:tasks.rados.rados.0.smithi029.stdout:101: snap_remove snap 4 2024-04-12T11:14:36.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:36 smithi029 ceph-mon[39999]: osdmap e84: 8 total, 8 up, 8 in 2024-04-12T11:14:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:36 smithi029 ceph-mon[29349]: osdmap e84: 8 total, 8 up, 8 in 2024-04-12T11:14:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:36 smithi087 ceph-mon[32917]: osdmap e84: 8 total, 8 up, 8 in 2024-04-12T11:14:37.375 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-12T11:14:37.577 INFO:tasks.rados.rados.0.smithi029.stdout:101: done (3 left) 2024-04-12T11:14:37.578 INFO:tasks.rados.rados.0.smithi029.stdout:102: write oid 41 current snap is 9 2024-04-12T11:14:37.578 INFO:tasks.rados.rados.0.smithi029.stdout:97: finishing write tid 2 to smithi02979277-3 2024-04-12T11:14:37.578 INFO:tasks.rados.rados.0.smithi029.stdout:97: finishing write tid 3 to smithi02979277-3 2024-04-12T11:14:37.578 INFO:tasks.rados.rados.0.smithi029.stdout:97: finishing write tid 4 to smithi02979277-3 2024-04-12T11:14:37.578 INFO:tasks.rados.rados.0.smithi029.stdout:97: oid 3 updating version 0 to 21 2024-04-12T11:14:37.578 INFO:tasks.rados.rados.0.smithi029.stdout:97: oid 3 updating version 21 to 22 2024-04-12T11:14:37.578 INFO:tasks.rados.rados.0.smithi029.stdout:97: oid 3 version 22 is already newer than 20 2024-04-12T11:14:37.578 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 22 (ObjNum 70 snap 9 seq_num 70) dirty exists 2024-04-12T11:14:37.578 INFO:tasks.rados.rados.0.smithi029.stdout:97: left oid 3 (ObjNum 70 snap 9 seq_num 70) 2024-04-12T11:14:37.578 INFO:tasks.rados.rados.0.smithi029.stdout:99: expect (ObjNum 45 snap 0 seq_num 45) 2024-04-12T11:14:37.604 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:14:37.679 INFO:tasks.rados.rados.0.smithi029.stdout:100: finishing write tid 1 to smithi02979277-36 2024-04-12T11:14:37.679 INFO:tasks.rados.rados.0.smithi029.stdout:100: finishing write tid 2 to smithi02979277-36 2024-04-12T11:14:37.679 INFO:tasks.rados.rados.0.smithi029.stdout:102: seq_num 73 ranges {426911=540354,1622009=726683,3121996=104915} 2024-04-12T11:14:37.684 INFO:tasks.rados.rados.0.smithi029.stdout:102: writing smithi02979277-41 from 426911 to 967265 tid 1 2024-04-12T11:14:37.691 INFO:tasks.rados.rados.0.smithi029.stdout:102: writing smithi02979277-41 from 1622009 to 2348692 tid 2 2024-04-12T11:14:37.697 INFO:tasks.rados.rados.0.smithi029.stdout:102: writing smithi02979277-41 from 3121996 to 3226911 tid 3 2024-04-12T11:14:37.697 INFO:tasks.rados.rados.0.smithi029.stdout:97: done (3 left) 2024-04-12T11:14:37.697 INFO:tasks.rados.rados.0.smithi029.stdout:99: done (2 left) 2024-04-12T11:14:37.697 INFO:tasks.rados.rados.0.smithi029.stdout:103: snap_remove snap 5 2024-04-12T11:14:37.862 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:37 smithi029 ceph-mon[39999]: pgmap v103: 89 pgs: 89 active+clean; 119 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 586 KiB/s wr, 10 op/s 2024-04-12T11:14:37.862 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:37 smithi029 ceph-mon[39999]: osdmap e85: 8 total, 8 up, 8 in 2024-04-12T11:14:37.862 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:37 smithi029 ceph-mon[29349]: pgmap v103: 89 pgs: 89 active+clean; 119 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 586 KiB/s wr, 10 op/s 2024-04-12T11:14:37.862 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:37 smithi029 ceph-mon[29349]: osdmap e85: 8 total, 8 up, 8 in 2024-04-12T11:14:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:37 smithi087 ceph-mon[32917]: pgmap v103: 89 pgs: 89 active+clean; 119 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 586 KiB/s wr, 10 op/s 2024-04-12T11:14:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:37 smithi087 ceph-mon[32917]: osdmap e85: 8 total, 8 up, 8 in 2024-04-12T11:14:38.581 INFO:tasks.rados.rados.0.smithi029.stdout:103: done (2 left) 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:104: delete oid 22 current snap is 9 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:100: finishing write tid 3 to smithi02979277-36 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:100: finishing write tid 4 to smithi02979277-36 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:100: finishing write tid 5 to smithi02979277-36 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:100: oid 36 updating version 0 to 10 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:100: oid 36 version 10 is already newer than 9 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:100: oid 36 version 10 is already newer than 8 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:100: oid 36 updating version 10 to 11 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 11 (ObjNum 72 snap 9 seq_num 72) dirty exists 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:100: left oid 36 (ObjNum 72 snap 9 seq_num 72) 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:102: finishing write tid 1 to smithi02979277-41 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:102: finishing write tid 2 to smithi02979277-41 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:102: finishing write tid 3 to smithi02979277-41 2024-04-12T11:14:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:102: finishing write tid 4 to smithi02979277-41 2024-04-12T11:14:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:102: finishing write tid 5 to smithi02979277-41 2024-04-12T11:14:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:102: oid 41 updating version 0 to 15 2024-04-12T11:14:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:102: oid 41 updating version 15 to 16 2024-04-12T11:14:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:102: oid 41 updating version 16 to 17 2024-04-12T11:14:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:102: oid 41 version 17 is already newer than 14 2024-04-12T11:14:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 17 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-04-12T11:14:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:102: left oid 41 (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:14:38.589 INFO:tasks.rados.rados.0.smithi029.stdout:100: done (2 left) 2024-04-12T11:14:38.590 INFO:tasks.rados.rados.0.smithi029.stdout:102: done (1 left) 2024-04-12T11:14:38.590 INFO:tasks.rados.rados.0.smithi029.stdout:104: done (0 left) 2024-04-12T11:14:38.590 INFO:tasks.rados.rados.0.smithi029.stdout:105: write oid 14 current snap is 9 2024-04-12T11:14:38.590 INFO:tasks.rados.rados.0.smithi029.stdout:105: seq_num 75 ranges {460525=690852,1622620=730352,3060489=200036} 2024-04-12T11:14:38.596 INFO:tasks.rados.rados.0.smithi029.stdout:105: writing smithi02979277-14 from 460525 to 1151377 tid 1 2024-04-12T11:14:38.603 INFO:tasks.rados.rados.0.smithi029.stdout:105: writing smithi02979277-14 from 1622620 to 2352972 tid 2 2024-04-12T11:14:38.609 INFO:tasks.rados.rados.0.smithi029.stdout:105: writing smithi02979277-14 from 3060489 to 3260525 tid 3 2024-04-12T11:14:38.609 INFO:tasks.rados.rados.0.smithi029.stdout:106: rollback oid 30 current snap is 9 2024-04-12T11:14:38.609 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 2 2024-04-12T11:14:38.609 INFO:tasks.rados.rados.0.smithi029.stdout:105: finishing write tid 1 to smithi02979277-14 2024-04-12T11:14:38.609 INFO:tasks.rados.rados.0.smithi029.stdout:105: finishing write tid 2 to smithi02979277-14 2024-04-12T11:14:38.609 INFO:tasks.rados.rados.0.smithi029.stdout:107: delete oid 10 current snap is 9 2024-04-12T11:14:38.611 INFO:tasks.rados.rados.0.smithi029.stdout:107: done (2 left) 2024-04-12T11:14:38.611 INFO:tasks.rados.rados.0.smithi029.stdout:108: rollback oid 37 current snap is 9 2024-04-12T11:14:38.611 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 2 2024-04-12T11:14:38.611 INFO:tasks.rados.rados.0.smithi029.stdout:109: write oid 13 current snap is 9 2024-04-12T11:14:38.611 INFO:tasks.rados.rados.0.smithi029.stdout:109: seq_num 77 ranges {494139=557703,1455937=734021,2948041=346098} 2024-04-12T11:14:38.617 INFO:tasks.rados.rados.0.smithi029.stdout:109: writing smithi02979277-13 from 494139 to 1051842 tid 1 2024-04-12T11:14:38.623 INFO:tasks.rados.rados.0.smithi029.stdout:109: writing smithi02979277-13 from 1455937 to 2189958 tid 2 2024-04-12T11:14:38.630 INFO:tasks.rados.rados.0.smithi029.stdout:109: writing smithi02979277-13 from 2948041 to 3294139 tid 3 2024-04-12T11:14:38.630 INFO:tasks.rados.rados.0.smithi029.stdout:110: write oid 25 current snap is 9 2024-04-12T11:14:38.630 INFO:tasks.rados.rados.0.smithi029.stdout:105: finishing write tid 3 to smithi02979277-14 2024-04-12T11:14:38.630 INFO:tasks.rados.rados.0.smithi029.stdout:106: finishing rollback tid 0 to smithi02979277-30 2024-04-12T11:14:38.630 INFO:tasks.rados.rados.0.smithi029.stdout:105: finishing write tid 4 to smithi02979277-14 2024-04-12T11:14:38.630 INFO:tasks.rados.rados.0.smithi029.stdout:105: finishing write tid 5 to smithi02979277-14 2024-04-12T11:14:38.630 INFO:tasks.rados.rados.0.smithi029.stdout:105: oid 14 updating version 0 to 26 2024-04-12T11:14:38.631 INFO:tasks.rados.rados.0.smithi029.stdout:105: oid 14 updating version 26 to 29 2024-04-12T11:14:38.631 INFO:tasks.rados.rados.0.smithi029.stdout:105: oid 14 version 29 is already newer than 27 2024-04-12T11:14:38.631 INFO:tasks.rados.rados.0.smithi029.stdout:105: oid 14 version 29 is already newer than 28 2024-04-12T11:14:38.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 29 (ObjNum 75 snap 9 seq_num 75) dirty exists 2024-04-12T11:14:38.631 INFO:tasks.rados.rados.0.smithi029.stdout:105: left oid 14 (ObjNum 75 snap 9 seq_num 75) 2024-04-12T11:14:38.631 INFO:tasks.rados.rados.0.smithi029.stdout:106: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:14:38.631 INFO:tasks.rados.rados.0.smithi029.stdout:106: finishing rollback tid 2 to smithi02979277-30 2024-04-12T11:14:38.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 22 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-04-12T11:14:38.631 INFO:tasks.rados.rados.0.smithi029.stdout:108: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:14:38.631 INFO:tasks.rados.rados.0.smithi029.stdout:110: seq_num 78 ranges {510946=632952,1798066=477679,2742758=457217} 2024-04-12T11:14:38.636 INFO:tasks.rados.rados.0.smithi029.stdout:110: writing smithi02979277-25 from 510946 to 1143898 tid 1 2024-04-12T11:14:38.642 INFO:tasks.rados.rados.0.smithi029.stdout:110: writing smithi02979277-25 from 1798066 to 2275745 tid 2 2024-04-12T11:14:38.648 INFO:tasks.rados.rados.0.smithi029.stdout:110: writing smithi02979277-25 from 2742758 to 3199975 tid 3 2024-04-12T11:14:38.648 INFO:tasks.rados.rados.0.smithi029.stdout:105: done (4 left) 2024-04-12T11:14:38.648 INFO:tasks.rados.rados.0.smithi029.stdout:106: done (3 left) 2024-04-12T11:14:38.648 INFO:tasks.rados.rados.0.smithi029.stdout:111: snap_create 2024-04-12T11:14:38.648 INFO:tasks.rados.rados.0.smithi029.stdout:108: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:14:38.648 INFO:tasks.rados.rados.0.smithi029.stdout:108: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:14:38.648 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 21 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-04-12T11:14:38.648 INFO:tasks.rados.rados.0.smithi029.stdout:109: finishing write tid 1 to smithi02979277-13 2024-04-12T11:14:38.648 INFO:tasks.rados.rados.0.smithi029.stdout:109: finishing write tid 2 to smithi02979277-13 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:109: finishing write tid 3 to smithi02979277-13 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:109: finishing write tid 4 to smithi02979277-13 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:109: finishing write tid 5 to smithi02979277-13 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:109: oid 13 updating version 0 to 19 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:109: oid 13 version 19 is already newer than 18 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:109: oid 13 updating version 19 to 21 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:109: oid 13 version 21 is already newer than 20 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 21 (ObjNum 77 snap 9 seq_num 77) dirty exists 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:109: left oid 13 (ObjNum 77 snap 9 seq_num 77) 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:110: finishing write tid 1 to smithi02979277-25 2024-04-12T11:14:38.649 INFO:tasks.rados.rados.0.smithi029.stdout:110: finishing write tid 2 to smithi02979277-25 2024-04-12T11:14:38.650 INFO:tasks.rados.rados.0.smithi029.stdout:110: finishing write tid 3 to smithi02979277-25 2024-04-12T11:14:38.650 INFO:tasks.rados.rados.0.smithi029.stdout:110: finishing write tid 4 to smithi02979277-25 2024-04-12T11:14:38.650 INFO:tasks.rados.rados.0.smithi029.stdout:110: finishing write tid 5 to smithi02979277-25 2024-04-12T11:14:38.650 INFO:tasks.rados.rados.0.smithi029.stdout:110: oid 25 updating version 0 to 24 2024-04-12T11:14:38.650 INFO:tasks.rados.rados.0.smithi029.stdout:110: oid 25 version 24 is already newer than 23 2024-04-12T11:14:38.650 INFO:tasks.rados.rados.0.smithi029.stdout:110: oid 25 updating version 24 to 25 2024-04-12T11:14:38.650 INFO:tasks.rados.rados.0.smithi029.stdout:110: oid 25 updating version 25 to 26 2024-04-12T11:14:38.650 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 26 (ObjNum 78 snap 9 seq_num 78) dirty exists 2024-04-12T11:14:38.650 INFO:tasks.rados.rados.0.smithi029.stdout:110: left oid 25 (ObjNum 78 snap 9 seq_num 78) 2024-04-12T11:14:38.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:38 smithi029 ceph-mon[39999]: osdmap e86: 8 total, 8 up, 8 in 2024-04-12T11:14:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:38 smithi029 ceph-mon[29349]: osdmap e86: 8 total, 8 up, 8 in 2024-04-12T11:14:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:38 smithi087 ceph-mon[32917]: osdmap e86: 8 total, 8 up, 8 in 2024-04-12T11:14:39.589 INFO:tasks.rados.rados.0.smithi029.stdout:108: done (3 left) 2024-04-12T11:14:39.589 INFO:tasks.rados.rados.0.smithi029.stdout:109: done (2 left) 2024-04-12T11:14:39.589 INFO:tasks.rados.rados.0.smithi029.stdout:110: done (1 left) 2024-04-12T11:14:39.589 INFO:tasks.rados.rados.0.smithi029.stdout:111: done (0 left) 2024-04-12T11:14:39.589 INFO:tasks.rados.rados.0.smithi029.stdout:112: write oid 47 current snap is 10 2024-04-12T11:14:39.589 INFO:tasks.rados.rados.0.smithi029.stdout:112: seq_num 79 ranges {527753=708201,1856548=737690,3286534=41219} 2024-04-12T11:14:39.598 INFO:tasks.rados.rados.0.smithi029.stdout:112: writing smithi02979277-47 from 527753 to 1235954 tid 1 2024-04-12T11:14:39.607 INFO:tasks.rados.rados.0.smithi029.stdout:112: writing smithi02979277-47 from 1856548 to 2594238 tid 2 2024-04-12T11:14:39.612 INFO:tasks.rados.rados.0.smithi029.stdout:112: writing smithi02979277-47 from 3286534 to 3327753 tid 3 2024-04-12T11:14:39.612 INFO:tasks.rados.rados.0.smithi029.stdout:113: delete oid 17 current snap is 10 2024-04-12T11:14:39.612 INFO:tasks.rados.rados.0.smithi029.stdout:112: finishing write tid 1 to smithi02979277-47 2024-04-12T11:14:39.612 INFO:tasks.rados.rados.0.smithi029.stdout:112: finishing write tid 2 to smithi02979277-47 2024-04-12T11:14:39.614 INFO:tasks.rados.rados.0.smithi029.stdout:112: finishing write tid 3 to smithi02979277-47 2024-04-12T11:14:39.614 INFO:tasks.rados.rados.0.smithi029.stdout:113: done (1 left) 2024-04-12T11:14:39.614 INFO:tasks.rados.rados.0.smithi029.stdout:114: snap_remove snap 2 2024-04-12T11:14:39.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:39 smithi029 ceph-mon[39999]: pgmap v106: 89 pgs: 89 active+clean; 119 MiB data, 404 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:39.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:39 smithi029 ceph-mon[39999]: osdmap e87: 8 total, 8 up, 8 in 2024-04-12T11:14:39.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:39 smithi029 ceph-mon[29349]: pgmap v106: 89 pgs: 89 active+clean; 119 MiB data, 404 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:39.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:39 smithi029 ceph-mon[29349]: osdmap e87: 8 total, 8 up, 8 in 2024-04-12T11:14:39.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:39 smithi087 ceph-mon[32917]: pgmap v106: 89 pgs: 89 active+clean; 119 MiB data, 404 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:39.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:39 smithi087 ceph-mon[32917]: osdmap e87: 8 total, 8 up, 8 in 2024-04-12T11:14:40.595 INFO:tasks.rados.rados.0.smithi029.stdout:114: done (1 left) 2024-04-12T11:14:40.595 INFO:tasks.rados.rados.0.smithi029.stdout:115: write oid 1 current snap is 10 2024-04-12T11:14:40.595 INFO:tasks.rados.rados.0.smithi029.stdout:115: seq_num 81 ranges {561367=458699,1573512=741359,2941380=419987} 2024-04-12T11:14:40.602 INFO:tasks.rados.rados.0.smithi029.stdout:115: writing smithi02979277-1 from 561367 to 1020066 tid 1 2024-04-12T11:14:40.609 INFO:tasks.rados.rados.0.smithi029.stdout:115: writing smithi02979277-1 from 1573512 to 2314871 tid 2 2024-04-12T11:14:40.616 INFO:tasks.rados.rados.0.smithi029.stdout:115: writing smithi02979277-1 from 2941380 to 3361367 tid 3 2024-04-12T11:14:40.616 INFO:tasks.rados.rados.0.smithi029.stdout:112: finishing write tid 4 to smithi02979277-47 2024-04-12T11:14:40.616 INFO:tasks.rados.rados.0.smithi029.stdout:112: finishing write tid 5 to smithi02979277-47 2024-04-12T11:14:40.616 INFO:tasks.rados.rados.0.smithi029.stdout:112: oid 47 updating version 0 to 33 2024-04-12T11:14:40.616 INFO:tasks.rados.rados.0.smithi029.stdout:112: oid 47 updating version 33 to 34 2024-04-12T11:14:40.616 INFO:tasks.rados.rados.0.smithi029.stdout:112: oid 47 version 34 is already newer than 31 2024-04-12T11:14:40.616 INFO:tasks.rados.rados.0.smithi029.stdout:112: oid 47 version 34 is already newer than 32 2024-04-12T11:14:40.616 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 34 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-04-12T11:14:40.616 INFO:tasks.rados.rados.0.smithi029.stdout:112: left oid 47 (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:14:40.616 INFO:tasks.rados.rados.0.smithi029.stdout:115: finishing write tid 1 to smithi02979277-1 2024-04-12T11:14:40.617 INFO:tasks.rados.rados.0.smithi029.stdout:115: finishing write tid 2 to smithi02979277-1 2024-04-12T11:14:40.617 INFO:tasks.rados.rados.0.smithi029.stdout:112: done (1 left) 2024-04-12T11:14:40.617 INFO:tasks.rados.rados.0.smithi029.stdout:116: read oid 12 snap -1 2024-04-12T11:14:40.617 INFO:tasks.rados.rados.0.smithi029.stdout:116: expect (ObjNum 68 snap 7 seq_num 68) 2024-04-12T11:14:40.617 INFO:tasks.rados.rados.0.smithi029.stdout:117: rollback oid 38 current snap is 10 2024-04-12T11:14:40.617 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 38 to 1 2024-04-12T11:14:40.617 INFO:tasks.rados.rados.0.smithi029.stdout:118: delete oid 39 current snap is 10 2024-04-12T11:14:40.619 INFO:tasks.rados.rados.0.smithi029.stdout:115: finishing write tid 3 to smithi02979277-1 2024-04-12T11:14:40.619 INFO:tasks.rados.rados.0.smithi029.stdout:115: finishing write tid 4 to smithi02979277-1 2024-04-12T11:14:40.619 INFO:tasks.rados.rados.0.smithi029.stdout:115: finishing write tid 5 to smithi02979277-1 2024-04-12T11:14:40.619 INFO:tasks.rados.rados.0.smithi029.stdout:115: oid 1 updating version 0 to 23 2024-04-12T11:14:40.619 INFO:tasks.rados.rados.0.smithi029.stdout:115: oid 1 updating version 23 to 24 2024-04-12T11:14:40.619 INFO:tasks.rados.rados.0.smithi029.stdout:115: oid 1 updating version 24 to 26 2024-04-12T11:14:40.619 INFO:tasks.rados.rados.0.smithi029.stdout:115: oid 1 version 26 is already newer than 25 2024-04-12T11:14:40.619 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 26 (ObjNum 81 snap 10 seq_num 81) dirty exists 2024-04-12T11:14:40.619 INFO:tasks.rados.rados.0.smithi029.stdout:115: left oid 1 (ObjNum 81 snap 10 seq_num 81) 2024-04-12T11:14:40.620 INFO:tasks.rados.rados.0.smithi029.stdout:116: expect (ObjNum 68 snap 7 seq_num 68) 2024-04-12T11:14:40.830 INFO:tasks.rados.rados.0.smithi029.stdout:117: finishing rollback tid 0 to smithi02979277-38 2024-04-12T11:14:40.830 INFO:tasks.rados.rados.0.smithi029.stdout:117: finishing rollback tid 1 to smithi02979277-38 2024-04-12T11:14:40.830 INFO:tasks.rados.rados.0.smithi029.stdout:117: finishing rollback tid 2 to smithi02979277-38 2024-04-12T11:14:40.830 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 17 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-04-12T11:14:40.830 INFO:tasks.rados.rados.0.smithi029.stdout:115: done (3 left) 2024-04-12T11:14:40.830 INFO:tasks.rados.rados.0.smithi029.stdout:116: done (2 left) 2024-04-12T11:14:40.831 INFO:tasks.rados.rados.0.smithi029.stdout:117: done (1 left) 2024-04-12T11:14:40.831 INFO:tasks.rados.rados.0.smithi029.stdout:118: done (0 left) 2024-04-12T11:14:40.831 INFO:tasks.rados.rados.0.smithi029.stdout:119: read oid 49 snap -1 2024-04-12T11:14:40.831 INFO:tasks.rados.rados.0.smithi029.stdout:119: expect (ObjNum 48 snap 0 seq_num 48) 2024-04-12T11:14:40.831 INFO:tasks.rados.rados.0.smithi029.stdout:120: read oid 13 snap -1 2024-04-12T11:14:40.831 INFO:tasks.rados.rados.0.smithi029.stdout:120: expect (ObjNum 77 snap 9 seq_num 77) 2024-04-12T11:14:40.831 INFO:tasks.rados.rados.0.smithi029.stdout:121: read oid 47 snap -1 2024-04-12T11:14:40.831 INFO:tasks.rados.rados.0.smithi029.stdout:121: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:14:40.831 INFO:tasks.rados.rados.0.smithi029.stdout:122: snap_remove snap 9 2024-04-12T11:14:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:40 smithi087 ceph-mon[32917]: osdmap e88: 8 total, 8 up, 8 in 2024-04-12T11:14:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:40 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1594578329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:40 smithi029 ceph-mon[39999]: osdmap e88: 8 total, 8 up, 8 in 2024-04-12T11:14:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:40 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1594578329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:40 smithi029 ceph-mon[29349]: osdmap e88: 8 total, 8 up, 8 in 2024-04-12T11:14:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:40 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1594578329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:41.513 INFO:tasks.rados.rados.0.smithi029.stdout:122: done (3 left) 2024-04-12T11:14:41.513 INFO:tasks.rados.rados.0.smithi029.stdout:123: write oid 27 current snap is 10 2024-04-12T11:14:41.513 INFO:tasks.rados.rados.0.smithi029.stdout:123: seq_num 83 ranges {594981=609197,1974123=745028,3279873=115108} 2024-04-12T11:14:41.521 INFO:tasks.rados.rados.0.smithi029.stdout:123: writing smithi02979277-27 from 594981 to 1204178 tid 1 2024-04-12T11:14:41.530 INFO:tasks.rados.rados.0.smithi029.stdout:123: writing smithi02979277-27 from 1974123 to 2719151 tid 2 2024-04-12T11:14:41.534 INFO:tasks.rados.rados.0.smithi029.stdout:123: writing smithi02979277-27 from 3279873 to 3394981 tid 3 2024-04-12T11:14:41.534 INFO:tasks.rados.rados.0.smithi029.stdout:124: write oid 28 current snap is 10 2024-04-12T11:14:41.534 INFO:tasks.rados.rados.0.smithi029.stdout:124: seq_num 84 ranges {611788=400799,1748958=488686,2907296=504492} 2024-04-12T11:14:41.539 INFO:tasks.rados.rados.0.smithi029.stdout:124: writing smithi02979277-28 from 611788 to 1012587 tid 1 2024-04-12T11:14:41.547 INFO:tasks.rados.rados.0.smithi029.stdout:124: writing smithi02979277-28 from 1748958 to 2237644 tid 2 2024-04-12T11:14:41.553 INFO:tasks.rados.rados.0.smithi029.stdout:124: writing smithi02979277-28 from 2907296 to 3411788 tid 3 2024-04-12T11:14:41.554 INFO:tasks.rados.rados.0.smithi029.stdout:125: write oid 44 current snap is 10 2024-04-12T11:14:41.554 INFO:tasks.rados.rados.0.smithi029.stdout:125: seq_num 85 ranges {628595=476048,1807440=632344,2934719=452474} 2024-04-12T11:14:41.560 INFO:tasks.rados.rados.0.smithi029.stdout:125: writing smithi02979277-44 from 628595 to 1104643 tid 1 2024-04-12T11:14:41.567 INFO:tasks.rados.rados.0.smithi029.stdout:125: writing smithi02979277-44 from 1807440 to 2439784 tid 2 2024-04-12T11:14:41.573 INFO:tasks.rados.rados.0.smithi029.stdout:125: writing smithi02979277-44 from 2934719 to 3387193 tid 3 2024-04-12T11:14:41.573 INFO:tasks.rados.rados.0.smithi029.stdout:126: snap_create 2024-04-12T11:14:41.573 INFO:tasks.rados.rados.0.smithi029.stdout:121: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:14:41.736 INFO:tasks.rados.rados.0.smithi029.stdout:119: expect (ObjNum 48 snap 0 seq_num 48) 2024-04-12T11:14:41.833 INFO:tasks.rados.rados.0.smithi029.stdout:120: expect (ObjNum 77 snap 9 seq_num 77) 2024-04-12T11:14:41.955 INFO:tasks.rados.rados.0.smithi029.stdout:123: finishing write tid 1 to smithi02979277-27 2024-04-12T11:14:41.955 INFO:tasks.rados.rados.0.smithi029.stdout:123: finishing write tid 2 to smithi02979277-27 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:123: finishing write tid 3 to smithi02979277-27 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:123: finishing write tid 4 to smithi02979277-27 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:123: finishing write tid 5 to smithi02979277-27 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:123: oid 27 updating version 0 to 31 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:123: oid 27 version 31 is already newer than 30 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:123: oid 27 version 31 is already newer than 28 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:123: oid 27 version 31 is already newer than 29 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 31 (ObjNum 83 snap 10 seq_num 83) dirty exists 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:123: left oid 27 (ObjNum 83 snap 10 seq_num 83) 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:124: finishing write tid 1 to smithi02979277-28 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:124: finishing write tid 2 to smithi02979277-28 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:124: finishing write tid 3 to smithi02979277-28 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:124: finishing write tid 4 to smithi02979277-28 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:124: finishing write tid 5 to smithi02979277-28 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:124: oid 28 updating version 0 to 36 2024-04-12T11:14:41.956 INFO:tasks.rados.rados.0.smithi029.stdout:124: oid 28 version 36 is already newer than 35 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:124: oid 28 version 36 is already newer than 33 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:124: oid 28 version 36 is already newer than 34 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 36 (ObjNum 84 snap 10 seq_num 84) dirty exists 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:124: left oid 28 (ObjNum 84 snap 10 seq_num 84) 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:125: finishing write tid 1 to smithi02979277-44 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:125: finishing write tid 2 to smithi02979277-44 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:125: finishing write tid 3 to smithi02979277-44 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:125: finishing write tid 4 to smithi02979277-44 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:125: finishing write tid 5 to smithi02979277-44 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:125: oid 44 updating version 0 to 32 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:125: oid 44 version 32 is already newer than 31 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:125: oid 44 updating version 32 to 33 2024-04-12T11:14:41.957 INFO:tasks.rados.rados.0.smithi029.stdout:125: oid 44 version 33 is already newer than 30 2024-04-12T11:14:41.958 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 33 (ObjNum 85 snap 10 seq_num 85) dirty exists 2024-04-12T11:14:41.958 INFO:tasks.rados.rados.0.smithi029.stdout:125: left oid 44 (ObjNum 85 snap 10 seq_num 85) 2024-04-12T11:14:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:41 smithi087 ceph-mon[32917]: pgmap v109: 89 pgs: 89 active+clean; 133 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2024-04-12T11:14:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:41 smithi087 ceph-mon[32917]: osdmap e89: 8 total, 8 up, 8 in 2024-04-12T11:14:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:41 smithi087 ceph-mon[32917]: osdmap e90: 8 total, 8 up, 8 in 2024-04-12T11:14:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:41 smithi029 ceph-mon[29349]: pgmap v109: 89 pgs: 89 active+clean; 133 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2024-04-12T11:14:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:41 smithi029 ceph-mon[29349]: osdmap e89: 8 total, 8 up, 8 in 2024-04-12T11:14:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:41 smithi029 ceph-mon[29349]: osdmap e90: 8 total, 8 up, 8 in 2024-04-12T11:14:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:41 smithi029 ceph-mon[39999]: pgmap v109: 89 pgs: 89 active+clean; 133 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2024-04-12T11:14:42.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:41 smithi029 ceph-mon[39999]: osdmap e89: 8 total, 8 up, 8 in 2024-04-12T11:14:42.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:41 smithi029 ceph-mon[39999]: osdmap e90: 8 total, 8 up, 8 in 2024-04-12T11:14:42.514 INFO:tasks.rados.rados.0.smithi029.stdout:119: done (6 left) 2024-04-12T11:14:42.515 INFO:tasks.rados.rados.0.smithi029.stdout:120: done (5 left) 2024-04-12T11:14:42.515 INFO:tasks.rados.rados.0.smithi029.stdout:121: done (4 left) 2024-04-12T11:14:42.515 INFO:tasks.rados.rados.0.smithi029.stdout:123: done (3 left) 2024-04-12T11:14:42.515 INFO:tasks.rados.rados.0.smithi029.stdout:124: done (2 left) 2024-04-12T11:14:42.515 INFO:tasks.rados.rados.0.smithi029.stdout:125: done (1 left) 2024-04-12T11:14:42.515 INFO:tasks.rados.rados.0.smithi029.stdout:126: done (0 left) 2024-04-12T11:14:42.515 INFO:tasks.rados.rados.0.smithi029.stdout:127: write oid 32 current snap is 11 2024-04-12T11:14:42.515 INFO:tasks.rados.rados.0.smithi029.stdout:127: seq_num 86 ranges {645402=551297,1749569=492355,2845789=599613} 2024-04-12T11:14:42.521 INFO:tasks.rados.rados.0.smithi029.stdout:127: writing smithi02979277-32 from 645402 to 1196699 tid 1 2024-04-12T11:14:42.526 INFO:tasks.rados.rados.0.smithi029.stdout:127: writing smithi02979277-32 from 1749569 to 2241924 tid 2 2024-04-12T11:14:42.533 INFO:tasks.rados.rados.0.smithi029.stdout:127: writing smithi02979277-32 from 2845789 to 3445402 tid 3 2024-04-12T11:14:42.533 INFO:tasks.rados.rados.0.smithi029.stdout:128: read oid 35 snap -1 2024-04-12T11:14:42.533 INFO:tasks.rados.rados.0.smithi029.stdout:128: expect (ObjNum 34 snap 0 seq_num 34) 2024-04-12T11:14:42.534 INFO:tasks.rados.rados.0.smithi029.stdout:129: read oid 11 snap -1 2024-04-12T11:14:42.534 INFO:tasks.rados.rados.0.smithi029.stdout:129: expect (ObjNum 10 snap 0 seq_num 10) 2024-04-12T11:14:42.534 INFO:tasks.rados.rados.0.smithi029.stdout:130: write oid 42 current snap is 11 2024-04-12T11:14:42.534 INFO:tasks.rados.rados.0.smithi029.stdout:130: seq_num 87 ranges {662209=626546,1808051=636013,2873212=588997} 2024-04-12T11:14:42.540 INFO:tasks.rados.rados.0.smithi029.stdout:130: writing smithi02979277-42 from 662209 to 1288755 tid 1 2024-04-12T11:14:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:130: writing smithi02979277-42 from 1808051 to 2444064 tid 2 2024-04-12T11:14:42.552 INFO:tasks.rados.rados.0.smithi029.stdout:130: writing smithi02979277-42 from 2873212 to 3462209 tid 3 2024-04-12T11:14:42.552 INFO:tasks.rados.rados.0.smithi029.stdout:131: snap_remove snap 10 2024-04-12T11:14:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:42 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3734545638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:42 smithi087 ceph-mon[32917]: osdmap e91: 8 total, 8 up, 8 in 2024-04-12T11:14:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:42 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3734545638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:42 smithi029 ceph-mon[29349]: osdmap e91: 8 total, 8 up, 8 in 2024-04-12T11:14:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:42 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3734545638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:42 smithi029 ceph-mon[39999]: osdmap e91: 8 total, 8 up, 8 in 2024-04-12T11:14:43.516 INFO:tasks.rados.rados.0.smithi029.stdout:131: done (4 left) 2024-04-12T11:14:43.516 INFO:tasks.rados.rados.0.smithi029.stdout:132: read oid 16 snap -1 2024-04-12T11:14:43.516 INFO:tasks.rados.rados.0.smithi029.stdout:132: expect (ObjNum 15 snap 0 seq_num 15) 2024-04-12T11:14:43.516 INFO:tasks.rados.rados.0.smithi029.stdout:133: snap_remove snap 3 2024-04-12T11:14:43.517 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:43 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:14:43] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.43.0" 2024-04-12T11:14:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:43 smithi087 ceph-mon[32917]: pgmap v112: 89 pgs: 89 active+clean; 133 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2024-04-12T11:14:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:43 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:43.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:43 smithi087 ceph-mon[32917]: osdmap e92: 8 total, 8 up, 8 in 2024-04-12T11:14:44.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:43 smithi029 ceph-mon[39999]: pgmap v112: 89 pgs: 89 active+clean; 133 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2024-04-12T11:14:44.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:43 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:44.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:43 smithi029 ceph-mon[39999]: osdmap e92: 8 total, 8 up, 8 in 2024-04-12T11:14:44.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:43 smithi029 ceph-mon[29349]: pgmap v112: 89 pgs: 89 active+clean; 133 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2024-04-12T11:14:44.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:43 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:44.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:43 smithi029 ceph-mon[29349]: osdmap e92: 8 total, 8 up, 8 in 2024-04-12T11:14:44.518 INFO:tasks.rados.rados.0.smithi029.stdout:133: done (5 left) 2024-04-12T11:14:44.518 INFO:tasks.rados.rados.0.smithi029.stdout:134: write oid 20 current snap is 11 2024-04-12T11:14:44.518 INFO:tasks.rados.rados.0.smithi029.stdout:134: seq_num 88 ranges {679016=701795,1866533=496024,2900635=402643} 2024-04-12T11:14:44.528 INFO:tasks.rados.rados.0.smithi029.stdout:134: writing smithi02979277-20 from 679016 to 1380811 tid 1 2024-04-12T11:14:44.535 INFO:tasks.rados.rados.0.smithi029.stdout:134: writing smithi02979277-20 from 1866533 to 2362557 tid 2 2024-04-12T11:14:44.541 INFO:tasks.rados.rados.0.smithi029.stdout:134: writing smithi02979277-20 from 2900635 to 3303278 tid 3 2024-04-12T11:14:44.541 INFO:tasks.rados.rados.0.smithi029.stdout:135: read oid 10 snap -1 2024-04-12T11:14:44.541 INFO:tasks.rados.rados.0.smithi029.stdout:135: expect deleted 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:127: finishing write tid 1 to smithi02979277-32 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:136: write oid 33 current snap is 11 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:127: finishing write tid 2 to smithi02979277-32 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:130: finishing write tid 1 to smithi02979277-42 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:130: finishing write tid 2 to smithi02979277-42 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:130: finishing write tid 3 to smithi02979277-42 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:130: finishing write tid 4 to smithi02979277-42 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:130: finishing write tid 5 to smithi02979277-42 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:130: oid 42 updating version 0 to 18 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:130: oid 42 version 18 is already newer than 16 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:130: oid 42 version 18 is already newer than 17 2024-04-12T11:14:44.542 INFO:tasks.rados.rados.0.smithi029.stdout:130: oid 42 version 18 is already newer than 15 2024-04-12T11:14:44.543 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 18 (ObjNum 87 snap 11 seq_num 87) dirty exists 2024-04-12T11:14:44.543 INFO:tasks.rados.rados.0.smithi029.stdout:130: left oid 42 (ObjNum 87 snap 11 seq_num 87) 2024-04-12T11:14:44.543 INFO:tasks.rados.rados.0.smithi029.stdout:128: expect (ObjNum 34 snap 0 seq_num 34) 2024-04-12T11:14:44.628 INFO:tasks.rados.rados.0.smithi029.stdout:127: finishing write tid 3 to smithi02979277-32 2024-04-12T11:14:44.628 INFO:tasks.rados.rados.0.smithi029.stdout:127: finishing write tid 4 to smithi02979277-32 2024-04-12T11:14:44.628 INFO:tasks.rados.rados.0.smithi029.stdout:136: seq_num 89 ranges {695823=777044,1925015=639682,3328058=167765} 2024-04-12T11:14:44.636 INFO:tasks.rados.rados.0.smithi029.stdout:136: writing smithi02979277-33 from 695823 to 1472867 tid 1 2024-04-12T11:14:44.642 INFO:tasks.rados.rados.0.smithi029.stdout:136: writing smithi02979277-33 from 1925015 to 2564697 tid 2 2024-04-12T11:14:44.648 INFO:tasks.rados.rados.0.smithi029.stdout:136: writing smithi02979277-33 from 3328058 to 3495823 tid 3 2024-04-12T11:14:44.648 INFO:tasks.rados.rados.0.smithi029.stdout:128: done (7 left) 2024-04-12T11:14:44.648 INFO:tasks.rados.rados.0.smithi029.stdout:130: done (6 left) 2024-04-12T11:14:44.648 INFO:tasks.rados.rados.0.smithi029.stdout:137: delete oid 8 current snap is 11 2024-04-12T11:14:44.648 INFO:tasks.rados.rados.0.smithi029.stdout:127: finishing write tid 5 to smithi02979277-32 2024-04-12T11:14:44.649 INFO:tasks.rados.rados.0.smithi029.stdout:127: oid 32 updating version 0 to 19 2024-04-12T11:14:44.649 INFO:tasks.rados.rados.0.smithi029.stdout:127: oid 32 version 19 is already newer than 18 2024-04-12T11:14:44.649 INFO:tasks.rados.rados.0.smithi029.stdout:127: oid 32 updating version 19 to 21 2024-04-12T11:14:44.649 INFO:tasks.rados.rados.0.smithi029.stdout:127: oid 32 version 21 is already newer than 20 2024-04-12T11:14:44.649 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 21 (ObjNum 86 snap 11 seq_num 86) dirty exists 2024-04-12T11:14:44.649 INFO:tasks.rados.rados.0.smithi029.stdout:127: left oid 32 (ObjNum 86 snap 11 seq_num 86) 2024-04-12T11:14:44.649 INFO:tasks.rados.rados.0.smithi029.stdout:129: expect (ObjNum 10 snap 0 seq_num 10) 2024-04-12T11:14:44.727 INFO:tasks.rados.rados.0.smithi029.stdout:132: expect (ObjNum 15 snap 0 seq_num 15) 2024-04-12T11:14:44.811 INFO:tasks.rados.rados.0.smithi029.stdout:134: finishing write tid 1 to smithi02979277-20 2024-04-12T11:14:44.811 INFO:tasks.rados.rados.0.smithi029.stdout:134: finishing write tid 2 to smithi02979277-20 2024-04-12T11:14:44.811 INFO:tasks.rados.rados.0.smithi029.stdout:127: done (6 left) 2024-04-12T11:14:44.811 INFO:tasks.rados.rados.0.smithi029.stdout:129: done (5 left) 2024-04-12T11:14:44.811 INFO:tasks.rados.rados.0.smithi029.stdout:132: done (4 left) 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:137: done (3 left) 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:138: snap_create 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:134: finishing write tid 3 to smithi02979277-20 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:134: finishing write tid 4 to smithi02979277-20 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:134: finishing write tid 5 to smithi02979277-20 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:134: oid 20 updating version 0 to 23 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:134: oid 20 updating version 23 to 26 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:134: oid 20 version 26 is already newer than 25 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:134: oid 20 version 26 is already newer than 24 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 26 (ObjNum 88 snap 11 seq_num 88) dirty exists 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:134: left oid 20 (ObjNum 88 snap 11 seq_num 88) 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:136: finishing write tid 1 to smithi02979277-33 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:136: finishing write tid 2 to smithi02979277-33 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:136: finishing write tid 3 to smithi02979277-33 2024-04-12T11:14:44.812 INFO:tasks.rados.rados.0.smithi029.stdout:136: finishing write tid 4 to smithi02979277-33 2024-04-12T11:14:44.813 INFO:tasks.rados.rados.0.smithi029.stdout:136: finishing write tid 5 to smithi02979277-33 2024-04-12T11:14:44.813 INFO:tasks.rados.rados.0.smithi029.stdout:136: oid 33 updating version 0 to 52 2024-04-12T11:14:44.813 INFO:tasks.rados.rados.0.smithi029.stdout:136: oid 33 version 52 is already newer than 49 2024-04-12T11:14:44.813 INFO:tasks.rados.rados.0.smithi029.stdout:136: oid 33 version 52 is already newer than 51 2024-04-12T11:14:44.813 INFO:tasks.rados.rados.0.smithi029.stdout:136: oid 33 version 52 is already newer than 50 2024-04-12T11:14:44.813 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 52 (ObjNum 89 snap 11 seq_num 89) dirty exists 2024-04-12T11:14:44.813 INFO:tasks.rados.rados.0.smithi029.stdout:136: left oid 33 (ObjNum 89 snap 11 seq_num 89) 2024-04-12T11:14:45.601 INFO:tasks.rados.rados.0.smithi029.stdout:134: done (3 left) 2024-04-12T11:14:45.602 INFO:tasks.rados.rados.0.smithi029.stdout:135: done (2 left) 2024-04-12T11:14:45.602 INFO:tasks.rados.rados.0.smithi029.stdout:136: done (1 left) 2024-04-12T11:14:45.602 INFO:tasks.rados.rados.0.smithi029.stdout:138: done (0 left) 2024-04-12T11:14:45.602 INFO:tasks.rados.rados.0.smithi029.stdout:139: snap_remove snap 1 2024-04-12T11:14:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:45 smithi029 ceph-mon[29349]: pgmap v115: 89 pgs: 89 active+clean; 146 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 2.6 MiB/s wr, 25 op/s 2024-04-12T11:14:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:45 smithi029 ceph-mon[29349]: osdmap e93: 8 total, 8 up, 8 in 2024-04-12T11:14:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:45 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/381000835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:45 smithi029 ceph-mon[39999]: pgmap v115: 89 pgs: 89 active+clean; 146 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 2.6 MiB/s wr, 25 op/s 2024-04-12T11:14:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:45 smithi029 ceph-mon[39999]: osdmap e93: 8 total, 8 up, 8 in 2024-04-12T11:14:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:45 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/381000835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:45.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:45 smithi087 ceph-mon[32917]: pgmap v115: 89 pgs: 89 active+clean; 146 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 2.6 MiB/s wr, 25 op/s 2024-04-12T11:14:45.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:45 smithi087 ceph-mon[32917]: osdmap e93: 8 total, 8 up, 8 in 2024-04-12T11:14:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:45 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/381000835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:46.516 INFO:tasks.rados.rados.0.smithi029.stdout:139: done (0 left) 2024-04-12T11:14:46.517 INFO:tasks.rados.rados.0.smithi029.stdout:140: snap_create 2024-04-12T11:14:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:46 smithi087 ceph-mon[32917]: osdmap e94: 8 total, 8 up, 8 in 2024-04-12T11:14:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:46 smithi087 ceph-mon[32917]: osdmap e95: 8 total, 8 up, 8 in 2024-04-12T11:14:47.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:46 smithi029 ceph-mon[39999]: osdmap e94: 8 total, 8 up, 8 in 2024-04-12T11:14:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:46 smithi029 ceph-mon[39999]: osdmap e95: 8 total, 8 up, 8 in 2024-04-12T11:14:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:46 smithi029 ceph-mon[29349]: osdmap e94: 8 total, 8 up, 8 in 2024-04-12T11:14:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:46 smithi029 ceph-mon[29349]: osdmap e95: 8 total, 8 up, 8 in 2024-04-12T11:14:47.516 INFO:tasks.rados.rados.0.smithi029.stdout:140: done (0 left) 2024-04-12T11:14:47.516 INFO:tasks.rados.rados.0.smithi029.stdout:141: read oid 31 snap -1 2024-04-12T11:14:47.516 INFO:tasks.rados.rados.0.smithi029.stdout:141: expect (ObjNum 30 snap 0 seq_num 30) 2024-04-12T11:14:47.516 INFO:tasks.rados.rados.0.smithi029.stdout:142: read oid 13 snap -1 2024-04-12T11:14:47.516 INFO:tasks.rados.rados.0.smithi029.stdout:142: expect (ObjNum 77 snap 9 seq_num 77) 2024-04-12T11:14:47.516 INFO:tasks.rados.rados.0.smithi029.stdout:143: read oid 7 snap -1 2024-04-12T11:14:47.516 INFO:tasks.rados.rados.0.smithi029.stdout:143: expect deleted 2024-04-12T11:14:47.516 INFO:tasks.rados.rados.0.smithi029.stdout:144: snap_create 2024-04-12T11:14:47.520 INFO:tasks.rados.rados.0.smithi029.stdout:141: expect (ObjNum 30 snap 0 seq_num 30) 2024-04-12T11:14:47.611 INFO:tasks.rados.rados.0.smithi029.stdout:142: expect (ObjNum 77 snap 9 seq_num 77) 2024-04-12T11:14:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:47 smithi087 ceph-mon[32917]: pgmap v118: 89 pgs: 89 active+clean; 146 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 2.6 MiB/s wr, 25 op/s 2024-04-12T11:14:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:47 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/996608643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:47 smithi087 ceph-mon[32917]: osdmap e96: 8 total, 8 up, 8 in 2024-04-12T11:14:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:47 smithi029 ceph-mon[29349]: pgmap v118: 89 pgs: 89 active+clean; 146 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 2.6 MiB/s wr, 25 op/s 2024-04-12T11:14:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:47 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/996608643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:47 smithi029 ceph-mon[29349]: osdmap e96: 8 total, 8 up, 8 in 2024-04-12T11:14:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:47 smithi029 ceph-mon[39999]: pgmap v118: 89 pgs: 89 active+clean; 146 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 2.6 MiB/s wr, 25 op/s 2024-04-12T11:14:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:47 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/996608643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:47 smithi029 ceph-mon[39999]: osdmap e96: 8 total, 8 up, 8 in 2024-04-12T11:14:48.517 INFO:tasks.rados.rados.0.smithi029.stdout:141: done (3 left) 2024-04-12T11:14:48.517 INFO:tasks.rados.rados.0.smithi029.stdout:142: done (2 left) 2024-04-12T11:14:48.517 INFO:tasks.rados.rados.0.smithi029.stdout:143: done (1 left) 2024-04-12T11:14:48.517 INFO:tasks.rados.rados.0.smithi029.stdout:144: done (0 left) 2024-04-12T11:14:48.517 INFO:tasks.rados.rados.0.smithi029.stdout:145: snap_create 2024-04-12T11:14:49.522 INFO:tasks.rados.rados.0.smithi029.stdout:145: done (0 left) 2024-04-12T11:14:49.522 INFO:tasks.rados.rados.0.smithi029.stdout:146: write oid 9 current snap is 15 2024-04-12T11:14:49.522 INFO:tasks.rados.rados.0.smithi029.stdout:146: seq_num 91 ranges {729437=527542,1925626=643351,3266551=262886} 2024-04-12T11:14:49.529 INFO:tasks.rados.rados.0.smithi029.stdout:146: writing smithi02979277-9 from 729437 to 1256979 tid 1 2024-04-12T11:14:49.536 INFO:tasks.rados.rados.0.smithi029.stdout:146: writing smithi02979277-9 from 1925626 to 2568977 tid 2 2024-04-12T11:14:49.542 INFO:tasks.rados.rados.0.smithi029.stdout:146: writing smithi02979277-9 from 3266551 to 3529437 tid 3 2024-04-12T11:14:49.542 INFO:tasks.rados.rados.0.smithi029.stdout:147: rollback oid 14 current snap is 15 2024-04-12T11:14:49.542 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 13 2024-04-12T11:14:49.542 INFO:tasks.rados.rados.0.smithi029.stdout:146: finishing write tid 1 to smithi02979277-9 2024-04-12T11:14:49.542 INFO:tasks.rados.rados.0.smithi029.stdout:146: finishing write tid 2 to smithi02979277-9 2024-04-12T11:14:49.543 INFO:tasks.rados.rados.0.smithi029.stdout:148: write oid 6 current snap is 15 2024-04-12T11:14:49.543 INFO:tasks.rados.rados.0.smithi029.stdout:148: seq_num 92 ranges {746244=719144,2100461=503362,3126680=419564} 2024-04-12T11:14:49.550 INFO:tasks.rados.rados.0.smithi029.stdout:148: writing smithi02979277-6 from 746244 to 1465388 tid 1 2024-04-12T11:14:49.556 INFO:tasks.rados.rados.0.smithi029.stdout:148: writing smithi02979277-6 from 2100461 to 2603823 tid 2 2024-04-12T11:14:49.562 INFO:tasks.rados.rados.0.smithi029.stdout:148: writing smithi02979277-6 from 3126680 to 3546244 tid 3 2024-04-12T11:14:49.562 INFO:tasks.rados.rados.0.smithi029.stdout:149: read oid 42 snap -1 2024-04-12T11:14:49.562 INFO:tasks.rados.rados.0.smithi029.stdout:149: expect (ObjNum 87 snap 11 seq_num 87) 2024-04-12T11:14:49.562 INFO:tasks.rados.rados.0.smithi029.stdout:150: snap_create 2024-04-12T11:14:49.562 INFO:tasks.rados.rados.0.smithi029.stdout:146: finishing write tid 3 to smithi02979277-9 2024-04-12T11:14:49.562 INFO:tasks.rados.rados.0.smithi029.stdout:146: finishing write tid 4 to smithi02979277-9 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:146: finishing write tid 5 to smithi02979277-9 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:146: oid 9 updating version 0 to 35 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:146: oid 9 version 35 is already newer than 34 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:146: oid 9 updating version 35 to 36 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:146: oid 9 version 36 is already newer than 33 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 36 (ObjNum 91 snap 15 seq_num 91) dirty exists 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:146: left oid 9 (ObjNum 91 snap 15 seq_num 91) 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:147: finishing rollback tid 0 to smithi02979277-14 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:147: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:147: finishing rollback tid 2 to smithi02979277-14 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 45 (ObjNum 75 snap 9 seq_num 75) dirty exists 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:148: finishing write tid 1 to smithi02979277-6 2024-04-12T11:14:49.563 INFO:tasks.rados.rados.0.smithi029.stdout:148: finishing write tid 2 to smithi02979277-6 2024-04-12T11:14:49.564 INFO:tasks.rados.rados.0.smithi029.stdout:148: finishing write tid 3 to smithi02979277-6 2024-04-12T11:14:49.564 INFO:tasks.rados.rados.0.smithi029.stdout:148: finishing write tid 4 to smithi02979277-6 2024-04-12T11:14:49.565 INFO:tasks.rados.rados.0.smithi029.stdout:148: finishing write tid 5 to smithi02979277-6 2024-04-12T11:14:49.565 INFO:tasks.rados.rados.0.smithi029.stdout:148: oid 6 updating version 0 to 23 2024-04-12T11:14:49.565 INFO:tasks.rados.rados.0.smithi029.stdout:148: oid 6 updating version 23 to 24 2024-04-12T11:14:49.565 INFO:tasks.rados.rados.0.smithi029.stdout:148: oid 6 updating version 24 to 26 2024-04-12T11:14:49.565 INFO:tasks.rados.rados.0.smithi029.stdout:148: oid 6 version 26 is already newer than 25 2024-04-12T11:14:49.565 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 26 (ObjNum 92 snap 15 seq_num 92) dirty exists 2024-04-12T11:14:49.565 INFO:tasks.rados.rados.0.smithi029.stdout:148: left oid 6 (ObjNum 92 snap 15 seq_num 92) 2024-04-12T11:14:49.573 INFO:tasks.rados.rados.0.smithi029.stdout:149: expect (ObjNum 87 snap 11 seq_num 87) 2024-04-12T11:14:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:49 smithi029 ceph-mon[29349]: pgmap v121: 89 pgs: 89 active+clean; 146 MiB data, 472 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:49 smithi029 ceph-mon[29349]: osdmap e97: 8 total, 8 up, 8 in 2024-04-12T11:14:49.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:49 smithi029 ceph-mon[39999]: pgmap v121: 89 pgs: 89 active+clean; 146 MiB data, 472 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:49.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:49 smithi029 ceph-mon[39999]: osdmap e97: 8 total, 8 up, 8 in 2024-04-12T11:14:49.769 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:49 smithi087 ceph-mon[32917]: pgmap v121: 89 pgs: 89 active+clean; 146 MiB data, 472 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:49.769 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:49 smithi087 ceph-mon[32917]: osdmap e97: 8 total, 8 up, 8 in 2024-04-12T11:14:50.527 INFO:tasks.rados.rados.0.smithi029.stdout:146: done (4 left) 2024-04-12T11:14:50.527 INFO:tasks.rados.rados.0.smithi029.stdout:147: done (3 left) 2024-04-12T11:14:50.527 INFO:tasks.rados.rados.0.smithi029.stdout:148: done (2 left) 2024-04-12T11:14:50.528 INFO:tasks.rados.rados.0.smithi029.stdout:149: done (1 left) 2024-04-12T11:14:50.528 INFO:tasks.rados.rados.0.smithi029.stdout:150: done (0 left) 2024-04-12T11:14:50.528 INFO:tasks.rados.rados.0.smithi029.stdout:151: write oid 46 current snap is 16 2024-04-12T11:14:50.528 INFO:tasks.rados.rados.0.smithi029.stdout:151: seq_num 93 ranges {763051=794393,2158943=647020,3437750=125301} 2024-04-12T11:14:50.535 INFO:tasks.rados.rados.0.smithi029.stdout:151: writing smithi02979277-46 from 763051 to 1557444 tid 1 2024-04-12T11:14:50.542 INFO:tasks.rados.rados.0.smithi029.stdout:151: writing smithi02979277-46 from 2158943 to 2805963 tid 2 2024-04-12T11:14:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:151: writing smithi02979277-46 from 3437750 to 3563051 tid 3 2024-04-12T11:14:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:152: read oid 25 snap -1 2024-04-12T11:14:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:152: expect (ObjNum 78 snap 9 seq_num 78) 2024-04-12T11:14:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:153: write oid 50 current snap is 16 2024-04-12T11:14:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:151: finishing write tid 1 to smithi02979277-46 2024-04-12T11:14:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:153: seq_num 94 ranges {779858=469642,1817425=507031,2781526=586628} 2024-04-12T11:14:50.553 INFO:tasks.rados.rados.0.smithi029.stdout:153: writing smithi02979277-50 from 779858 to 1249500 tid 1 2024-04-12T11:14:50.559 INFO:tasks.rados.rados.0.smithi029.stdout:153: writing smithi02979277-50 from 1817425 to 2324456 tid 2 2024-04-12T11:14:50.565 INFO:tasks.rados.rados.0.smithi029.stdout:153: writing smithi02979277-50 from 2781526 to 3368154 tid 3 2024-04-12T11:14:50.565 INFO:tasks.rados.rados.0.smithi029.stdout:154: read oid 6 snap 12 2024-04-12T11:14:50.565 INFO:tasks.rados.rados.0.smithi029.stdout:154: expect (ObjNum 5 snap 0 seq_num 5) 2024-04-12T11:14:50.566 INFO:tasks.rados.rados.0.smithi029.stdout:155: snap_remove snap 8 2024-04-12T11:14:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:50 smithi087 ceph-mon[32917]: osdmap e98: 8 total, 8 up, 8 in 2024-04-12T11:14:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:50 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/4177651547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:50 smithi029 ceph-mon[39999]: osdmap e98: 8 total, 8 up, 8 in 2024-04-12T11:14:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:50 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/4177651547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:50 smithi029 ceph-mon[29349]: osdmap e98: 8 total, 8 up, 8 in 2024-04-12T11:14:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:50 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/4177651547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:51.516 INFO:tasks.rados.rados.0.smithi029.stdout:155: done (4 left) 2024-04-12T11:14:51.516 INFO:tasks.rados.rados.0.smithi029.stdout:156: snap_remove snap 6 2024-04-12T11:14:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:51 smithi087 ceph-mon[32917]: pgmap v124: 89 pgs: 89 active+clean; 150 MiB data, 485 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 778 KiB/s wr, 6 op/s 2024-04-12T11:14:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:51 smithi087 ceph-mon[32917]: osdmap e99: 8 total, 8 up, 8 in 2024-04-12T11:14:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:51 smithi087 ceph-mon[32917]: osdmap e100: 8 total, 8 up, 8 in 2024-04-12T11:14:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:51 smithi029 ceph-mon[39999]: pgmap v124: 89 pgs: 89 active+clean; 150 MiB data, 485 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 778 KiB/s wr, 6 op/s 2024-04-12T11:14:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:51 smithi029 ceph-mon[39999]: osdmap e99: 8 total, 8 up, 8 in 2024-04-12T11:14:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:51 smithi029 ceph-mon[39999]: osdmap e100: 8 total, 8 up, 8 in 2024-04-12T11:14:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:51 smithi029 ceph-mon[29349]: pgmap v124: 89 pgs: 89 active+clean; 150 MiB data, 485 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 778 KiB/s wr, 6 op/s 2024-04-12T11:14:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:51 smithi029 ceph-mon[29349]: osdmap e99: 8 total, 8 up, 8 in 2024-04-12T11:14:52.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:51 smithi029 ceph-mon[29349]: osdmap e100: 8 total, 8 up, 8 in 2024-04-12T11:14:52.518 INFO:tasks.rados.rados.0.smithi029.stdout:156: done (4 left) 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:157: delete oid 16 current snap is 16 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:151: finishing write tid 2 to smithi02979277-46 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:151: finishing write tid 3 to smithi02979277-46 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:151: finishing write tid 4 to smithi02979277-46 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:151: finishing write tid 5 to smithi02979277-46 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:151: oid 46 updating version 0 to 35 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:151: oid 46 version 35 is already newer than 33 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:151: oid 46 version 35 is already newer than 34 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:151: oid 46 version 35 is already newer than 32 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 35 (ObjNum 93 snap 16 seq_num 93) dirty exists 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:151: left oid 46 (ObjNum 93 snap 16 seq_num 93) 2024-04-12T11:14:52.519 INFO:tasks.rados.rados.0.smithi029.stdout:152: expect (ObjNum 78 snap 9 seq_num 78) 2024-04-12T11:14:52.722 INFO:tasks.rados.rados.0.smithi029.stdout:153: finishing write tid 1 to smithi02979277-50 2024-04-12T11:14:52.722 INFO:tasks.rados.rados.0.smithi029.stdout:153: finishing write tid 2 to smithi02979277-50 2024-04-12T11:14:52.722 INFO:tasks.rados.rados.0.smithi029.stdout:153: finishing write tid 3 to smithi02979277-50 2024-04-12T11:14:52.722 INFO:tasks.rados.rados.0.smithi029.stdout:153: finishing write tid 4 to smithi02979277-50 2024-04-12T11:14:52.722 INFO:tasks.rados.rados.0.smithi029.stdout:153: finishing write tid 5 to smithi02979277-50 2024-04-12T11:14:52.722 INFO:tasks.rados.rados.0.smithi029.stdout:153: oid 50 updating version 0 to 62 2024-04-12T11:14:52.722 INFO:tasks.rados.rados.0.smithi029.stdout:153: oid 50 version 62 is already newer than 60 2024-04-12T11:14:52.723 INFO:tasks.rados.rados.0.smithi029.stdout:153: oid 50 updating version 62 to 63 2024-04-12T11:14:52.723 INFO:tasks.rados.rados.0.smithi029.stdout:153: oid 50 version 63 is already newer than 61 2024-04-12T11:14:52.723 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 63 (ObjNum 94 snap 16 seq_num 94) dirty exists 2024-04-12T11:14:52.723 INFO:tasks.rados.rados.0.smithi029.stdout:153: left oid 50 (ObjNum 94 snap 16 seq_num 94) 2024-04-12T11:14:52.723 INFO:tasks.rados.rados.0.smithi029.stdout:154: expect (ObjNum 5 snap 0 seq_num 5) 2024-04-12T11:14:52.795 INFO:tasks.rados.rados.0.smithi029.stdout:151: done (4 left) 2024-04-12T11:14:52.795 INFO:tasks.rados.rados.0.smithi029.stdout:152: done (3 left) 2024-04-12T11:14:52.795 INFO:tasks.rados.rados.0.smithi029.stdout:153: done (2 left) 2024-04-12T11:14:52.795 INFO:tasks.rados.rados.0.smithi029.stdout:154: done (1 left) 2024-04-12T11:14:52.795 INFO:tasks.rados.rados.0.smithi029.stdout:157: done (0 left) 2024-04-12T11:14:52.795 INFO:tasks.rados.rados.0.smithi029.stdout:158: write oid 29 current snap is 16 2024-04-12T11:14:52.796 INFO:tasks.rados.rados.0.smithi029.stdout:158: seq_num 96 ranges {413472=620140,1818036=510700,3120019=493453} 2024-04-12T11:14:52.800 INFO:tasks.rados.rados.0.smithi029.stdout:158: writing smithi02979277-29 from 413472 to 1033612 tid 1 2024-04-12T11:14:52.808 INFO:tasks.rados.rados.0.smithi029.stdout:158: writing smithi02979277-29 from 1818036 to 2328736 tid 2 2024-04-12T11:14:52.815 INFO:tasks.rados.rados.0.smithi029.stdout:158: writing smithi02979277-29 from 3120019 to 3613472 tid 3 2024-04-12T11:14:52.815 INFO:tasks.rados.rados.0.smithi029.stdout:159: read oid 41 snap -1 2024-04-12T11:14:52.816 INFO:tasks.rados.rados.0.smithi029.stdout:159: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:14:52.816 INFO:tasks.rados.rados.0.smithi029.stdout:160: delete oid 28 current snap is 16 2024-04-12T11:14:52.816 INFO:tasks.rados.rados.0.smithi029.stdout:158: finishing write tid 1 to smithi02979277-29 2024-04-12T11:14:52.816 INFO:tasks.rados.rados.0.smithi029.stdout:158: finishing write tid 2 to smithi02979277-29 2024-04-12T11:14:52.818 INFO:tasks.rados.rados.0.smithi029.stdout:160: done (2 left) 2024-04-12T11:14:52.818 INFO:tasks.rados.rados.0.smithi029.stdout:161: write oid 44 current snap is 16 2024-04-12T11:14:52.818 INFO:tasks.rados.rados.0.smithi029.stdout:161: seq_num 98 ranges {447086=770638,1935000=798016,3458512=188574} 2024-04-12T11:14:52.825 INFO:tasks.rados.rados.0.smithi029.stdout:161: writing smithi02979277-44 from 447086 to 1217724 tid 1 2024-04-12T11:14:52.836 INFO:tasks.rados.rados.0.smithi029.stdout:161: writing smithi02979277-44 from 1935000 to 2733016 tid 2 2024-04-12T11:14:52.841 INFO:tasks.rados.rados.0.smithi029.stdout:161: writing smithi02979277-44 from 3458512 to 3647086 tid 3 2024-04-12T11:14:52.841 INFO:tasks.rados.rados.0.smithi029.stdout:162: snap_remove snap 12 2024-04-12T11:14:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:52 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1159384672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:52.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:52 smithi087 ceph-mon[32917]: osdmap e101: 8 total, 8 up, 8 in 2024-04-12T11:14:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:52 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1159384672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:52 smithi029 ceph-mon[29349]: osdmap e101: 8 total, 8 up, 8 in 2024-04-12T11:14:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:52 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1159384672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:52 smithi029 ceph-mon[39999]: osdmap e101: 8 total, 8 up, 8 in 2024-04-12T11:14:53.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:14:53 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:14:53] "GET /metrics HTTP/1.1" 200 36253 "" "Prometheus/2.43.0" 2024-04-12T11:14:53.543 INFO:tasks.rados.rados.0.smithi029.stdout:162: done (3 left) 2024-04-12T11:14:53.543 INFO:tasks.rados.rados.0.smithi029.stdout:163: rollback oid 15 current snap is 16 2024-04-12T11:14:53.543 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 15 to 0 2024-04-12T11:14:53.543 INFO:tasks.rados.rados.0.smithi029.stdout:164: read oid 21 snap -1 2024-04-12T11:14:53.543 INFO:tasks.rados.rados.0.smithi029.stdout:164: expect (ObjNum 20 snap 0 seq_num 20) 2024-04-12T11:14:53.543 INFO:tasks.rados.rados.0.smithi029.stdout:165: read oid 17 snap 13 2024-04-12T11:14:53.543 INFO:tasks.rados.rados.0.smithi029.stdout:165: expect deleted 2024-04-12T11:14:53.544 INFO:tasks.rados.rados.0.smithi029.stdout:166: read oid 18 snap 13 2024-04-12T11:14:53.544 INFO:tasks.rados.rados.0.smithi029.stdout:166: expect (ObjNum 17 snap 0 seq_num 17) 2024-04-12T11:14:53.544 INFO:tasks.rados.rados.0.smithi029.stdout:167: write oid 24 current snap is 16 2024-04-12T11:14:53.544 INFO:tasks.rados.rados.0.smithi029.stdout:167: seq_num 99 ranges {463893=562240,1593482=658027,2685935=559341} 2024-04-12T11:14:53.548 INFO:tasks.rados.rados.0.smithi029.stdout:167: writing smithi02979277-24 from 463893 to 1026133 tid 1 2024-04-12T11:14:53.556 INFO:tasks.rados.rados.0.smithi029.stdout:167: writing smithi02979277-24 from 1593482 to 2251509 tid 2 2024-04-12T11:14:53.563 INFO:tasks.rados.rados.0.smithi029.stdout:167: writing smithi02979277-24 from 2685935 to 3245276 tid 3 2024-04-12T11:14:53.563 INFO:tasks.rados.rados.0.smithi029.stdout:168: write oid 30 current snap is 16 2024-04-12T11:14:53.563 INFO:tasks.rados.rados.0.smithi029.stdout:158: finishing write tid 3 to smithi02979277-29 2024-04-12T11:14:53.563 INFO:tasks.rados.rados.0.smithi029.stdout:158: finishing write tid 4 to smithi02979277-29 2024-04-12T11:14:53.563 INFO:tasks.rados.rados.0.smithi029.stdout:168: seq_num 100 ranges {480700=637489,1651964=401685,2713358=770613} 2024-04-12T11:14:53.570 INFO:tasks.rados.rados.0.smithi029.stdout:168: writing smithi02979277-30 from 480700 to 1118189 tid 1 2024-04-12T11:14:53.575 INFO:tasks.rados.rados.0.smithi029.stdout:168: writing smithi02979277-30 from 1651964 to 2053649 tid 2 2024-04-12T11:14:53.584 INFO:tasks.rados.rados.0.smithi029.stdout:168: writing smithi02979277-30 from 2713358 to 3483971 tid 3 2024-04-12T11:14:53.584 INFO:tasks.rados.rados.0.smithi029.stdout:169: write oid 26 current snap is 16 2024-04-12T11:14:53.584 INFO:tasks.rados.rados.0.smithi029.stdout:158: finishing write tid 5 to smithi02979277-29 2024-04-12T11:14:53.584 INFO:tasks.rados.rados.0.smithi029.stdout:158: oid 29 updating version 0 to 34 2024-04-12T11:14:53.584 INFO:tasks.rados.rados.0.smithi029.stdout:158: oid 29 updating version 34 to 35 2024-04-12T11:14:53.584 INFO:tasks.rados.rados.0.smithi029.stdout:158: oid 29 version 35 is already newer than 33 2024-04-12T11:14:53.584 INFO:tasks.rados.rados.0.smithi029.stdout:158: oid 29 updating version 35 to 36 2024-04-12T11:14:53.584 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 36 (ObjNum 96 snap 16 seq_num 96) dirty exists 2024-04-12T11:14:53.584 INFO:tasks.rados.rados.0.smithi029.stdout:158: left oid 29 (ObjNum 96 snap 16 seq_num 96) 2024-04-12T11:14:53.584 INFO:tasks.rados.rados.0.smithi029.stdout:159: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:14:53.746 INFO:tasks.rados.rados.0.smithi029.stdout:161: finishing write tid 1 to smithi02979277-44 2024-04-12T11:14:53.746 INFO:tasks.rados.rados.0.smithi029.stdout:161: finishing write tid 2 to smithi02979277-44 2024-04-12T11:14:53.746 INFO:tasks.rados.rados.0.smithi029.stdout:161: finishing write tid 3 to smithi02979277-44 2024-04-12T11:14:53.746 INFO:tasks.rados.rados.0.smithi029.stdout:161: finishing write tid 4 to smithi02979277-44 2024-04-12T11:14:53.746 INFO:tasks.rados.rados.0.smithi029.stdout:161: finishing write tid 5 to smithi02979277-44 2024-04-12T11:14:53.746 INFO:tasks.rados.rados.0.smithi029.stdout:161: oid 44 updating version 0 to 62 2024-04-12T11:14:53.747 INFO:tasks.rados.rados.0.smithi029.stdout:161: oid 44 version 62 is already newer than 61 2024-04-12T11:14:53.747 INFO:tasks.rados.rados.0.smithi029.stdout:161: oid 44 version 62 is already newer than 60 2024-04-12T11:14:53.747 INFO:tasks.rados.rados.0.smithi029.stdout:161: oid 44 version 62 is already newer than 59 2024-04-12T11:14:53.747 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 62 (ObjNum 98 snap 16 seq_num 98) dirty exists 2024-04-12T11:14:53.747 INFO:tasks.rados.rados.0.smithi029.stdout:161: left oid 44 (ObjNum 98 snap 16 seq_num 98) 2024-04-12T11:14:53.747 INFO:tasks.rados.rados.0.smithi029.stdout:163: finishing rollback tid 0 to smithi02979277-15 2024-04-12T11:14:53.747 INFO:tasks.rados.rados.0.smithi029.stdout:163: finishing rollback tid 1 to smithi02979277-15 2024-04-12T11:14:53.747 INFO:tasks.rados.rados.0.smithi029.stdout:163: finishing rollback tid 2 to smithi02979277-15 2024-04-12T11:14:53.747 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 57 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-04-12T11:14:53.747 INFO:tasks.rados.rados.0.smithi029.stdout:169: seq_num 101 ranges {497507=712738,1710446=661696,3140781=556726} 2024-04-12T11:14:53.752 INFO:tasks.rados.rados.0.smithi029.stdout:169: writing smithi02979277-26 from 497507 to 1210245 tid 1 2024-04-12T11:14:53.758 INFO:tasks.rados.rados.0.smithi029.stdout:169: writing smithi02979277-26 from 1710446 to 2372142 tid 2 2024-04-12T11:14:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:169: writing smithi02979277-26 from 3140781 to 3697507 tid 3 2024-04-12T11:14:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:158: done (9 left) 2024-04-12T11:14:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:159: done (8 left) 2024-04-12T11:14:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:161: done (7 left) 2024-04-12T11:14:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:163: done (6 left) 2024-04-12T11:14:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:170: write oid 40 current snap is 16 2024-04-12T11:14:53.767 INFO:tasks.rados.rados.0.smithi029.stdout:164: expect (ObjNum 20 snap 0 seq_num 20) 2024-04-12T11:14:53.847 INFO:tasks.rados.rados.0.smithi029.stdout:167: finishing write tid 1 to smithi02979277-24 2024-04-12T11:14:53.847 INFO:tasks.rados.rados.0.smithi029.stdout:166: expect (ObjNum 17 snap 0 seq_num 17) 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:167: finishing write tid 2 to smithi02979277-24 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:167: finishing write tid 3 to smithi02979277-24 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:167: finishing write tid 4 to smithi02979277-24 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:167: finishing write tid 5 to smithi02979277-24 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:167: oid 24 updating version 0 to 40 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:167: oid 24 updating version 40 to 41 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:167: oid 24 version 41 is already newer than 38 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:167: oid 24 version 41 is already newer than 39 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 41 (ObjNum 99 snap 16 seq_num 99) dirty exists 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:167: left oid 24 (ObjNum 99 snap 16 seq_num 99) 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:168: finishing write tid 1 to smithi02979277-30 2024-04-12T11:14:53.922 INFO:tasks.rados.rados.0.smithi029.stdout:168: finishing write tid 2 to smithi02979277-30 2024-04-12T11:14:53.923 INFO:tasks.rados.rados.0.smithi029.stdout:170: seq_num 102 ranges {514314=787987,1768928=405354,2768204=509510} 2024-04-12T11:14:53.928 INFO:tasks.rados.rados.0.smithi029.stdout:170: writing smithi02979277-40 from 514314 to 1302301 tid 1 2024-04-12T11:14:53.933 INFO:tasks.rados.rados.0.smithi029.stdout:170: writing smithi02979277-40 from 1768928 to 2174282 tid 2 2024-04-12T11:14:53.940 INFO:tasks.rados.rados.0.smithi029.stdout:170: writing smithi02979277-40 from 2768204 to 3277714 tid 3 2024-04-12T11:14:53.940 INFO:tasks.rados.rados.0.smithi029.stdout:164: done (6 left) 2024-04-12T11:14:53.940 INFO:tasks.rados.rados.0.smithi029.stdout:165: done (5 left) 2024-04-12T11:14:53.940 INFO:tasks.rados.rados.0.smithi029.stdout:166: done (4 left) 2024-04-12T11:14:53.940 INFO:tasks.rados.rados.0.smithi029.stdout:167: done (3 left) 2024-04-12T11:14:53.940 INFO:tasks.rados.rados.0.smithi029.stdout:171: rollback oid 12 current snap is 16 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 15 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:168: finishing write tid 3 to smithi02979277-30 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:168: finishing write tid 4 to smithi02979277-30 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:168: finishing write tid 5 to smithi02979277-30 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:168: oid 30 updating version 0 to 59 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:168: oid 30 updating version 59 to 60 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:168: oid 30 updating version 60 to 62 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:168: oid 30 version 62 is already newer than 61 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 62 (ObjNum 100 snap 16 seq_num 100) dirty exists 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:168: left oid 30 (ObjNum 100 snap 16 seq_num 100) 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:169: finishing write tid 1 to smithi02979277-26 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:169: finishing write tid 2 to smithi02979277-26 2024-04-12T11:14:53.941 INFO:tasks.rados.rados.0.smithi029.stdout:168: done (3 left) 2024-04-12T11:14:53.942 INFO:tasks.rados.rados.0.smithi029.stdout:172: rollback oid 44 current snap is 16 2024-04-12T11:14:53.942 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 44 to 14 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:169: finishing write tid 3 to smithi02979277-26 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:169: finishing write tid 4 to smithi02979277-26 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:169: finishing write tid 5 to smithi02979277-26 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:169: oid 26 updating version 0 to 46 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:169: oid 26 version 46 is already newer than 43 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:169: oid 26 version 46 is already newer than 44 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:169: oid 26 version 46 is already newer than 45 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 46 (ObjNum 101 snap 16 seq_num 101) dirty exists 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:169: left oid 26 (ObjNum 101 snap 16 seq_num 101) 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:170: finishing write tid 1 to smithi02979277-40 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:170: finishing write tid 2 to smithi02979277-40 2024-04-12T11:14:53.943 INFO:tasks.rados.rados.0.smithi029.stdout:169: done (3 left) 2024-04-12T11:14:53.944 INFO:tasks.rados.rados.0.smithi029.stdout:173: write oid 18 current snap is 16 2024-04-12T11:14:53.944 INFO:tasks.rados.rados.0.smithi029.stdout:173: seq_num 103 ranges {531121=463236,1711057=665365,3079274=651847} 2024-04-12T11:14:53.945 INFO:tasks.rados.rados.0.smithi029.stdout:173: writing smithi02979277-18 from 531121 to 994357 tid 1 2024-04-12T11:14:53.952 INFO:tasks.rados.rados.0.smithi029.stdout:173: writing smithi02979277-18 from 1711057 to 2376422 tid 2 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:173: writing smithi02979277-18 from 3079274 to 3731121 tid 3 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:174: read oid 8 snap -1 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:174: expect deleted 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:175: read oid 50 snap -1 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:175: expect (ObjNum 94 snap 16 seq_num 94) 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:176: rollback oid 39 current snap is 16 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 15 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:172: finishing rollback tid 0 to smithi02979277-44 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:171: finishing rollback tid 0 to smithi02979277-12 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:172: finishing rollback tid 1 to smithi02979277-44 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:177: write oid 11 current snap is 16 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:170: finishing write tid 3 to smithi02979277-40 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:171: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:14:53.961 INFO:tasks.rados.rados.0.smithi029.stdout:177: seq_num 104 ranges {547928=538485,1769539=409023,2706697=532054} 2024-04-12T11:14:53.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:53 smithi087 ceph-mon[32917]: pgmap v127: 89 pgs: 89 active+clean; 150 MiB data, 485 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 778 KiB/s wr, 6 op/s 2024-04-12T11:14:53.966 INFO:tasks.rados.rados.0.smithi029.stdout:177: writing smithi02979277-11 from 547928 to 1086413 tid 1 2024-04-12T11:14:53.972 INFO:tasks.rados.rados.0.smithi029.stdout:177: writing smithi02979277-11 from 1769539 to 2178562 tid 2 2024-04-12T11:14:53.980 INFO:tasks.rados.rados.0.smithi029.stdout:177: writing smithi02979277-11 from 2706697 to 3238751 tid 3 2024-04-12T11:14:53.980 INFO:tasks.rados.rados.0.smithi029.stdout:178: read oid 13 snap 11 2024-04-12T11:14:53.980 INFO:tasks.rados.rados.0.smithi029.stdout:178: expect (ObjNum 77 snap 9 seq_num 77) 2024-04-12T11:14:53.980 INFO:tasks.rados.rados.0.smithi029.stdout:179: rollback oid 30 current snap is 16 2024-04-12T11:14:53.980 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 14 2024-04-12T11:14:53.980 INFO:tasks.rados.rados.0.smithi029.stdout:170: finishing write tid 4 to smithi02979277-40 2024-04-12T11:14:53.980 INFO:tasks.rados.rados.0.smithi029.stdout:171: finishing rollback tid 2 to smithi02979277-12 2024-04-12T11:14:53.980 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 64 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-04-12T11:14:53.980 INFO:tasks.rados.rados.0.smithi029.stdout:172: finishing rollback tid 2 to smithi02979277-44 2024-04-12T11:14:53.980 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 67 (ObjNum 85 snap 10 seq_num 85) dirty exists 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:171: done (9 left) 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:172: done (8 left) 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:180: rollback oid 48 current snap is 16 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 11 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:170: finishing write tid 5 to smithi02979277-40 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:170: oid 40 updating version 0 to 51 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:170: oid 40 version 51 is already newer than 50 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:170: oid 40 version 51 is already newer than 49 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:170: oid 40 version 51 is already newer than 48 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 51 (ObjNum 102 snap 16 seq_num 102) dirty exists 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:170: left oid 40 (ObjNum 102 snap 16 seq_num 102) 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:173: finishing write tid 1 to smithi02979277-18 2024-04-12T11:14:53.981 INFO:tasks.rados.rados.0.smithi029.stdout:173: finishing write tid 2 to smithi02979277-18 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:176: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 5 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:173: finishing write tid 3 to smithi02979277-18 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:173: finishing write tid 4 to smithi02979277-18 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:173: finishing write tid 5 to smithi02979277-18 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:173: oid 18 updating version 0 to 53 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:173: oid 18 updating version 53 to 54 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:173: oid 18 updating version 54 to 55 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:173: oid 18 version 55 is already newer than 52 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 55 (ObjNum 103 snap 16 seq_num 103) dirty exists 2024-04-12T11:14:53.982 INFO:tasks.rados.rados.0.smithi029.stdout:173: left oid 18 (ObjNum 103 snap 16 seq_num 103) 2024-04-12T11:14:53.983 INFO:tasks.rados.rados.0.smithi029.stdout:177: finishing write tid 1 to smithi02979277-11 2024-04-12T11:14:53.983 INFO:tasks.rados.rados.0.smithi029.stdout:175: expect (ObjNum 94 snap 16 seq_num 94) 2024-04-12T11:14:54.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:53 smithi029 ceph-mon[39999]: pgmap v127: 89 pgs: 89 active+clean; 150 MiB data, 485 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 778 KiB/s wr, 6 op/s 2024-04-12T11:14:54.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:53 smithi029 ceph-mon[29349]: pgmap v127: 89 pgs: 89 active+clean; 150 MiB data, 485 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 778 KiB/s wr, 6 op/s 2024-04-12T11:14:54.188 INFO:tasks.rados.rados.0.smithi029.stdout:177: finishing write tid 2 to smithi02979277-11 2024-04-12T11:14:54.188 INFO:tasks.rados.rados.0.smithi029.stdout:170: done (8 left) 2024-04-12T11:14:54.188 INFO:tasks.rados.rados.0.smithi029.stdout:173: done (7 left) 2024-04-12T11:14:54.188 INFO:tasks.rados.rados.0.smithi029.stdout:174: done (6 left) 2024-04-12T11:14:54.188 INFO:tasks.rados.rados.0.smithi029.stdout:175: done (5 left) 2024-04-12T11:14:54.188 INFO:tasks.rados.rados.0.smithi029.stdout:176: done (4 left) 2024-04-12T11:14:54.188 INFO:tasks.rados.rados.0.smithi029.stdout:181: read oid 37 snap 14 2024-04-12T11:14:54.188 INFO:tasks.rados.rados.0.smithi029.stdout:181: expect (ObjNum 36 snap 0 seq_num 36) 2024-04-12T11:14:54.188 INFO:tasks.rados.rados.0.smithi029.stdout:182: read oid 34 snap -1 2024-04-12T11:14:54.188 INFO:tasks.rados.rados.0.smithi029.stdout:182: expect (ObjNum 33 snap 0 seq_num 33) 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:183: write oid 17 current snap is 16 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:177: finishing write tid 3 to smithi02979277-11 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:177: finishing write tid 4 to smithi02979277-11 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:180: finishing rollback tid 0 to smithi02979277-48 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:180: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:177: finishing write tid 5 to smithi02979277-11 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:177: oid 11 updating version 0 to 56 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:177: oid 11 version 56 is already newer than 54 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:177: oid 11 version 56 is already newer than 55 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:177: oid 11 version 56 is already newer than 53 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 56 (ObjNum 104 snap 16 seq_num 104) dirty exists 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:177: left oid 11 (ObjNum 104 snap 16 seq_num 104) 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:180: finishing rollback tid 2 to smithi02979277-48 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 56 (ObjNum 66 snap 5 seq_num 66) dirty exists 2024-04-12T11:14:54.189 INFO:tasks.rados.rados.0.smithi029.stdout:179: finishing rollback tid 0 to smithi02979277-30 2024-04-12T11:14:54.190 INFO:tasks.rados.rados.0.smithi029.stdout:179: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:14:54.190 INFO:tasks.rados.rados.0.smithi029.stdout:183: seq_num 105 ranges {564735=613734,1828021=669034,3134120=630615} 2024-04-12T11:14:54.194 INFO:tasks.rados.rados.0.smithi029.stdout:183: writing smithi02979277-17 from 564735 to 1178469 tid 1 2024-04-12T11:14:54.201 INFO:tasks.rados.rados.0.smithi029.stdout:183: writing smithi02979277-17 from 1828021 to 2497055 tid 2 2024-04-12T11:14:54.208 INFO:tasks.rados.rados.0.smithi029.stdout:183: writing smithi02979277-17 from 3134120 to 3764735 tid 3 2024-04-12T11:14:54.208 INFO:tasks.rados.rados.0.smithi029.stdout:177: done (6 left) 2024-04-12T11:14:54.208 INFO:tasks.rados.rados.0.smithi029.stdout:180: done (5 left) 2024-04-12T11:14:54.208 INFO:tasks.rados.rados.0.smithi029.stdout:184: write oid 27 current snap is 16 2024-04-12T11:14:54.209 INFO:tasks.rados.rados.0.smithi029.stdout:179: finishing rollback tid 2 to smithi02979277-30 2024-04-12T11:14:54.209 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 65 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-04-12T11:14:54.209 INFO:tasks.rados.rados.0.smithi029.stdout:178: expect (ObjNum 77 snap 9 seq_num 77) 2024-04-12T11:14:54.334 INFO:tasks.rados.rados.0.smithi029.stdout:182: expect (ObjNum 33 snap 0 seq_num 33) 2024-04-12T11:14:54.421 INFO:tasks.rados.rados.0.smithi029.stdout:181: expect (ObjNum 36 snap 0 seq_num 36) 2024-04-12T11:14:54.504 INFO:tasks.rados.rados.0.smithi029.stdout:183: finishing write tid 1 to smithi02979277-17 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:183: finishing write tid 2 to smithi02979277-17 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:183: finishing write tid 3 to smithi02979277-17 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:183: finishing write tid 4 to smithi02979277-17 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:183: finishing write tid 5 to smithi02979277-17 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:183: oid 17 updating version 0 to 59 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:183: oid 17 version 59 is already newer than 58 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:183: oid 17 version 59 is already newer than 57 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:183: oid 17 version 59 is already newer than 56 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 59 (ObjNum 105 snap 16 seq_num 105) dirty exists 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:183: left oid 17 (ObjNum 105 snap 16 seq_num 105) 2024-04-12T11:14:54.505 INFO:tasks.rados.rados.0.smithi029.stdout:184: seq_num 106 ranges {581542=688983,1886503=412692,2761543=670951} 2024-04-12T11:14:54.511 INFO:tasks.rados.rados.0.smithi029.stdout:184: writing smithi02979277-27 from 581542 to 1270525 tid 1 2024-04-12T11:14:54.517 INFO:tasks.rados.rados.0.smithi029.stdout:184: writing smithi02979277-27 from 1886503 to 2299195 tid 2 2024-04-12T11:14:54.523 INFO:tasks.rados.rados.0.smithi029.stdout:184: writing smithi02979277-27 from 2761543 to 3432494 tid 3 2024-04-12T11:14:54.524 INFO:tasks.rados.rados.0.smithi029.stdout:178: done (5 left) 2024-04-12T11:14:54.524 INFO:tasks.rados.rados.0.smithi029.stdout:179: done (4 left) 2024-04-12T11:14:54.524 INFO:tasks.rados.rados.0.smithi029.stdout:181: done (3 left) 2024-04-12T11:14:54.524 INFO:tasks.rados.rados.0.smithi029.stdout:182: done (2 left) 2024-04-12T11:14:54.524 INFO:tasks.rados.rados.0.smithi029.stdout:183: done (1 left) 2024-04-12T11:14:54.524 INFO:tasks.rados.rados.0.smithi029.stdout:185: delete oid 2 current snap is 16 2024-04-12T11:14:54.524 INFO:tasks.rados.rados.0.smithi029.stdout:184: finishing write tid 1 to smithi02979277-27 2024-04-12T11:14:54.524 INFO:tasks.rados.rados.0.smithi029.stdout:184: finishing write tid 2 to smithi02979277-27 2024-04-12T11:14:54.525 INFO:tasks.rados.rados.0.smithi029.stdout:185: done (1 left) 2024-04-12T11:14:54.525 INFO:tasks.rados.rados.0.smithi029.stdout:186: delete oid 35 current snap is 16 2024-04-12T11:14:54.526 INFO:tasks.rados.rados.0.smithi029.stdout:184: finishing write tid 3 to smithi02979277-27 2024-04-12T11:14:54.526 INFO:tasks.rados.rados.0.smithi029.stdout:184: finishing write tid 4 to smithi02979277-27 2024-04-12T11:14:54.526 INFO:tasks.rados.rados.0.smithi029.stdout:184: finishing write tid 5 to smithi02979277-27 2024-04-12T11:14:54.526 INFO:tasks.rados.rados.0.smithi029.stdout:184: oid 27 updating version 0 to 70 2024-04-12T11:14:54.526 INFO:tasks.rados.rados.0.smithi029.stdout:184: oid 27 version 70 is already newer than 67 2024-04-12T11:14:54.526 INFO:tasks.rados.rados.0.smithi029.stdout:184: oid 27 version 70 is already newer than 69 2024-04-12T11:14:54.526 INFO:tasks.rados.rados.0.smithi029.stdout:184: oid 27 version 70 is already newer than 68 2024-04-12T11:14:54.526 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 70 (ObjNum 106 snap 16 seq_num 106) dirty exists 2024-04-12T11:14:54.526 INFO:tasks.rados.rados.0.smithi029.stdout:184: left oid 27 (ObjNum 106 snap 16 seq_num 106) 2024-04-12T11:14:54.527 INFO:tasks.rados.rados.0.smithi029.stdout:184: done (1 left) 2024-04-12T11:14:54.527 INFO:tasks.rados.rados.0.smithi029.stdout:186: done (0 left) 2024-04-12T11:14:54.527 INFO:tasks.rados.rados.0.smithi029.stdout:187: snap_create 2024-04-12T11:14:54.547 INFO:tasks.rados.rados.0.smithi029.stdout:187: done (0 left) 2024-04-12T11:14:54.547 INFO:tasks.rados.rados.0.smithi029.stdout:188: read oid 42 snap -1 2024-04-12T11:14:54.547 INFO:tasks.rados.rados.0.smithi029.stdout:188: expect (ObjNum 87 snap 11 seq_num 87) 2024-04-12T11:14:54.547 INFO:tasks.rados.rados.0.smithi029.stdout:189: rollback oid 30 current snap is 17 2024-04-12T11:14:54.547 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 14 2024-04-12T11:14:54.547 INFO:tasks.rados.rados.0.smithi029.stdout:190: delete oid 20 current snap is 17 2024-04-12T11:14:54.549 INFO:tasks.rados.rados.0.smithi029.stdout:190: done (2 left) 2024-04-12T11:14:54.549 INFO:tasks.rados.rados.0.smithi029.stdout:191: snap_remove snap 0 2024-04-12T11:14:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:54 smithi087 ceph-mon[32917]: osdmap e102: 8 total, 8 up, 8 in 2024-04-12T11:14:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:54 smithi029 ceph-mon[29349]: osdmap e102: 8 total, 8 up, 8 in 2024-04-12T11:14:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:54 smithi029 ceph-mon[39999]: osdmap e102: 8 total, 8 up, 8 in 2024-04-12T11:14:55.548 INFO:tasks.rados.rados.0.smithi029.stdout:191: done (2 left) 2024-04-12T11:14:55.549 INFO:tasks.rados.rados.0.smithi029.stdout:192: snap_remove snap 15 2024-04-12T11:14:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:55 smithi087 ceph-mon[32917]: pgmap v130: 89 pgs: 89 active+clean; 174 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-04-12T11:14:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:55 smithi087 ceph-mon[32917]: osdmap e103: 8 total, 8 up, 8 in 2024-04-12T11:14:55.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:55 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2313121969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:55 smithi029 ceph-mon[39999]: pgmap v130: 89 pgs: 89 active+clean; 174 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-04-12T11:14:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:55 smithi029 ceph-mon[39999]: osdmap e103: 8 total, 8 up, 8 in 2024-04-12T11:14:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:55 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2313121969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:55 smithi029 ceph-mon[29349]: pgmap v130: 89 pgs: 89 active+clean; 174 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-04-12T11:14:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:55 smithi029 ceph-mon[29349]: osdmap e103: 8 total, 8 up, 8 in 2024-04-12T11:14:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:55 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2313121969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:56.519 INFO:tasks.rados.rados.0.smithi029.stdout:192: done (2 left) 2024-04-12T11:14:56.519 INFO:tasks.rados.rados.0.smithi029.stdout:193: read oid 11 snap 11 2024-04-12T11:14:56.519 INFO:tasks.rados.rados.0.smithi029.stdout:193: expect (ObjNum 10 snap 0 seq_num 10) 2024-04-12T11:14:56.519 INFO:tasks.rados.rados.0.smithi029.stdout:194: snap_create 2024-04-12T11:14:56.519 INFO:tasks.rados.rados.0.smithi029.stdout:189: finishing rollback tid 0 to smithi02979277-30 2024-04-12T11:14:56.519 INFO:tasks.rados.rados.0.smithi029.stdout:189: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:14:56.520 INFO:tasks.rados.rados.0.smithi029.stdout:189: finishing rollback tid 2 to smithi02979277-30 2024-04-12T11:14:56.520 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 74 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-04-12T11:14:56.520 INFO:tasks.rados.rados.0.smithi029.stdout:188: expect (ObjNum 87 snap 11 seq_num 87) 2024-04-12T11:14:56.682 INFO:tasks.rados.rados.0.smithi029.stdout:193: expect (ObjNum 10 snap 0 seq_num 10) 2024-04-12T11:14:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:56 smithi087 ceph-mon[32917]: osdmap e104: 8 total, 8 up, 8 in 2024-04-12T11:14:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:56 smithi087 ceph-mon[32917]: osdmap e105: 8 total, 8 up, 8 in 2024-04-12T11:14:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:56 smithi029 ceph-mon[39999]: osdmap e104: 8 total, 8 up, 8 in 2024-04-12T11:14:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:56 smithi029 ceph-mon[39999]: osdmap e105: 8 total, 8 up, 8 in 2024-04-12T11:14:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:56 smithi029 ceph-mon[29349]: osdmap e104: 8 total, 8 up, 8 in 2024-04-12T11:14:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:56 smithi029 ceph-mon[29349]: osdmap e105: 8 total, 8 up, 8 in 2024-04-12T11:14:57.518 INFO:tasks.rados.rados.0.smithi029.stdout:188: done (3 left) 2024-04-12T11:14:57.519 INFO:tasks.rados.rados.0.smithi029.stdout:189: done (2 left) 2024-04-12T11:14:57.519 INFO:tasks.rados.rados.0.smithi029.stdout:193: done (1 left) 2024-04-12T11:14:57.519 INFO:tasks.rados.rados.0.smithi029.stdout:194: done (0 left) 2024-04-12T11:14:57.519 INFO:tasks.rados.rados.0.smithi029.stdout:195: delete oid 45 current snap is 18 2024-04-12T11:14:57.522 INFO:tasks.rados.rados.0.smithi029.stdout:195: done (0 left) 2024-04-12T11:14:57.522 INFO:tasks.rados.rados.0.smithi029.stdout:196: read oid 12 snap 13 2024-04-12T11:14:57.522 INFO:tasks.rados.rados.0.smithi029.stdout:196: expect (ObjNum 68 snap 7 seq_num 68) 2024-04-12T11:14:57.522 INFO:tasks.rados.rados.0.smithi029.stdout:197: rollback oid 44 current snap is 18 2024-04-12T11:14:57.522 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 44 to 17 2024-04-12T11:14:57.522 INFO:tasks.rados.rados.0.smithi029.stdout:198: snap_remove snap 14 2024-04-12T11:14:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:57 smithi087 ceph-mon[32917]: pgmap v133: 89 pgs: 89 active+clean; 174 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-04-12T11:14:57.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:57 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2521299143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:57.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:57 smithi087 ceph-mon[32917]: osdmap e106: 8 total, 8 up, 8 in 2024-04-12T11:14:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:57 smithi029 ceph-mon[29349]: pgmap v133: 89 pgs: 89 active+clean; 174 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-04-12T11:14:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:57 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2521299143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:57 smithi029 ceph-mon[29349]: osdmap e106: 8 total, 8 up, 8 in 2024-04-12T11:14:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:57 smithi029 ceph-mon[39999]: pgmap v133: 89 pgs: 89 active+clean; 174 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-04-12T11:14:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:57 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2521299143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:14:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:57 smithi029 ceph-mon[39999]: osdmap e106: 8 total, 8 up, 8 in 2024-04-12T11:14:58.526 INFO:tasks.rados.rados.0.smithi029.stdout:198: done (2 left) 2024-04-12T11:14:58.526 INFO:tasks.rados.rados.0.smithi029.stdout:199: delete oid 24 current snap is 18 2024-04-12T11:14:58.526 INFO:tasks.rados.rados.0.smithi029.stdout:197: finishing rollback tid 0 to smithi02979277-44 2024-04-12T11:14:58.526 INFO:tasks.rados.rados.0.smithi029.stdout:197: finishing rollback tid 1 to smithi02979277-44 2024-04-12T11:14:58.526 INFO:tasks.rados.rados.0.smithi029.stdout:197: finishing rollback tid 2 to smithi02979277-44 2024-04-12T11:14:58.526 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 79 (ObjNum 85 snap 10 seq_num 85) dirty exists 2024-04-12T11:14:58.526 INFO:tasks.rados.rados.0.smithi029.stdout:196: expect (ObjNum 68 snap 7 seq_num 68) 2024-04-12T11:14:58.742 INFO:tasks.rados.rados.0.smithi029.stdout:196: done (2 left) 2024-04-12T11:14:58.742 INFO:tasks.rados.rados.0.smithi029.stdout:197: done (1 left) 2024-04-12T11:14:58.742 INFO:tasks.rados.rados.0.smithi029.stdout:199: done (0 left) 2024-04-12T11:14:58.742 INFO:tasks.rados.rados.0.smithi029.stdout:200: read oid 9 snap 13 2024-04-12T11:14:58.742 INFO:tasks.rados.rados.0.smithi029.stdout:200: expect (ObjNum 8 snap 0 seq_num 8) 2024-04-12T11:14:58.742 INFO:tasks.rados.rados.0.smithi029.stdout:201: rollback oid 45 current snap is 18 2024-04-12T11:14:58.743 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 45 to 16 2024-04-12T11:14:58.743 INFO:tasks.rados.rados.0.smithi029.stdout:202: rollback oid 14 current snap is 18 2024-04-12T11:14:58.743 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 13 2024-04-12T11:14:58.743 INFO:tasks.rados.rados.0.smithi029.stdout:203: write oid 34 current snap is 18 2024-04-12T11:14:58.743 INFO:tasks.rados.rados.0.smithi029.stdout:203: seq_num 112 ranges {682384=456830,1721042=423699,2809728=454936,3731357=151027} 2024-04-12T11:14:58.748 INFO:tasks.rados.rados.0.smithi029.stdout:203: writing smithi02979277-34 from 682384 to 1139214 tid 1 2024-04-12T11:14:58.753 INFO:tasks.rados.rados.0.smithi029.stdout:203: writing smithi02979277-34 from 1721042 to 2144741 tid 2 2024-04-12T11:14:58.760 INFO:tasks.rados.rados.0.smithi029.stdout:203: writing smithi02979277-34 from 2809728 to 3264664 tid 3 2024-04-12T11:14:58.764 INFO:tasks.rados.rados.0.smithi029.stdout:203: writing smithi02979277-34 from 3731357 to 3882384 tid 4 2024-04-12T11:14:58.764 INFO:tasks.rados.rados.0.smithi029.stdout:201: finishing rollback tid 1 to smithi02979277-45 2024-04-12T11:14:58.764 INFO:tasks.rados.rados.0.smithi029.stdout:201: finishing rollback tid 2 to smithi02979277-45 2024-04-12T11:14:58.764 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 80 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-04-12T11:14:58.764 INFO:tasks.rados.rados.0.smithi029.stdout:200: expect (ObjNum 8 snap 0 seq_num 8) 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:202: finishing rollback tid 0 to smithi02979277-14 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:202: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:200: done (3 left) 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:201: done (2 left) 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:204: snap_create 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:202: finishing rollback tid 2 to smithi02979277-14 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 75 (ObjNum 75 snap 9 seq_num 75) dirty exists 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:203: finishing write tid 1 to smithi02979277-34 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:203: finishing write tid 2 to smithi02979277-34 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:203: finishing write tid 3 to smithi02979277-34 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:203: finishing write tid 4 to smithi02979277-34 2024-04-12T11:14:58.845 INFO:tasks.rados.rados.0.smithi029.stdout:203: finishing write tid 5 to smithi02979277-34 2024-04-12T11:14:58.846 INFO:tasks.rados.rados.0.smithi029.stdout:203: finishing write tid 6 to smithi02979277-34 2024-04-12T11:14:58.846 INFO:tasks.rados.rados.0.smithi029.stdout:203: oid 34 updating version 0 to 52 2024-04-12T11:14:58.846 INFO:tasks.rados.rados.0.smithi029.stdout:203: oid 34 version 52 is already newer than 50 2024-04-12T11:14:58.846 INFO:tasks.rados.rados.0.smithi029.stdout:203: oid 34 version 52 is already newer than 48 2024-04-12T11:14:58.846 INFO:tasks.rados.rados.0.smithi029.stdout:203: oid 34 version 52 is already newer than 51 2024-04-12T11:14:58.846 INFO:tasks.rados.rados.0.smithi029.stdout:203: oid 34 version 52 is already newer than 49 2024-04-12T11:14:58.846 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 52 (ObjNum 112 snap 18 seq_num 112) dirty exists 2024-04-12T11:14:58.846 INFO:tasks.rados.rados.0.smithi029.stdout:203: left oid 34 (ObjNum 112 snap 18 seq_num 112) 2024-04-12T11:14:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:58 smithi087 ceph-mon[32917]: osdmap e107: 8 total, 8 up, 8 in 2024-04-12T11:14:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:58 smithi029 ceph-mon[39999]: osdmap e107: 8 total, 8 up, 8 in 2024-04-12T11:14:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:14:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:58 smithi029 ceph-mon[29349]: osdmap e107: 8 total, 8 up, 8 in 2024-04-12T11:14:59.568 INFO:tasks.rados.rados.0.smithi029.stdout:202: done (2 left) 2024-04-12T11:14:59.569 INFO:tasks.rados.rados.0.smithi029.stdout:203: done (1 left) 2024-04-12T11:14:59.569 INFO:tasks.rados.rados.0.smithi029.stdout:204: done (0 left) 2024-04-12T11:14:59.569 INFO:tasks.rados.rados.0.smithi029.stdout:205: write oid 49 current snap is 19 2024-04-12T11:14:59.569 INFO:tasks.rados.rados.0.smithi029.stdout:205: seq_num 113 ranges {699191=532079,1779524=567357,2837151=666208} 2024-04-12T11:14:59.575 INFO:tasks.rados.rados.0.smithi029.stdout:205: writing smithi02979277-49 from 699191 to 1231270 tid 1 2024-04-12T11:14:59.581 INFO:tasks.rados.rados.0.smithi029.stdout:205: writing smithi02979277-49 from 1779524 to 2346881 tid 2 2024-04-12T11:14:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:205: writing smithi02979277-49 from 2837151 to 3503359 tid 3 2024-04-12T11:14:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:206: snap_remove snap 18 2024-04-12T11:14:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:59 smithi087 ceph-mon[32917]: pgmap v136: 89 pgs: 89 active+clean; 174 MiB data, 523 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:14:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:14:59 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/667875669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:00.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:59 smithi029 ceph-mon[29349]: pgmap v136: 89 pgs: 89 active+clean; 174 MiB data, 523 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:00.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:14:59 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/667875669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:59 smithi029 ceph-mon[39999]: pgmap v136: 89 pgs: 89 active+clean; 174 MiB data, 523 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:14:59 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/667875669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:00.574 INFO:tasks.rados.rados.0.smithi029.stdout:206: done (1 left) 2024-04-12T11:15:00.574 INFO:tasks.rados.rados.0.smithi029.stdout:207: rollback oid 12 current snap is 19 2024-04-12T11:15:00.574 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 11 2024-04-12T11:15:00.574 INFO:tasks.rados.rados.0.smithi029.stdout:208: snap_remove snap 13 2024-04-12T11:15:00.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:00 smithi087 ceph-mon[32917]: osdmap e108: 8 total, 8 up, 8 in 2024-04-12T11:15:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:00 smithi029 ceph-mon[39999]: osdmap e108: 8 total, 8 up, 8 in 2024-04-12T11:15:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:00 smithi029 ceph-mon[29349]: osdmap e108: 8 total, 8 up, 8 in 2024-04-12T11:15:01.519 INFO:tasks.rados.rados.0.smithi029.stdout:208: done (2 left) 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:209: delete oid 18 current snap is 19 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:205: finishing write tid 1 to smithi02979277-49 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:205: finishing write tid 2 to smithi02979277-49 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:205: finishing write tid 3 to smithi02979277-49 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:205: finishing write tid 4 to smithi02979277-49 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:205: finishing write tid 5 to smithi02979277-49 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:205: oid 49 updating version 0 to 55 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:205: oid 49 updating version 55 to 57 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:205: oid 49 version 57 is already newer than 54 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:205: oid 49 version 57 is already newer than 56 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 57 (ObjNum 113 snap 19 seq_num 113) dirty exists 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:205: left oid 49 (ObjNum 113 snap 19 seq_num 113) 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:207: finishing rollback tid 0 to smithi02979277-12 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:207: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:15:01.520 INFO:tasks.rados.rados.0.smithi029.stdout:207: finishing rollback tid 2 to smithi02979277-12 2024-04-12T11:15:01.521 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 84 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-04-12T11:15:01.527 INFO:tasks.rados.rados.0.smithi029.stdout:205: done (2 left) 2024-04-12T11:15:01.527 INFO:tasks.rados.rados.0.smithi029.stdout:207: done (1 left) 2024-04-12T11:15:01.527 INFO:tasks.rados.rados.0.smithi029.stdout:209: done (0 left) 2024-04-12T11:15:01.528 INFO:tasks.rados.rados.0.smithi029.stdout:210: read oid 19 snap -1 2024-04-12T11:15:01.528 INFO:tasks.rados.rados.0.smithi029.stdout:210: expect (ObjNum 51 snap 0 seq_num 51) 2024-04-12T11:15:01.528 INFO:tasks.rados.rados.0.smithi029.stdout:211: snap_remove snap 11 2024-04-12T11:15:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:01 smithi087 ceph-mon[32917]: pgmap v139: 89 pgs: 89 active+clean; 149 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-04-12T11:15:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:01 smithi087 ceph-mon[32917]: osdmap e109: 8 total, 8 up, 8 in 2024-04-12T11:15:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:01 smithi087 ceph-mon[32917]: osdmap e110: 8 total, 8 up, 8 in 2024-04-12T11:15:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:01 smithi029 ceph-mon[39999]: pgmap v139: 89 pgs: 89 active+clean; 149 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-04-12T11:15:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:01 smithi029 ceph-mon[39999]: osdmap e109: 8 total, 8 up, 8 in 2024-04-12T11:15:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:01 smithi029 ceph-mon[39999]: osdmap e110: 8 total, 8 up, 8 in 2024-04-12T11:15:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:01 smithi029 ceph-mon[29349]: pgmap v139: 89 pgs: 89 active+clean; 149 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-04-12T11:15:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:01 smithi029 ceph-mon[29349]: osdmap e109: 8 total, 8 up, 8 in 2024-04-12T11:15:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:01 smithi029 ceph-mon[29349]: osdmap e110: 8 total, 8 up, 8 in 2024-04-12T11:15:02.521 INFO:tasks.rados.rados.0.smithi029.stdout:211: done (1 left) 2024-04-12T11:15:02.521 INFO:tasks.rados.rados.0.smithi029.stdout:212: delete oid 2 current snap is 19 2024-04-12T11:15:02.522 INFO:tasks.rados.rados.0.smithi029.stdout:210: expect (ObjNum 51 snap 0 seq_num 51) 2024-04-12T11:15:02.664 INFO:tasks.rados.rados.0.smithi029.stdout:210: done (1 left) 2024-04-12T11:15:02.664 INFO:tasks.rados.rados.0.smithi029.stdout:212: done (0 left) 2024-04-12T11:15:02.664 INFO:tasks.rados.rados.0.smithi029.stdout:213: snap_create 2024-04-12T11:15:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:02 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3430280580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:02 smithi087 ceph-mon[32917]: osdmap e111: 8 total, 8 up, 8 in 2024-04-12T11:15:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:02 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3430280580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:02 smithi029 ceph-mon[39999]: osdmap e111: 8 total, 8 up, 8 in 2024-04-12T11:15:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:02 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3430280580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:02 smithi029 ceph-mon[29349]: osdmap e111: 8 total, 8 up, 8 in 2024-04-12T11:15:03.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:15:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:15:03] "GET /metrics HTTP/1.1" 200 36258 "" "Prometheus/2.43.0" 2024-04-12T11:15:03.581 INFO:tasks.rados.rados.0.smithi029.stdout:213: done (0 left) 2024-04-12T11:15:03.581 INFO:tasks.rados.rados.0.smithi029.stdout:214: delete oid 46 current snap is 20 2024-04-12T11:15:03.585 INFO:tasks.rados.rados.0.smithi029.stdout:214: done (0 left) 2024-04-12T11:15:03.585 INFO:tasks.rados.rados.0.smithi029.stdout:215: write oid 16 current snap is 20 2024-04-12T11:15:03.585 INFO:tasks.rados.rados.0.smithi029.stdout:215: seq_num 117 ranges {766419=549428,2013452=574695,3346843=427649} 2024-04-12T11:15:03.592 INFO:tasks.rados.rados.0.smithi029.stdout:215: writing smithi02979277-16 from 766419 to 1315847 tid 1 2024-04-12T11:15:03.599 INFO:tasks.rados.rados.0.smithi029.stdout:215: writing smithi02979277-16 from 2013452 to 2588147 tid 2 2024-04-12T11:15:03.606 INFO:tasks.rados.rados.0.smithi029.stdout:215: writing smithi02979277-16 from 3346843 to 3774492 tid 3 2024-04-12T11:15:03.606 INFO:tasks.rados.rados.0.smithi029.stdout:216: read oid 30 snap -1 2024-04-12T11:15:03.606 INFO:tasks.rados.rados.0.smithi029.stdout:216: expect (ObjNum 29 snap 0 seq_num 29) 2024-04-12T11:15:03.606 INFO:tasks.rados.rados.0.smithi029.stdout:217: snap_remove snap 16 2024-04-12T11:15:03.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:03 smithi087 ceph-mon[32917]: pgmap v142: 89 pgs: 89 active+clean; 149 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-04-12T11:15:04.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:03 smithi029 ceph-mon[39999]: pgmap v142: 89 pgs: 89 active+clean; 149 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-04-12T11:15:04.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:03 smithi029 ceph-mon[29349]: pgmap v142: 89 pgs: 89 active+clean; 149 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-04-12T11:15:04.591 INFO:tasks.rados.rados.0.smithi029.stdout:217: done (2 left) 2024-04-12T11:15:04.591 INFO:tasks.rados.rados.0.smithi029.stdout:218: snap_remove snap 17 2024-04-12T11:15:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:04 smithi029 ceph-mon[29349]: osdmap e112: 8 total, 8 up, 8 in 2024-04-12T11:15:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:04 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1121993324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:04 smithi029 ceph-mon[39999]: osdmap e112: 8 total, 8 up, 8 in 2024-04-12T11:15:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:04 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1121993324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:04 smithi087 ceph-mon[32917]: osdmap e112: 8 total, 8 up, 8 in 2024-04-12T11:15:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:04 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1121993324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:05.593 INFO:tasks.rados.rados.0.smithi029.stdout:218: done (2 left) 2024-04-12T11:15:05.593 INFO:tasks.rados.rados.0.smithi029.stdout:219: write oid 18 current snap is 20 2024-04-12T11:15:05.593 INFO:tasks.rados.rados.0.smithi029.stdout:215: finishing write tid 1 to smithi02979277-16 2024-04-12T11:15:05.594 INFO:tasks.rados.rados.0.smithi029.stdout:219: seq_num 118 ranges {783226=624677,2071934=434706,2974266=638921} 2024-04-12T11:15:05.601 INFO:tasks.rados.rados.0.smithi029.stdout:219: writing smithi02979277-18 from 783226 to 1407903 tid 1 2024-04-12T11:15:05.608 INFO:tasks.rados.rados.0.smithi029.stdout:219: writing smithi02979277-18 from 2071934 to 2506640 tid 2 2024-04-12T11:15:05.614 INFO:tasks.rados.rados.0.smithi029.stdout:219: writing smithi02979277-18 from 2974266 to 3613187 tid 3 2024-04-12T11:15:05.614 INFO:tasks.rados.rados.0.smithi029.stdout:220: rollback oid 7 current snap is 20 2024-04-12T11:15:05.614 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 19 2024-04-12T11:15:05.614 INFO:tasks.rados.rados.0.smithi029.stdout:215: finishing write tid 2 to smithi02979277-16 2024-04-12T11:15:05.614 INFO:tasks.rados.rados.0.smithi029.stdout:216: expect (ObjNum 29 snap 0 seq_num 29) 2024-04-12T11:15:05.707 INFO:tasks.rados.rados.0.smithi029.stdout:215: finishing write tid 3 to smithi02979277-16 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:215: finishing write tid 4 to smithi02979277-16 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:215: finishing write tid 5 to smithi02979277-16 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:215: oid 16 updating version 0 to 87 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:215: oid 16 version 87 is already newer than 84 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:215: oid 16 version 87 is already newer than 86 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:215: oid 16 version 87 is already newer than 85 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 87 (ObjNum 117 snap 20 seq_num 117) dirty exists 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:215: left oid 16 (ObjNum 117 snap 20 seq_num 117) 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:219: finishing write tid 1 to smithi02979277-18 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:219: finishing write tid 2 to smithi02979277-18 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:215: done (3 left) 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:216: done (2 left) 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:221: snap_create 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:220: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:15:05.708 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 0 (ObjNum 129 snap 0 seq_num 861950551) dirty dne 2024-04-12T11:15:05.709 INFO:tasks.rados.rados.0.smithi029.stdout:219: finishing write tid 3 to smithi02979277-18 2024-04-12T11:15:05.709 INFO:tasks.rados.rados.0.smithi029.stdout:219: finishing write tid 4 to smithi02979277-18 2024-04-12T11:15:05.709 INFO:tasks.rados.rados.0.smithi029.stdout:219: finishing write tid 5 to smithi02979277-18 2024-04-12T11:15:05.709 INFO:tasks.rados.rados.0.smithi029.stdout:219: oid 18 updating version 0 to 81 2024-04-12T11:15:05.709 INFO:tasks.rados.rados.0.smithi029.stdout:219: oid 18 updating version 81 to 82 2024-04-12T11:15:05.709 INFO:tasks.rados.rados.0.smithi029.stdout:219: oid 18 version 82 is already newer than 78 2024-04-12T11:15:05.709 INFO:tasks.rados.rados.0.smithi029.stdout:219: oid 18 updating version 82 to 83 2024-04-12T11:15:05.709 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 83 (ObjNum 118 snap 20 seq_num 118) dirty exists 2024-04-12T11:15:05.709 INFO:tasks.rados.rados.0.smithi029.stdout:219: left oid 18 (ObjNum 118 snap 20 seq_num 118) 2024-04-12T11:15:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:05 smithi087 ceph-mon[32917]: pgmap v145: 89 pgs: 89 active+clean; 131 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 811 KiB/s wr, 8 op/s 2024-04-12T11:15:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:05 smithi087 ceph-mon[32917]: osdmap e113: 8 total, 8 up, 8 in 2024-04-12T11:15:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:05 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:05 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:15:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:05 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:05 smithi029 ceph-mon[29349]: pgmap v145: 89 pgs: 89 active+clean; 131 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 811 KiB/s wr, 8 op/s 2024-04-12T11:15:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:05 smithi029 ceph-mon[29349]: osdmap e113: 8 total, 8 up, 8 in 2024-04-12T11:15:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:05 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:05 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:15:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:05 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:05 smithi029 ceph-mon[39999]: pgmap v145: 89 pgs: 89 active+clean; 131 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 811 KiB/s wr, 8 op/s 2024-04-12T11:15:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:05 smithi029 ceph-mon[39999]: osdmap e113: 8 total, 8 up, 8 in 2024-04-12T11:15:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:05 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:05 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:15:06.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:05 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:219: done (2 left) 2024-04-12T11:15:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:220: done (1 left) 2024-04-12T11:15:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:221: done (0 left) 2024-04-12T11:15:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:222: delete oid 28 current snap is 21 2024-04-12T11:15:06.521 INFO:tasks.rados.rados.0.smithi029.stdout:222: done (0 left) 2024-04-12T11:15:06.521 INFO:tasks.rados.rados.0.smithi029.stdout:223: snap_create 2024-04-12T11:15:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:06 smithi087 ceph-mon[32917]: Upgrade: Target is version 19.0.0-2874-g2cee1857 (unknown) 2024-04-12T11:15:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:06 smithi087 ceph-mon[32917]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d'] 2024-04-12T11:15:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:06 smithi087 ceph-mon[32917]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-12T11:15:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:06 smithi087 ceph-mon[32917]: osdmap e114: 8 total, 8 up, 8 in 2024-04-12T11:15:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:06 smithi087 ceph-mon[32917]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d on smithi087 2024-04-12T11:15:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:06 smithi087 ceph-mon[32917]: osdmap e115: 8 total, 8 up, 8 in 2024-04-12T11:15:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[39999]: Upgrade: Target is version 19.0.0-2874-g2cee1857 (unknown) 2024-04-12T11:15:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[39999]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d'] 2024-04-12T11:15:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[39999]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-12T11:15:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[39999]: osdmap e114: 8 total, 8 up, 8 in 2024-04-12T11:15:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[39999]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d on smithi087 2024-04-12T11:15:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[39999]: osdmap e115: 8 total, 8 up, 8 in 2024-04-12T11:15:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[29349]: Upgrade: Target is version 19.0.0-2874-g2cee1857 (unknown) 2024-04-12T11:15:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[29349]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d'] 2024-04-12T11:15:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[29349]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-12T11:15:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[29349]: osdmap e114: 8 total, 8 up, 8 in 2024-04-12T11:15:07.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[29349]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d on smithi087 2024-04-12T11:15:07.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:06 smithi029 ceph-mon[29349]: osdmap e115: 8 total, 8 up, 8 in 2024-04-12T11:15:07.522 INFO:tasks.rados.rados.0.smithi029.stdout:223: done (0 left) 2024-04-12T11:15:07.522 INFO:tasks.rados.rados.0.smithi029.stdout:224: read oid 49 snap -1 2024-04-12T11:15:07.522 INFO:tasks.rados.rados.0.smithi029.stdout:224: expect (ObjNum 113 snap 19 seq_num 113) 2024-04-12T11:15:07.522 INFO:tasks.rados.rados.0.smithi029.stdout:225: read oid 23 snap 21 2024-04-12T11:15:07.522 INFO:tasks.rados.rados.0.smithi029.stdout:225: expect deleted 2024-04-12T11:15:07.523 INFO:tasks.rados.rados.0.smithi029.stdout:226: write oid 7 current snap is 22 2024-04-12T11:15:07.523 INFO:tasks.rados.rados.0.smithi029.stdout:226: seq_num 120 ranges {416840=775175,1672545=344295} 2024-04-12T11:15:07.528 INFO:tasks.rados.rados.0.smithi029.stdout:226: writing smithi02979277-7 from 416840 to 1192015 tid 1 2024-04-12T11:15:07.533 INFO:tasks.rados.rados.0.smithi029.stdout:226: writing smithi02979277-7 from 1672545 to 2016840 tid 2 2024-04-12T11:15:07.533 INFO:tasks.rados.rados.0.smithi029.stdout:227: snap_create 2024-04-12T11:15:07.533 INFO:tasks.rados.rados.0.smithi029.stdout:224: expect (ObjNum 113 snap 19 seq_num 113) 2024-04-12T11:15:07.716 INFO:tasks.rados.rados.0.smithi029.stdout:226: finishing write tid 1 to smithi02979277-7 2024-04-12T11:15:07.716 INFO:tasks.rados.rados.0.smithi029.stdout:226: finishing write tid 2 to smithi02979277-7 2024-04-12T11:15:07.716 INFO:tasks.rados.rados.0.smithi029.stdout:226: finishing write tid 3 to smithi02979277-7 2024-04-12T11:15:07.716 INFO:tasks.rados.rados.0.smithi029.stdout:226: finishing write tid 4 to smithi02979277-7 2024-04-12T11:15:07.716 INFO:tasks.rados.rados.0.smithi029.stdout:226: oid 7 updating version 0 to 101 2024-04-12T11:15:07.716 INFO:tasks.rados.rados.0.smithi029.stdout:226: oid 7 updating version 101 to 102 2024-04-12T11:15:07.716 INFO:tasks.rados.rados.0.smithi029.stdout:226: oid 7 updating version 102 to 103 2024-04-12T11:15:07.716 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 103 (ObjNum 120 snap 22 seq_num 120) dirty exists 2024-04-12T11:15:07.716 INFO:tasks.rados.rados.0.smithi029.stdout:226: left oid 7 (ObjNum 120 snap 22 seq_num 120) 2024-04-12T11:15:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:07 smithi087 ceph-mon[32917]: pgmap v148: 89 pgs: 89 active+clean; 131 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 811 KiB/s wr, 8 op/s 2024-04-12T11:15:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:07 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2221093016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:07 smithi087 ceph-mon[32917]: osdmap e116: 8 total, 8 up, 8 in 2024-04-12T11:15:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:07 smithi029 ceph-mon[39999]: pgmap v148: 89 pgs: 89 active+clean; 131 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 811 KiB/s wr, 8 op/s 2024-04-12T11:15:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:07 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2221093016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:07 smithi029 ceph-mon[39999]: osdmap e116: 8 total, 8 up, 8 in 2024-04-12T11:15:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:07 smithi029 ceph-mon[29349]: pgmap v148: 89 pgs: 89 active+clean; 131 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 811 KiB/s wr, 8 op/s 2024-04-12T11:15:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:07 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2221093016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:07 smithi029 ceph-mon[29349]: osdmap e116: 8 total, 8 up, 8 in 2024-04-12T11:15:08.523 INFO:tasks.rados.rados.0.smithi029.stdout:224: done (3 left) 2024-04-12T11:15:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:225: done (2 left) 2024-04-12T11:15:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:226: done (1 left) 2024-04-12T11:15:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:227: done (0 left) 2024-04-12T11:15:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:228: rollback oid 47 current snap is 23 2024-04-12T11:15:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 19 2024-04-12T11:15:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:229: delete oid 28 current snap is 23 2024-04-12T11:15:08.526 INFO:tasks.rados.rados.0.smithi029.stdout:229: done (1 left) 2024-04-12T11:15:08.526 INFO:tasks.rados.rados.0.smithi029.stdout:230: snap_remove snap 21 2024-04-12T11:15:09.527 INFO:tasks.rados.rados.0.smithi029.stdout:230: done (1 left) 2024-04-12T11:15:09.527 INFO:tasks.rados.rados.0.smithi029.stdout:231: write oid 28 current snap is 23 2024-04-12T11:15:09.527 INFO:tasks.rados.rados.0.smithi029.stdout:231: seq_num 122 ranges {450454=642026,1505862=544592} 2024-04-12T11:15:09.532 INFO:tasks.rados.rados.0.smithi029.stdout:231: writing smithi02979277-28 from 450454 to 1092480 tid 1 2024-04-12T11:15:09.538 INFO:tasks.rados.rados.0.smithi029.stdout:231: writing smithi02979277-28 from 1505862 to 2050454 tid 2 2024-04-12T11:15:09.538 INFO:tasks.rados.rados.0.smithi029.stdout:232: read oid 40 snap -1 2024-04-12T11:15:09.538 INFO:tasks.rados.rados.0.smithi029.stdout:232: expect (ObjNum 102 snap 16 seq_num 102) 2024-04-12T11:15:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:233: read oid 46 snap -1 2024-04-12T11:15:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:233: expect deleted 2024-04-12T11:15:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:234: read oid 15 snap -1 2024-04-12T11:15:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:234: expect (ObjNum 14 snap 0 seq_num 14) 2024-04-12T11:15:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:235: delete oid 27 current snap is 23 2024-04-12T11:15:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:228: finishing rollback tid 0 to smithi02979277-47 2024-04-12T11:15:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:228: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:15:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:228: finishing rollback tid 2 to smithi02979277-47 2024-04-12T11:15:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 107 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-04-12T11:15:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:231: finishing write tid 1 to smithi02979277-28 2024-04-12T11:15:09.542 INFO:tasks.rados.rados.0.smithi029.stdout:231: finishing write tid 2 to smithi02979277-28 2024-04-12T11:15:09.542 INFO:tasks.rados.rados.0.smithi029.stdout:231: finishing write tid 3 to smithi02979277-28 2024-04-12T11:15:09.542 INFO:tasks.rados.rados.0.smithi029.stdout:231: finishing write tid 4 to smithi02979277-28 2024-04-12T11:15:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:231: oid 28 updating version 0 to 80 2024-04-12T11:15:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:231: oid 28 updating version 80 to 81 2024-04-12T11:15:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:231: oid 28 version 81 is already newer than 79 2024-04-12T11:15:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 81 (ObjNum 122 snap 23 seq_num 122) dirty exists 2024-04-12T11:15:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:231: left oid 28 (ObjNum 122 snap 23 seq_num 122) 2024-04-12T11:15:09.549 INFO:tasks.rados.rados.0.smithi029.stdout:232: expect (ObjNum 102 snap 16 seq_num 102) 2024-04-12T11:15:09.719 INFO:tasks.rados.rados.0.smithi029.stdout:234: expect (ObjNum 14 snap 0 seq_num 14) 2024-04-12T11:15:09.795 INFO:tasks.rados.rados.0.smithi029.stdout:228: done (5 left) 2024-04-12T11:15:09.795 INFO:tasks.rados.rados.0.smithi029.stdout:231: done (4 left) 2024-04-12T11:15:09.795 INFO:tasks.rados.rados.0.smithi029.stdout:232: done (3 left) 2024-04-12T11:15:09.795 INFO:tasks.rados.rados.0.smithi029.stdout:233: done (2 left) 2024-04-12T11:15:09.795 INFO:tasks.rados.rados.0.smithi029.stdout:234: done (1 left) 2024-04-12T11:15:09.795 INFO:tasks.rados.rados.0.smithi029.stdout:235: done (0 left) 2024-04-12T11:15:09.795 INFO:tasks.rados.rados.0.smithi029.stdout:236: write oid 29 current snap is 23 2024-04-12T11:15:09.795 INFO:tasks.rados.rados.0.smithi029.stdout:236: seq_num 124 ranges {484068=792524,1906473=177595} 2024-04-12T11:15:09.801 INFO:tasks.rados.rados.0.smithi029.stdout:236: writing smithi02979277-29 from 484068 to 1276592 tid 1 2024-04-12T11:15:09.806 INFO:tasks.rados.rados.0.smithi029.stdout:236: writing smithi02979277-29 from 1906473 to 2084068 tid 2 2024-04-12T11:15:09.806 INFO:tasks.rados.rados.0.smithi029.stdout:237: write oid 1 current snap is 23 2024-04-12T11:15:09.806 INFO:tasks.rados.rados.0.smithi029.stdout:237: seq_num 125 ranges {500875=467773,1564955=535920} 2024-04-12T11:15:09.811 INFO:tasks.rados.rados.0.smithi029.stdout:237: writing smithi02979277-1 from 500875 to 968648 tid 1 2024-04-12T11:15:09.817 INFO:tasks.rados.rados.0.smithi029.stdout:237: writing smithi02979277-1 from 1564955 to 2100875 tid 2 2024-04-12T11:15:09.818 INFO:tasks.rados.rados.0.smithi029.stdout:238: snap_create 2024-04-12T11:15:09.818 INFO:tasks.rados.rados.0.smithi029.stdout:236: finishing write tid 1 to smithi02979277-29 2024-04-12T11:15:09.818 INFO:tasks.rados.rados.0.smithi029.stdout:236: finishing write tid 2 to smithi02979277-29 2024-04-12T11:15:09.818 INFO:tasks.rados.rados.0.smithi029.stdout:236: finishing write tid 3 to smithi02979277-29 2024-04-12T11:15:09.818 INFO:tasks.rados.rados.0.smithi029.stdout:236: finishing write tid 4 to smithi02979277-29 2024-04-12T11:15:09.818 INFO:tasks.rados.rados.0.smithi029.stdout:236: oid 29 updating version 0 to 66 2024-04-12T11:15:09.818 INFO:tasks.rados.rados.0.smithi029.stdout:236: oid 29 updating version 66 to 67 2024-04-12T11:15:09.818 INFO:tasks.rados.rados.0.smithi029.stdout:236: oid 29 updating version 67 to 68 2024-04-12T11:15:09.818 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 68 (ObjNum 124 snap 23 seq_num 124) dirty exists 2024-04-12T11:15:09.818 INFO:tasks.rados.rados.0.smithi029.stdout:236: left oid 29 (ObjNum 124 snap 23 seq_num 124) 2024-04-12T11:15:09.819 INFO:tasks.rados.rados.0.smithi029.stdout:237: finishing write tid 1 to smithi02979277-1 2024-04-12T11:15:09.820 INFO:tasks.rados.rados.0.smithi029.stdout:237: finishing write tid 2 to smithi02979277-1 2024-04-12T11:15:09.820 INFO:tasks.rados.rados.0.smithi029.stdout:237: finishing write tid 3 to smithi02979277-1 2024-04-12T11:15:09.820 INFO:tasks.rados.rados.0.smithi029.stdout:237: finishing write tid 4 to smithi02979277-1 2024-04-12T11:15:09.820 INFO:tasks.rados.rados.0.smithi029.stdout:237: oid 1 updating version 0 to 93 2024-04-12T11:15:09.820 INFO:tasks.rados.rados.0.smithi029.stdout:237: oid 1 version 93 is already newer than 91 2024-04-12T11:15:09.820 INFO:tasks.rados.rados.0.smithi029.stdout:237: oid 1 version 93 is already newer than 92 2024-04-12T11:15:09.820 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 93 (ObjNum 125 snap 23 seq_num 125) dirty exists 2024-04-12T11:15:09.821 INFO:tasks.rados.rados.0.smithi029.stdout:237: left oid 1 (ObjNum 125 snap 23 seq_num 125) 2024-04-12T11:15:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:09 smithi087 ceph-mon[32917]: pgmap v151: 89 pgs: 89 active+clean; 131 MiB data, 448 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:09 smithi087 ceph-mon[32917]: osdmap e117: 8 total, 8 up, 8 in 2024-04-12T11:15:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:09 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1949150163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:09 smithi029 ceph-mon[29349]: pgmap v151: 89 pgs: 89 active+clean; 131 MiB data, 448 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:09 smithi029 ceph-mon[29349]: osdmap e117: 8 total, 8 up, 8 in 2024-04-12T11:15:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:09 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1949150163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:10.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:09 smithi029 ceph-mon[39999]: pgmap v151: 89 pgs: 89 active+clean; 131 MiB data, 448 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:10.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:09 smithi029 ceph-mon[39999]: osdmap e117: 8 total, 8 up, 8 in 2024-04-12T11:15:10.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:09 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1949150163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:236: done (2 left) 2024-04-12T11:15:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:237: done (1 left) 2024-04-12T11:15:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:238: done (0 left) 2024-04-12T11:15:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:239: read oid 44 snap -1 2024-04-12T11:15:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:239: expect (ObjNum 85 snap 10 seq_num 85) 2024-04-12T11:15:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:240: write oid 22 current snap is 24 2024-04-12T11:15:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:240: seq_num 126 ranges {517682=543022,1623437=494245} 2024-04-12T11:15:10.538 INFO:tasks.rados.rados.0.smithi029.stdout:240: writing smithi02979277-22 from 517682 to 1060704 tid 1 2024-04-12T11:15:10.544 INFO:tasks.rados.rados.0.smithi029.stdout:240: writing smithi02979277-22 from 1623437 to 2117682 tid 2 2024-04-12T11:15:10.544 INFO:tasks.rados.rados.0.smithi029.stdout:241: snap_create 2024-04-12T11:15:10.544 INFO:tasks.rados.rados.0.smithi029.stdout:239: expect (ObjNum 85 snap 10 seq_num 85) 2024-04-12T11:15:10.709 INFO:tasks.rados.rados.0.smithi029.stdout:240: finishing write tid 1 to smithi02979277-22 2024-04-12T11:15:10.709 INFO:tasks.rados.rados.0.smithi029.stdout:240: finishing write tid 2 to smithi02979277-22 2024-04-12T11:15:10.709 INFO:tasks.rados.rados.0.smithi029.stdout:240: finishing write tid 3 to smithi02979277-22 2024-04-12T11:15:10.709 INFO:tasks.rados.rados.0.smithi029.stdout:240: finishing write tid 4 to smithi02979277-22 2024-04-12T11:15:10.709 INFO:tasks.rados.rados.0.smithi029.stdout:240: oid 22 updating version 0 to 96 2024-04-12T11:15:10.709 INFO:tasks.rados.rados.0.smithi029.stdout:240: oid 22 updating version 96 to 97 2024-04-12T11:15:10.709 INFO:tasks.rados.rados.0.smithi029.stdout:240: oid 22 updating version 97 to 98 2024-04-12T11:15:10.709 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 98 (ObjNum 126 snap 24 seq_num 126) dirty exists 2024-04-12T11:15:10.710 INFO:tasks.rados.rados.0.smithi029.stdout:240: left oid 22 (ObjNum 126 snap 24 seq_num 126) 2024-04-12T11:15:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:10 smithi087 ceph-mon[32917]: osdmap e118: 8 total, 8 up, 8 in 2024-04-12T11:15:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:10 smithi029 ceph-mon[29349]: osdmap e118: 8 total, 8 up, 8 in 2024-04-12T11:15:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:10 smithi029 ceph-mon[39999]: osdmap e118: 8 total, 8 up, 8 in 2024-04-12T11:15:11.520 INFO:tasks.rados.rados.0.smithi029.stdout:239: done (2 left) 2024-04-12T11:15:11.520 INFO:tasks.rados.rados.0.smithi029.stdout:240: done (1 left) 2024-04-12T11:15:11.521 INFO:tasks.rados.rados.0.smithi029.stdout:241: done (0 left) 2024-04-12T11:15:11.521 INFO:tasks.rados.rados.0.smithi029.stdout:242: rollback oid 28 current snap is 25 2024-04-12T11:15:11.521 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 28 to 24 2024-04-12T11:15:11.521 INFO:tasks.rados.rados.0.smithi029.stdout:243: delete oid 29 current snap is 25 2024-04-12T11:15:11.525 INFO:tasks.rados.rados.0.smithi029.stdout:243: done (1 left) 2024-04-12T11:15:11.525 INFO:tasks.rados.rados.0.smithi029.stdout:244: rollback oid 26 current snap is 25 2024-04-12T11:15:11.525 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 24 2024-04-12T11:15:11.525 INFO:tasks.rados.rados.0.smithi029.stdout:245: rollback oid 16 current snap is 25 2024-04-12T11:15:11.525 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 16 to 20 2024-04-12T11:15:11.525 INFO:tasks.rados.rados.0.smithi029.stdout:246: snap_create 2024-04-12T11:15:11.526 INFO:tasks.rados.rados.0.smithi029.stdout:242: finishing rollback tid 0 to smithi02979277-28 2024-04-12T11:15:11.526 INFO:tasks.rados.rados.0.smithi029.stdout:242: finishing rollback tid 1 to smithi02979277-28 2024-04-12T11:15:11.527 INFO:tasks.rados.rados.0.smithi029.stdout:242: finishing rollback tid 2 to smithi02979277-28 2024-04-12T11:15:11.527 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 85 (ObjNum 122 snap 23 seq_num 122) dirty exists 2024-04-12T11:15:11.527 INFO:tasks.rados.rados.0.smithi029.stdout:244: finishing rollback tid 0 to smithi02979277-26 2024-04-12T11:15:11.528 INFO:tasks.rados.rados.0.smithi029.stdout:244: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:15:11.528 INFO:tasks.rados.rados.0.smithi029.stdout:244: finishing rollback tid 2 to smithi02979277-26 2024-04-12T11:15:11.528 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 120 (ObjNum 101 snap 16 seq_num 101) dirty exists 2024-04-12T11:15:11.529 INFO:tasks.rados.rados.0.smithi029.stdout:245: finishing rollback tid 0 to smithi02979277-16 2024-04-12T11:15:11.529 INFO:tasks.rados.rados.0.smithi029.stdout:245: finishing rollback tid 1 to smithi02979277-16 2024-04-12T11:15:11.529 INFO:tasks.rados.rados.0.smithi029.stdout:245: finishing rollback tid 2 to smithi02979277-16 2024-04-12T11:15:11.529 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 95 (ObjNum 117 snap 20 seq_num 117) dirty exists 2024-04-12T11:15:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:11 smithi087 ceph-mon[32917]: pgmap v154: 89 pgs: 89 active+clean; 120 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 689 KiB/s wr, 5 op/s 2024-04-12T11:15:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:11 smithi087 ceph-mon[32917]: osdmap e119: 8 total, 8 up, 8 in 2024-04-12T11:15:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:11 smithi087 ceph-mon[32917]: osdmap e120: 8 total, 8 up, 8 in 2024-04-12T11:15:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:11 smithi029 ceph-mon[29349]: pgmap v154: 89 pgs: 89 active+clean; 120 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 689 KiB/s wr, 5 op/s 2024-04-12T11:15:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:11 smithi029 ceph-mon[29349]: osdmap e119: 8 total, 8 up, 8 in 2024-04-12T11:15:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:11 smithi029 ceph-mon[29349]: osdmap e120: 8 total, 8 up, 8 in 2024-04-12T11:15:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:11 smithi029 ceph-mon[39999]: pgmap v154: 89 pgs: 89 active+clean; 120 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 689 KiB/s wr, 5 op/s 2024-04-12T11:15:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:11 smithi029 ceph-mon[39999]: osdmap e119: 8 total, 8 up, 8 in 2024-04-12T11:15:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:11 smithi029 ceph-mon[39999]: osdmap e120: 8 total, 8 up, 8 in 2024-04-12T11:15:12.523 INFO:tasks.rados.rados.0.smithi029.stdout:242: done (3 left) 2024-04-12T11:15:12.523 INFO:tasks.rados.rados.0.smithi029.stdout:244: done (2 left) 2024-04-12T11:15:12.523 INFO:tasks.rados.rados.0.smithi029.stdout:245: done (1 left) 2024-04-12T11:15:12.523 INFO:tasks.rados.rados.0.smithi029.stdout:246: done (0 left) 2024-04-12T11:15:12.523 INFO:tasks.rados.rados.0.smithi029.stdout:247: snap_create 2024-04-12T11:15:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:12 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2229261340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:12 smithi087 ceph-mon[32917]: osdmap e121: 8 total, 8 up, 8 in 2024-04-12T11:15:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:12 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2229261340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:12 smithi029 ceph-mon[29349]: osdmap e121: 8 total, 8 up, 8 in 2024-04-12T11:15:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:12 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2229261340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:12 smithi029 ceph-mon[39999]: osdmap e121: 8 total, 8 up, 8 in 2024-04-12T11:15:13.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:15:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:15:13] "GET /metrics HTTP/1.1" 200 36258 "" "Prometheus/2.43.0" 2024-04-12T11:15:13.523 INFO:tasks.rados.rados.0.smithi029.stdout:247: done (0 left) 2024-04-12T11:15:13.524 INFO:tasks.rados.rados.0.smithi029.stdout:248: write oid 48 current snap is 27 2024-04-12T11:15:13.524 INFO:tasks.rados.rados.0.smithi029.stdout:248: seq_num 128 ranges {551296=693520,2024048=127248} 2024-04-12T11:15:13.530 INFO:tasks.rados.rados.0.smithi029.stdout:248: writing smithi02979277-48 from 551296 to 1244816 tid 1 2024-04-12T11:15:13.536 INFO:tasks.rados.rados.0.smithi029.stdout:248: writing smithi02979277-48 from 2024048 to 2151296 tid 2 2024-04-12T11:15:13.536 INFO:tasks.rados.rados.0.smithi029.stdout:249: write oid 44 current snap is 27 2024-04-12T11:15:13.536 INFO:tasks.rados.rados.0.smithi029.stdout:248: finishing write tid 1 to smithi02979277-48 2024-04-12T11:15:13.536 INFO:tasks.rados.rados.0.smithi029.stdout:249: seq_num 129 ranges {568103=768769,2082530=85573} 2024-04-12T11:15:13.543 INFO:tasks.rados.rados.0.smithi029.stdout:249: writing smithi02979277-44 from 568103 to 1336872 tid 1 2024-04-12T11:15:13.547 INFO:tasks.rados.rados.0.smithi029.stdout:249: writing smithi02979277-44 from 2082530 to 2168103 tid 2 2024-04-12T11:15:13.548 INFO:tasks.rados.rados.0.smithi029.stdout:250: snap_remove snap 26 2024-04-12T11:15:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:13 smithi087 ceph-mon[32917]: pgmap v157: 89 pgs: 89 active+clean; 120 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 689 KiB/s wr, 5 op/s 2024-04-12T11:15:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:13 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:13 smithi087 ceph-mon[32917]: osdmap e122: 8 total, 8 up, 8 in 2024-04-12T11:15:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:13 smithi029 ceph-mon[29349]: pgmap v157: 89 pgs: 89 active+clean; 120 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 689 KiB/s wr, 5 op/s 2024-04-12T11:15:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:13 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:13 smithi029 ceph-mon[29349]: osdmap e122: 8 total, 8 up, 8 in 2024-04-12T11:15:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:13 smithi029 ceph-mon[39999]: pgmap v157: 89 pgs: 89 active+clean; 120 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 689 KiB/s wr, 5 op/s 2024-04-12T11:15:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:13 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:13 smithi029 ceph-mon[39999]: osdmap e122: 8 total, 8 up, 8 in 2024-04-12T11:15:14.543 INFO:tasks.rados.rados.0.smithi029.stdout:250: done (2 left) 2024-04-12T11:15:14.545 INFO:tasks.rados.rados.0.smithi029.stdout:251: write oid 8 current snap is 27 2024-04-12T11:15:14.545 INFO:tasks.rados.rados.0.smithi029.stdout:248: finishing write tid 2 to smithi02979277-48 2024-04-12T11:15:14.545 INFO:tasks.rados.rados.0.smithi029.stdout:251: seq_num 130 ranges {584910=560371,1857365=327545} 2024-04-12T11:15:14.549 INFO:tasks.rados.rados.0.smithi029.stdout:251: writing smithi02979277-8 from 584910 to 1145281 tid 1 2024-04-12T11:15:14.555 INFO:tasks.rados.rados.0.smithi029.stdout:251: writing smithi02979277-8 from 1857365 to 2184910 tid 2 2024-04-12T11:15:14.555 INFO:tasks.rados.rados.0.smithi029.stdout:252: snap_create 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:248: finishing write tid 3 to smithi02979277-48 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:248: finishing write tid 4 to smithi02979277-48 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:248: oid 48 updating version 0 to 88 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:248: oid 48 updating version 88 to 89 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:248: oid 48 version 89 is already newer than 87 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 89 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:248: left oid 48 (ObjNum 128 snap 27 seq_num 128) 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:249: finishing write tid 1 to smithi02979277-44 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:249: finishing write tid 2 to smithi02979277-44 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:249: finishing write tid 3 to smithi02979277-44 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:249: finishing write tid 4 to smithi02979277-44 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:249: oid 44 updating version 0 to 95 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:249: oid 44 updating version 95 to 97 2024-04-12T11:15:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:249: oid 44 version 97 is already newer than 96 2024-04-12T11:15:14.557 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 97 (ObjNum 129 snap 27 seq_num 129) dirty exists 2024-04-12T11:15:14.557 INFO:tasks.rados.rados.0.smithi029.stdout:249: left oid 44 (ObjNum 129 snap 27 seq_num 129) 2024-04-12T11:15:14.557 INFO:tasks.rados.rados.0.smithi029.stdout:251: finishing write tid 1 to smithi02979277-8 2024-04-12T11:15:14.557 INFO:tasks.rados.rados.0.smithi029.stdout:251: finishing write tid 2 to smithi02979277-8 2024-04-12T11:15:14.557 INFO:tasks.rados.rados.0.smithi029.stdout:251: finishing write tid 3 to smithi02979277-8 2024-04-12T11:15:14.558 INFO:tasks.rados.rados.0.smithi029.stdout:251: finishing write tid 4 to smithi02979277-8 2024-04-12T11:15:14.558 INFO:tasks.rados.rados.0.smithi029.stdout:251: oid 8 updating version 0 to 14 2024-04-12T11:15:14.558 INFO:tasks.rados.rados.0.smithi029.stdout:251: oid 8 updating version 14 to 16 2024-04-12T11:15:14.558 INFO:tasks.rados.rados.0.smithi029.stdout:251: oid 8 version 16 is already newer than 15 2024-04-12T11:15:14.558 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 16 (ObjNum 130 snap 27 seq_num 130) dirty exists 2024-04-12T11:15:14.558 INFO:tasks.rados.rados.0.smithi029.stdout:251: left oid 8 (ObjNum 130 snap 27 seq_num 130) 2024-04-12T11:15:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:14 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3720877228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:14 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3720877228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:14 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3720877228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:15.545 INFO:tasks.rados.rados.0.smithi029.stdout:248: done (3 left) 2024-04-12T11:15:15.545 INFO:tasks.rados.rados.0.smithi029.stdout:249: done (2 left) 2024-04-12T11:15:15.545 INFO:tasks.rados.rados.0.smithi029.stdout:251: done (1 left) 2024-04-12T11:15:15.545 INFO:tasks.rados.rados.0.smithi029.stdout:252: done (0 left) 2024-04-12T11:15:15.545 INFO:tasks.rados.rados.0.smithi029.stdout:253: snap_remove snap 27 2024-04-12T11:15:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:15 smithi087 ceph-mon[32917]: pgmap v160: 89 pgs: 89 active+clean; 128 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-04-12T11:15:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:15 smithi087 ceph-mon[32917]: osdmap e123: 8 total, 8 up, 8 in 2024-04-12T11:15:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:15 smithi029 ceph-mon[39999]: pgmap v160: 89 pgs: 89 active+clean; 128 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-04-12T11:15:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:15 smithi029 ceph-mon[39999]: osdmap e123: 8 total, 8 up, 8 in 2024-04-12T11:15:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:15 smithi029 ceph-mon[29349]: pgmap v160: 89 pgs: 89 active+clean; 128 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-04-12T11:15:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:15 smithi029 ceph-mon[29349]: osdmap e123: 8 total, 8 up, 8 in 2024-04-12T11:15:16.522 INFO:tasks.rados.rados.0.smithi029.stdout:253: done (0 left) 2024-04-12T11:15:16.522 INFO:tasks.rados.rados.0.smithi029.stdout:254: read oid 29 snap -1 2024-04-12T11:15:16.522 INFO:tasks.rados.rados.0.smithi029.stdout:254: expect deleted 2024-04-12T11:15:16.522 INFO:tasks.rados.rados.0.smithi029.stdout:255: read oid 15 snap -1 2024-04-12T11:15:16.522 INFO:tasks.rados.rados.0.smithi029.stdout:255: expect (ObjNum 14 snap 0 seq_num 14) 2024-04-12T11:15:16.522 INFO:tasks.rados.rados.0.smithi029.stdout:256: delete oid 36 current snap is 28 2024-04-12T11:15:16.526 INFO:tasks.rados.rados.0.smithi029.stdout:255: expect (ObjNum 14 snap 0 seq_num 14) 2024-04-12T11:15:16.602 INFO:tasks.rados.rados.0.smithi029.stdout:254: done (2 left) 2024-04-12T11:15:16.602 INFO:tasks.rados.rados.0.smithi029.stdout:255: done (1 left) 2024-04-12T11:15:16.602 INFO:tasks.rados.rados.0.smithi029.stdout:256: done (0 left) 2024-04-12T11:15:16.602 INFO:tasks.rados.rados.0.smithi029.stdout:257: delete oid 44 current snap is 28 2024-04-12T11:15:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:257: done (0 left) 2024-04-12T11:15:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:258: read oid 29 snap -1 2024-04-12T11:15:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:258: expect deleted 2024-04-12T11:15:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:259: write oid 19 current snap is 28 2024-04-12T11:15:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:259: seq_num 133 ranges {635331=786118,1916458=318873} 2024-04-12T11:15:16.611 INFO:tasks.rados.rados.0.smithi029.stdout:259: writing smithi02979277-19 from 635331 to 1421449 tid 1 2024-04-12T11:15:16.616 INFO:tasks.rados.rados.0.smithi029.stdout:259: writing smithi02979277-19 from 1916458 to 2235331 tid 2 2024-04-12T11:15:16.616 INFO:tasks.rados.rados.0.smithi029.stdout:260: snap_remove snap 19 2024-04-12T11:15:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:16 smithi087 ceph-mon[32917]: osdmap e124: 8 total, 8 up, 8 in 2024-04-12T11:15:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:16 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/541725277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:16 smithi087 ceph-mon[32917]: osdmap e125: 8 total, 8 up, 8 in 2024-04-12T11:15:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:16 smithi029 ceph-mon[29349]: osdmap e124: 8 total, 8 up, 8 in 2024-04-12T11:15:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:16 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/541725277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:16 smithi029 ceph-mon[29349]: osdmap e125: 8 total, 8 up, 8 in 2024-04-12T11:15:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:16 smithi029 ceph-mon[39999]: osdmap e124: 8 total, 8 up, 8 in 2024-04-12T11:15:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:16 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/541725277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:16 smithi029 ceph-mon[39999]: osdmap e125: 8 total, 8 up, 8 in 2024-04-12T11:15:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:260: done (2 left) 2024-04-12T11:15:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:261: write oid 36 current snap is 28 2024-04-12T11:15:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:261: seq_num 134 ranges {652138=461367,1574940=677198} 2024-04-12T11:15:17.567 INFO:tasks.rados.rados.0.smithi029.stdout:261: writing smithi02979277-36 from 652138 to 1113505 tid 1 2024-04-12T11:15:17.573 INFO:tasks.rados.rados.0.smithi029.stdout:261: writing smithi02979277-36 from 1574940 to 2252138 tid 2 2024-04-12T11:15:17.574 INFO:tasks.rados.rados.0.smithi029.stdout:262: read oid 2 snap -1 2024-04-12T11:15:17.574 INFO:tasks.rados.rados.0.smithi029.stdout:262: expect deleted 2024-04-12T11:15:17.574 INFO:tasks.rados.rados.0.smithi029.stdout:263: write oid 26 current snap is 28 2024-04-12T11:15:17.574 INFO:tasks.rados.rados.0.smithi029.stdout:263: seq_num 135 ranges {668945=536616,1633422=491363} 2024-04-12T11:15:17.580 INFO:tasks.rados.rados.0.smithi029.stdout:263: writing smithi02979277-26 from 668945 to 1205561 tid 1 2024-04-12T11:15:17.586 INFO:tasks.rados.rados.0.smithi029.stdout:263: writing smithi02979277-26 from 1633422 to 2124785 tid 2 2024-04-12T11:15:17.586 INFO:tasks.rados.rados.0.smithi029.stdout:264: read oid 37 snap -1 2024-04-12T11:15:17.586 INFO:tasks.rados.rados.0.smithi029.stdout:264: expect (ObjNum 36 snap 0 seq_num 36) 2024-04-12T11:15:17.586 INFO:tasks.rados.rados.0.smithi029.stdout:265: read oid 43 snap -1 2024-04-12T11:15:17.586 INFO:tasks.rados.rados.0.smithi029.stdout:265: expect (ObjNum 42 snap 0 seq_num 42) 2024-04-12T11:15:17.586 INFO:tasks.rados.rados.0.smithi029.stdout:266: write oid 8 current snap is 28 2024-04-12T11:15:17.586 INFO:tasks.rados.rados.0.smithi029.stdout:266: seq_num 136 ranges {685752=611865,1975551=310201} 2024-04-12T11:15:17.594 INFO:tasks.rados.rados.0.smithi029.stdout:266: writing smithi02979277-8 from 685752 to 1297617 tid 1 2024-04-12T11:15:17.600 INFO:tasks.rados.rados.0.smithi029.stdout:266: writing smithi02979277-8 from 1975551 to 2285752 tid 2 2024-04-12T11:15:17.600 INFO:tasks.rados.rados.0.smithi029.stdout:267: rollback oid 13 current snap is 28 2024-04-12T11:15:17.600 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 13 to 24 2024-04-12T11:15:17.600 INFO:tasks.rados.rados.0.smithi029.stdout:259: finishing write tid 1 to smithi02979277-19 2024-04-12T11:15:17.600 INFO:tasks.rados.rados.0.smithi029.stdout:259: finishing write tid 2 to smithi02979277-19 2024-04-12T11:15:17.600 INFO:tasks.rados.rados.0.smithi029.stdout:259: finishing write tid 3 to smithi02979277-19 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:259: finishing write tid 4 to smithi02979277-19 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:259: oid 19 updating version 0 to 98 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:259: oid 19 version 98 is already newer than 97 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:259: oid 19 updating version 98 to 99 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 99 (ObjNum 133 snap 28 seq_num 133) dirty exists 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:259: left oid 19 (ObjNum 133 snap 28 seq_num 133) 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:261: finishing write tid 1 to smithi02979277-36 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:261: finishing write tid 2 to smithi02979277-36 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:261: finishing write tid 3 to smithi02979277-36 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:261: finishing write tid 4 to smithi02979277-36 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:261: oid 36 updating version 0 to 51 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:261: oid 36 updating version 51 to 53 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:261: oid 36 version 53 is already newer than 52 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 53 (ObjNum 134 snap 28 seq_num 134) dirty exists 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:261: left oid 36 (ObjNum 134 snap 28 seq_num 134) 2024-04-12T11:15:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:263: finishing write tid 1 to smithi02979277-26 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:258: done (8 left) 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:259: done (7 left) 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:261: done (6 left) 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:262: done (5 left) 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:268: write oid 1 current snap is 28 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:263: finishing write tid 2 to smithi02979277-26 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:263: finishing write tid 3 to smithi02979277-26 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:263: finishing write tid 4 to smithi02979277-26 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:263: oid 26 updating version 0 to 124 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:263: oid 26 updating version 124 to 126 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:263: oid 26 version 126 is already newer than 125 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 126 (ObjNum 135 snap 28 seq_num 135) dirty exists 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:263: left oid 26 (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:15:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:264: expect (ObjNum 36 snap 0 seq_num 36) 2024-04-12T11:15:17.684 INFO:tasks.rados.rados.0.smithi029.stdout:265: expect (ObjNum 42 snap 0 seq_num 42) 2024-04-12T11:15:17.774 INFO:tasks.rados.rados.0.smithi029.stdout:266: finishing write tid 1 to smithi02979277-8 2024-04-12T11:15:17.774 INFO:tasks.rados.rados.0.smithi029.stdout:266: finishing write tid 2 to smithi02979277-8 2024-04-12T11:15:17.774 INFO:tasks.rados.rados.0.smithi029.stdout:267: finishing rollback tid 0 to smithi02979277-13 2024-04-12T11:15:17.774 INFO:tasks.rados.rados.0.smithi029.stdout:267: finishing rollback tid 1 to smithi02979277-13 2024-04-12T11:15:17.774 INFO:tasks.rados.rados.0.smithi029.stdout:267: finishing rollback tid 2 to smithi02979277-13 2024-04-12T11:15:17.774 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 87 (ObjNum 77 snap 9 seq_num 77) dirty exists 2024-04-12T11:15:17.775 INFO:tasks.rados.rados.0.smithi029.stdout:266: finishing write tid 3 to smithi02979277-8 2024-04-12T11:15:17.775 INFO:tasks.rados.rados.0.smithi029.stdout:266: finishing write tid 4 to smithi02979277-8 2024-04-12T11:15:17.775 INFO:tasks.rados.rados.0.smithi029.stdout:266: oid 8 updating version 0 to 18 2024-04-12T11:15:17.775 INFO:tasks.rados.rados.0.smithi029.stdout:266: oid 8 updating version 18 to 19 2024-04-12T11:15:17.775 INFO:tasks.rados.rados.0.smithi029.stdout:266: oid 8 version 19 is already newer than 17 2024-04-12T11:15:17.775 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 19 (ObjNum 136 snap 28 seq_num 136) dirty exists 2024-04-12T11:15:17.775 INFO:tasks.rados.rados.0.smithi029.stdout:266: left oid 8 (ObjNum 136 snap 28 seq_num 136) 2024-04-12T11:15:17.775 INFO:tasks.rados.rados.0.smithi029.stdout:268: seq_num 137 ranges {702559=403467,1750386=495032} 2024-04-12T11:15:17.780 INFO:tasks.rados.rados.0.smithi029.stdout:268: writing smithi02979277-1 from 702559 to 1106026 tid 1 2024-04-12T11:15:17.787 INFO:tasks.rados.rados.0.smithi029.stdout:268: writing smithi02979277-1 from 1750386 to 2245418 tid 2 2024-04-12T11:15:17.787 INFO:tasks.rados.rados.0.smithi029.stdout:263: done (5 left) 2024-04-12T11:15:17.787 INFO:tasks.rados.rados.0.smithi029.stdout:264: done (4 left) 2024-04-12T11:15:17.787 INFO:tasks.rados.rados.0.smithi029.stdout:265: done (3 left) 2024-04-12T11:15:17.787 INFO:tasks.rados.rados.0.smithi029.stdout:266: done (2 left) 2024-04-12T11:15:17.787 INFO:tasks.rados.rados.0.smithi029.stdout:267: done (1 left) 2024-04-12T11:15:17.787 INFO:tasks.rados.rados.0.smithi029.stdout:269: read oid 32 snap -1 2024-04-12T11:15:17.787 INFO:tasks.rados.rados.0.smithi029.stdout:269: expect (ObjNum 86 snap 11 seq_num 86) 2024-04-12T11:15:17.787 INFO:tasks.rados.rados.0.smithi029.stdout:270: rollback oid 8 current snap is 28 2024-04-12T11:15:17.788 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 24 2024-04-12T11:15:17.788 INFO:tasks.rados.rados.0.smithi029.stdout:268: finishing write tid 1 to smithi02979277-1 2024-04-12T11:15:17.788 INFO:tasks.rados.rados.0.smithi029.stdout:271: snap_create 2024-04-12T11:15:17.789 INFO:tasks.rados.rados.0.smithi029.stdout:270: finishing rollback tid 0 to smithi02979277-8 2024-04-12T11:15:17.789 INFO:tasks.rados.rados.0.smithi029.stdout:270: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:15:17.789 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 21 (ObjNum 289 snap 0 seq_num 862924704) dirty dne 2024-04-12T11:15:17.790 INFO:tasks.rados.rados.0.smithi029.stdout:268: finishing write tid 2 to smithi02979277-1 2024-04-12T11:15:17.790 INFO:tasks.rados.rados.0.smithi029.stdout:268: finishing write tid 3 to smithi02979277-1 2024-04-12T11:15:17.790 INFO:tasks.rados.rados.0.smithi029.stdout:268: finishing write tid 4 to smithi02979277-1 2024-04-12T11:15:17.790 INFO:tasks.rados.rados.0.smithi029.stdout:268: oid 1 updating version 0 to 108 2024-04-12T11:15:17.790 INFO:tasks.rados.rados.0.smithi029.stdout:268: oid 1 version 108 is already newer than 107 2024-04-12T11:15:17.790 INFO:tasks.rados.rados.0.smithi029.stdout:268: oid 1 version 108 is already newer than 106 2024-04-12T11:15:17.790 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 108 (ObjNum 137 snap 28 seq_num 137) dirty exists 2024-04-12T11:15:17.790 INFO:tasks.rados.rados.0.smithi029.stdout:268: left oid 1 (ObjNum 137 snap 28 seq_num 137) 2024-04-12T11:15:17.790 INFO:tasks.rados.rados.0.smithi029.stdout:269: expect (ObjNum 86 snap 11 seq_num 86) 2024-04-12T11:15:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:17 smithi087 ceph-mon[32917]: pgmap v163: 89 pgs: 89 active+clean; 128 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-04-12T11:15:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:17 smithi029 ceph-mon[39999]: pgmap v163: 89 pgs: 89 active+clean; 128 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-04-12T11:15:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:17 smithi029 ceph-mon[29349]: pgmap v163: 89 pgs: 89 active+clean; 128 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-04-12T11:15:18.565 INFO:tasks.rados.rados.0.smithi029.stdout:268: done (3 left) 2024-04-12T11:15:18.565 INFO:tasks.rados.rados.0.smithi029.stdout:269: done (2 left) 2024-04-12T11:15:18.565 INFO:tasks.rados.rados.0.smithi029.stdout:270: done (1 left) 2024-04-12T11:15:18.565 INFO:tasks.rados.rados.0.smithi029.stdout:271: done (0 left) 2024-04-12T11:15:18.565 INFO:tasks.rados.rados.0.smithi029.stdout:272: snap_create 2024-04-12T11:15:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:18 smithi087 ceph-mon[32917]: osdmap e126: 8 total, 8 up, 8 in 2024-04-12T11:15:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:18 smithi029 ceph-mon[29349]: osdmap e126: 8 total, 8 up, 8 in 2024-04-12T11:15:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:18 smithi029 ceph-mon[39999]: osdmap e126: 8 total, 8 up, 8 in 2024-04-12T11:15:19.566 INFO:tasks.rados.rados.0.smithi029.stdout:272: done (0 left) 2024-04-12T11:15:19.566 INFO:tasks.rados.rados.0.smithi029.stdout:273: write oid 31 current snap is 30 2024-04-12T11:15:19.566 INFO:tasks.rados.rados.0.smithi029.stdout:273: seq_num 138 ranges {719366=478716,1808868=510498} 2024-04-12T11:15:19.572 INFO:tasks.rados.rados.0.smithi029.stdout:273: writing smithi02979277-31 from 719366 to 1198082 tid 1 2024-04-12T11:15:19.579 INFO:tasks.rados.rados.0.smithi029.stdout:273: writing smithi02979277-31 from 1808868 to 2319366 tid 2 2024-04-12T11:15:19.579 INFO:tasks.rados.rados.0.smithi029.stdout:274: delete oid 38 current snap is 30 2024-04-12T11:15:19.579 INFO:tasks.rados.rados.0.smithi029.stdout:273: finishing write tid 1 to smithi02979277-31 2024-04-12T11:15:19.581 INFO:tasks.rados.rados.0.smithi029.stdout:273: finishing write tid 2 to smithi02979277-31 2024-04-12T11:15:19.582 INFO:tasks.rados.rados.0.smithi029.stdout:273: finishing write tid 3 to smithi02979277-31 2024-04-12T11:15:19.582 INFO:tasks.rados.rados.0.smithi029.stdout:273: finishing write tid 4 to smithi02979277-31 2024-04-12T11:15:19.582 INFO:tasks.rados.rados.0.smithi029.stdout:273: oid 31 updating version 0 to 96 2024-04-12T11:15:19.582 INFO:tasks.rados.rados.0.smithi029.stdout:273: oid 31 updating version 96 to 97 2024-04-12T11:15:19.582 INFO:tasks.rados.rados.0.smithi029.stdout:273: oid 31 version 97 is already newer than 95 2024-04-12T11:15:19.582 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 97 (ObjNum 138 snap 30 seq_num 138) dirty exists 2024-04-12T11:15:19.582 INFO:tasks.rados.rados.0.smithi029.stdout:273: left oid 31 (ObjNum 138 snap 30 seq_num 138) 2024-04-12T11:15:19.584 INFO:tasks.rados.rados.0.smithi029.stdout:273: done (1 left) 2024-04-12T11:15:19.584 INFO:tasks.rados.rados.0.smithi029.stdout:274: done (0 left) 2024-04-12T11:15:19.584 INFO:tasks.rados.rados.0.smithi029.stdout:275: rollback oid 20 current snap is 30 2024-04-12T11:15:19.584 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 20 to 25 2024-04-12T11:15:19.584 INFO:tasks.rados.rados.0.smithi029.stdout:276: snap_create 2024-04-12T11:15:19.585 INFO:tasks.rados.rados.0.smithi029.stdout:275: finishing rollback tid 1 to smithi02979277-20 2024-04-12T11:15:19.585 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 0 (ObjNum 129 snap 0 seq_num 1801074456) dirty dne 2024-04-12T11:15:19.867 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:19 smithi087 ceph-mon[32917]: pgmap v166: 89 pgs: 89 active+clean; 128 MiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:19.868 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:19 smithi087 ceph-mon[32917]: osdmap e127: 8 total, 8 up, 8 in 2024-04-12T11:15:19.868 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:19 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3907369257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:19 smithi029 ceph-mon[29349]: pgmap v166: 89 pgs: 89 active+clean; 128 MiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:19 smithi029 ceph-mon[29349]: osdmap e127: 8 total, 8 up, 8 in 2024-04-12T11:15:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:19 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3907369257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:19 smithi029 ceph-mon[39999]: pgmap v166: 89 pgs: 89 active+clean; 128 MiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:19 smithi029 ceph-mon[39999]: osdmap e127: 8 total, 8 up, 8 in 2024-04-12T11:15:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:19 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3907369257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:20.574 INFO:tasks.rados.rados.0.smithi029.stdout:275: done (1 left) 2024-04-12T11:15:20.574 INFO:tasks.rados.rados.0.smithi029.stdout:276: done (0 left) 2024-04-12T11:15:20.574 INFO:tasks.rados.rados.0.smithi029.stdout:277: write oid 19 current snap is 31 2024-04-12T11:15:20.574 INFO:tasks.rados.rados.0.smithi029.stdout:277: seq_num 140 ranges {752980=629214,1809479=543501} 2024-04-12T11:15:20.581 INFO:tasks.rados.rados.0.smithi029.stdout:277: writing smithi02979277-19 from 752980 to 1382194 tid 1 2024-04-12T11:15:20.587 INFO:tasks.rados.rados.0.smithi029.stdout:277: writing smithi02979277-19 from 1809479 to 2352980 tid 2 2024-04-12T11:15:20.587 INFO:tasks.rados.rados.0.smithi029.stdout:278: snap_remove snap 29 2024-04-12T11:15:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:20 smithi087 ceph-mon[32917]: osdmap e128: 8 total, 8 up, 8 in 2024-04-12T11:15:21.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:20 smithi029 ceph-mon[39999]: osdmap e128: 8 total, 8 up, 8 in 2024-04-12T11:15:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:20 smithi029 ceph-mon[29349]: osdmap e128: 8 total, 8 up, 8 in 2024-04-12T11:15:21.522 INFO:tasks.rados.rados.0.smithi029.stdout:278: done (1 left) 2024-04-12T11:15:21.523 INFO:tasks.rados.rados.0.smithi029.stdout:279: delete oid 33 current snap is 31 2024-04-12T11:15:21.523 INFO:tasks.rados.rados.0.smithi029.stdout:277: finishing write tid 1 to smithi02979277-19 2024-04-12T11:15:21.523 INFO:tasks.rados.rados.0.smithi029.stdout:277: finishing write tid 2 to smithi02979277-19 2024-04-12T11:15:21.523 INFO:tasks.rados.rados.0.smithi029.stdout:277: finishing write tid 3 to smithi02979277-19 2024-04-12T11:15:21.523 INFO:tasks.rados.rados.0.smithi029.stdout:277: finishing write tid 4 to smithi02979277-19 2024-04-12T11:15:21.523 INFO:tasks.rados.rados.0.smithi029.stdout:277: oid 19 updating version 0 to 103 2024-04-12T11:15:21.523 INFO:tasks.rados.rados.0.smithi029.stdout:277: oid 19 updating version 103 to 105 2024-04-12T11:15:21.523 INFO:tasks.rados.rados.0.smithi029.stdout:277: oid 19 version 105 is already newer than 104 2024-04-12T11:15:21.523 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 105 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-04-12T11:15:21.523 INFO:tasks.rados.rados.0.smithi029.stdout:277: left oid 19 (ObjNum 140 snap 31 seq_num 140) 2024-04-12T11:15:21.530 INFO:tasks.rados.rados.0.smithi029.stdout:277: done (1 left) 2024-04-12T11:15:21.530 INFO:tasks.rados.rados.0.smithi029.stdout:279: done (0 left) 2024-04-12T11:15:21.530 INFO:tasks.rados.rados.0.smithi029.stdout:280: snap_remove snap 24 2024-04-12T11:15:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:21 smithi087 ceph-mon[32917]: pgmap v169: 89 pgs: 89 active+clean; 129 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-04-12T11:15:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:21 smithi087 ceph-mon[32917]: osdmap e129: 8 total, 8 up, 8 in 2024-04-12T11:15:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:21 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/418265946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:21 smithi087 ceph-mon[32917]: osdmap e130: 8 total, 8 up, 8 in 2024-04-12T11:15:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:21 smithi029 ceph-mon[29349]: pgmap v169: 89 pgs: 89 active+clean; 129 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-04-12T11:15:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:21 smithi029 ceph-mon[29349]: osdmap e129: 8 total, 8 up, 8 in 2024-04-12T11:15:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:21 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/418265946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:21 smithi029 ceph-mon[29349]: osdmap e130: 8 total, 8 up, 8 in 2024-04-12T11:15:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:21 smithi029 ceph-mon[39999]: pgmap v169: 89 pgs: 89 active+clean; 129 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-04-12T11:15:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:21 smithi029 ceph-mon[39999]: osdmap e129: 8 total, 8 up, 8 in 2024-04-12T11:15:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:21 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/418265946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:21 smithi029 ceph-mon[39999]: osdmap e130: 8 total, 8 up, 8 in 2024-04-12T11:15:22.525 INFO:tasks.rados.rados.0.smithi029.stdout:280: done (0 left) 2024-04-12T11:15:22.526 INFO:tasks.rados.rados.0.smithi029.stdout:281: rollback oid 49 current snap is 31 2024-04-12T11:15:22.526 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 49 to 22 2024-04-12T11:15:22.526 INFO:tasks.rados.rados.0.smithi029.stdout:282: snap_remove snap 25 2024-04-12T11:15:23.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:15:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:15:23] "GET /metrics HTTP/1.1" 200 36251 "" "Prometheus/2.43.0" 2024-04-12T11:15:23.525 INFO:tasks.rados.rados.0.smithi029.stdout:282: done (1 left) 2024-04-12T11:15:23.526 INFO:tasks.rados.rados.0.smithi029.stdout:283: read oid 9 snap -1 2024-04-12T11:15:23.526 INFO:tasks.rados.rados.0.smithi029.stdout:283: expect (ObjNum 91 snap 15 seq_num 91) 2024-04-12T11:15:23.526 INFO:tasks.rados.rados.0.smithi029.stdout:284: snap_create 2024-04-12T11:15:23.526 INFO:tasks.rados.rados.0.smithi029.stdout:281: finishing rollback tid 0 to smithi02979277-49 2024-04-12T11:15:23.526 INFO:tasks.rados.rados.0.smithi029.stdout:281: finishing rollback tid 1 to smithi02979277-49 2024-04-12T11:15:23.526 INFO:tasks.rados.rados.0.smithi029.stdout:281: finishing rollback tid 2 to smithi02979277-49 2024-04-12T11:15:23.526 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 75 (ObjNum 113 snap 19 seq_num 113) dirty exists 2024-04-12T11:15:23.532 INFO:tasks.rados.rados.0.smithi029.stdout:283: expect (ObjNum 91 snap 15 seq_num 91) 2024-04-12T11:15:23.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:23 smithi087 ceph-mon[32917]: pgmap v172: 89 pgs: 89 active+clean; 129 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-04-12T11:15:23.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:23 smithi087 ceph-mon[32917]: osdmap e131: 8 total, 8 up, 8 in 2024-04-12T11:15:24.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:23 smithi029 ceph-mon[29349]: pgmap v172: 89 pgs: 89 active+clean; 129 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-04-12T11:15:24.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:23 smithi029 ceph-mon[29349]: osdmap e131: 8 total, 8 up, 8 in 2024-04-12T11:15:24.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:23 smithi029 ceph-mon[39999]: pgmap v172: 89 pgs: 89 active+clean; 129 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-04-12T11:15:24.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:23 smithi029 ceph-mon[39999]: osdmap e131: 8 total, 8 up, 8 in 2024-04-12T11:15:24.527 INFO:tasks.rados.rados.0.smithi029.stdout:281: done (2 left) 2024-04-12T11:15:24.527 INFO:tasks.rados.rados.0.smithi029.stdout:283: done (1 left) 2024-04-12T11:15:24.527 INFO:tasks.rados.rados.0.smithi029.stdout:284: done (0 left) 2024-04-12T11:15:24.528 INFO:tasks.rados.rados.0.smithi029.stdout:285: snap_create 2024-04-12T11:15:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:24 smithi087 ceph-mon[32917]: osdmap e132: 8 total, 8 up, 8 in 2024-04-12T11:15:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:24 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/394900283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:24 smithi087 ceph-mon[32917]: osdmap e133: 8 total, 8 up, 8 in 2024-04-12T11:15:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:24 smithi029 ceph-mon[29349]: osdmap e132: 8 total, 8 up, 8 in 2024-04-12T11:15:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:24 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/394900283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:24 smithi029 ceph-mon[29349]: osdmap e133: 8 total, 8 up, 8 in 2024-04-12T11:15:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:24 smithi029 ceph-mon[39999]: osdmap e132: 8 total, 8 up, 8 in 2024-04-12T11:15:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:24 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/394900283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:24 smithi029 ceph-mon[39999]: osdmap e133: 8 total, 8 up, 8 in 2024-04-12T11:15:25.529 INFO:tasks.rados.rados.0.smithi029.stdout:285: done (0 left) 2024-04-12T11:15:25.529 INFO:tasks.rados.rados.0.smithi029.stdout:286: snap_create 2024-04-12T11:15:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:25 smithi087 ceph-mon[32917]: pgmap v175: 89 pgs: 89 active+clean; 131 MiB data, 473 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 528 KiB/s wr, 6 op/s 2024-04-12T11:15:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:25 smithi087 ceph-mon[32917]: osdmap e134: 8 total, 8 up, 8 in 2024-04-12T11:15:26.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:25 smithi029 ceph-mon[29349]: pgmap v175: 89 pgs: 89 active+clean; 131 MiB data, 473 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 528 KiB/s wr, 6 op/s 2024-04-12T11:15:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:25 smithi029 ceph-mon[29349]: osdmap e134: 8 total, 8 up, 8 in 2024-04-12T11:15:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:25 smithi029 ceph-mon[39999]: pgmap v175: 89 pgs: 89 active+clean; 131 MiB data, 473 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 528 KiB/s wr, 6 op/s 2024-04-12T11:15:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:25 smithi029 ceph-mon[39999]: osdmap e134: 8 total, 8 up, 8 in 2024-04-12T11:15:26.524 INFO:tasks.rados.rados.0.smithi029.stdout:286: done (0 left) 2024-04-12T11:15:26.525 INFO:tasks.rados.rados.0.smithi029.stdout:287: write oid 13 current snap is 34 2024-04-12T11:15:26.525 INFO:tasks.rados.rados.0.smithi029.stdout:287: seq_num 142 ranges {786594=779712,2326443=60151} 2024-04-12T11:15:26.533 INFO:tasks.rados.rados.0.smithi029.stdout:287: writing smithi02979277-13 from 786594 to 1566306 tid 1 2024-04-12T11:15:26.537 INFO:tasks.rados.rados.0.smithi029.stdout:287: writing smithi02979277-13 from 2326443 to 2386594 tid 2 2024-04-12T11:15:26.538 INFO:tasks.rados.rados.0.smithi029.stdout:288: write oid 29 current snap is 34 2024-04-12T11:15:26.538 INFO:tasks.rados.rados.0.smithi029.stdout:288: seq_num 143 ranges {403401=454961,1584925=506039} 2024-04-12T11:15:26.542 INFO:tasks.rados.rados.0.smithi029.stdout:288: writing smithi02979277-29 from 403401 to 858362 tid 1 2024-04-12T11:15:26.550 INFO:tasks.rados.rados.0.smithi029.stdout:288: writing smithi02979277-29 from 1584925 to 2090964 tid 2 2024-04-12T11:15:26.550 INFO:tasks.rados.rados.0.smithi029.stdout:289: read oid 30 snap -1 2024-04-12T11:15:26.550 INFO:tasks.rados.rados.0.smithi029.stdout:289: expect (ObjNum 29 snap 0 seq_num 29) 2024-04-12T11:15:26.550 INFO:tasks.rados.rados.0.smithi029.stdout:290: delete oid 22 current snap is 34 2024-04-12T11:15:26.550 INFO:tasks.rados.rados.0.smithi029.stdout:287: finishing write tid 1 to smithi02979277-13 2024-04-12T11:15:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:287: finishing write tid 2 to smithi02979277-13 2024-04-12T11:15:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:287: finishing write tid 3 to smithi02979277-13 2024-04-12T11:15:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:287: finishing write tid 4 to smithi02979277-13 2024-04-12T11:15:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:287: oid 13 updating version 0 to 94 2024-04-12T11:15:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:287: oid 13 updating version 94 to 95 2024-04-12T11:15:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:287: oid 13 version 95 is already newer than 93 2024-04-12T11:15:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 95 (ObjNum 142 snap 34 seq_num 142) dirty exists 2024-04-12T11:15:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:287: left oid 13 (ObjNum 142 snap 34 seq_num 142) 2024-04-12T11:15:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:288: finishing write tid 1 to smithi02979277-29 2024-04-12T11:15:26.553 INFO:tasks.rados.rados.0.smithi029.stdout:288: finishing write tid 2 to smithi02979277-29 2024-04-12T11:15:26.554 INFO:tasks.rados.rados.0.smithi029.stdout:288: finishing write tid 3 to smithi02979277-29 2024-04-12T11:15:26.554 INFO:tasks.rados.rados.0.smithi029.stdout:287: done (3 left) 2024-04-12T11:15:26.554 INFO:tasks.rados.rados.0.smithi029.stdout:290: done (2 left) 2024-04-12T11:15:26.554 INFO:tasks.rados.rados.0.smithi029.stdout:291: write oid 7 current snap is 34 2024-04-12T11:15:26.554 INFO:tasks.rados.rados.0.smithi029.stdout:291: seq_num 145 ranges {437015=721812,1701889=509708} 2024-04-12T11:15:26.559 INFO:tasks.rados.rados.0.smithi029.stdout:291: writing smithi02979277-7 from 437015 to 1158827 tid 1 2024-04-12T11:15:26.566 INFO:tasks.rados.rados.0.smithi029.stdout:291: writing smithi02979277-7 from 1701889 to 2211597 tid 2 2024-04-12T11:15:26.566 INFO:tasks.rados.rados.0.smithi029.stdout:292: delete oid 38 current snap is 34 2024-04-12T11:15:26.566 INFO:tasks.rados.rados.0.smithi029.stdout:289: expect (ObjNum 29 snap 0 seq_num 29) 2024-04-12T11:15:26.651 INFO:tasks.rados.rados.0.smithi029.stdout:288: finishing write tid 4 to smithi02979277-29 2024-04-12T11:15:26.651 INFO:tasks.rados.rados.0.smithi029.stdout:288: oid 29 updating version 0 to 75 2024-04-12T11:15:26.651 INFO:tasks.rados.rados.0.smithi029.stdout:288: oid 29 updating version 75 to 77 2024-04-12T11:15:26.651 INFO:tasks.rados.rados.0.smithi029.stdout:288: oid 29 version 77 is already newer than 76 2024-04-12T11:15:26.651 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 77 (ObjNum 143 snap 34 seq_num 143) dirty exists 2024-04-12T11:15:26.651 INFO:tasks.rados.rados.0.smithi029.stdout:288: left oid 29 (ObjNum 143 snap 34 seq_num 143) 2024-04-12T11:15:26.651 INFO:tasks.rados.rados.0.smithi029.stdout:291: finishing write tid 1 to smithi02979277-7 2024-04-12T11:15:26.652 INFO:tasks.rados.rados.0.smithi029.stdout:288: done (3 left) 2024-04-12T11:15:26.652 INFO:tasks.rados.rados.0.smithi029.stdout:289: done (2 left) 2024-04-12T11:15:26.652 INFO:tasks.rados.rados.0.smithi029.stdout:292: done (1 left) 2024-04-12T11:15:26.652 INFO:tasks.rados.rados.0.smithi029.stdout:293: write oid 6 current snap is 34 2024-04-12T11:15:26.652 INFO:tasks.rados.rados.0.smithi029.stdout:291: finishing write tid 2 to smithi02979277-7 2024-04-12T11:15:26.652 INFO:tasks.rados.rados.0.smithi029.stdout:291: finishing write tid 3 to smithi02979277-7 2024-04-12T11:15:26.652 INFO:tasks.rados.rados.0.smithi029.stdout:293: seq_num 147 ranges {470629=472310,1418853=513377} 2024-04-12T11:15:26.656 INFO:tasks.rados.rados.0.smithi029.stdout:293: writing smithi02979277-6 from 470629 to 942939 tid 1 2024-04-12T11:15:26.662 INFO:tasks.rados.rados.0.smithi029.stdout:293: writing smithi02979277-6 from 1418853 to 1932230 tid 2 2024-04-12T11:15:26.662 INFO:tasks.rados.rados.0.smithi029.stdout:294: write oid 11 current snap is 34 2024-04-12T11:15:26.662 INFO:tasks.rados.rados.0.smithi029.stdout:291: finishing write tid 4 to smithi02979277-7 2024-04-12T11:15:26.662 INFO:tasks.rados.rados.0.smithi029.stdout:291: oid 7 updating version 0 to 114 2024-04-12T11:15:26.662 INFO:tasks.rados.rados.0.smithi029.stdout:291: oid 7 updating version 114 to 115 2024-04-12T11:15:26.662 INFO:tasks.rados.rados.0.smithi029.stdout:291: oid 7 version 115 is already newer than 113 2024-04-12T11:15:26.662 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 115 (ObjNum 145 snap 34 seq_num 145) dirty exists 2024-04-12T11:15:26.663 INFO:tasks.rados.rados.0.smithi029.stdout:291: left oid 7 (ObjNum 145 snap 34 seq_num 145) 2024-04-12T11:15:26.663 INFO:tasks.rados.rados.0.smithi029.stdout:293: finishing write tid 1 to smithi02979277-6 2024-04-12T11:15:26.663 INFO:tasks.rados.rados.0.smithi029.stdout:294: seq_num 148 ranges {487436=547559,1760982=657035} 2024-04-12T11:15:26.667 INFO:tasks.rados.rados.0.smithi029.stdout:294: writing smithi02979277-11 from 487436 to 1034995 tid 1 2024-04-12T11:15:26.676 INFO:tasks.rados.rados.0.smithi029.stdout:294: writing smithi02979277-11 from 1760982 to 2418017 tid 2 2024-04-12T11:15:26.676 INFO:tasks.rados.rados.0.smithi029.stdout:291: done (2 left) 2024-04-12T11:15:26.676 INFO:tasks.rados.rados.0.smithi029.stdout:295: rollback oid 7 current snap is 34 2024-04-12T11:15:26.676 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 30 2024-04-12T11:15:26.676 INFO:tasks.rados.rados.0.smithi029.stdout:293: finishing write tid 2 to smithi02979277-6 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:293: finishing write tid 3 to smithi02979277-6 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:293: finishing write tid 4 to smithi02979277-6 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:293: oid 6 updating version 0 to 61 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:293: oid 6 version 61 is already newer than 59 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:293: oid 6 version 61 is already newer than 60 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 61 (ObjNum 147 snap 34 seq_num 147) dirty exists 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:293: left oid 6 (ObjNum 147 snap 34 seq_num 147) 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:294: finishing write tid 1 to smithi02979277-11 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:293: done (2 left) 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:296: rollback oid 10 current snap is 34 2024-04-12T11:15:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 10 to 20 2024-04-12T11:15:26.678 INFO:tasks.rados.rados.0.smithi029.stdout:297: delete oid 8 current snap is 34 2024-04-12T11:15:26.678 INFO:tasks.rados.rados.0.smithi029.stdout:296: finishing rollback tid 1 to smithi02979277-10 2024-04-12T11:15:26.678 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 0 (ObjNum 97 snap 0 seq_num 4229981600) dirty dne 2024-04-12T11:15:26.678 INFO:tasks.rados.rados.0.smithi029.stdout:296: done (3 left) 2024-04-12T11:15:26.678 INFO:tasks.rados.rados.0.smithi029.stdout:297: done (2 left) 2024-04-12T11:15:26.679 INFO:tasks.rados.rados.0.smithi029.stdout:298: snap_remove snap 28 2024-04-12T11:15:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:26 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/388408519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:26 smithi087 ceph-mon[32917]: osdmap e135: 8 total, 8 up, 8 in 2024-04-12T11:15:27.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:26 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/388408519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:26 smithi029 ceph-mon[29349]: osdmap e135: 8 total, 8 up, 8 in 2024-04-12T11:15:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:26 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/388408519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:26 smithi029 ceph-mon[39999]: osdmap e135: 8 total, 8 up, 8 in 2024-04-12T11:15:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:295: finishing rollback tid 0 to smithi02979277-7 2024-04-12T11:15:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:295: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:295: finishing rollback tid 2 to smithi02979277-7 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 118 (ObjNum 120 snap 22 seq_num 120) dirty exists 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:294: finishing write tid 2 to smithi02979277-11 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:294: finishing write tid 3 to smithi02979277-11 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:294: finishing write tid 4 to smithi02979277-11 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:294: oid 11 updating version 0 to 132 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:294: oid 11 updating version 132 to 134 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:294: oid 11 version 134 is already newer than 133 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 134 (ObjNum 148 snap 34 seq_num 148) dirty exists 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:294: left oid 11 (ObjNum 148 snap 34 seq_num 148) 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:294: done (2 left) 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:295: done (1 left) 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:298: done (0 left) 2024-04-12T11:15:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:299: snap_remove snap 32 2024-04-12T11:15:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:27 smithi087 ceph-mon[32917]: pgmap v178: 89 pgs: 89 active+clean; 131 MiB data, 473 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 528 KiB/s wr, 6 op/s 2024-04-12T11:15:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:27 smithi029 ceph-mon[29349]: pgmap v178: 89 pgs: 89 active+clean; 131 MiB data, 473 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 528 KiB/s wr, 6 op/s 2024-04-12T11:15:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:27 smithi029 ceph-mon[39999]: pgmap v178: 89 pgs: 89 active+clean; 131 MiB data, 473 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 528 KiB/s wr, 6 op/s 2024-04-12T11:15:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:299: done (0 left) 2024-04-12T11:15:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:300: rollback oid 36 current snap is 34 2024-04-12T11:15:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 36 to 30 2024-04-12T11:15:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:301: snap_create 2024-04-12T11:15:28.561 INFO:tasks.rados.rados.0.smithi029.stdout:300: finishing rollback tid 0 to smithi02979277-36 2024-04-12T11:15:28.561 INFO:tasks.rados.rados.0.smithi029.stdout:300: finishing rollback tid 1 to smithi02979277-36 2024-04-12T11:15:28.561 INFO:tasks.rados.rados.0.smithi029.stdout:300: finishing rollback tid 2 to smithi02979277-36 2024-04-12T11:15:28.561 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 67 (ObjNum 134 snap 28 seq_num 134) dirty exists 2024-04-12T11:15:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:28 smithi087 ceph-mon[32917]: osdmap e136: 8 total, 8 up, 8 in 2024-04-12T11:15:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:28 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:28 smithi029 ceph-mon[29349]: osdmap e136: 8 total, 8 up, 8 in 2024-04-12T11:15:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:28 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:28 smithi029 ceph-mon[39999]: osdmap e136: 8 total, 8 up, 8 in 2024-04-12T11:15:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:28 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:29.557 INFO:tasks.rados.rados.0.smithi029.stdout:300: done (1 left) 2024-04-12T11:15:29.557 INFO:tasks.rados.rados.0.smithi029.stdout:301: done (0 left) 2024-04-12T11:15:29.557 INFO:tasks.rados.rados.0.smithi029.stdout:302: write oid 3 current snap is 35 2024-04-12T11:15:29.558 INFO:tasks.rados.rados.0.smithi029.stdout:302: seq_num 150 ranges {521050=698057,1877946=643104} 2024-04-12T11:15:29.564 INFO:tasks.rados.rados.0.smithi029.stdout:302: writing smithi02979277-3 from 521050 to 1219107 tid 1 2024-04-12T11:15:29.571 INFO:tasks.rados.rados.0.smithi029.stdout:302: writing smithi02979277-3 from 1877946 to 2521050 tid 2 2024-04-12T11:15:29.571 INFO:tasks.rados.rados.0.smithi029.stdout:303: read oid 25 snap 30 2024-04-12T11:15:29.571 INFO:tasks.rados.rados.0.smithi029.stdout:303: expect (ObjNum 78 snap 9 seq_num 78) 2024-04-12T11:15:29.572 INFO:tasks.rados.rados.0.smithi029.stdout:304: snap_create 2024-04-12T11:15:29.572 INFO:tasks.rados.rados.0.smithi029.stdout:302: finishing write tid 1 to smithi02979277-3 2024-04-12T11:15:29.575 INFO:tasks.rados.rados.0.smithi029.stdout:302: finishing write tid 2 to smithi02979277-3 2024-04-12T11:15:29.575 INFO:tasks.rados.rados.0.smithi029.stdout:302: finishing write tid 3 to smithi02979277-3 2024-04-12T11:15:29.575 INFO:tasks.rados.rados.0.smithi029.stdout:302: finishing write tid 4 to smithi02979277-3 2024-04-12T11:15:29.575 INFO:tasks.rados.rados.0.smithi029.stdout:302: oid 3 updating version 0 to 142 2024-04-12T11:15:29.576 INFO:tasks.rados.rados.0.smithi029.stdout:302: oid 3 version 142 is already newer than 140 2024-04-12T11:15:29.576 INFO:tasks.rados.rados.0.smithi029.stdout:302: oid 3 version 142 is already newer than 141 2024-04-12T11:15:29.576 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 142 (ObjNum 150 snap 35 seq_num 150) dirty exists 2024-04-12T11:15:29.576 INFO:tasks.rados.rados.0.smithi029.stdout:302: left oid 3 (ObjNum 150 snap 35 seq_num 150) 2024-04-12T11:15:29.578 INFO:tasks.rados.rados.0.smithi029.stdout:303: expect (ObjNum 78 snap 9 seq_num 78) 2024-04-12T11:15:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:29 smithi087 ceph-mon[32917]: pgmap v181: 89 pgs: 89 active+clean; 131 MiB data, 473 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:29 smithi087 ceph-mon[32917]: osdmap e137: 8 total, 8 up, 8 in 2024-04-12T11:15:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:29 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3424312611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:30.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:29 smithi029 ceph-mon[29349]: pgmap v181: 89 pgs: 89 active+clean; 131 MiB data, 473 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:30.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:29 smithi029 ceph-mon[29349]: osdmap e137: 8 total, 8 up, 8 in 2024-04-12T11:15:30.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:29 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3424312611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:30.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:29 smithi029 ceph-mon[39999]: pgmap v181: 89 pgs: 89 active+clean; 131 MiB data, 473 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:30.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:29 smithi029 ceph-mon[39999]: osdmap e137: 8 total, 8 up, 8 in 2024-04-12T11:15:30.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:29 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3424312611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:30.568 INFO:tasks.rados.rados.0.smithi029.stdout:302: done (2 left) 2024-04-12T11:15:30.568 INFO:tasks.rados.rados.0.smithi029.stdout:303: done (1 left) 2024-04-12T11:15:30.568 INFO:tasks.rados.rados.0.smithi029.stdout:304: done (0 left) 2024-04-12T11:15:30.568 INFO:tasks.rados.rados.0.smithi029.stdout:305: snap_remove snap 34 2024-04-12T11:15:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:30 smithi087 ceph-mon[32917]: osdmap e138: 8 total, 8 up, 8 in 2024-04-12T11:15:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:30 smithi029 ceph-mon[39999]: osdmap e138: 8 total, 8 up, 8 in 2024-04-12T11:15:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:30 smithi029 ceph-mon[29349]: osdmap e138: 8 total, 8 up, 8 in 2024-04-12T11:15:31.526 INFO:tasks.rados.rados.0.smithi029.stdout:305: done (0 left) 2024-04-12T11:15:31.526 INFO:tasks.rados.rados.0.smithi029.stdout:306: read oid 30 snap -1 2024-04-12T11:15:31.526 INFO:tasks.rados.rados.0.smithi029.stdout:306: expect (ObjNum 29 snap 0 seq_num 29) 2024-04-12T11:15:31.526 INFO:tasks.rados.rados.0.smithi029.stdout:307: write oid 9 current snap is 36 2024-04-12T11:15:31.526 INFO:tasks.rados.rados.0.smithi029.stdout:307: seq_num 151 ranges {537857=773306,1936428=520715} 2024-04-12T11:15:31.533 INFO:tasks.rados.rados.0.smithi029.stdout:307: writing smithi02979277-9 from 537857 to 1311163 tid 1 2024-04-12T11:15:31.540 INFO:tasks.rados.rados.0.smithi029.stdout:307: writing smithi02979277-9 from 1936428 to 2457143 tid 2 2024-04-12T11:15:31.540 INFO:tasks.rados.rados.0.smithi029.stdout:308: delete oid 1 current snap is 36 2024-04-12T11:15:31.540 INFO:tasks.rados.rados.0.smithi029.stdout:306: expect (ObjNum 29 snap 0 seq_num 29) 2024-04-12T11:15:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:307: finishing write tid 1 to smithi02979277-9 2024-04-12T11:15:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:307: finishing write tid 2 to smithi02979277-9 2024-04-12T11:15:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:307: finishing write tid 3 to smithi02979277-9 2024-04-12T11:15:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:307: finishing write tid 4 to smithi02979277-9 2024-04-12T11:15:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:307: oid 9 updating version 0 to 82 2024-04-12T11:15:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:307: oid 9 updating version 82 to 83 2024-04-12T11:15:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:307: oid 9 version 83 is already newer than 81 2024-04-12T11:15:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 83 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-04-12T11:15:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:307: left oid 9 (ObjNum 151 snap 36 seq_num 151) 2024-04-12T11:15:31.627 INFO:tasks.rados.rados.0.smithi029.stdout:306: done (2 left) 2024-04-12T11:15:31.627 INFO:tasks.rados.rados.0.smithi029.stdout:307: done (1 left) 2024-04-12T11:15:31.627 INFO:tasks.rados.rados.0.smithi029.stdout:308: done (0 left) 2024-04-12T11:15:31.627 INFO:tasks.rados.rados.0.smithi029.stdout:309: write oid 49 current snap is 36 2024-04-12T11:15:31.627 INFO:tasks.rados.rados.0.smithi029.stdout:309: seq_num 153 ranges {571471=640157,1653392=524384} 2024-04-12T11:15:31.632 INFO:tasks.rados.rados.0.smithi029.stdout:309: writing smithi02979277-49 from 571471 to 1211628 tid 1 2024-04-12T11:15:31.638 INFO:tasks.rados.rados.0.smithi029.stdout:309: writing smithi02979277-49 from 1653392 to 2177776 tid 2 2024-04-12T11:15:31.638 INFO:tasks.rados.rados.0.smithi029.stdout:310: snap_create 2024-04-12T11:15:31.638 INFO:tasks.rados.rados.0.smithi029.stdout:309: finishing write tid 1 to smithi02979277-49 2024-04-12T11:15:31.640 INFO:tasks.rados.rados.0.smithi029.stdout:309: finishing write tid 2 to smithi02979277-49 2024-04-12T11:15:31.640 INFO:tasks.rados.rados.0.smithi029.stdout:309: finishing write tid 3 to smithi02979277-49 2024-04-12T11:15:31.640 INFO:tasks.rados.rados.0.smithi029.stdout:309: finishing write tid 4 to smithi02979277-49 2024-04-12T11:15:31.640 INFO:tasks.rados.rados.0.smithi029.stdout:309: oid 49 updating version 0 to 86 2024-04-12T11:15:31.640 INFO:tasks.rados.rados.0.smithi029.stdout:309: oid 49 version 86 is already newer than 85 2024-04-12T11:15:31.640 INFO:tasks.rados.rados.0.smithi029.stdout:309: oid 49 updating version 86 to 87 2024-04-12T11:15:31.640 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 87 (ObjNum 153 snap 36 seq_num 153) dirty exists 2024-04-12T11:15:31.640 INFO:tasks.rados.rados.0.smithi029.stdout:309: left oid 49 (ObjNum 153 snap 36 seq_num 153) 2024-04-12T11:15:31.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:31 smithi029 ceph-mon[39999]: pgmap v184: 89 pgs: 89 active+clean; 137 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:15:31.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:31 smithi029 ceph-mon[39999]: osdmap e139: 8 total, 8 up, 8 in 2024-04-12T11:15:31.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:31 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3116729412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:31.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:31 smithi029 ceph-mon[39999]: osdmap e140: 8 total, 8 up, 8 in 2024-04-12T11:15:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:31 smithi029 ceph-mon[29349]: pgmap v184: 89 pgs: 89 active+clean; 137 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:15:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:31 smithi029 ceph-mon[29349]: osdmap e139: 8 total, 8 up, 8 in 2024-04-12T11:15:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:31 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3116729412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:31 smithi029 ceph-mon[29349]: osdmap e140: 8 total, 8 up, 8 in 2024-04-12T11:15:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:31 smithi087 ceph-mon[32917]: pgmap v184: 89 pgs: 89 active+clean; 137 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:15:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:31 smithi087 ceph-mon[32917]: osdmap e139: 8 total, 8 up, 8 in 2024-04-12T11:15:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:31 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3116729412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:31 smithi087 ceph-mon[32917]: osdmap e140: 8 total, 8 up, 8 in 2024-04-12T11:15:32.573 INFO:tasks.rados.rados.0.smithi029.stdout:309: done (1 left) 2024-04-12T11:15:32.573 INFO:tasks.rados.rados.0.smithi029.stdout:310: done (0 left) 2024-04-12T11:15:32.573 INFO:tasks.rados.rados.0.smithi029.stdout:311: snap_create 2024-04-12T11:15:33.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:15:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:15:33] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.43.0" 2024-04-12T11:15:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:311: done (0 left) 2024-04-12T11:15:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:312: read oid 35 snap -1 2024-04-12T11:15:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:312: expect deleted 2024-04-12T11:15:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:313: delete oid 29 current snap is 38 2024-04-12T11:15:33.578 INFO:tasks.rados.rados.0.smithi029.stdout:312: done (1 left) 2024-04-12T11:15:33.578 INFO:tasks.rados.rados.0.smithi029.stdout:313: done (0 left) 2024-04-12T11:15:33.578 INFO:tasks.rados.rados.0.smithi029.stdout:314: snap_remove snap 22 2024-04-12T11:15:33.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:33 smithi087 ceph-mon[32917]: pgmap v187: 89 pgs: 89 active+clean; 137 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:15:33.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:33 smithi087 ceph-mon[32917]: osdmap e141: 8 total, 8 up, 8 in 2024-04-12T11:15:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:33 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3589632536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:34.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:33 smithi029 ceph-mon[29349]: pgmap v187: 89 pgs: 89 active+clean; 137 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:15:34.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:33 smithi029 ceph-mon[29349]: osdmap e141: 8 total, 8 up, 8 in 2024-04-12T11:15:34.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:33 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3589632536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:34.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:33 smithi029 ceph-mon[39999]: pgmap v187: 89 pgs: 89 active+clean; 137 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:15:34.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:33 smithi029 ceph-mon[39999]: osdmap e141: 8 total, 8 up, 8 in 2024-04-12T11:15:34.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:33 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3589632536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:34.585 INFO:tasks.rados.rados.0.smithi029.stdout:314: done (0 left) 2024-04-12T11:15:34.585 INFO:tasks.rados.rados.0.smithi029.stdout:315: snap_remove snap 30 2024-04-12T11:15:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:34 smithi087 ceph-mon[32917]: osdmap e142: 8 total, 8 up, 8 in 2024-04-12T11:15:35.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:34 smithi029 ceph-mon[39999]: osdmap e142: 8 total, 8 up, 8 in 2024-04-12T11:15:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:34 smithi029 ceph-mon[29349]: osdmap e142: 8 total, 8 up, 8 in 2024-04-12T11:15:35.591 INFO:tasks.rados.rados.0.smithi029.stdout:315: done (0 left) 2024-04-12T11:15:35.592 INFO:tasks.rados.rados.0.smithi029.stdout:316: write oid 29 current snap is 38 2024-04-12T11:15:35.592 INFO:tasks.rados.rados.0.smithi029.stdout:316: seq_num 155 ranges {605085=790655,2170356=434729} 2024-04-12T11:15:35.599 INFO:tasks.rados.rados.0.smithi029.stdout:316: writing smithi02979277-29 from 605085 to 1395740 tid 1 2024-04-12T11:15:35.607 INFO:tasks.rados.rados.0.smithi029.stdout:316: writing smithi02979277-29 from 2170356 to 2605085 tid 2 2024-04-12T11:15:35.607 INFO:tasks.rados.rados.0.smithi029.stdout:317: snap_create 2024-04-12T11:15:35.608 INFO:tasks.rados.rados.0.smithi029.stdout:316: finishing write tid 1 to smithi02979277-29 2024-04-12T11:15:35.610 INFO:tasks.rados.rados.0.smithi029.stdout:316: finishing write tid 2 to smithi02979277-29 2024-04-12T11:15:35.610 INFO:tasks.rados.rados.0.smithi029.stdout:316: finishing write tid 3 to smithi02979277-29 2024-04-12T11:15:35.610 INFO:tasks.rados.rados.0.smithi029.stdout:316: finishing write tid 4 to smithi02979277-29 2024-04-12T11:15:35.610 INFO:tasks.rados.rados.0.smithi029.stdout:316: oid 29 updating version 0 to 83 2024-04-12T11:15:35.610 INFO:tasks.rados.rados.0.smithi029.stdout:316: oid 29 updating version 83 to 84 2024-04-12T11:15:35.610 INFO:tasks.rados.rados.0.smithi029.stdout:316: oid 29 version 84 is already newer than 82 2024-04-12T11:15:35.610 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 84 (ObjNum 155 snap 38 seq_num 155) dirty exists 2024-04-12T11:15:35.610 INFO:tasks.rados.rados.0.smithi029.stdout:316: left oid 29 (ObjNum 155 snap 38 seq_num 155) 2024-04-12T11:15:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:35 smithi087 ceph-mon[32917]: pgmap v190: 89 pgs: 89 active+clean; 141 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 929 KiB/s wr, 8 op/s 2024-04-12T11:15:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:35 smithi087 ceph-mon[32917]: osdmap e143: 8 total, 8 up, 8 in 2024-04-12T11:15:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:35 smithi029 ceph-mon[39999]: pgmap v190: 89 pgs: 89 active+clean; 141 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 929 KiB/s wr, 8 op/s 2024-04-12T11:15:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:35 smithi029 ceph-mon[39999]: osdmap e143: 8 total, 8 up, 8 in 2024-04-12T11:15:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:35 smithi029 ceph-mon[29349]: pgmap v190: 89 pgs: 89 active+clean; 141 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 929 KiB/s wr, 8 op/s 2024-04-12T11:15:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:35 smithi029 ceph-mon[29349]: osdmap e143: 8 total, 8 up, 8 in 2024-04-12T11:15:36.528 INFO:tasks.rados.rados.0.smithi029.stdout:316: done (1 left) 2024-04-12T11:15:36.528 INFO:tasks.rados.rados.0.smithi029.stdout:317: done (0 left) 2024-04-12T11:15:36.528 INFO:tasks.rados.rados.0.smithi029.stdout:318: write oid 7 current snap is 39 2024-04-12T11:15:36.528 INFO:tasks.rados.rados.0.smithi029.stdout:318: seq_num 156 ranges {621892=465904,1712485=671711} 2024-04-12T11:15:36.533 INFO:tasks.rados.rados.0.smithi029.stdout:318: writing smithi02979277-7 from 621892 to 1087796 tid 1 2024-04-12T11:15:36.541 INFO:tasks.rados.rados.0.smithi029.stdout:318: writing smithi02979277-7 from 1712485 to 2384196 tid 2 2024-04-12T11:15:36.542 INFO:tasks.rados.rados.0.smithi029.stdout:319: rollback oid 50 current snap is 39 2024-04-12T11:15:36.542 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 50 to 36 2024-04-12T11:15:36.542 INFO:tasks.rados.rados.0.smithi029.stdout:318: finishing write tid 1 to smithi02979277-7 2024-04-12T11:15:36.542 INFO:tasks.rados.rados.0.smithi029.stdout:320: read oid 36 snap -1 2024-04-12T11:15:36.542 INFO:tasks.rados.rados.0.smithi029.stdout:320: expect (ObjNum 134 snap 28 seq_num 134) 2024-04-12T11:15:36.542 INFO:tasks.rados.rados.0.smithi029.stdout:321: rollback oid 44 current snap is 39 2024-04-12T11:15:36.542 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 44 to 37 2024-04-12T11:15:36.542 INFO:tasks.rados.rados.0.smithi029.stdout:322: write oid 17 current snap is 39 2024-04-12T11:15:36.542 INFO:tasks.rados.rados.0.smithi029.stdout:322: seq_num 157 ranges {638699=541153,1770967=531722} 2024-04-12T11:15:36.548 INFO:tasks.rados.rados.0.smithi029.stdout:322: writing smithi02979277-17 from 638699 to 1179852 tid 1 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:322: writing smithi02979277-17 from 1770967 to 2302689 tid 2 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:323: snap_create 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:321: finishing rollback tid 1 to smithi02979277-44 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 100 (ObjNum 33 snap 0 seq_num 861964147) dirty dne 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:318: finishing write tid 2 to smithi02979277-7 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:318: finishing write tid 3 to smithi02979277-7 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:318: finishing write tid 4 to smithi02979277-7 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:318: oid 7 updating version 0 to 132 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:318: oid 7 version 132 is already newer than 131 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:318: oid 7 version 132 is already newer than 130 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 132 (ObjNum 156 snap 39 seq_num 156) dirty exists 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:318: left oid 7 (ObjNum 156 snap 39 seq_num 156) 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:319: finishing rollback tid 0 to smithi02979277-50 2024-04-12T11:15:36.555 INFO:tasks.rados.rados.0.smithi029.stdout:319: finishing rollback tid 1 to smithi02979277-50 2024-04-12T11:15:36.556 INFO:tasks.rados.rados.0.smithi029.stdout:319: finishing rollback tid 2 to smithi02979277-50 2024-04-12T11:15:36.556 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 122 (ObjNum 94 snap 16 seq_num 94) dirty exists 2024-04-12T11:15:36.556 INFO:tasks.rados.rados.0.smithi029.stdout:322: finishing write tid 1 to smithi02979277-17 2024-04-12T11:15:36.558 INFO:tasks.rados.rados.0.smithi029.stdout:320: expect (ObjNum 134 snap 28 seq_num 134) 2024-04-12T11:15:36.632 INFO:tasks.rados.rados.0.smithi029.stdout:322: finishing write tid 2 to smithi02979277-17 2024-04-12T11:15:36.632 INFO:tasks.rados.rados.0.smithi029.stdout:322: finishing write tid 3 to smithi02979277-17 2024-04-12T11:15:36.632 INFO:tasks.rados.rados.0.smithi029.stdout:322: finishing write tid 4 to smithi02979277-17 2024-04-12T11:15:36.632 INFO:tasks.rados.rados.0.smithi029.stdout:322: oid 17 updating version 0 to 107 2024-04-12T11:15:36.633 INFO:tasks.rados.rados.0.smithi029.stdout:322: oid 17 version 107 is already newer than 106 2024-04-12T11:15:36.633 INFO:tasks.rados.rados.0.smithi029.stdout:322: oid 17 version 107 is already newer than 105 2024-04-12T11:15:36.633 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 107 (ObjNum 157 snap 39 seq_num 157) dirty exists 2024-04-12T11:15:36.633 INFO:tasks.rados.rados.0.smithi029.stdout:322: left oid 17 (ObjNum 157 snap 39 seq_num 157) 2024-04-12T11:15:36.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:36 smithi087 ceph-mon[32917]: osdmap e144: 8 total, 8 up, 8 in 2024-04-12T11:15:36.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:36 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2495186994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:36.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:36 smithi087 ceph-mon[32917]: osdmap e145: 8 total, 8 up, 8 in 2024-04-12T11:15:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:36 smithi029 ceph-mon[29349]: osdmap e144: 8 total, 8 up, 8 in 2024-04-12T11:15:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:36 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2495186994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:36 smithi029 ceph-mon[29349]: osdmap e145: 8 total, 8 up, 8 in 2024-04-12T11:15:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:36 smithi029 ceph-mon[39999]: osdmap e144: 8 total, 8 up, 8 in 2024-04-12T11:15:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:36 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2495186994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:36 smithi029 ceph-mon[39999]: osdmap e145: 8 total, 8 up, 8 in 2024-04-12T11:15:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:318: done (5 left) 2024-04-12T11:15:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:319: done (4 left) 2024-04-12T11:15:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:320: done (3 left) 2024-04-12T11:15:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:321: done (2 left) 2024-04-12T11:15:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:322: done (1 left) 2024-04-12T11:15:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:323: done (0 left) 2024-04-12T11:15:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:324: write oid 40 current snap is 40 2024-04-12T11:15:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:324: seq_num 158 ranges {655506=616402,1829449=675380} 2024-04-12T11:15:37.536 INFO:tasks.rados.rados.0.smithi029.stdout:324: writing smithi02979277-40 from 655506 to 1271908 tid 1 2024-04-12T11:15:37.543 INFO:tasks.rados.rados.0.smithi029.stdout:324: writing smithi02979277-40 from 1829449 to 2504829 tid 2 2024-04-12T11:15:37.543 INFO:tasks.rados.rados.0.smithi029.stdout:325: snap_create 2024-04-12T11:15:37.543 INFO:tasks.rados.rados.0.smithi029.stdout:324: finishing write tid 1 to smithi02979277-40 2024-04-12T11:15:37.546 INFO:tasks.rados.rados.0.smithi029.stdout:324: finishing write tid 2 to smithi02979277-40 2024-04-12T11:15:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:324: finishing write tid 3 to smithi02979277-40 2024-04-12T11:15:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:324: finishing write tid 4 to smithi02979277-40 2024-04-12T11:15:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:324: oid 40 updating version 0 to 156 2024-04-12T11:15:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:324: oid 40 updating version 156 to 157 2024-04-12T11:15:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:324: oid 40 updating version 157 to 158 2024-04-12T11:15:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 158 (ObjNum 158 snap 40 seq_num 158) dirty exists 2024-04-12T11:15:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:324: left oid 40 (ObjNum 158 snap 40 seq_num 158) 2024-04-12T11:15:37.592 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:37 smithi087 ceph-mon[32917]: pgmap v193: 89 pgs: 89 active+clean; 141 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 928 KiB/s wr, 8 op/s 2024-04-12T11:15:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:37 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:37 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:15:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:37 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:15:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:37 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:15:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:37 smithi087 ceph-mon[32917]: osdmap e146: 8 total, 8 up, 8 in 2024-04-12T11:15:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[29349]: pgmap v193: 89 pgs: 89 active+clean; 141 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 928 KiB/s wr, 8 op/s 2024-04-12T11:15:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:15:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:15:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:15:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[29349]: osdmap e146: 8 total, 8 up, 8 in 2024-04-12T11:15:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[39999]: pgmap v193: 89 pgs: 89 active+clean; 141 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 928 KiB/s wr, 8 op/s 2024-04-12T11:15:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:15:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:15:38.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:15:38.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:37 smithi029 ceph-mon[39999]: osdmap e146: 8 total, 8 up, 8 in 2024-04-12T11:15:38.530 INFO:tasks.rados.rados.0.smithi029.stdout:324: done (1 left) 2024-04-12T11:15:38.530 INFO:tasks.rados.rados.0.smithi029.stdout:325: done (0 left) 2024-04-12T11:15:38.530 INFO:tasks.rados.rados.0.smithi029.stdout:326: write oid 33 current snap is 41 2024-04-12T11:15:38.530 INFO:tasks.rados.rados.0.smithi029.stdout:326: seq_num 159 ranges {672313=691651,1887931=419038} 2024-04-12T11:15:38.537 INFO:tasks.rados.rados.0.smithi029.stdout:326: writing smithi02979277-33 from 672313 to 1363964 tid 1 2024-04-12T11:15:38.543 INFO:tasks.rados.rados.0.smithi029.stdout:326: writing smithi02979277-33 from 1887931 to 2306969 tid 2 2024-04-12T11:15:38.543 INFO:tasks.rados.rados.0.smithi029.stdout:326: finishing write tid 1 to smithi02979277-33 2024-04-12T11:15:38.543 INFO:tasks.rados.rados.0.smithi029.stdout:327: read oid 18 snap 23 2024-04-12T11:15:38.543 INFO:tasks.rados.rados.0.smithi029.stdout:327: expect (ObjNum 118 snap 20 seq_num 118) 2024-04-12T11:15:38.543 INFO:tasks.rados.rados.0.smithi029.stdout:328: read oid 48 snap 40 2024-04-12T11:15:38.543 INFO:tasks.rados.rados.0.smithi029.stdout:328: expect (ObjNum 128 snap 27 seq_num 128) 2024-04-12T11:15:38.543 INFO:tasks.rados.rados.0.smithi029.stdout:329: write oid 49 current snap is 41 2024-04-12T11:15:38.543 INFO:tasks.rados.rados.0.smithi029.stdout:329: seq_num 160 ranges {689120=483253,1946413=679049} 2024-04-12T11:15:38.550 INFO:tasks.rados.rados.0.smithi029.stdout:329: writing smithi02979277-49 from 689120 to 1172373 tid 1 2024-04-12T11:15:38.558 INFO:tasks.rados.rados.0.smithi029.stdout:329: writing smithi02979277-49 from 1946413 to 2625462 tid 2 2024-04-12T11:15:38.559 INFO:tasks.rados.rados.0.smithi029.stdout:330: snap_create 2024-04-12T11:15:38.559 INFO:tasks.rados.rados.0.smithi029.stdout:326: finishing write tid 2 to smithi02979277-33 2024-04-12T11:15:38.559 INFO:tasks.rados.rados.0.smithi029.stdout:326: finishing write tid 3 to smithi02979277-33 2024-04-12T11:15:38.559 INFO:tasks.rados.rados.0.smithi029.stdout:326: finishing write tid 4 to smithi02979277-33 2024-04-12T11:15:38.559 INFO:tasks.rados.rados.0.smithi029.stdout:326: oid 33 updating version 0 to 124 2024-04-12T11:15:38.559 INFO:tasks.rados.rados.0.smithi029.stdout:326: oid 33 updating version 124 to 125 2024-04-12T11:15:38.559 INFO:tasks.rados.rados.0.smithi029.stdout:326: oid 33 version 125 is already newer than 123 2024-04-12T11:15:38.559 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 125 (ObjNum 159 snap 41 seq_num 159) dirty exists 2024-04-12T11:15:38.559 INFO:tasks.rados.rados.0.smithi029.stdout:326: left oid 33 (ObjNum 159 snap 41 seq_num 159) 2024-04-12T11:15:38.559 INFO:tasks.rados.rados.0.smithi029.stdout:328: expect (ObjNum 128 snap 27 seq_num 128) 2024-04-12T11:15:38.716 INFO:tasks.rados.rados.0.smithi029.stdout:329: finishing write tid 1 to smithi02979277-49 2024-04-12T11:15:38.716 INFO:tasks.rados.rados.0.smithi029.stdout:327: expect (ObjNum 118 snap 20 seq_num 118) 2024-04-12T11:15:38.850 INFO:tasks.rados.rados.0.smithi029.stdout:329: finishing write tid 2 to smithi02979277-49 2024-04-12T11:15:38.850 INFO:tasks.rados.rados.0.smithi029.stdout:329: finishing write tid 3 to smithi02979277-49 2024-04-12T11:15:38.850 INFO:tasks.rados.rados.0.smithi029.stdout:329: finishing write tid 4 to smithi02979277-49 2024-04-12T11:15:38.850 INFO:tasks.rados.rados.0.smithi029.stdout:329: oid 49 updating version 0 to 97 2024-04-12T11:15:38.850 INFO:tasks.rados.rados.0.smithi029.stdout:329: oid 49 updating version 97 to 99 2024-04-12T11:15:38.850 INFO:tasks.rados.rados.0.smithi029.stdout:329: oid 49 version 99 is already newer than 98 2024-04-12T11:15:38.850 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 99 (ObjNum 160 snap 41 seq_num 160) dirty exists 2024-04-12T11:15:38.850 INFO:tasks.rados.rados.0.smithi029.stdout:329: left oid 49 (ObjNum 160 snap 41 seq_num 160) 2024-04-12T11:15:38.925 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:38 smithi087 ceph-mon[32917]: Upgrade: Updating mgr.x 2024-04-12T11:15:38.925 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:38 smithi087 ceph-mon[32917]: Deploying daemon mgr.x on smithi087 2024-04-12T11:15:38.925 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:38 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3680000735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:38.926 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:38 smithi087 ceph-mon[32917]: osdmap e147: 8 total, 8 up, 8 in 2024-04-12T11:15:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:38 smithi029 ceph-mon[29349]: Upgrade: Updating mgr.x 2024-04-12T11:15:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:38 smithi029 ceph-mon[29349]: Deploying daemon mgr.x on smithi087 2024-04-12T11:15:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:38 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3680000735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:38 smithi029 ceph-mon[29349]: osdmap e147: 8 total, 8 up, 8 in 2024-04-12T11:15:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:38 smithi029 ceph-mon[39999]: Upgrade: Updating mgr.x 2024-04-12T11:15:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:38 smithi029 ceph-mon[39999]: Deploying daemon mgr.x on smithi087 2024-04-12T11:15:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:38 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3680000735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:38 smithi029 ceph-mon[39999]: osdmap e147: 8 total, 8 up, 8 in 2024-04-12T11:15:39.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:39 smithi087 systemd[1]: Stopping Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:15:39.532 INFO:tasks.rados.rados.0.smithi029.stdout:326: done (4 left) 2024-04-12T11:15:39.533 INFO:tasks.rados.rados.0.smithi029.stdout:327: done (3 left) 2024-04-12T11:15:39.533 INFO:tasks.rados.rados.0.smithi029.stdout:328: done (2 left) 2024-04-12T11:15:39.533 INFO:tasks.rados.rados.0.smithi029.stdout:329: done (1 left) 2024-04-12T11:15:39.533 INFO:tasks.rados.rados.0.smithi029.stdout:330: done (0 left) 2024-04-12T11:15:39.533 INFO:tasks.rados.rados.0.smithi029.stdout:331: snap_remove snap 39 2024-04-12T11:15:39.781 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:39 smithi087 podman[66744]: 2024-04-12 11:15:39.4837277 +0000 UTC m=+0.127719037 container died 6117969a131454edf79c97d1d333181f327a53f4faf670858971fc1dad4a0448 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=reef-d540eba, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:15:39.781 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:39 smithi087 ceph-mon[32917]: pgmap v196: 89 pgs: 89 active+clean; 141 MiB data, 501 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:39.781 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:39 smithi087 ceph-mon[32917]: osdmap e148: 8 total, 8 up, 8 in 2024-04-12T11:15:40.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:39 smithi029 ceph-mon[29349]: pgmap v196: 89 pgs: 89 active+clean; 141 MiB data, 501 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:40.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:39 smithi029 ceph-mon[29349]: osdmap e148: 8 total, 8 up, 8 in 2024-04-12T11:15:40.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:39 smithi029 ceph-mon[39999]: pgmap v196: 89 pgs: 89 active+clean; 141 MiB data, 501 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:40.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:39 smithi029 ceph-mon[39999]: osdmap e148: 8 total, 8 up, 8 in 2024-04-12T11:15:40.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:39 smithi087 podman[66744]: 2024-04-12 11:15:39.84637808 +0000 UTC m=+0.490369419 container cleanup 6117969a131454edf79c97d1d333181f327a53f4faf670858971fc1dad4a0448 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, GIT_CLEAN=True, RELEASE=reef-d540eba, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2) 2024-04-12T11:15:40.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:39 smithi087 bash[66744]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x 2024-04-12T11:15:40.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 podman[66758]: 2024-04-12 11:15:40.014147205 +0000 UTC m=+0.525961274 container remove 6117969a131454edf79c97d1d333181f327a53f4faf670858971fc1dad4a0448 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , RELEASE=reef-d540eba, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T11:15:40.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-12T11:15:40.533 INFO:tasks.rados.rados.0.smithi029.stdout:331: done (0 left) 2024-04-12T11:15:40.533 INFO:tasks.rados.rados.0.smithi029.stdout:332: rollback oid 33 current snap is 42 2024-04-12T11:15:40.533 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 38 2024-04-12T11:15:40.533 INFO:tasks.rados.rados.0.smithi029.stdout:333: snap_remove snap 33 2024-04-12T11:15:40.669 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-12T11:15:40.669 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 systemd[1]: Stopped Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:15:40.669 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.x.service: Consumed 17.964s CPU time. 2024-04-12T11:15:40.670 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 systemd[1]: Starting Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:15:40.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 podman[66893]: 2024-04-12 11:15:40.572176161 +0000 UTC m=+0.017755814 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:15:40.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 podman[66893]: 2024-04-12 11:15:40.706683187 +0000 UTC m=+0.152262816 container create ba0eae540f953c975fcb912e9a91454623fc4f90b4b11c7c5b844d1dbd7bb42c (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, RELEASE=wip-rf-comb1-2cee185, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-12T11:15:41.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 podman[66893]: 2024-04-12 11:15:40.967686523 +0000 UTC m=+0.413266153 container init ba0eae540f953c975fcb912e9a91454623fc4f90b4b11c7c5b844d1dbd7bb42c (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:15:41.467 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 podman[66893]: 2024-04-12 11:15:40.97014691 +0000 UTC m=+0.415726543 container start ba0eae540f953c975fcb912e9a91454623fc4f90b4b11c7c5b844d1dbd7bb42c (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, maintainer=Guillaume Abrioux ) 2024-04-12T11:15:41.467 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:40 smithi087 ceph-mgr[66912]: -- 172.21.15.87:0/4094483832 <== mon.1 v2:172.21.15.87:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5640bd65e5a0 con 0x5640bd633000 2024-04-12T11:15:41.467 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:41 smithi087 bash[66893]: ba0eae540f953c975fcb912e9a91454623fc4f90b4b11c7c5b844d1dbd7bb42c 2024-04-12T11:15:41.467 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:41 smithi087 systemd[1]: Started Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:15:41.528 INFO:tasks.rados.rados.0.smithi029.stdout:333: done (1 left) 2024-04-12T11:15:41.529 INFO:tasks.rados.rados.0.smithi029.stdout:334: rollback oid 2 current snap is 42 2024-04-12T11:15:41.529 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 2 to 37 2024-04-12T11:15:41.529 INFO:tasks.rados.rados.0.smithi029.stdout:332: finishing rollback tid 0 to smithi02979277-33 2024-04-12T11:15:41.529 INFO:tasks.rados.rados.0.smithi029.stdout:332: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:15:41.530 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 128 (ObjNum 273 snap 0 seq_num 1479980447) dirty dne 2024-04-12T11:15:41.530 INFO:tasks.rados.rados.0.smithi029.stdout:332: done (1 left) 2024-04-12T11:15:41.530 INFO:tasks.rados.rados.0.smithi029.stdout:335: write oid 48 current snap is 42 2024-04-12T11:15:41.530 INFO:tasks.rados.rados.0.smithi029.stdout:335: seq_num 161 ranges {705927=558502,2004895=422707} 2024-04-12T11:15:41.537 INFO:tasks.rados.rados.0.smithi029.stdout:335: writing smithi02979277-48 from 705927 to 1264429 tid 1 2024-04-12T11:15:41.544 INFO:tasks.rados.rados.0.smithi029.stdout:335: writing smithi02979277-48 from 2004895 to 2427602 tid 2 2024-04-12T11:15:41.544 INFO:tasks.rados.rados.0.smithi029.stdout:336: snap_create 2024-04-12T11:15:41.544 INFO:tasks.rados.rados.0.smithi029.stdout:334: finishing rollback tid 1 to smithi02979277-2 2024-04-12T11:15:41.544 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 0 (ObjNum 81 snap 0 seq_num 1800831200) dirty dne 2024-04-12T11:15:41.544 INFO:tasks.rados.rados.0.smithi029.stdout:335: finishing write tid 1 to smithi02979277-48 2024-04-12T11:15:41.547 INFO:tasks.rados.rados.0.smithi029.stdout:335: finishing write tid 2 to smithi02979277-48 2024-04-12T11:15:41.547 INFO:tasks.rados.rados.0.smithi029.stdout:335: finishing write tid 3 to smithi02979277-48 2024-04-12T11:15:41.547 INFO:tasks.rados.rados.0.smithi029.stdout:335: finishing write tid 4 to smithi02979277-48 2024-04-12T11:15:41.547 INFO:tasks.rados.rados.0.smithi029.stdout:335: oid 48 updating version 0 to 120 2024-04-12T11:15:41.547 INFO:tasks.rados.rados.0.smithi029.stdout:335: oid 48 version 120 is already newer than 119 2024-04-12T11:15:41.547 INFO:tasks.rados.rados.0.smithi029.stdout:335: oid 48 version 120 is already newer than 118 2024-04-12T11:15:41.547 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 120 (ObjNum 161 snap 42 seq_num 161) dirty exists 2024-04-12T11:15:41.547 INFO:tasks.rados.rados.0.smithi029.stdout:335: left oid 48 (ObjNum 161 snap 42 seq_num 161) 2024-04-12T11:15:41.805 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[39999]: pgmap v199: 89 pgs: 89 active+clean; 149 MiB data, 517 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-04-12T11:15:41.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[39999]: osdmap e149: 8 total, 8 up, 8 in 2024-04-12T11:15:41.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3720179827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:41.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:41.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:41.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:15:41.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[39999]: osdmap e150: 8 total, 8 up, 8 in 2024-04-12T11:15:41.806 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[29349]: pgmap v199: 89 pgs: 89 active+clean; 149 MiB data, 517 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-04-12T11:15:41.806 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[29349]: osdmap e149: 8 total, 8 up, 8 in 2024-04-12T11:15:41.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3720179827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:41.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:41.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:41.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:15:41.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:41 smithi029 ceph-mon[29349]: osdmap e150: 8 total, 8 up, 8 in 2024-04-12T11:15:41.822 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:41 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:41.553+0000 7fef0a014200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:15:41.822 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:41 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:41.665+0000 7fef0a014200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:15:41.822 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:41 smithi087 ceph-mon[32917]: pgmap v199: 89 pgs: 89 active+clean; 149 MiB data, 517 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-04-12T11:15:41.823 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:41 smithi087 ceph-mon[32917]: osdmap e149: 8 total, 8 up, 8 in 2024-04-12T11:15:41.823 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:41 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3720179827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:41.823 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:41 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:41.823 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:41 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:41.823 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:41 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:15:41.823 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:41 smithi087 ceph-mon[32917]: osdmap e150: 8 total, 8 up, 8 in 2024-04-12T11:15:42.486 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:42.346+0000 7fef0a014200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:15:42.486 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:42.442+0000 7fef0a014200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:15:42.543 INFO:tasks.rados.rados.0.smithi029.stdout:334: done (2 left) 2024-04-12T11:15:42.543 INFO:tasks.rados.rados.0.smithi029.stdout:335: done (1 left) 2024-04-12T11:15:42.543 INFO:tasks.rados.rados.0.smithi029.stdout:336: done (0 left) 2024-04-12T11:15:42.543 INFO:tasks.rados.rados.0.smithi029.stdout:337: read oid 36 snap -1 2024-04-12T11:15:42.543 INFO:tasks.rados.rados.0.smithi029.stdout:337: expect (ObjNum 134 snap 28 seq_num 134) 2024-04-12T11:15:42.544 INFO:tasks.rados.rados.0.smithi029.stdout:338: delete oid 35 current snap is 43 2024-04-12T11:15:42.545 INFO:tasks.rados.rados.0.smithi029.stdout:338: done (1 left) 2024-04-12T11:15:42.545 INFO:tasks.rados.rados.0.smithi029.stdout:339: rollback oid 35 current snap is 43 2024-04-12T11:15:42.545 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 35 to 35 2024-04-12T11:15:42.545 INFO:tasks.rados.rados.0.smithi029.stdout:340: write oid 25 current snap is 43 2024-04-12T11:15:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:340: seq_num 163 ranges {739541=709000,2121859=426376} 2024-04-12T11:15:42.553 INFO:tasks.rados.rados.0.smithi029.stdout:340: writing smithi02979277-25 from 739541 to 1448541 tid 1 2024-04-12T11:15:42.560 INFO:tasks.rados.rados.0.smithi029.stdout:340: writing smithi02979277-25 from 2121859 to 2548235 tid 2 2024-04-12T11:15:42.560 INFO:tasks.rados.rados.0.smithi029.stdout:341: rollback oid 29 current snap is 43 2024-04-12T11:15:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 29 to 40 2024-04-12T11:15:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:339: finishing rollback tid 1 to smithi02979277-35 2024-04-12T11:15:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 0 (ObjNum 49 snap 0 seq_num 862096796) dirty dne 2024-04-12T11:15:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:337: expect (ObjNum 134 snap 28 seq_num 134) 2024-04-12T11:15:42.635 INFO:tasks.rados.rados.0.smithi029.stdout:340: finishing write tid 1 to smithi02979277-25 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:341: finishing rollback tid 0 to smithi02979277-29 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:340: finishing write tid 2 to smithi02979277-25 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:341: finishing rollback tid 1 to smithi02979277-29 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:340: finishing write tid 3 to smithi02979277-25 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:340: finishing write tid 4 to smithi02979277-25 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:340: oid 25 updating version 0 to 114 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:340: oid 25 updating version 114 to 115 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:340: oid 25 version 115 is already newer than 113 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 115 (ObjNum 163 snap 43 seq_num 163) dirty exists 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:340: left oid 25 (ObjNum 163 snap 43 seq_num 163) 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:341: finishing rollback tid 2 to smithi02979277-29 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 88 (ObjNum 155 snap 38 seq_num 155) dirty exists 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:337: done (3 left) 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:339: done (2 left) 2024-04-12T11:15:42.636 INFO:tasks.rados.rados.0.smithi029.stdout:340: done (1 left) 2024-04-12T11:15:42.637 INFO:tasks.rados.rados.0.smithi029.stdout:341: done (0 left) 2024-04-12T11:15:42.637 INFO:tasks.rados.rados.0.smithi029.stdout:342: delete oid 6 current snap is 43 2024-04-12T11:15:42.637 INFO:tasks.rados.rados.0.smithi029.stdout:342: done (0 left) 2024-04-12T11:15:42.637 INFO:tasks.rados.rados.0.smithi029.stdout:343: delete oid 3 current snap is 43 2024-04-12T11:15:42.639 INFO:tasks.rados.rados.0.smithi029.stdout:343: done (0 left) 2024-04-12T11:15:42.639 INFO:tasks.rados.rados.0.smithi029.stdout:344: write oid 7 current snap is 43 2024-04-12T11:15:42.639 INFO:tasks.rados.rados.0.smithi029.stdout:344: seq_num 166 ranges {789962=534747,1780952=690056} 2024-04-12T11:15:42.646 INFO:tasks.rados.rados.0.smithi029.stdout:344: writing smithi02979277-7 from 789962 to 1324709 tid 1 2024-04-12T11:15:42.653 INFO:tasks.rados.rados.0.smithi029.stdout:344: writing smithi02979277-7 from 1780952 to 2471008 tid 2 2024-04-12T11:15:42.653 INFO:tasks.rados.rados.0.smithi029.stdout:345: read oid 31 snap -1 2024-04-12T11:15:42.653 INFO:tasks.rados.rados.0.smithi029.stdout:345: expect (ObjNum 138 snap 30 seq_num 138) 2024-04-12T11:15:42.653 INFO:tasks.rados.rados.0.smithi029.stdout:346: write oid 12 current snap is 43 2024-04-12T11:15:42.653 INFO:tasks.rados.rados.0.smithi029.stdout:344: finishing write tid 1 to smithi02979277-7 2024-04-12T11:15:42.654 INFO:tasks.rados.rados.0.smithi029.stdout:346: seq_num 167 ranges {406769=609996,1439434=433714,2303522=503247} 2024-04-12T11:15:42.659 INFO:tasks.rados.rados.0.smithi029.stdout:346: writing smithi02979277-12 from 406769 to 1016765 tid 1 2024-04-12T11:15:42.663 INFO:tasks.rados.rados.0.smithi029.stdout:346: writing smithi02979277-12 from 1439434 to 1873148 tid 2 2024-04-12T11:15:42.669 INFO:tasks.rados.rados.0.smithi029.stdout:346: writing smithi02979277-12 from 2303522 to 2806769 tid 3 2024-04-12T11:15:42.669 INFO:tasks.rados.rados.0.smithi029.stdout:347: rollback oid 36 current snap is 43 2024-04-12T11:15:42.669 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 36 to 40 2024-04-12T11:15:42.669 INFO:tasks.rados.rados.0.smithi029.stdout:344: finishing write tid 2 to smithi02979277-7 2024-04-12T11:15:42.670 INFO:tasks.rados.rados.0.smithi029.stdout:344: finishing write tid 3 to smithi02979277-7 2024-04-12T11:15:42.670 INFO:tasks.rados.rados.0.smithi029.stdout:344: finishing write tid 4 to smithi02979277-7 2024-04-12T11:15:42.670 INFO:tasks.rados.rados.0.smithi029.stdout:344: oid 7 updating version 0 to 137 2024-04-12T11:15:42.670 INFO:tasks.rados.rados.0.smithi029.stdout:344: oid 7 version 137 is already newer than 136 2024-04-12T11:15:42.670 INFO:tasks.rados.rados.0.smithi029.stdout:344: oid 7 updating version 137 to 138 2024-04-12T11:15:42.670 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 138 (ObjNum 166 snap 43 seq_num 166) dirty exists 2024-04-12T11:15:42.670 INFO:tasks.rados.rados.0.smithi029.stdout:344: left oid 7 (ObjNum 166 snap 43 seq_num 166) 2024-04-12T11:15:42.670 INFO:tasks.rados.rados.0.smithi029.stdout:345: expect (ObjNum 138 snap 30 seq_num 138) 2024-04-12T11:15:42.775 INFO:tasks.rados.rados.0.smithi029.stdout:346: finishing write tid 1 to smithi02979277-12 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:346: finishing write tid 2 to smithi02979277-12 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:344: done (3 left) 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:345: done (2 left) 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:348: snap_create 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:346: finishing write tid 3 to smithi02979277-12 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:347: finishing rollback tid 0 to smithi02979277-36 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:346: finishing write tid 4 to smithi02979277-12 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:346: finishing write tid 5 to smithi02979277-12 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:346: oid 12 updating version 0 to 142 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:346: oid 12 version 142 is already newer than 140 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:346: oid 12 updating version 142 to 143 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:346: oid 12 version 143 is already newer than 141 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 143 (ObjNum 167 snap 43 seq_num 167) dirty exists 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:346: left oid 12 (ObjNum 167 snap 43 seq_num 167) 2024-04-12T11:15:42.776 INFO:tasks.rados.rados.0.smithi029.stdout:347: finishing rollback tid 1 to smithi02979277-36 2024-04-12T11:15:42.777 INFO:tasks.rados.rados.0.smithi029.stdout:347: finishing rollback tid 2 to smithi02979277-36 2024-04-12T11:15:42.777 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 87 (ObjNum 134 snap 28 seq_num 134) dirty exists 2024-04-12T11:15:42.814 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:42.599+0000 7fef0a014200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:15:42.814 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:42.728+0000 7fef0a014200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:15:42.815 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:42.814+0000 7fef0a014200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:15:43.396 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:15:43 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:15:43] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.43.0" 2024-04-12T11:15:43.404 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:43.099+0000 7fef0a014200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:15:43.405 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:43.222+0000 7fef0a014200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:15:43.405 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:43.312+0000 7fef0a014200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:15:43.544 INFO:tasks.rados.rados.0.smithi029.stdout:346: done (2 left) 2024-04-12T11:15:43.545 INFO:tasks.rados.rados.0.smithi029.stdout:347: done (1 left) 2024-04-12T11:15:43.545 INFO:tasks.rados.rados.0.smithi029.stdout:348: done (0 left) 2024-04-12T11:15:43.545 INFO:tasks.rados.rados.0.smithi029.stdout:349: write oid 13 current snap is 44 2024-04-12T11:15:43.545 INFO:tasks.rados.rados.0.smithi029.stdout:349: seq_num 168 ranges {423576=401598,1497916=693725} 2024-04-12T11:15:43.548 INFO:tasks.rados.rados.0.smithi029.stdout:349: writing smithi02979277-13 from 423576 to 825174 tid 1 2024-04-12T11:15:43.556 INFO:tasks.rados.rados.0.smithi029.stdout:349: writing smithi02979277-13 from 1497916 to 2191641 tid 2 2024-04-12T11:15:43.556 INFO:tasks.rados.rados.0.smithi029.stdout:350: write oid 44 current snap is 44 2024-04-12T11:15:43.556 INFO:tasks.rados.rados.0.smithi029.stdout:350: seq_num 169 ranges {440383=476847,1556398=437383,2474721=365662} 2024-04-12T11:15:43.561 INFO:tasks.rados.rados.0.smithi029.stdout:350: writing smithi02979277-44 from 440383 to 917230 tid 1 2024-04-12T11:15:43.567 INFO:tasks.rados.rados.0.smithi029.stdout:350: writing smithi02979277-44 from 1556398 to 1993781 tid 2 2024-04-12T11:15:43.572 INFO:tasks.rados.rados.0.smithi029.stdout:350: writing smithi02979277-44 from 2474721 to 2840383 tid 3 2024-04-12T11:15:43.572 INFO:tasks.rados.rados.0.smithi029.stdout:351: write oid 21 current snap is 44 2024-04-12T11:15:43.572 INFO:tasks.rados.rados.0.smithi029.stdout:349: finishing write tid 1 to smithi02979277-13 2024-04-12T11:15:43.572 INFO:tasks.rados.rados.0.smithi029.stdout:349: finishing write tid 2 to smithi02979277-13 2024-04-12T11:15:43.572 INFO:tasks.rados.rados.0.smithi029.stdout:349: finishing write tid 3 to smithi02979277-13 2024-04-12T11:15:43.572 INFO:tasks.rados.rados.0.smithi029.stdout:349: finishing write tid 4 to smithi02979277-13 2024-04-12T11:15:43.572 INFO:tasks.rados.rados.0.smithi029.stdout:349: oid 13 updating version 0 to 118 2024-04-12T11:15:43.572 INFO:tasks.rados.rados.0.smithi029.stdout:349: oid 13 updating version 118 to 119 2024-04-12T11:15:43.572 INFO:tasks.rados.rados.0.smithi029.stdout:349: oid 13 version 119 is already newer than 117 2024-04-12T11:15:43.573 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 119 (ObjNum 168 snap 44 seq_num 168) dirty exists 2024-04-12T11:15:43.573 INFO:tasks.rados.rados.0.smithi029.stdout:349: left oid 13 (ObjNum 168 snap 44 seq_num 168) 2024-04-12T11:15:43.573 INFO:tasks.rados.rados.0.smithi029.stdout:350: finishing write tid 1 to smithi02979277-44 2024-04-12T11:15:43.573 INFO:tasks.rados.rados.0.smithi029.stdout:350: finishing write tid 2 to smithi02979277-44 2024-04-12T11:15:43.573 INFO:tasks.rados.rados.0.smithi029.stdout:351: seq_num 170 ranges {457190=552096,1614880=581041,2785791=71399} 2024-04-12T11:15:43.577 INFO:tasks.rados.rados.0.smithi029.stdout:351: writing smithi02979277-21 from 457190 to 1009286 tid 1 2024-04-12T11:15:43.584 INFO:tasks.rados.rados.0.smithi029.stdout:351: writing smithi02979277-21 from 1614880 to 2195921 tid 2 2024-04-12T11:15:43.588 INFO:tasks.rados.rados.0.smithi029.stdout:351: writing smithi02979277-21 from 2785791 to 2857190 tid 3 2024-04-12T11:15:43.588 INFO:tasks.rados.rados.0.smithi029.stdout:349: done (2 left) 2024-04-12T11:15:43.588 INFO:tasks.rados.rados.0.smithi029.stdout:352: write oid 7 current snap is 44 2024-04-12T11:15:43.588 INFO:tasks.rados.rados.0.smithi029.stdout:350: finishing write tid 3 to smithi02979277-44 2024-04-12T11:15:43.588 INFO:tasks.rados.rados.0.smithi029.stdout:350: finishing write tid 4 to smithi02979277-44 2024-04-12T11:15:43.588 INFO:tasks.rados.rados.0.smithi029.stdout:350: finishing write tid 5 to smithi02979277-44 2024-04-12T11:15:43.588 INFO:tasks.rados.rados.0.smithi029.stdout:350: oid 44 updating version 0 to 128 2024-04-12T11:15:43.588 INFO:tasks.rados.rados.0.smithi029.stdout:350: oid 44 updating version 128 to 130 2024-04-12T11:15:43.588 INFO:tasks.rados.rados.0.smithi029.stdout:350: oid 44 version 130 is already newer than 129 2024-04-12T11:15:43.588 INFO:tasks.rados.rados.0.smithi029.stdout:350: oid 44 updating version 130 to 131 2024-04-12T11:15:43.589 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 131 (ObjNum 169 snap 44 seq_num 169) dirty exists 2024-04-12T11:15:43.589 INFO:tasks.rados.rados.0.smithi029.stdout:350: left oid 44 (ObjNum 169 snap 44 seq_num 169) 2024-04-12T11:15:43.589 INFO:tasks.rados.rados.0.smithi029.stdout:351: finishing write tid 1 to smithi02979277-21 2024-04-12T11:15:43.589 INFO:tasks.rados.rados.0.smithi029.stdout:351: finishing write tid 2 to smithi02979277-21 2024-04-12T11:15:43.589 INFO:tasks.rados.rados.0.smithi029.stdout:352: seq_num 171 ranges {473997=627345,1673362=441052,2529567=344430} 2024-04-12T11:15:43.593 INFO:tasks.rados.rados.0.smithi029.stdout:352: writing smithi02979277-7 from 473997 to 1101342 tid 1 2024-04-12T11:15:43.599 INFO:tasks.rados.rados.0.smithi029.stdout:352: writing smithi02979277-7 from 1673362 to 2114414 tid 2 2024-04-12T11:15:43.603 INFO:tasks.rados.rados.0.smithi029.stdout:352: writing smithi02979277-7 from 2529567 to 2873997 tid 3 2024-04-12T11:15:43.603 INFO:tasks.rados.rados.0.smithi029.stdout:350: done (2 left) 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:353: snap_create 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:351: finishing write tid 3 to smithi02979277-21 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:351: finishing write tid 4 to smithi02979277-21 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:351: finishing write tid 5 to smithi02979277-21 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:351: oid 21 updating version 0 to 135 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:351: oid 21 updating version 135 to 136 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:351: oid 21 version 136 is already newer than 133 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:351: oid 21 version 136 is already newer than 134 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 136 (ObjNum 170 snap 44 seq_num 170) dirty exists 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:351: left oid 21 (ObjNum 170 snap 44 seq_num 170) 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:352: finishing write tid 1 to smithi02979277-7 2024-04-12T11:15:43.604 INFO:tasks.rados.rados.0.smithi029.stdout:352: finishing write tid 2 to smithi02979277-7 2024-04-12T11:15:43.605 INFO:tasks.rados.rados.0.smithi029.stdout:352: finishing write tid 3 to smithi02979277-7 2024-04-12T11:15:43.606 INFO:tasks.rados.rados.0.smithi029.stdout:352: finishing write tid 4 to smithi02979277-7 2024-04-12T11:15:43.606 INFO:tasks.rados.rados.0.smithi029.stdout:352: finishing write tid 5 to smithi02979277-7 2024-04-12T11:15:43.606 INFO:tasks.rados.rados.0.smithi029.stdout:352: oid 7 updating version 0 to 146 2024-04-12T11:15:43.606 INFO:tasks.rados.rados.0.smithi029.stdout:352: oid 7 updating version 146 to 147 2024-04-12T11:15:43.606 INFO:tasks.rados.rados.0.smithi029.stdout:352: oid 7 updating version 147 to 148 2024-04-12T11:15:43.606 INFO:tasks.rados.rados.0.smithi029.stdout:352: oid 7 version 148 is already newer than 145 2024-04-12T11:15:43.606 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 148 (ObjNum 171 snap 44 seq_num 171) dirty exists 2024-04-12T11:15:43.606 INFO:tasks.rados.rados.0.smithi029.stdout:352: left oid 7 (ObjNum 171 snap 44 seq_num 171) 2024-04-12T11:15:43.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:43.413+0000 7fef0a014200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:15:43.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:43.615+0000 7fef0a014200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:15:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:43 smithi087 ceph-mon[32917]: pgmap v202: 89 pgs: 89 active+clean; 149 MiB data, 517 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-04-12T11:15:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:43 smithi087 ceph-mon[32917]: osdmap e151: 8 total, 8 up, 8 in 2024-04-12T11:15:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:43 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/983907861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:43 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:43 smithi029 ceph-mon[29349]: pgmap v202: 89 pgs: 89 active+clean; 149 MiB data, 517 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-04-12T11:15:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:43 smithi029 ceph-mon[29349]: osdmap e151: 8 total, 8 up, 8 in 2024-04-12T11:15:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:43 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/983907861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:43 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:43 smithi029 ceph-mon[39999]: pgmap v202: 89 pgs: 89 active+clean; 149 MiB data, 517 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-04-12T11:15:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:43 smithi029 ceph-mon[39999]: osdmap e151: 8 total, 8 up, 8 in 2024-04-12T11:15:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:43 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/983907861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:43 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:44.552 INFO:tasks.rados.rados.0.smithi029.stdout:351: done (2 left) 2024-04-12T11:15:44.552 INFO:tasks.rados.rados.0.smithi029.stdout:352: done (1 left) 2024-04-12T11:15:44.552 INFO:tasks.rados.rados.0.smithi029.stdout:353: done (0 left) 2024-04-12T11:15:44.552 INFO:tasks.rados.rados.0.smithi029.stdout:354: rollback oid 19 current snap is 45 2024-04-12T11:15:44.553 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 19 to 38 2024-04-12T11:15:44.553 INFO:tasks.rados.rados.0.smithi029.stdout:355: read oid 36 snap -1 2024-04-12T11:15:44.553 INFO:tasks.rados.rados.0.smithi029.stdout:355: expect (ObjNum 134 snap 28 seq_num 134) 2024-04-12T11:15:44.553 INFO:tasks.rados.rados.0.smithi029.stdout:356: write oid 27 current snap is 45 2024-04-12T11:15:44.553 INFO:tasks.rados.rados.0.smithi029.stdout:356: seq_num 172 ranges {490804=702594,1615491=584710,2724284=166520} 2024-04-12T11:15:44.559 INFO:tasks.rados.rados.0.smithi029.stdout:356: writing smithi02979277-27 from 490804 to 1193398 tid 1 2024-04-12T11:15:44.564 INFO:tasks.rados.rados.0.smithi029.stdout:356: writing smithi02979277-27 from 1615491 to 2200201 tid 2 2024-04-12T11:15:44.569 INFO:tasks.rados.rados.0.smithi029.stdout:356: writing smithi02979277-27 from 2724284 to 2890804 tid 3 2024-04-12T11:15:44.569 INFO:tasks.rados.rados.0.smithi029.stdout:357: snap_create 2024-04-12T11:15:44.569 INFO:tasks.rados.rados.0.smithi029.stdout:354: finishing rollback tid 0 to smithi02979277-19 2024-04-12T11:15:44.569 INFO:tasks.rados.rados.0.smithi029.stdout:354: finishing rollback tid 1 to smithi02979277-19 2024-04-12T11:15:44.569 INFO:tasks.rados.rados.0.smithi029.stdout:354: finishing rollback tid 2 to smithi02979277-19 2024-04-12T11:15:44.569 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 125 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-04-12T11:15:44.569 INFO:tasks.rados.rados.0.smithi029.stdout:355: expect (ObjNum 134 snap 28 seq_num 134) 2024-04-12T11:15:44.642 INFO:tasks.rados.rados.0.smithi029.stdout:356: finishing write tid 1 to smithi02979277-27 2024-04-12T11:15:44.642 INFO:tasks.rados.rados.0.smithi029.stdout:356: finishing write tid 2 to smithi02979277-27 2024-04-12T11:15:44.642 INFO:tasks.rados.rados.0.smithi029.stdout:356: finishing write tid 3 to smithi02979277-27 2024-04-12T11:15:44.642 INFO:tasks.rados.rados.0.smithi029.stdout:356: finishing write tid 4 to smithi02979277-27 2024-04-12T11:15:44.642 INFO:tasks.rados.rados.0.smithi029.stdout:356: finishing write tid 5 to smithi02979277-27 2024-04-12T11:15:44.642 INFO:tasks.rados.rados.0.smithi029.stdout:356: oid 27 updating version 0 to 115 2024-04-12T11:15:44.642 INFO:tasks.rados.rados.0.smithi029.stdout:356: oid 27 version 115 is already newer than 113 2024-04-12T11:15:44.643 INFO:tasks.rados.rados.0.smithi029.stdout:356: oid 27 version 115 is already newer than 114 2024-04-12T11:15:44.643 INFO:tasks.rados.rados.0.smithi029.stdout:356: oid 27 updating version 115 to 116 2024-04-12T11:15:44.643 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 116 (ObjNum 172 snap 45 seq_num 172) dirty exists 2024-04-12T11:15:44.643 INFO:tasks.rados.rados.0.smithi029.stdout:356: left oid 27 (ObjNum 172 snap 45 seq_num 172) 2024-04-12T11:15:44.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:44 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:44.604+0000 7fef0a014200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:15:44.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:44 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:44.737+0000 7fef0a014200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:15:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:44 smithi087 ceph-mon[32917]: osdmap e152: 8 total, 8 up, 8 in 2024-04-12T11:15:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:44 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:44 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:45.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:44 smithi029 ceph-mon[29349]: osdmap e152: 8 total, 8 up, 8 in 2024-04-12T11:15:45.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:44 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:45.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:44 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:44 smithi029 ceph-mon[39999]: osdmap e152: 8 total, 8 up, 8 in 2024-04-12T11:15:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:44 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:44 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:45.366 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:45.075+0000 7fef0a014200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:15:45.366 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:45.166+0000 7fef0a014200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:15:45.366 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:45.366+0000 7fef0a014200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:15:45.556 INFO:tasks.rados.rados.0.smithi029.stdout:354: done (3 left) 2024-04-12T11:15:45.556 INFO:tasks.rados.rados.0.smithi029.stdout:355: done (2 left) 2024-04-12T11:15:45.556 INFO:tasks.rados.rados.0.smithi029.stdout:356: done (1 left) 2024-04-12T11:15:45.556 INFO:tasks.rados.rados.0.smithi029.stdout:357: done (0 left) 2024-04-12T11:15:45.556 INFO:tasks.rados.rados.0.smithi029.stdout:358: write oid 28 current snap is 46 2024-04-12T11:15:45.557 INFO:tasks.rados.rados.0.smithi029.stdout:358: seq_num 173 ranges {507611=777843,2073973=444721} 2024-04-12T11:15:45.563 INFO:tasks.rados.rados.0.smithi029.stdout:358: writing smithi02979277-28 from 507611 to 1285454 tid 1 2024-04-12T11:15:45.570 INFO:tasks.rados.rados.0.smithi029.stdout:358: writing smithi02979277-28 from 2073973 to 2518694 tid 2 2024-04-12T11:15:45.570 INFO:tasks.rados.rados.0.smithi029.stdout:359: snap_create 2024-04-12T11:15:45.570 INFO:tasks.rados.rados.0.smithi029.stdout:358: finishing write tid 1 to smithi02979277-28 2024-04-12T11:15:45.572 INFO:tasks.rados.rados.0.smithi029.stdout:358: finishing write tid 2 to smithi02979277-28 2024-04-12T11:15:45.572 INFO:tasks.rados.rados.0.smithi029.stdout:358: finishing write tid 3 to smithi02979277-28 2024-04-12T11:15:45.572 INFO:tasks.rados.rados.0.smithi029.stdout:358: finishing write tid 4 to smithi02979277-28 2024-04-12T11:15:45.572 INFO:tasks.rados.rados.0.smithi029.stdout:358: oid 28 updating version 0 to 123 2024-04-12T11:15:45.572 INFO:tasks.rados.rados.0.smithi029.stdout:358: oid 28 version 123 is already newer than 122 2024-04-12T11:15:45.572 INFO:tasks.rados.rados.0.smithi029.stdout:358: oid 28 updating version 123 to 124 2024-04-12T11:15:45.572 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 124 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-04-12T11:15:45.572 INFO:tasks.rados.rados.0.smithi029.stdout:358: left oid 28 (ObjNum 173 snap 46 seq_num 173) 2024-04-12T11:15:45.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:45.555+0000 7fef0a014200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:15:45.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:45 smithi087 ceph-mon[32917]: pgmap v205: 89 pgs: 89 active+clean; 162 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:15:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:45 smithi087 ceph-mon[32917]: osdmap e153: 8 total, 8 up, 8 in 2024-04-12T11:15:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:45 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1876966801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:46.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:45 smithi029 ceph-mon[29349]: pgmap v205: 89 pgs: 89 active+clean; 162 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:15:46.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:45 smithi029 ceph-mon[29349]: osdmap e153: 8 total, 8 up, 8 in 2024-04-12T11:15:46.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:45 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1876966801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:46.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:45 smithi029 ceph-mon[39999]: pgmap v205: 89 pgs: 89 active+clean; 162 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:15:46.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:45 smithi029 ceph-mon[39999]: osdmap e153: 8 total, 8 up, 8 in 2024-04-12T11:15:46.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:45 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1876966801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:46.202 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:45 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:45.847+0000 7fef0a014200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:15:46.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:46.107+0000 7fef0a014200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:15:46.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:46.346+0000 7fef0a014200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:15:46.532 INFO:tasks.rados.rados.0.smithi029.stdout:358: done (1 left) 2024-04-12T11:15:46.532 INFO:tasks.rados.rados.0.smithi029.stdout:359: done (0 left) 2024-04-12T11:15:46.532 INFO:tasks.rados.rados.0.smithi029.stdout:360: snap_remove snap 45 2024-04-12T11:15:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:46 smithi087 ceph-mon[32917]: osdmap e154: 8 total, 8 up, 8 in 2024-04-12T11:15:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:46 smithi087 ceph-mon[32917]: osdmap e155: 8 total, 8 up, 8 in 2024-04-12T11:15:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:46 smithi029 ceph-mon[29349]: osdmap e154: 8 total, 8 up, 8 in 2024-04-12T11:15:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:46 smithi029 ceph-mon[29349]: osdmap e155: 8 total, 8 up, 8 in 2024-04-12T11:15:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:46 smithi029 ceph-mon[39999]: osdmap e154: 8 total, 8 up, 8 in 2024-04-12T11:15:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:46 smithi029 ceph-mon[39999]: osdmap e155: 8 total, 8 up, 8 in 2024-04-12T11:15:47.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:47.185+0000 7fef0a014200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:15:47.534 INFO:tasks.rados.rados.0.smithi029.stdout:360: done (0 left) 2024-04-12T11:15:47.534 INFO:tasks.rados.rados.0.smithi029.stdout:361: read oid 32 snap -1 2024-04-12T11:15:47.534 INFO:tasks.rados.rados.0.smithi029.stdout:361: expect (ObjNum 86 snap 11 seq_num 86) 2024-04-12T11:15:47.534 INFO:tasks.rados.rados.0.smithi029.stdout:362: read oid 10 snap 43 2024-04-12T11:15:47.535 INFO:tasks.rados.rados.0.smithi029.stdout:362: expect deleted 2024-04-12T11:15:47.535 INFO:tasks.rados.rados.0.smithi029.stdout:363: read oid 26 snap -1 2024-04-12T11:15:47.535 INFO:tasks.rados.rados.0.smithi029.stdout:363: expect (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:15:47.535 INFO:tasks.rados.rados.0.smithi029.stdout:364: read oid 2 snap -1 2024-04-12T11:15:47.535 INFO:tasks.rados.rados.0.smithi029.stdout:364: expect deleted 2024-04-12T11:15:47.536 INFO:tasks.rados.rados.0.smithi029.stdout:365: delete oid 40 current snap is 47 2024-04-12T11:15:47.539 INFO:tasks.rados.rados.0.smithi029.stdout:361: expect (ObjNum 86 snap 11 seq_num 86) 2024-04-12T11:15:47.694 INFO:tasks.rados.rados.0.smithi029.stdout:363: expect (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:15:47.810 INFO:tasks.rados.rados.0.smithi029.stdout:361: done (4 left) 2024-04-12T11:15:47.810 INFO:tasks.rados.rados.0.smithi029.stdout:362: done (3 left) 2024-04-12T11:15:47.810 INFO:tasks.rados.rados.0.smithi029.stdout:363: done (2 left) 2024-04-12T11:15:47.810 INFO:tasks.rados.rados.0.smithi029.stdout:364: done (1 left) 2024-04-12T11:15:47.810 INFO:tasks.rados.rados.0.smithi029.stdout:365: done (0 left) 2024-04-12T11:15:47.810 INFO:tasks.rados.rados.0.smithi029.stdout:366: read oid 10 snap -1 2024-04-12T11:15:47.811 INFO:tasks.rados.rados.0.smithi029.stdout:366: expect deleted 2024-04-12T11:15:47.811 INFO:tasks.rados.rados.0.smithi029.stdout:367: rollback oid 8 current snap is 47 2024-04-12T11:15:47.811 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 38 2024-04-12T11:15:47.811 INFO:tasks.rados.rados.0.smithi029.stdout:368: rollback oid 40 current snap is 47 2024-04-12T11:15:47.811 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 40 to 43 2024-04-12T11:15:47.811 INFO:tasks.rados.rados.0.smithi029.stdout:369: write oid 50 current snap is 47 2024-04-12T11:15:47.811 INFO:tasks.rados.rados.0.smithi029.stdout:369: seq_num 175 ranges {541225=644694,1907290=448390} 2024-04-12T11:15:47.816 INFO:tasks.rados.rados.0.smithi029.stdout:369: writing smithi02979277-50 from 541225 to 1185919 tid 1 2024-04-12T11:15:47.823 INFO:tasks.rados.rados.0.smithi029.stdout:369: writing smithi02979277-50 from 1907290 to 2355680 tid 2 2024-04-12T11:15:47.823 INFO:tasks.rados.rados.0.smithi029.stdout:370: write oid 35 current snap is 47 2024-04-12T11:15:47.823 INFO:tasks.rados.rados.0.smithi029.stdout:370: seq_num 176 ranges {558032=719943,1849419=592048} 2024-04-12T11:15:47.829 INFO:tasks.rados.rados.0.smithi029.stdout:370: writing smithi02979277-35 from 558032 to 1277975 tid 1 2024-04-12T11:15:47.836 INFO:tasks.rados.rados.0.smithi029.stdout:370: writing smithi02979277-35 from 1849419 to 2441467 tid 2 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:371: read oid 21 snap -1 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:371: expect (ObjNum 170 snap 44 seq_num 170) 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:372: delete oid 34 current snap is 47 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:367: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:368: finishing rollback tid 1 to smithi02979277-40 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:368: finishing rollback tid 2 to smithi02979277-40 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 173 (ObjNum 158 snap 40 seq_num 158) dirty exists 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:369: finishing write tid 1 to smithi02979277-50 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:369: finishing write tid 2 to smithi02979277-50 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:369: finishing write tid 3 to smithi02979277-50 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:369: finishing write tid 4 to smithi02979277-50 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:369: oid 50 updating version 0 to 134 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:369: oid 50 updating version 134 to 136 2024-04-12T11:15:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:369: oid 50 version 136 is already newer than 135 2024-04-12T11:15:47.838 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 136 (ObjNum 175 snap 47 seq_num 175) dirty exists 2024-04-12T11:15:47.838 INFO:tasks.rados.rados.0.smithi029.stdout:369: left oid 50 (ObjNum 175 snap 47 seq_num 175) 2024-04-12T11:15:47.838 INFO:tasks.rados.rados.0.smithi029.stdout:370: finishing write tid 1 to smithi02979277-35 2024-04-12T11:15:47.839 INFO:tasks.rados.rados.0.smithi029.stdout:366: done (6 left) 2024-04-12T11:15:47.839 INFO:tasks.rados.rados.0.smithi029.stdout:367: done (5 left) 2024-04-12T11:15:47.839 INFO:tasks.rados.rados.0.smithi029.stdout:368: done (4 left) 2024-04-12T11:15:47.839 INFO:tasks.rados.rados.0.smithi029.stdout:369: done (3 left) 2024-04-12T11:15:47.840 INFO:tasks.rados.rados.0.smithi029.stdout:372: done (2 left) 2024-04-12T11:15:47.840 INFO:tasks.rados.rados.0.smithi029.stdout:373: write oid 11 current snap is 47 2024-04-12T11:15:47.840 INFO:tasks.rados.rados.0.smithi029.stdout:373: seq_num 178 ranges {591646=470441,1566383=595717,2888822=102824} 2024-04-12T11:15:47.845 INFO:tasks.rados.rados.0.smithi029.stdout:373: writing smithi02979277-11 from 591646 to 1062087 tid 1 2024-04-12T11:15:47.851 INFO:tasks.rados.rados.0.smithi029.stdout:373: writing smithi02979277-11 from 1566383 to 2162100 tid 2 2024-04-12T11:15:47.856 INFO:tasks.rados.rados.0.smithi029.stdout:373: writing smithi02979277-11 from 2888822 to 2991646 tid 3 2024-04-12T11:15:47.856 INFO:tasks.rados.rados.0.smithi029.stdout:374: write oid 45 current snap is 47 2024-04-12T11:15:47.856 INFO:tasks.rados.rados.0.smithi029.stdout:374: seq_num 179 ranges {608453=545690,1624865=455728,2632598=375855} 2024-04-12T11:15:47.862 INFO:tasks.rados.rados.0.smithi029.stdout:374: writing smithi02979277-45 from 608453 to 1154143 tid 1 2024-04-12T11:15:47.867 INFO:tasks.rados.rados.0.smithi029.stdout:374: writing smithi02979277-45 from 1624865 to 2080593 tid 2 2024-04-12T11:15:47.873 INFO:tasks.rados.rados.0.smithi029.stdout:374: writing smithi02979277-45 from 2632598 to 3008453 tid 3 2024-04-12T11:15:47.873 INFO:tasks.rados.rados.0.smithi029.stdout:375: rollback oid 40 current snap is 47 2024-04-12T11:15:47.873 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 40 to 46 2024-04-12T11:15:47.873 INFO:tasks.rados.rados.0.smithi029.stdout:370: finishing write tid 2 to smithi02979277-35 2024-04-12T11:15:47.873 INFO:tasks.rados.rados.0.smithi029.stdout:370: finishing write tid 3 to smithi02979277-35 2024-04-12T11:15:47.873 INFO:tasks.rados.rados.0.smithi029.stdout:370: finishing write tid 4 to smithi02979277-35 2024-04-12T11:15:47.873 INFO:tasks.rados.rados.0.smithi029.stdout:370: oid 35 updating version 0 to 176 2024-04-12T11:15:47.873 INFO:tasks.rados.rados.0.smithi029.stdout:370: oid 35 version 176 is already newer than 174 2024-04-12T11:15:47.874 INFO:tasks.rados.rados.0.smithi029.stdout:370: oid 35 version 176 is already newer than 175 2024-04-12T11:15:47.874 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 176 (ObjNum 176 snap 47 seq_num 176) dirty exists 2024-04-12T11:15:47.874 INFO:tasks.rados.rados.0.smithi029.stdout:370: left oid 35 (ObjNum 176 snap 47 seq_num 176) 2024-04-12T11:15:47.874 INFO:tasks.rados.rados.0.smithi029.stdout:371: expect (ObjNum 170 snap 44 seq_num 170) 2024-04-12T11:15:47.874 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:47.765+0000 7fef0a014200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:15:47.874 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:47 smithi087 ceph-mon[32917]: pgmap v208: 89 pgs: 89 active+clean; 162 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:15:47.874 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:47 smithi087 ceph-mon[32917]: osdmap e156: 8 total, 8 up, 8 in 2024-04-12T11:15:48.003 INFO:tasks.rados.rados.0.smithi029.stdout:373: finishing write tid 1 to smithi02979277-11 2024-04-12T11:15:48.003 INFO:tasks.rados.rados.0.smithi029.stdout:373: finishing write tid 2 to smithi02979277-11 2024-04-12T11:15:48.003 INFO:tasks.rados.rados.0.smithi029.stdout:373: finishing write tid 3 to smithi02979277-11 2024-04-12T11:15:48.003 INFO:tasks.rados.rados.0.smithi029.stdout:373: finishing write tid 4 to smithi02979277-11 2024-04-12T11:15:48.003 INFO:tasks.rados.rados.0.smithi029.stdout:373: finishing write tid 5 to smithi02979277-11 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:373: oid 11 updating version 0 to 180 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:373: oid 11 version 180 is already newer than 178 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:373: oid 11 version 180 is already newer than 179 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:373: oid 11 updating version 180 to 181 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 181 (ObjNum 178 snap 47 seq_num 178) dirty exists 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:373: left oid 11 (ObjNum 178 snap 47 seq_num 178) 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:374: finishing write tid 1 to smithi02979277-45 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:370: done (4 left) 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:371: done (3 left) 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:373: done (2 left) 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:376: delete oid 33 current snap is 47 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:374: finishing write tid 2 to smithi02979277-45 2024-04-12T11:15:48.004 INFO:tasks.rados.rados.0.smithi029.stdout:375: finishing rollback tid 0 to smithi02979277-40 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:375: finishing rollback tid 1 to smithi02979277-40 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:374: finishing write tid 3 to smithi02979277-45 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:375: finishing rollback tid 2 to smithi02979277-40 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 184 (ObjNum 158 snap 40 seq_num 158) dirty exists 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:374: finishing write tid 4 to smithi02979277-45 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:374: finishing write tid 5 to smithi02979277-45 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:374: oid 45 updating version 0 to 151 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:374: oid 45 version 151 is already newer than 150 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:374: oid 45 updating version 151 to 153 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:374: oid 45 version 153 is already newer than 152 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 153 (ObjNum 179 snap 47 seq_num 179) dirty exists 2024-04-12T11:15:48.005 INFO:tasks.rados.rados.0.smithi029.stdout:374: left oid 45 (ObjNum 179 snap 47 seq_num 179) 2024-04-12T11:15:48.006 INFO:tasks.rados.rados.0.smithi029.stdout:374: done (2 left) 2024-04-12T11:15:48.006 INFO:tasks.rados.rados.0.smithi029.stdout:375: done (1 left) 2024-04-12T11:15:48.006 INFO:tasks.rados.rados.0.smithi029.stdout:376: done (0 left) 2024-04-12T11:15:48.006 INFO:tasks.rados.rados.0.smithi029.stdout:377: snap_remove snap 37 2024-04-12T11:15:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:47 smithi029 ceph-mon[29349]: pgmap v208: 89 pgs: 89 active+clean; 162 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:15:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:47 smithi029 ceph-mon[29349]: osdmap e156: 8 total, 8 up, 8 in 2024-04-12T11:15:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:47 smithi029 ceph-mon[39999]: pgmap v208: 89 pgs: 89 active+clean; 162 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:15:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:47 smithi029 ceph-mon[39999]: osdmap e156: 8 total, 8 up, 8 in 2024-04-12T11:15:48.202 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:47.873+0000 7fef0a014200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:15:48.202 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:48.093+0000 7fef0a014200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:15:48.202 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:15:48] ENGINE Bus STARTING 2024-04-12T11:15:48.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: CherryPy Checker: 2024-04-12T11:15:48.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: The Application mounted at '' has an empty config. 2024-04-12T11:15:48.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:15:48.467 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:15:48] ENGINE Serving on http://:::9283 2024-04-12T11:15:48.467 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:15:48] ENGINE Bus STARTED 2024-04-12T11:15:48.576 INFO:tasks.rados.rados.0.smithi029.stdout:377: done (0 left) 2024-04-12T11:15:48.576 INFO:tasks.rados.rados.0.smithi029.stdout:378: read oid 7 snap -1 2024-04-12T11:15:48.576 INFO:tasks.rados.rados.0.smithi029.stdout:378: expect (ObjNum 171 snap 44 seq_num 171) 2024-04-12T11:15:48.576 INFO:tasks.rados.rados.0.smithi029.stdout:379: delete oid 3 current snap is 47 2024-04-12T11:15:48.581 INFO:tasks.rados.rados.0.smithi029.stdout:379: done (1 left) 2024-04-12T11:15:48.581 INFO:tasks.rados.rados.0.smithi029.stdout:380: snap_create 2024-04-12T11:15:48.582 INFO:tasks.rados.rados.0.smithi029.stdout:378: expect (ObjNum 171 snap 44 seq_num 171) 2024-04-12T11:15:48.962 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1553146733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:48.962 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-mon[32917]: Standby manager daemon x restarted 2024-04-12T11:15:48.962 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-mon[32917]: Standby manager daemon x started 2024-04-12T11:15:48.962 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:15:48.963 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:15:48.963 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:15:48.963 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:48 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:15:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1553146733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[29349]: Standby manager daemon x restarted 2024-04-12T11:15:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[29349]: Standby manager daemon x started 2024-04-12T11:15:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:15:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:15:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:15:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:15:49.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1553146733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:49.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[39999]: Standby manager daemon x restarted 2024-04-12T11:15:49.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[39999]: Standby manager daemon x started 2024-04-12T11:15:49.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:15:49.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:15:49.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:15:49.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:48 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:15:49.575 INFO:tasks.rados.rados.0.smithi029.stdout:378: done (1 left) 2024-04-12T11:15:49.575 INFO:tasks.rados.rados.0.smithi029.stdout:380: done (0 left) 2024-04-12T11:15:49.575 INFO:tasks.rados.rados.0.smithi029.stdout:381: read oid 34 snap -1 2024-04-12T11:15:49.576 INFO:tasks.rados.rados.0.smithi029.stdout:381: expect deleted 2024-04-12T11:15:49.576 INFO:tasks.rados.rados.0.smithi029.stdout:382: read oid 47 snap -1 2024-04-12T11:15:49.576 INFO:tasks.rados.rados.0.smithi029.stdout:382: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:15:49.576 INFO:tasks.rados.rados.0.smithi029.stdout:383: write oid 25 current snap is 48 2024-04-12T11:15:49.576 INFO:tasks.rados.rados.0.smithi029.stdout:383: seq_num 182 ranges {658874=771437,2083958=603055} 2024-04-12T11:15:49.583 INFO:tasks.rados.rados.0.smithi029.stdout:383: writing smithi02979277-25 from 658874 to 1430311 tid 1 2024-04-12T11:15:49.591 INFO:tasks.rados.rados.0.smithi029.stdout:383: writing smithi02979277-25 from 2083958 to 2687013 tid 2 2024-04-12T11:15:49.591 INFO:tasks.rados.rados.0.smithi029.stdout:384: rollback oid 1 current snap is 48 2024-04-12T11:15:49.591 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 1 to 35 2024-04-12T11:15:49.591 INFO:tasks.rados.rados.0.smithi029.stdout:382: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:15:49.746 INFO:tasks.rados.rados.0.smithi029.stdout:384: finishing rollback tid 1 to smithi02979277-1 2024-04-12T11:15:49.746 INFO:tasks.rados.rados.0.smithi029.stdout:384: finishing rollback tid 2 to smithi02979277-1 2024-04-12T11:15:49.746 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 140 (ObjNum 137 snap 28 seq_num 137) dirty exists 2024-04-12T11:15:49.746 INFO:tasks.rados.rados.0.smithi029.stdout:383: finishing write tid 1 to smithi02979277-25 2024-04-12T11:15:49.746 INFO:tasks.rados.rados.0.smithi029.stdout:381: done (3 left) 2024-04-12T11:15:49.746 INFO:tasks.rados.rados.0.smithi029.stdout:382: done (2 left) 2024-04-12T11:15:49.747 INFO:tasks.rados.rados.0.smithi029.stdout:384: done (1 left) 2024-04-12T11:15:49.747 INFO:tasks.rados.rados.0.smithi029.stdout:385: write oid 38 current snap is 48 2024-04-12T11:15:49.747 INFO:tasks.rados.rados.0.smithi029.stdout:383: finishing write tid 2 to smithi02979277-25 2024-04-12T11:15:49.747 INFO:tasks.rados.rados.0.smithi029.stdout:383: finishing write tid 3 to smithi02979277-25 2024-04-12T11:15:49.747 INFO:tasks.rados.rados.0.smithi029.stdout:385: seq_num 183 ranges {675681=563039,1858793=746713,3025937=49744} 2024-04-12T11:15:49.752 INFO:tasks.rados.rados.0.smithi029.stdout:385: writing smithi02979277-38 from 675681 to 1238720 tid 1 2024-04-12T11:15:49.760 INFO:tasks.rados.rados.0.smithi029.stdout:385: writing smithi02979277-38 from 1858793 to 2605506 tid 2 2024-04-12T11:15:49.763 INFO:tasks.rados.rados.0.smithi029.stdout:385: writing smithi02979277-38 from 3025937 to 3075681 tid 3 2024-04-12T11:15:49.763 INFO:tasks.rados.rados.0.smithi029.stdout:386: snap_create 2024-04-12T11:15:49.763 INFO:tasks.rados.rados.0.smithi029.stdout:383: finishing write tid 4 to smithi02979277-25 2024-04-12T11:15:49.763 INFO:tasks.rados.rados.0.smithi029.stdout:383: oid 25 updating version 0 to 129 2024-04-12T11:15:49.763 INFO:tasks.rados.rados.0.smithi029.stdout:383: oid 25 version 129 is already newer than 127 2024-04-12T11:15:49.763 INFO:tasks.rados.rados.0.smithi029.stdout:383: oid 25 version 129 is already newer than 128 2024-04-12T11:15:49.763 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 129 (ObjNum 182 snap 48 seq_num 182) dirty exists 2024-04-12T11:15:49.763 INFO:tasks.rados.rados.0.smithi029.stdout:383: left oid 25 (ObjNum 182 snap 48 seq_num 182) 2024-04-12T11:15:49.763 INFO:tasks.rados.rados.0.smithi029.stdout:385: finishing write tid 1 to smithi02979277-38 2024-04-12T11:15:49.764 INFO:tasks.rados.rados.0.smithi029.stdout:385: finishing write tid 2 to smithi02979277-38 2024-04-12T11:15:49.764 INFO:tasks.rados.rados.0.smithi029.stdout:385: finishing write tid 3 to smithi02979277-38 2024-04-12T11:15:49.764 INFO:tasks.rados.rados.0.smithi029.stdout:385: finishing write tid 4 to smithi02979277-38 2024-04-12T11:15:49.765 INFO:tasks.rados.rados.0.smithi029.stdout:385: finishing write tid 5 to smithi02979277-38 2024-04-12T11:15:49.765 INFO:tasks.rados.rados.0.smithi029.stdout:385: oid 38 updating version 0 to 100 2024-04-12T11:15:49.765 INFO:tasks.rados.rados.0.smithi029.stdout:385: oid 38 version 100 is already newer than 99 2024-04-12T11:15:49.765 INFO:tasks.rados.rados.0.smithi029.stdout:385: oid 38 version 100 is already newer than 97 2024-04-12T11:15:49.765 INFO:tasks.rados.rados.0.smithi029.stdout:385: oid 38 version 100 is already newer than 98 2024-04-12T11:15:49.765 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 100 (ObjNum 183 snap 48 seq_num 183) dirty exists 2024-04-12T11:15:49.765 INFO:tasks.rados.rados.0.smithi029.stdout:385: left oid 38 (ObjNum 183 snap 48 seq_num 183) 2024-04-12T11:15:49.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:49 smithi029 ceph-mon[29349]: pgmap v211: 89 pgs: 89 active+clean; 162 MiB data, 544 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:49.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:49 smithi029 ceph-mon[29349]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-12T11:15:49.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:49 smithi029 ceph-mon[29349]: osdmap e157: 8 total, 8 up, 8 in 2024-04-12T11:15:49.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:49 smithi029 ceph-mon[39999]: pgmap v211: 89 pgs: 89 active+clean; 162 MiB data, 544 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:49.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:49 smithi029 ceph-mon[39999]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-12T11:15:49.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:49 smithi029 ceph-mon[39999]: osdmap e157: 8 total, 8 up, 8 in 2024-04-12T11:15:49.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:49 smithi087 ceph-mon[32917]: pgmap v211: 89 pgs: 89 active+clean; 162 MiB data, 544 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:15:49.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:49 smithi087 ceph-mon[32917]: mgrmap e21: y(active, since 3m), standbys: x 2024-04-12T11:15:49.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:49 smithi087 ceph-mon[32917]: osdmap e157: 8 total, 8 up, 8 in 2024-04-12T11:15:50.580 INFO:tasks.rados.rados.0.smithi029.stdout:383: done (2 left) 2024-04-12T11:15:50.580 INFO:tasks.rados.rados.0.smithi029.stdout:385: done (1 left) 2024-04-12T11:15:50.580 INFO:tasks.rados.rados.0.smithi029.stdout:386: done (0 left) 2024-04-12T11:15:50.580 INFO:tasks.rados.rados.0.smithi029.stdout:387: read oid 39 snap -1 2024-04-12T11:15:50.580 INFO:tasks.rados.rados.0.smithi029.stdout:387: expect deleted 2024-04-12T11:15:50.581 INFO:tasks.rados.rados.0.smithi029.stdout:388: delete oid 22 current snap is 49 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:387: done (1 left) 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:388: done (0 left) 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:389: rollback oid 46 current snap is 49 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 46 to 38 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:390: read oid 7 snap -1 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:390: expect (ObjNum 171 snap 44 seq_num 171) 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:391: read oid 39 snap -1 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:391: expect deleted 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:392: rollback oid 45 current snap is 49 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 45 to 48 2024-04-12T11:15:50.595 INFO:tasks.rados.rados.0.smithi029.stdout:393: snap_remove snap 23 2024-04-12T11:15:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:50 smithi087 ceph-mon[32917]: osdmap e158: 8 total, 8 up, 8 in 2024-04-12T11:15:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:50 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3480599628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:51.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:50 smithi029 ceph-mon[39999]: osdmap e158: 8 total, 8 up, 8 in 2024-04-12T11:15:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:50 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3480599628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:50 smithi029 ceph-mon[29349]: osdmap e158: 8 total, 8 up, 8 in 2024-04-12T11:15:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:50 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3480599628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:51.533 INFO:tasks.rados.rados.0.smithi029.stdout:393: done (4 left) 2024-04-12T11:15:51.534 INFO:tasks.rados.rados.0.smithi029.stdout:394: write oid 14 current snap is 49 2024-04-12T11:15:51.534 INFO:tasks.rados.rados.0.smithi029.stdout:389: finishing rollback tid 1 to smithi02979277-46 2024-04-12T11:15:51.534 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 0 (ObjNum 1801044128 snap 21891 seq_num 304) dirty dne 2024-04-12T11:15:51.534 INFO:tasks.rados.rados.0.smithi029.stdout:390: expect (ObjNum 171 snap 44 seq_num 171) 2024-04-12T11:15:51.702 INFO:tasks.rados.rados.0.smithi029.stdout:392: finishing rollback tid 0 to smithi02979277-45 2024-04-12T11:15:51.702 INFO:tasks.rados.rados.0.smithi029.stdout:392: finishing rollback tid 1 to smithi02979277-45 2024-04-12T11:15:51.702 INFO:tasks.rados.rados.0.smithi029.stdout:392: finishing rollback tid 2 to smithi02979277-45 2024-04-12T11:15:51.702 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 163 (ObjNum 179 snap 47 seq_num 179) dirty exists 2024-04-12T11:15:51.702 INFO:tasks.rados.rados.0.smithi029.stdout:394: seq_num 185 ranges {709295=713537,1859404=750382} 2024-04-12T11:15:51.709 INFO:tasks.rados.rados.0.smithi029.stdout:394: writing smithi02979277-14 from 709295 to 1422832 tid 1 2024-04-12T11:15:51.716 INFO:tasks.rados.rados.0.smithi029.stdout:394: writing smithi02979277-14 from 1859404 to 2609786 tid 2 2024-04-12T11:15:51.716 INFO:tasks.rados.rados.0.smithi029.stdout:389: done (4 left) 2024-04-12T11:15:51.716 INFO:tasks.rados.rados.0.smithi029.stdout:390: done (3 left) 2024-04-12T11:15:51.716 INFO:tasks.rados.rados.0.smithi029.stdout:391: done (2 left) 2024-04-12T11:15:51.717 INFO:tasks.rados.rados.0.smithi029.stdout:392: done (1 left) 2024-04-12T11:15:51.717 INFO:tasks.rados.rados.0.smithi029.stdout:395: write oid 23 current snap is 49 2024-04-12T11:15:51.717 INFO:tasks.rados.rados.0.smithi029.stdout:394: finishing write tid 1 to smithi02979277-14 2024-04-12T11:15:51.717 INFO:tasks.rados.rados.0.smithi029.stdout:395: seq_num 186 ranges {726102=788786,1917886=610393,3108206=17896} 2024-04-12T11:15:51.724 INFO:tasks.rados.rados.0.smithi029.stdout:395: writing smithi02979277-23 from 726102 to 1514888 tid 1 2024-04-12T11:15:51.730 INFO:tasks.rados.rados.0.smithi029.stdout:395: writing smithi02979277-23 from 1917886 to 2528279 tid 2 2024-04-12T11:15:51.734 INFO:tasks.rados.rados.0.smithi029.stdout:395: writing smithi02979277-23 from 3108206 to 3126102 tid 3 2024-04-12T11:15:51.734 INFO:tasks.rados.rados.0.smithi029.stdout:396: write oid 35 current snap is 49 2024-04-12T11:15:51.734 INFO:tasks.rados.rados.0.smithi029.stdout:394: finishing write tid 2 to smithi02979277-14 2024-04-12T11:15:51.734 INFO:tasks.rados.rados.0.smithi029.stdout:396: seq_num 187 ranges {742909=464035,1976368=754051} 2024-04-12T11:15:51.740 INFO:tasks.rados.rados.0.smithi029.stdout:396: writing smithi02979277-35 from 742909 to 1206944 tid 1 2024-04-12T11:15:51.749 INFO:tasks.rados.rados.0.smithi029.stdout:396: writing smithi02979277-35 from 1976368 to 2730419 tid 2 2024-04-12T11:15:51.749 INFO:tasks.rados.rados.0.smithi029.stdout:397: read oid 16 snap 47 2024-04-12T11:15:51.749 INFO:tasks.rados.rados.0.smithi029.stdout:397: expect (ObjNum 117 snap 20 seq_num 117) 2024-04-12T11:15:51.749 INFO:tasks.rados.rados.0.smithi029.stdout:398: snap_create 2024-04-12T11:15:51.749 INFO:tasks.rados.rados.0.smithi029.stdout:394: finishing write tid 3 to smithi02979277-14 2024-04-12T11:15:51.749 INFO:tasks.rados.rados.0.smithi029.stdout:394: finishing write tid 4 to smithi02979277-14 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:394: oid 14 updating version 0 to 135 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:394: oid 14 version 135 is already newer than 134 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:394: oid 14 version 135 is already newer than 133 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 135 (ObjNum 185 snap 49 seq_num 185) dirty exists 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:394: left oid 14 (ObjNum 185 snap 49 seq_num 185) 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:395: finishing write tid 1 to smithi02979277-23 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:395: finishing write tid 2 to smithi02979277-23 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:395: finishing write tid 3 to smithi02979277-23 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:395: finishing write tid 4 to smithi02979277-23 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:395: finishing write tid 5 to smithi02979277-23 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:395: oid 23 updating version 0 to 148 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:395: oid 23 version 148 is already newer than 141 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:395: oid 23 version 148 is already newer than 147 2024-04-12T11:15:51.750 INFO:tasks.rados.rados.0.smithi029.stdout:395: oid 23 version 148 is already newer than 146 2024-04-12T11:15:51.751 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 148 (ObjNum 186 snap 49 seq_num 186) dirty exists 2024-04-12T11:15:51.751 INFO:tasks.rados.rados.0.smithi029.stdout:395: left oid 23 (ObjNum 186 snap 49 seq_num 186) 2024-04-12T11:15:51.751 INFO:tasks.rados.rados.0.smithi029.stdout:396: finishing write tid 1 to smithi02979277-35 2024-04-12T11:15:51.753 INFO:tasks.rados.rados.0.smithi029.stdout:396: finishing write tid 2 to smithi02979277-35 2024-04-12T11:15:51.753 INFO:tasks.rados.rados.0.smithi029.stdout:396: finishing write tid 3 to smithi02979277-35 2024-04-12T11:15:51.753 INFO:tasks.rados.rados.0.smithi029.stdout:396: finishing write tid 4 to smithi02979277-35 2024-04-12T11:15:51.753 INFO:tasks.rados.rados.0.smithi029.stdout:396: oid 35 updating version 0 to 203 2024-04-12T11:15:51.753 INFO:tasks.rados.rados.0.smithi029.stdout:396: oid 35 version 203 is already newer than 202 2024-04-12T11:15:51.753 INFO:tasks.rados.rados.0.smithi029.stdout:396: oid 35 version 203 is already newer than 201 2024-04-12T11:15:51.754 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 203 (ObjNum 187 snap 49 seq_num 187) dirty exists 2024-04-12T11:15:51.754 INFO:tasks.rados.rados.0.smithi029.stdout:396: left oid 35 (ObjNum 187 snap 49 seq_num 187) 2024-04-12T11:15:51.755 INFO:tasks.rados.rados.0.smithi029.stdout:397: expect (ObjNum 117 snap 20 seq_num 117) 2024-04-12T11:15:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:51 smithi087 ceph-mon[32917]: pgmap v214: 89 pgs: 89 active+clean; 175 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-04-12T11:15:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:51 smithi087 ceph-mon[32917]: osdmap e159: 8 total, 8 up, 8 in 2024-04-12T11:15:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:51 smithi087 ceph-mon[32917]: osdmap e160: 8 total, 8 up, 8 in 2024-04-12T11:15:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:51 smithi029 ceph-mon[29349]: pgmap v214: 89 pgs: 89 active+clean; 175 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-04-12T11:15:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:51 smithi029 ceph-mon[29349]: osdmap e159: 8 total, 8 up, 8 in 2024-04-12T11:15:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:51 smithi029 ceph-mon[29349]: osdmap e160: 8 total, 8 up, 8 in 2024-04-12T11:15:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:51 smithi029 ceph-mon[39999]: pgmap v214: 89 pgs: 89 active+clean; 175 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-04-12T11:15:52.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:51 smithi029 ceph-mon[39999]: osdmap e159: 8 total, 8 up, 8 in 2024-04-12T11:15:52.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:51 smithi029 ceph-mon[39999]: osdmap e160: 8 total, 8 up, 8 in 2024-04-12T11:15:52.585 INFO:tasks.rados.rados.0.smithi029.stdout:394: done (4 left) 2024-04-12T11:15:52.585 INFO:tasks.rados.rados.0.smithi029.stdout:395: done (3 left) 2024-04-12T11:15:52.585 INFO:tasks.rados.rados.0.smithi029.stdout:396: done (2 left) 2024-04-12T11:15:52.585 INFO:tasks.rados.rados.0.smithi029.stdout:397: done (1 left) 2024-04-12T11:15:52.585 INFO:tasks.rados.rados.0.smithi029.stdout:398: done (0 left) 2024-04-12T11:15:52.585 INFO:tasks.rados.rados.0.smithi029.stdout:399: read oid 47 snap 46 2024-04-12T11:15:52.585 INFO:tasks.rados.rados.0.smithi029.stdout:399: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:15:52.585 INFO:tasks.rados.rados.0.smithi029.stdout:400: rollback oid 24 current snap is 50 2024-04-12T11:15:52.585 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 24 to 20 2024-04-12T11:15:52.586 INFO:tasks.rados.rados.0.smithi029.stdout:401: delete oid 8 current snap is 50 2024-04-12T11:15:52.587 INFO:tasks.rados.rados.0.smithi029.stdout:400: finishing rollback tid 1 to smithi02979277-24 2024-04-12T11:15:52.587 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 0 (ObjNum 33 snap 0 seq_num 1801094896) dirty dne 2024-04-12T11:15:52.588 INFO:tasks.rados.rados.0.smithi029.stdout:400: done (2 left) 2024-04-12T11:15:52.588 INFO:tasks.rados.rados.0.smithi029.stdout:401: done (1 left) 2024-04-12T11:15:52.588 INFO:tasks.rados.rados.0.smithi029.stdout:402: snap_remove snap 20 2024-04-12T11:15:53.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:15:53 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:15:53] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.43.0" 2024-04-12T11:15:53.591 INFO:tasks.rados.rados.0.smithi029.stdout:399: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:15:53.749 INFO:tasks.rados.rados.0.smithi029.stdout:399: done (1 left) 2024-04-12T11:15:53.749 INFO:tasks.rados.rados.0.smithi029.stdout:402: done (0 left) 2024-04-12T11:15:53.749 INFO:tasks.rados.rados.0.smithi029.stdout:403: snap_create 2024-04-12T11:15:53.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:53 smithi087 ceph-mon[32917]: pgmap v217: 89 pgs: 89 active+clean; 175 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-04-12T11:15:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:53 smithi087 ceph-mon[32917]: osdmap e161: 8 total, 8 up, 8 in 2024-04-12T11:15:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:53 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2255573114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:53 smithi029 ceph-mon[39999]: pgmap v217: 89 pgs: 89 active+clean; 175 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-04-12T11:15:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:53 smithi029 ceph-mon[39999]: osdmap e161: 8 total, 8 up, 8 in 2024-04-12T11:15:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:53 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2255573114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:54.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:53 smithi029 ceph-mon[29349]: pgmap v217: 89 pgs: 89 active+clean; 175 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-04-12T11:15:54.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:53 smithi029 ceph-mon[29349]: osdmap e161: 8 total, 8 up, 8 in 2024-04-12T11:15:54.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:53 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2255573114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:54.596 INFO:tasks.rados.rados.0.smithi029.stdout:403: done (0 left) 2024-04-12T11:15:54.596 INFO:tasks.rados.rados.0.smithi029.stdout:404: write oid 45 current snap is 51 2024-04-12T11:15:54.596 INFO:tasks.rados.rados.0.smithi029.stdout:404: seq_num 189 ranges {776523=614533,1976979=757720} 2024-04-12T11:15:54.603 INFO:tasks.rados.rados.0.smithi029.stdout:404: writing smithi02979277-45 from 776523 to 1391056 tid 1 2024-04-12T11:15:54.611 INFO:tasks.rados.rados.0.smithi029.stdout:404: writing smithi02979277-45 from 1976979 to 2734699 tid 2 2024-04-12T11:15:54.611 INFO:tasks.rados.rados.0.smithi029.stdout:405: write oid 9 current snap is 51 2024-04-12T11:15:54.611 INFO:tasks.rados.rados.0.smithi029.stdout:405: seq_num 190 ranges {793330=689782,2035461=617731,3101545=91785} 2024-04-12T11:15:54.619 INFO:tasks.rados.rados.0.smithi029.stdout:405: writing smithi02979277-9 from 793330 to 1483112 tid 1 2024-04-12T11:15:54.625 INFO:tasks.rados.rados.0.smithi029.stdout:405: writing smithi02979277-9 from 2035461 to 2653192 tid 2 2024-04-12T11:15:54.629 INFO:tasks.rados.rados.0.smithi029.stdout:405: writing smithi02979277-9 from 3101545 to 3193330 tid 3 2024-04-12T11:15:54.629 INFO:tasks.rados.rados.0.smithi029.stdout:406: write oid 21 current snap is 51 2024-04-12T11:15:54.629 INFO:tasks.rados.rados.0.smithi029.stdout:404: finishing write tid 1 to smithi02979277-45 2024-04-12T11:15:54.629 INFO:tasks.rados.rados.0.smithi029.stdout:404: finishing write tid 2 to smithi02979277-45 2024-04-12T11:15:54.629 INFO:tasks.rados.rados.0.smithi029.stdout:406: seq_num 191 ranges {410137=481384,1410296=761389,2728968=481169} 2024-04-12T11:15:54.633 INFO:tasks.rados.rados.0.smithi029.stdout:406: writing smithi02979277-21 from 410137 to 891521 tid 1 2024-04-12T11:15:54.641 INFO:tasks.rados.rados.0.smithi029.stdout:406: writing smithi02979277-21 from 1410296 to 2171685 tid 2 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:406: writing smithi02979277-21 from 2728968 to 3210137 tid 3 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:407: rollback oid 26 current snap is 51 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 46 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:404: finishing write tid 3 to smithi02979277-45 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:404: finishing write tid 4 to smithi02979277-45 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:404: oid 45 updating version 0 to 178 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:404: oid 45 version 178 is already newer than 177 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:404: oid 45 updating version 178 to 179 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 179 (ObjNum 189 snap 51 seq_num 189) dirty exists 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:404: left oid 45 (ObjNum 189 snap 51 seq_num 189) 2024-04-12T11:15:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:405: finishing write tid 1 to smithi02979277-9 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:405: finishing write tid 2 to smithi02979277-9 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:405: finishing write tid 3 to smithi02979277-9 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:405: finishing write tid 4 to smithi02979277-9 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:405: finishing write tid 5 to smithi02979277-9 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:405: oid 9 updating version 0 to 119 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:405: oid 9 version 119 is already newer than 118 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:405: oid 9 version 119 is already newer than 117 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:405: oid 9 updating version 119 to 120 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 120 (ObjNum 190 snap 51 seq_num 190) dirty exists 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:405: left oid 9 (ObjNum 190 snap 51 seq_num 190) 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:406: finishing write tid 1 to smithi02979277-21 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:406: finishing write tid 2 to smithi02979277-21 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:404: done (3 left) 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:405: done (2 left) 2024-04-12T11:15:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:408: write oid 4 current snap is 51 2024-04-12T11:15:54.650 INFO:tasks.rados.rados.0.smithi029.stdout:408: seq_num 192 ranges {426944=556633,1468778=621400,2872744=354200} 2024-04-12T11:15:54.652 INFO:tasks.rados.rados.0.smithi029.stdout:408: writing smithi02979277-4 from 426944 to 983577 tid 1 2024-04-12T11:15:54.658 INFO:tasks.rados.rados.0.smithi029.stdout:408: writing smithi02979277-4 from 1468778 to 2090178 tid 2 2024-04-12T11:15:54.665 INFO:tasks.rados.rados.0.smithi029.stdout:408: writing smithi02979277-4 from 2872744 to 3226944 tid 3 2024-04-12T11:15:54.665 INFO:tasks.rados.rados.0.smithi029.stdout:409: snap_create 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:406: finishing write tid 3 to smithi02979277-21 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:406: finishing write tid 4 to smithi02979277-21 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:406: finishing write tid 5 to smithi02979277-21 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:406: oid 21 updating version 0 to 154 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:406: oid 21 updating version 154 to 155 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:406: oid 21 updating version 155 to 156 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:406: oid 21 updating version 156 to 157 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 157 (ObjNum 191 snap 51 seq_num 191) dirty exists 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:406: left oid 21 (ObjNum 191 snap 51 seq_num 191) 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:407: finishing rollback tid 0 to smithi02979277-26 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:407: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:407: finishing rollback tid 2 to smithi02979277-26 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 215 (ObjNum 135 snap 28 seq_num 135) dirty exists 2024-04-12T11:15:54.666 INFO:tasks.rados.rados.0.smithi029.stdout:408: finishing write tid 1 to smithi02979277-4 2024-04-12T11:15:54.667 INFO:tasks.rados.rados.0.smithi029.stdout:408: finishing write tid 2 to smithi02979277-4 2024-04-12T11:15:54.667 INFO:tasks.rados.rados.0.smithi029.stdout:408: finishing write tid 3 to smithi02979277-4 2024-04-12T11:15:54.668 INFO:tasks.rados.rados.0.smithi029.stdout:408: finishing write tid 4 to smithi02979277-4 2024-04-12T11:15:54.668 INFO:tasks.rados.rados.0.smithi029.stdout:408: finishing write tid 5 to smithi02979277-4 2024-04-12T11:15:54.668 INFO:tasks.rados.rados.0.smithi029.stdout:408: oid 4 updating version 0 to 144 2024-04-12T11:15:54.668 INFO:tasks.rados.rados.0.smithi029.stdout:408: oid 4 updating version 144 to 145 2024-04-12T11:15:54.668 INFO:tasks.rados.rados.0.smithi029.stdout:408: oid 4 version 145 is already newer than 143 2024-04-12T11:15:54.668 INFO:tasks.rados.rados.0.smithi029.stdout:408: oid 4 version 145 is already newer than 142 2024-04-12T11:15:54.668 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 145 (ObjNum 192 snap 51 seq_num 192) dirty exists 2024-04-12T11:15:54.668 INFO:tasks.rados.rados.0.smithi029.stdout:408: left oid 4 (ObjNum 192 snap 51 seq_num 192) 2024-04-12T11:15:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:54 smithi087 ceph-mon[32917]: osdmap e162: 8 total, 8 up, 8 in 2024-04-12T11:15:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:54 smithi029 ceph-mon[29349]: osdmap e162: 8 total, 8 up, 8 in 2024-04-12T11:15:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:54 smithi029 ceph-mon[39999]: osdmap e162: 8 total, 8 up, 8 in 2024-04-12T11:15:55.600 INFO:tasks.rados.rados.0.smithi029.stdout:406: done (3 left) 2024-04-12T11:15:55.601 INFO:tasks.rados.rados.0.smithi029.stdout:407: done (2 left) 2024-04-12T11:15:55.601 INFO:tasks.rados.rados.0.smithi029.stdout:408: done (1 left) 2024-04-12T11:15:55.601 INFO:tasks.rados.rados.0.smithi029.stdout:409: done (0 left) 2024-04-12T11:15:55.601 INFO:tasks.rados.rados.0.smithi029.stdout:410: snap_create 2024-04-12T11:15:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:55 smithi087 ceph-mon[32917]: pgmap v220: 89 pgs: 89 active+clean; 158 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-04-12T11:15:55.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:55 smithi087 ceph-mon[32917]: osdmap e163: 8 total, 8 up, 8 in 2024-04-12T11:15:55.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:55 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/861783417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:55 smithi029 ceph-mon[29349]: pgmap v220: 89 pgs: 89 active+clean; 158 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-04-12T11:15:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:55 smithi029 ceph-mon[29349]: osdmap e163: 8 total, 8 up, 8 in 2024-04-12T11:15:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:55 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/861783417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:55 smithi029 ceph-mon[39999]: pgmap v220: 89 pgs: 89 active+clean; 158 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-04-12T11:15:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:55 smithi029 ceph-mon[39999]: osdmap e163: 8 total, 8 up, 8 in 2024-04-12T11:15:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:55 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/861783417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:56.534 INFO:tasks.rados.rados.0.smithi029.stdout:410: done (0 left) 2024-04-12T11:15:56.534 INFO:tasks.rados.rados.0.smithi029.stdout:411: write oid 8 current snap is 53 2024-04-12T11:15:56.534 INFO:tasks.rados.rados.0.smithi029.stdout:411: seq_num 193 ranges {443751=631882,1810907=765058,3067461=176290} 2024-04-12T11:15:56.540 INFO:tasks.rados.rados.0.smithi029.stdout:411: writing smithi02979277-8 from 443751 to 1075633 tid 1 2024-04-12T11:15:56.548 INFO:tasks.rados.rados.0.smithi029.stdout:411: writing smithi02979277-8 from 1810907 to 2575965 tid 2 2024-04-12T11:15:56.552 INFO:tasks.rados.rados.0.smithi029.stdout:411: writing smithi02979277-8 from 3067461 to 3243751 tid 3 2024-04-12T11:15:56.552 INFO:tasks.rados.rados.0.smithi029.stdout:412: snap_create 2024-04-12T11:15:56.552 INFO:tasks.rados.rados.0.smithi029.stdout:411: finishing write tid 1 to smithi02979277-8 2024-04-12T11:15:56.552 INFO:tasks.rados.rados.0.smithi029.stdout:411: finishing write tid 2 to smithi02979277-8 2024-04-12T11:15:56.554 INFO:tasks.rados.rados.0.smithi029.stdout:411: finishing write tid 3 to smithi02979277-8 2024-04-12T11:15:56.554 INFO:tasks.rados.rados.0.smithi029.stdout:411: finishing write tid 4 to smithi02979277-8 2024-04-12T11:15:56.554 INFO:tasks.rados.rados.0.smithi029.stdout:411: finishing write tid 5 to smithi02979277-8 2024-04-12T11:15:56.554 INFO:tasks.rados.rados.0.smithi029.stdout:411: oid 8 updating version 0 to 25 2024-04-12T11:15:56.554 INFO:tasks.rados.rados.0.smithi029.stdout:411: oid 8 updating version 25 to 26 2024-04-12T11:15:56.554 INFO:tasks.rados.rados.0.smithi029.stdout:411: oid 8 updating version 26 to 27 2024-04-12T11:15:56.554 INFO:tasks.rados.rados.0.smithi029.stdout:411: oid 8 updating version 27 to 28 2024-04-12T11:15:56.554 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 28 (ObjNum 193 snap 53 seq_num 193) dirty exists 2024-04-12T11:15:56.554 INFO:tasks.rados.rados.0.smithi029.stdout:411: left oid 8 (ObjNum 193 snap 53 seq_num 193) 2024-04-12T11:15:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:56 smithi087 ceph-mon[32917]: osdmap e164: 8 total, 8 up, 8 in 2024-04-12T11:15:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:56 smithi087 ceph-mon[32917]: osdmap e165: 8 total, 8 up, 8 in 2024-04-12T11:15:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:56 smithi029 ceph-mon[29349]: osdmap e164: 8 total, 8 up, 8 in 2024-04-12T11:15:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:56 smithi029 ceph-mon[29349]: osdmap e165: 8 total, 8 up, 8 in 2024-04-12T11:15:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:56 smithi029 ceph-mon[39999]: osdmap e164: 8 total, 8 up, 8 in 2024-04-12T11:15:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:56 smithi029 ceph-mon[39999]: osdmap e165: 8 total, 8 up, 8 in 2024-04-12T11:15:57.534 INFO:tasks.rados.rados.0.smithi029.stdout:411: done (1 left) 2024-04-12T11:15:57.534 INFO:tasks.rados.rados.0.smithi029.stdout:412: done (0 left) 2024-04-12T11:15:57.535 INFO:tasks.rados.rados.0.smithi029.stdout:413: snap_create 2024-04-12T11:15:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:57 smithi087 ceph-mon[32917]: pgmap v223: 89 pgs: 89 active+clean; 158 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-04-12T11:15:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:57 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:57 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:57 smithi087 ceph-mon[32917]: osdmap e166: 8 total, 8 up, 8 in 2024-04-12T11:15:57.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:57 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2950147348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:57 smithi029 ceph-mon[29349]: pgmap v223: 89 pgs: 89 active+clean; 158 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-04-12T11:15:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:57 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:57 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:57 smithi029 ceph-mon[29349]: osdmap e166: 8 total, 8 up, 8 in 2024-04-12T11:15:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:57 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2950147348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:57 smithi029 ceph-mon[39999]: pgmap v223: 89 pgs: 89 active+clean; 158 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-04-12T11:15:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:57 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:57 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:58.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:57 smithi029 ceph-mon[39999]: osdmap e166: 8 total, 8 up, 8 in 2024-04-12T11:15:58.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:57 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2950147348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:15:58.538 INFO:tasks.rados.rados.0.smithi029.stdout:413: done (0 left) 2024-04-12T11:15:58.538 INFO:tasks.rados.rados.0.smithi029.stdout:414: snap_remove snap 41 2024-04-12T11:15:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:59.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:59.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:58 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:59.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:58 smithi087 ceph-mon[32917]: osdmap e167: 8 total, 8 up, 8 in 2024-04-12T11:15:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:58 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:58 smithi029 ceph-mon[29349]: osdmap e167: 8 total, 8 up, 8 in 2024-04-12T11:15:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:15:59.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:58 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:15:59.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:58 smithi029 ceph-mon[39999]: osdmap e167: 8 total, 8 up, 8 in 2024-04-12T11:15:59.539 INFO:tasks.rados.rados.0.smithi029.stdout:414: done (0 left) 2024-04-12T11:15:59.539 INFO:tasks.rados.rados.0.smithi029.stdout:415: rollback oid 9 current snap is 55 2024-04-12T11:15:59.539 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 9 to 35 2024-04-12T11:15:59.539 INFO:tasks.rados.rados.0.smithi029.stdout:416: rollback oid 11 current snap is 55 2024-04-12T11:15:59.539 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 11 to 49 2024-04-12T11:15:59.540 INFO:tasks.rados.rados.0.smithi029.stdout:417: write oid 25 current snap is 55 2024-04-12T11:15:59.540 INFO:tasks.rados.rados.0.smithi029.stdout:417: seq_num 194 ranges {460558=707131,1869389=508716,3094884=165674} 2024-04-12T11:15:59.546 INFO:tasks.rados.rados.0.smithi029.stdout:417: writing smithi02979277-25 from 460558 to 1167689 tid 1 2024-04-12T11:15:59.553 INFO:tasks.rados.rados.0.smithi029.stdout:417: writing smithi02979277-25 from 1869389 to 2378105 tid 2 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:417: writing smithi02979277-25 from 3094884 to 3260558 tid 3 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:418: write oid 5 current snap is 55 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:415: finishing rollback tid 0 to smithi02979277-9 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:415: finishing rollback tid 1 to smithi02979277-9 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:415: finishing rollback tid 2 to smithi02979277-9 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 126 (ObjNum 91 snap 15 seq_num 91) dirty exists 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:416: finishing rollback tid 0 to smithi02979277-11 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:416: finishing rollback tid 1 to smithi02979277-11 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:416: finishing rollback tid 2 to smithi02979277-11 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 219 (ObjNum 178 snap 47 seq_num 178) dirty exists 2024-04-12T11:15:59.559 INFO:tasks.rados.rados.0.smithi029.stdout:417: finishing write tid 1 to smithi02979277-25 2024-04-12T11:15:59.560 INFO:tasks.rados.rados.0.smithi029.stdout:417: finishing write tid 2 to smithi02979277-25 2024-04-12T11:15:59.560 INFO:tasks.rados.rados.0.smithi029.stdout:418: seq_num 195 ranges {477365=782380,1927871=768727,3122307=155058} 2024-04-12T11:15:59.565 INFO:tasks.rados.rados.0.smithi029.stdout:418: writing smithi02979277-5 from 477365 to 1259745 tid 1 2024-04-12T11:15:59.573 INFO:tasks.rados.rados.0.smithi029.stdout:418: writing smithi02979277-5 from 1927871 to 2696598 tid 2 2024-04-12T11:15:59.577 INFO:tasks.rados.rados.0.smithi029.stdout:418: writing smithi02979277-5 from 3122307 to 3277365 tid 3 2024-04-12T11:15:59.578 INFO:tasks.rados.rados.0.smithi029.stdout:415: done (3 left) 2024-04-12T11:15:59.578 INFO:tasks.rados.rados.0.smithi029.stdout:416: done (2 left) 2024-04-12T11:15:59.578 INFO:tasks.rados.rados.0.smithi029.stdout:419: read oid 12 snap -1 2024-04-12T11:15:59.578 INFO:tasks.rados.rados.0.smithi029.stdout:419: expect (ObjNum 167 snap 43 seq_num 167) 2024-04-12T11:15:59.578 INFO:tasks.rados.rados.0.smithi029.stdout:420: write oid 16 current snap is 55 2024-04-12T11:15:59.578 INFO:tasks.rados.rados.0.smithi029.stdout:417: finishing write tid 3 to smithi02979277-25 2024-04-12T11:15:59.578 INFO:tasks.rados.rados.0.smithi029.stdout:420: seq_num 196 ranges {494172=457629,1586353=512385,2749730=544442} 2024-04-12T11:15:59.583 INFO:tasks.rados.rados.0.smithi029.stdout:420: writing smithi02979277-16 from 494172 to 951801 tid 1 2024-04-12T11:15:59.589 INFO:tasks.rados.rados.0.smithi029.stdout:420: writing smithi02979277-16 from 1586353 to 2098738 tid 2 2024-04-12T11:15:59.596 INFO:tasks.rados.rados.0.smithi029.stdout:420: writing smithi02979277-16 from 2749730 to 3294172 tid 3 2024-04-12T11:15:59.596 INFO:tasks.rados.rados.0.smithi029.stdout:421: rollback oid 3 current snap is 55 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 35 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:417: finishing write tid 4 to smithi02979277-25 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:417: finishing write tid 5 to smithi02979277-25 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:417: oid 25 updating version 0 to 147 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:417: oid 25 updating version 147 to 149 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:417: oid 25 version 149 is already newer than 148 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:417: oid 25 updating version 149 to 150 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 150 (ObjNum 194 snap 55 seq_num 194) dirty exists 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:417: left oid 25 (ObjNum 194 snap 55 seq_num 194) 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:418: finishing write tid 1 to smithi02979277-5 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:418: finishing write tid 2 to smithi02979277-5 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:417: done (4 left) 2024-04-12T11:15:59.597 INFO:tasks.rados.rados.0.smithi029.stdout:422: write oid 22 current snap is 55 2024-04-12T11:15:59.598 INFO:tasks.rados.rados.0.smithi029.stdout:418: finishing write tid 3 to smithi02979277-5 2024-04-12T11:15:59.598 INFO:tasks.rados.rados.0.smithi029.stdout:418: finishing write tid 4 to smithi02979277-5 2024-04-12T11:15:59.598 INFO:tasks.rados.rados.0.smithi029.stdout:422: seq_num 197 ranges {510979=532878,1528482=772396,3060800=250179} 2024-04-12T11:15:59.602 INFO:tasks.rados.rados.0.smithi029.stdout:422: writing smithi02979277-22 from 510979 to 1043857 tid 1 2024-04-12T11:15:59.609 INFO:tasks.rados.rados.0.smithi029.stdout:422: writing smithi02979277-22 from 1528482 to 2300878 tid 2 2024-04-12T11:15:59.615 INFO:tasks.rados.rados.0.smithi029.stdout:422: writing smithi02979277-22 from 3060800 to 3310979 tid 3 2024-04-12T11:15:59.615 INFO:tasks.rados.rados.0.smithi029.stdout:423: snap_create 2024-04-12T11:15:59.615 INFO:tasks.rados.rados.0.smithi029.stdout:418: finishing write tid 5 to smithi02979277-5 2024-04-12T11:15:59.615 INFO:tasks.rados.rados.0.smithi029.stdout:418: oid 5 updating version 0 to 153 2024-04-12T11:15:59.615 INFO:tasks.rados.rados.0.smithi029.stdout:418: oid 5 version 153 is already newer than 152 2024-04-12T11:15:59.615 INFO:tasks.rados.rados.0.smithi029.stdout:418: oid 5 version 153 is already newer than 151 2024-04-12T11:15:59.615 INFO:tasks.rados.rados.0.smithi029.stdout:418: oid 5 updating version 153 to 154 2024-04-12T11:15:59.616 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 154 (ObjNum 195 snap 55 seq_num 195) dirty exists 2024-04-12T11:15:59.616 INFO:tasks.rados.rados.0.smithi029.stdout:418: left oid 5 (ObjNum 195 snap 55 seq_num 195) 2024-04-12T11:15:59.616 INFO:tasks.rados.rados.0.smithi029.stdout:419: expect (ObjNum 167 snap 43 seq_num 167) 2024-04-12T11:15:59.737 INFO:tasks.rados.rados.0.smithi029.stdout:420: finishing write tid 1 to smithi02979277-16 2024-04-12T11:15:59.737 INFO:tasks.rados.rados.0.smithi029.stdout:420: finishing write tid 2 to smithi02979277-16 2024-04-12T11:15:59.737 INFO:tasks.rados.rados.0.smithi029.stdout:421: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:15:59.737 INFO:tasks.rados.rados.0.smithi029.stdout:421: finishing rollback tid 2 to smithi02979277-3 2024-04-12T11:15:59.737 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 229 (ObjNum 150 snap 35 seq_num 150) dirty exists 2024-04-12T11:15:59.737 INFO:tasks.rados.rados.0.smithi029.stdout:420: finishing write tid 3 to smithi02979277-16 2024-04-12T11:15:59.737 INFO:tasks.rados.rados.0.smithi029.stdout:420: finishing write tid 4 to smithi02979277-16 2024-04-12T11:15:59.737 INFO:tasks.rados.rados.0.smithi029.stdout:420: finishing write tid 5 to smithi02979277-16 2024-04-12T11:15:59.737 INFO:tasks.rados.rados.0.smithi029.stdout:420: oid 16 updating version 0 to 157 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:420: oid 16 version 157 is already newer than 156 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:420: oid 16 updating version 157 to 158 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:420: oid 16 updating version 158 to 159 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 159 (ObjNum 196 snap 55 seq_num 196) dirty exists 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:420: left oid 16 (ObjNum 196 snap 55 seq_num 196) 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:422: finishing write tid 1 to smithi02979277-22 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:422: finishing write tid 2 to smithi02979277-22 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:422: finishing write tid 3 to smithi02979277-22 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:422: finishing write tid 4 to smithi02979277-22 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:422: finishing write tid 5 to smithi02979277-22 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:422: oid 22 updating version 0 to 159 2024-04-12T11:15:59.738 INFO:tasks.rados.rados.0.smithi029.stdout:422: oid 22 updating version 159 to 160 2024-04-12T11:15:59.739 INFO:tasks.rados.rados.0.smithi029.stdout:422: oid 22 version 160 is already newer than 157 2024-04-12T11:15:59.739 INFO:tasks.rados.rados.0.smithi029.stdout:422: oid 22 version 160 is already newer than 158 2024-04-12T11:15:59.739 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 160 (ObjNum 197 snap 55 seq_num 197) dirty exists 2024-04-12T11:15:59.739 INFO:tasks.rados.rados.0.smithi029.stdout:422: left oid 22 (ObjNum 197 snap 55 seq_num 197) 2024-04-12T11:16:00.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:59 smithi087 ceph-mon[32917]: pgmap v226: 89 pgs: 89 active+clean; 158 MiB data, 561 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:00.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:15:59 smithi087 ceph-mon[32917]: osdmap e168: 8 total, 8 up, 8 in 2024-04-12T11:16:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:59 smithi029 ceph-mon[29349]: pgmap v226: 89 pgs: 89 active+clean; 158 MiB data, 561 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:15:59 smithi029 ceph-mon[29349]: osdmap e168: 8 total, 8 up, 8 in 2024-04-12T11:16:00.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:59 smithi029 ceph-mon[39999]: pgmap v226: 89 pgs: 89 active+clean; 158 MiB data, 561 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:00.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:15:59 smithi029 ceph-mon[39999]: osdmap e168: 8 total, 8 up, 8 in 2024-04-12T11:16:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:418: done (5 left) 2024-04-12T11:16:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:419: done (4 left) 2024-04-12T11:16:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:420: done (3 left) 2024-04-12T11:16:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:421: done (2 left) 2024-04-12T11:16:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:422: done (1 left) 2024-04-12T11:16:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:423: done (0 left) 2024-04-12T11:16:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:424: rollback oid 48 current snap is 56 2024-04-12T11:16:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 36 2024-04-12T11:16:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:425: snap_create 2024-04-12T11:16:00.542 INFO:tasks.rados.rados.0.smithi029.stdout:424: finishing rollback tid 0 to smithi02979277-48 2024-04-12T11:16:00.542 INFO:tasks.rados.rados.0.smithi029.stdout:424: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:16:00.542 INFO:tasks.rados.rados.0.smithi029.stdout:424: finishing rollback tid 2 to smithi02979277-48 2024-04-12T11:16:00.542 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 148 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-04-12T11:16:01.205 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:00 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1481182942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:01.205 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:00 smithi087 ceph-mon[32917]: osdmap e169: 8 total, 8 up, 8 in 2024-04-12T11:16:01.219 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:00 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1481182942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:01.219 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:00 smithi029 ceph-mon[39999]: osdmap e169: 8 total, 8 up, 8 in 2024-04-12T11:16:01.220 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:00 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1481182942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:01.220 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:00 smithi029 ceph-mon[29349]: osdmap e169: 8 total, 8 up, 8 in 2024-04-12T11:16:01.399 INFO:tasks.rados.rados.0.smithi029.stdout:424: done (1 left) 2024-04-12T11:16:01.399 INFO:tasks.rados.rados.0.smithi029.stdout:425: done (0 left) 2024-04-12T11:16:01.399 INFO:tasks.rados.rados.0.smithi029.stdout:426: snap_remove snap 56 2024-04-12T11:16:01.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:16:01] ENGINE Bus STOPPING 2024-04-12T11:16:01.467 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:16:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T11:16:01.467 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:16:01] ENGINE Bus STOPPED 2024-04-12T11:16:01.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:01.396+0000 7f9e93758700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-12T11:16:01.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ignoring --setuser ceph since I am not root 2024-04-12T11:16:01.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ignoring --setgroup ceph since I am not root 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:16:01] ENGINE Bus STARTING 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:01.521+0000 7fee6a463700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:01.521+0000 7fee6a463700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:01.521+0000 7fee6a463700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:01.521+0000 7fee6a463700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:01.521+0000 7fee6a463700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:01.521+0000 7fee67c5e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:01.521+0000 7fee67c5e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:01.521+0000 7fee67c5e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:01.521+0000 7fee67c5e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:01.521+0000 7fee67c5e700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:01.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:16:01] ENGINE Serving on http://:::9283 2024-04-12T11:16:01.968 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:16:01] ENGINE Bus STARTED 2024-04-12T11:16:02.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:01.984+0000 7f1b9a9f0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:16:02.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:02.082+0000 7f1b9a9f0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: pgmap v229: 89 pgs: 89 active+clean; 169 MiB data, 559 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: Activating manager daemon x 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: osdmap e170: 8 total, 8 up, 8 in 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-12T11:16:02.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: mgrmap e22: x(active, starting, since 0.00802451s) 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:16:02.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: pgmap v229: 89 pgs: 89 active+clean; 169 MiB data, 559 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: Activating manager daemon x 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: osdmap e170: 8 total, 8 up, 8 in 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:16:02.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: mgrmap e22: x(active, starting, since 0.00802451s) 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:16:02.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: Manager daemon x is now available 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: Manager daemon x is now available 2024-04-12T11:16:02.273 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:02.274 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:02.274 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:16:02.274 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:16:02.274 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T11:16:02.274 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:01 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T11:16:02.400 INFO:tasks.rados.rados.0.smithi029.stdout:426: done (0 left) 2024-04-12T11:16:02.400 INFO:tasks.rados.rados.0.smithi029.stdout:427: delete oid 27 current snap is 57 2024-04-12T11:16:02.402 INFO:tasks.rados.rados.0.smithi029.stdout:427: done (0 left) 2024-04-12T11:16:02.402 INFO:tasks.rados.rados.0.smithi029.stdout:428: rollback oid 33 current snap is 57 2024-04-12T11:16:02.402 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 51 2024-04-12T11:16:02.402 INFO:tasks.rados.rados.0.smithi029.stdout:429: read oid 45 snap -1 2024-04-12T11:16:02.402 INFO:tasks.rados.rados.0.smithi029.stdout:429: expect (ObjNum 189 snap 51 seq_num 189) 2024-04-12T11:16:02.402 INFO:tasks.rados.rados.0.smithi029.stdout:430: read oid 26 snap -1 2024-04-12T11:16:02.402 INFO:tasks.rados.rados.0.smithi029.stdout:430: expect (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:16:02.403 INFO:tasks.rados.rados.0.smithi029.stdout:431: write oid 43 current snap is 57 2024-04-12T11:16:02.403 INFO:tasks.rados.rados.0.smithi029.stdout:431: seq_num 199 ranges {544593=799729,1761799=776065,2948352=396241} 2024-04-12T11:16:02.411 INFO:tasks.rados.rados.0.smithi029.stdout:431: writing smithi02979277-43 from 544593 to 1344322 tid 1 2024-04-12T11:16:02.417 INFO:tasks.rados.rados.0.smithi029.stdout:431: writing smithi02979277-43 from 1761799 to 2537864 tid 2 2024-04-12T11:16:02.423 INFO:tasks.rados.rados.0.smithi029.stdout:431: writing smithi02979277-43 from 2948352 to 3344593 tid 3 2024-04-12T11:16:02.423 INFO:tasks.rados.rados.0.smithi029.stdout:432: read oid 47 snap -1 2024-04-12T11:16:02.423 INFO:tasks.rados.rados.0.smithi029.stdout:432: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:16:02.423 INFO:tasks.rados.rados.0.smithi029.stdout:433: read oid 31 snap -1 2024-04-12T11:16:02.423 INFO:tasks.rados.rados.0.smithi029.stdout:433: expect (ObjNum 138 snap 30 seq_num 138) 2024-04-12T11:16:02.423 INFO:tasks.rados.rados.0.smithi029.stdout:434: write oid 30 current snap is 57 2024-04-12T11:16:02.423 INFO:tasks.rados.rados.0.smithi029.stdout:434: seq_num 200 ranges {561400=474978,1820281=519723,2859422=457579} 2024-04-12T11:16:02.428 INFO:tasks.rados.rados.0.smithi029.stdout:434: writing smithi02979277-30 from 561400 to 1036378 tid 1 2024-04-12T11:16:02.435 INFO:tasks.rados.rados.0.smithi029.stdout:434: writing smithi02979277-30 from 1820281 to 2340004 tid 2 2024-04-12T11:16:02.441 INFO:tasks.rados.rados.0.smithi029.stdout:434: writing smithi02979277-30 from 2859422 to 3317001 tid 3 2024-04-12T11:16:02.441 INFO:tasks.rados.rados.0.smithi029.stdout:435: write oid 21 current snap is 57 2024-04-12T11:16:02.441 INFO:tasks.rados.rados.0.smithi029.stdout:435: seq_num 201 ranges {578207=550227,1762410=779734,3286845=91362} 2024-04-12T11:16:02.447 INFO:tasks.rados.rados.0.smithi029.stdout:435: writing smithi02979277-21 from 578207 to 1128434 tid 1 2024-04-12T11:16:02.455 INFO:tasks.rados.rados.0.smithi029.stdout:435: writing smithi02979277-21 from 1762410 to 2542144 tid 2 2024-04-12T11:16:02.460 INFO:tasks.rados.rados.0.smithi029.stdout:435: writing smithi02979277-21 from 3286845 to 3378207 tid 3 2024-04-12T11:16:02.460 INFO:tasks.rados.rados.0.smithi029.stdout:436: rollback oid 1 current snap is 57 2024-04-12T11:16:02.460 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 1 to 43 2024-04-12T11:16:02.460 INFO:tasks.rados.rados.0.smithi029.stdout:429: expect (ObjNum 189 snap 51 seq_num 189) 2024-04-12T11:16:02.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: pgmap v229: 89 pgs: 89 active+clean; 169 MiB data, 559 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: Activating manager daemon x 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: osdmap e170: 8 total, 8 up, 8 in 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:16:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.14472 172.21.15.29:0/643587911' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: mgrmap e22: x(active, starting, since 0.00802451s) 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:16:02.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:16:02.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:16:02.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: Manager daemon x is now available 2024-04-12T11:16:02.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:02.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:02.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:16:02.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:16:02.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T11:16:02.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:01 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-12T11:16:02.629 INFO:tasks.rados.rados.0.smithi029.stdout:428: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:16:02.629 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 0 (ObjNum 289 snap 0 seq_num 1800876112) dirty dne 2024-04-12T11:16:02.629 INFO:tasks.rados.rados.0.smithi029.stdout:430: expect (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:16:02.744 INFO:tasks.rados.rados.0.smithi029.stdout:431: finishing write tid 1 to smithi02979277-43 2024-04-12T11:16:02.745 INFO:tasks.rados.rados.0.smithi029.stdout:431: finishing write tid 2 to smithi02979277-43 2024-04-12T11:16:02.745 INFO:tasks.rados.rados.0.smithi029.stdout:433: expect (ObjNum 138 snap 30 seq_num 138) 2024-04-12T11:16:02.846 INFO:tasks.rados.rados.0.smithi029.stdout:431: finishing write tid 3 to smithi02979277-43 2024-04-12T11:16:02.846 INFO:tasks.rados.rados.0.smithi029.stdout:431: finishing write tid 4 to smithi02979277-43 2024-04-12T11:16:02.847 INFO:tasks.rados.rados.0.smithi029.stdout:432: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:16:03.002 INFO:tasks.rados.rados.0.smithi029.stdout:431: finishing write tid 5 to smithi02979277-43 2024-04-12T11:16:03.002 INFO:tasks.rados.rados.0.smithi029.stdout:431: oid 43 updating version 0 to 111 2024-04-12T11:16:03.002 INFO:tasks.rados.rados.0.smithi029.stdout:431: oid 43 version 111 is already newer than 109 2024-04-12T11:16:03.002 INFO:tasks.rados.rados.0.smithi029.stdout:431: oid 43 version 111 is already newer than 108 2024-04-12T11:16:03.002 INFO:tasks.rados.rados.0.smithi029.stdout:431: oid 43 version 111 is already newer than 110 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 111 (ObjNum 199 snap 57 seq_num 199) dirty exists 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:431: left oid 43 (ObjNum 199 snap 57 seq_num 199) 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:434: finishing write tid 1 to smithi02979277-30 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:434: finishing write tid 2 to smithi02979277-30 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:434: finishing write tid 3 to smithi02979277-30 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:434: finishing write tid 4 to smithi02979277-30 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:428: done (8 left) 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:429: done (7 left) 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:430: done (6 left) 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:431: done (5 left) 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:432: done (4 left) 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:433: done (3 left) 2024-04-12T11:16:03.003 INFO:tasks.rados.rados.0.smithi029.stdout:437: snap_remove snap 42 2024-04-12T11:16:03.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-mon[29349]: Deploying cephadm binary to smithi087 2024-04-12T11:16:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-mon[29349]: Deploying cephadm binary to smithi029 2024-04-12T11:16:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3056929726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-mon[29349]: osdmap e171: 8 total, 8 up, 8 in 2024-04-12T11:16:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-mon[29349]: mgrmap e23: x(active, since 1.01611s) 2024-04-12T11:16:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-mon[39999]: Deploying cephadm binary to smithi087 2024-04-12T11:16:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-mon[39999]: Deploying cephadm binary to smithi029 2024-04-12T11:16:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3056929726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-mon[39999]: osdmap e171: 8 total, 8 up, 8 in 2024-04-12T11:16:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-mon[39999]: mgrmap e23: x(active, since 1.01611s) 2024-04-12T11:16:03.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:02.760+0000 7f1b9a9f0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:16:03.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:02 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:02.855+0000 7f1b9a9f0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:16:03.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:03.000+0000 7f1b9a9f0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:16:03.453 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:03.097+0000 7f1b9a9f0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:16:03.454 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:03.183+0000 7f1b9a9f0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:16:03.454 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:03.452+0000 7f1b9a9f0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:16:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:02 smithi087 ceph-mon[32917]: Deploying cephadm binary to smithi087 2024-04-12T11:16:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:02 smithi087 ceph-mon[32917]: Deploying cephadm binary to smithi029 2024-04-12T11:16:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:02 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3056929726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:02 smithi087 ceph-mon[32917]: osdmap e171: 8 total, 8 up, 8 in 2024-04-12T11:16:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:02 smithi087 ceph-mon[32917]: mgrmap e23: x(active, since 1.01611s) 2024-04-12T11:16:03.751 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:03.565+0000 7f1b9a9f0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:16:03.751 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:03.654+0000 7f1b9a9f0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:16:03.751 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:03.750+0000 7f1b9a9f0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:16:03.974 INFO:tasks.rados.rados.0.smithi029.stdout:437: done (3 left) 2024-04-12T11:16:03.974 INFO:tasks.rados.rados.0.smithi029.stdout:438: write oid 3 current snap is 57 2024-04-12T11:16:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:438: seq_num 202 ranges {595014=625476,1820892=523392,2797915=596476} 2024-04-12T11:16:03.981 INFO:tasks.rados.rados.0.smithi029.stdout:438: writing smithi02979277-3 from 595014 to 1220490 tid 1 2024-04-12T11:16:03.988 INFO:tasks.rados.rados.0.smithi029.stdout:438: writing smithi02979277-3 from 1820892 to 2344284 tid 2 2024-04-12T11:16:03.994 INFO:tasks.rados.rados.0.smithi029.stdout:438: writing smithi02979277-3 from 2797915 to 3394391 tid 3 2024-04-12T11:16:03.994 INFO:tasks.rados.rados.0.smithi029.stdout:439: write oid 15 current snap is 57 2024-04-12T11:16:03.994 INFO:tasks.rados.rados.0.smithi029.stdout:434: finishing write tid 5 to smithi02979277-30 2024-04-12T11:16:03.994 INFO:tasks.rados.rados.0.smithi029.stdout:434: oid 30 updating version 0 to 125 2024-04-12T11:16:03.995 INFO:tasks.rados.rados.0.smithi029.stdout:434: oid 30 version 125 is already newer than 123 2024-04-12T11:16:03.995 INFO:tasks.rados.rados.0.smithi029.stdout:434: oid 30 version 125 is already newer than 122 2024-04-12T11:16:03.995 INFO:tasks.rados.rados.0.smithi029.stdout:434: oid 30 version 125 is already newer than 124 2024-04-12T11:16:03.995 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 125 (ObjNum 200 snap 57 seq_num 200) dirty exists 2024-04-12T11:16:03.995 INFO:tasks.rados.rados.0.smithi029.stdout:434: left oid 30 (ObjNum 200 snap 57 seq_num 200) 2024-04-12T11:16:03.995 INFO:tasks.rados.rados.0.smithi029.stdout:435: finishing write tid 1 to smithi02979277-21 2024-04-12T11:16:03.995 INFO:tasks.rados.rados.0.smithi029.stdout:435: finishing write tid 2 to smithi02979277-21 2024-04-12T11:16:03.995 INFO:tasks.rados.rados.0.smithi029.stdout:439: seq_num 203 ranges {611821=700725,1879374=783403,3341691=70130} 2024-04-12T11:16:04.001 INFO:tasks.rados.rados.0.smithi029.stdout:439: writing smithi02979277-15 from 611821 to 1312546 tid 1 2024-04-12T11:16:04.008 INFO:tasks.rados.rados.0.smithi029.stdout:439: writing smithi02979277-15 from 1879374 to 2662777 tid 2 2024-04-12T11:16:04.013 INFO:tasks.rados.rados.0.smithi029.stdout:439: writing smithi02979277-15 from 3341691 to 3411821 tid 3 2024-04-12T11:16:04.013 INFO:tasks.rados.rados.0.smithi029.stdout:434: done (4 left) 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:440: snap_create 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:435: finishing write tid 3 to smithi02979277-21 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:435: finishing write tid 4 to smithi02979277-21 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:435: finishing write tid 5 to smithi02979277-21 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:435: oid 21 updating version 0 to 163 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:435: oid 21 updating version 163 to 164 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:435: oid 21 version 164 is already newer than 162 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:435: oid 21 version 164 is already newer than 161 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 164 (ObjNum 201 snap 57 seq_num 201) dirty exists 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:435: left oid 21 (ObjNum 201 snap 57 seq_num 201) 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:436: finishing rollback tid 0 to smithi02979277-1 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:436: finishing rollback tid 1 to smithi02979277-1 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 167 (ObjNum 33 snap 0 seq_num 862035138) dirty dne 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:438: finishing write tid 1 to smithi02979277-3 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:438: finishing write tid 2 to smithi02979277-3 2024-04-12T11:16:04.014 INFO:tasks.rados.rados.0.smithi029.stdout:438: finishing write tid 3 to smithi02979277-3 2024-04-12T11:16:04.015 INFO:tasks.rados.rados.0.smithi029.stdout:438: finishing write tid 4 to smithi02979277-3 2024-04-12T11:16:04.015 INFO:tasks.rados.rados.0.smithi029.stdout:438: finishing write tid 5 to smithi02979277-3 2024-04-12T11:16:04.015 INFO:tasks.rados.rados.0.smithi029.stdout:438: oid 3 updating version 0 to 242 2024-04-12T11:16:04.015 INFO:tasks.rados.rados.0.smithi029.stdout:438: oid 3 version 242 is already newer than 231 2024-04-12T11:16:04.015 INFO:tasks.rados.rados.0.smithi029.stdout:438: oid 3 version 242 is already newer than 241 2024-04-12T11:16:04.015 INFO:tasks.rados.rados.0.smithi029.stdout:438: oid 3 version 242 is already newer than 236 2024-04-12T11:16:04.015 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 242 (ObjNum 202 snap 57 seq_num 202) dirty exists 2024-04-12T11:16:04.015 INFO:tasks.rados.rados.0.smithi029.stdout:438: left oid 3 (ObjNum 202 snap 57 seq_num 202) 2024-04-12T11:16:04.015 INFO:tasks.rados.rados.0.smithi029.stdout:439: finishing write tid 1 to smithi02979277-15 2024-04-12T11:16:04.015 INFO:tasks.rados.rados.0.smithi029.stdout:439: finishing write tid 2 to smithi02979277-15 2024-04-12T11:16:04.016 INFO:tasks.rados.rados.0.smithi029.stdout:439: finishing write tid 3 to smithi02979277-15 2024-04-12T11:16:04.016 INFO:tasks.rados.rados.0.smithi029.stdout:439: finishing write tid 4 to smithi02979277-15 2024-04-12T11:16:04.016 INFO:tasks.rados.rados.0.smithi029.stdout:439: finishing write tid 5 to smithi02979277-15 2024-04-12T11:16:04.016 INFO:tasks.rados.rados.0.smithi029.stdout:439: oid 15 updating version 0 to 131 2024-04-12T11:16:04.016 INFO:tasks.rados.rados.0.smithi029.stdout:439: oid 15 version 131 is already newer than 129 2024-04-12T11:16:04.017 INFO:tasks.rados.rados.0.smithi029.stdout:439: oid 15 updating version 131 to 132 2024-04-12T11:16:04.017 INFO:tasks.rados.rados.0.smithi029.stdout:439: oid 15 version 132 is already newer than 130 2024-04-12T11:16:04.017 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 132 (ObjNum 203 snap 57 seq_num 203) dirty exists 2024-04-12T11:16:04.017 INFO:tasks.rados.rados.0.smithi029.stdout:439: left oid 15 (ObjNum 203 snap 57 seq_num 203) 2024-04-12T11:16:04.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-mon[29349]: [12/Apr/2024:11:16:02] ENGINE Bus STARTING 2024-04-12T11:16:04.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-mon[29349]: [12/Apr/2024:11:16:02] ENGINE Serving on http://172.21.15.87:8765 2024-04-12T11:16:04.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-mon[29349]: [12/Apr/2024:11:16:02] ENGINE Serving on https://172.21.15.87:7150 2024-04-12T11:16:04.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-mon[29349]: [12/Apr/2024:11:16:02] ENGINE Bus STARTED 2024-04-12T11:16:04.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-mon[29349]: pgmap v4: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:04.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:03.929+0000 7f1b9a9f0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:16:04.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-mon[39999]: [12/Apr/2024:11:16:02] ENGINE Bus STARTING 2024-04-12T11:16:04.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-mon[39999]: [12/Apr/2024:11:16:02] ENGINE Serving on http://172.21.15.87:8765 2024-04-12T11:16:04.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-mon[39999]: [12/Apr/2024:11:16:02] ENGINE Serving on https://172.21.15.87:7150 2024-04-12T11:16:04.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-mon[39999]: [12/Apr/2024:11:16:02] ENGINE Bus STARTED 2024-04-12T11:16:04.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:03 smithi029 ceph-mon[39999]: pgmap v4: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:04.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:03 smithi087 ceph-mon[32917]: [12/Apr/2024:11:16:02] ENGINE Bus STARTING 2024-04-12T11:16:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:03 smithi087 ceph-mon[32917]: [12/Apr/2024:11:16:02] ENGINE Serving on http://172.21.15.87:8765 2024-04-12T11:16:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:03 smithi087 ceph-mon[32917]: [12/Apr/2024:11:16:02] ENGINE Serving on https://172.21.15.87:7150 2024-04-12T11:16:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:03 smithi087 ceph-mon[32917]: [12/Apr/2024:11:16:02] ENGINE Bus STARTED 2024-04-12T11:16:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:03 smithi087 ceph-mon[32917]: pgmap v4: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:04.983 INFO:tasks.rados.rados.0.smithi029.stdout:435: done (4 left) 2024-04-12T11:16:04.984 INFO:tasks.rados.rados.0.smithi029.stdout:436: done (3 left) 2024-04-12T11:16:04.984 INFO:tasks.rados.rados.0.smithi029.stdout:438: done (2 left) 2024-04-12T11:16:04.984 INFO:tasks.rados.rados.0.smithi029.stdout:439: done (1 left) 2024-04-12T11:16:04.984 INFO:tasks.rados.rados.0.smithi029.stdout:440: done (0 left) 2024-04-12T11:16:04.984 INFO:tasks.rados.rados.0.smithi029.stdout:441: snap_remove snap 46 2024-04-12T11:16:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:04 smithi029 ceph-mon[29349]: pgmap v5: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:04 smithi029 ceph-mon[29349]: osdmap e172: 8 total, 8 up, 8 in 2024-04-12T11:16:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:04 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/415529123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:04 smithi029 ceph-mon[39999]: pgmap v5: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:04 smithi029 ceph-mon[39999]: osdmap e172: 8 total, 8 up, 8 in 2024-04-12T11:16:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:04 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/415529123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:05.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:04.941+0000 7f1b9a9f0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:16:05.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:05 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:05.080+0000 7f1b9a9f0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:16:05.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:04 smithi087 ceph-mon[32917]: pgmap v5: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:05.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:04 smithi087 ceph-mon[32917]: osdmap e172: 8 total, 8 up, 8 in 2024-04-12T11:16:05.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:04 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/415529123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:05.724 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:05 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:05.424+0000 7f1b9a9f0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:16:05.724 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:05 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:05.519+0000 7f1b9a9f0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:16:05.724 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:05 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:05.724+0000 7f1b9a9f0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:16:05.983 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:05 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:05.918+0000 7f1b9a9f0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:16:05.987 INFO:tasks.rados.rados.0.smithi029.stdout:441: done (0 left) 2024-04-12T11:16:05.987 INFO:tasks.rados.rados.0.smithi029.stdout:442: read oid 36 snap -1 2024-04-12T11:16:05.987 INFO:tasks.rados.rados.0.smithi029.stdout:442: expect (ObjNum 134 snap 28 seq_num 134) 2024-04-12T11:16:05.987 INFO:tasks.rados.rados.0.smithi029.stdout:443: read oid 35 snap 54 2024-04-12T11:16:05.987 INFO:tasks.rados.rados.0.smithi029.stdout:443: expect (ObjNum 187 snap 49 seq_num 187) 2024-04-12T11:16:05.987 INFO:tasks.rados.rados.0.smithi029.stdout:444: snap_remove snap 51 2024-04-12T11:16:06.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:05 smithi029 ceph-mon[39999]: mgrmap e24: x(active, since 3s) 2024-04-12T11:16:06.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:05 smithi029 ceph-mon[39999]: osdmap e173: 8 total, 8 up, 8 in 2024-04-12T11:16:06.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:05 smithi029 ceph-mon[29349]: mgrmap e24: x(active, since 3s) 2024-04-12T11:16:06.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:05 smithi029 ceph-mon[29349]: osdmap e173: 8 total, 8 up, 8 in 2024-04-12T11:16:06.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:06.221+0000 7f1b9a9f0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:16:06.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:05 smithi087 ceph-mon[32917]: mgrmap e24: x(active, since 3s) 2024-04-12T11:16:06.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:05 smithi087 ceph-mon[32917]: osdmap e173: 8 total, 8 up, 8 in 2024-04-12T11:16:06.535 INFO:tasks.rados.rados.0.smithi029.stdout:444: done (2 left) 2024-04-12T11:16:06.535 INFO:tasks.rados.rados.0.smithi029.stdout:445: rollback oid 37 current snap is 58 2024-04-12T11:16:06.535 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 53 2024-04-12T11:16:06.535 INFO:tasks.rados.rados.0.smithi029.stdout:446: snap_remove snap 50 2024-04-12T11:16:06.746 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:06.486+0000 7f1b9a9f0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:16:06.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:06.691Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-04-12T11:16:06.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:06.693Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-04-12T11:16:06.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:06.694Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-04-12T11:16:06.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:06.694Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-04-12T11:16:06.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:06.694Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-04-12T11:16:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:06 smithi029 ceph-mon[39999]: pgmap v8: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:06 smithi029 ceph-mon[39999]: osdmap e174: 8 total, 8 up, 8 in 2024-04-12T11:16:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:06 smithi029 ceph-mon[39999]: osdmap e175: 8 total, 8 up, 8 in 2024-04-12T11:16:07.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:06.746+0000 7f1b9a9f0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:16:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:06 smithi029 ceph-mon[29349]: pgmap v8: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:06 smithi029 ceph-mon[29349]: osdmap e174: 8 total, 8 up, 8 in 2024-04-12T11:16:07.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:06 smithi029 ceph-mon[29349]: osdmap e175: 8 total, 8 up, 8 in 2024-04-12T11:16:07.267 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:06 smithi087 ceph-mon[32917]: pgmap v8: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:07.268 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:06 smithi087 ceph-mon[32917]: osdmap e174: 8 total, 8 up, 8 in 2024-04-12T11:16:07.268 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:06 smithi087 ceph-mon[32917]: osdmap e175: 8 total, 8 up, 8 in 2024-04-12T11:16:07.538 INFO:tasks.rados.rados.0.smithi029.stdout:446: done (3 left) 2024-04-12T11:16:07.538 INFO:tasks.rados.rados.0.smithi029.stdout:447: write oid 17 current snap is 58 2024-04-12T11:16:07.538 INFO:tasks.rados.rados.0.smithi029.stdout:447: seq_num 204 ranges {628628=775974,1937856=527061,3252761=175867} 2024-04-12T11:16:07.547 INFO:tasks.rados.rados.0.smithi029.stdout:447: writing smithi02979277-17 from 628628 to 1404602 tid 1 2024-04-12T11:16:07.554 INFO:tasks.rados.rados.0.smithi029.stdout:447: writing smithi02979277-17 from 1937856 to 2464917 tid 2 2024-04-12T11:16:07.560 INFO:tasks.rados.rados.0.smithi029.stdout:447: writing smithi02979277-17 from 3252761 to 3428628 tid 3 2024-04-12T11:16:07.560 INFO:tasks.rados.rados.0.smithi029.stdout:448: read oid 21 snap 43 2024-04-12T11:16:07.560 INFO:tasks.rados.rados.0.smithi029.stdout:448: expect (ObjNum 20 snap 0 seq_num 20) 2024-04-12T11:16:07.560 INFO:tasks.rados.rados.0.smithi029.stdout:449: read oid 2 snap -1 2024-04-12T11:16:07.561 INFO:tasks.rados.rados.0.smithi029.stdout:449: expect deleted 2024-04-12T11:16:07.561 INFO:tasks.rados.rados.0.smithi029.stdout:450: write oid 43 current snap is 58 2024-04-12T11:16:07.561 INFO:tasks.rados.rados.0.smithi029.stdout:450: seq_num 205 ranges {645435=451223,1879985=787072,3280184=165251} 2024-04-12T11:16:07.567 INFO:tasks.rados.rados.0.smithi029.stdout:450: writing smithi02979277-43 from 645435 to 1096658 tid 1 2024-04-12T11:16:07.577 INFO:tasks.rados.rados.0.smithi029.stdout:450: writing smithi02979277-43 from 1879985 to 2667057 tid 2 2024-04-12T11:16:07.582 INFO:tasks.rados.rados.0.smithi029.stdout:450: writing smithi02979277-43 from 3280184 to 3445435 tid 3 2024-04-12T11:16:07.582 INFO:tasks.rados.rados.0.smithi029.stdout:451: snap_create 2024-04-12T11:16:07.582 INFO:tasks.rados.rados.0.smithi029.stdout:442: expect (ObjNum 134 snap 28 seq_num 134) 2024-04-12T11:16:07.656 INFO:tasks.rados.rados.0.smithi029.stdout:443: expect (ObjNum 187 snap 49 seq_num 187) 2024-04-12T11:16:07.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:07 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:07.662+0000 7f1b9a9f0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:16:07.807 INFO:tasks.rados.rados.0.smithi029.stdout:445: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:16:07.807 INFO:tasks.rados.rados.0.smithi029.stdout:445: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:16:07.807 INFO:tasks.rados.rados.0.smithi029.stdout:445: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:16:07.807 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 134 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-04-12T11:16:07.807 INFO:tasks.rados.rados.0.smithi029.stdout:447: finishing write tid 1 to smithi02979277-17 2024-04-12T11:16:07.807 INFO:tasks.rados.rados.0.smithi029.stdout:447: finishing write tid 2 to smithi02979277-17 2024-04-12T11:16:07.807 INFO:tasks.rados.rados.0.smithi029.stdout:447: finishing write tid 3 to smithi02979277-17 2024-04-12T11:16:07.808 INFO:tasks.rados.rados.0.smithi029.stdout:447: finishing write tid 4 to smithi02979277-17 2024-04-12T11:16:07.808 INFO:tasks.rados.rados.0.smithi029.stdout:447: finishing write tid 5 to smithi02979277-17 2024-04-12T11:16:07.808 INFO:tasks.rados.rados.0.smithi029.stdout:447: oid 17 updating version 0 to 160 2024-04-12T11:16:07.808 INFO:tasks.rados.rados.0.smithi029.stdout:447: oid 17 updating version 160 to 163 2024-04-12T11:16:07.808 INFO:tasks.rados.rados.0.smithi029.stdout:447: oid 17 updating version 163 to 164 2024-04-12T11:16:07.808 INFO:tasks.rados.rados.0.smithi029.stdout:447: oid 17 updating version 164 to 165 2024-04-12T11:16:07.808 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 165 (ObjNum 204 snap 58 seq_num 204) dirty exists 2024-04-12T11:16:07.808 INFO:tasks.rados.rados.0.smithi029.stdout:447: left oid 17 (ObjNum 204 snap 58 seq_num 204) 2024-04-12T11:16:07.808 INFO:tasks.rados.rados.0.smithi029.stdout:448: expect (ObjNum 20 snap 0 seq_num 20) 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:450: finishing write tid 1 to smithi02979277-43 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:450: finishing write tid 2 to smithi02979277-43 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:450: finishing write tid 3 to smithi02979277-43 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:450: finishing write tid 4 to smithi02979277-43 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:450: finishing write tid 5 to smithi02979277-43 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:450: oid 43 updating version 0 to 129 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:450: oid 43 version 129 is already newer than 127 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:450: oid 43 version 129 is already newer than 128 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:450: oid 43 updating version 129 to 130 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 130 (ObjNum 205 snap 58 seq_num 205) dirty exists 2024-04-12T11:16:07.891 INFO:tasks.rados.rados.0.smithi029.stdout:450: left oid 43 (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:16:08.229 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:07 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/273264118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:08.230 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:07 smithi087 ceph-mon[32917]: osdmap e176: 8 total, 8 up, 8 in 2024-04-12T11:16:08.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:07 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/273264118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:08.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:07 smithi029 ceph-mon[29349]: osdmap e176: 8 total, 8 up, 8 in 2024-04-12T11:16:08.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:07 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/273264118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:08.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:07 smithi029 ceph-mon[39999]: osdmap e176: 8 total, 8 up, 8 in 2024-04-12T11:16:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:442: done (7 left) 2024-04-12T11:16:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:443: done (6 left) 2024-04-12T11:16:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:445: done (5 left) 2024-04-12T11:16:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:447: done (4 left) 2024-04-12T11:16:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:448: done (3 left) 2024-04-12T11:16:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:449: done (2 left) 2024-04-12T11:16:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:450: done (1 left) 2024-04-12T11:16:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:451: done (0 left) 2024-04-12T11:16:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:452: snap_create 2024-04-12T11:16:08.619 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:08.326+0000 7f1b9a9f0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:16:08.620 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:08.429+0000 7f1b9a9f0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:16:08.989 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[29349]: pgmap v11: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:08.989 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[29349]: osdmap e177: 8 total, 8 up, 8 in 2024-04-12T11:16:08.989 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[29349]: Standby manager daemon y started 2024-04-12T11:16:08.989 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T11:16:08.989 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:08.665+0000 7f1b9a9f0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:16:08.990 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:16:08] ENGINE Bus STARTING 2024-04-12T11:16:08.990 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: CherryPy Checker: 2024-04-12T11:16:08.990 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: The Application mounted at '' has an empty config. 2024-04-12T11:16:08.990 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: 2024-04-12T11:16:08.990 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:16:08] ENGINE Serving on http://:::9283 2024-04-12T11:16:08.990 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: [12/Apr/2024:11:16:08] ENGINE Bus STARTED 2024-04-12T11:16:09.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:08 smithi087 ceph-mon[32917]: pgmap v11: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:09.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:08 smithi087 ceph-mon[32917]: osdmap e177: 8 total, 8 up, 8 in 2024-04-12T11:16:09.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:08 smithi087 ceph-mon[32917]: Standby manager daemon y started 2024-04-12T11:16:09.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:08 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T11:16:09.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:08 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:16:09.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:08 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T11:16:09.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:08 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:16:09.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:16:09.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T11:16:09.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:16:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[39999]: pgmap v11: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 180 MiB data, 573 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[39999]: osdmap e177: 8 total, 8 up, 8 in 2024-04-12T11:16:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[39999]: Standby manager daemon y started 2024-04-12T11:16:09.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T11:16:09.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:16:09.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T11:16:09.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:08 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.29:0/2281795399' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:16:09.538 INFO:tasks.rados.rados.0.smithi029.stdout:452: done (0 left) 2024-04-12T11:16:09.539 INFO:tasks.rados.rados.0.smithi029.stdout:453: delete oid 44 current snap is 60 2024-04-12T11:16:09.542 INFO:tasks.rados.rados.0.smithi029.stdout:453: done (0 left) 2024-04-12T11:16:09.542 INFO:tasks.rados.rados.0.smithi029.stdout:454: rollback oid 27 current snap is 60 2024-04-12T11:16:09.542 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 35 2024-04-12T11:16:09.542 INFO:tasks.rados.rados.0.smithi029.stdout:455: rollback oid 7 current snap is 60 2024-04-12T11:16:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 57 2024-04-12T11:16:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:456: read oid 25 snap -1 2024-04-12T11:16:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:456: expect (ObjNum 194 snap 55 seq_num 194) 2024-04-12T11:16:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:457: read oid 48 snap -1 2024-04-12T11:16:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:457: expect (ObjNum 128 snap 27 seq_num 128) 2024-04-12T11:16:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:458: write oid 8 current snap is 60 2024-04-12T11:16:09.543 INFO:tasks.rados.rados.0.smithi029.stdout:458: seq_num 207 ranges {679049=718074,2113302=674388,3335030=144019} 2024-04-12T11:16:09.550 INFO:tasks.rados.rados.0.smithi029.stdout:458: writing smithi02979277-8 from 679049 to 1397123 tid 1 2024-04-12T11:16:09.557 INFO:tasks.rados.rados.0.smithi029.stdout:458: writing smithi02979277-8 from 2113302 to 2787690 tid 2 2024-04-12T11:16:09.561 INFO:tasks.rados.rados.0.smithi029.stdout:458: writing smithi02979277-8 from 3335030 to 3479049 tid 3 2024-04-12T11:16:09.561 INFO:tasks.rados.rados.0.smithi029.stdout:459: snap_remove snap 55 2024-04-12T11:16:10.546 INFO:tasks.rados.rados.0.smithi029.stdout:459: done (5 left) 2024-04-12T11:16:10.546 INFO:tasks.rados.rados.0.smithi029.stdout:460: snap_remove snap 52 2024-04-12T11:16:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:10 smithi087 ceph-mon[32917]: pgmap v14: 89 pgs: 89 active+clean; 189 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 45 op/s 2024-04-12T11:16:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:10 smithi087 ceph-mon[32917]: osdmap e178: 8 total, 8 up, 8 in 2024-04-12T11:16:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:10 smithi087 ceph-mon[32917]: mgrmap e25: x(active, since 8s), standbys: y 2024-04-12T11:16:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:10 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:16:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:10 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:10 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:10 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/286795646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[29349]: pgmap v14: 89 pgs: 89 active+clean; 189 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 45 op/s 2024-04-12T11:16:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[29349]: osdmap e178: 8 total, 8 up, 8 in 2024-04-12T11:16:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[29349]: mgrmap e25: x(active, since 8s), standbys: y 2024-04-12T11:16:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:16:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/286795646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[39999]: pgmap v14: 89 pgs: 89 active+clean; 189 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 45 op/s 2024-04-12T11:16:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[39999]: osdmap e178: 8 total, 8 up, 8 in 2024-04-12T11:16:11.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[39999]: mgrmap e25: x(active, since 8s), standbys: y 2024-04-12T11:16:11.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:16:11.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:11.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:11.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:10 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/286795646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:11.537 INFO:tasks.rados.rados.0.smithi029.stdout:460: done (5 left) 2024-04-12T11:16:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:461: snap_create 2024-04-12T11:16:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:454: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:16:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 120 (ObjNum 129 snap 0 seq_num 1800803248) dirty dne 2024-04-12T11:16:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:455: finishing rollback tid 0 to smithi02979277-7 2024-04-12T11:16:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:455: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:16:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:455: finishing rollback tid 2 to smithi02979277-7 2024-04-12T11:16:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 199 (ObjNum 171 snap 44 seq_num 171) dirty exists 2024-04-12T11:16:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:457: expect (ObjNum 128 snap 27 seq_num 128) 2024-04-12T11:16:11.697 INFO:tasks.rados.rados.0.smithi029.stdout:458: finishing write tid 1 to smithi02979277-8 2024-04-12T11:16:11.697 INFO:tasks.rados.rados.0.smithi029.stdout:456: expect (ObjNum 194 snap 55 seq_num 194) 2024-04-12T11:16:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: osdmap e179: 8 total, 8 up, 8 in 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi029 to 2481M 2024-04-12T11:16:11.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:11.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:11 smithi087 ceph-mon[32917]: osdmap e180: 8 total, 8 up, 8 in 2024-04-12T11:16:11.993 INFO:tasks.rados.rados.0.smithi029.stdout:458: finishing write tid 2 to smithi02979277-8 2024-04-12T11:16:11.993 INFO:tasks.rados.rados.0.smithi029.stdout:458: finishing write tid 3 to smithi02979277-8 2024-04-12T11:16:11.993 INFO:tasks.rados.rados.0.smithi029.stdout:458: finishing write tid 4 to smithi02979277-8 2024-04-12T11:16:11.993 INFO:tasks.rados.rados.0.smithi029.stdout:458: finishing write tid 5 to smithi02979277-8 2024-04-12T11:16:11.993 INFO:tasks.rados.rados.0.smithi029.stdout:458: oid 8 updating version 0 to 32 2024-04-12T11:16:11.993 INFO:tasks.rados.rados.0.smithi029.stdout:458: oid 8 version 32 is already newer than 30 2024-04-12T11:16:11.993 INFO:tasks.rados.rados.0.smithi029.stdout:458: oid 8 version 32 is already newer than 31 2024-04-12T11:16:11.993 INFO:tasks.rados.rados.0.smithi029.stdout:458: oid 8 updating version 32 to 33 2024-04-12T11:16:11.993 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 33 (ObjNum 207 snap 60 seq_num 207) dirty exists 2024-04-12T11:16:11.993 INFO:tasks.rados.rados.0.smithi029.stdout:458: left oid 8 (ObjNum 207 snap 60 seq_num 207) 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: osdmap e179: 8 total, 8 up, 8 in 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi029 to 2481M 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[29349]: osdmap e180: 8 total, 8 up, 8 in 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: osdmap e179: 8 total, 8 up, 8 in 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:12.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi029 to 2481M 2024-04-12T11:16:12.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:12.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:11 smithi029 ceph-mon[39999]: osdmap e180: 8 total, 8 up, 8 in 2024-04-12T11:16:12.538 INFO:tasks.rados.rados.0.smithi029.stdout:454: done (5 left) 2024-04-12T11:16:12.538 INFO:tasks.rados.rados.0.smithi029.stdout:455: done (4 left) 2024-04-12T11:16:12.538 INFO:tasks.rados.rados.0.smithi029.stdout:456: done (3 left) 2024-04-12T11:16:12.539 INFO:tasks.rados.rados.0.smithi029.stdout:457: done (2 left) 2024-04-12T11:16:12.539 INFO:tasks.rados.rados.0.smithi029.stdout:458: done (1 left) 2024-04-12T11:16:12.539 INFO:tasks.rados.rados.0.smithi029.stdout:461: done (0 left) 2024-04-12T11:16:12.539 INFO:tasks.rados.rados.0.smithi029.stdout:462: delete oid 40 current snap is 61 2024-04-12T11:16:12.541 INFO:tasks.rados.rados.0.smithi029.stdout:462: done (0 left) 2024-04-12T11:16:12.541 INFO:tasks.rados.rados.0.smithi029.stdout:463: rollback oid 3 current snap is 61 2024-04-12T11:16:12.541 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 47 2024-04-12T11:16:12.541 INFO:tasks.rados.rados.0.smithi029.stdout:464: rollback oid 19 current snap is 61 2024-04-12T11:16:12.541 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 19 to 60 2024-04-12T11:16:12.541 INFO:tasks.rados.rados.0.smithi029.stdout:465: snap_remove snap 53 2024-04-12T11:16:13.067 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:12 smithi029 ceph-mon[29349]: pgmap v17: 89 pgs: 89 active+clean; 189 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 45 op/s 2024-04-12T11:16:13.067 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:12 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:13.067 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:12 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:13.067 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:12 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2922523066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:13.067 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:12 smithi029 ceph-mon[29349]: osdmap e181: 8 total, 8 up, 8 in 2024-04-12T11:16:13.068 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:12 smithi029 ceph-mon[39999]: pgmap v17: 89 pgs: 89 active+clean; 189 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 45 op/s 2024-04-12T11:16:13.068 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:12 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:13.068 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:12 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:13.068 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:12 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2922523066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:13.068 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:12 smithi029 ceph-mon[39999]: osdmap e181: 8 total, 8 up, 8 in 2024-04-12T11:16:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:12 smithi087 ceph-mon[32917]: pgmap v17: 89 pgs: 89 active+clean; 189 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 45 op/s 2024-04-12T11:16:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:12 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:12 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:12 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2922523066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:13.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:12 smithi087 ceph-mon[32917]: osdmap e181: 8 total, 8 up, 8 in 2024-04-12T11:16:13.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[29581]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:16:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-12T11:16:13.541 INFO:tasks.rados.rados.0.smithi029.stdout:465: done (2 left) 2024-04-12T11:16:13.541 INFO:tasks.rados.rados.0.smithi029.stdout:466: delete oid 29 current snap is 61 2024-04-12T11:16:13.541 INFO:tasks.rados.rados.0.smithi029.stdout:463: finishing rollback tid 0 to smithi02979277-3 2024-04-12T11:16:13.541 INFO:tasks.rados.rados.0.smithi029.stdout:463: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:16:13.541 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 263 (ObjNum 289 snap 0 seq_num 862158154) dirty dne 2024-04-12T11:16:13.541 INFO:tasks.rados.rados.0.smithi029.stdout:464: finishing rollback tid 0 to smithi02979277-19 2024-04-12T11:16:13.541 INFO:tasks.rados.rados.0.smithi029.stdout:464: finishing rollback tid 1 to smithi02979277-19 2024-04-12T11:16:13.541 INFO:tasks.rados.rados.0.smithi029.stdout:464: finishing rollback tid 2 to smithi02979277-19 2024-04-12T11:16:13.541 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 189 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-04-12T11:16:13.547 INFO:tasks.rados.rados.0.smithi029.stdout:463: done (2 left) 2024-04-12T11:16:13.547 INFO:tasks.rados.rados.0.smithi029.stdout:464: done (1 left) 2024-04-12T11:16:13.547 INFO:tasks.rados.rados.0.smithi029.stdout:466: done (0 left) 2024-04-12T11:16:13.547 INFO:tasks.rados.rados.0.smithi029.stdout:467: snap_create 2024-04-12T11:16:14.545 INFO:tasks.rados.rados.0.smithi029.stdout:467: done (0 left) 2024-04-12T11:16:14.545 INFO:tasks.rados.rados.0.smithi029.stdout:468: read oid 26 snap -1 2024-04-12T11:16:14.545 INFO:tasks.rados.rados.0.smithi029.stdout:468: expect (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:16:14.545 INFO:tasks.rados.rados.0.smithi029.stdout:469: delete oid 16 current snap is 62 2024-04-12T11:16:14.548 INFO:tasks.rados.rados.0.smithi029.stdout:469: done (1 left) 2024-04-12T11:16:14.548 INFO:tasks.rados.rados.0.smithi029.stdout:470: snap_remove snap 38 2024-04-12T11:16:14.930 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:14 smithi087 ceph-mon[32917]: pgmap v20: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:16:14.930 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:14 smithi087 ceph-mon[32917]: osdmap e182: 8 total, 8 up, 8 in 2024-04-12T11:16:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:14 smithi029 ceph-mon[39999]: pgmap v20: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:16:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:14 smithi029 ceph-mon[39999]: osdmap e182: 8 total, 8 up, 8 in 2024-04-12T11:16:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:14 smithi029 ceph-mon[29349]: pgmap v20: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:16:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:14 smithi029 ceph-mon[29349]: osdmap e182: 8 total, 8 up, 8 in 2024-04-12T11:16:15.550 INFO:tasks.rados.rados.0.smithi029.stdout:470: done (1 left) 2024-04-12T11:16:15.550 INFO:tasks.rados.rados.0.smithi029.stdout:471: snap_remove snap 44 2024-04-12T11:16:15.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: osdmap e183: 8 total, 8 up, 8 in 2024-04-12T11:16:15.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1042020115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:15 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: osdmap e183: 8 total, 8 up, 8 in 2024-04-12T11:16:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1042020115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:16.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: osdmap e183: 8 total, 8 up, 8 in 2024-04-12T11:16:16.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1042020115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:16.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T11:16:16.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:16.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:15 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:16.542 INFO:tasks.rados.rados.0.smithi029.stdout:471: done (1 left) 2024-04-12T11:16:16.542 INFO:tasks.rados.rados.0.smithi029.stdout:472: snap_create 2024-04-12T11:16:16.543 INFO:tasks.rados.rados.0.smithi029.stdout:468: expect (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:16:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: pgmap v23: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: Adjusting osd_memory_target on smithi087 to 2480M 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: osdmap e184: 8 total, 8 up, 8 in 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:16 smithi087 ceph-mon[32917]: osdmap e185: 8 total, 8 up, 8 in 2024-04-12T11:16:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: pgmap v23: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:16:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: Adjusting osd_memory_target on smithi087 to 2480M 2024-04-12T11:16:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:16:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:16:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: osdmap e184: 8 total, 8 up, 8 in 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[29349]: osdmap e185: 8 total, 8 up, 8 in 2024-04-12T11:16:17.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: pgmap v23: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: Adjusting osd_memory_target on smithi087 to 2480M 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: osdmap e184: 8 total, 8 up, 8 in 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:17.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:17.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:17.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:17.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:16 smithi029 ceph-mon[39999]: osdmap e185: 8 total, 8 up, 8 in 2024-04-12T11:16:17.542 INFO:tasks.rados.rados.0.smithi029.stdout:468: done (1 left) 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:472: done (0 left) 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:473: read oid 27 snap -1 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:473: expect deleted 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:474: rollback oid 48 current snap is 63 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 31 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:475: read oid 17 snap 35 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:475: expect (ObjNum 105 snap 16 seq_num 105) 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:476: read oid 14 snap -1 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:476: expect (ObjNum 185 snap 49 seq_num 185) 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:477: write oid 46 current snap is 63 2024-04-12T11:16:17.543 INFO:tasks.rados.rados.0.smithi029.stdout:477: seq_num 211 ranges {746277=619070,1830877=681726,2928369=617908} 2024-04-12T11:16:17.549 INFO:tasks.rados.rados.0.smithi029.stdout:477: writing smithi02979277-46 from 746277 to 1365347 tid 1 2024-04-12T11:16:17.556 INFO:tasks.rados.rados.0.smithi029.stdout:477: writing smithi02979277-46 from 1830877 to 2512603 tid 2 2024-04-12T11:16:17.563 INFO:tasks.rados.rados.0.smithi029.stdout:477: writing smithi02979277-46 from 2928369 to 3546277 tid 3 2024-04-12T11:16:17.563 INFO:tasks.rados.rados.0.smithi029.stdout:478: snap_remove snap 54 2024-04-12T11:16:17.634 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:17 smithi087 ceph-mon[32917]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:17.634 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:17 smithi087 ceph-mon[32917]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:17.634 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:17 smithi087 ceph-mon[32917]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T11:16:17.635 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:17 smithi087 ceph-mon[32917]: Reconfiguring daemon prometheus.a on smithi087 2024-04-12T11:16:17.635 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:17 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3364076332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:17.635 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:17 smithi087 ceph-mon[32917]: osdmap e186: 8 total, 8 up, 8 in 2024-04-12T11:16:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[39999]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[39999]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T11:16:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[39999]: Reconfiguring daemon prometheus.a on smithi087 2024-04-12T11:16:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3364076332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[39999]: osdmap e186: 8 total, 8 up, 8 in 2024-04-12T11:16:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[29349]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:18.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[29349]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:18.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[29349]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T11:16:18.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[29349]: Reconfiguring daemon prometheus.a on smithi087 2024-04-12T11:16:18.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3364076332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:18.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:17 smithi029 ceph-mon[29349]: osdmap e186: 8 total, 8 up, 8 in 2024-04-12T11:16:18.570 INFO:tasks.rados.rados.0.smithi029.stdout:478: done (5 left) 2024-04-12T11:16:18.570 INFO:tasks.rados.rados.0.smithi029.stdout:479: read oid 4 snap -1 2024-04-12T11:16:18.570 INFO:tasks.rados.rados.0.smithi029.stdout:479: expect (ObjNum 192 snap 51 seq_num 192) 2024-04-12T11:16:18.571 INFO:tasks.rados.rados.0.smithi029.stdout:480: rollback oid 7 current snap is 63 2024-04-12T11:16:18.571 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 57 2024-04-12T11:16:18.571 INFO:tasks.rados.rados.0.smithi029.stdout:474: finishing rollback tid 0 to smithi02979277-48 2024-04-12T11:16:18.571 INFO:tasks.rados.rados.0.smithi029.stdout:474: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:16:18.571 INFO:tasks.rados.rados.0.smithi029.stdout:474: finishing rollback tid 2 to smithi02979277-48 2024-04-12T11:16:18.571 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 176 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-04-12T11:16:18.571 INFO:tasks.rados.rados.0.smithi029.stdout:477: finishing write tid 1 to smithi02979277-46 2024-04-12T11:16:18.571 INFO:tasks.rados.rados.0.smithi029.stdout:476: expect (ObjNum 185 snap 49 seq_num 185) 2024-04-12T11:16:18.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:18 smithi087 ceph-mon[32917]: pgmap v26: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:18.763 INFO:tasks.rados.rados.0.smithi029.stdout:477: finishing write tid 2 to smithi02979277-46 2024-04-12T11:16:18.763 INFO:tasks.rados.rados.0.smithi029.stdout:475: expect (ObjNum 105 snap 16 seq_num 105) 2024-04-12T11:16:18.886 INFO:tasks.rados.rados.0.smithi029.stdout:477: finishing write tid 3 to smithi02979277-46 2024-04-12T11:16:18.886 INFO:tasks.rados.rados.0.smithi029.stdout:477: finishing write tid 4 to smithi02979277-46 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:473: done (6 left) 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:474: done (5 left) 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:475: done (4 left) 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:476: done (3 left) 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:481: rollback oid 14 current snap is 63 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 57 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:477: finishing write tid 5 to smithi02979277-46 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:477: oid 46 updating version 0 to 107 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:477: oid 46 updating version 107 to 108 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:477: oid 46 version 108 is already newer than 106 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:477: oid 46 version 108 is already newer than 105 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 108 (ObjNum 211 snap 63 seq_num 211) dirty exists 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:477: left oid 46 (ObjNum 211 snap 63 seq_num 211) 2024-04-12T11:16:18.887 INFO:tasks.rados.rados.0.smithi029.stdout:480: finishing rollback tid 0 to smithi02979277-7 2024-04-12T11:16:18.888 INFO:tasks.rados.rados.0.smithi029.stdout:480: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:16:18.888 INFO:tasks.rados.rados.0.smithi029.stdout:480: finishing rollback tid 2 to smithi02979277-7 2024-04-12T11:16:18.888 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 221 (ObjNum 171 snap 44 seq_num 171) dirty exists 2024-04-12T11:16:18.888 INFO:tasks.rados.rados.0.smithi029.stdout:479: expect (ObjNum 192 snap 51 seq_num 192) 2024-04-12T11:16:18.992 INFO:tasks.rados.rados.0.smithi029.stdout:481: finishing rollback tid 0 to smithi02979277-14 2024-04-12T11:16:18.993 INFO:tasks.rados.rados.0.smithi029.stdout:481: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:16:18.993 INFO:tasks.rados.rados.0.smithi029.stdout:481: finishing rollback tid 2 to smithi02979277-14 2024-04-12T11:16:18.993 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 223 (ObjNum 185 snap 49 seq_num 185) dirty exists 2024-04-12T11:16:18.993 INFO:tasks.rados.rados.0.smithi029.stdout:477: done (3 left) 2024-04-12T11:16:18.993 INFO:tasks.rados.rados.0.smithi029.stdout:479: done (2 left) 2024-04-12T11:16:18.993 INFO:tasks.rados.rados.0.smithi029.stdout:480: done (1 left) 2024-04-12T11:16:18.993 INFO:tasks.rados.rados.0.smithi029.stdout:481: done (0 left) 2024-04-12T11:16:18.993 INFO:tasks.rados.rados.0.smithi029.stdout:482: rollback oid 42 current snap is 63 2024-04-12T11:16:18.993 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 59 2024-04-12T11:16:18.993 INFO:tasks.rados.rados.0.smithi029.stdout:483: snap_remove snap 31 2024-04-12T11:16:19.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:18 smithi029 ceph-mon[29349]: pgmap v26: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:18 smithi029 ceph-mon[39999]: pgmap v26: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:19.575 INFO:tasks.rados.rados.0.smithi029.stdout:482: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:16:19.575 INFO:tasks.rados.rados.0.smithi029.stdout:482: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:16:19.575 INFO:tasks.rados.rados.0.smithi029.stdout:482: finishing rollback tid 2 to smithi02979277-42 2024-04-12T11:16:19.575 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 34 (ObjNum 87 snap 11 seq_num 87) dirty exists 2024-04-12T11:16:19.575 INFO:tasks.rados.rados.0.smithi029.stdout:482: done (1 left) 2024-04-12T11:16:19.575 INFO:tasks.rados.rados.0.smithi029.stdout:483: done (0 left) 2024-04-12T11:16:19.576 INFO:tasks.rados.rados.0.smithi029.stdout:484: delete oid 22 current snap is 63 2024-04-12T11:16:19.587 INFO:tasks.rados.rados.0.smithi029.stdout:484: done (0 left) 2024-04-12T11:16:19.587 INFO:tasks.rados.rados.0.smithi029.stdout:485: read oid 37 snap -1 2024-04-12T11:16:19.587 INFO:tasks.rados.rados.0.smithi029.stdout:485: expect (ObjNum 36 snap 0 seq_num 36) 2024-04-12T11:16:19.587 INFO:tasks.rados.rados.0.smithi029.stdout:486: read oid 47 snap 48 2024-04-12T11:16:19.587 INFO:tasks.rados.rados.0.smithi029.stdout:486: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:16:19.587 INFO:tasks.rados.rados.0.smithi029.stdout:487: snap_remove snap 49 2024-04-12T11:16:19.842 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:19 smithi087 ceph-mon[32917]: osdmap e187: 8 total, 8 up, 8 in 2024-04-12T11:16:19.842 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:19 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1473417724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:19.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:19 smithi029 ceph-mon[39999]: osdmap e187: 8 total, 8 up, 8 in 2024-04-12T11:16:19.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:19 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1473417724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:19 smithi029 ceph-mon[29349]: osdmap e187: 8 total, 8 up, 8 in 2024-04-12T11:16:19.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:19 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1473417724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:20.216 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:19 smithi087 systemd[1]: Stopping Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:16:20.505 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.229Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-12T11:16:20.505 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.229Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-12T11:16:20.505 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.229Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-12T11:16:20.505 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.229Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-12T11:16:20.506 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.229Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-12T11:16:20.506 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.229Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-12T11:16:20.506 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.229Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-12T11:16:20.506 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.229Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-12T11:16:20.506 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.230Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-12T11:16:20.506 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.230Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-12T11:16:20.506 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.230Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-12T11:16:20.506 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[65051]: ts=2024-04-12T11:16:20.230Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-12T11:16:20.506 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 podman[75329]: 2024-04-12 11:16:20.239305478 +0000 UTC m=+0.126225717 container died 06b93cbb7d0ce332957f7651c0e0c2420d7623e754a17415c085da0f09473b7d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:20.580 INFO:tasks.rados.rados.0.smithi029.stdout:487: done (2 left) 2024-04-12T11:16:20.580 INFO:tasks.rados.rados.0.smithi029.stdout:488: rollback oid 17 current snap is 63 2024-04-12T11:16:20.580 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 36 2024-04-12T11:16:20.580 INFO:tasks.rados.rados.0.smithi029.stdout:489: write oid 19 current snap is 63 2024-04-12T11:16:20.580 INFO:tasks.rados.rados.0.smithi029.stdout:489: seq_num 213 ranges {779891=485921,1947841=685395,3383215=196676} 2024-04-12T11:16:20.588 INFO:tasks.rados.rados.0.smithi029.stdout:489: writing smithi02979277-19 from 779891 to 1265812 tid 1 2024-04-12T11:16:20.596 INFO:tasks.rados.rados.0.smithi029.stdout:489: writing smithi02979277-19 from 1947841 to 2633236 tid 2 2024-04-12T11:16:20.601 INFO:tasks.rados.rados.0.smithi029.stdout:489: writing smithi02979277-19 from 3383215 to 3579891 tid 3 2024-04-12T11:16:20.601 INFO:tasks.rados.rados.0.smithi029.stdout:490: delete oid 39 current snap is 63 2024-04-12T11:16:20.601 INFO:tasks.rados.rados.0.smithi029.stdout:485: expect (ObjNum 36 snap 0 seq_num 36) 2024-04-12T11:16:20.686 INFO:tasks.rados.rados.0.smithi029.stdout:486: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:16:20.840 INFO:tasks.rados.rados.0.smithi029.stdout:488: finishing rollback tid 0 to smithi02979277-17 2024-04-12T11:16:20.840 INFO:tasks.rados.rados.0.smithi029.stdout:488: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:16:20.840 INFO:tasks.rados.rados.0.smithi029.stdout:488: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:16:20.840 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 203 (ObjNum 105 snap 16 seq_num 105) dirty exists 2024-04-12T11:16:20.840 INFO:tasks.rados.rados.0.smithi029.stdout:489: finishing write tid 1 to smithi02979277-19 2024-04-12T11:16:20.840 INFO:tasks.rados.rados.0.smithi029.stdout:489: finishing write tid 2 to smithi02979277-19 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:485: done (4 left) 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:486: done (3 left) 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:488: done (2 left) 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:490: done (1 left) 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:491: write oid 42 current snap is 63 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:489: finishing write tid 3 to smithi02979277-19 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:489: finishing write tid 4 to smithi02979277-19 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:489: finishing write tid 5 to smithi02979277-19 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:489: oid 19 updating version 0 to 243 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:489: oid 19 version 243 is already newer than 241 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:489: oid 19 updating version 243 to 244 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:489: oid 19 version 244 is already newer than 242 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 244 (ObjNum 213 snap 63 seq_num 213) dirty exists 2024-04-12T11:16:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:489: left oid 19 (ObjNum 213 snap 63 seq_num 213) 2024-04-12T11:16:20.842 INFO:tasks.rados.rados.0.smithi029.stdout:491: seq_num 215 ranges {413505=636419,1664805=689064,3038061=492071} 2024-04-12T11:16:20.845 INFO:tasks.rados.rados.0.smithi029.stdout:491: writing smithi02979277-42 from 413505 to 1049924 tid 1 2024-04-12T11:16:20.852 INFO:tasks.rados.rados.0.smithi029.stdout:491: writing smithi02979277-42 from 1664805 to 2353869 tid 2 2024-04-12T11:16:20.859 INFO:tasks.rados.rados.0.smithi029.stdout:491: writing smithi02979277-42 from 3038061 to 3530132 tid 3 2024-04-12T11:16:20.860 INFO:tasks.rados.rados.0.smithi029.stdout:489: done (1 left) 2024-04-12T11:16:20.860 INFO:tasks.rados.rados.0.smithi029.stdout:492: read oid 20 snap 62 2024-04-12T11:16:20.860 INFO:tasks.rados.rados.0.smithi029.stdout:492: expect deleted 2024-04-12T11:16:20.860 INFO:tasks.rados.rados.0.smithi029.stdout:493: write oid 28 current snap is 63 2024-04-12T11:16:20.860 INFO:tasks.rados.rados.0.smithi029.stdout:491: finishing write tid 1 to smithi02979277-42 2024-04-12T11:16:20.860 INFO:tasks.rados.rados.0.smithi029.stdout:491: finishing write tid 2 to smithi02979277-42 2024-04-12T11:16:20.860 INFO:tasks.rados.rados.0.smithi029.stdout:493: seq_num 216 ranges {430312=711668,1723287=549075,2781837=703343} 2024-04-12T11:16:20.865 INFO:tasks.rados.rados.0.smithi029.stdout:493: writing smithi02979277-28 from 430312 to 1141980 tid 1 2024-04-12T11:16:20.871 INFO:tasks.rados.rados.0.smithi029.stdout:493: writing smithi02979277-28 from 1723287 to 2272362 tid 2 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:493: writing smithi02979277-28 from 2781837 to 3485180 tid 3 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:494: read oid 27 snap -1 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:494: expect deleted 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:495: rollback oid 41 current snap is 63 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 60 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:491: finishing write tid 3 to smithi02979277-42 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:491: finishing write tid 4 to smithi02979277-42 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:491: finishing write tid 5 to smithi02979277-42 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:491: oid 42 updating version 0 to 42 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:491: oid 42 version 42 is already newer than 41 2024-04-12T11:16:20.879 INFO:tasks.rados.rados.0.smithi029.stdout:491: oid 42 version 42 is already newer than 39 2024-04-12T11:16:20.880 INFO:tasks.rados.rados.0.smithi029.stdout:491: oid 42 version 42 is already newer than 40 2024-04-12T11:16:20.880 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 42 (ObjNum 215 snap 63 seq_num 215) dirty exists 2024-04-12T11:16:20.880 INFO:tasks.rados.rados.0.smithi029.stdout:491: left oid 42 (ObjNum 215 snap 63 seq_num 215) 2024-04-12T11:16:20.880 INFO:tasks.rados.rados.0.smithi029.stdout:493: finishing write tid 1 to smithi02979277-28 2024-04-12T11:16:20.880 INFO:tasks.rados.rados.0.smithi029.stdout:493: finishing write tid 2 to smithi02979277-28 2024-04-12T11:16:20.880 INFO:tasks.rados.rados.0.smithi029.stdout:491: done (4 left) 2024-04-12T11:16:20.880 INFO:tasks.rados.rados.0.smithi029.stdout:492: done (3 left) 2024-04-12T11:16:20.880 INFO:tasks.rados.rados.0.smithi029.stdout:496: delete oid 1 current snap is 63 2024-04-12T11:16:20.881 INFO:tasks.rados.rados.0.smithi029.stdout:493: finishing write tid 3 to smithi02979277-28 2024-04-12T11:16:20.881 INFO:tasks.rados.rados.0.smithi029.stdout:493: finishing write tid 4 to smithi02979277-28 2024-04-12T11:16:20.881 INFO:tasks.rados.rados.0.smithi029.stdout:495: finishing rollback tid 0 to smithi02979277-41 2024-04-12T11:16:20.881 INFO:tasks.rados.rados.0.smithi029.stdout:493: finishing write tid 5 to smithi02979277-28 2024-04-12T11:16:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:493: oid 28 updating version 0 to 188 2024-04-12T11:16:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:493: oid 28 version 188 is already newer than 186 2024-04-12T11:16:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:493: oid 28 updating version 188 to 189 2024-04-12T11:16:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:493: oid 28 version 189 is already newer than 187 2024-04-12T11:16:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 189 (ObjNum 216 snap 63 seq_num 216) dirty exists 2024-04-12T11:16:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:493: left oid 28 (ObjNum 216 snap 63 seq_num 216) 2024-04-12T11:16:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:495: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:16:20.883 INFO:tasks.rados.rados.0.smithi029.stdout:495: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:16:20.883 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 225 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-04-12T11:16:20.883 INFO:tasks.rados.rados.0.smithi029.stdout:493: done (3 left) 2024-04-12T11:16:20.883 INFO:tasks.rados.rados.0.smithi029.stdout:494: done (2 left) 2024-04-12T11:16:20.883 INFO:tasks.rados.rados.0.smithi029.stdout:495: done (1 left) 2024-04-12T11:16:20.883 INFO:tasks.rados.rados.0.smithi029.stdout:496: done (0 left) 2024-04-12T11:16:20.883 INFO:tasks.rados.rados.0.smithi029.stdout:497: write oid 5 current snap is 63 2024-04-12T11:16:20.883 INFO:tasks.rados.rados.0.smithi029.stdout:497: seq_num 218 ranges {463926=462166,1723898=436391,2603977=725887} 2024-04-12T11:16:20.887 INFO:tasks.rados.rados.0.smithi029.stdout:497: writing smithi02979277-5 from 463926 to 926092 tid 1 2024-04-12T11:16:20.894 INFO:tasks.rados.rados.0.smithi029.stdout:497: writing smithi02979277-5 from 1723898 to 2160289 tid 2 2024-04-12T11:16:20.901 INFO:tasks.rados.rados.0.smithi029.stdout:497: writing smithi02979277-5 from 2603977 to 3329864 tid 3 2024-04-12T11:16:20.901 INFO:tasks.rados.rados.0.smithi029.stdout:498: rollback oid 41 current snap is 63 2024-04-12T11:16:20.901 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 48 2024-04-12T11:16:20.901 INFO:tasks.rados.rados.0.smithi029.stdout:497: finishing write tid 1 to smithi02979277-5 2024-04-12T11:16:20.901 INFO:tasks.rados.rados.0.smithi029.stdout:497: finishing write tid 2 to smithi02979277-5 2024-04-12T11:16:20.901 INFO:tasks.rados.rados.0.smithi029.stdout:499: write oid 9 current snap is 63 2024-04-12T11:16:20.901 INFO:tasks.rados.rados.0.smithi029.stdout:499: seq_num 219 ranges {480733=537415,1782380=696402,3031400=537159} 2024-04-12T11:16:20.906 INFO:tasks.rados.rados.0.smithi029.stdout:499: writing smithi02979277-9 from 480733 to 1018148 tid 1 2024-04-12T11:16:20.914 INFO:tasks.rados.rados.0.smithi029.stdout:499: writing smithi02979277-9 from 1782380 to 2478782 tid 2 2024-04-12T11:16:20.921 INFO:tasks.rados.rados.0.smithi029.stdout:499: writing smithi02979277-9 from 3031400 to 3568559 tid 3 2024-04-12T11:16:20.921 INFO:tasks.rados.rados.0.smithi029.stdout:500: read oid 43 snap -1 2024-04-12T11:16:20.921 INFO:tasks.rados.rados.0.smithi029.stdout:500: expect (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:501: write oid 38 current snap is 63 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:498: finishing rollback tid 0 to smithi02979277-41 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:498: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:498: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 229 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:497: finishing write tid 3 to smithi02979277-5 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:497: finishing write tid 4 to smithi02979277-5 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:497: finishing write tid 5 to smithi02979277-5 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:497: oid 5 updating version 0 to 249 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:497: oid 5 version 249 is already newer than 246 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:497: oid 5 version 249 is already newer than 248 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:497: oid 5 version 249 is already newer than 247 2024-04-12T11:16:20.922 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 249 (ObjNum 218 snap 63 seq_num 218) dirty exists 2024-04-12T11:16:20.923 INFO:tasks.rados.rados.0.smithi029.stdout:497: left oid 5 (ObjNum 218 snap 63 seq_num 218) 2024-04-12T11:16:20.923 INFO:tasks.rados.rados.0.smithi029.stdout:499: finishing write tid 1 to smithi02979277-9 2024-04-12T11:16:20.923 INFO:tasks.rados.rados.0.smithi029.stdout:499: finishing write tid 2 to smithi02979277-9 2024-04-12T11:16:20.923 INFO:tasks.rados.rados.0.smithi029.stdout:501: seq_num 220 ranges {497540=612664,1840862=440060,3058823=464784} 2024-04-12T11:16:20.927 INFO:tasks.rados.rados.0.smithi029.stdout:501: writing smithi02979277-38 from 497540 to 1110204 tid 1 2024-04-12T11:16:20.933 INFO:tasks.rados.rados.0.smithi029.stdout:501: writing smithi02979277-38 from 1840862 to 2280922 tid 2 2024-04-12T11:16:20.940 INFO:tasks.rados.rados.0.smithi029.stdout:501: writing smithi02979277-38 from 3058823 to 3523607 tid 3 2024-04-12T11:16:20.941 INFO:tasks.rados.rados.0.smithi029.stdout:497: done (4 left) 2024-04-12T11:16:20.941 INFO:tasks.rados.rados.0.smithi029.stdout:498: done (3 left) 2024-04-12T11:16:20.941 INFO:tasks.rados.rados.0.smithi029.stdout:502: snap_remove snap 48 2024-04-12T11:16:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-mon[32917]: pgmap v29: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:16:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:20 smithi087 ceph-mon[32917]: osdmap e188: 8 total, 8 up, 8 in 2024-04-12T11:16:20.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 podman[75329]: 2024-04-12 11:16:20.704466543 +0000 UTC m=+0.591386789 container cleanup 06b93cbb7d0ce332957f7651c0e0c2420d7623e754a17415c085da0f09473b7d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:20.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 bash[75329]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a 2024-04-12T11:16:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:20 smithi029 ceph-mon[29349]: pgmap v29: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:16:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:20 smithi029 ceph-mon[29349]: osdmap e188: 8 total, 8 up, 8 in 2024-04-12T11:16:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:20 smithi029 ceph-mon[39999]: pgmap v29: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:16:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:20 smithi029 ceph-mon[39999]: osdmap e188: 8 total, 8 up, 8 in 2024-04-12T11:16:21.331 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:20 smithi087 podman[75343]: 2024-04-12 11:16:20.997697057 +0000 UTC m=+0.760340194 container remove 06b93cbb7d0ce332957f7651c0e0c2420d7623e754a17415c085da0f09473b7d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:21.332 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-12T11:16:21.332 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 systemd[1]: Stopped Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:16:21.332 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@prometheus.a.service: Consumed 1.325s CPU time. 2024-04-12T11:16:21.332 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 systemd[1]: Starting Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:16:21.545 INFO:tasks.rados.rados.0.smithi029.stdout:502: done (3 left) 2024-04-12T11:16:21.545 INFO:tasks.rados.rados.0.smithi029.stdout:503: delete oid 12 current snap is 63 2024-04-12T11:16:21.545 INFO:tasks.rados.rados.0.smithi029.stdout:499: finishing write tid 3 to smithi02979277-9 2024-04-12T11:16:21.545 INFO:tasks.rados.rados.0.smithi029.stdout:499: finishing write tid 4 to smithi02979277-9 2024-04-12T11:16:21.546 INFO:tasks.rados.rados.0.smithi029.stdout:499: finishing write tid 5 to smithi02979277-9 2024-04-12T11:16:21.546 INFO:tasks.rados.rados.0.smithi029.stdout:499: oid 9 updating version 0 to 176 2024-04-12T11:16:21.546 INFO:tasks.rados.rados.0.smithi029.stdout:499: oid 9 updating version 176 to 178 2024-04-12T11:16:21.546 INFO:tasks.rados.rados.0.smithi029.stdout:499: oid 9 version 178 is already newer than 177 2024-04-12T11:16:21.546 INFO:tasks.rados.rados.0.smithi029.stdout:499: oid 9 updating version 178 to 179 2024-04-12T11:16:21.546 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 179 (ObjNum 219 snap 63 seq_num 219) dirty exists 2024-04-12T11:16:21.546 INFO:tasks.rados.rados.0.smithi029.stdout:499: left oid 9 (ObjNum 219 snap 63 seq_num 219) 2024-04-12T11:16:21.546 INFO:tasks.rados.rados.0.smithi029.stdout:500: expect (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:16:21.592 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-mon[32917]: osdmap e189: 8 total, 8 up, 8 in 2024-04-12T11:16:21.592 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-mon[32917]: osdmap e190: 8 total, 8 up, 8 in 2024-04-12T11:16:21.593 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 podman[75438]: 2024-04-12 11:16:21.236029553 +0000 UTC m=+0.018894150 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-12T11:16:21.593 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 podman[75438]: 2024-04-12 11:16:21.373742376 +0000 UTC m=+0.156606950 container create 354d7b898c83c64ae8e4cfa42dbe32a778d4cc1837de7291a2c3f5b403e8ea42 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:21.593 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 podman[75438]: 2024-04-12 11:16:21.593022336 +0000 UTC m=+0.375886910 container init 354d7b898c83c64ae8e4cfa42dbe32a778d4cc1837de7291a2c3f5b403e8ea42 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:21.772 INFO:tasks.rados.rados.0.smithi029.stdout:501: finishing write tid 1 to smithi02979277-38 2024-04-12T11:16:21.772 INFO:tasks.rados.rados.0.smithi029.stdout:501: finishing write tid 2 to smithi02979277-38 2024-04-12T11:16:21.772 INFO:tasks.rados.rados.0.smithi029.stdout:501: finishing write tid 3 to smithi02979277-38 2024-04-12T11:16:21.772 INFO:tasks.rados.rados.0.smithi029.stdout:501: finishing write tid 4 to smithi02979277-38 2024-04-12T11:16:21.772 INFO:tasks.rados.rados.0.smithi029.stdout:501: finishing write tid 5 to smithi02979277-38 2024-04-12T11:16:21.772 INFO:tasks.rados.rados.0.smithi029.stdout:501: oid 38 updating version 0 to 110 2024-04-12T11:16:21.772 INFO:tasks.rados.rados.0.smithi029.stdout:501: oid 38 updating version 110 to 113 2024-04-12T11:16:21.772 INFO:tasks.rados.rados.0.smithi029.stdout:501: oid 38 version 113 is already newer than 112 2024-04-12T11:16:21.772 INFO:tasks.rados.rados.0.smithi029.stdout:501: oid 38 version 113 is already newer than 111 2024-04-12T11:16:21.772 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 113 (ObjNum 220 snap 63 seq_num 220) dirty exists 2024-04-12T11:16:21.773 INFO:tasks.rados.rados.0.smithi029.stdout:501: left oid 38 (ObjNum 220 snap 63 seq_num 220) 2024-04-12T11:16:21.773 INFO:tasks.rados.rados.0.smithi029.stdout:499: done (3 left) 2024-04-12T11:16:21.773 INFO:tasks.rados.rados.0.smithi029.stdout:500: done (2 left) 2024-04-12T11:16:21.773 INFO:tasks.rados.rados.0.smithi029.stdout:501: done (1 left) 2024-04-12T11:16:21.773 INFO:tasks.rados.rados.0.smithi029.stdout:503: done (0 left) 2024-04-12T11:16:21.773 INFO:tasks.rados.rados.0.smithi029.stdout:504: snap_create 2024-04-12T11:16:21.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 podman[75438]: 2024-04-12 11:16:21.595832991 +0000 UTC m=+0.378697577 container start 354d7b898c83c64ae8e4cfa42dbe32a778d4cc1837de7291a2c3f5b403e8ea42 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:21.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.619Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-12T11:16:21.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.619Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-12T11:16:21.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.619Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi087 (none))" 2024-04-12T11:16:21.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.619Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-12T11:16:21.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.619Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-12T11:16:21.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.620Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-12T11:16:21.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.621Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-12T11:16:21.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.622Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-12T11:16:21.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.622Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.624Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.624Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.674µs 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.624Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.625Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.631Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.631Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.631Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.042µs wal_replay_duration=7.699546ms wbl_replay_duration=108ns total_replay_duration=7.736342ms 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.633Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.633Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.633Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.652Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.661362ms db_storage=765ns remote_storage=912ns web_handler=426ns query_engine=564ns scrape=384.362µs scrape_sd=82.803µs notify=16.406µs notify_sd=14.34µs rules=18.724165ms tracing=13.135µs 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.652Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-12T11:16:21.968 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:21.653Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-12T11:16:21.969 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 bash[75438]: 354d7b898c83c64ae8e4cfa42dbe32a778d4cc1837de7291a2c3f5b403e8ea42 2024-04-12T11:16:21.969 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:21 smithi087 systemd[1]: Started Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:16:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:21 smithi029 ceph-mon[39999]: osdmap e189: 8 total, 8 up, 8 in 2024-04-12T11:16:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:21 smithi029 ceph-mon[39999]: osdmap e190: 8 total, 8 up, 8 in 2024-04-12T11:16:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:21 smithi029 ceph-mon[29349]: osdmap e189: 8 total, 8 up, 8 in 2024-04-12T11:16:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:21 smithi029 ceph-mon[29349]: osdmap e190: 8 total, 8 up, 8 in 2024-04-12T11:16:22.732 INFO:tasks.rados.rados.0.smithi029.stdout:504: done (0 left) 2024-04-12T11:16:22.733 INFO:tasks.rados.rados.0.smithi029.stdout:505: write oid 32 current snap is 64 2024-04-12T11:16:22.733 INFO:tasks.rados.rados.0.smithi029.stdout:505: seq_num 222 ranges {531154=479515,1557826=443729,2713669=487328,3679295=51859} 2024-04-12T11:16:22.737 INFO:tasks.rados.rados.0.smithi029.stdout:505: writing smithi02979277-32 from 531154 to 1010669 tid 1 2024-04-12T11:16:22.743 INFO:tasks.rados.rados.0.smithi029.stdout:505: writing smithi02979277-32 from 1557826 to 2001555 tid 2 2024-04-12T11:16:22.750 INFO:tasks.rados.rados.0.smithi029.stdout:505: writing smithi02979277-32 from 2713669 to 3200997 tid 3 2024-04-12T11:16:22.753 INFO:tasks.rados.rados.0.smithi029.stdout:505: writing smithi02979277-32 from 3679295 to 3731154 tid 4 2024-04-12T11:16:22.753 INFO:tasks.rados.rados.0.smithi029.stdout:506: read oid 47 snap -1 2024-04-12T11:16:22.754 INFO:tasks.rados.rados.0.smithi029.stdout:506: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:16:22.754 INFO:tasks.rados.rados.0.smithi029.stdout:507: write oid 21 current snap is 64 2024-04-12T11:16:22.754 INFO:tasks.rados.rados.0.smithi029.stdout:505: finishing write tid 1 to smithi02979277-32 2024-04-12T11:16:22.754 INFO:tasks.rados.rados.0.smithi029.stdout:507: seq_num 223 ranges {547961=554764,1616308=703740,2741092=698600} 2024-04-12T11:16:22.759 INFO:tasks.rados.rados.0.smithi029.stdout:507: writing smithi02979277-21 from 547961 to 1102725 tid 1 2024-04-12T11:16:22.766 INFO:tasks.rados.rados.0.smithi029.stdout:507: writing smithi02979277-21 from 1616308 to 2320048 tid 2 2024-04-12T11:16:22.772 INFO:tasks.rados.rados.0.smithi029.stdout:507: writing smithi02979277-21 from 2741092 to 3439692 tid 3 2024-04-12T11:16:22.772 INFO:tasks.rados.rados.0.smithi029.stdout:508: rollback oid 11 current snap is 64 2024-04-12T11:16:22.772 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 11 to 61 2024-04-12T11:16:22.773 INFO:tasks.rados.rados.0.smithi029.stdout:505: finishing write tid 2 to smithi02979277-32 2024-04-12T11:16:22.773 INFO:tasks.rados.rados.0.smithi029.stdout:505: finishing write tid 3 to smithi02979277-32 2024-04-12T11:16:22.773 INFO:tasks.rados.rados.0.smithi029.stdout:506: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:16:22.928 INFO:tasks.rados.rados.0.smithi029.stdout:505: finishing write tid 4 to smithi02979277-32 2024-04-12T11:16:22.928 INFO:tasks.rados.rados.0.smithi029.stdout:505: finishing write tid 5 to smithi02979277-32 2024-04-12T11:16:22.928 INFO:tasks.rados.rados.0.smithi029.stdout:505: finishing write tid 6 to smithi02979277-32 2024-04-12T11:16:22.928 INFO:tasks.rados.rados.0.smithi029.stdout:505: oid 32 updating version 0 to 169 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:505: oid 32 updating version 169 to 171 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:505: oid 32 updating version 171 to 172 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:505: oid 32 version 172 is already newer than 168 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:505: oid 32 version 172 is already newer than 170 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 172 (ObjNum 222 snap 64 seq_num 222) dirty exists 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:505: left oid 32 (ObjNum 222 snap 64 seq_num 222) 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:507: finishing write tid 1 to smithi02979277-21 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:507: finishing write tid 2 to smithi02979277-21 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:508: finishing rollback tid 0 to smithi02979277-11 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:508: finishing rollback tid 1 to smithi02979277-11 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:508: finishing rollback tid 2 to smithi02979277-11 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 311 (ObjNum 178 snap 47 seq_num 178) dirty exists 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:507: finishing write tid 3 to smithi02979277-21 2024-04-12T11:16:22.929 INFO:tasks.rados.rados.0.smithi029.stdout:507: finishing write tid 4 to smithi02979277-21 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:507: finishing write tid 5 to smithi02979277-21 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:507: oid 21 updating version 0 to 242 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:507: oid 21 version 242 is already newer than 240 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:507: oid 21 version 242 is already newer than 239 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:507: oid 21 version 242 is already newer than 241 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 242 (ObjNum 223 snap 64 seq_num 223) dirty exists 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:507: left oid 21 (ObjNum 223 snap 64 seq_num 223) 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:505: done (3 left) 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:506: done (2 left) 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:507: done (1 left) 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:508: done (0 left) 2024-04-12T11:16:22.930 INFO:tasks.rados.rados.0.smithi029.stdout:509: snap_create 2024-04-12T11:16:23.003 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[29349]: pgmap v32: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:16:23.003 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:23.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:23.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:23.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[29349]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:23.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:23.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[29349]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T11:16:23.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[29349]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T11:16:23.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1282146347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:23.004 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[39999]: pgmap v32: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:16:23.004 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:23.004 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:23.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:23.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[39999]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:23.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:23.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[39999]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T11:16:23.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[39999]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T11:16:23.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:22 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1282146347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:22 smithi087 ceph-mon[32917]: pgmap v32: 89 pgs: 89 active+clean; 193 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:16:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:22 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:22 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:22 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:22 smithi087 ceph-mon[32917]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:22 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:22 smithi087 ceph-mon[32917]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T11:16:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:22 smithi087 ceph-mon[32917]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-12T11:16:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:22 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1282146347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:23.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[29349]: osdmap e191: 8 total, 8 up, 8 in 2024-04-12T11:16:23.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:23.738 INFO:tasks.rados.rados.0.smithi029.stdout:509: done (0 left) 2024-04-12T11:16:23.738 INFO:tasks.rados.rados.0.smithi029.stdout:510: delete oid 45 current snap is 65 2024-04-12T11:16:23.743 INFO:tasks.rados.rados.0.smithi029.stdout:510: done (0 left) 2024-04-12T11:16:23.743 INFO:tasks.rados.rados.0.smithi029.stdout:511: snap_create 2024-04-12T11:16:24.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:16:24.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:16:24.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:16:24.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:24.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[39999]: osdmap e191: 8 total, 8 up, 8 in 2024-04-12T11:16:24.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:24.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:16:24.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:16:24.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:16:24.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:23 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:24.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:23 smithi087 ceph-mon[32917]: osdmap e191: 8 total, 8 up, 8 in 2024-04-12T11:16:24.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:23 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:24.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:23 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:16:24.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:23 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:16:24.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:23 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:16:24.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:23 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:511: done (0 left) 2024-04-12T11:16:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:512: read oid 2 snap -1 2024-04-12T11:16:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:512: expect deleted 2024-04-12T11:16:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:513: rollback oid 25 current snap is 66 2024-04-12T11:16:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 25 to 64 2024-04-12T11:16:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:514: write oid 50 current snap is 66 2024-04-12T11:16:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:514: seq_num 225 ranges {581575=705262,1733272=707409,3195938=437497} 2024-04-12T11:16:24.749 INFO:tasks.rados.rados.0.smithi029.stdout:514: writing smithi02979277-50 from 581575 to 1286837 tid 1 2024-04-12T11:16:24.756 INFO:tasks.rados.rados.0.smithi029.stdout:514: writing smithi02979277-50 from 1733272 to 2440681 tid 2 2024-04-12T11:16:24.763 INFO:tasks.rados.rados.0.smithi029.stdout:514: writing smithi02979277-50 from 3195938 to 3633435 tid 3 2024-04-12T11:16:24.763 INFO:tasks.rados.rados.0.smithi029.stdout:515: read oid 34 snap -1 2024-04-12T11:16:24.763 INFO:tasks.rados.rados.0.smithi029.stdout:515: expect deleted 2024-04-12T11:16:24.763 INFO:tasks.rados.rados.0.smithi029.stdout:516: read oid 1 snap 43 2024-04-12T11:16:24.763 INFO:tasks.rados.rados.0.smithi029.stdout:516: expect deleted 2024-04-12T11:16:24.763 INFO:tasks.rados.rados.0.smithi029.stdout:517: read oid 39 snap -1 2024-04-12T11:16:24.763 INFO:tasks.rados.rados.0.smithi029.stdout:517: expect deleted 2024-04-12T11:16:24.763 INFO:tasks.rados.rados.0.smithi029.stdout:518: write oid 27 current snap is 66 2024-04-12T11:16:24.764 INFO:tasks.rados.rados.0.smithi029.stdout:518: seq_num 226 ranges {598382=780511,2075401=451067,3107008=648769} 2024-04-12T11:16:24.770 INFO:tasks.rados.rados.0.smithi029.stdout:518: writing smithi02979277-27 from 598382 to 1378893 tid 1 2024-04-12T11:16:24.776 INFO:tasks.rados.rados.0.smithi029.stdout:518: writing smithi02979277-27 from 2075401 to 2526468 tid 2 2024-04-12T11:16:24.784 INFO:tasks.rados.rados.0.smithi029.stdout:518: writing smithi02979277-27 from 3107008 to 3755777 tid 3 2024-04-12T11:16:24.784 INFO:tasks.rados.rados.0.smithi029.stdout:519: write oid 44 current snap is 66 2024-04-12T11:16:24.784 INFO:tasks.rados.rados.0.smithi029.stdout:519: seq_num 227 ranges {615189=455760,1733883=711078,3134431=460041} 2024-04-12T11:16:24.789 INFO:tasks.rados.rados.0.smithi029.stdout:519: writing smithi02979277-44 from 615189 to 1070949 tid 1 2024-04-12T11:16:24.797 INFO:tasks.rados.rados.0.smithi029.stdout:519: writing smithi02979277-44 from 1733883 to 2444961 tid 2 2024-04-12T11:16:24.805 INFO:tasks.rados.rados.0.smithi029.stdout:519: writing smithi02979277-44 from 3134431 to 3594472 tid 3 2024-04-12T11:16:24.805 INFO:tasks.rados.rados.0.smithi029.stdout:520: read oid 3 snap 59 2024-04-12T11:16:24.805 INFO:tasks.rados.rados.0.smithi029.stdout:520: expect (ObjNum 202 snap 57 seq_num 202) 2024-04-12T11:16:24.805 INFO:tasks.rados.rados.0.smithi029.stdout:521: rollback oid 9 current snap is 66 2024-04-12T11:16:24.805 INFO:tasks.rados.rados.0.smithi029.stdout:513: finishing rollback tid 0 to smithi02979277-25 2024-04-12T11:16:24.805 INFO:tasks.rados.rados.0.smithi029.stdout:513: finishing rollback tid 1 to smithi02979277-25 2024-04-12T11:16:24.805 INFO:tasks.rados.rados.0.smithi029.stdout:513: finishing rollback tid 2 to smithi02979277-25 2024-04-12T11:16:24.805 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 211 (ObjNum 194 snap 55 seq_num 194) dirty exists 2024-04-12T11:16:24.805 INFO:tasks.rados.rados.0.smithi029.stdout:514: finishing write tid 1 to smithi02979277-50 2024-04-12T11:16:24.805 INFO:tasks.rados.rados.0.smithi029.stdout:514: finishing write tid 2 to smithi02979277-50 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 9 to 40 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:514: finishing write tid 3 to smithi02979277-50 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:514: finishing write tid 4 to smithi02979277-50 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:514: finishing write tid 5 to smithi02979277-50 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:514: oid 50 updating version 0 to 179 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:514: oid 50 version 179 is already newer than 178 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:514: oid 50 version 179 is already newer than 176 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:514: oid 50 version 179 is already newer than 177 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 179 (ObjNum 225 snap 66 seq_num 225) dirty exists 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:514: left oid 50 (ObjNum 225 snap 66 seq_num 225) 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:512: done (9 left) 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:513: done (8 left) 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:514: done (7 left) 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:515: done (6 left) 2024-04-12T11:16:24.806 INFO:tasks.rados.rados.0.smithi029.stdout:516: done (5 left) 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:517: done (4 left) 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:522: snap_create 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:518: finishing write tid 1 to smithi02979277-27 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:518: finishing write tid 2 to smithi02979277-27 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:518: finishing write tid 3 to smithi02979277-27 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:518: finishing write tid 4 to smithi02979277-27 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:518: finishing write tid 5 to smithi02979277-27 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:518: oid 27 updating version 0 to 202 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:518: oid 27 version 202 is already newer than 201 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:518: oid 27 version 202 is already newer than 200 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:518: oid 27 updating version 202 to 203 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 203 (ObjNum 226 snap 66 seq_num 226) dirty exists 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:518: left oid 27 (ObjNum 226 snap 66 seq_num 226) 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:519: finishing write tid 1 to smithi02979277-44 2024-04-12T11:16:24.807 INFO:tasks.rados.rados.0.smithi029.stdout:519: finishing write tid 2 to smithi02979277-44 2024-04-12T11:16:24.808 INFO:tasks.rados.rados.0.smithi029.stdout:519: finishing write tid 3 to smithi02979277-44 2024-04-12T11:16:24.808 INFO:tasks.rados.rados.0.smithi029.stdout:519: finishing write tid 4 to smithi02979277-44 2024-04-12T11:16:24.809 INFO:tasks.rados.rados.0.smithi029.stdout:519: finishing write tid 5 to smithi02979277-44 2024-04-12T11:16:24.809 INFO:tasks.rados.rados.0.smithi029.stdout:519: oid 44 updating version 0 to 243 2024-04-12T11:16:24.809 INFO:tasks.rados.rados.0.smithi029.stdout:519: oid 44 updating version 243 to 245 2024-04-12T11:16:24.809 INFO:tasks.rados.rados.0.smithi029.stdout:519: oid 44 updating version 245 to 246 2024-04-12T11:16:24.809 INFO:tasks.rados.rados.0.smithi029.stdout:519: oid 44 version 246 is already newer than 244 2024-04-12T11:16:24.809 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 246 (ObjNum 227 snap 66 seq_num 227) dirty exists 2024-04-12T11:16:24.809 INFO:tasks.rados.rados.0.smithi029.stdout:519: left oid 44 (ObjNum 227 snap 66 seq_num 227) 2024-04-12T11:16:24.809 INFO:tasks.rados.rados.0.smithi029.stdout:521: finishing rollback tid 0 to smithi02979277-9 2024-04-12T11:16:24.809 INFO:tasks.rados.rados.0.smithi029.stdout:521: finishing rollback tid 1 to smithi02979277-9 2024-04-12T11:16:24.810 INFO:tasks.rados.rados.0.smithi029.stdout:521: finishing rollback tid 2 to smithi02979277-9 2024-04-12T11:16:24.810 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 187 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-04-12T11:16:24.810 INFO:tasks.rados.rados.0.smithi029.stdout:520: expect (ObjNum 202 snap 57 seq_num 202) 2024-04-12T11:16:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:24 smithi029 ceph-mon[29349]: pgmap v35: 89 pgs: 89 active+clean; 194 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-04-12T11:16:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:24 smithi029 ceph-mon[29349]: Upgrade: Updating mgr.y 2024-04-12T11:16:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:24 smithi029 ceph-mon[29349]: Deploying daemon mgr.y on smithi029 2024-04-12T11:16:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:24 smithi029 ceph-mon[29349]: osdmap e192: 8 total, 8 up, 8 in 2024-04-12T11:16:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:24 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3032023885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:24 smithi029 ceph-mon[39999]: pgmap v35: 89 pgs: 89 active+clean; 194 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-04-12T11:16:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:24 smithi029 ceph-mon[39999]: Upgrade: Updating mgr.y 2024-04-12T11:16:25.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:24 smithi029 ceph-mon[39999]: Deploying daemon mgr.y on smithi029 2024-04-12T11:16:25.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:24 smithi029 ceph-mon[39999]: osdmap e192: 8 total, 8 up, 8 in 2024-04-12T11:16:25.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:24 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3032023885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:24 smithi087 ceph-mon[32917]: pgmap v35: 89 pgs: 89 active+clean; 194 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-04-12T11:16:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:24 smithi087 ceph-mon[32917]: Upgrade: Updating mgr.y 2024-04-12T11:16:25.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:24 smithi087 ceph-mon[32917]: Deploying daemon mgr.y on smithi029 2024-04-12T11:16:25.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:24 smithi087 ceph-mon[32917]: osdmap e192: 8 total, 8 up, 8 in 2024-04-12T11:16:25.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:24 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3032023885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:25.745 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:25 smithi029 systemd[1]: Stopping Ceph mgr.y for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:16:25.749 INFO:tasks.rados.rados.0.smithi029.stdout:518: done (4 left) 2024-04-12T11:16:25.749 INFO:tasks.rados.rados.0.smithi029.stdout:519: done (3 left) 2024-04-12T11:16:25.749 INFO:tasks.rados.rados.0.smithi029.stdout:520: done (2 left) 2024-04-12T11:16:25.749 INFO:tasks.rados.rados.0.smithi029.stdout:521: done (1 left) 2024-04-12T11:16:25.749 INFO:tasks.rados.rados.0.smithi029.stdout:522: done (0 left) 2024-04-12T11:16:25.749 INFO:tasks.rados.rados.0.smithi029.stdout:523: delete oid 22 current snap is 67 2024-04-12T11:16:25.752 INFO:tasks.rados.rados.0.smithi029.stdout:523: done (0 left) 2024-04-12T11:16:25.753 INFO:tasks.rados.rados.0.smithi029.stdout:524: snap_create 2024-04-12T11:16:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:25 smithi029 ceph-mon[29349]: osdmap e193: 8 total, 8 up, 8 in 2024-04-12T11:16:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:25 smithi029 ceph-mon[39999]: osdmap e193: 8 total, 8 up, 8 in 2024-04-12T11:16:26.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:25 smithi029 podman[84365]: 2024-04-12 11:16:25.745890922 +0000 UTC m=+0.123752293 container died 314601fec666f567978eb64350aaa551e54d3e23c9abbd0f1c595750425cc7db (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=reef-d540eba, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403) 2024-04-12T11:16:26.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:25 smithi087 ceph-mon[32917]: osdmap e193: 8 total, 8 up, 8 in 2024-04-12T11:16:26.491 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:26 smithi029 podman[84365]: 2024-04-12 11:16:26.219823688 +0000 UTC m=+0.597685043 container cleanup 314601fec666f567978eb64350aaa551e54d3e23c9abbd0f1c595750425cc7db (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=reef-d540eba, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:16:26.491 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:26 smithi029 bash[84365]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y 2024-04-12T11:16:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:524: done (0 left) 2024-04-12T11:16:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:525: snap_remove snap 67 2024-04-12T11:16:26.746 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:26 smithi029 ceph-mon[39999]: pgmap v38: 89 pgs: 89 active+clean; 194 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-04-12T11:16:26.747 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:26 smithi029 ceph-mon[39999]: osdmap e194: 8 total, 8 up, 8 in 2024-04-12T11:16:26.747 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:26 smithi029 ceph-mon[39999]: osdmap e195: 8 total, 8 up, 8 in 2024-04-12T11:16:26.747 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:26 smithi029 ceph-mon[29349]: pgmap v38: 89 pgs: 89 active+clean; 194 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-04-12T11:16:26.747 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:26 smithi029 ceph-mon[29349]: osdmap e194: 8 total, 8 up, 8 in 2024-04-12T11:16:26.747 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:26 smithi029 ceph-mon[29349]: osdmap e195: 8 total, 8 up, 8 in 2024-04-12T11:16:26.747 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:26 smithi029 podman[84379]: 2024-04-12 11:16:26.491449285 +0000 UTC m=+0.748484457 container remove 314601fec666f567978eb64350aaa551e54d3e23c9abbd0f1c595750425cc7db (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=reef-d540eba, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD) 2024-04-12T11:16:26.748 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:26 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-12T11:16:26.748 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:26 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-12T11:16:26.748 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:26 smithi029 systemd[1]: Stopped Ceph mgr.y for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:16:26.748 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:26 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.y.service: Consumed 1min 4.192s CPU time. 2024-04-12T11:16:27.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:26 smithi029 systemd[1]: Starting Ceph mgr.y for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:16:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:26 smithi087 ceph-mon[32917]: pgmap v38: 89 pgs: 89 active+clean; 194 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-04-12T11:16:27.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:26 smithi087 ceph-mon[32917]: osdmap e194: 8 total, 8 up, 8 in 2024-04-12T11:16:27.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:26 smithi087 ceph-mon[32917]: osdmap e195: 8 total, 8 up, 8 in 2024-04-12T11:16:27.400 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:27 smithi029 podman[84532]: 2024-04-12 11:16:27.126953034 +0000 UTC m=+0.111972301 container create 05c1874892626f277133aa21c0c3bc2c0ba76a92c2bfce5fc5785af0d7dbb257 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-12T11:16:27.400 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:27 smithi029 podman[84532]: 2024-04-12 11:16:27.033129041 +0000 UTC m=+0.018148282 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:16:27.400 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:27 smithi029 podman[84532]: 2024-04-12 11:16:27.373363223 +0000 UTC m=+0.358382457 container init 05c1874892626f277133aa21c0c3bc2c0ba76a92c2bfce5fc5785af0d7dbb257 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-12T11:16:27.400 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:27 smithi029 podman[84532]: 2024-04-12 11:16:27.375910212 +0000 UTC m=+0.360929437 container start 05c1874892626f277133aa21c0c3bc2c0ba76a92c2bfce5fc5785af0d7dbb257 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-12T11:16:27.546 INFO:tasks.rados.rados.0.smithi029.stdout:525: done (0 left) 2024-04-12T11:16:27.546 INFO:tasks.rados.rados.0.smithi029.stdout:526: write oid 45 current snap is 68 2024-04-12T11:16:27.547 INFO:tasks.rados.rados.0.smithi029.stdout:526: seq_num 229 ranges {648803=722611,1967200=714747,3421983=426820} 2024-04-12T11:16:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:526: writing smithi02979277-45 from 648803 to 1371414 tid 1 2024-04-12T11:16:27.561 INFO:tasks.rados.rados.0.smithi029.stdout:526: writing smithi02979277-45 from 1967200 to 2681947 tid 2 2024-04-12T11:16:27.567 INFO:tasks.rados.rados.0.smithi029.stdout:526: writing smithi02979277-45 from 3421983 to 3848803 tid 3 2024-04-12T11:16:27.567 INFO:tasks.rados.rados.0.smithi029.stdout:527: snap_remove snap 62 2024-04-12T11:16:27.750 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mgr[84550]: -- 172.21.15.29:0/825849030 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56033ea0e5a0 con 0x56033e9e2800 2024-04-12T11:16:27.750 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:27 smithi029 bash[84532]: 05c1874892626f277133aa21c0c3bc2c0ba76a92c2bfce5fc5785af0d7dbb257 2024-04-12T11:16:27.750 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:27 smithi029 systemd[1]: Started Ceph mgr.y for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:16:28.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:27.965+0000 7f16c7730200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:16:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1848884095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mon[29349]: osdmap e196: 8 total, 8 up, 8 in 2024-04-12T11:16:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:28.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1848884095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:28.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mon[39999]: osdmap e196: 8 total, 8 up, 8 in 2024-04-12T11:16:28.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:28.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:28.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:27 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:28.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:27 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1848884095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:28.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:27 smithi087 ceph-mon[32917]: osdmap e196: 8 total, 8 up, 8 in 2024-04-12T11:16:28.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:27 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:28.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:27 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:28.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:27 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:28.374 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:28 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:28.067+0000 7f16c7730200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:527: done (1 left) 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:528: rollback oid 12 current snap is 68 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 66 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:526: finishing write tid 1 to smithi02979277-45 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:526: finishing write tid 2 to smithi02979277-45 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:526: finishing write tid 3 to smithi02979277-45 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:526: finishing write tid 4 to smithi02979277-45 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:526: finishing write tid 5 to smithi02979277-45 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:526: oid 45 updating version 0 to 243 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:526: oid 45 version 243 is already newer than 242 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:526: oid 45 version 243 is already newer than 241 2024-04-12T11:16:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:526: oid 45 version 243 is already newer than 240 2024-04-12T11:16:28.549 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 243 (ObjNum 229 snap 68 seq_num 229) dirty exists 2024-04-12T11:16:28.549 INFO:tasks.rados.rados.0.smithi029.stdout:526: left oid 45 (ObjNum 229 snap 68 seq_num 229) 2024-04-12T11:16:28.549 INFO:tasks.rados.rados.0.smithi029.stdout:526: done (1 left) 2024-04-12T11:16:28.549 INFO:tasks.rados.rados.0.smithi029.stdout:529: snap_create 2024-04-12T11:16:28.553 INFO:tasks.rados.rados.0.smithi029.stdout:528: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:16:28.553 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 237 (ObjNum 81 snap 0 seq_num 0) dirty dne 2024-04-12T11:16:28.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:28 smithi029 ceph-mon[39999]: pgmap v41: 89 pgs: 89 active+clean; 194 MiB data, 621 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:28.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:28 smithi029 ceph-mon[39999]: osdmap e197: 8 total, 8 up, 8 in 2024-04-12T11:16:28.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:28 smithi029 ceph-mon[29349]: pgmap v41: 89 pgs: 89 active+clean; 194 MiB data, 621 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:28.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:28 smithi029 ceph-mon[29349]: osdmap e197: 8 total, 8 up, 8 in 2024-04-12T11:16:28.930 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:28 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:28.772+0000 7f16c7730200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:16:28.930 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:28 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:28.874+0000 7f16c7730200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:16:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:28 smithi087 ceph-mon[32917]: pgmap v41: 89 pgs: 89 active+clean; 194 MiB data, 621 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:28 smithi087 ceph-mon[32917]: osdmap e197: 8 total, 8 up, 8 in 2024-04-12T11:16:29.231 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:29.034+0000 7f16c7730200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:16:29.231 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:29.138+0000 7f16c7730200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:16:29.231 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:29.229+0000 7f16c7730200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:16:29.548 INFO:tasks.rados.rados.0.smithi029.stdout:528: done (1 left) 2024-04-12T11:16:29.548 INFO:tasks.rados.rados.0.smithi029.stdout:529: done (0 left) 2024-04-12T11:16:29.548 INFO:tasks.rados.rados.0.smithi029.stdout:530: rollback oid 7 current snap is 69 2024-04-12T11:16:29.548 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 59 2024-04-12T11:16:29.548 INFO:tasks.rados.rados.0.smithi029.stdout:531: read oid 4 snap -1 2024-04-12T11:16:29.549 INFO:tasks.rados.rados.0.smithi029.stdout:531: expect (ObjNum 192 snap 51 seq_num 192) 2024-04-12T11:16:29.549 INFO:tasks.rados.rados.0.smithi029.stdout:532: write oid 34 current snap is 69 2024-04-12T11:16:29.549 INFO:tasks.rados.rados.0.smithi029.stdout:532: seq_num 230 ranges {665610=797860,1909329=458405,2816700=410210} 2024-04-12T11:16:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:532: writing smithi02979277-34 from 665610 to 1463470 tid 1 2024-04-12T11:16:29.561 INFO:tasks.rados.rados.0.smithi029.stdout:532: writing smithi02979277-34 from 1909329 to 2367734 tid 2 2024-04-12T11:16:29.566 INFO:tasks.rados.rados.0.smithi029.stdout:532: writing smithi02979277-34 from 2816700 to 3226910 tid 3 2024-04-12T11:16:29.566 INFO:tasks.rados.rados.0.smithi029.stdout:533: snap_create 2024-04-12T11:16:29.567 INFO:tasks.rados.rados.0.smithi029.stdout:530: finishing rollback tid 0 to smithi02979277-7 2024-04-12T11:16:29.567 INFO:tasks.rados.rados.0.smithi029.stdout:530: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:16:29.567 INFO:tasks.rados.rados.0.smithi029.stdout:530: finishing rollback tid 2 to smithi02979277-7 2024-04-12T11:16:29.567 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 254 (ObjNum 171 snap 44 seq_num 171) dirty exists 2024-04-12T11:16:29.567 INFO:tasks.rados.rados.0.smithi029.stdout:532: finishing write tid 1 to smithi02979277-34 2024-04-12T11:16:29.567 INFO:tasks.rados.rados.0.smithi029.stdout:531: expect (ObjNum 192 snap 51 seq_num 192) 2024-04-12T11:16:29.674 INFO:tasks.rados.rados.0.smithi029.stdout:532: finishing write tid 2 to smithi02979277-34 2024-04-12T11:16:29.674 INFO:tasks.rados.rados.0.smithi029.stdout:532: finishing write tid 3 to smithi02979277-34 2024-04-12T11:16:29.674 INFO:tasks.rados.rados.0.smithi029.stdout:532: finishing write tid 4 to smithi02979277-34 2024-04-12T11:16:29.675 INFO:tasks.rados.rados.0.smithi029.stdout:532: finishing write tid 5 to smithi02979277-34 2024-04-12T11:16:29.675 INFO:tasks.rados.rados.0.smithi029.stdout:532: oid 34 updating version 0 to 177 2024-04-12T11:16:29.675 INFO:tasks.rados.rados.0.smithi029.stdout:532: oid 34 version 177 is already newer than 175 2024-04-12T11:16:29.675 INFO:tasks.rados.rados.0.smithi029.stdout:532: oid 34 updating version 177 to 178 2024-04-12T11:16:29.675 INFO:tasks.rados.rados.0.smithi029.stdout:532: oid 34 version 178 is already newer than 176 2024-04-12T11:16:29.675 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 178 (ObjNum 230 snap 69 seq_num 230) dirty exists 2024-04-12T11:16:29.675 INFO:tasks.rados.rados.0.smithi029.stdout:532: left oid 34 (ObjNum 230 snap 69 seq_num 230) 2024-04-12T11:16:29.757 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/347069054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:29.758 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:29.516+0000 7f16c7730200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:16:29.758 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:29.641+0000 7f16c7730200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:16:29.758 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:29.741+0000 7f16c7730200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:16:30.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/347069054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:30.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-mon[39999]: osdmap e198: 8 total, 8 up, 8 in 2024-04-12T11:16:30.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-mon[29349]: osdmap e198: 8 total, 8 up, 8 in 2024-04-12T11:16:30.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:29.872+0000 7f16c7730200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:16:30.051 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:29 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/347069054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:30.051 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:29 smithi087 ceph-mon[32917]: osdmap e198: 8 total, 8 up, 8 in 2024-04-12T11:16:30.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:30 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:30.086+0000 7f16c7730200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:16:30.548 INFO:tasks.rados.rados.0.smithi029.stdout:530: done (3 left) 2024-04-12T11:16:30.548 INFO:tasks.rados.rados.0.smithi029.stdout:531: done (2 left) 2024-04-12T11:16:30.548 INFO:tasks.rados.rados.0.smithi029.stdout:532: done (1 left) 2024-04-12T11:16:30.548 INFO:tasks.rados.rados.0.smithi029.stdout:533: done (0 left) 2024-04-12T11:16:30.548 INFO:tasks.rados.rados.0.smithi029.stdout:534: delete oid 49 current snap is 70 2024-04-12T11:16:30.549 INFO:tasks.rados.rados.0.smithi029.stdout:534: done (0 left) 2024-04-12T11:16:30.549 INFO:tasks.rados.rados.0.smithi029.stdout:535: snap_remove snap 60 2024-04-12T11:16:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:30 smithi029 ceph-mon[39999]: pgmap v44: 89 pgs: 89 active+clean; 209 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-04-12T11:16:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:30 smithi029 ceph-mon[39999]: osdmap e199: 8 total, 8 up, 8 in 2024-04-12T11:16:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:30 smithi029 ceph-mon[29349]: pgmap v44: 89 pgs: 89 active+clean; 209 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-04-12T11:16:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:30 smithi029 ceph-mon[29349]: osdmap e199: 8 total, 8 up, 8 in 2024-04-12T11:16:31.151 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:30 smithi087 ceph-mon[32917]: pgmap v44: 89 pgs: 89 active+clean; 209 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-04-12T11:16:31.151 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:30 smithi087 ceph-mon[32917]: osdmap e199: 8 total, 8 up, 8 in 2024-04-12T11:16:31.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:31.148+0000 7f16c7730200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:16:31.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:31.288+0000 7f16c7730200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:16:31.552 INFO:tasks.rados.rados.0.smithi029.stdout:535: done (0 left) 2024-04-12T11:16:31.552 INFO:tasks.rados.rados.0.smithi029.stdout:536: write oid 45 current snap is 70 2024-04-12T11:16:31.552 INFO:tasks.rados.rados.0.smithi029.stdout:536: seq_num 232 ranges {699224=548358,2026293=462074,3271546=432754} 2024-04-12T11:16:31.560 INFO:tasks.rados.rados.0.smithi029.stdout:536: writing smithi02979277-45 from 699224 to 1247582 tid 1 2024-04-12T11:16:31.567 INFO:tasks.rados.rados.0.smithi029.stdout:536: writing smithi02979277-45 from 2026293 to 2488367 tid 2 2024-04-12T11:16:31.574 INFO:tasks.rados.rados.0.smithi029.stdout:536: writing smithi02979277-45 from 3271546 to 3704300 tid 3 2024-04-12T11:16:31.574 INFO:tasks.rados.rados.0.smithi029.stdout:537: delete oid 10 current snap is 70 2024-04-12T11:16:31.574 INFO:tasks.rados.rados.0.smithi029.stdout:536: finishing write tid 1 to smithi02979277-45 2024-04-12T11:16:31.574 INFO:tasks.rados.rados.0.smithi029.stdout:536: finishing write tid 2 to smithi02979277-45 2024-04-12T11:16:31.575 INFO:tasks.rados.rados.0.smithi029.stdout:537: done (1 left) 2024-04-12T11:16:31.575 INFO:tasks.rados.rados.0.smithi029.stdout:538: write oid 48 current snap is 70 2024-04-12T11:16:31.575 INFO:tasks.rados.rados.0.smithi029.stdout:538: seq_num 234 ranges {732838=698856,2026904=465743,3210039=571651} 2024-04-12T11:16:31.583 INFO:tasks.rados.rados.0.smithi029.stdout:538: writing smithi02979277-48 from 732838 to 1431694 tid 1 2024-04-12T11:16:31.589 INFO:tasks.rados.rados.0.smithi029.stdout:538: writing smithi02979277-48 from 2026904 to 2492647 tid 2 2024-04-12T11:16:31.596 INFO:tasks.rados.rados.0.smithi029.stdout:538: writing smithi02979277-48 from 3210039 to 3781690 tid 3 2024-04-12T11:16:31.596 INFO:tasks.rados.rados.0.smithi029.stdout:539: snap_create 2024-04-12T11:16:31.596 INFO:tasks.rados.rados.0.smithi029.stdout:536: finishing write tid 3 to smithi02979277-45 2024-04-12T11:16:31.596 INFO:tasks.rados.rados.0.smithi029.stdout:536: finishing write tid 4 to smithi02979277-45 2024-04-12T11:16:31.597 INFO:tasks.rados.rados.0.smithi029.stdout:536: finishing write tid 5 to smithi02979277-45 2024-04-12T11:16:31.597 INFO:tasks.rados.rados.0.smithi029.stdout:536: oid 45 updating version 0 to 265 2024-04-12T11:16:31.597 INFO:tasks.rados.rados.0.smithi029.stdout:536: oid 45 version 265 is already newer than 264 2024-04-12T11:16:31.597 INFO:tasks.rados.rados.0.smithi029.stdout:536: oid 45 version 265 is already newer than 263 2024-04-12T11:16:31.597 INFO:tasks.rados.rados.0.smithi029.stdout:536: oid 45 version 265 is already newer than 260 2024-04-12T11:16:31.597 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 265 (ObjNum 232 snap 70 seq_num 232) dirty exists 2024-04-12T11:16:31.597 INFO:tasks.rados.rados.0.smithi029.stdout:536: left oid 45 (ObjNum 232 snap 70 seq_num 232) 2024-04-12T11:16:31.597 INFO:tasks.rados.rados.0.smithi029.stdout:538: finishing write tid 1 to smithi02979277-48 2024-04-12T11:16:31.597 INFO:tasks.rados.rados.0.smithi029.stdout:538: finishing write tid 2 to smithi02979277-48 2024-04-12T11:16:31.599 INFO:tasks.rados.rados.0.smithi029.stdout:538: finishing write tid 3 to smithi02979277-48 2024-04-12T11:16:31.599 INFO:tasks.rados.rados.0.smithi029.stdout:538: finishing write tid 4 to smithi02979277-48 2024-04-12T11:16:31.599 INFO:tasks.rados.rados.0.smithi029.stdout:538: finishing write tid 5 to smithi02979277-48 2024-04-12T11:16:31.599 INFO:tasks.rados.rados.0.smithi029.stdout:538: oid 48 updating version 0 to 207 2024-04-12T11:16:31.599 INFO:tasks.rados.rados.0.smithi029.stdout:538: oid 48 version 207 is already newer than 204 2024-04-12T11:16:31.599 INFO:tasks.rados.rados.0.smithi029.stdout:538: oid 48 version 207 is already newer than 206 2024-04-12T11:16:31.599 INFO:tasks.rados.rados.0.smithi029.stdout:538: oid 48 version 207 is already newer than 205 2024-04-12T11:16:31.599 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 207 (ObjNum 234 snap 70 seq_num 234) dirty exists 2024-04-12T11:16:31.599 INFO:tasks.rados.rados.0.smithi029.stdout:538: left oid 48 (ObjNum 234 snap 70 seq_num 234) 2024-04-12T11:16:32.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:32.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:32.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:32.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-mon[39999]: osdmap e200: 8 total, 8 up, 8 in 2024-04-12T11:16:32.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1258045607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:32.007 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:32.007 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:32.007 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:32.007 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-mon[29349]: osdmap e200: 8 total, 8 up, 8 in 2024-04-12T11:16:32.008 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1258045607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:32.008 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:31.687+0000 7f16c7730200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:16:32.008 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:31 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:31.787+0000 7f16c7730200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:16:32.008 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:32.004+0000 7f16c7730200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:16:32.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:31 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:32.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:31 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:32.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:31 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:32.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:31 smithi087 ceph-mon[32917]: osdmap e200: 8 total, 8 up, 8 in 2024-04-12T11:16:32.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:31 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1258045607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:32.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:32.205+0000 7f16c7730200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:16:32.551 INFO:tasks.rados.rados.0.smithi029.stdout:536: done (2 left) 2024-04-12T11:16:32.551 INFO:tasks.rados.rados.0.smithi029.stdout:538: done (1 left) 2024-04-12T11:16:32.552 INFO:tasks.rados.rados.0.smithi029.stdout:539: done (0 left) 2024-04-12T11:16:32.552 INFO:tasks.rados.rados.0.smithi029.stdout:540: read oid 42 snap -1 2024-04-12T11:16:32.552 INFO:tasks.rados.rados.0.smithi029.stdout:540: expect (ObjNum 215 snap 63 seq_num 215) 2024-04-12T11:16:32.552 INFO:tasks.rados.rados.0.smithi029.stdout:541: snap_create 2024-04-12T11:16:32.564 INFO:tasks.rados.rados.0.smithi029.stdout:540: expect (ObjNum 215 snap 63 seq_num 215) 2024-04-12T11:16:32.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:32.516+0000 7f16c7730200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:16:32.924 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:16:32] "GET /metrics HTTP/1.1" 200 36252 "" "Prometheus/2.43.0" 2024-04-12T11:16:33.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[39999]: pgmap v47: 89 pgs: 89 active+clean; 209 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-04-12T11:16:33.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[39999]: osdmap e201: 8 total, 8 up, 8 in 2024-04-12T11:16:33.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[29349]: pgmap v47: 89 pgs: 89 active+clean; 209 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-04-12T11:16:33.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[29349]: osdmap e201: 8 total, 8 up, 8 in 2024-04-12T11:16:33.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.056 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:32 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:32.794+0000 7f16c7730200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:16:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:32 smithi087 ceph-mon[32917]: pgmap v47: 89 pgs: 89 active+clean; 209 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.1 MiB/s wr, 13 op/s 2024-04-12T11:16:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:32 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:32 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:32 smithi087 ceph-mon[32917]: osdmap e201: 8 total, 8 up, 8 in 2024-04-12T11:16:33.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:32 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:32 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:33.310 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:33.054+0000 7f16c7730200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:16:33.553 INFO:tasks.rados.rados.0.smithi029.stdout:540: done (1 left) 2024-04-12T11:16:33.553 INFO:tasks.rados.rados.0.smithi029.stdout:541: done (0 left) 2024-04-12T11:16:33.553 INFO:tasks.rados.rados.0.smithi029.stdout:542: delete oid 14 current snap is 72 2024-04-12T11:16:33.557 INFO:tasks.rados.rados.0.smithi029.stdout:542: done (0 left) 2024-04-12T11:16:33.557 INFO:tasks.rados.rados.0.smithi029.stdout:543: write oid 24 current snap is 72 2024-04-12T11:16:33.557 INFO:tasks.rados.rados.0.smithi029.stdout:543: seq_num 236 ranges {766452=565707,1860221=469412,2981238=594195} 2024-04-12T11:16:33.565 INFO:tasks.rados.rados.0.smithi029.stdout:543: writing smithi02979277-24 from 766452 to 1332159 tid 1 2024-04-12T11:16:33.572 INFO:tasks.rados.rados.0.smithi029.stdout:543: writing smithi02979277-24 from 1860221 to 2329633 tid 2 2024-04-12T11:16:33.580 INFO:tasks.rados.rados.0.smithi029.stdout:543: writing smithi02979277-24 from 2981238 to 3575433 tid 3 2024-04-12T11:16:33.580 INFO:tasks.rados.rados.0.smithi029.stdout:544: read oid 16 snap 43 2024-04-12T11:16:33.580 INFO:tasks.rados.rados.0.smithi029.stdout:544: expect (ObjNum 117 snap 20 seq_num 117) 2024-04-12T11:16:33.580 INFO:tasks.rados.rados.0.smithi029.stdout:545: snap_remove snap 61 2024-04-12T11:16:33.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:33.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:16:34.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:34.004+0000 7f16c7730200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:16:34.558 INFO:tasks.rados.rados.0.smithi029.stdout:545: done (2 left) 2024-04-12T11:16:34.558 INFO:tasks.rados.rados.0.smithi029.stdout:546: snap_create 2024-04-12T11:16:34.558 INFO:tasks.rados.rados.0.smithi029.stdout:543: finishing write tid 1 to smithi02979277-24 2024-04-12T11:16:34.558 INFO:tasks.rados.rados.0.smithi029.stdout:543: finishing write tid 2 to smithi02979277-24 2024-04-12T11:16:34.558 INFO:tasks.rados.rados.0.smithi029.stdout:543: finishing write tid 3 to smithi02979277-24 2024-04-12T11:16:34.558 INFO:tasks.rados.rados.0.smithi029.stdout:543: finishing write tid 4 to smithi02979277-24 2024-04-12T11:16:34.558 INFO:tasks.rados.rados.0.smithi029.stdout:543: finishing write tid 5 to smithi02979277-24 2024-04-12T11:16:34.559 INFO:tasks.rados.rados.0.smithi029.stdout:543: oid 24 updating version 0 to 321 2024-04-12T11:16:34.559 INFO:tasks.rados.rados.0.smithi029.stdout:543: oid 24 version 321 is already newer than 320 2024-04-12T11:16:34.559 INFO:tasks.rados.rados.0.smithi029.stdout:543: oid 24 updating version 321 to 323 2024-04-12T11:16:34.559 INFO:tasks.rados.rados.0.smithi029.stdout:543: oid 24 version 323 is already newer than 322 2024-04-12T11:16:34.559 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 323 (ObjNum 236 snap 72 seq_num 236) dirty exists 2024-04-12T11:16:34.559 INFO:tasks.rados.rados.0.smithi029.stdout:543: left oid 24 (ObjNum 236 snap 72 seq_num 236) 2024-04-12T11:16:34.559 INFO:tasks.rados.rados.0.smithi029.stdout:544: expect (ObjNum 117 snap 20 seq_num 117) 2024-04-12T11:16:34.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-mon[39999]: pgmap v50: 89 pgs: 89 active+clean; 220 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-04-12T11:16:34.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-mon[39999]: osdmap e202: 8 total, 8 up, 8 in 2024-04-12T11:16:34.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1125576274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:34.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-mon[29349]: pgmap v50: 89 pgs: 89 active+clean; 220 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-04-12T11:16:34.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-mon[29349]: osdmap e202: 8 total, 8 up, 8 in 2024-04-12T11:16:34.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1125576274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:34.958 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:34.618+0000 7f16c7730200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:16:34.958 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:34.723+0000 7f16c7730200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:16:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:34 smithi087 ceph-mon[32917]: pgmap v50: 89 pgs: 89 active+clean; 220 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-04-12T11:16:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:34 smithi087 ceph-mon[32917]: osdmap e202: 8 total, 8 up, 8 in 2024-04-12T11:16:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:34 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1125576274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:34.957+0000 7f16c7730200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:16:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:16:34] ENGINE Bus STARTING 2024-04-12T11:16:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: CherryPy Checker: 2024-04-12T11:16:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: The Application mounted at '' has an empty config. 2024-04-12T11:16:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:16:35] ENGINE Serving on http://:::9283 2024-04-12T11:16:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:16:35] ENGINE Bus STARTED 2024-04-12T11:16:35.566 INFO:tasks.rados.rados.0.smithi029.stdout:543: done (2 left) 2024-04-12T11:16:35.567 INFO:tasks.rados.rados.0.smithi029.stdout:544: done (1 left) 2024-04-12T11:16:35.567 INFO:tasks.rados.rados.0.smithi029.stdout:546: done (0 left) 2024-04-12T11:16:35.567 INFO:tasks.rados.rados.0.smithi029.stdout:547: write oid 42 current snap is 73 2024-04-12T11:16:35.567 INFO:tasks.rados.rados.0.smithi029.stdout:547: seq_num 237 ranges {783259=640956,1918703=613070,3008661=405467,3821939=161320} 2024-04-12T11:16:35.574 INFO:tasks.rados.rados.0.smithi029.stdout:547: writing smithi02979277-42 from 783259 to 1424215 tid 1 2024-04-12T11:16:35.580 INFO:tasks.rados.rados.0.smithi029.stdout:547: writing smithi02979277-42 from 1918703 to 2531773 tid 2 2024-04-12T11:16:35.585 INFO:tasks.rados.rados.0.smithi029.stdout:547: writing smithi02979277-42 from 3008661 to 3414128 tid 3 2024-04-12T11:16:35.588 INFO:tasks.rados.rados.0.smithi029.stdout:547: writing smithi02979277-42 from 3821939 to 3983259 tid 4 2024-04-12T11:16:35.588 INFO:tasks.rados.rados.0.smithi029.stdout:548: snap_remove snap 47 2024-04-12T11:16:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[29349]: osdmap e203: 8 total, 8 up, 8 in 2024-04-12T11:16:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[29349]: Standby manager daemon y restarted 2024-04-12T11:16:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[29349]: Standby manager daemon y started 2024-04-12T11:16:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T11:16:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:16:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T11:16:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:16:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[39999]: osdmap e203: 8 total, 8 up, 8 in 2024-04-12T11:16:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[39999]: Standby manager daemon y restarted 2024-04-12T11:16:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[39999]: Standby manager daemon y started 2024-04-12T11:16:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T11:16:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:16:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T11:16:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:35 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:16:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:35 smithi087 ceph-mon[32917]: osdmap e203: 8 total, 8 up, 8 in 2024-04-12T11:16:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:35 smithi087 ceph-mon[32917]: Standby manager daemon y restarted 2024-04-12T11:16:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:35 smithi087 ceph-mon[32917]: Standby manager daemon y started 2024-04-12T11:16:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:35 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-12T11:16:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:35 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:16:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:35 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-12T11:16:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:35 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:16:36.325 INFO:tasks.rados.rados.0.smithi029.stdout:548: done (1 left) 2024-04-12T11:16:36.325 INFO:tasks.rados.rados.0.smithi029.stdout:549: snap_create 2024-04-12T11:16:36.325 INFO:tasks.rados.rados.0.smithi029.stdout:547: finishing write tid 1 to smithi02979277-42 2024-04-12T11:16:36.325 INFO:tasks.rados.rados.0.smithi029.stdout:547: finishing write tid 2 to smithi02979277-42 2024-04-12T11:16:36.325 INFO:tasks.rados.rados.0.smithi029.stdout:547: finishing write tid 3 to smithi02979277-42 2024-04-12T11:16:36.326 INFO:tasks.rados.rados.0.smithi029.stdout:547: finishing write tid 4 to smithi02979277-42 2024-04-12T11:16:36.326 INFO:tasks.rados.rados.0.smithi029.stdout:547: finishing write tid 5 to smithi02979277-42 2024-04-12T11:16:36.326 INFO:tasks.rados.rados.0.smithi029.stdout:547: finishing write tid 6 to smithi02979277-42 2024-04-12T11:16:36.326 INFO:tasks.rados.rados.0.smithi029.stdout:547: oid 42 updating version 0 to 53 2024-04-12T11:16:36.326 INFO:tasks.rados.rados.0.smithi029.stdout:547: oid 42 updating version 53 to 56 2024-04-12T11:16:36.326 INFO:tasks.rados.rados.0.smithi029.stdout:547: oid 42 version 56 is already newer than 55 2024-04-12T11:16:36.326 INFO:tasks.rados.rados.0.smithi029.stdout:547: oid 42 version 56 is already newer than 54 2024-04-12T11:16:36.326 INFO:tasks.rados.rados.0.smithi029.stdout:547: oid 42 version 56 is already newer than 52 2024-04-12T11:16:36.326 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 56 (ObjNum 237 snap 73 seq_num 237) dirty exists 2024-04-12T11:16:36.326 INFO:tasks.rados.rados.0.smithi029.stdout:547: left oid 42 (ObjNum 237 snap 73 seq_num 237) 2024-04-12T11:16:36.350 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:16:36] ENGINE Bus STOPPING 2024-04-12T11:16:36.350 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:16:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T11:16:36.350 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:16:36] ENGINE Bus STOPPED 2024-04-12T11:16:36.546 INFO:tasks.rados.rados.0.smithi029.stdout:547: done (1 left) 2024-04-12T11:16:36.546 INFO:tasks.rados.rados.0.smithi029.stdout:549: done (0 left) 2024-04-12T11:16:36.546 INFO:tasks.rados.rados.0.smithi029.stdout:550: snap_remove snap 69 2024-04-12T11:16:36.579 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:36.324+0000 7feefd5ff700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-12T11:16:36.579 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: ignoring --setuser ceph since I am not root 2024-04-12T11:16:36.579 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: ignoring --setgroup ceph since I am not root 2024-04-12T11:16:36.579 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mgr[66912]: -- 172.21.15.87:0/4292724778 <== mon.1 v2:172.21.15.87:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5560dc9265a0 con 0x5560dc8fa800 2024-04-12T11:16:36.580 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: pgmap v53: 89 pgs: 89 active+clean; 220 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-04-12T11:16:36.580 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: osdmap e204: 8 total, 8 up, 8 in 2024-04-12T11:16:36.580 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: mgrmap e26: x(active, since 34s), standbys: y 2024-04-12T11:16:36.580 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:36.580 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:36.580 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:36.580 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:36.580 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:36.580 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:36.580 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: Activating manager daemon y 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: osdmap e205: 8 total, 8 up, 8 in 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.24772 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: mgrmap e27: y(active, starting, since 0.00657216s) 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:16:36.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: Manager daemon y is now available 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:16:36.582 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-mon[32917]: osdmap e206: 8 total, 8 up, 8 in 2024-04-12T11:16:36.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:16:36] ENGINE Bus STARTING 2024-04-12T11:16:36.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:36.466+0000 7f1628383700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:36.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:36.466+0000 7f1628383700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:36.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:36.466+0000 7f1628383700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:36.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:36.466+0000 7f1628383700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:36.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:36.466+0000 7f1628383700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:36.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:36.467+0000 7f162437b700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:36.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:36.467+0000 7f162437b700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:36.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:36.467+0000 7f162437b700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:36.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:36.467+0000 7f162437b700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:36.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:16:36.467+0000 7f162437b700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:16:36.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:16:36] ENGINE Serving on http://:::9283 2024-04-12T11:16:36.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:16:36] ENGINE Bus STARTED 2024-04-12T11:16:36.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: pgmap v53: 89 pgs: 89 active+clean; 220 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-04-12T11:16:36.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: osdmap e204: 8 total, 8 up, 8 in 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: mgrmap e26: x(active, since 34s), standbys: y 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: Activating manager daemon y 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: osdmap e205: 8 total, 8 up, 8 in 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.24772 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-12T11:16:36.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: mgrmap e27: y(active, starting, since 0.00657216s) 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:16:36.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: Manager daemon y is now available 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[29349]: osdmap e206: 8 total, 8 up, 8 in 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: pgmap v53: 89 pgs: 89 active+clean; 220 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: osdmap e204: 8 total, 8 up, 8 in 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: mgrmap e26: x(active, since 34s), standbys: y 2024-04-12T11:16:36.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.24772 172.21.15.87:0/4172545233' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: Activating manager daemon y 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: osdmap e205: 8 total, 8 up, 8 in 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.24772 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: mgrmap e27: y(active, starting, since 0.00657216s) 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:16:36.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:16:36.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:16:36.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:16:36.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: Manager daemon y is now available 2024-04-12T11:16:36.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:36.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:36.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:16:36.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:16:36.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:36 smithi029 ceph-mon[39999]: osdmap e206: 8 total, 8 up, 8 in 2024-04-12T11:16:37.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:36 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:36.906+0000 7fa910273200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:16:37.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:37.003+0000 7fa910273200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:16:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:550: done (0 left) 2024-04-12T11:16:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:551: rollback oid 39 current snap is 74 2024-04-12T11:16:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 68 2024-04-12T11:16:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:552: read oid 9 snap -1 2024-04-12T11:16:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:552: expect (ObjNum 151 snap 36 seq_num 151) 2024-04-12T11:16:37.547 INFO:tasks.rados.rados.0.smithi029.stdout:553: snap_create 2024-04-12T11:16:37.548 INFO:tasks.rados.rados.0.smithi029.stdout:551: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:16:37.548 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 0 (ObjNum 129 snap 0 seq_num 1800825872) dirty dne 2024-04-12T11:16:37.551 INFO:tasks.rados.rados.0.smithi029.stdout:552: expect (ObjNum 151 snap 36 seq_num 151) 2024-04-12T11:16:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1498161157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[39999]: [12/Apr/2024:11:16:36] ENGINE Bus STARTING 2024-04-12T11:16:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[39999]: [12/Apr/2024:11:16:36] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:16:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[39999]: [12/Apr/2024:11:16:36] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:16:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[39999]: [12/Apr/2024:11:16:36] ENGINE Bus STARTED 2024-04-12T11:16:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[39999]: mgrmap e28: y(active, since 1.00885s) 2024-04-12T11:16:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[39999]: pgmap v4: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[39999]: osdmap e207: 8 total, 8 up, 8 in 2024-04-12T11:16:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1498161157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[29349]: [12/Apr/2024:11:16:36] ENGINE Bus STARTING 2024-04-12T11:16:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[29349]: [12/Apr/2024:11:16:36] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:16:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[29349]: [12/Apr/2024:11:16:36] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:16:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[29349]: [12/Apr/2024:11:16:36] ENGINE Bus STARTED 2024-04-12T11:16:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[29349]: mgrmap e28: y(active, since 1.00885s) 2024-04-12T11:16:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[29349]: pgmap v4: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:37 smithi029 ceph-mon[29349]: osdmap e207: 8 total, 8 up, 8 in 2024-04-12T11:16:37.799 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:37.705+0000 7fa910273200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:16:37.799 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:37.798+0000 7fa910273200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:16:37.799 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1498161157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:37.799 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-mon[32917]: [12/Apr/2024:11:16:36] ENGINE Bus STARTING 2024-04-12T11:16:37.799 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-mon[32917]: [12/Apr/2024:11:16:36] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:16:37.799 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-mon[32917]: [12/Apr/2024:11:16:36] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:16:37.799 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-mon[32917]: [12/Apr/2024:11:16:36] ENGINE Bus STARTED 2024-04-12T11:16:37.800 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-mon[32917]: mgrmap e28: y(active, since 1.00885s) 2024-04-12T11:16:37.800 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-mon[32917]: pgmap v4: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:37.800 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-mon[32917]: osdmap e207: 8 total, 8 up, 8 in 2024-04-12T11:16:38.059 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:37.941+0000 7fa910273200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:16:38.342 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:38 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:38.058+0000 7fa910273200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:16:38.343 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:38 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:38.146+0000 7fa910273200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:16:38.547 INFO:tasks.rados.rados.0.smithi029.stdout:551: done (2 left) 2024-04-12T11:16:38.547 INFO:tasks.rados.rados.0.smithi029.stdout:552: done (1 left) 2024-04-12T11:16:38.547 INFO:tasks.rados.rados.0.smithi029.stdout:553: done (0 left) 2024-04-12T11:16:38.547 INFO:tasks.rados.rados.0.smithi029.stdout:554: write oid 40 current snap is 75 2024-04-12T11:16:38.547 INFO:tasks.rados.rados.0.smithi029.stdout:554: seq_num 238 ranges {400066=716205,1860832=139234} 2024-04-12T11:16:38.553 INFO:tasks.rados.rados.0.smithi029.stdout:554: writing smithi02979277-40 from 400066 to 1116271 tid 1 2024-04-12T11:16:38.558 INFO:tasks.rados.rados.0.smithi029.stdout:554: writing smithi02979277-40 from 1860832 to 2000066 tid 2 2024-04-12T11:16:38.558 INFO:tasks.rados.rados.0.smithi029.stdout:555: read oid 4 snap 74 2024-04-12T11:16:38.558 INFO:tasks.rados.rados.0.smithi029.stdout:555: expect (ObjNum 192 snap 51 seq_num 192) 2024-04-12T11:16:38.558 INFO:tasks.rados.rados.0.smithi029.stdout:556: write oid 24 current snap is 75 2024-04-12T11:16:38.558 INFO:tasks.rados.rados.0.smithi029.stdout:556: seq_num 239 ranges {416873=791454,1919314=97559} 2024-04-12T11:16:38.565 INFO:tasks.rados.rados.0.smithi029.stdout:556: writing smithi02979277-24 from 416873 to 1208327 tid 1 2024-04-12T11:16:38.570 INFO:tasks.rados.rados.0.smithi029.stdout:556: writing smithi02979277-24 from 1919314 to 2016873 tid 2 2024-04-12T11:16:38.570 INFO:tasks.rados.rados.0.smithi029.stdout:557: rollback oid 23 current snap is 75 2024-04-12T11:16:38.570 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 23 to 66 2024-04-12T11:16:38.570 INFO:tasks.rados.rados.0.smithi029.stdout:554: finishing write tid 1 to smithi02979277-40 2024-04-12T11:16:38.571 INFO:tasks.rados.rados.0.smithi029.stdout:554: finishing write tid 2 to smithi02979277-40 2024-04-12T11:16:38.571 INFO:tasks.rados.rados.0.smithi029.stdout:554: finishing write tid 3 to smithi02979277-40 2024-04-12T11:16:38.571 INFO:tasks.rados.rados.0.smithi029.stdout:554: finishing write tid 4 to smithi02979277-40 2024-04-12T11:16:38.571 INFO:tasks.rados.rados.0.smithi029.stdout:554: oid 40 updating version 0 to 336 2024-04-12T11:16:38.571 INFO:tasks.rados.rados.0.smithi029.stdout:554: oid 40 version 336 is already newer than 335 2024-04-12T11:16:38.571 INFO:tasks.rados.rados.0.smithi029.stdout:554: oid 40 version 336 is already newer than 334 2024-04-12T11:16:38.571 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 336 (ObjNum 238 snap 75 seq_num 238) dirty exists 2024-04-12T11:16:38.571 INFO:tasks.rados.rados.0.smithi029.stdout:554: left oid 40 (ObjNum 238 snap 75 seq_num 238) 2024-04-12T11:16:38.571 INFO:tasks.rados.rados.0.smithi029.stdout:555: expect (ObjNum 192 snap 51 seq_num 192) 2024-04-12T11:16:38.633 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:38 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:38.423+0000 7fa910273200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:16:38.633 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:38 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:38.538+0000 7fa910273200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:556: finishing write tid 1 to smithi02979277-24 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:556: finishing write tid 2 to smithi02979277-24 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:556: finishing write tid 3 to smithi02979277-24 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:556: finishing write tid 4 to smithi02979277-24 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:556: oid 24 updating version 0 to 339 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:556: oid 24 updating version 339 to 340 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:556: oid 24 version 340 is already newer than 338 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 340 (ObjNum 239 snap 75 seq_num 239) dirty exists 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:556: left oid 24 (ObjNum 239 snap 75 seq_num 239) 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:557: finishing rollback tid 0 to smithi02979277-23 2024-04-12T11:16:38.677 INFO:tasks.rados.rados.0.smithi029.stdout:554: done (3 left) 2024-04-12T11:16:38.678 INFO:tasks.rados.rados.0.smithi029.stdout:555: done (2 left) 2024-04-12T11:16:38.678 INFO:tasks.rados.rados.0.smithi029.stdout:556: done (1 left) 2024-04-12T11:16:38.678 INFO:tasks.rados.rados.0.smithi029.stdout:558: read oid 17 snap -1 2024-04-12T11:16:38.678 INFO:tasks.rados.rados.0.smithi029.stdout:558: expect (ObjNum 105 snap 16 seq_num 105) 2024-04-12T11:16:38.678 INFO:tasks.rados.rados.0.smithi029.stdout:559: write oid 16 current snap is 75 2024-04-12T11:16:38.678 INFO:tasks.rados.rados.0.smithi029.stdout:557: finishing rollback tid 1 to smithi02979277-23 2024-04-12T11:16:38.678 INFO:tasks.rados.rados.0.smithi029.stdout:559: seq_num 240 ranges {433680=466703,1577796=455884} 2024-04-12T11:16:38.681 INFO:tasks.rados.rados.0.smithi029.stdout:559: writing smithi02979277-16 from 433680 to 900383 tid 1 2024-04-12T11:16:38.688 INFO:tasks.rados.rados.0.smithi029.stdout:559: writing smithi02979277-16 from 1577796 to 2033680 tid 2 2024-04-12T11:16:38.688 INFO:tasks.rados.rados.0.smithi029.stdout:560: delete oid 45 current snap is 75 2024-04-12T11:16:38.688 INFO:tasks.rados.rados.0.smithi029.stdout:557: finishing rollback tid 2 to smithi02979277-23 2024-04-12T11:16:38.689 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 198 (ObjNum 186 snap 49 seq_num 186) dirty exists 2024-04-12T11:16:38.689 INFO:tasks.rados.rados.0.smithi029.stdout:558: expect (ObjNum 105 snap 16 seq_num 105) 2024-04-12T11:16:38.817 INFO:tasks.rados.rados.0.smithi029.stdout:559: finishing write tid 1 to smithi02979277-16 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:559: finishing write tid 2 to smithi02979277-16 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:559: finishing write tid 3 to smithi02979277-16 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:557: done (3 left) 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:558: done (2 left) 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:560: done (1 left) 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:561: write oid 27 current snap is 75 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:559: finishing write tid 4 to smithi02979277-16 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:559: oid 16 updating version 0 to 292 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:559: oid 16 updating version 292 to 294 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:559: oid 16 version 294 is already newer than 293 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 294 (ObjNum 240 snap 75 seq_num 240) dirty exists 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:559: left oid 16 (ObjNum 240 snap 75 seq_num 240) 2024-04-12T11:16:38.818 INFO:tasks.rados.rados.0.smithi029.stdout:561: seq_num 242 ranges {467294=617201,1578407=488887} 2024-04-12T11:16:38.823 INFO:tasks.rados.rados.0.smithi029.stdout:561: writing smithi02979277-27 from 467294 to 1084495 tid 1 2024-04-12T11:16:38.830 INFO:tasks.rados.rados.0.smithi029.stdout:561: writing smithi02979277-27 from 1578407 to 2067294 tid 2 2024-04-12T11:16:38.830 INFO:tasks.rados.rados.0.smithi029.stdout:559: done (1 left) 2024-04-12T11:16:38.830 INFO:tasks.rados.rados.0.smithi029.stdout:562: rollback oid 21 current snap is 75 2024-04-12T11:16:38.830 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 21 to 73 2024-04-12T11:16:38.830 INFO:tasks.rados.rados.0.smithi029.stdout:561: finishing write tid 1 to smithi02979277-27 2024-04-12T11:16:38.830 INFO:tasks.rados.rados.0.smithi029.stdout:563: delete oid 44 current snap is 75 2024-04-12T11:16:38.832 INFO:tasks.rados.rados.0.smithi029.stdout:561: finishing write tid 2 to smithi02979277-27 2024-04-12T11:16:38.832 INFO:tasks.rados.rados.0.smithi029.stdout:561: finishing write tid 3 to smithi02979277-27 2024-04-12T11:16:38.832 INFO:tasks.rados.rados.0.smithi029.stdout:561: finishing write tid 4 to smithi02979277-27 2024-04-12T11:16:38.832 INFO:tasks.rados.rados.0.smithi029.stdout:561: oid 27 updating version 0 to 213 2024-04-12T11:16:38.832 INFO:tasks.rados.rados.0.smithi029.stdout:561: oid 27 version 213 is already newer than 211 2024-04-12T11:16:38.833 INFO:tasks.rados.rados.0.smithi029.stdout:561: oid 27 version 213 is already newer than 212 2024-04-12T11:16:38.833 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 213 (ObjNum 242 snap 75 seq_num 242) dirty exists 2024-04-12T11:16:38.833 INFO:tasks.rados.rados.0.smithi029.stdout:561: left oid 27 (ObjNum 242 snap 75 seq_num 242) 2024-04-12T11:16:38.833 INFO:tasks.rados.rados.0.smithi029.stdout:562: finishing rollback tid 0 to smithi02979277-21 2024-04-12T11:16:38.833 INFO:tasks.rados.rados.0.smithi029.stdout:562: finishing rollback tid 1 to smithi02979277-21 2024-04-12T11:16:38.834 INFO:tasks.rados.rados.0.smithi029.stdout:562: finishing rollback tid 2 to smithi02979277-21 2024-04-12T11:16:38.834 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 268 (ObjNum 223 snap 64 seq_num 223) dirty exists 2024-04-12T11:16:38.835 INFO:tasks.rados.rados.0.smithi029.stdout:561: done (2 left) 2024-04-12T11:16:38.835 INFO:tasks.rados.rados.0.smithi029.stdout:562: done (1 left) 2024-04-12T11:16:38.835 INFO:tasks.rados.rados.0.smithi029.stdout:563: done (0 left) 2024-04-12T11:16:38.835 INFO:tasks.rados.rados.0.smithi029.stdout:564: delete oid 27 current snap is 75 2024-04-12T11:16:38.836 INFO:tasks.rados.rados.0.smithi029.stdout:564: done (0 left) 2024-04-12T11:16:38.836 INFO:tasks.rados.rados.0.smithi029.stdout:565: delete oid 24 current snap is 75 2024-04-12T11:16:38.838 INFO:tasks.rados.rados.0.smithi029.stdout:565: done (0 left) 2024-04-12T11:16:38.838 INFO:tasks.rados.rados.0.smithi029.stdout:566: write oid 4 current snap is 75 2024-04-12T11:16:38.838 INFO:tasks.rados.rados.0.smithi029.stdout:566: seq_num 246 ranges {534522=634550,1812335=322187} 2024-04-12T11:16:38.845 INFO:tasks.rados.rados.0.smithi029.stdout:566: writing smithi02979277-4 from 534522 to 1169072 tid 1 2024-04-12T11:16:38.851 INFO:tasks.rados.rados.0.smithi029.stdout:566: writing smithi02979277-4 from 1812335 to 2134522 tid 2 2024-04-12T11:16:38.851 INFO:tasks.rados.rados.0.smithi029.stdout:567: read oid 31 snap -1 2024-04-12T11:16:38.851 INFO:tasks.rados.rados.0.smithi029.stdout:567: expect (ObjNum 138 snap 30 seq_num 138) 2024-04-12T11:16:38.851 INFO:tasks.rados.rados.0.smithi029.stdout:566: finishing write tid 1 to smithi02979277-4 2024-04-12T11:16:38.851 INFO:tasks.rados.rados.0.smithi029.stdout:568: snap_create 2024-04-12T11:16:38.854 INFO:tasks.rados.rados.0.smithi029.stdout:566: finishing write tid 2 to smithi02979277-4 2024-04-12T11:16:38.854 INFO:tasks.rados.rados.0.smithi029.stdout:566: finishing write tid 3 to smithi02979277-4 2024-04-12T11:16:38.854 INFO:tasks.rados.rados.0.smithi029.stdout:566: finishing write tid 4 to smithi02979277-4 2024-04-12T11:16:38.854 INFO:tasks.rados.rados.0.smithi029.stdout:566: oid 4 updating version 0 to 298 2024-04-12T11:16:38.854 INFO:tasks.rados.rados.0.smithi029.stdout:566: oid 4 version 298 is already newer than 297 2024-04-12T11:16:38.854 INFO:tasks.rados.rados.0.smithi029.stdout:566: oid 4 version 298 is already newer than 296 2024-04-12T11:16:38.855 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 298 (ObjNum 246 snap 75 seq_num 246) dirty exists 2024-04-12T11:16:38.855 INFO:tasks.rados.rados.0.smithi029.stdout:566: left oid 4 (ObjNum 246 snap 75 seq_num 246) 2024-04-12T11:16:38.855 INFO:tasks.rados.rados.0.smithi029.stdout:567: expect (ObjNum 138 snap 30 seq_num 138) 2024-04-12T11:16:38.938 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:38 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:38.631+0000 7fa910273200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:16:38.938 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:38 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:38.730+0000 7fa910273200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:16:38.938 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:38 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:38.937+0000 7fa910273200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:16:39.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:39 smithi087 ceph-mon[32917]: pgmap v6: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:39.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:39 smithi087 ceph-mon[32917]: osdmap e208: 8 total, 8 up, 8 in 2024-04-12T11:16:39.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:39 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/4090864665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:39.554 INFO:tasks.rados.rados.0.smithi029.stdout:566: done (2 left) 2024-04-12T11:16:39.554 INFO:tasks.rados.rados.0.smithi029.stdout:567: done (1 left) 2024-04-12T11:16:39.554 INFO:tasks.rados.rados.0.smithi029.stdout:568: done (0 left) 2024-04-12T11:16:39.554 INFO:tasks.rados.rados.0.smithi029.stdout:569: write oid 36 current snap is 76 2024-04-12T11:16:39.554 INFO:tasks.rados.rados.0.smithi029.stdout:569: seq_num 247 ranges {551329=709799,1870817=280512} 2024-04-12T11:16:39.560 INFO:tasks.rados.rados.0.smithi029.stdout:569: writing smithi02979277-36 from 551329 to 1261128 tid 1 2024-04-12T11:16:39.565 INFO:tasks.rados.rados.0.smithi029.stdout:569: writing smithi02979277-36 from 1870817 to 2151329 tid 2 2024-04-12T11:16:39.566 INFO:tasks.rados.rados.0.smithi029.stdout:570: rollback oid 12 current snap is 76 2024-04-12T11:16:39.566 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 66 2024-04-12T11:16:39.566 INFO:tasks.rados.rados.0.smithi029.stdout:571: write oid 18 current snap is 76 2024-04-12T11:16:39.566 INFO:tasks.rados.rados.0.smithi029.stdout:571: seq_num 248 ranges {568136=785048,1929299=238837} 2024-04-12T11:16:39.573 INFO:tasks.rados.rados.0.smithi029.stdout:571: writing smithi02979277-18 from 568136 to 1353184 tid 1 2024-04-12T11:16:39.578 INFO:tasks.rados.rados.0.smithi029.stdout:571: writing smithi02979277-18 from 1929299 to 2168136 tid 2 2024-04-12T11:16:39.578 INFO:tasks.rados.rados.0.smithi029.stdout:572: rollback oid 21 current snap is 76 2024-04-12T11:16:39.578 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 21 to 66 2024-04-12T11:16:39.578 INFO:tasks.rados.rados.0.smithi029.stdout:570: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:16:39.578 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 237 (ObjNum 97 snap 0 seq_num 0) dirty dne 2024-04-12T11:16:39.578 INFO:tasks.rados.rados.0.smithi029.stdout:569: finishing write tid 1 to smithi02979277-36 2024-04-12T11:16:39.578 INFO:tasks.rados.rados.0.smithi029.stdout:569: finishing write tid 2 to smithi02979277-36 2024-04-12T11:16:39.579 INFO:tasks.rados.rados.0.smithi029.stdout:569: finishing write tid 3 to smithi02979277-36 2024-04-12T11:16:39.579 INFO:tasks.rados.rados.0.smithi029.stdout:569: finishing write tid 4 to smithi02979277-36 2024-04-12T11:16:39.579 INFO:tasks.rados.rados.0.smithi029.stdout:569: oid 36 updating version 0 to 118 2024-04-12T11:16:39.579 INFO:tasks.rados.rados.0.smithi029.stdout:569: oid 36 updating version 118 to 119 2024-04-12T11:16:39.579 INFO:tasks.rados.rados.0.smithi029.stdout:569: oid 36 version 119 is already newer than 117 2024-04-12T11:16:39.579 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 119 (ObjNum 247 snap 76 seq_num 247) dirty exists 2024-04-12T11:16:39.579 INFO:tasks.rados.rados.0.smithi029.stdout:569: left oid 36 (ObjNum 247 snap 76 seq_num 247) 2024-04-12T11:16:39.579 INFO:tasks.rados.rados.0.smithi029.stdout:569: done (3 left) 2024-04-12T11:16:39.579 INFO:tasks.rados.rados.0.smithi029.stdout:570: done (2 left) 2024-04-12T11:16:39.579 INFO:tasks.rados.rados.0.smithi029.stdout:573: snap_remove snap 72 2024-04-12T11:16:39.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:39 smithi029 ceph-mon[39999]: pgmap v6: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:39.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:39 smithi029 ceph-mon[39999]: osdmap e208: 8 total, 8 up, 8 in 2024-04-12T11:16:39.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:39 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/4090864665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:39 smithi029 ceph-mon[29349]: pgmap v6: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:39 smithi029 ceph-mon[29349]: osdmap e208: 8 total, 8 up, 8 in 2024-04-12T11:16:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:39 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/4090864665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:40.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:39 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:39.957+0000 7fa910273200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:16:40.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:40 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:40.097+0000 7fa910273200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:16:40.562 INFO:tasks.rados.rados.0.smithi029.stdout:573: done (2 left) 2024-04-12T11:16:40.562 INFO:tasks.rados.rados.0.smithi029.stdout:574: read oid 15 snap -1 2024-04-12T11:16:40.563 INFO:tasks.rados.rados.0.smithi029.stdout:574: expect (ObjNum 203 snap 57 seq_num 203) 2024-04-12T11:16:40.563 INFO:tasks.rados.rados.0.smithi029.stdout:575: snap_remove snap 64 2024-04-12T11:16:40.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:40 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:40.443+0000 7fa910273200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:16:40.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:40 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:40.534+0000 7fa910273200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:16:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:40 smithi087 ceph-mon[32917]: mgrmap e29: y(active, since 3s) 2024-04-12T11:16:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:40 smithi087 ceph-mon[32917]: osdmap e209: 8 total, 8 up, 8 in 2024-04-12T11:16:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:40 smithi087 ceph-mon[32917]: pgmap v9: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:40 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:40 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:40 smithi029 ceph-mon[39999]: mgrmap e29: y(active, since 3s) 2024-04-12T11:16:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:40 smithi029 ceph-mon[39999]: osdmap e209: 8 total, 8 up, 8 in 2024-04-12T11:16:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:40 smithi029 ceph-mon[39999]: pgmap v9: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:40 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:40 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:40 smithi029 ceph-mon[29349]: mgrmap e29: y(active, since 3s) 2024-04-12T11:16:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:40 smithi029 ceph-mon[29349]: osdmap e209: 8 total, 8 up, 8 in 2024-04-12T11:16:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:40 smithi029 ceph-mon[29349]: pgmap v9: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:40 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:40 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:41.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:40 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:40.746+0000 7fa910273200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:16:41.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:40 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:40.938+0000 7fa910273200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:16:41.499 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:41 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:41.233+0000 7fa910273200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:16:41.550 INFO:tasks.rados.rados.0.smithi029.stdout:575: done (3 left) 2024-04-12T11:16:41.550 INFO:tasks.rados.rados.0.smithi029.stdout:576: snap_create 2024-04-12T11:16:41.550 INFO:tasks.rados.rados.0.smithi029.stdout:571: finishing write tid 1 to smithi02979277-18 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:571: finishing write tid 2 to smithi02979277-18 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:571: finishing write tid 3 to smithi02979277-18 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:571: finishing write tid 4 to smithi02979277-18 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:571: oid 18 updating version 0 to 230 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:571: oid 18 version 230 is already newer than 229 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:571: oid 18 updating version 230 to 231 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 231 (ObjNum 248 snap 76 seq_num 248) dirty exists 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:571: left oid 18 (ObjNum 248 snap 76 seq_num 248) 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:572: finishing rollback tid 0 to smithi02979277-21 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:572: finishing rollback tid 1 to smithi02979277-21 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:572: finishing rollback tid 2 to smithi02979277-21 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 274 (ObjNum 223 snap 64 seq_num 223) dirty exists 2024-04-12T11:16:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:574: expect (ObjNum 203 snap 57 seq_num 203) 2024-04-12T11:16:41.765 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:41 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:41.498+0000 7fa910273200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:16:41.765 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:41 smithi087 ceph-mon[32917]: osdmap e210: 8 total, 8 up, 8 in 2024-04-12T11:16:41.765 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:41 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:41.766 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:41 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:41.766 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:41 smithi087 ceph-mon[32917]: osdmap e211: 8 total, 8 up, 8 in 2024-04-12T11:16:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:41 smithi029 ceph-mon[29349]: osdmap e210: 8 total, 8 up, 8 in 2024-04-12T11:16:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:41 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:41 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:41 smithi029 ceph-mon[29349]: osdmap e211: 8 total, 8 up, 8 in 2024-04-12T11:16:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:41 smithi029 ceph-mon[39999]: osdmap e210: 8 total, 8 up, 8 in 2024-04-12T11:16:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:41 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:41 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:41 smithi029 ceph-mon[39999]: osdmap e211: 8 total, 8 up, 8 in 2024-04-12T11:16:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:571: done (3 left) 2024-04-12T11:16:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:572: done (2 left) 2024-04-12T11:16:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:574: done (1 left) 2024-04-12T11:16:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:576: done (0 left) 2024-04-12T11:16:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:577: snap_create 2024-04-12T11:16:42.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:41 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:41.764+0000 7fa910273200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:16:42.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:42.602+0000 7fa910273200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:16:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:42 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1587898446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:42 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:42 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:42 smithi087 ceph-mon[32917]: pgmap v12: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:42 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:42 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:42 smithi087 ceph-mon[32917]: osdmap e212: 8 total, 8 up, 8 in 2024-04-12T11:16:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1587898446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[29349]: pgmap v12: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[29349]: osdmap e212: 8 total, 8 up, 8 in 2024-04-12T11:16:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1587898446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[39999]: pgmap v12: 89 pgs: 89 active+clean; 220 MiB data, 673 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:42 smithi029 ceph-mon[39999]: osdmap e212: 8 total, 8 up, 8 in 2024-04-12T11:16:43.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:43.165+0000 7fa910273200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:16:43.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:43.263+0000 7fa910273200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:16:43.549 INFO:tasks.rados.rados.0.smithi029.stdout:577: done (0 left) 2024-04-12T11:16:43.549 INFO:tasks.rados.rados.0.smithi029.stdout:578: snap_remove snap 65 2024-04-12T11:16:43.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[29349]: Standby manager daemon x started 2024-04-12T11:16:43.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:16:43.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:16:43.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:16:43.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:16:43.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[29349]: osdmap e213: 8 total, 8 up, 8 in 2024-04-12T11:16:43.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[39999]: Standby manager daemon x started 2024-04-12T11:16:43.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:16:43.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:16:43.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:16:43.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:16:43.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:43 smithi029 ceph-mon[39999]: osdmap e213: 8 total, 8 up, 8 in 2024-04-12T11:16:43.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:16:43.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-mon[32917]: Standby manager daemon x started 2024-04-12T11:16:43.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:16:43.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:16:43.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:16:43.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1998125966' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:16:43.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-mon[32917]: osdmap e213: 8 total, 8 up, 8 in 2024-04-12T11:16:43.968 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:43.483+0000 7fa910273200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:16:43.968 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:16:43] ENGINE Bus STARTING 2024-04-12T11:16:43.968 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: CherryPy Checker: 2024-04-12T11:16:43.968 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: The Application mounted at '' has an empty config. 2024-04-12T11:16:43.968 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: 2024-04-12T11:16:43.968 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:16:43] ENGINE Serving on http://:::9283 2024-04-12T11:16:43.968 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[66908]: [12/Apr/2024:11:16:43] ENGINE Bus STARTED 2024-04-12T11:16:44.553 INFO:tasks.rados.rados.0.smithi029.stdout:578: done (0 left) 2024-04-12T11:16:44.553 INFO:tasks.rados.rados.0.smithi029.stdout:579: delete oid 39 current snap is 78 2024-04-12T11:16:44.556 INFO:tasks.rados.rados.0.smithi029.stdout:579: done (0 left) 2024-04-12T11:16:44.556 INFO:tasks.rados.rados.0.smithi029.stdout:580: delete oid 19 current snap is 78 2024-04-12T11:16:44.561 INFO:tasks.rados.rados.0.smithi029.stdout:580: done (0 left) 2024-04-12T11:16:44.561 INFO:tasks.rados.rados.0.smithi029.stdout:581: read oid 6 snap -1 2024-04-12T11:16:44.562 INFO:tasks.rados.rados.0.smithi029.stdout:581: expect deleted 2024-04-12T11:16:44.562 INFO:tasks.rados.rados.0.smithi029.stdout:582: write oid 45 current snap is 78 2024-04-12T11:16:44.562 INFO:tasks.rados.rados.0.smithi029.stdout:582: seq_num 251 ranges {618557=727148,2104745=113812} 2024-04-12T11:16:44.569 INFO:tasks.rados.rados.0.smithi029.stdout:582: writing smithi02979277-45 from 618557 to 1345705 tid 1 2024-04-12T11:16:44.574 INFO:tasks.rados.rados.0.smithi029.stdout:582: writing smithi02979277-45 from 2104745 to 2218557 tid 2 2024-04-12T11:16:44.574 INFO:tasks.rados.rados.0.smithi029.stdout:583: snap_create 2024-04-12T11:16:44.574 INFO:tasks.rados.rados.0.smithi029.stdout:582: finishing write tid 1 to smithi02979277-45 2024-04-12T11:16:44.576 INFO:tasks.rados.rados.0.smithi029.stdout:582: finishing write tid 2 to smithi02979277-45 2024-04-12T11:16:44.576 INFO:tasks.rados.rados.0.smithi029.stdout:582: finishing write tid 3 to smithi02979277-45 2024-04-12T11:16:44.576 INFO:tasks.rados.rados.0.smithi029.stdout:582: finishing write tid 4 to smithi02979277-45 2024-04-12T11:16:44.576 INFO:tasks.rados.rados.0.smithi029.stdout:582: oid 45 updating version 0 to 292 2024-04-12T11:16:44.576 INFO:tasks.rados.rados.0.smithi029.stdout:582: oid 45 version 292 is already newer than 291 2024-04-12T11:16:44.576 INFO:tasks.rados.rados.0.smithi029.stdout:582: oid 45 updating version 292 to 293 2024-04-12T11:16:44.576 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 293 (ObjNum 251 snap 78 seq_num 251) dirty exists 2024-04-12T11:16:44.576 INFO:tasks.rados.rados.0.smithi029.stdout:582: left oid 45 (ObjNum 251 snap 78 seq_num 251) 2024-04-12T11:16:44.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:44 smithi029 ceph-mon[29349]: mgrmap e30: y(active, since 7s), standbys: x 2024-04-12T11:16:44.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:44 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:16:44.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:44 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/812035875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:44.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:44 smithi029 ceph-mon[29349]: pgmap v15: 89 pgs: 89 active+clean; 227 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 51 op/s 2024-04-12T11:16:44.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:44 smithi029 ceph-mon[29349]: osdmap e214: 8 total, 8 up, 8 in 2024-04-12T11:16:44.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:44 smithi029 ceph-mon[39999]: mgrmap e30: y(active, since 7s), standbys: x 2024-04-12T11:16:44.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:44 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:16:44.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:44 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/812035875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:44.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:44 smithi029 ceph-mon[39999]: pgmap v15: 89 pgs: 89 active+clean; 227 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 51 op/s 2024-04-12T11:16:44.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:44 smithi029 ceph-mon[39999]: osdmap e214: 8 total, 8 up, 8 in 2024-04-12T11:16:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:44 smithi087 ceph-mon[32917]: mgrmap e30: y(active, since 7s), standbys: x 2024-04-12T11:16:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:44 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:16:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:44 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/812035875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:44 smithi087 ceph-mon[32917]: pgmap v15: 89 pgs: 89 active+clean; 227 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 51 op/s 2024-04-12T11:16:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:44 smithi087 ceph-mon[32917]: osdmap e214: 8 total, 8 up, 8 in 2024-04-12T11:16:45.552 INFO:tasks.rados.rados.0.smithi029.stdout:581: done (2 left) 2024-04-12T11:16:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:582: done (1 left) 2024-04-12T11:16:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:583: done (0 left) 2024-04-12T11:16:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:584: write oid 2 current snap is 79 2024-04-12T11:16:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:584: seq_num 252 ranges {635364=402397,1763227=472137} 2024-04-12T11:16:45.558 INFO:tasks.rados.rados.0.smithi029.stdout:584: writing smithi02979277-2 from 635364 to 1037761 tid 1 2024-04-12T11:16:45.565 INFO:tasks.rados.rados.0.smithi029.stdout:584: writing smithi02979277-2 from 1763227 to 2235364 tid 2 2024-04-12T11:16:45.565 INFO:tasks.rados.rados.0.smithi029.stdout:585: write oid 17 current snap is 79 2024-04-12T11:16:45.565 INFO:tasks.rados.rados.0.smithi029.stdout:584: finishing write tid 1 to smithi02979277-2 2024-04-12T11:16:45.565 INFO:tasks.rados.rados.0.smithi029.stdout:585: seq_num 253 ranges {652171=477646,1821709=430462} 2024-04-12T11:16:45.571 INFO:tasks.rados.rados.0.smithi029.stdout:585: writing smithi02979277-17 from 652171 to 1129817 tid 1 2024-04-12T11:16:45.577 INFO:tasks.rados.rados.0.smithi029.stdout:585: writing smithi02979277-17 from 1821709 to 2252171 tid 2 2024-04-12T11:16:45.577 INFO:tasks.rados.rados.0.smithi029.stdout:586: write oid 30 current snap is 79 2024-04-12T11:16:45.577 INFO:tasks.rados.rados.0.smithi029.stdout:584: finishing write tid 2 to smithi02979277-2 2024-04-12T11:16:45.577 INFO:tasks.rados.rados.0.smithi029.stdout:586: seq_num 254 ranges {668978=552895,1763838=505140} 2024-04-12T11:16:45.584 INFO:tasks.rados.rados.0.smithi029.stdout:586: writing smithi02979277-30 from 668978 to 1221873 tid 1 2024-04-12T11:16:45.590 INFO:tasks.rados.rados.0.smithi029.stdout:586: writing smithi02979277-30 from 1763838 to 2268978 tid 2 2024-04-12T11:16:45.590 INFO:tasks.rados.rados.0.smithi029.stdout:587: delete oid 23 current snap is 79 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:584: finishing write tid 3 to smithi02979277-2 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:584: finishing write tid 4 to smithi02979277-2 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:584: oid 2 updating version 0 to 13 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:584: oid 2 updating version 13 to 14 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:584: oid 2 version 14 is already newer than 12 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 14 (ObjNum 252 snap 79 seq_num 252) dirty exists 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:584: left oid 2 (ObjNum 252 snap 79 seq_num 252) 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:585: finishing write tid 1 to smithi02979277-17 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:585: finishing write tid 2 to smithi02979277-17 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:585: finishing write tid 3 to smithi02979277-17 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:585: finishing write tid 4 to smithi02979277-17 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:585: oid 17 updating version 0 to 244 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:585: oid 17 updating version 244 to 245 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:585: oid 17 version 245 is already newer than 243 2024-04-12T11:16:45.591 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 245 (ObjNum 253 snap 79 seq_num 253) dirty exists 2024-04-12T11:16:45.592 INFO:tasks.rados.rados.0.smithi029.stdout:585: left oid 17 (ObjNum 253 snap 79 seq_num 253) 2024-04-12T11:16:45.592 INFO:tasks.rados.rados.0.smithi029.stdout:586: finishing write tid 1 to smithi02979277-30 2024-04-12T11:16:45.592 INFO:tasks.rados.rados.0.smithi029.stdout:584: done (3 left) 2024-04-12T11:16:45.592 INFO:tasks.rados.rados.0.smithi029.stdout:585: done (2 left) 2024-04-12T11:16:45.592 INFO:tasks.rados.rados.0.smithi029.stdout:587: done (1 left) 2024-04-12T11:16:45.592 INFO:tasks.rados.rados.0.smithi029.stdout:588: read oid 39 snap 43 2024-04-12T11:16:45.592 INFO:tasks.rados.rados.0.smithi029.stdout:588: expect deleted 2024-04-12T11:16:45.593 INFO:tasks.rados.rados.0.smithi029.stdout:589: snap_remove snap 70 2024-04-12T11:16:46.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:46.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:46.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:46.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:46.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[39999]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:16:46.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:16:46.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[39999]: osdmap e215: 8 total, 8 up, 8 in 2024-04-12T11:16:46.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:46.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:46.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:46.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:46.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[29349]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:16:46.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[29349]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:16:46.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:46 smithi029 ceph-mon[29349]: osdmap e215: 8 total, 8 up, 8 in 2024-04-12T11:16:46.548 INFO:tasks.rados.rados.0.smithi029.stdout:589: done (2 left) 2024-04-12T11:16:46.548 INFO:tasks.rados.rados.0.smithi029.stdout:590: rollback oid 22 current snap is 79 2024-04-12T11:16:46.548 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 22 to 73 2024-04-12T11:16:46.548 INFO:tasks.rados.rados.0.smithi029.stdout:591: rollback oid 28 current snap is 79 2024-04-12T11:16:46.549 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 28 to 59 2024-04-12T11:16:46.549 INFO:tasks.rados.rados.0.smithi029.stdout:592: delete oid 1 current snap is 79 2024-04-12T11:16:46.549 INFO:tasks.rados.rados.0.smithi029.stdout:586: finishing write tid 2 to smithi02979277-30 2024-04-12T11:16:46.549 INFO:tasks.rados.rados.0.smithi029.stdout:586: finishing write tid 3 to smithi02979277-30 2024-04-12T11:16:46.549 INFO:tasks.rados.rados.0.smithi029.stdout:586: finishing write tid 4 to smithi02979277-30 2024-04-12T11:16:46.549 INFO:tasks.rados.rados.0.smithi029.stdout:586: oid 30 updating version 0 to 219 2024-04-12T11:16:46.549 INFO:tasks.rados.rados.0.smithi029.stdout:586: oid 30 version 219 is already newer than 218 2024-04-12T11:16:46.549 INFO:tasks.rados.rados.0.smithi029.stdout:586: oid 30 updating version 219 to 220 2024-04-12T11:16:46.549 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 220 (ObjNum 254 snap 79 seq_num 254) dirty exists 2024-04-12T11:16:46.550 INFO:tasks.rados.rados.0.smithi029.stdout:586: left oid 30 (ObjNum 254 snap 79 seq_num 254) 2024-04-12T11:16:46.551 INFO:tasks.rados.rados.0.smithi029.stdout:586: done (4 left) 2024-04-12T11:16:46.551 INFO:tasks.rados.rados.0.smithi029.stdout:588: done (3 left) 2024-04-12T11:16:46.551 INFO:tasks.rados.rados.0.smithi029.stdout:592: done (2 left) 2024-04-12T11:16:46.551 INFO:tasks.rados.rados.0.smithi029.stdout:593: read oid 5 snap 71 2024-04-12T11:16:46.551 INFO:tasks.rados.rados.0.smithi029.stdout:593: expect (ObjNum 218 snap 63 seq_num 218) 2024-04-12T11:16:46.552 INFO:tasks.rados.rados.0.smithi029.stdout:594: write oid 49 current snap is 79 2024-04-12T11:16:46.552 INFO:tasks.rados.rados.0.smithi029.stdout:594: seq_num 257 ranges {719399=778642,1939284=380115} 2024-04-12T11:16:46.560 INFO:tasks.rados.rados.0.smithi029.stdout:594: writing smithi02979277-49 from 719399 to 1498041 tid 1 2024-04-12T11:16:46.565 INFO:tasks.rados.rados.0.smithi029.stdout:594: writing smithi02979277-49 from 1939284 to 2319399 tid 2 2024-04-12T11:16:46.565 INFO:tasks.rados.rados.0.smithi029.stdout:595: snap_create 2024-04-12T11:16:46.565 INFO:tasks.rados.rados.0.smithi029.stdout:591: finishing rollback tid 0 to smithi02979277-28 2024-04-12T11:16:46.565 INFO:tasks.rados.rados.0.smithi029.stdout:590: finishing rollback tid 1 to smithi02979277-22 2024-04-12T11:16:46.565 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 170 (ObjNum 129 snap 0 seq_num 1800863464) dirty dne 2024-04-12T11:16:46.565 INFO:tasks.rados.rados.0.smithi029.stdout:591: finishing rollback tid 1 to smithi02979277-28 2024-04-12T11:16:46.566 INFO:tasks.rados.rados.0.smithi029.stdout:591: finishing rollback tid 2 to smithi02979277-28 2024-04-12T11:16:46.566 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 225 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-04-12T11:16:46.566 INFO:tasks.rados.rados.0.smithi029.stdout:594: finishing write tid 1 to smithi02979277-49 2024-04-12T11:16:46.566 INFO:tasks.rados.rados.0.smithi029.stdout:593: expect (ObjNum 218 snap 63 seq_num 218) 2024-04-12T11:16:46.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:46 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:46.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:46 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:46.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:46 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:46 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:16:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:46 smithi087 ceph-mon[32917]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:16:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:46 smithi087 ceph-mon[32917]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:16:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:46 smithi087 ceph-mon[32917]: osdmap e215: 8 total, 8 up, 8 in 2024-04-12T11:16:46.762 INFO:tasks.rados.rados.0.smithi029.stdout:594: finishing write tid 2 to smithi02979277-49 2024-04-12T11:16:46.762 INFO:tasks.rados.rados.0.smithi029.stdout:594: finishing write tid 3 to smithi02979277-49 2024-04-12T11:16:46.762 INFO:tasks.rados.rados.0.smithi029.stdout:594: finishing write tid 4 to smithi02979277-49 2024-04-12T11:16:46.762 INFO:tasks.rados.rados.0.smithi029.stdout:594: oid 49 updating version 0 to 218 2024-04-12T11:16:46.762 INFO:tasks.rados.rados.0.smithi029.stdout:594: oid 49 version 218 is already newer than 217 2024-04-12T11:16:46.762 INFO:tasks.rados.rados.0.smithi029.stdout:594: oid 49 version 218 is already newer than 216 2024-04-12T11:16:46.762 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 218 (ObjNum 257 snap 79 seq_num 257) dirty exists 2024-04-12T11:16:46.762 INFO:tasks.rados.rados.0.smithi029.stdout:594: left oid 49 (ObjNum 257 snap 79 seq_num 257) 2024-04-12T11:16:47.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: pgmap v18: 89 pgs: 89 active+clean; 227 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 51 op/s 2024-04-12T11:16:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T11:16:47.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2131494449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:47.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:47 smithi087 ceph-mon[32917]: osdmap e216: 8 total, 8 up, 8 in 2024-04-12T11:16:47.549 INFO:tasks.rados.rados.0.smithi029.stdout:590: done (4 left) 2024-04-12T11:16:47.550 INFO:tasks.rados.rados.0.smithi029.stdout:591: done (3 left) 2024-04-12T11:16:47.550 INFO:tasks.rados.rados.0.smithi029.stdout:593: done (2 left) 2024-04-12T11:16:47.550 INFO:tasks.rados.rados.0.smithi029.stdout:594: done (1 left) 2024-04-12T11:16:47.550 INFO:tasks.rados.rados.0.smithi029.stdout:595: done (0 left) 2024-04-12T11:16:47.550 INFO:tasks.rados.rados.0.smithi029.stdout:596: snap_create 2024-04-12T11:16:47.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:47.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:47.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:47.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:47.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:47.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: pgmap v18: 89 pgs: 89 active+clean; 227 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 51 op/s 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2131494449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[29349]: osdmap e216: 8 total, 8 up, 8 in 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:16:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: pgmap v18: 89 pgs: 89 active+clean; 227 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 51 op/s 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2131494449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:47 smithi029 ceph-mon[39999]: osdmap e216: 8 total, 8 up, 8 in 2024-04-12T11:16:48.553 INFO:tasks.rados.rados.0.smithi029.stdout:596: done (0 left) 2024-04-12T11:16:48.553 INFO:tasks.rados.rados.0.smithi029.stdout:597: read oid 13 snap -1 2024-04-12T11:16:48.553 INFO:tasks.rados.rados.0.smithi029.stdout:597: expect (ObjNum 168 snap 44 seq_num 168) 2024-04-12T11:16:48.553 INFO:tasks.rados.rados.0.smithi029.stdout:598: snap_remove snap 35 2024-04-12T11:16:48.615 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:48 smithi087 ceph-mon[32917]: Reconfiguring daemon prometheus.a on smithi087 2024-04-12T11:16:48.615 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:48 smithi087 ceph-mon[32917]: osdmap e217: 8 total, 8 up, 8 in 2024-04-12T11:16:48.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:48 smithi029 ceph-mon[39999]: Reconfiguring daemon prometheus.a on smithi087 2024-04-12T11:16:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:48 smithi029 ceph-mon[39999]: osdmap e217: 8 total, 8 up, 8 in 2024-04-12T11:16:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:48 smithi029 ceph-mon[29349]: Reconfiguring daemon prometheus.a on smithi087 2024-04-12T11:16:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:48 smithi029 ceph-mon[29349]: osdmap e217: 8 total, 8 up, 8 in 2024-04-12T11:16:49.555 INFO:tasks.rados.rados.0.smithi029.stdout:598: done (1 left) 2024-04-12T11:16:49.555 INFO:tasks.rados.rados.0.smithi029.stdout:599: write oid 12 current snap is 81 2024-04-12T11:16:49.555 INFO:tasks.rados.rados.0.smithi029.stdout:599: seq_num 258 ranges {736206=453891,1881413=454793} 2024-04-12T11:16:49.561 INFO:tasks.rados.rados.0.smithi029.stdout:599: writing smithi02979277-12 from 736206 to 1190097 tid 1 2024-04-12T11:16:49.569 INFO:tasks.rados.rados.0.smithi029.stdout:599: writing smithi02979277-12 from 1881413 to 2336206 tid 2 2024-04-12T11:16:49.569 INFO:tasks.rados.rados.0.smithi029.stdout:600: read oid 41 snap 40 2024-04-12T11:16:49.569 INFO:tasks.rados.rados.0.smithi029.stdout:600: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:16:49.569 INFO:tasks.rados.rados.0.smithi029.stdout:601: write oid 34 current snap is 81 2024-04-12T11:16:49.569 INFO:tasks.rados.rados.0.smithi029.stdout:601: seq_num 259 ranges {753013=645493,2056248=296765} 2024-04-12T11:16:49.577 INFO:tasks.rados.rados.0.smithi029.stdout:601: writing smithi02979277-34 from 753013 to 1398506 tid 1 2024-04-12T11:16:49.582 INFO:tasks.rados.rados.0.smithi029.stdout:601: writing smithi02979277-34 from 2056248 to 2353013 tid 2 2024-04-12T11:16:49.582 INFO:tasks.rados.rados.0.smithi029.stdout:602: delete oid 8 current snap is 81 2024-04-12T11:16:49.583 INFO:tasks.rados.rados.0.smithi029.stdout:597: expect (ObjNum 168 snap 44 seq_num 168) 2024-04-12T11:16:49.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:49 smithi029 ceph-mon[39999]: pgmap v21: 89 pgs: 89 active+clean; 237 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:16:49.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:49 smithi029 ceph-mon[39999]: osdmap e218: 8 total, 8 up, 8 in 2024-04-12T11:16:49.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:49 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2554121089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:49.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:49 smithi029 ceph-mon[29349]: pgmap v21: 89 pgs: 89 active+clean; 237 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:16:49.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:49 smithi029 ceph-mon[29349]: osdmap e218: 8 total, 8 up, 8 in 2024-04-12T11:16:49.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:49 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2554121089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:49.670 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:49 smithi087 ceph-mon[32917]: pgmap v21: 89 pgs: 89 active+clean; 237 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:16:49.670 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:49 smithi087 ceph-mon[32917]: osdmap e218: 8 total, 8 up, 8 in 2024-04-12T11:16:49.670 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:49 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2554121089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:49.709 INFO:tasks.rados.rados.0.smithi029.stdout:599: finishing write tid 1 to smithi02979277-12 2024-04-12T11:16:49.709 INFO:tasks.rados.rados.0.smithi029.stdout:599: finishing write tid 2 to smithi02979277-12 2024-04-12T11:16:49.709 INFO:tasks.rados.rados.0.smithi029.stdout:599: finishing write tid 3 to smithi02979277-12 2024-04-12T11:16:49.709 INFO:tasks.rados.rados.0.smithi029.stdout:599: finishing write tid 4 to smithi02979277-12 2024-04-12T11:16:49.710 INFO:tasks.rados.rados.0.smithi029.stdout:599: oid 12 updating version 0 to 300 2024-04-12T11:16:49.710 INFO:tasks.rados.rados.0.smithi029.stdout:599: oid 12 updating version 300 to 304 2024-04-12T11:16:49.710 INFO:tasks.rados.rados.0.smithi029.stdout:599: oid 12 version 304 is already newer than 303 2024-04-12T11:16:49.710 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 304 (ObjNum 258 snap 81 seq_num 258) dirty exists 2024-04-12T11:16:49.710 INFO:tasks.rados.rados.0.smithi029.stdout:599: left oid 12 (ObjNum 258 snap 81 seq_num 258) 2024-04-12T11:16:49.710 INFO:tasks.rados.rados.0.smithi029.stdout:601: finishing write tid 1 to smithi02979277-34 2024-04-12T11:16:49.710 INFO:tasks.rados.rados.0.smithi029.stdout:600: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:16:49.852 INFO:tasks.rados.rados.0.smithi029.stdout:601: finishing write tid 2 to smithi02979277-34 2024-04-12T11:16:49.852 INFO:tasks.rados.rados.0.smithi029.stdout:601: finishing write tid 3 to smithi02979277-34 2024-04-12T11:16:49.852 INFO:tasks.rados.rados.0.smithi029.stdout:601: finishing write tid 4 to smithi02979277-34 2024-04-12T11:16:49.852 INFO:tasks.rados.rados.0.smithi029.stdout:601: oid 34 updating version 0 to 201 2024-04-12T11:16:49.852 INFO:tasks.rados.rados.0.smithi029.stdout:601: oid 34 updating version 201 to 202 2024-04-12T11:16:49.852 INFO:tasks.rados.rados.0.smithi029.stdout:601: oid 34 version 202 is already newer than 200 2024-04-12T11:16:49.852 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 202 (ObjNum 259 snap 81 seq_num 259) dirty exists 2024-04-12T11:16:49.853 INFO:tasks.rados.rados.0.smithi029.stdout:601: left oid 34 (ObjNum 259 snap 81 seq_num 259) 2024-04-12T11:16:49.853 INFO:tasks.rados.rados.0.smithi029.stdout:597: done (4 left) 2024-04-12T11:16:49.853 INFO:tasks.rados.rados.0.smithi029.stdout:599: done (3 left) 2024-04-12T11:16:49.853 INFO:tasks.rados.rados.0.smithi029.stdout:600: done (2 left) 2024-04-12T11:16:49.853 INFO:tasks.rados.rados.0.smithi029.stdout:601: done (1 left) 2024-04-12T11:16:49.853 INFO:tasks.rados.rados.0.smithi029.stdout:602: done (0 left) 2024-04-12T11:16:49.853 INFO:tasks.rados.rados.0.smithi029.stdout:603: snap_remove snap 71 2024-04-12T11:16:49.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:49 smithi087 systemd[1]: Stopping Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:16:50.359 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[75452]: ts=2024-04-12T11:16:50.083Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-12T11:16:50.360 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 podman[77952]: 2024-04-12 11:16:50.093329029 +0000 UTC m=+0.136368309 container died 354d7b898c83c64ae8e4cfa42dbe32a778d4cc1837de7291a2c3f5b403e8ea42 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:50.561 INFO:tasks.rados.rados.0.smithi029.stdout:603: done (0 left) 2024-04-12T11:16:50.562 INFO:tasks.rados.rados.0.smithi029.stdout:604: snap_create 2024-04-12T11:16:50.652 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 podman[77952]: 2024-04-12 11:16:50.457904582 +0000 UTC m=+0.500943865 container cleanup 354d7b898c83c64ae8e4cfa42dbe32a778d4cc1837de7291a2c3f5b403e8ea42 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:50.652 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 bash[77952]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a 2024-04-12T11:16:50.652 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-mon[32917]: osdmap e219: 8 total, 8 up, 8 in 2024-04-12T11:16:50.652 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:50 smithi087 ceph-mon[32917]: pgmap v24: 89 pgs: 89 active+clean; 237 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:16:50.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:50 smithi029 ceph-mon[29349]: osdmap e219: 8 total, 8 up, 8 in 2024-04-12T11:16:50.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:50 smithi029 ceph-mon[29349]: pgmap v24: 89 pgs: 89 active+clean; 237 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:16:50.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:50 smithi029 ceph-mon[39999]: osdmap e219: 8 total, 8 up, 8 in 2024-04-12T11:16:50.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:50 smithi029 ceph-mon[39999]: pgmap v24: 89 pgs: 89 active+clean; 237 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:16:50.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 podman[77966]: 2024-04-12 11:16:50.751107173 +0000 UTC m=+0.658212019 container remove 354d7b898c83c64ae8e4cfa42dbe32a778d4cc1837de7291a2c3f5b403e8ea42 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:50.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-12T11:16:50.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 systemd[1]: Stopped Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:16:50.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@prometheus.a.service: Consumed 1.034s CPU time. 2024-04-12T11:16:50.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:50 smithi087 systemd[1]: Starting Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:16:51.322 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 podman[78072]: 2024-04-12 11:16:50.976354043 +0000 UTC m=+0.019206472 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 podman[78072]: 2024-04-12 11:16:51.085301107 +0000 UTC m=+0.128153536 container create 569140fc5a5359a99754861b08c344a1a019ea6bb52f543817e907a3e0effbac (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 podman[78072]: 2024-04-12 11:16:51.293471424 +0000 UTC m=+0.336323854 container init 569140fc5a5359a99754861b08c344a1a019ea6bb52f543817e907a3e0effbac (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 podman[78072]: 2024-04-12 11:16:51.296094813 +0000 UTC m=+0.338947255 container start 569140fc5a5359a99754861b08c344a1a019ea6bb52f543817e907a3e0effbac (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.319Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.320Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.320Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi087 (none))" 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.320Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.320Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.321Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.322Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.322Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-12T11:16:51.323 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.322Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-12T11:16:51.549 INFO:tasks.rados.rados.0.smithi029.stdout:604: done (0 left) 2024-04-12T11:16:51.549 INFO:tasks.rados.rados.0.smithi029.stdout:605: write oid 46 current snap is 82 2024-04-12T11:16:51.549 INFO:tasks.rados.rados.0.smithi029.stdout:605: seq_num 261 ranges {786627=795991,2173212=213415} 2024-04-12T11:16:51.559 INFO:tasks.rados.rados.0.smithi029.stdout:605: writing smithi02979277-46 from 786627 to 1582618 tid 1 2024-04-12T11:16:51.564 INFO:tasks.rados.rados.0.smithi029.stdout:605: writing smithi02979277-46 from 2173212 to 2386627 tid 2 2024-04-12T11:16:51.565 INFO:tasks.rados.rados.0.smithi029.stdout:606: read oid 50 snap -1 2024-04-12T11:16:51.565 INFO:tasks.rados.rados.0.smithi029.stdout:606: expect (ObjNum 225 snap 66 seq_num 225) 2024-04-12T11:16:51.565 INFO:tasks.rados.rados.0.smithi029.stdout:607: delete oid 4 current snap is 82 2024-04-12T11:16:51.567 INFO:tasks.rados.rados.0.smithi029.stdout:605: finishing write tid 1 to smithi02979277-46 2024-04-12T11:16:51.567 INFO:tasks.rados.rados.0.smithi029.stdout:605: finishing write tid 2 to smithi02979277-46 2024-04-12T11:16:51.567 INFO:tasks.rados.rados.0.smithi029.stdout:607: done (2 left) 2024-04-12T11:16:51.567 INFO:tasks.rados.rados.0.smithi029.stdout:608: write oid 27 current snap is 82 2024-04-12T11:16:51.567 INFO:tasks.rados.rados.0.smithi029.stdout:608: seq_num 263 ranges {420241=546489,1373823=544414} 2024-04-12T11:16:51.573 INFO:tasks.rados.rados.0.smithi029.stdout:608: writing smithi02979277-27 from 420241 to 966730 tid 1 2024-04-12T11:16:51.580 INFO:tasks.rados.rados.0.smithi029.stdout:608: writing smithi02979277-27 from 1373823 to 1918237 tid 2 2024-04-12T11:16:51.580 INFO:tasks.rados.rados.0.smithi029.stdout:609: write oid 18 current snap is 82 2024-04-12T11:16:51.580 INFO:tasks.rados.rados.0.smithi029.stdout:609: seq_num 264 ranges {437048=621738,1832305=404425} 2024-04-12T11:16:51.587 INFO:tasks.rados.rados.0.smithi029.stdout:609: writing smithi02979277-18 from 437048 to 1058786 tid 1 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:609: writing smithi02979277-18 from 1832305 to 2236730 tid 2 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:610: write oid 44 current snap is 82 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:605: finishing write tid 3 to smithi02979277-46 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:605: finishing write tid 4 to smithi02979277-46 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:605: oid 46 updating version 0 to 125 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:605: oid 46 version 125 is already newer than 124 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:605: oid 46 version 125 is already newer than 123 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 125 (ObjNum 261 snap 82 seq_num 261) dirty exists 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:605: left oid 46 (ObjNum 261 snap 82 seq_num 261) 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:608: finishing write tid 1 to smithi02979277-27 2024-04-12T11:16:51.595 INFO:tasks.rados.rados.0.smithi029.stdout:606: expect (ObjNum 225 snap 66 seq_num 225) 2024-04-12T11:16:51.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-mon[32917]: osdmap e220: 8 total, 8 up, 8 in 2024-04-12T11:16:51.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2609812464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-mon[32917]: osdmap e221: 8 total, 8 up, 8 in 2024-04-12T11:16:51.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.326Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-12T11:16:51.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.326Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.891µs 2024-04-12T11:16:51.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.326Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-12T11:16:51.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.328Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-12T11:16:51.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.334Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-12T11:16:51.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.342Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-12T11:16:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.342Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-12T11:16:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.342Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=40.353µs wal_replay_duration=16.145916ms wbl_replay_duration=101ns total_replay_duration=16.418609ms 2024-04-12T11:16:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.344Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-12T11:16:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.344Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-12T11:16:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.344Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-12T11:16:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.367Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.57204ms db_storage=913ns remote_storage=1.229µs web_handler=372ns query_engine=659ns scrape=461.142µs scrape_sd=115.826µs notify=11.042µs notify_sd=16.682µs rules=21.582ms tracing=5.48µs 2024-04-12T11:16:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.367Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-12T11:16:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:51.367Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-12T11:16:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 bash[78072]: 569140fc5a5359a99754861b08c344a1a019ea6bb52f543817e907a3e0effbac 2024-04-12T11:16:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:51 smithi087 systemd[1]: Started Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:16:51.888 INFO:tasks.rados.rados.0.smithi029.stdout:608: finishing write tid 2 to smithi02979277-27 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:608: finishing write tid 3 to smithi02979277-27 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:608: finishing write tid 4 to smithi02979277-27 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:608: oid 27 updating version 0 to 235 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:608: oid 27 version 235 is already newer than 233 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:608: oid 27 version 235 is already newer than 234 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 235 (ObjNum 263 snap 82 seq_num 263) dirty exists 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:608: left oid 27 (ObjNum 263 snap 82 seq_num 263) 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:609: finishing write tid 1 to smithi02979277-18 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:609: finishing write tid 2 to smithi02979277-18 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:609: finishing write tid 3 to smithi02979277-18 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:609: finishing write tid 4 to smithi02979277-18 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:609: oid 18 updating version 0 to 265 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:609: oid 18 version 265 is already newer than 264 2024-04-12T11:16:51.889 INFO:tasks.rados.rados.0.smithi029.stdout:609: oid 18 version 265 is already newer than 263 2024-04-12T11:16:51.890 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 265 (ObjNum 264 snap 82 seq_num 264) dirty exists 2024-04-12T11:16:51.890 INFO:tasks.rados.rados.0.smithi029.stdout:609: left oid 18 (ObjNum 264 snap 82 seq_num 264) 2024-04-12T11:16:51.890 INFO:tasks.rados.rados.0.smithi029.stdout:610: seq_num 265 ranges {453855=696987,1890787=548083} 2024-04-12T11:16:51.896 INFO:tasks.rados.rados.0.smithi029.stdout:610: writing smithi02979277-44 from 453855 to 1150842 tid 1 2024-04-12T11:16:51.904 INFO:tasks.rados.rados.0.smithi029.stdout:610: writing smithi02979277-44 from 1890787 to 2438870 tid 2 2024-04-12T11:16:51.904 INFO:tasks.rados.rados.0.smithi029.stdout:605: done (4 left) 2024-04-12T11:16:51.904 INFO:tasks.rados.rados.0.smithi029.stdout:606: done (3 left) 2024-04-12T11:16:51.904 INFO:tasks.rados.rados.0.smithi029.stdout:608: done (2 left) 2024-04-12T11:16:51.904 INFO:tasks.rados.rados.0.smithi029.stdout:609: done (1 left) 2024-04-12T11:16:51.904 INFO:tasks.rados.rados.0.smithi029.stdout:611: snap_create 2024-04-12T11:16:51.905 INFO:tasks.rados.rados.0.smithi029.stdout:610: finishing write tid 1 to smithi02979277-44 2024-04-12T11:16:51.907 INFO:tasks.rados.rados.0.smithi029.stdout:610: finishing write tid 2 to smithi02979277-44 2024-04-12T11:16:51.907 INFO:tasks.rados.rados.0.smithi029.stdout:610: finishing write tid 3 to smithi02979277-44 2024-04-12T11:16:51.907 INFO:tasks.rados.rados.0.smithi029.stdout:610: finishing write tid 4 to smithi02979277-44 2024-04-12T11:16:51.907 INFO:tasks.rados.rados.0.smithi029.stdout:610: oid 44 updating version 0 to 299 2024-04-12T11:16:51.907 INFO:tasks.rados.rados.0.smithi029.stdout:610: oid 44 version 299 is already newer than 298 2024-04-12T11:16:51.907 INFO:tasks.rados.rados.0.smithi029.stdout:610: oid 44 updating version 299 to 300 2024-04-12T11:16:51.907 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 300 (ObjNum 265 snap 82 seq_num 265) dirty exists 2024-04-12T11:16:51.907 INFO:tasks.rados.rados.0.smithi029.stdout:610: left oid 44 (ObjNum 265 snap 82 seq_num 265) 2024-04-12T11:16:51.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[29349]: osdmap e220: 8 total, 8 up, 8 in 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2609812464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[29349]: osdmap e221: 8 total, 8 up, 8 in 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[39999]: osdmap e220: 8 total, 8 up, 8 in 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2609812464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:51.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:51.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:51.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:51 smithi029 ceph-mon[39999]: osdmap e221: 8 total, 8 up, 8 in 2024-04-12T11:16:52.566 INFO:tasks.rados.rados.0.smithi029.stdout:610: done (1 left) 2024-04-12T11:16:52.566 INFO:tasks.rados.rados.0.smithi029.stdout:611: done (0 left) 2024-04-12T11:16:52.566 INFO:tasks.rados.rados.0.smithi029.stdout:612: write oid 25 current snap is 83 2024-04-12T11:16:52.566 INFO:tasks.rados.rados.0.smithi029.stdout:612: seq_num 266 ranges {470662=772236,1832916=408094} 2024-04-12T11:16:52.572 INFO:tasks.rados.rados.0.smithi029.stdout:612: writing smithi02979277-25 from 470662 to 1242898 tid 1 2024-04-12T11:16:52.578 INFO:tasks.rados.rados.0.smithi029.stdout:612: writing smithi02979277-25 from 1832916 to 2241010 tid 2 2024-04-12T11:16:52.578 INFO:tasks.rados.rados.0.smithi029.stdout:613: snap_create 2024-04-12T11:16:52.578 INFO:tasks.rados.rados.0.smithi029.stdout:612: finishing write tid 1 to smithi02979277-25 2024-04-12T11:16:52.581 INFO:tasks.rados.rados.0.smithi029.stdout:612: finishing write tid 2 to smithi02979277-25 2024-04-12T11:16:52.581 INFO:tasks.rados.rados.0.smithi029.stdout:612: finishing write tid 3 to smithi02979277-25 2024-04-12T11:16:52.581 INFO:tasks.rados.rados.0.smithi029.stdout:612: finishing write tid 4 to smithi02979277-25 2024-04-12T11:16:52.581 INFO:tasks.rados.rados.0.smithi029.stdout:612: oid 25 updating version 0 to 267 2024-04-12T11:16:52.581 INFO:tasks.rados.rados.0.smithi029.stdout:612: oid 25 updating version 267 to 269 2024-04-12T11:16:52.581 INFO:tasks.rados.rados.0.smithi029.stdout:612: oid 25 version 269 is already newer than 268 2024-04-12T11:16:52.581 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 269 (ObjNum 266 snap 83 seq_num 266) dirty exists 2024-04-12T11:16:52.581 INFO:tasks.rados.rados.0.smithi029.stdout:612: left oid 25 (ObjNum 266 snap 83 seq_num 266) 2024-04-12T11:16:52.683 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:52 smithi087 ceph-mon[32917]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:52.683 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:52 smithi087 ceph-mon[32917]: pgmap v27: 89 pgs: 89 active+clean; 237 MiB data, 709 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:52 smithi029 ceph-mon[39999]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:52 smithi029 ceph-mon[39999]: pgmap v27: 89 pgs: 89 active+clean; 237 MiB data, 709 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:52 smithi029 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T11:16:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:52 smithi029 ceph-mon[29349]: pgmap v27: 89 pgs: 89 active+clean; 237 MiB data, 709 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:16:53.570 INFO:tasks.rados.rados.0.smithi029.stdout:612: done (1 left) 2024-04-12T11:16:53.571 INFO:tasks.rados.rados.0.smithi029.stdout:613: done (0 left) 2024-04-12T11:16:53.571 INFO:tasks.rados.rados.0.smithi029.stdout:614: delete oid 25 current snap is 84 2024-04-12T11:16:53.575 INFO:tasks.rados.rados.0.smithi029.stdout:614: done (0 left) 2024-04-12T11:16:53.575 INFO:tasks.rados.rados.0.smithi029.stdout:615: snap_remove snap 77 2024-04-12T11:16:53.845 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:53 smithi087 ceph-mon[32917]: osdmap e222: 8 total, 8 up, 8 in 2024-04-12T11:16:53.845 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:53 smithi087 ceph-mon[32917]: Upgrade: Updating mgr.x 2024-04-12T11:16:53.845 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:53 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:16:53.845 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:53 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:16:53.845 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:53 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:53.845 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:53 smithi087 ceph-mon[32917]: Deploying daemon mgr.x on smithi087 2024-04-12T11:16:53.846 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:16:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:16:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:16:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[39999]: osdmap e222: 8 total, 8 up, 8 in 2024-04-12T11:16:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[39999]: Upgrade: Updating mgr.x 2024-04-12T11:16:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:16:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:16:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[39999]: Deploying daemon mgr.x on smithi087 2024-04-12T11:16:54.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[29349]: osdmap e222: 8 total, 8 up, 8 in 2024-04-12T11:16:54.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[29349]: Upgrade: Updating mgr.x 2024-04-12T11:16:54.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:16:54.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:16:54.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:16:54.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:53 smithi029 ceph-mon[29349]: Deploying daemon mgr.x on smithi087 2024-04-12T11:16:54.578 INFO:tasks.rados.rados.0.smithi029.stdout:615: done (0 left) 2024-04-12T11:16:54.578 INFO:tasks.rados.rados.0.smithi029.stdout:616: snap_create 2024-04-12T11:16:54.741 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:54 smithi087 ceph-mon[32917]: osdmap e223: 8 total, 8 up, 8 in 2024-04-12T11:16:54.741 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:54 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/702127176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:54.741 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:54 smithi087 ceph-mon[32917]: pgmap v30: 89 pgs: 89 active+clean; 240 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:16:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:54 smithi029 ceph-mon[39999]: osdmap e223: 8 total, 8 up, 8 in 2024-04-12T11:16:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:54 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/702127176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:54 smithi029 ceph-mon[39999]: pgmap v30: 89 pgs: 89 active+clean; 240 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:16:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:54 smithi029 ceph-mon[29349]: osdmap e223: 8 total, 8 up, 8 in 2024-04-12T11:16:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:54 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/702127176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:54 smithi029 ceph-mon[29349]: pgmap v30: 89 pgs: 89 active+clean; 240 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:16:55.025 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:54 smithi087 systemd[1]: Stopping Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:16:55.329 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:55 smithi087 podman[78411]: 2024-04-12 11:16:55.024986092 +0000 UTC m=+0.143946670 container died ba0eae540f953c975fcb912e9a91454623fc4f90b4b11c7c5b844d1dbd7bb42c (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409) 2024-04-12T11:16:55.583 INFO:tasks.rados.rados.0.smithi029.stdout:616: done (0 left) 2024-04-12T11:16:55.583 INFO:tasks.rados.rados.0.smithi029.stdout:617: snap_create 2024-04-12T11:16:55.670 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:55 smithi087 ceph-mon[32917]: osdmap e224: 8 total, 8 up, 8 in 2024-04-12T11:16:55.670 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:55 smithi087 podman[78411]: 2024-04-12 11:16:55.427297214 +0000 UTC m=+0.546257789 container cleanup ba0eae540f953c975fcb912e9a91454623fc4f90b4b11c7c5b844d1dbd7bb42c (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-rf-comb1-2cee185, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-12T11:16:55.670 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:55 smithi087 bash[78411]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x 2024-04-12T11:16:55.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:55 smithi087 podman[78425]: 2024-04-12 11:16:55.670343042 +0000 UTC m=+0.649196267 container remove ba0eae540f953c975fcb912e9a91454623fc4f90b4b11c7c5b844d1dbd7bb42c (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-12T11:16:55.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:55 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-12T11:16:55.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:55 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-12T11:16:55.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:55 smithi087 systemd[1]: Stopped Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:16:55.967 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:55 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.x.service: Consumed 24.527s CPU time. 2024-04-12T11:16:56.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:55 smithi029 ceph-mon[39999]: osdmap e224: 8 total, 8 up, 8 in 2024-04-12T11:16:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:55 smithi029 ceph-mon[29349]: osdmap e224: 8 total, 8 up, 8 in 2024-04-12T11:16:56.238 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:55 smithi087 systemd[1]: Starting Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:16:56.238 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:56 smithi087 podman[78556]: 2024-04-12 11:16:56.106121478 +0000 UTC m=+0.020530652 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:16:56.532 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:56 smithi087 podman[78556]: 2024-04-12 11:16:56.238516377 +0000 UTC m=+0.152925529 container create 5a46135dfdbc3347e8cb43cdef1220f8ca45f69854c290c1c106e6ad098589ec (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, RELEASE=wip-rf-comb1-2cee185, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-12T11:16:56.533 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:56 smithi087 podman[78556]: 2024-04-12 11:16:56.441092473 +0000 UTC m=+0.355501624 container init 5a46135dfdbc3347e8cb43cdef1220f8ca45f69854c290c1c106e6ad098589ec (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-rf-comb1-2cee185, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-12T11:16:56.533 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:56 smithi087 podman[78556]: 2024-04-12 11:16:56.443531607 +0000 UTC m=+0.357940759 container start 5a46135dfdbc3347e8cb43cdef1220f8ca45f69854c290c1c106e6ad098589ec (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-12T11:16:56.533 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:56 smithi087 ceph-mgr[78574]: -- 172.21.15.87:0/809580762 <== mon.1 v2:172.21.15.87:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560a151365a0 con 0x560a1510a800 2024-04-12T11:16:56.550 INFO:tasks.rados.rados.0.smithi029.stdout:617: done (0 left) 2024-04-12T11:16:56.550 INFO:tasks.rados.rados.0.smithi029.stdout:618: write oid 9 current snap is 86 2024-04-12T11:16:56.550 INFO:tasks.rados.rados.0.smithi029.stdout:618: seq_num 268 ranges {504276=639087,1666233=695410} 2024-04-12T11:16:56.556 INFO:tasks.rados.rados.0.smithi029.stdout:618: writing smithi02979277-9 from 504276 to 1143363 tid 1 2024-04-12T11:16:56.563 INFO:tasks.rados.rados.0.smithi029.stdout:618: writing smithi02979277-9 from 1666233 to 2361643 tid 2 2024-04-12T11:16:56.563 INFO:tasks.rados.rados.0.smithi029.stdout:619: write oid 15 current snap is 86 2024-04-12T11:16:56.563 INFO:tasks.rados.rados.0.smithi029.stdout:619: seq_num 269 ranges {521083=714336,1724715=555421} 2024-04-12T11:16:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:619: writing smithi02979277-15 from 521083 to 1235419 tid 1 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:619: writing smithi02979277-15 from 1724715 to 2280136 tid 2 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:620: read oid 50 snap -1 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:620: expect (ObjNum 225 snap 66 seq_num 225) 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:618: finishing write tid 1 to smithi02979277-9 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:618: finishing write tid 2 to smithi02979277-9 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:618: finishing write tid 3 to smithi02979277-9 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:621: read oid 47 snap 63 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:621: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:622: read oid 34 snap -1 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:622: expect (ObjNum 259 snap 81 seq_num 259) 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:623: write oid 13 current snap is 86 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:618: finishing write tid 4 to smithi02979277-9 2024-04-12T11:16:56.577 INFO:tasks.rados.rados.0.smithi029.stdout:618: oid 9 updating version 0 to 228 2024-04-12T11:16:56.578 INFO:tasks.rados.rados.0.smithi029.stdout:618: oid 9 version 228 is already newer than 227 2024-04-12T11:16:56.578 INFO:tasks.rados.rados.0.smithi029.stdout:618: oid 9 version 228 is already newer than 226 2024-04-12T11:16:56.578 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 228 (ObjNum 268 snap 86 seq_num 268) dirty exists 2024-04-12T11:16:56.578 INFO:tasks.rados.rados.0.smithi029.stdout:618: left oid 9 (ObjNum 268 snap 86 seq_num 268) 2024-04-12T11:16:56.578 INFO:tasks.rados.rados.0.smithi029.stdout:619: finishing write tid 1 to smithi02979277-15 2024-04-12T11:16:56.578 INFO:tasks.rados.rados.0.smithi029.stdout:623: seq_num 270 ranges {537890=789585,1783197=699079} 2024-04-12T11:16:56.583 INFO:tasks.rados.rados.0.smithi029.stdout:623: writing smithi02979277-13 from 537890 to 1327475 tid 1 2024-04-12T11:16:56.590 INFO:tasks.rados.rados.0.smithi029.stdout:623: writing smithi02979277-13 from 1783197 to 2482276 tid 2 2024-04-12T11:16:56.590 INFO:tasks.rados.rados.0.smithi029.stdout:618: done (5 left) 2024-04-12T11:16:56.590 INFO:tasks.rados.rados.0.smithi029.stdout:624: read oid 12 snap 73 2024-04-12T11:16:56.590 INFO:tasks.rados.rados.0.smithi029.stdout:624: expect deleted 2024-04-12T11:16:56.590 INFO:tasks.rados.rados.0.smithi029.stdout:625: snap_remove snap 36 2024-04-12T11:16:56.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:56 smithi087 bash[78556]: 5a46135dfdbc3347e8cb43cdef1220f8ca45f69854c290c1c106e6ad098589ec 2024-04-12T11:16:56.966 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:56 smithi087 systemd[1]: Started Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:16:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:56 smithi087 ceph-mon[32917]: osdmap e225: 8 total, 8 up, 8 in 2024-04-12T11:16:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:56 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/830566120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:56 smithi087 ceph-mon[32917]: pgmap v33: 89 pgs: 89 active+clean; 240 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:16:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:56 smithi087 ceph-mon[32917]: osdmap e226: 8 total, 8 up, 8 in 2024-04-12T11:16:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:56 smithi029 ceph-mon[39999]: osdmap e225: 8 total, 8 up, 8 in 2024-04-12T11:16:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:56 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/830566120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:56 smithi029 ceph-mon[39999]: pgmap v33: 89 pgs: 89 active+clean; 240 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:16:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:56 smithi029 ceph-mon[39999]: osdmap e226: 8 total, 8 up, 8 in 2024-04-12T11:16:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:56 smithi029 ceph-mon[29349]: osdmap e225: 8 total, 8 up, 8 in 2024-04-12T11:16:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:56 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/830566120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:56 smithi029 ceph-mon[29349]: pgmap v33: 89 pgs: 89 active+clean; 240 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:16:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:56 smithi029 ceph-mon[29349]: osdmap e226: 8 total, 8 up, 8 in 2024-04-12T11:16:57.462 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:57.013+0000 7fbeb2c04200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:16:57.462 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:57.112+0000 7fbeb2c04200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:16:57.592 INFO:tasks.rados.rados.0.smithi029.stdout:625: done (6 left) 2024-04-12T11:16:57.592 INFO:tasks.rados.rados.0.smithi029.stdout:626: snap_remove snap 74 2024-04-12T11:16:57.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:57 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:57.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:57 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:57.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:57 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:57.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:57 smithi087 ceph-mon[32917]: osdmap e227: 8 total, 8 up, 8 in 2024-04-12T11:16:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:57 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:57 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:57 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:57 smithi029 ceph-mon[39999]: osdmap e227: 8 total, 8 up, 8 in 2024-04-12T11:16:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:57 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:57 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:16:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:57 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:16:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:57 smithi029 ceph-mon[29349]: osdmap e227: 8 total, 8 up, 8 in 2024-04-12T11:16:58.079 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:57.811+0000 7fbeb2c04200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:16:58.079 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:57.936+0000 7fbeb2c04200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:16:58.422 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:58 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:58.078+0000 7fbeb2c04200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:16:58.422 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:58 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:58.177+0000 7fbeb2c04200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:16:58.422 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:58 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:58.273+0000 7fbeb2c04200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:16:58.592 INFO:tasks.rados.rados.0.smithi029.stdout:626: done (6 left) 2024-04-12T11:16:58.593 INFO:tasks.rados.rados.0.smithi029.stdout:627: rollback oid 45 current snap is 86 2024-04-12T11:16:58.593 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 45 to 83 2024-04-12T11:16:58.593 INFO:tasks.rados.rados.0.smithi029.stdout:622: expect (ObjNum 259 snap 81 seq_num 259) 2024-04-12T11:16:58.706 INFO:tasks.rados.rados.0.smithi029.stdout:621: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:16:58.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:58 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:58.541+0000 7fbeb2c04200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:16:58.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:58 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:58.667+0000 7fbeb2c04200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:16:58.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:58 smithi087 ceph-mon[32917]: pgmap v36: 89 pgs: 89 active+clean; 245 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:16:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:58 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3247737241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:16:58 smithi087 ceph-mon[32917]: osdmap e228: 8 total, 8 up, 8 in 2024-04-12T11:16:58.860 INFO:tasks.rados.rados.0.smithi029.stdout:619: finishing write tid 2 to smithi02979277-15 2024-04-12T11:16:58.860 INFO:tasks.rados.rados.0.smithi029.stdout:619: finishing write tid 3 to smithi02979277-15 2024-04-12T11:16:58.860 INFO:tasks.rados.rados.0.smithi029.stdout:621: done (6 left) 2024-04-12T11:16:58.860 INFO:tasks.rados.rados.0.smithi029.stdout:622: done (5 left) 2024-04-12T11:16:58.860 INFO:tasks.rados.rados.0.smithi029.stdout:628: snap_remove snap 68 2024-04-12T11:16:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:58 smithi029 ceph-mon[39999]: pgmap v36: 89 pgs: 89 active+clean; 245 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:16:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:58 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3247737241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:16:58 smithi029 ceph-mon[39999]: osdmap e228: 8 total, 8 up, 8 in 2024-04-12T11:16:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:58 smithi029 ceph-mon[29349]: pgmap v36: 89 pgs: 89 active+clean; 245 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:16:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:58 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3247737241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:16:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:16:58 smithi029 ceph-mon[29349]: osdmap e228: 8 total, 8 up, 8 in 2024-04-12T11:16:59.059 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:58 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:58.772+0000 7fbeb2c04200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:16:59.059 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:58 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:58.881+0000 7fbeb2c04200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:16:59.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:16:59 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:16:59.064+0000 7fbeb2c04200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:16:59.626 INFO:tasks.rados.rados.0.smithi029.stdout:628: done (5 left) 2024-04-12T11:16:59.626 INFO:tasks.rados.rados.0.smithi029.stdout:629: read oid 21 snap -1 2024-04-12T11:16:59.626 INFO:tasks.rados.rados.0.smithi029.stdout:629: expect (ObjNum 223 snap 64 seq_num 223) 2024-04-12T11:16:59.626 INFO:tasks.rados.rados.0.smithi029.stdout:630: write oid 35 current snap is 86 2024-04-12T11:16:59.626 INFO:tasks.rados.rados.0.smithi029.stdout:619: finishing write tid 4 to smithi02979277-15 2024-04-12T11:16:59.626 INFO:tasks.rados.rados.0.smithi029.stdout:619: oid 15 updating version 0 to 240 2024-04-12T11:16:59.626 INFO:tasks.rados.rados.0.smithi029.stdout:619: oid 15 updating version 240 to 241 2024-04-12T11:16:59.626 INFO:tasks.rados.rados.0.smithi029.stdout:619: oid 15 version 241 is already newer than 239 2024-04-12T11:16:59.627 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 241 (ObjNum 269 snap 86 seq_num 269) dirty exists 2024-04-12T11:16:59.627 INFO:tasks.rados.rados.0.smithi029.stdout:619: left oid 15 (ObjNum 269 snap 86 seq_num 269) 2024-04-12T11:16:59.627 INFO:tasks.rados.rados.0.smithi029.stdout:620: expect (ObjNum 225 snap 66 seq_num 225) 2024-04-12T11:16:59.921 INFO:tasks.rados.rados.0.smithi029.stdout:623: finishing write tid 1 to smithi02979277-13 2024-04-12T11:16:59.922 INFO:tasks.rados.rados.0.smithi029.stdout:623: finishing write tid 2 to smithi02979277-13 2024-04-12T11:16:59.922 INFO:tasks.rados.rados.0.smithi029.stdout:623: finishing write tid 3 to smithi02979277-13 2024-04-12T11:16:59.922 INFO:tasks.rados.rados.0.smithi029.stdout:623: finishing write tid 4 to smithi02979277-13 2024-04-12T11:16:59.922 INFO:tasks.rados.rados.0.smithi029.stdout:623: oid 13 updating version 0 to 280 2024-04-12T11:16:59.922 INFO:tasks.rados.rados.0.smithi029.stdout:623: oid 13 version 280 is already newer than 279 2024-04-12T11:16:59.922 INFO:tasks.rados.rados.0.smithi029.stdout:623: oid 13 updating version 280 to 281 2024-04-12T11:16:59.922 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 281 (ObjNum 270 snap 86 seq_num 270) dirty exists 2024-04-12T11:16:59.922 INFO:tasks.rados.rados.0.smithi029.stdout:623: left oid 13 (ObjNum 270 snap 86 seq_num 270) 2024-04-12T11:16:59.922 INFO:tasks.rados.rados.0.smithi029.stdout:630: seq_num 271 ranges {554697=464834,1725326=559090} 2024-04-12T11:16:59.927 INFO:tasks.rados.rados.0.smithi029.stdout:630: writing smithi02979277-35 from 554697 to 1019531 tid 1 2024-04-12T11:16:59.934 INFO:tasks.rados.rados.0.smithi029.stdout:630: writing smithi02979277-35 from 1725326 to 2284416 tid 2 2024-04-12T11:16:59.934 INFO:tasks.rados.rados.0.smithi029.stdout:619: done (6 left) 2024-04-12T11:16:59.934 INFO:tasks.rados.rados.0.smithi029.stdout:620: done (5 left) 2024-04-12T11:16:59.934 INFO:tasks.rados.rados.0.smithi029.stdout:623: done (4 left) 2024-04-12T11:16:59.934 INFO:tasks.rados.rados.0.smithi029.stdout:624: done (3 left) 2024-04-12T11:16:59.934 INFO:tasks.rados.rados.0.smithi029.stdout:631: snap_remove snap 76 2024-04-12T11:17:00.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:00.097+0000 7fbeb2c04200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:17:00.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:00.236+0000 7fbeb2c04200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:17:00.628 INFO:tasks.rados.rados.0.smithi029.stdout:631: done (3 left) 2024-04-12T11:17:00.628 INFO:tasks.rados.rados.0.smithi029.stdout:632: write oid 33 current snap is 86 2024-04-12T11:17:00.628 INFO:tasks.rados.rados.0.smithi029.stdout:627: finishing rollback tid 0 to smithi02979277-45 2024-04-12T11:17:00.628 INFO:tasks.rados.rados.0.smithi029.stdout:627: finishing rollback tid 1 to smithi02979277-45 2024-04-12T11:17:00.628 INFO:tasks.rados.rados.0.smithi029.stdout:627: finishing rollback tid 2 to smithi02979277-45 2024-04-12T11:17:00.628 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 318 (ObjNum 251 snap 78 seq_num 251) dirty exists 2024-04-12T11:17:00.628 INFO:tasks.rados.rados.0.smithi029.stdout:629: expect (ObjNum 223 snap 64 seq_num 223) 2024-04-12T11:17:00.904 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:00.600+0000 7fbeb2c04200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:17:00.905 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:00.696+0000 7fbeb2c04200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:17:00.905 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:00 smithi087 ceph-mon[32917]: osdmap e229: 8 total, 8 up, 8 in 2024-04-12T11:17:00.905 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:00 smithi087 ceph-mon[32917]: pgmap v39: 89 pgs: 89 active+clean; 245 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:17:00.905 INFO:tasks.rados.rados.0.smithi029.stdout:630: finishing write tid 1 to smithi02979277-35 2024-04-12T11:17:00.906 INFO:tasks.rados.rados.0.smithi029.stdout:630: finishing write tid 2 to smithi02979277-35 2024-04-12T11:17:00.906 INFO:tasks.rados.rados.0.smithi029.stdout:630: finishing write tid 3 to smithi02979277-35 2024-04-12T11:17:00.906 INFO:tasks.rados.rados.0.smithi029.stdout:630: finishing write tid 4 to smithi02979277-35 2024-04-12T11:17:00.906 INFO:tasks.rados.rados.0.smithi029.stdout:630: oid 35 updating version 0 to 365 2024-04-12T11:17:00.906 INFO:tasks.rados.rados.0.smithi029.stdout:630: oid 35 version 365 is already newer than 363 2024-04-12T11:17:00.906 INFO:tasks.rados.rados.0.smithi029.stdout:630: oid 35 version 365 is already newer than 364 2024-04-12T11:17:00.906 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 365 (ObjNum 271 snap 86 seq_num 271) dirty exists 2024-04-12T11:17:00.906 INFO:tasks.rados.rados.0.smithi029.stdout:630: left oid 35 (ObjNum 271 snap 86 seq_num 271) 2024-04-12T11:17:00.906 INFO:tasks.rados.rados.0.smithi029.stdout:632: seq_num 272 ranges {571504=540083,1783808=702748} 2024-04-12T11:17:00.911 INFO:tasks.rados.rados.0.smithi029.stdout:632: writing smithi02979277-33 from 571504 to 1111587 tid 1 2024-04-12T11:17:00.919 INFO:tasks.rados.rados.0.smithi029.stdout:632: writing smithi02979277-33 from 1783808 to 2486556 tid 2 2024-04-12T11:17:00.919 INFO:tasks.rados.rados.0.smithi029.stdout:627: done (3 left) 2024-04-12T11:17:00.919 INFO:tasks.rados.rados.0.smithi029.stdout:629: done (2 left) 2024-04-12T11:17:00.919 INFO:tasks.rados.rados.0.smithi029.stdout:630: done (1 left) 2024-04-12T11:17:00.919 INFO:tasks.rados.rados.0.smithi029.stdout:633: delete oid 45 current snap is 86 2024-04-12T11:17:00.919 INFO:tasks.rados.rados.0.smithi029.stdout:632: finishing write tid 1 to smithi02979277-33 2024-04-12T11:17:00.921 INFO:tasks.rados.rados.0.smithi029.stdout:633: done (1 left) 2024-04-12T11:17:00.921 INFO:tasks.rados.rados.0.smithi029.stdout:634: snap_create 2024-04-12T11:17:00.922 INFO:tasks.rados.rados.0.smithi029.stdout:632: finishing write tid 2 to smithi02979277-33 2024-04-12T11:17:00.922 INFO:tasks.rados.rados.0.smithi029.stdout:632: finishing write tid 3 to smithi02979277-33 2024-04-12T11:17:00.922 INFO:tasks.rados.rados.0.smithi029.stdout:632: finishing write tid 4 to smithi02979277-33 2024-04-12T11:17:00.922 INFO:tasks.rados.rados.0.smithi029.stdout:632: oid 33 updating version 0 to 229 2024-04-12T11:17:00.922 INFO:tasks.rados.rados.0.smithi029.stdout:632: oid 33 updating version 229 to 230 2024-04-12T11:17:00.922 INFO:tasks.rados.rados.0.smithi029.stdout:632: oid 33 version 230 is already newer than 228 2024-04-12T11:17:00.922 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 230 (ObjNum 272 snap 86 seq_num 272) dirty exists 2024-04-12T11:17:00.923 INFO:tasks.rados.rados.0.smithi029.stdout:632: left oid 33 (ObjNum 272 snap 86 seq_num 272) 2024-04-12T11:17:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:00 smithi029 ceph-mon[29349]: osdmap e229: 8 total, 8 up, 8 in 2024-04-12T11:17:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:00 smithi029 ceph-mon[29349]: pgmap v39: 89 pgs: 89 active+clean; 245 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:17:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:00 smithi029 ceph-mon[39999]: osdmap e229: 8 total, 8 up, 8 in 2024-04-12T11:17:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:00 smithi029 ceph-mon[39999]: pgmap v39: 89 pgs: 89 active+clean; 245 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:17:01.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:00.903+0000 7fbeb2c04200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:17:01.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:01.104+0000 7fbeb2c04200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:17:01.552 INFO:tasks.rados.rados.0.smithi029.stdout:632: done (1 left) 2024-04-12T11:17:01.552 INFO:tasks.rados.rados.0.smithi029.stdout:634: done (0 left) 2024-04-12T11:17:01.552 INFO:tasks.rados.rados.0.smithi029.stdout:635: rollback oid 19 current snap is 87 2024-04-12T11:17:01.553 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 19 to 83 2024-04-12T11:17:01.553 INFO:tasks.rados.rados.0.smithi029.stdout:636: rollback oid 6 current snap is 87 2024-04-12T11:17:01.553 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 6 to 84 2024-04-12T11:17:01.553 INFO:tasks.rados.rados.0.smithi029.stdout:637: snap_remove snap 43 2024-04-12T11:17:01.710 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:01.412+0000 7fbeb2c04200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:17:01.711 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:01.709+0000 7fbeb2c04200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:17:01.711 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:01 smithi087 ceph-mon[32917]: osdmap e230: 8 total, 8 up, 8 in 2024-04-12T11:17:01.711 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:01 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2937291475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:01.711 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:01 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:01.711 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:01 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:01.711 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:01 smithi087 ceph-mon[32917]: osdmap e231: 8 total, 8 up, 8 in 2024-04-12T11:17:01.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:01 smithi029 ceph-mon[29349]: osdmap e230: 8 total, 8 up, 8 in 2024-04-12T11:17:01.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:01 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/2937291475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:01.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:01 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:01.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:01 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:01.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:01 smithi029 ceph-mon[29349]: osdmap e231: 8 total, 8 up, 8 in 2024-04-12T11:17:01.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:01 smithi029 ceph-mon[39999]: osdmap e230: 8 total, 8 up, 8 in 2024-04-12T11:17:01.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:01 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2937291475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:01.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:01 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:01.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:01 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:01.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:01 smithi029 ceph-mon[39999]: osdmap e231: 8 total, 8 up, 8 in 2024-04-12T11:17:02.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:01.966+0000 7fbeb2c04200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:17:02.553 INFO:tasks.rados.rados.0.smithi029.stdout:637: done (2 left) 2024-04-12T11:17:02.554 INFO:tasks.rados.rados.0.smithi029.stdout:638: read oid 26 snap -1 2024-04-12T11:17:02.554 INFO:tasks.rados.rados.0.smithi029.stdout:638: expect (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:17:02.554 INFO:tasks.rados.rados.0.smithi029.stdout:639: rollback oid 14 current snap is 87 2024-04-12T11:17:02.554 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 82 2024-04-12T11:17:02.554 INFO:tasks.rados.rados.0.smithi029.stdout:640: snap_create 2024-04-12T11:17:02.554 INFO:tasks.rados.rados.0.smithi029.stdout:635: finishing rollback tid 1 to smithi02979277-19 2024-04-12T11:17:02.554 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 310 (ObjNum 49 snap 0 seq_num 4294967295) dirty dne 2024-04-12T11:17:02.554 INFO:tasks.rados.rados.0.smithi029.stdout:636: finishing rollback tid 1 to smithi02979277-6 2024-04-12T11:17:02.555 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 83 (ObjNum 49 snap 0 seq_num 1777349616) dirty dne 2024-04-12T11:17:02.562 INFO:tasks.rados.rados.0.smithi029.stdout:638: expect (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:17:02.680 INFO:tasks.rados.rados.0.smithi029.stdout:639: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:17:02.680 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 273 (ObjNum 33 snap 0 seq_num 1801043936) dirty dne 2024-04-12T11:17:03.097 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:02.849+0000 7fbeb2c04200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:17:03.431 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:03.431 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:03.431 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-mon[32917]: pgmap v42: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 554 KiB/s wr, 4 op/s 2024-04-12T11:17:03.431 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-mon[32917]: osdmap e232: 8 total, 8 up, 8 in 2024-04-12T11:17:03.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:03 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:03.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:03 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:03.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:03 smithi029 ceph-mon[29349]: pgmap v42: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 554 KiB/s wr, 4 op/s 2024-04-12T11:17:03.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:03 smithi029 ceph-mon[29349]: osdmap e232: 8 total, 8 up, 8 in 2024-04-12T11:17:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:03 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:03 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:03 smithi029 ceph-mon[39999]: pgmap v42: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 554 KiB/s wr, 4 op/s 2024-04-12T11:17:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:03 smithi029 ceph-mon[39999]: osdmap e232: 8 total, 8 up, 8 in 2024-04-12T11:17:03.554 INFO:tasks.rados.rados.0.smithi029.stdout:635: done (4 left) 2024-04-12T11:17:03.554 INFO:tasks.rados.rados.0.smithi029.stdout:636: done (3 left) 2024-04-12T11:17:03.554 INFO:tasks.rados.rados.0.smithi029.stdout:638: done (2 left) 2024-04-12T11:17:03.554 INFO:tasks.rados.rados.0.smithi029.stdout:639: done (1 left) 2024-04-12T11:17:03.554 INFO:tasks.rados.rados.0.smithi029.stdout:640: done (0 left) 2024-04-12T11:17:03.555 INFO:tasks.rados.rados.0.smithi029.stdout:641: read oid 18 snap -1 2024-04-12T11:17:03.555 INFO:tasks.rados.rados.0.smithi029.stdout:641: expect (ObjNum 264 snap 82 seq_num 264) 2024-04-12T11:17:03.555 INFO:tasks.rados.rados.0.smithi029.stdout:642: snap_remove snap 87 2024-04-12T11:17:03.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:03.430+0000 7fbeb2c04200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:17:03.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:03.534+0000 7fbeb2c04200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:17:03.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:17:03.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:17:04.104 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:03.762+0000 7fbeb2c04200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:17:04.104 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: [12/Apr/2024:11:17:03] ENGINE Bus STARTING 2024-04-12T11:17:04.105 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: CherryPy Checker: 2024-04-12T11:17:04.105 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: The Application mounted at '' has an empty config. 2024-04-12T11:17:04.105 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:04.105 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: [12/Apr/2024:11:17:03] ENGINE Serving on http://:::9283 2024-04-12T11:17:04.105 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: [12/Apr/2024:11:17:03] ENGINE Bus STARTED 2024-04-12T11:17:04.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:04 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/674135967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:04.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:04 smithi087 ceph-mon[32917]: osdmap e233: 8 total, 8 up, 8 in 2024-04-12T11:17:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:04 smithi087 ceph-mon[32917]: Standby manager daemon x restarted 2024-04-12T11:17:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:04 smithi087 ceph-mon[32917]: Standby manager daemon x started 2024-04-12T11:17:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:04 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:17:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:04 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:17:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:04 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:17:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:04 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:17:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/674135967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[39999]: osdmap e233: 8 total, 8 up, 8 in 2024-04-12T11:17:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[39999]: Standby manager daemon x restarted 2024-04-12T11:17:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[39999]: Standby manager daemon x started 2024-04-12T11:17:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:17:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:17:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:17:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:17:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/674135967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[29349]: osdmap e233: 8 total, 8 up, 8 in 2024-04-12T11:17:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[29349]: Standby manager daemon x restarted 2024-04-12T11:17:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[29349]: Standby manager daemon x started 2024-04-12T11:17:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:17:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:17:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:17:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-mon[29349]: from='mgr.? 172.21.15.87:0/1628447999' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:17:04.566 INFO:tasks.rados.rados.0.smithi029.stdout:642: done (1 left) 2024-04-12T11:17:04.566 INFO:tasks.rados.rados.0.smithi029.stdout:643: read oid 13 snap -1 2024-04-12T11:17:04.566 INFO:tasks.rados.rados.0.smithi029.stdout:643: expect (ObjNum 270 snap 86 seq_num 270) 2024-04-12T11:17:04.566 INFO:tasks.rados.rados.0.smithi029.stdout:644: delete oid 39 current snap is 88 2024-04-12T11:17:04.566 INFO:tasks.rados.rados.0.smithi029.stdout:641: expect (ObjNum 264 snap 82 seq_num 264) 2024-04-12T11:17:04.705 INFO:tasks.rados.rados.0.smithi029.stdout:643: expect (ObjNum 270 snap 86 seq_num 270) 2024-04-12T11:17:04.848 INFO:tasks.rados.rados.0.smithi029.stdout:641: done (2 left) 2024-04-12T11:17:04.848 INFO:tasks.rados.rados.0.smithi029.stdout:643: done (1 left) 2024-04-12T11:17:04.848 INFO:tasks.rados.rados.0.smithi029.stdout:644: done (0 left) 2024-04-12T11:17:04.848 INFO:tasks.rados.rados.0.smithi029.stdout:645: read oid 40 snap -1 2024-04-12T11:17:04.848 INFO:tasks.rados.rados.0.smithi029.stdout:645: expect (ObjNum 238 snap 75 seq_num 238) 2024-04-12T11:17:04.848 INFO:tasks.rados.rados.0.smithi029.stdout:646: read oid 11 snap -1 2024-04-12T11:17:04.848 INFO:tasks.rados.rados.0.smithi029.stdout:646: expect (ObjNum 178 snap 47 seq_num 178) 2024-04-12T11:17:04.848 INFO:tasks.rados.rados.0.smithi029.stdout:647: snap_remove snap 83 2024-04-12T11:17:05.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:17:04] "GET /metrics HTTP/1.1" 200 36250 "" "Prometheus/2.43.0" 2024-04-12T11:17:05.372 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:05 smithi087 ceph-mon[32917]: pgmap v45: 89 pgs: 89 active+clean; 239 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 554 KiB/s wr, 5 op/s 2024-04-12T11:17:05.373 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:05 smithi087 ceph-mon[32917]: mgrmap e31: y(active, since 28s), standbys: x 2024-04-12T11:17:05.373 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:05 smithi087 ceph-mon[32917]: osdmap e234: 8 total, 8 up, 8 in 2024-04-12T11:17:05.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:05 smithi029 ceph-mon[39999]: pgmap v45: 89 pgs: 89 active+clean; 239 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 554 KiB/s wr, 5 op/s 2024-04-12T11:17:05.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:05 smithi029 ceph-mon[39999]: mgrmap e31: y(active, since 28s), standbys: x 2024-04-12T11:17:05.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:05 smithi029 ceph-mon[39999]: osdmap e234: 8 total, 8 up, 8 in 2024-04-12T11:17:05.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:05 smithi029 ceph-mon[29349]: pgmap v45: 89 pgs: 89 active+clean; 239 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 554 KiB/s wr, 5 op/s 2024-04-12T11:17:05.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:05 smithi029 ceph-mon[29349]: mgrmap e31: y(active, since 28s), standbys: x 2024-04-12T11:17:05.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:05 smithi029 ceph-mon[29349]: osdmap e234: 8 total, 8 up, 8 in 2024-04-12T11:17:05.568 INFO:tasks.rados.rados.0.smithi029.stdout:647: done (2 left) 2024-04-12T11:17:05.568 INFO:tasks.rados.rados.0.smithi029.stdout:648: read oid 18 snap -1 2024-04-12T11:17:05.568 INFO:tasks.rados.rados.0.smithi029.stdout:648: expect (ObjNum 264 snap 82 seq_num 264) 2024-04-12T11:17:05.569 INFO:tasks.rados.rados.0.smithi029.stdout:649: read oid 25 snap -1 2024-04-12T11:17:05.569 INFO:tasks.rados.rados.0.smithi029.stdout:649: expect deleted 2024-04-12T11:17:05.569 INFO:tasks.rados.rados.0.smithi029.stdout:650: delete oid 14 current snap is 88 2024-04-12T11:17:05.569 INFO:tasks.rados.rados.0.smithi029.stdout:645: expect (ObjNum 238 snap 75 seq_num 238) 2024-04-12T11:17:05.637 INFO:tasks.rados.rados.0.smithi029.stdout:646: expect (ObjNum 178 snap 47 seq_num 178) 2024-04-12T11:17:05.832 INFO:tasks.rados.rados.0.smithi029.stdout:648: expect (ObjNum 264 snap 82 seq_num 264) 2024-04-12T11:17:05.971 INFO:tasks.rados.rados.0.smithi029.stdout:645: done (4 left) 2024-04-12T11:17:05.971 INFO:tasks.rados.rados.0.smithi029.stdout:646: done (3 left) 2024-04-12T11:17:05.971 INFO:tasks.rados.rados.0.smithi029.stdout:648: done (2 left) 2024-04-12T11:17:05.972 INFO:tasks.rados.rados.0.smithi029.stdout:649: done (1 left) 2024-04-12T11:17:05.972 INFO:tasks.rados.rados.0.smithi029.stdout:650: done (0 left) 2024-04-12T11:17:05.972 INFO:tasks.rados.rados.0.smithi029.stdout:651: snap_remove snap 57 2024-04-12T11:17:06.554 INFO:tasks.rados.rados.0.smithi029.stdout:651: done (0 left) 2024-04-12T11:17:06.554 INFO:tasks.rados.rados.0.smithi029.stdout:652: write oid 29 current snap is 88 2024-04-12T11:17:06.554 INFO:tasks.rados.rados.0.smithi029.stdout:652: seq_num 276 ranges {638732=557432,1617736=710086} 2024-04-12T11:17:06.561 INFO:tasks.rados.rados.0.smithi029.stdout:652: writing smithi02979277-29 from 638732 to 1196164 tid 1 2024-04-12T11:17:06.568 INFO:tasks.rados.rados.0.smithi029.stdout:652: writing smithi02979277-29 from 1617736 to 2327822 tid 2 2024-04-12T11:17:06.568 INFO:tasks.rados.rados.0.smithi029.stdout:653: delete oid 28 current snap is 88 2024-04-12T11:17:06.569 INFO:tasks.rados.rados.0.smithi029.stdout:652: finishing write tid 1 to smithi02979277-29 2024-04-12T11:17:06.571 INFO:tasks.rados.rados.0.smithi029.stdout:653: done (1 left) 2024-04-12T11:17:06.571 INFO:tasks.rados.rados.0.smithi029.stdout:654: delete oid 7 current snap is 88 2024-04-12T11:17:06.571 INFO:tasks.rados.rados.0.smithi029.stdout:652: finishing write tid 2 to smithi02979277-29 2024-04-12T11:17:06.572 INFO:tasks.rados.rados.0.smithi029.stdout:652: finishing write tid 3 to smithi02979277-29 2024-04-12T11:17:06.572 INFO:tasks.rados.rados.0.smithi029.stdout:652: finishing write tid 4 to smithi02979277-29 2024-04-12T11:17:06.572 INFO:tasks.rados.rados.0.smithi029.stdout:652: oid 29 updating version 0 to 235 2024-04-12T11:17:06.572 INFO:tasks.rados.rados.0.smithi029.stdout:652: oid 29 version 235 is already newer than 234 2024-04-12T11:17:06.572 INFO:tasks.rados.rados.0.smithi029.stdout:652: oid 29 version 235 is already newer than 233 2024-04-12T11:17:06.572 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 235 (ObjNum 276 snap 88 seq_num 276) dirty exists 2024-04-12T11:17:06.572 INFO:tasks.rados.rados.0.smithi029.stdout:652: left oid 29 (ObjNum 276 snap 88 seq_num 276) 2024-04-12T11:17:06.573 INFO:tasks.rados.rados.0.smithi029.stdout:652: done (1 left) 2024-04-12T11:17:06.573 INFO:tasks.rados.rados.0.smithi029.stdout:654: done (0 left) 2024-04-12T11:17:06.573 INFO:tasks.rados.rados.0.smithi029.stdout:655: read oid 4 snap -1 2024-04-12T11:17:06.573 INFO:tasks.rados.rados.0.smithi029.stdout:655: expect deleted 2024-04-12T11:17:06.573 INFO:tasks.rados.rados.0.smithi029.stdout:656: rollback oid 16 current snap is 88 2024-04-12T11:17:06.573 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 16 to 86 2024-04-12T11:17:06.573 INFO:tasks.rados.rados.0.smithi029.stdout:657: write oid 45 current snap is 88 2024-04-12T11:17:06.573 INFO:tasks.rados.rados.0.smithi029.stdout:657: seq_num 279 ranges {689153=783179,2076829=457413} 2024-04-12T11:17:06.580 INFO:tasks.rados.rados.0.smithi029.stdout:657: writing smithi02979277-45 from 689153 to 1472332 tid 1 2024-04-12T11:17:06.587 INFO:tasks.rados.rados.0.smithi029.stdout:657: writing smithi02979277-45 from 2076829 to 2534242 tid 2 2024-04-12T11:17:06.587 INFO:tasks.rados.rados.0.smithi029.stdout:658: delete oid 11 current snap is 88 2024-04-12T11:17:06.587 INFO:tasks.rados.rados.0.smithi029.stdout:656: finishing rollback tid 0 to smithi02979277-16 2024-04-12T11:17:06.587 INFO:tasks.rados.rados.0.smithi029.stdout:656: finishing rollback tid 1 to smithi02979277-16 2024-04-12T11:17:06.587 INFO:tasks.rados.rados.0.smithi029.stdout:656: finishing rollback tid 2 to smithi02979277-16 2024-04-12T11:17:06.587 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 383 (ObjNum 240 snap 75 seq_num 240) dirty exists 2024-04-12T11:17:06.587 INFO:tasks.rados.rados.0.smithi029.stdout:657: finishing write tid 1 to smithi02979277-45 2024-04-12T11:17:06.589 INFO:tasks.rados.rados.0.smithi029.stdout:655: done (3 left) 2024-04-12T11:17:06.589 INFO:tasks.rados.rados.0.smithi029.stdout:656: done (2 left) 2024-04-12T11:17:06.589 INFO:tasks.rados.rados.0.smithi029.stdout:658: done (1 left) 2024-04-12T11:17:06.589 INFO:tasks.rados.rados.0.smithi029.stdout:659: read oid 41 snap -1 2024-04-12T11:17:06.589 INFO:tasks.rados.rados.0.smithi029.stdout:659: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:17:06.589 INFO:tasks.rados.rados.0.smithi029.stdout:660: read oid 8 snap 63 2024-04-12T11:17:06.589 INFO:tasks.rados.rados.0.smithi029.stdout:660: expect (ObjNum 207 snap 60 seq_num 207) 2024-04-12T11:17:06.589 INFO:tasks.rados.rados.0.smithi029.stdout:661: rollback oid 10 current snap is 88 2024-04-12T11:17:06.589 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 10 to 78 2024-04-12T11:17:06.590 INFO:tasks.rados.rados.0.smithi029.stdout:662: snap_remove snap 58 2024-04-12T11:17:06.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: osdmap e235: 8 total, 8 up, 8 in 2024-04-12T11:17:06.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1887064597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: Upgrade: Setting container_image for all mgr 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-12T11:17:06.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: Upgrade: It appears safe to stop mon.a 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: pgmap v48: 89 pgs: 89 active+clean; 239 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[39999]: osdmap e236: 8 total, 8 up, 8 in 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: osdmap e235: 8 total, 8 up, 8 in 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/1887064597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.892 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: Upgrade: Setting container_image for all mgr 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: Upgrade: It appears safe to stop mon.a 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: pgmap v48: 89 pgs: 89 active+clean; 239 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:17:06.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:06.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:06 smithi029 ceph-mon[29349]: osdmap e236: 8 total, 8 up, 8 in 2024-04-12T11:17:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: osdmap e235: 8 total, 8 up, 8 in 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1887064597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: Upgrade: Setting container_image for all mgr 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-12T11:17:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-12T11:17:06.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-12T11:17:06.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-12T11:17:06.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T11:17:06.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-12T11:17:06.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: Upgrade: It appears safe to stop mon.a 2024-04-12T11:17:06.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: pgmap v48: 89 pgs: 89 active+clean; 239 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:17:06.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:06.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:06 smithi087 ceph-mon[32917]: osdmap e236: 8 total, 8 up, 8 in 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:657: finishing write tid 2 to smithi02979277-45 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:657: finishing write tid 3 to smithi02979277-45 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:657: finishing write tid 4 to smithi02979277-45 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:657: oid 45 updating version 0 to 342 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:657: oid 45 version 342 is already newer than 341 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:657: oid 45 version 342 is already newer than 340 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 342 (ObjNum 279 snap 88 seq_num 279) dirty exists 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:657: left oid 45 (ObjNum 279 snap 88 seq_num 279) 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:661: finishing rollback tid 1 to smithi02979277-10 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:17:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:660: expect (ObjNum 207 snap 60 seq_num 207) 2024-04-12T11:17:07.734 INFO:tasks.rados.rados.0.smithi029.stdout:659: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:17:07.875 INFO:tasks.rados.rados.0.smithi029.stdout:657: done (4 left) 2024-04-12T11:17:07.875 INFO:tasks.rados.rados.0.smithi029.stdout:659: done (3 left) 2024-04-12T11:17:07.875 INFO:tasks.rados.rados.0.smithi029.stdout:660: done (2 left) 2024-04-12T11:17:07.875 INFO:tasks.rados.rados.0.smithi029.stdout:661: done (1 left) 2024-04-12T11:17:07.875 INFO:tasks.rados.rados.0.smithi029.stdout:662: done (0 left) 2024-04-12T11:17:07.875 INFO:tasks.rados.rados.0.smithi029.stdout:663: read oid 16 snap -1 2024-04-12T11:17:07.875 INFO:tasks.rados.rados.0.smithi029.stdout:663: expect (ObjNum 240 snap 75 seq_num 240) 2024-04-12T11:17:07.876 INFO:tasks.rados.rados.0.smithi029.stdout:664: snap_remove snap 75 2024-04-12T11:17:08.576 INFO:tasks.rados.rados.0.smithi029.stdout:664: done (1 left) 2024-04-12T11:17:08.576 INFO:tasks.rados.rados.0.smithi029.stdout:665: delete oid 32 current snap is 88 2024-04-12T11:17:08.576 INFO:tasks.rados.rados.0.smithi029.stdout:663: expect (ObjNum 240 snap 75 seq_num 240) 2024-04-12T11:17:08.648 INFO:tasks.rados.rados.0.smithi029.stdout:663: done (1 left) 2024-04-12T11:17:08.648 INFO:tasks.rados.rados.0.smithi029.stdout:665: done (0 left) 2024-04-12T11:17:08.648 INFO:tasks.rados.rados.0.smithi029.stdout:666: rollback oid 37 current snap is 88 2024-04-12T11:17:08.648 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 66 2024-04-12T11:17:08.649 INFO:tasks.rados.rados.0.smithi029.stdout:667: rollback oid 21 current snap is 88 2024-04-12T11:17:08.649 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 21 to 66 2024-04-12T11:17:08.649 INFO:tasks.rados.rados.0.smithi029.stdout:668: write oid 40 current snap is 88 2024-04-12T11:17:08.649 INFO:tasks.rados.rados.0.smithi029.stdout:668: seq_num 282 ranges {739574=725279,1968628=721093} 2024-04-12T11:17:08.656 INFO:tasks.rados.rados.0.smithi029.stdout:668: writing smithi02979277-40 from 739574 to 1464853 tid 1 2024-04-12T11:17:08.663 INFO:tasks.rados.rados.0.smithi029.stdout:668: writing smithi02979277-40 from 1968628 to 2689721 tid 2 2024-04-12T11:17:08.663 INFO:tasks.rados.rados.0.smithi029.stdout:669: write oid 42 current snap is 88 2024-04-12T11:17:08.663 INFO:tasks.rados.rados.0.smithi029.stdout:669: seq_num 283 ranges {756381=400528,1910757=464751} 2024-04-12T11:17:08.669 INFO:tasks.rados.rados.0.smithi029.stdout:669: writing smithi02979277-42 from 756381 to 1156909 tid 1 2024-04-12T11:17:08.676 INFO:tasks.rados.rados.0.smithi029.stdout:669: writing smithi02979277-42 from 1910757 to 2375508 tid 2 2024-04-12T11:17:08.676 INFO:tasks.rados.rados.0.smithi029.stdout:670: snap_create 2024-04-12T11:17:08.676 INFO:tasks.rados.rados.0.smithi029.stdout:666: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:17:08.676 INFO:tasks.rados.rados.0.smithi029.stdout:666: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:666: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 266 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:667: finishing rollback tid 0 to smithi02979277-21 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:667: finishing rollback tid 1 to smithi02979277-21 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:667: finishing rollback tid 2 to smithi02979277-21 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 334 (ObjNum 223 snap 64 seq_num 223) dirty exists 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:668: finishing write tid 1 to smithi02979277-40 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:668: finishing write tid 2 to smithi02979277-40 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:668: finishing write tid 3 to smithi02979277-40 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:668: finishing write tid 4 to smithi02979277-40 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:668: oid 40 updating version 0 to 401 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:668: oid 40 version 401 is already newer than 399 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:668: oid 40 version 401 is already newer than 400 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 401 (ObjNum 282 snap 88 seq_num 282) dirty exists 2024-04-12T11:17:08.677 INFO:tasks.rados.rados.0.smithi029.stdout:668: left oid 40 (ObjNum 282 snap 88 seq_num 282) 2024-04-12T11:17:08.678 INFO:tasks.rados.rados.0.smithi029.stdout:669: finishing write tid 1 to smithi02979277-42 2024-04-12T11:17:08.678 INFO:tasks.rados.rados.0.smithi029.stdout:669: finishing write tid 2 to smithi02979277-42 2024-04-12T11:17:08.679 INFO:tasks.rados.rados.0.smithi029.stdout:669: finishing write tid 3 to smithi02979277-42 2024-04-12T11:17:08.679 INFO:tasks.rados.rados.0.smithi029.stdout:669: finishing write tid 4 to smithi02979277-42 2024-04-12T11:17:08.679 INFO:tasks.rados.rados.0.smithi029.stdout:669: oid 42 updating version 0 to 85 2024-04-12T11:17:08.679 INFO:tasks.rados.rados.0.smithi029.stdout:669: oid 42 version 85 is already newer than 84 2024-04-12T11:17:08.679 INFO:tasks.rados.rados.0.smithi029.stdout:669: oid 42 updating version 85 to 86 2024-04-12T11:17:08.679 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 86 (ObjNum 283 snap 88 seq_num 283) dirty exists 2024-04-12T11:17:08.679 INFO:tasks.rados.rados.0.smithi029.stdout:669: left oid 42 (ObjNum 283 snap 88 seq_num 283) 2024-04-12T11:17:08.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[39999]: osdmap e237: 8 total, 8 up, 8 in 2024-04-12T11:17:08.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[39999]: Upgrade: Updating mon.a 2024-04-12T11:17:08.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:08.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:17:08.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:17:08.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:08.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[39999]: Deploying daemon mon.a on smithi029 2024-04-12T11:17:08.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/3978152146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:08.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[39999]: pgmap v51: 89 pgs: 89 active+clean; 239 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 613 KiB/s wr, 21 op/s 2024-04-12T11:17:08.884 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[29349]: osdmap e237: 8 total, 8 up, 8 in 2024-04-12T11:17:08.884 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[29349]: Upgrade: Updating mon.a 2024-04-12T11:17:08.884 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:08.884 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:17:08.884 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:17:08.884 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[29349]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:08.884 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[29349]: Deploying daemon mon.a on smithi029 2024-04-12T11:17:08.884 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[29349]: from='client.? 172.21.15.29:0/3978152146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:08.884 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:08 smithi029 ceph-mon[29349]: pgmap v51: 89 pgs: 89 active+clean; 239 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 613 KiB/s wr, 21 op/s 2024-04-12T11:17:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:08 smithi087 ceph-mon[32917]: osdmap e237: 8 total, 8 up, 8 in 2024-04-12T11:17:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:08 smithi087 ceph-mon[32917]: Upgrade: Updating mon.a 2024-04-12T11:17:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:08 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' 2024-04-12T11:17:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:08 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:17:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:08 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:17:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:08 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:08 smithi087 ceph-mon[32917]: Deploying daemon mon.a on smithi029 2024-04-12T11:17:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:08 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/3978152146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:08 smithi087 ceph-mon[32917]: pgmap v51: 89 pgs: 89 active+clean; 239 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 613 KiB/s wr, 21 op/s 2024-04-12T11:17:09.582 INFO:tasks.rados.rados.0.smithi029.stdout:666: done (4 left) 2024-04-12T11:17:09.582 INFO:tasks.rados.rados.0.smithi029.stdout:667: done (3 left) 2024-04-12T11:17:09.582 INFO:tasks.rados.rados.0.smithi029.stdout:668: done (2 left) 2024-04-12T11:17:09.582 INFO:tasks.rados.rados.0.smithi029.stdout:669: done (1 left) 2024-04-12T11:17:09.582 INFO:tasks.rados.rados.0.smithi029.stdout:670: done (0 left) 2024-04-12T11:17:09.582 INFO:tasks.rados.rados.0.smithi029.stdout:671: rollback oid 28 current snap is 89 2024-04-12T11:17:09.582 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 28 to 84 2024-04-12T11:17:09.582 INFO:tasks.rados.rados.0.smithi029.stdout:672: snap_remove snap 79 2024-04-12T11:17:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:09 smithi087 ceph-mon[32917]: osdmap e238: 8 total, 8 up, 8 in 2024-04-12T11:17:10.007 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:09 smithi029 ceph-mon[39999]: osdmap e238: 8 total, 8 up, 8 in 2024-04-12T11:17:10.008 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:09 smithi029 ceph-mon[29349]: osdmap e238: 8 total, 8 up, 8 in 2024-04-12T11:17:10.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:10 smithi029 systemd[1]: Stopping Ceph mon.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:17:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:10 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[29345]: 2024-04-12T11:17:10.582+0000 7f1ecbbc0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:17:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:10 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[29345]: 2024-04-12T11:17:10.582+0000 7f1ecbbc0700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-12T11:17:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:10 smithi029 podman[93475]: 2024-04-12 11:17:10.817148947 +0000 UTC m=+0.355599774 container died 1cf52b0b92d9eba320e090e01a740447595c03c3cdef85d9effe11eb945eeb7b (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=reef-d540eba, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T11:17:11.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:10 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:17:10] ENGINE Bus STOPPING 2024-04-12T11:17:11.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:10 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:17:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T11:17:11.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:10 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:17:10] ENGINE Bus STOPPED 2024-04-12T11:17:11.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:10 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:17:10] ENGINE Bus STARTING 2024-04-12T11:17:11.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:10 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:17:10] ENGINE Serving on http://:::9283 2024-04-12T11:17:11.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:10 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:17:10] ENGINE Bus STARTED 2024-04-12T11:17:11.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:11 smithi029 podman[93475]: 2024-04-12 11:17:11.301584287 +0000 UTC m=+0.840035143 container cleanup 1cf52b0b92d9eba320e090e01a740447595c03c3cdef85d9effe11eb945eeb7b (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=reef-d540eba) 2024-04-12T11:17:11.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:11 smithi029 bash[93475]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a 2024-04-12T11:17:11.976 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:11 smithi029 podman[93502]: 2024-04-12 11:17:11.606636389 +0000 UTC m=+0.787913061 container remove 1cf52b0b92d9eba320e090e01a740447595c03c3cdef85d9effe11eb945eeb7b (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=reef-d540eba, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True) 2024-04-12T11:17:11.976 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:11 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-12T11:17:11.976 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:11 smithi029 systemd[1]: Stopped Ceph mon.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:17:11.976 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:11 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.a.service: Consumed 13.460s CPU time. 2024-04-12T11:17:12.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:11 smithi029 systemd[1]: Starting Ceph mon.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:17:12.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 podman[93631]: 2024-04-12 11:17:12.183585902 +0000 UTC m=+0.110713597 container create d1506def0998659ccd935211736c68234566f534f7e49f954fde3a94f4c75644 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:17:12.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 podman[93631]: 2024-04-12 11:17:12.090687365 +0000 UTC m=+0.017815072 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:17:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 podman[93631]: 2024-04-12 11:17:12.440254541 +0000 UTC m=+0.367382234 container init d1506def0998659ccd935211736c68234566f534f7e49f954fde3a94f4c75644 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:17:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 podman[93631]: 2024-04-12 11:17:12.443227803 +0000 UTC m=+0.370355499 container start d1506def0998659ccd935211736c68234566f534f7e49f954fde3a94f4c75644 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-12T11:17:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T11:17:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev), process ceph-mon, pid 2 2024-04-12T11:17:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: pidfile_write: ignore empty --pid-file 2024-04-12T11:17:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: load: jerasure load: lrc 2024-04-12T11:17:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: RocksDB version: 7.9.2 2024-04-12T11:17:12.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Git sha 0 2024-04-12T11:17:12.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Compile date 2024-04-11 21:23:14 2024-04-12T11:17:12.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: DB SUMMARY 2024-04-12T11:17:12.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: DB Session ID: VJE585GURAIYE14GSZ5J 2024-04-12T11:17:12.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: CURRENT file: CURRENT 2024-04-12T11:17:12.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T11:17:12.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1267 Bytes 2024-04-12T11:17:12.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000029.sst 2024-04-12T11:17:12.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000027.log size: 2505366 ; 2024-04-12T11:17:12.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.error_if_exists: 0 2024-04-12T11:17:12.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.create_if_missing: 0 2024-04-12T11:17:12.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.paranoid_checks: 1 2024-04-12T11:17:12.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T11:17:12.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T11:17:12.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T11:17:12.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.env: 0x55ac40158cc0 2024-04-12T11:17:12.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.info_log: 0x55ac42529bc0 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.statistics: (nil) 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.use_fsync: 0 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_log_file_size: 0 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.allow_fallocate: 1 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T11:17:12.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.use_direct_reads: 0 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.db_log_dir: 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.wal_dir: 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T11:17:12.773 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.write_buffer_manager: 0x55ac424fa1e0 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.unordered_write: 0 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T11:17:12.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T11:17:12.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T11:17:12.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T11:17:12.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.row_cache: None 2024-04-12T11:17:12.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.wal_filter: None 2024-04-12T11:17:12.776 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T11:17:12.776 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T11:17:12.776 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.two_write_queues: 0 2024-04-12T11:17:12.776 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T11:17:12.776 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.wal_compression: 0 2024-04-12T11:17:12.776 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.atomic_flush: 0 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.log_readahead_size: 0 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_background_jobs: 2 2024-04-12T11:17:12.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_background_compactions: -1 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_subcompactions: 1 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_open_files: -1 2024-04-12T11:17:12.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_background_flushes: -1 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Compression algorithms supported: 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: kZSTD supported: 0 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: kXpressCompression supported: 0 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: kZlibCompression supported: 1 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: kSnappyCompression supported: 1 2024-04-12T11:17:12.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: kLZ4Compression supported: 1 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: kBZip2Compression supported: 0 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.merge_operator: 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_filter: None 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T11:17:12.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ac42529da0) 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks: 1 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_top_level_index_and_filter: 1 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_type: 0 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_index_type: 0 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_shortening: 1 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: checksum: 4 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: no_block_cache: 0 2024-04-12T11:17:12.781 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache: 0x55ac4252d090 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_name: BinnedLRUCache 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_options: 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: capacity : 536870912 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: num_shard_bits : 4 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: strict_capacity_limit : 0 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: high_pri_pool_ratio: 0.000 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_compressed: (nil) 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: persistent_cache: (nil) 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size: 4096 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size_deviation: 10 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_restart_interval: 16 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_block_restart_interval: 1 2024-04-12T11:17:12.782 INFO:journalctl@ceph.mon.a.smithi029.stdout: metadata_block_size: 4096 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: partition_filters: 0 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: use_delta_encoding: 1 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: filter_policy: bloomfilter 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: whole_key_filtering: 1 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: verify_compression: 0 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: read_amp_bytes_per_bit: 0 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: format_version: 5 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_index_compression: 1 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_align: 0 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_auto_readahead_size: 262144 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: prepopulate_block_cache: 0 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: initial_auto_readahead_size: 8192 2024-04-12T11:17:12.783 INFO:journalctl@ceph.mon.a.smithi029.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compression: NoCompression 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.num_levels: 7 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T11:17:12.784 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T11:17:12.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T11:17:12.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T11:17:12.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T11:17:12.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.inplace_update_support: 0 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.bloom_locality: 0 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.max_successive_merges: 0 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T11:17:12.789 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.ttl: 2592000 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.enable_blob_files: false 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.min_blob_size: 0 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T11:17:12.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 31, last_sequence is 12743, log_number is 27,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 27 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 27 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f2a020de-3b3d-46d1-b09f-6540083d5ac9 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920632472066, "job": 1, "event": "recovery_started", "wal_files": [27]} 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #27 mode 2 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920632479730, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 32, "file_size": 2364421, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12744, "largest_seqno": 13902, "table_properties": {"data_size": 2357835, "index_size": 4197, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 13304, "raw_average_key_size": 25, "raw_value_size": 2345770, "raw_average_value_size": 4417, "num_data_blocks": 186, "num_entries": 531, "num_filter_entries": 531, "num_deletions": 24, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712920632, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f2a020de-3b3d-46d1-b09f-6540083d5ac9", "db_session_id": "VJE585GURAIYE14GSZ5J", "orig_file_number": 32, "seqno_to_time_mapping": "N/A"}} 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920632479836, "job": 1, "event": "recovery_finished"} 2024-04-12T11:17:12.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/version_set.cc:5047] Creating manifest 34 2024-04-12T11:17:12.792 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T11:17:12.792 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000027.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T11:17:12.792 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ac4261c000 2024-04-12T11:17:12.792 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: DB pointer 0x55ac42606000 2024-04-12T11:17:12.792 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T11:17:12.792 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T11:17:12.792 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** DB Stats ** 2024-04-12T11:17:12.792 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T11:17:12.792 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T11:17:12.792 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: L0 1/0 2.25 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 555.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: L6 1/0 7.49 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: Sum 2/0 9.74 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 555.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 555.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T11:17:12.793 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 555.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T11:17:12.794 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative compaction: 0.00 GB write, 225.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval compaction: 0.00 GB write, 225.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout: Block cache BinnedLRUCache@0x55ac4252d090#2 capacity: 512.00 MB usage: 5.98 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: starting mon.a rank 0 at public addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] at bind addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: mon.a@-1(???) e3 preinit fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: mon.a@-1(???).mds e9 new map 2024-04-12T11:17:12.795 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: mon.a@-1(???).mds e9 print_map 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: e9 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: legacy client fscid: 1 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: Filesystem 'foo' (1) 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: fs_name foo 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: epoch 8 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: created 2024-04-12T11:12:57.763670+0000 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: modified 2024-04-12T11:13:16.631701+0000 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: tableserver 0 2024-04-12T11:17:12.796 INFO:journalctl@ceph.mon.a.smithi029.stdout: root 0 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: session_timeout 60 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: session_autoclose 300 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_file_size 1099511627776 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_xattr_size 65536 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: required_client_features {} 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: last_failure 0 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: last_failure_osd_epoch 0 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_mds 1 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: in 0 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: up {0=14661} 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: failed 2024-04-12T11:17:12.797 INFO:journalctl@ceph.mon.a.smithi029.stdout: damaged 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: stopped 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_pools [4] 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: metadata_pool 3 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: inline_data disabled 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: balancer 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: bal_rank_mask -1 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: standby_count_wanted 1 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: qdb_cluster leader: 0 members: 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: [mds.foo.smithi029.scbuux{0:14661} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:17:12.798 INFO:journalctl@ceph.mon.a.smithi029.stdout: Standby daemons: 2024-04-12T11:17:12.799 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-04-12T11:17:12.799 INFO:journalctl@ceph.mon.a.smithi029.stdout: [mds.foo.smithi087.mbflzu{-1:24502} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.87:6832/2304329976,v1:172.21.15.87:6833/2304329976] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T11:17:12.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: mon.a@-1(???).osd e239 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-12T11:17:12.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: mon.a@-1(???).osd e239 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T11:17:12.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: mon.a@-1(???).osd e239 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T11:17:12.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: mon.a@-1(???).osd e239 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T11:17:12.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 ceph-mon[93650]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-04-12T11:17:12.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 bash[93631]: d1506def0998659ccd935211736c68234566f534f7e49f954fde3a94f4c75644 2024-04-12T11:17:12.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:12 smithi029 systemd[1]: Started Ceph mon.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:17:13.898 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:17:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:17:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:17:13.903 INFO:tasks.rados.rados.0.smithi029.stdout:672: done (1 left) 2024-04-12T11:17:13.903 INFO:tasks.rados.rados.0.smithi029.stdout:673: delete oid 29 current snap is 89 2024-04-12T11:17:13.903 INFO:tasks.rados.rados.0.smithi029.stdout:671: finishing rollback tid 1 to smithi02979277-28 2024-04-12T11:17:13.903 INFO:tasks.rados.rados.0.smithi029.stdout:671: finishing rollback tid 2 to smithi02979277-28 2024-04-12T11:17:13.903 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 264 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-04-12T11:17:13.910 INFO:tasks.rados.rados.0.smithi029.stdout:671: done (1 left) 2024-04-12T11:17:13.911 INFO:tasks.rados.rados.0.smithi029.stdout:673: done (0 left) 2024-04-12T11:17:13.911 INFO:tasks.rados.rados.0.smithi029.stdout:674: read oid 5 snap -1 2024-04-12T11:17:13.911 INFO:tasks.rados.rados.0.smithi029.stdout:674: expect (ObjNum 218 snap 63 seq_num 218) 2024-04-12T11:17:13.911 INFO:tasks.rados.rados.0.smithi029.stdout:675: snap_remove snap 81 2024-04-12T11:17:14.091 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:13 smithi029 ceph-mon[39999]: osdmap e239: 8 total, 8 up, 8 in 2024-04-12T11:17:14.092 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:13 smithi029 ceph-mon[39999]: pgmap v54: 89 pgs: 89 active+clean; 239 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 613 KiB/s wr, 21 op/s 2024-04-12T11:17:14.092 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:13 smithi029 ceph-mon[93650]: osdmap e239: 8 total, 8 up, 8 in 2024-04-12T11:17:14.092 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:13 smithi029 ceph-mon[93650]: pgmap v54: 89 pgs: 89 active+clean; 239 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 613 KiB/s wr, 21 op/s 2024-04-12T11:17:14.092 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ignoring --setuser ceph since I am not root 2024-04-12T11:17:14.092 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ignoring --setgroup ceph since I am not root 2024-04-12T11:17:14.092 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:13 smithi029 ceph-mgr[84550]: -- 172.21.15.29:0/896968067 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55de4130c5a0 con 0x55de412e0800 2024-04-12T11:17:14.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:13 smithi087 ceph-mon[32917]: osdmap e239: 8 total, 8 up, 8 in 2024-04-12T11:17:14.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:13 smithi087 ceph-mon[32917]: pgmap v54: 89 pgs: 89 active+clean; 239 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 613 KiB/s wr, 21 op/s 2024-04-12T11:17:14.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: ignoring --setuser ceph since I am not root 2024-04-12T11:17:14.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: ignoring --setgroup ceph since I am not root 2024-04-12T11:17:14.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:13 smithi087 ceph-mgr[78574]: -- 172.21.15.87:0/3589054603 <== mon.1 v2:172.21.15.87:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d6b8ab85a0 con 0x55d6b8a8d000 2024-04-12T11:17:14.897 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:14.480+0000 7fa7435fe200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:17:14.898 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:14.573+0000 7fa7435fe200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:17:14.898 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:14.535+0000 7f4c6959f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-12T11:17:14.898 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:14.633+0000 7f4c6959f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-12T11:17:14.921 INFO:tasks.rados.rados.0.smithi029.stdout:675: done (1 left) 2024-04-12T11:17:14.921 INFO:tasks.rados.rados.0.smithi029.stdout:676: write oid 24 current snap is 89 2024-04-12T11:17:14.921 INFO:tasks.rados.rados.0.smithi029.stdout:676: seq_num 285 ranges {789995=551026,2027721=468420} 2024-04-12T11:17:14.929 INFO:tasks.rados.rados.0.smithi029.stdout:676: writing smithi02979277-24 from 789995 to 1341021 tid 1 2024-04-12T11:17:14.936 INFO:tasks.rados.rados.0.smithi029.stdout:676: writing smithi02979277-24 from 2027721 to 2496141 tid 2 2024-04-12T11:17:14.936 INFO:tasks.rados.rados.0.smithi029.stdout:677: snap_remove snap 73 2024-04-12T11:17:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:17:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:17:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:17:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: from='client.15297 172.21.15.29:0/3573662231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: pgmap v55: 89 pgs: 89 active+clean; 241 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 667 KiB/s wr, 16 op/s 2024-04-12T11:17:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: mon.a calling monitor election 2024-04-12T11:17:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2673034903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:15.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:17:15.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: monmap epoch 3 2024-04-12T11:17:15.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:17:15.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: last_changed 2024-04-12T11:07:50.917802+0000 2024-04-12T11:17:15.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: created 2024-04-12T11:06:03.531247+0000 2024-04-12T11:17:15.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: min_mon_release 18 (reef) 2024-04-12T11:17:15.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: election_strategy: 1 2024-04-12T11:17:15.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-04-12T11:17:15.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: 1: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.b 2024-04-12T11:17:15.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-04-12T11:17:15.219 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:17:15.219 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: osdmap e240: 8 total, 8 up, 8 in 2024-04-12T11:17:15.219 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: mgrmap e31: y(active, since 37s), standbys: x 2024-04-12T11:17:15.219 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: overall HEALTH_OK 2024-04-12T11:17:15.219 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: mgrmap e32: y(active, since 37s), standbys: x 2024-04-12T11:17:15.219 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:14 smithi087 ceph-mon[32917]: from='mgr.25037 ' entity='mgr.y' 2024-04-12T11:17:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:17:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:17:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: from='client.15297 172.21.15.29:0/3573662231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: pgmap v55: 89 pgs: 89 active+clean; 241 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 667 KiB/s wr, 16 op/s 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: mon.a calling monitor election 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2673034903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: monmap epoch 3 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: last_changed 2024-04-12T11:07:50.917802+0000 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: created 2024-04-12T11:06:03.531247+0000 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: min_mon_release 18 (reef) 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: election_strategy: 1 2024-04-12T11:17:15.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-04-12T11:17:15.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: 1: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.b 2024-04-12T11:17:15.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-04-12T11:17:15.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:17:15.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: osdmap e240: 8 total, 8 up, 8 in 2024-04-12T11:17:15.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: mgrmap e31: y(active, since 37s), standbys: x 2024-04-12T11:17:15.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: overall HEALTH_OK 2024-04-12T11:17:15.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: mgrmap e32: y(active, since 37s), standbys: x 2024-04-12T11:17:15.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[93650]: from='mgr.25037 ' entity='mgr.y' 2024-04-12T11:17:15.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: from='mgr.25037 172.21.15.29:0/3429870179' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: from='client.15297 172.21.15.29:0/3573662231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: pgmap v55: 89 pgs: 89 active+clean; 241 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 667 KiB/s wr, 16 op/s 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: mon.a calling monitor election 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2673034903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: monmap epoch 3 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: last_changed 2024-04-12T11:07:50.917802+0000 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: created 2024-04-12T11:06:03.531247+0000 2024-04-12T11:17:15.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: min_mon_release 18 (reef) 2024-04-12T11:17:15.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: election_strategy: 1 2024-04-12T11:17:15.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-04-12T11:17:15.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: 1: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.b 2024-04-12T11:17:15.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-04-12T11:17:15.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:17:15.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: osdmap e240: 8 total, 8 up, 8 in 2024-04-12T11:17:15.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: mgrmap e31: y(active, since 37s), standbys: x 2024-04-12T11:17:15.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: overall HEALTH_OK 2024-04-12T11:17:15.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: mgrmap e32: y(active, since 37s), standbys: x 2024-04-12T11:17:15.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:14 smithi029 ceph-mon[39999]: from='mgr.25037 ' entity='mgr.y' 2024-04-12T11:17:15.548 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:15.219+0000 7fa7435fe200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:17:15.548 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:15.312+0000 7fa7435fe200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:17:15.548 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:15.453+0000 7fa7435fe200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:17:15.548 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:15.548+0000 7fa7435fe200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:17:15.592 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:15.318+0000 7f4c6959f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-12T11:17:15.592 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:15.414+0000 7f4c6959f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-12T11:17:15.592 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:15.559+0000 7f4c6959f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-12T11:17:15.593 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev)": 1 2024-04-12T11:17:15.895 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:15.630+0000 7fa7435fe200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:17:15.922 INFO:tasks.rados.rados.0.smithi029.stdout:677: done (2 left) 2024-04-12T11:17:15.922 INFO:tasks.rados.rados.0.smithi029.stdout:678: write oid 30 current snap is 89 2024-04-12T11:17:15.922 INFO:tasks.rados.rados.0.smithi029.stdout:678: seq_num 286 ranges {406802=626275,1686203=728431} 2024-04-12T11:17:15.928 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:15.662+0000 7f4c6959f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-12T11:17:15.928 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:15 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:15.747+0000 7f4c6959f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-12T11:17:15.929 INFO:tasks.rados.rados.0.smithi029.stdout:678: writing smithi02979277-30 from 406802 to 1033077 tid 1 2024-04-12T11:17:15.937 INFO:tasks.rados.rados.0.smithi029.stdout:678: writing smithi02979277-30 from 1686203 to 2414634 tid 2 2024-04-12T11:17:15.938 INFO:tasks.rados.rados.0.smithi029.stdout:679: read oid 13 snap 78 2024-04-12T11:17:15.938 INFO:tasks.rados.rados.0.smithi029.stdout:679: expect (ObjNum 168 snap 44 seq_num 168) 2024-04-12T11:17:15.938 INFO:tasks.rados.rados.0.smithi029.stdout:680: read oid 19 snap 85 2024-04-12T11:17:15.938 INFO:tasks.rados.rados.0.smithi029.stdout:680: expect deleted 2024-04-12T11:17:15.938 INFO:tasks.rados.rados.0.smithi029.stdout:681: write oid 22 current snap is 89 2024-04-12T11:17:15.938 INFO:tasks.rados.rados.0.smithi029.stdout:674: expect (ObjNum 218 snap 63 seq_num 218) 2024-04-12T11:17:16.134 INFO:tasks.rados.rados.0.smithi029.stdout:676: finishing write tid 1 to smithi02979277-24 2024-04-12T11:17:16.134 INFO:tasks.rados.rados.0.smithi029.stdout:676: finishing write tid 2 to smithi02979277-24 2024-04-12T11:17:16.134 INFO:tasks.rados.rados.0.smithi029.stdout:676: finishing write tid 3 to smithi02979277-24 2024-04-12T11:17:16.134 INFO:tasks.rados.rados.0.smithi029.stdout:676: finishing write tid 4 to smithi02979277-24 2024-04-12T11:17:16.135 INFO:tasks.rados.rados.0.smithi029.stdout:676: oid 24 updating version 0 to 408 2024-04-12T11:17:16.135 INFO:tasks.rados.rados.0.smithi029.stdout:676: oid 24 updating version 408 to 415 2024-04-12T11:17:16.135 INFO:tasks.rados.rados.0.smithi029.stdout:676: oid 24 updating version 415 to 416 2024-04-12T11:17:16.135 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 416 (ObjNum 285 snap 89 seq_num 285) dirty exists 2024-04-12T11:17:16.135 INFO:tasks.rados.rados.0.smithi029.stdout:676: left oid 24 (ObjNum 285 snap 89 seq_num 285) 2024-04-12T11:17:16.135 INFO:tasks.rados.rados.0.smithi029.stdout:678: finishing write tid 1 to smithi02979277-30 2024-04-12T11:17:16.135 INFO:tasks.rados.rados.0.smithi029.stdout:678: finishing write tid 2 to smithi02979277-30 2024-04-12T11:17:16.135 INFO:tasks.rados.rados.0.smithi029.stdout:678: finishing write tid 3 to smithi02979277-30 2024-04-12T11:17:16.135 INFO:tasks.rados.rados.0.smithi029.stdout:681: seq_num 287 ranges {423609=701524,1628332=472089,2648987=174622} 2024-04-12T11:17:16.140 INFO:tasks.rados.rados.0.smithi029.stdout:681: writing smithi02979277-22 from 423609 to 1125133 tid 1 2024-04-12T11:17:16.145 INFO:tasks.rados.rados.0.smithi029.stdout:681: writing smithi02979277-22 from 1628332 to 2100421 tid 2 2024-04-12T11:17:16.150 INFO:tasks.rados.rados.0.smithi029.stdout:681: writing smithi02979277-22 from 2648987 to 2823609 tid 3 2024-04-12T11:17:16.150 INFO:tasks.rados.rados.0.smithi029.stdout:674: done (5 left) 2024-04-12T11:17:16.150 INFO:tasks.rados.rados.0.smithi029.stdout:676: done (4 left) 2024-04-12T11:17:16.150 INFO:tasks.rados.rados.0.smithi029.stdout:680: done (3 left) 2024-04-12T11:17:16.150 INFO:tasks.rados.rados.0.smithi029.stdout:682: read oid 40 snap 88 2024-04-12T11:17:16.150 INFO:tasks.rados.rados.0.smithi029.stdout:682: expect (ObjNum 282 snap 88 seq_num 282) 2024-04-12T11:17:16.150 INFO:tasks.rados.rados.0.smithi029.stdout:683: snap_create 2024-04-12T11:17:16.150 INFO:tasks.rados.rados.0.smithi029.stdout:678: finishing write tid 4 to smithi02979277-30 2024-04-12T11:17:16.150 INFO:tasks.rados.rados.0.smithi029.stdout:678: oid 30 updating version 0 to 291 2024-04-12T11:17:16.151 INFO:tasks.rados.rados.0.smithi029.stdout:678: oid 30 updating version 291 to 293 2024-04-12T11:17:16.151 INFO:tasks.rados.rados.0.smithi029.stdout:678: oid 30 version 293 is already newer than 292 2024-04-12T11:17:16.151 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 293 (ObjNum 286 snap 89 seq_num 286) dirty exists 2024-04-12T11:17:16.151 INFO:tasks.rados.rados.0.smithi029.stdout:678: left oid 30 (ObjNum 286 snap 89 seq_num 286) 2024-04-12T11:17:16.151 INFO:tasks.rados.rados.0.smithi029.stdout:679: expect (ObjNum 168 snap 44 seq_num 168) 2024-04-12T11:17:16.196 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:15 smithi087 ceph-mon[32917]: osdmap e241: 8 total, 8 up, 8 in 2024-04-12T11:17:16.196 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:15 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2960565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:16.197 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:15.895+0000 7fa7435fe200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:17:16.197 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:16.014+0000 7fa7435fe200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:17:16.197 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:16.099+0000 7fa7435fe200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:17:16.197 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:16.196+0000 7fa7435fe200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:17:16.235 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:15 smithi029 ceph-mon[93650]: osdmap e241: 8 total, 8 up, 8 in 2024-04-12T11:17:16.235 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:15 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2960565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:16.235 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:15 smithi029 ceph-mon[39999]: osdmap e241: 8 total, 8 up, 8 in 2024-04-12T11:17:16.235 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:15 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/2960565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:17:16.235 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:16 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:16.030+0000 7f4c6959f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-12T11:17:16.235 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:16 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:16.143+0000 7f4c6959f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:681: finishing write tid 1 to smithi02979277-22 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:681: finishing write tid 2 to smithi02979277-22 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:681: finishing write tid 3 to smithi02979277-22 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:681: finishing write tid 4 to smithi02979277-22 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:681: finishing write tid 5 to smithi02979277-22 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:681: oid 22 updating version 0 to 252 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:681: oid 22 version 252 is already newer than 250 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:681: oid 22 version 252 is already newer than 249 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:681: oid 22 version 252 is already newer than 251 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 252 (ObjNum 287 snap 89 seq_num 287) dirty exists 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:681: left oid 22 (ObjNum 287 snap 89 seq_num 287) 2024-04-12T11:17:16.276 INFO:tasks.rados.rados.0.smithi029.stdout:682: expect (ObjNum 282 snap 88 seq_num 282) 2024-04-12T11:17:16.428 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch ps' 2024-04-12T11:17:16.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:16.373+0000 7fa7435fe200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:17:16.495 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:16 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:16.234+0000 7f4c6959f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-12T11:17:16.495 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:16 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:16.332+0000 7f4c6959f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-12T11:17:16.693 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:17:16.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:16 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:16.518+0000 7f4c6959f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-12T11:17:16.947 INFO:tasks.rados.rados.0.smithi029.stdout:678: done (4 left) 2024-04-12T11:17:16.947 INFO:tasks.rados.rados.0.smithi029.stdout:679: done (3 left) 2024-04-12T11:17:16.947 INFO:tasks.rados.rados.0.smithi029.stdout:681: done (2 left) 2024-04-12T11:17:16.947 INFO:tasks.rados.rados.0.smithi029.stdout:682: done (1 left) 2024-04-12T11:17:16.947 INFO:tasks.rados.rados.0.smithi029.stdout:683: done (0 left) 2024-04-12T11:17:16.947 INFO:tasks.rados.rados.0.smithi029.stdout:684: write oid 15 current snap is 90 2024-04-12T11:17:16.948 INFO:tasks.rados.rados.0.smithi029.stdout:684: seq_num 288 ranges {440416=776773,1686814=732100} 2024-04-12T11:17:16.954 INFO:tasks.rados.rados.0.smithi029.stdout:684: writing smithi02979277-15 from 440416 to 1217189 tid 1 2024-04-12T11:17:16.961 INFO:tasks.rados.rados.0.smithi029.stdout:684: writing smithi02979277-15 from 1686814 to 2418914 tid 2 2024-04-12T11:17:16.962 INFO:tasks.rados.rados.0.smithi029.stdout:685: rollback oid 5 current snap is 90 2024-04-12T11:17:16.962 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 5 to 85 2024-04-12T11:17:16.962 INFO:tasks.rados.rados.0.smithi029.stdout:684: finishing write tid 1 to smithi02979277-15 2024-04-12T11:17:16.962 INFO:tasks.rados.rados.0.smithi029.stdout:686: read oid 36 snap -1 2024-04-12T11:17:16.963 INFO:tasks.rados.rados.0.smithi029.stdout:686: expect (ObjNum 247 snap 76 seq_num 247) 2024-04-12T11:17:16.963 INFO:tasks.rados.rados.0.smithi029.stdout:687: snap_remove snap 59 2024-04-12T11:17:17.066 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:16 smithi029 ceph-mon[39999]: osdmap e242: 8 total, 8 up, 8 in 2024-04-12T11:17:17.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:16 smithi029 ceph-mon[93650]: osdmap e242: 8 total, 8 up, 8 in 2024-04-12T11:17:17.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:16 smithi087 ceph-mon[32917]: osdmap e242: 8 total, 8 up, 8 in 2024-04-12T11:17:17.491 INFO:tasks.rados.rados.0.smithi029.stdout:687: done (3 left) 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:688: read oid 2 snap -1 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:688: expect (ObjNum 252 snap 79 seq_num 252) 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:684: finishing write tid 2 to smithi02979277-15 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:684: finishing write tid 3 to smithi02979277-15 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:684: finishing write tid 4 to smithi02979277-15 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:684: oid 15 updating version 0 to 297 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:684: oid 15 version 297 is already newer than 296 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:684: oid 15 version 297 is already newer than 295 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 297 (ObjNum 288 snap 90 seq_num 288) dirty exists 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:684: left oid 15 (ObjNum 288 snap 90 seq_num 288) 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:685: finishing rollback tid 0 to smithi02979277-5 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:685: finishing rollback tid 1 to smithi02979277-5 2024-04-12T11:17:17.492 INFO:tasks.rados.rados.0.smithi029.stdout:685: finishing rollback tid 2 to smithi02979277-5 2024-04-12T11:17:17.493 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 415 (ObjNum 218 snap 63 seq_num 218) dirty exists 2024-04-12T11:17:17.493 INFO:tasks.rados.rados.0.smithi029.stdout:686: expect (ObjNum 247 snap 76 seq_num 247) 2024-04-12T11:17:17.594 INFO:tasks.rados.rados.0.smithi029.stdout:688: expect (ObjNum 252 snap 79 seq_num 252) 2024-04-12T11:17:17.671 INFO:tasks.rados.rados.0.smithi029.stdout:684: done (3 left) 2024-04-12T11:17:17.671 INFO:tasks.rados.rados.0.smithi029.stdout:685: done (2 left) 2024-04-12T11:17:17.671 INFO:tasks.rados.rados.0.smithi029.stdout:686: done (1 left) 2024-04-12T11:17:17.671 INFO:tasks.rados.rados.0.smithi029.stdout:688: done (0 left) 2024-04-12T11:17:17.672 INFO:tasks.rados.rados.0.smithi029.stdout:689: snap_create 2024-04-12T11:17:17.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:17.332+0000 7fa7435fe200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:17:17.716 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:17.460+0000 7fa7435fe200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:17:17.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:17 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:17.558+0000 7f4c6959f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-12T11:17:17.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:17 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:17.695+0000 7f4c6959f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-12T11:17:18.095 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:17 smithi087 ceph-mon[32917]: osdmap e243: 8 total, 8 up, 8 in 2024-04-12T11:17:18.096 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:17 smithi087 ceph-mon[32917]: osdmap e244: 8 total, 8 up, 8 in 2024-04-12T11:17:18.096 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:17.804+0000 7fa7435fe200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:17:18.096 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:17.895+0000 7fa7435fe200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:17:18.201 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:17 smithi029 ceph-mon[39999]: osdmap e243: 8 total, 8 up, 8 in 2024-04-12T11:17:18.201 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:17 smithi029 ceph-mon[39999]: osdmap e244: 8 total, 8 up, 8 in 2024-04-12T11:17:18.202 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:17 smithi029 ceph-mon[93650]: osdmap e243: 8 total, 8 up, 8 in 2024-04-12T11:17:18.202 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:17 smithi029 ceph-mon[93650]: osdmap e244: 8 total, 8 up, 8 in 2024-04-12T11:17:18.202 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:18 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:18.043+0000 7f4c6959f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-12T11:17:18.202 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:18 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:18.138+0000 7f4c6959f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-12T11:17:18.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:18 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:18.095+0000 7fa7435fe200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:17:18.466 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:18 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:18.289+0000 7fa7435fe200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:17:18.471 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:18 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:18.347+0000 7f4c6959f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:17:18.492 INFO:tasks.rados.rados.0.smithi029.stdout:689: done (0 left) 2024-04-12T11:17:18.492 INFO:tasks.rados.rados.0.smithi029.stdout:690: write oid 27 current snap is 91 2024-04-12T11:17:18.492 INFO:tasks.rados.rados.0.smithi029.stdout:690: seq_num 289 ranges {457223=568375,1461649=475758,2420186=437037} 2024-04-12T11:17:18.499 INFO:tasks.rados.rados.0.smithi029.stdout:690: writing smithi02979277-27 from 457223 to 1025598 tid 1 2024-04-12T11:17:18.505 INFO:tasks.rados.rados.0.smithi029.stdout:690: writing smithi02979277-27 from 1461649 to 1937407 tid 2 2024-04-12T11:17:18.512 INFO:tasks.rados.rados.0.smithi029.stdout:690: writing smithi02979277-27 from 2420186 to 2857223 tid 3 2024-04-12T11:17:18.512 INFO:tasks.rados.rados.0.smithi029.stdout:691: read oid 16 snap -1 2024-04-12T11:17:18.512 INFO:tasks.rados.rados.0.smithi029.stdout:691: expect (ObjNum 240 snap 75 seq_num 240) 2024-04-12T11:17:18.512 INFO:tasks.rados.rados.0.smithi029.stdout:692: snap_remove snap 66 2024-04-12T11:17:18.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:18 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:18.569+0000 7f4c6959f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-12T11:17:18.843 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:18 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:18.583+0000 7fa7435fe200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:17:19.133 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:18 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:18.870+0000 7f4c6959f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-12T11:17:19.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:18 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:18.843+0000 7fa7435fe200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:17:19.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:19 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:19.081+0000 7fa7435fe200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:17:19.385 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:19 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:19.132+0000 7f4c6959f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:690: finishing write tid 1 to smithi02979277-27 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:690: finishing write tid 2 to smithi02979277-27 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:690: finishing write tid 3 to smithi02979277-27 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:690: finishing write tid 4 to smithi02979277-27 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:690: finishing write tid 5 to smithi02979277-27 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:690: oid 27 updating version 0 to 306 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:690: oid 27 version 306 is already newer than 305 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:690: oid 27 version 306 is already newer than 303 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:690: oid 27 version 306 is already newer than 304 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 306 (ObjNum 289 snap 91 seq_num 289) dirty exists 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:690: left oid 27 (ObjNum 289 snap 91 seq_num 289) 2024-04-12T11:17:19.498 INFO:tasks.rados.rados.0.smithi029.stdout:691: expect (ObjNum 240 snap 75 seq_num 240) 2024-04-12T11:17:19.569 INFO:tasks.rados.rados.0.smithi029.stdout:690: done (2 left) 2024-04-12T11:17:19.570 INFO:tasks.rados.rados.0.smithi029.stdout:691: done (1 left) 2024-04-12T11:17:19.570 INFO:tasks.rados.rados.0.smithi029.stdout:692: done (0 left) 2024-04-12T11:17:19.570 INFO:tasks.rados.rados.0.smithi029.stdout:693: rollback oid 42 current snap is 91 2024-04-12T11:17:19.570 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 86 2024-04-12T11:17:19.570 INFO:tasks.rados.rados.0.smithi029.stdout:694: snap_create 2024-04-12T11:17:19.572 INFO:tasks.rados.rados.0.smithi029.stdout:693: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:17:19.572 INFO:tasks.rados.rados.0.smithi029.stdout:693: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:17:19.572 INFO:tasks.rados.rados.0.smithi029.stdout:693: finishing rollback tid 2 to smithi02979277-42 2024-04-12T11:17:19.572 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 100 (ObjNum 237 snap 73 seq_num 237) dirty exists 2024-04-12T11:17:19.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:19 smithi029 ceph-mon[93650]: osdmap e245: 8 total, 8 up, 8 in 2024-04-12T11:17:19.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:19 smithi029 ceph-mon[39999]: osdmap e245: 8 total, 8 up, 8 in 2024-04-12T11:17:19.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:19 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:19.384+0000 7f4c6959f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-12T11:17:19.900 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:19 smithi087 ceph-mon[32917]: osdmap e245: 8 total, 8 up, 8 in 2024-04-12T11:17:20.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:19 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:19.900+0000 7fa7435fe200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:17:20.487 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:20.484+0000 7fa7435fe200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:17:20.495 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:20 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:20.235+0000 7f4c6959f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-12T11:17:20.503 INFO:tasks.rados.rados.0.smithi029.stdout:693: done (1 left) 2024-04-12T11:17:20.503 INFO:tasks.rados.rados.0.smithi029.stdout:694: done (0 left) 2024-04-12T11:17:20.503 INFO:tasks.rados.rados.0.smithi029.stdout:695: read oid 17 snap 91 2024-04-12T11:17:20.503 INFO:tasks.rados.rados.0.smithi029.stdout:695: expect (ObjNum 253 snap 79 seq_num 253) 2024-04-12T11:17:20.503 INFO:tasks.rados.rados.0.smithi029.stdout:696: rollback oid 3 current snap is 92 2024-04-12T11:17:20.503 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 82 2024-04-12T11:17:20.503 INFO:tasks.rados.rados.0.smithi029.stdout:697: rollback oid 39 current snap is 92 2024-04-12T11:17:20.503 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 80 2024-04-12T11:17:20.504 INFO:tasks.rados.rados.0.smithi029.stdout:698: write oid 37 current snap is 92 2024-04-12T11:17:20.504 INFO:tasks.rados.rados.0.smithi029.stdout:698: seq_num 290 ranges {474030=643624,1520131=735769} 2024-04-12T11:17:20.509 INFO:tasks.rados.rados.0.smithi029.stdout:698: writing smithi02979277-37 from 474030 to 1117654 tid 1 2024-04-12T11:17:20.516 INFO:tasks.rados.rados.0.smithi029.stdout:698: writing smithi02979277-37 from 1520131 to 2255900 tid 2 2024-04-12T11:17:20.516 INFO:tasks.rados.rados.0.smithi029.stdout:699: snap_create 2024-04-12T11:17:20.516 INFO:tasks.rados.rados.0.smithi029.stdout:697: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:17:20.517 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 0 (ObjNum 49 snap 0 seq_num 1800751040) dirty dne 2024-04-12T11:17:20.517 INFO:tasks.rados.rados.0.smithi029.stdout:696: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:17:20.517 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 263 (ObjNum 129 snap 0 seq_num 862129041) dirty dne 2024-04-12T11:17:20.517 INFO:tasks.rados.rados.0.smithi029.stdout:698: finishing write tid 1 to smithi02979277-37 2024-04-12T11:17:20.517 INFO:tasks.rados.rados.0.smithi029.stdout:695: expect (ObjNum 253 snap 79 seq_num 253) 2024-04-12T11:17:20.611 INFO:tasks.rados.rados.0.smithi029.stdout:698: finishing write tid 2 to smithi02979277-37 2024-04-12T11:17:20.611 INFO:tasks.rados.rados.0.smithi029.stdout:698: finishing write tid 3 to smithi02979277-37 2024-04-12T11:17:20.611 INFO:tasks.rados.rados.0.smithi029.stdout:698: finishing write tid 4 to smithi02979277-37 2024-04-12T11:17:20.611 INFO:tasks.rados.rados.0.smithi029.stdout:698: oid 37 updating version 0 to 294 2024-04-12T11:17:20.611 INFO:tasks.rados.rados.0.smithi029.stdout:698: oid 37 version 294 is already newer than 293 2024-04-12T11:17:20.611 INFO:tasks.rados.rados.0.smithi029.stdout:698: oid 37 version 294 is already newer than 292 2024-04-12T11:17:20.611 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 294 (ObjNum 290 snap 92 seq_num 290) dirty exists 2024-04-12T11:17:20.611 INFO:tasks.rados.rados.0.smithi029.stdout:698: left oid 37 (ObjNum 290 snap 92 seq_num 290) 2024-04-12T11:17:20.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:20 smithi029 ceph-mon[39999]: osdmap e246: 8 total, 8 up, 8 in 2024-04-12T11:17:20.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:20 smithi029 ceph-mon[93650]: osdmap e246: 8 total, 8 up, 8 in 2024-04-12T11:17:20.824 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:20 smithi087 ceph-mon[32917]: osdmap e246: 8 total, 8 up, 8 in 2024-04-12T11:17:20.824 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:20.595+0000 7fa7435fe200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:17:21.158 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:20 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:20.827+0000 7f4c6959f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-12T11:17:21.158 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:20 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:20.929+0000 7f4c6959f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-12T11:17:21.169 INFO:tasks.rados.rados.0.smithi029.stdout:695: done (4 left) 2024-04-12T11:17:21.169 INFO:tasks.rados.rados.0.smithi029.stdout:696: done (3 left) 2024-04-12T11:17:21.169 INFO:tasks.rados.rados.0.smithi029.stdout:697: done (2 left) 2024-04-12T11:17:21.169 INFO:tasks.rados.rados.0.smithi029.stdout:698: done (1 left) 2024-04-12T11:17:21.169 INFO:tasks.rados.rados.0.smithi029.stdout:699: done (0 left) 2024-04-12T11:17:21.170 INFO:tasks.rados.rados.0.smithi029.stdout:700: rollback oid 45 current snap is 93 2024-04-12T11:17:21.170 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 45 to 78 2024-04-12T11:17:21.170 INFO:tasks.rados.rados.0.smithi029.stdout:701: read oid 9 snap -1 2024-04-12T11:17:21.170 INFO:tasks.rados.rados.0.smithi029.stdout:701: expect (ObjNum 268 snap 86 seq_num 268) 2024-04-12T11:17:21.170 INFO:tasks.rados.rados.0.smithi029.stdout:702: write oid 10 current snap is 93 2024-04-12T11:17:21.170 INFO:tasks.rados.rados.0.smithi029.stdout:702: seq_num 291 ranges {490837=718873,1862260=479427,2875032=15805} 2024-04-12T11:17:21.176 INFO:tasks.rados.rados.0.smithi029.stdout:702: writing smithi02979277-10 from 490837 to 1209710 tid 1 2024-04-12T11:17:21.183 INFO:tasks.rados.rados.0.smithi029.stdout:702: writing smithi02979277-10 from 1862260 to 2341687 tid 2 2024-04-12T11:17:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:702: writing smithi02979277-10 from 2875032 to 2890837 tid 3 2024-04-12T11:17:21.187 INFO:tasks.rados.rados.0.smithi029.stdout:703: write oid 46 current snap is 93 2024-04-12T11:17:21.187 INFO:tasks.rados.rados.0.smithi029.stdout:700: finishing rollback tid 0 to smithi02979277-45 2024-04-12T11:17:21.187 INFO:tasks.rados.rados.0.smithi029.stdout:700: finishing rollback tid 1 to smithi02979277-45 2024-04-12T11:17:21.187 INFO:tasks.rados.rados.0.smithi029.stdout:700: finishing rollback tid 2 to smithi02979277-45 2024-04-12T11:17:21.187 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 374 (ObjNum 251 snap 78 seq_num 251) dirty exists 2024-04-12T11:17:21.187 INFO:tasks.rados.rados.0.smithi029.stdout:701: expect (ObjNum 268 snap 86 seq_num 268) 2024-04-12T11:17:21.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:20.822+0000 7fa7435fe200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:17:21.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: [12/Apr/2024:11:17:20] ENGINE Bus STARTING 2024-04-12T11:17:21.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: CherryPy Checker: 2024-04-12T11:17:21.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: The Application mounted at '' has an empty config. 2024-04-12T11:17:21.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: 2024-04-12T11:17:21.216 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: [12/Apr/2024:11:17:20] ENGINE Serving on http://:::9283 2024-04-12T11:17:21.217 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 11:17:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x[78570]: [12/Apr/2024:11:17:20] ENGINE Bus STARTED 2024-04-12T11:17:21.360 INFO:tasks.rados.rados.0.smithi029.stdout:702: finishing write tid 1 to smithi02979277-10 2024-04-12T11:17:21.360 INFO:tasks.rados.rados.0.smithi029.stdout:702: finishing write tid 2 to smithi02979277-10 2024-04-12T11:17:21.360 INFO:tasks.rados.rados.0.smithi029.stdout:703: seq_num 292 ranges {507644=794122,1920742=623085} 2024-04-12T11:17:21.368 INFO:tasks.rados.rados.0.smithi029.stdout:703: writing smithi02979277-46 from 507644 to 1301766 tid 1 2024-04-12T11:17:21.376 INFO:tasks.rados.rados.0.smithi029.stdout:703: writing smithi02979277-46 from 1920742 to 2543827 tid 2 2024-04-12T11:17:21.376 INFO:tasks.rados.rados.0.smithi029.stdout:700: done (3 left) 2024-04-12T11:17:21.376 INFO:tasks.rados.rados.0.smithi029.stdout:701: done (2 left) 2024-04-12T11:17:21.376 INFO:tasks.rados.rados.0.smithi029.stdout:704: write oid 44 current snap is 93 2024-04-12T11:17:21.376 INFO:tasks.rados.rados.0.smithi029.stdout:702: finishing write tid 3 to smithi02979277-10 2024-04-12T11:17:21.377 INFO:tasks.rados.rados.0.smithi029.stdout:702: finishing write tid 4 to smithi02979277-10 2024-04-12T11:17:21.377 INFO:tasks.rados.rados.0.smithi029.stdout:704: seq_num 293 ranges {524451=469371,1579224=483096,2529878=394573} 2024-04-12T11:17:21.382 INFO:tasks.rados.rados.0.smithi029.stdout:704: writing smithi02979277-44 from 524451 to 993822 tid 1 2024-04-12T11:17:21.388 INFO:tasks.rados.rados.0.smithi029.stdout:704: writing smithi02979277-44 from 1579224 to 2062320 tid 2 2024-04-12T11:17:21.393 INFO:tasks.rados.rados.0.smithi029.stdout:704: writing smithi02979277-44 from 2529878 to 2924451 tid 3 2024-04-12T11:17:21.393 INFO:tasks.rados.rados.0.smithi029.stdout:705: write oid 6 current snap is 93 2024-04-12T11:17:21.393 INFO:tasks.rados.rados.0.smithi029.stdout:702: finishing write tid 5 to smithi02979277-10 2024-04-12T11:17:21.393 INFO:tasks.rados.rados.0.smithi029.stdout:702: oid 10 updating version 0 to 280 2024-04-12T11:17:21.393 INFO:tasks.rados.rados.0.smithi029.stdout:702: oid 10 version 280 is already newer than 279 2024-04-12T11:17:21.393 INFO:tasks.rados.rados.0.smithi029.stdout:702: oid 10 updating version 280 to 282 2024-04-12T11:17:21.394 INFO:tasks.rados.rados.0.smithi029.stdout:702: oid 10 version 282 is already newer than 281 2024-04-12T11:17:21.394 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 282 (ObjNum 291 snap 93 seq_num 291) dirty exists 2024-04-12T11:17:21.394 INFO:tasks.rados.rados.0.smithi029.stdout:702: left oid 10 (ObjNum 291 snap 93 seq_num 291) 2024-04-12T11:17:21.394 INFO:tasks.rados.rados.0.smithi029.stdout:705: seq_num 294 ranges {541258=544620,1637706=626754,2840948=100310} 2024-04-12T11:17:21.399 INFO:tasks.rados.rados.0.smithi029.stdout:705: writing smithi02979277-6 from 541258 to 1085878 tid 1 2024-04-12T11:17:21.405 INFO:tasks.rados.rados.0.smithi029.stdout:705: writing smithi02979277-6 from 1637706 to 2264460 tid 2 2024-04-12T11:17:21.410 INFO:tasks.rados.rados.0.smithi029.stdout:705: writing smithi02979277-6 from 2840948 to 2941258 tid 3 2024-04-12T11:17:21.410 INFO:tasks.rados.rados.0.smithi029.stdout:702: done (3 left) 2024-04-12T11:17:21.410 INFO:tasks.rados.rados.0.smithi029.stdout:706: read oid 37 snap -1 2024-04-12T11:17:21.410 INFO:tasks.rados.rados.0.smithi029.stdout:706: expect (ObjNum 290 snap 92 seq_num 290) 2024-04-12T11:17:21.410 INFO:tasks.rados.rados.0.smithi029.stdout:707: write oid 24 current snap is 93 2024-04-12T11:17:21.410 INFO:tasks.rados.rados.0.smithi029.stdout:703: finishing write tid 1 to smithi02979277-46 2024-04-12T11:17:21.410 INFO:tasks.rados.rados.0.smithi029.stdout:707: seq_num 295 ranges {558065=619869,1579835=486765,2468371=489694} 2024-04-12T11:17:21.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.157+0000 7f4c6959f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-12T11:17:21.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:17:21] ENGINE Bus STARTING 2024-04-12T11:17:21.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.304+0000 7f4bca12f700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:17:21.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.304+0000 7f4bca12f700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:17:21.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.304+0000 7f4bca12f700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:17:21.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.304+0000 7f4bca12f700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:17:21.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.304+0000 7f4bca12f700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:17:21.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.304+0000 7f4bc792a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:17:21.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.304+0000 7f4bc792a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:17:21.413 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.304+0000 7f4bc792a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:17:21.414 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.304+0000 7f4bc792a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:17:21.414 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.304+0000 7f4bc792a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-12T11:17:21.414 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: CherryPy Checker: 2024-04-12T11:17:21.414 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: The Application mounted at '' has an empty config. 2024-04-12T11:17:21.414 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:21.414 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:17:21] ENGINE Serving on http://:::9283 2024-04-12T11:17:21.414 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:11:17:21] ENGINE Bus STARTED 2024-04-12T11:17:21.416 INFO:tasks.rados.rados.0.smithi029.stdout:707: writing smithi02979277-24 from 558065 to 1177934 tid 1 2024-04-12T11:17:21.422 INFO:tasks.rados.rados.0.smithi029.stdout:707: writing smithi02979277-24 from 1579835 to 2066600 tid 2 2024-04-12T11:17:21.427 INFO:tasks.rados.rados.0.smithi029.stdout:707: writing smithi02979277-24 from 2468371 to 2958065 tid 3 2024-04-12T11:17:21.427 INFO:tasks.rados.rados.0.smithi029.stdout:708: read oid 23 snap -1 2024-04-12T11:17:21.427 INFO:tasks.rados.rados.0.smithi029.stdout:708: expect deleted 2024-04-12T11:17:21.427 INFO:tasks.rados.rados.0.smithi029.stdout:709: write oid 14 current snap is 93 2024-04-12T11:17:21.427 INFO:tasks.rados.rados.0.smithi029.stdout:703: finishing write tid 2 to smithi02979277-46 2024-04-12T11:17:21.427 INFO:tasks.rados.rados.0.smithi029.stdout:703: finishing write tid 3 to smithi02979277-46 2024-04-12T11:17:21.427 INFO:tasks.rados.rados.0.smithi029.stdout:703: finishing write tid 4 to smithi02979277-46 2024-04-12T11:17:21.428 INFO:tasks.rados.rados.0.smithi029.stdout:703: oid 46 updating version 0 to 151 2024-04-12T11:17:21.428 INFO:tasks.rados.rados.0.smithi029.stdout:703: oid 46 updating version 151 to 153 2024-04-12T11:17:21.428 INFO:tasks.rados.rados.0.smithi029.stdout:703: oid 46 version 153 is already newer than 152 2024-04-12T11:17:21.428 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 153 (ObjNum 292 snap 93 seq_num 292) dirty exists 2024-04-12T11:17:21.428 INFO:tasks.rados.rados.0.smithi029.stdout:703: left oid 46 (ObjNum 292 snap 93 seq_num 292) 2024-04-12T11:17:21.428 INFO:tasks.rados.rados.0.smithi029.stdout:704: finishing write tid 1 to smithi02979277-44 2024-04-12T11:17:21.428 INFO:tasks.rados.rados.0.smithi029.stdout:704: finishing write tid 2 to smithi02979277-44 2024-04-12T11:17:21.428 INFO:tasks.rados.rados.0.smithi029.stdout:709: seq_num 296 ranges {574872=695118,2038317=630423} 2024-04-12T11:17:21.434 INFO:tasks.rados.rados.0.smithi029.stdout:709: writing smithi02979277-14 from 574872 to 1269990 tid 1 2024-04-12T11:17:21.442 INFO:tasks.rados.rados.0.smithi029.stdout:709: writing smithi02979277-14 from 2038317 to 2668740 tid 2 2024-04-12T11:17:21.442 INFO:tasks.rados.rados.0.smithi029.stdout:703: done (6 left) 2024-04-12T11:17:21.442 INFO:tasks.rados.rados.0.smithi029.stdout:710: write oid 18 current snap is 93 2024-04-12T11:17:21.442 INFO:tasks.rados.rados.0.smithi029.stdout:704: finishing write tid 3 to smithi02979277-44 2024-04-12T11:17:21.442 INFO:tasks.rados.rados.0.smithi029.stdout:704: finishing write tid 4 to smithi02979277-44 2024-04-12T11:17:21.442 INFO:tasks.rados.rados.0.smithi029.stdout:710: seq_num 297 ranges {591679=486720,1813152=490434} 2024-04-12T11:17:21.448 INFO:tasks.rados.rados.0.smithi029.stdout:710: writing smithi02979277-18 from 591679 to 1078399 tid 1 2024-04-12T11:17:21.455 INFO:tasks.rados.rados.0.smithi029.stdout:710: writing smithi02979277-18 from 1813152 to 2303586 tid 2 2024-04-12T11:17:21.455 INFO:tasks.rados.rados.0.smithi029.stdout:711: read oid 41 snap -1 2024-04-12T11:17:21.455 INFO:tasks.rados.rados.0.smithi029.stdout:711: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:17:21.455 INFO:tasks.rados.rados.0.smithi029.stdout:712: delete oid 50 current snap is 93 2024-04-12T11:17:21.455 INFO:tasks.rados.rados.0.smithi029.stdout:704: finishing write tid 5 to smithi02979277-44 2024-04-12T11:17:21.455 INFO:tasks.rados.rados.0.smithi029.stdout:704: oid 44 updating version 0 to 357 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:704: oid 44 version 357 is already newer than 354 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:704: oid 44 version 357 is already newer than 355 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:704: oid 44 version 357 is already newer than 356 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 357 (ObjNum 293 snap 93 seq_num 293) dirty exists 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:704: left oid 44 (ObjNum 293 snap 93 seq_num 293) 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:705: finishing write tid 1 to smithi02979277-6 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:705: finishing write tid 2 to smithi02979277-6 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:705: finishing write tid 3 to smithi02979277-6 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:705: finishing write tid 4 to smithi02979277-6 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:705: finishing write tid 5 to smithi02979277-6 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:705: oid 6 updating version 0 to 157 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:705: oid 6 updating version 157 to 160 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:705: oid 6 version 160 is already newer than 158 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:705: oid 6 version 160 is already newer than 159 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 160 (ObjNum 294 snap 93 seq_num 294) dirty exists 2024-04-12T11:17:21.456 INFO:tasks.rados.rados.0.smithi029.stdout:705: left oid 6 (ObjNum 294 snap 93 seq_num 294) 2024-04-12T11:17:21.457 INFO:tasks.rados.rados.0.smithi029.stdout:706: expect (ObjNum 290 snap 92 seq_num 290) 2024-04-12T11:17:21.589 INFO:tasks.rados.rados.0.smithi029.stdout:707: finishing write tid 1 to smithi02979277-24 2024-04-12T11:17:21.589 INFO:tasks.rados.rados.0.smithi029.stdout:707: finishing write tid 2 to smithi02979277-24 2024-04-12T11:17:21.589 INFO:tasks.rados.rados.0.smithi029.stdout:704: done (8 left) 2024-04-12T11:17:21.589 INFO:tasks.rados.rados.0.smithi029.stdout:705: done (7 left) 2024-04-12T11:17:21.589 INFO:tasks.rados.rados.0.smithi029.stdout:706: done (6 left) 2024-04-12T11:17:21.589 INFO:tasks.rados.rados.0.smithi029.stdout:712: done (5 left) 2024-04-12T11:17:21.589 INFO:tasks.rados.rados.0.smithi029.stdout:713: snap_remove snap 40 2024-04-12T11:17:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: osdmap e247: 8 total, 8 up, 8 in 2024-04-12T11:17:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: Standby manager daemon x restarted 2024-04-12T11:17:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: Standby manager daemon x started 2024-04-12T11:17:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:17:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:17:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:17:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:17:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: Active manager daemon y restarted 2024-04-12T11:17:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: Activating manager daemon y 2024-04-12T11:17:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: osdmap e248: 8 total, 8 up, 8 in 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: mgrmap e33: y(active, starting, since 0.00676281s), standbys: x 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:17:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:17:21.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:17:21.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:17:21.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:17:21.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:17:21.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: Manager daemon y is now available 2024-04-12T11:17:21.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:17:21.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:21.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:17:21.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:17:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: osdmap e247: 8 total, 8 up, 8 in 2024-04-12T11:17:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: Standby manager daemon x restarted 2024-04-12T11:17:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: Standby manager daemon x started 2024-04-12T11:17:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:17:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:17:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:17:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: Active manager daemon y restarted 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: Activating manager daemon y 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: osdmap e248: 8 total, 8 up, 8 in 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: mgrmap e33: y(active, starting, since 0.00676281s), standbys: x 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:17:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: Manager daemon y is now available 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:17:21.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: osdmap e247: 8 total, 8 up, 8 in 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: Standby manager daemon x restarted 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: Standby manager daemon x started 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.? 172.21.15.87:0/2880424617' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: Active manager daemon y restarted 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: Activating manager daemon y 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: osdmap e248: 8 total, 8 up, 8 in 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: mgrmap e33: y(active, starting, since 0.00676281s), standbys: x 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.scbuux"}]: dispatch 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi087.mbflzu"}]: dispatch 2024-04-12T11:17:21.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: Manager daemon y is now available 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-12T11:17:21.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-12T11:17:22.188 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T11:17:22.188 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (3m) 41s ago 4m 27.8M - 0.25.0 c8568f914cd2 5f36b9c37cec 2024-04-12T11:17:22.188 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi087 *:3000 running (4m) 21s ago 4m 90.6M - 9.4.7 954c08fa6188 d4ba4d587fdb 2024-04-12T11:17:22.188 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.scbuux smithi029 running (4m) 41s ago 4m 15.2M - 18.2.2-1187-gd540ebac bb2c5eac26a4 a659ce0bccaf 2024-04-12T11:17:22.188 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi087.mbflzu smithi087 running (4m) 21s ago 4m 16.3M - 18.2.2-1187-gd540ebac bb2c5eac26a4 64bafd1e6a4c 2024-04-12T11:17:22.188 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi087 *:8443,9283,8765 running (26s) 21s ago 9m 66.5M - 19.0.0-2874-g2cee1857 4a0e59462345 5a46135dfdbc 2024-04-12T11:17:22.188 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (55s) 41s ago 11m 478M - 19.0.0-2874-g2cee1857 4a0e59462345 05c187489262 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (11m) 41s ago 11m 52.2M 2048M 18.2.2-1187-gd540ebac bb2c5eac26a4 1cf52b0b92d9 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi087 running (9m) 21s ago 9m 47.2M 2048M 18.2.2-1187-gd540ebac bb2c5eac26a4 adfeefa5d84f 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (9m) 41s ago 9m 46.0M 2048M 18.2.2-1187-gd540ebac bb2c5eac26a4 903a61eba1ba 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (4m) 41s ago 4m 15.4M - 1.5.0 0da6a335fe13 32124106e912 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi087 *:9100 running (4m) 21s ago 4m 16.1M - 1.5.0 0da6a335fe13 78910b2d30c6 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (8m) 41s ago 8m 79.9M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 f65c324b0084 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (8m) 41s ago 8m 64.1M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 bb552e04f578 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (8m) 41s ago 8m 67.7M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 dbb7abb8911b 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (7m) 41s ago 7m 60.3M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 a9b5c72d15da 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi087 running (7m) 21s ago 7m 75.0M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 ab3a8e864afd 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi087 running (6m) 21s ago 6m 77.4M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 573e2f90e4da 2024-04-12T11:17:22.189 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi087 running (6m) 21s ago 6m 61.4M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 2b67d3f477dd 2024-04-12T11:17:22.190 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi087 running (6m) 21s ago 6m 86.5M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 9c484b9f0790 2024-04-12T11:17:22.190 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi087 *:9095 running (31s) 21s ago 5m 36.0M - 2.43.0 a07b618ecd1d 569140fc5a53 2024-04-12T11:17:22.508 INFO:tasks.rados.rados.0.smithi029.stdout:713: done (5 left) 2024-04-12T11:17:22.508 INFO:tasks.rados.rados.0.smithi029.stdout:714: read oid 35 snap -1 2024-04-12T11:17:22.508 INFO:tasks.rados.rados.0.smithi029.stdout:714: expect (ObjNum 271 snap 86 seq_num 271) 2024-04-12T11:17:22.508 INFO:tasks.rados.rados.0.smithi029.stdout:715: read oid 34 snap -1 2024-04-12T11:17:22.508 INFO:tasks.rados.rados.0.smithi029.stdout:715: expect (ObjNum 259 snap 81 seq_num 259) 2024-04-12T11:17:22.508 INFO:tasks.rados.rados.0.smithi029.stdout:716: rollback oid 37 current snap is 93 2024-04-12T11:17:22.508 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 91 2024-04-12T11:17:22.508 INFO:tasks.rados.rados.0.smithi029.stdout:717: write oid 39 current snap is 93 2024-04-12T11:17:22.508 INFO:tasks.rados.rados.0.smithi029.stdout:717: seq_num 299 ranges {625293=637218,1813763=494103,2978063=47230} 2024-04-12T11:17:22.515 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:22 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:17:22.175+0000 7f4c1f740700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-12T11:17:22.515 INFO:tasks.rados.rados.0.smithi029.stdout:717: writing smithi02979277-39 from 625293 to 1262511 tid 1 2024-04-12T11:17:22.522 INFO:tasks.rados.rados.0.smithi029.stdout:717: writing smithi02979277-39 from 1813763 to 2307866 tid 2 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:717: writing smithi02979277-39 from 2978063 to 3025293 tid 3 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:707: finishing write tid 3 to smithi02979277-24 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:707: finishing write tid 4 to smithi02979277-24 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:707: finishing write tid 5 to smithi02979277-24 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:707: oid 24 updating version 0 to 439 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:707: oid 24 version 439 is already newer than 438 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:707: oid 24 updating version 439 to 440 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:707: oid 24 version 440 is already newer than 437 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 440 (ObjNum 295 snap 93 seq_num 295) dirty exists 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:707: left oid 24 (ObjNum 295 snap 93 seq_num 295) 2024-04-12T11:17:22.527 INFO:tasks.rados.rados.0.smithi029.stdout:709: finishing write tid 1 to smithi02979277-14 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:709: finishing write tid 2 to smithi02979277-14 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:709: finishing write tid 3 to smithi02979277-14 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:709: finishing write tid 4 to smithi02979277-14 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:709: oid 14 updating version 0 to 434 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:709: oid 14 updating version 434 to 435 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:709: oid 14 updating version 435 to 436 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 436 (ObjNum 296 snap 93 seq_num 296) dirty exists 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:709: left oid 14 (ObjNum 296 snap 93 seq_num 296) 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:710: finishing write tid 1 to smithi02979277-18 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:710: finishing write tid 2 to smithi02979277-18 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:710: finishing write tid 3 to smithi02979277-18 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:710: finishing write tid 4 to smithi02979277-18 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:710: oid 18 updating version 0 to 389 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:710: oid 18 version 389 is already newer than 388 2024-04-12T11:17:22.528 INFO:tasks.rados.rados.0.smithi029.stdout:710: oid 18 version 389 is already newer than 387 2024-04-12T11:17:22.529 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 389 (ObjNum 297 snap 93 seq_num 297) dirty exists 2024-04-12T11:17:22.529 INFO:tasks.rados.rados.0.smithi029.stdout:710: left oid 18 (ObjNum 297 snap 93 seq_num 297) 2024-04-12T11:17:22.529 INFO:tasks.rados.rados.0.smithi029.stdout:711: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:17:22.672 INFO:tasks.rados.rados.0.smithi029.stdout:715: expect (ObjNum 259 snap 81 seq_num 259) 2024-04-12T11:17:22.782 INFO:tasks.rados.rados.0.smithi029.stdout:714: expect (ObjNum 271 snap 86 seq_num 271) 2024-04-12T11:17:22.925 INFO:tasks.rados.rados.0.smithi029.stdout:717: finishing write tid 1 to smithi02979277-39 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:716: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:716: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:707: done (8 left) 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:708: done (7 left) 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:709: done (6 left) 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:710: done (5 left) 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:711: done (4 left) 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:714: done (3 left) 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:715: done (2 left) 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:718: write oid 23 current snap is 93 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:716: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 302 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:717: finishing write tid 2 to smithi02979277-39 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:717: finishing write tid 3 to smithi02979277-39 2024-04-12T11:17:22.926 INFO:tasks.rados.rados.0.smithi029.stdout:717: finishing write tid 4 to smithi02979277-39 2024-04-12T11:17:22.927 INFO:tasks.rados.rados.0.smithi029.stdout:717: finishing write tid 5 to smithi02979277-39 2024-04-12T11:17:22.927 INFO:tasks.rados.rados.0.smithi029.stdout:717: oid 39 updating version 0 to 24 2024-04-12T11:17:22.927 INFO:tasks.rados.rados.0.smithi029.stdout:717: oid 39 updating version 24 to 26 2024-04-12T11:17:22.927 INFO:tasks.rados.rados.0.smithi029.stdout:717: oid 39 version 26 is already newer than 23 2024-04-12T11:17:22.927 INFO:tasks.rados.rados.0.smithi029.stdout:717: oid 39 version 26 is already newer than 25 2024-04-12T11:17:22.927 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 26 (ObjNum 299 snap 93 seq_num 299) dirty exists 2024-04-12T11:17:22.927 INFO:tasks.rados.rados.0.smithi029.stdout:717: left oid 39 (ObjNum 299 snap 93 seq_num 299) 2024-04-12T11:17:22.927 INFO:tasks.rados.rados.0.smithi029.stdout:718: seq_num 300 ranges {642100=712467,1872245=637761} 2024-04-12T11:17:22.932 INFO:tasks.rados.rados.0.smithi029.stdout:718: writing smithi02979277-23 from 642100 to 1354567 tid 1 2024-04-12T11:17:22.939 INFO:tasks.rados.rados.0.smithi029.stdout:718: writing smithi02979277-23 from 1872245 to 2510006 tid 2 2024-04-12T11:17:22.939 INFO:tasks.rados.rados.0.smithi029.stdout:716: done (2 left) 2024-04-12T11:17:22.939 INFO:tasks.rados.rados.0.smithi029.stdout:717: done (1 left) 2024-04-12T11:17:22.940 INFO:tasks.rados.rados.0.smithi029.stdout:719: write oid 47 current snap is 93 2024-04-12T11:17:22.940 INFO:tasks.rados.rados.0.smithi029.stdout:718: finishing write tid 1 to smithi02979277-23 2024-04-12T11:17:22.940 INFO:tasks.rados.rados.0.smithi029.stdout:719: seq_num 301 ranges {658907=787716,1930727=497772,3032909=25998} 2024-04-12T11:17:22.947 INFO:tasks.rados.rados.0.smithi029.stdout:719: writing smithi02979277-47 from 658907 to 1446623 tid 1 2024-04-12T11:17:22.950 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch upgrade pause' 2024-04-12T11:17:22.953 INFO:tasks.rados.rados.0.smithi029.stdout:719: writing smithi02979277-47 from 1930727 to 2428499 tid 2 2024-04-12T11:17:22.957 INFO:tasks.rados.rados.0.smithi029.stdout:719: writing smithi02979277-47 from 3032909 to 3058907 tid 3 2024-04-12T11:17:22.957 INFO:tasks.rados.rados.0.smithi029.stdout:720: write oid 32 current snap is 93 2024-04-12T11:17:22.957 INFO:tasks.rados.rados.0.smithi029.stdout:718: finishing write tid 2 to smithi02979277-23 2024-04-12T11:17:22.957 INFO:tasks.rados.rados.0.smithi029.stdout:718: finishing write tid 3 to smithi02979277-23 2024-04-12T11:17:22.957 INFO:tasks.rados.rados.0.smithi029.stdout:718: finishing write tid 4 to smithi02979277-23 2024-04-12T11:17:22.957 INFO:tasks.rados.rados.0.smithi029.stdout:718: oid 23 updating version 0 to 267 2024-04-12T11:17:22.957 INFO:tasks.rados.rados.0.smithi029.stdout:718: oid 23 version 267 is already newer than 265 2024-04-12T11:17:22.957 INFO:tasks.rados.rados.0.smithi029.stdout:718: oid 23 version 267 is already newer than 266 2024-04-12T11:17:22.957 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 267 (ObjNum 300 snap 93 seq_num 300) dirty exists 2024-04-12T11:17:22.957 INFO:tasks.rados.rados.0.smithi029.stdout:718: left oid 23 (ObjNum 300 snap 93 seq_num 300) 2024-04-12T11:17:22.958 INFO:tasks.rados.rados.0.smithi029.stdout:719: finishing write tid 1 to smithi02979277-47 2024-04-12T11:17:22.958 INFO:tasks.rados.rados.0.smithi029.stdout:719: finishing write tid 2 to smithi02979277-47 2024-04-12T11:17:22.958 INFO:tasks.rados.rados.0.smithi029.stdout:720: seq_num 302 ranges {675714=462965,1589209=641430,2943979=131735} 2024-04-12T11:17:22.963 INFO:tasks.rados.rados.0.smithi029.stdout:720: writing smithi02979277-32 from 675714 to 1138679 tid 1 2024-04-12T11:17:22.969 INFO:tasks.rados.rados.0.smithi029.stdout:720: writing smithi02979277-32 from 1589209 to 2230639 tid 2 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:720: writing smithi02979277-32 from 2943979 to 3075714 tid 3 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:718: done (2 left) 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:721: snap_create 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:719: finishing write tid 3 to smithi02979277-47 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:719: finishing write tid 4 to smithi02979277-47 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:719: finishing write tid 5 to smithi02979277-47 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:719: oid 47 updating version 0 to 382 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:719: oid 47 updating version 382 to 385 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:719: oid 47 version 385 is already newer than 383 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:719: oid 47 version 385 is already newer than 384 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 385 (ObjNum 301 snap 93 seq_num 301) dirty exists 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:719: left oid 47 (ObjNum 301 snap 93 seq_num 301) 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:720: finishing write tid 1 to smithi02979277-32 2024-04-12T11:17:22.975 INFO:tasks.rados.rados.0.smithi029.stdout:720: finishing write tid 2 to smithi02979277-32 2024-04-12T11:17:22.976 INFO:tasks.rados.rados.0.smithi029.stdout:720: finishing write tid 3 to smithi02979277-32 2024-04-12T11:17:22.977 INFO:tasks.rados.rados.0.smithi029.stdout:720: finishing write tid 4 to smithi02979277-32 2024-04-12T11:17:22.977 INFO:tasks.rados.rados.0.smithi029.stdout:720: finishing write tid 5 to smithi02979277-32 2024-04-12T11:17:22.977 INFO:tasks.rados.rados.0.smithi029.stdout:720: oid 32 updating version 0 to 273 2024-04-12T11:17:22.977 INFO:tasks.rados.rados.0.smithi029.stdout:720: oid 32 version 273 is already newer than 270 2024-04-12T11:17:22.977 INFO:tasks.rados.rados.0.smithi029.stdout:720: oid 32 version 273 is already newer than 271 2024-04-12T11:17:22.977 INFO:tasks.rados.rados.0.smithi029.stdout:720: oid 32 version 273 is already newer than 272 2024-04-12T11:17:22.977 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 273 (ObjNum 302 snap 93 seq_num 302) dirty exists 2024-04-12T11:17:22.977 INFO:tasks.rados.rados.0.smithi029.stdout:720: left oid 32 (ObjNum 302 snap 93 seq_num 302) 2024-04-12T11:17:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:23 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:23 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:23 smithi087 ceph-mon[32917]: mgrmap e34: y(active, since 1.01163s), standbys: x 2024-04-12T11:17:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:23 smithi087 ceph-mon[32917]: [12/Apr/2024:11:17:22] ENGINE Bus STARTING 2024-04-12T11:17:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:23 smithi087 ceph-mon[32917]: [12/Apr/2024:11:17:22] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:17:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:23 smithi087 ceph-mon[32917]: [12/Apr/2024:11:17:22] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:17:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:23 smithi087 ceph-mon[32917]: [12/Apr/2024:11:17:22] ENGINE Bus STARTED 2024-04-12T11:17:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:23 smithi087 ceph-mon[32917]: osdmap e249: 8 total, 8 up, 8 in 2024-04-12T11:17:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[39999]: mgrmap e34: y(active, since 1.01163s), standbys: x 2024-04-12T11:17:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[39999]: [12/Apr/2024:11:17:22] ENGINE Bus STARTING 2024-04-12T11:17:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[39999]: [12/Apr/2024:11:17:22] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:17:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[39999]: [12/Apr/2024:11:17:22] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:17:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[39999]: [12/Apr/2024:11:17:22] ENGINE Bus STARTED 2024-04-12T11:17:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[39999]: osdmap e249: 8 total, 8 up, 8 in 2024-04-12T11:17:23.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:23.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:23.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[93650]: mgrmap e34: y(active, since 1.01163s), standbys: x 2024-04-12T11:17:23.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[93650]: [12/Apr/2024:11:17:22] ENGINE Bus STARTING 2024-04-12T11:17:23.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[93650]: [12/Apr/2024:11:17:22] ENGINE Serving on https://172.21.15.29:7150 2024-04-12T11:17:23.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[93650]: [12/Apr/2024:11:17:22] ENGINE Serving on http://172.21.15.29:8765 2024-04-12T11:17:23.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[93650]: [12/Apr/2024:11:17:22] ENGINE Bus STARTED 2024-04-12T11:17:23.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:23 smithi029 ceph-mon[93650]: osdmap e249: 8 total, 8 up, 8 in 2024-04-12T11:17:23.507 INFO:tasks.rados.rados.0.smithi029.stdout:719: done (2 left) 2024-04-12T11:17:23.507 INFO:tasks.rados.rados.0.smithi029.stdout:720: done (1 left) 2024-04-12T11:17:23.507 INFO:tasks.rados.rados.0.smithi029.stdout:721: done (0 left) 2024-04-12T11:17:23.507 INFO:tasks.rados.rados.0.smithi029.stdout:722: snap_remove snap 63 2024-04-12T11:17:23.807 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:17:23.893 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:17:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:17:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:17:24.514 INFO:tasks.rados.rados.0.smithi029.stdout:722: done (0 left) 2024-04-12T11:17:24.514 INFO:tasks.rados.rados.0.smithi029.stdout:723: read oid 6 snap -1 2024-04-12T11:17:24.514 INFO:tasks.rados.rados.0.smithi029.stdout:723: expect (ObjNum 294 snap 93 seq_num 294) 2024-04-12T11:17:24.514 INFO:tasks.rados.rados.0.smithi029.stdout:724: delete oid 29 current snap is 94 2024-04-12T11:17:24.518 INFO:tasks.rados.rados.0.smithi029.stdout:724: done (1 left) 2024-04-12T11:17:24.519 INFO:tasks.rados.rados.0.smithi029.stdout:725: write oid 46 current snap is 94 2024-04-12T11:17:24.519 INFO:tasks.rados.rados.0.smithi029.stdout:725: seq_num 304 ranges {709328=613463,1989820=645099} 2024-04-12T11:17:24.527 INFO:tasks.rados.rados.0.smithi029.stdout:725: writing smithi02979277-46 from 709328 to 1322791 tid 1 2024-04-12T11:17:24.534 INFO:tasks.rados.rados.0.smithi029.stdout:725: writing smithi02979277-46 from 1989820 to 2634919 tid 2 2024-04-12T11:17:24.534 INFO:tasks.rados.rados.0.smithi029.stdout:726: read oid 18 snap -1 2024-04-12T11:17:24.534 INFO:tasks.rados.rados.0.smithi029.stdout:726: expect (ObjNum 297 snap 93 seq_num 297) 2024-04-12T11:17:24.535 INFO:tasks.rados.rados.0.smithi029.stdout:727: rollback oid 35 current snap is 94 2024-04-12T11:17:24.535 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 35 to 84 2024-04-12T11:17:24.535 INFO:tasks.rados.rados.0.smithi029.stdout:723: expect (ObjNum 294 snap 93 seq_num 294) 2024-04-12T11:17:24.635 INFO:tasks.rados.rados.0.smithi029.stdout:725: finishing write tid 1 to smithi02979277-46 2024-04-12T11:17:24.635 INFO:tasks.rados.rados.0.smithi029.stdout:723: done (3 left) 2024-04-12T11:17:24.635 INFO:tasks.rados.rados.0.smithi029.stdout:728: write oid 31 current snap is 94 2024-04-12T11:17:24.635 INFO:tasks.rados.rados.0.smithi029.stdout:725: finishing write tid 2 to smithi02979277-46 2024-04-12T11:17:24.635 INFO:tasks.rados.rados.0.smithi029.stdout:725: finishing write tid 3 to smithi02979277-46 2024-04-12T11:17:24.635 INFO:tasks.rados.rados.0.smithi029.stdout:728: seq_num 305 ranges {726135=405065,1764655=788757,3026248=99887} 2024-04-12T11:17:24.640 INFO:tasks.rados.rados.0.smithi029.stdout:728: writing smithi02979277-31 from 726135 to 1131200 tid 1 2024-04-12T11:17:24.648 INFO:tasks.rados.rados.0.smithi029.stdout:728: writing smithi02979277-31 from 1764655 to 2553412 tid 2 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:728: writing smithi02979277-31 from 3026248 to 3126135 tid 3 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:729: write oid 32 current snap is 94 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:727: finishing rollback tid 0 to smithi02979277-35 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:727: finishing rollback tid 1 to smithi02979277-35 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:725: finishing write tid 4 to smithi02979277-46 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:725: oid 46 updating version 0 to 158 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:725: oid 46 updating version 158 to 159 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:725: oid 46 version 159 is already newer than 157 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 159 (ObjNum 304 snap 94 seq_num 304) dirty exists 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:725: left oid 46 (ObjNum 304 snap 94 seq_num 304) 2024-04-12T11:17:24.652 INFO:tasks.rados.rados.0.smithi029.stdout:727: finishing rollback tid 2 to smithi02979277-35 2024-04-12T11:17:24.653 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 456 (ObjNum 187 snap 49 seq_num 187) dirty exists 2024-04-12T11:17:24.653 INFO:tasks.rados.rados.0.smithi029.stdout:729: seq_num 306 ranges {742942=480314,1823137=648768} 2024-04-12T11:17:24.658 INFO:tasks.rados.rados.0.smithi029.stdout:729: writing smithi02979277-32 from 742942 to 1223256 tid 1 2024-04-12T11:17:24.665 INFO:tasks.rados.rados.0.smithi029.stdout:729: writing smithi02979277-32 from 1823137 to 2471905 tid 2 2024-04-12T11:17:24.665 INFO:tasks.rados.rados.0.smithi029.stdout:725: done (4 left) 2024-04-12T11:17:24.665 INFO:tasks.rados.rados.0.smithi029.stdout:727: done (3 left) 2024-04-12T11:17:24.666 INFO:tasks.rados.rados.0.smithi029.stdout:730: delete oid 45 current snap is 94 2024-04-12T11:17:24.666 INFO:tasks.rados.rados.0.smithi029.stdout:726: expect (ObjNum 297 snap 93 seq_num 297) 2024-04-12T11:17:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:24 smithi029 ceph-mon[93650]: pgmap v5: 89 pgs: 89 active+clean; 237 MiB data, 727 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:24 smithi029 ceph-mon[93650]: osdmap e250: 8 total, 8 up, 8 in 2024-04-12T11:17:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:24 smithi029 ceph-mon[93650]: mgrmap e35: y(active, since 2s), standbys: x 2024-04-12T11:17:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:24 smithi029 ceph-mon[39999]: pgmap v5: 89 pgs: 89 active+clean; 237 MiB data, 727 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:24 smithi029 ceph-mon[39999]: osdmap e250: 8 total, 8 up, 8 in 2024-04-12T11:17:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:24 smithi029 ceph-mon[39999]: mgrmap e35: y(active, since 2s), standbys: x 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:728: finishing write tid 1 to smithi02979277-31 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:728: finishing write tid 2 to smithi02979277-31 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:728: finishing write tid 3 to smithi02979277-31 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:728: finishing write tid 4 to smithi02979277-31 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:728: finishing write tid 5 to smithi02979277-31 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:728: oid 31 updating version 0 to 295 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:728: oid 31 version 295 is already newer than 294 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:728: oid 31 version 295 is already newer than 293 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:728: oid 31 version 295 is already newer than 292 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 295 (ObjNum 305 snap 94 seq_num 305) dirty exists 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:728: left oid 31 (ObjNum 305 snap 94 seq_num 305) 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:729: finishing write tid 1 to smithi02979277-32 2024-04-12T11:17:24.872 INFO:tasks.rados.rados.0.smithi029.stdout:729: finishing write tid 2 to smithi02979277-32 2024-04-12T11:17:24.873 INFO:tasks.rados.rados.0.smithi029.stdout:729: finishing write tid 3 to smithi02979277-32 2024-04-12T11:17:24.873 INFO:tasks.rados.rados.0.smithi029.stdout:726: done (3 left) 2024-04-12T11:17:24.873 INFO:tasks.rados.rados.0.smithi029.stdout:728: done (2 left) 2024-04-12T11:17:24.873 INFO:tasks.rados.rados.0.smithi029.stdout:730: done (1 left) 2024-04-12T11:17:24.873 INFO:tasks.rados.rados.0.smithi029.stdout:731: snap_remove snap 93 2024-04-12T11:17:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:24 smithi087 ceph-mon[32917]: pgmap v5: 89 pgs: 89 active+clean; 237 MiB data, 727 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:24 smithi087 ceph-mon[32917]: osdmap e250: 8 total, 8 up, 8 in 2024-04-12T11:17:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:24 smithi087 ceph-mon[32917]: mgrmap e35: y(active, since 2s), standbys: x 2024-04-12T11:17:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:17:24] "GET /metrics HTTP/1.1" 200 33532 "" "Prometheus/2.43.0" 2024-04-12T11:17:25.522 INFO:tasks.rados.rados.0.smithi029.stdout:731: done (1 left) 2024-04-12T11:17:25.522 INFO:tasks.rados.rados.0.smithi029.stdout:732: write oid 47 current snap is 94 2024-04-12T11:17:25.522 INFO:tasks.rados.rados.0.smithi029.stdout:729: finishing write tid 4 to smithi02979277-32 2024-04-12T11:17:25.522 INFO:tasks.rados.rados.0.smithi029.stdout:729: oid 32 updating version 0 to 280 2024-04-12T11:17:25.522 INFO:tasks.rados.rados.0.smithi029.stdout:729: oid 32 version 280 is already newer than 279 2024-04-12T11:17:25.522 INFO:tasks.rados.rados.0.smithi029.stdout:729: oid 32 version 280 is already newer than 278 2024-04-12T11:17:25.522 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 280 (ObjNum 306 snap 94 seq_num 306) dirty exists 2024-04-12T11:17:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:729: left oid 32 (ObjNum 306 snap 94 seq_num 306) 2024-04-12T11:17:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:732: seq_num 308 ranges {776556=630812,1823748=652437,3108517=68039} 2024-04-12T11:17:25.529 INFO:tasks.rados.rados.0.smithi029.stdout:732: writing smithi02979277-47 from 776556 to 1407368 tid 1 2024-04-12T11:17:25.536 INFO:tasks.rados.rados.0.smithi029.stdout:732: writing smithi02979277-47 from 1823748 to 2476185 tid 2 2024-04-12T11:17:25.540 INFO:tasks.rados.rados.0.smithi029.stdout:732: writing smithi02979277-47 from 3108517 to 3176556 tid 3 2024-04-12T11:17:25.540 INFO:tasks.rados.rados.0.smithi029.stdout:729: done (1 left) 2024-04-12T11:17:25.540 INFO:tasks.rados.rados.0.smithi029.stdout:733: delete oid 19 current snap is 94 2024-04-12T11:17:25.540 INFO:tasks.rados.rados.0.smithi029.stdout:732: finishing write tid 1 to smithi02979277-47 2024-04-12T11:17:25.540 INFO:tasks.rados.rados.0.smithi029.stdout:732: finishing write tid 2 to smithi02979277-47 2024-04-12T11:17:25.541 INFO:tasks.rados.rados.0.smithi029.stdout:733: done (1 left) 2024-04-12T11:17:25.541 INFO:tasks.rados.rados.0.smithi029.stdout:734: write oid 34 current snap is 94 2024-04-12T11:17:25.541 INFO:tasks.rados.rados.0.smithi029.stdout:734: seq_num 310 ranges {410170=781310,1940712=656106,3163363=46807} 2024-04-12T11:17:25.548 INFO:tasks.rados.rados.0.smithi029.stdout:734: writing smithi02979277-34 from 410170 to 1191480 tid 1 2024-04-12T11:17:25.556 INFO:tasks.rados.rados.0.smithi029.stdout:734: writing smithi02979277-34 from 1940712 to 2596818 tid 2 2024-04-12T11:17:25.559 INFO:tasks.rados.rados.0.smithi029.stdout:734: writing smithi02979277-34 from 3163363 to 3210170 tid 3 2024-04-12T11:17:25.559 INFO:tasks.rados.rados.0.smithi029.stdout:735: write oid 33 current snap is 94 2024-04-12T11:17:25.560 INFO:tasks.rados.rados.0.smithi029.stdout:732: finishing write tid 3 to smithi02979277-47 2024-04-12T11:17:25.560 INFO:tasks.rados.rados.0.smithi029.stdout:735: seq_num 311 ranges {426977=456559,1482841=799764,2958080=268897} 2024-04-12T11:17:25.564 INFO:tasks.rados.rados.0.smithi029.stdout:735: writing smithi02979277-33 from 426977 to 883536 tid 1 2024-04-12T11:17:25.572 INFO:tasks.rados.rados.0.smithi029.stdout:735: writing smithi02979277-33 from 1482841 to 2282605 tid 2 2024-04-12T11:17:25.579 INFO:tasks.rados.rados.0.smithi029.stdout:735: writing smithi02979277-33 from 2958080 to 3226977 tid 3 2024-04-12T11:17:25.579 INFO:tasks.rados.rados.0.smithi029.stdout:736: rollback oid 37 current snap is 94 2024-04-12T11:17:25.579 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 85 2024-04-12T11:17:25.579 INFO:tasks.rados.rados.0.smithi029.stdout:732: finishing write tid 4 to smithi02979277-47 2024-04-12T11:17:25.579 INFO:tasks.rados.rados.0.smithi029.stdout:732: finishing write tid 5 to smithi02979277-47 2024-04-12T11:17:25.579 INFO:tasks.rados.rados.0.smithi029.stdout:732: oid 47 updating version 0 to 394 2024-04-12T11:17:25.579 INFO:tasks.rados.rados.0.smithi029.stdout:732: oid 47 updating version 394 to 397 2024-04-12T11:17:25.579 INFO:tasks.rados.rados.0.smithi029.stdout:732: oid 47 updating version 397 to 398 2024-04-12T11:17:25.580 INFO:tasks.rados.rados.0.smithi029.stdout:732: oid 47 updating version 398 to 399 2024-04-12T11:17:25.580 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 399 (ObjNum 308 snap 94 seq_num 308) dirty exists 2024-04-12T11:17:25.580 INFO:tasks.rados.rados.0.smithi029.stdout:732: left oid 47 (ObjNum 308 snap 94 seq_num 308) 2024-04-12T11:17:25.580 INFO:tasks.rados.rados.0.smithi029.stdout:734: finishing write tid 1 to smithi02979277-34 2024-04-12T11:17:25.580 INFO:tasks.rados.rados.0.smithi029.stdout:734: finishing write tid 2 to smithi02979277-34 2024-04-12T11:17:25.580 INFO:tasks.rados.rados.0.smithi029.stdout:732: done (3 left) 2024-04-12T11:17:25.580 INFO:tasks.rados.rados.0.smithi029.stdout:737: rollback oid 46 current snap is 94 2024-04-12T11:17:25.580 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 46 to 90 2024-04-12T11:17:25.581 INFO:tasks.rados.rados.0.smithi029.stdout:734: finishing write tid 3 to smithi02979277-34 2024-04-12T11:17:25.581 INFO:tasks.rados.rados.0.smithi029.stdout:734: finishing write tid 4 to smithi02979277-34 2024-04-12T11:17:25.581 INFO:tasks.rados.rados.0.smithi029.stdout:734: finishing write tid 5 to smithi02979277-34 2024-04-12T11:17:25.581 INFO:tasks.rados.rados.0.smithi029.stdout:734: oid 34 updating version 0 to 282 2024-04-12T11:17:25.581 INFO:tasks.rados.rados.0.smithi029.stdout:734: oid 34 updating version 282 to 286 2024-04-12T11:17:25.581 INFO:tasks.rados.rados.0.smithi029.stdout:734: oid 34 version 286 is already newer than 285 2024-04-12T11:17:25.581 INFO:tasks.rados.rados.0.smithi029.stdout:734: oid 34 updating version 286 to 287 2024-04-12T11:17:25.581 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 287 (ObjNum 310 snap 94 seq_num 310) dirty exists 2024-04-12T11:17:25.581 INFO:tasks.rados.rados.0.smithi029.stdout:734: left oid 34 (ObjNum 310 snap 94 seq_num 310) 2024-04-12T11:17:25.581 INFO:tasks.rados.rados.0.smithi029.stdout:735: finishing write tid 1 to smithi02979277-33 2024-04-12T11:17:25.582 INFO:tasks.rados.rados.0.smithi029.stdout:735: finishing write tid 2 to smithi02979277-33 2024-04-12T11:17:25.582 INFO:tasks.rados.rados.0.smithi029.stdout:734: done (3 left) 2024-04-12T11:17:25.582 INFO:tasks.rados.rados.0.smithi029.stdout:738: read oid 17 snap 89 2024-04-12T11:17:25.582 INFO:tasks.rados.rados.0.smithi029.stdout:738: expect (ObjNum 253 snap 79 seq_num 253) 2024-04-12T11:17:25.582 INFO:tasks.rados.rados.0.smithi029.stdout:739: read oid 22 snap -1 2024-04-12T11:17:25.582 INFO:tasks.rados.rados.0.smithi029.stdout:739: expect (ObjNum 287 snap 89 seq_num 287) 2024-04-12T11:17:25.582 INFO:tasks.rados.rados.0.smithi029.stdout:740: snap_create 2024-04-12T11:17:25.583 INFO:tasks.rados.rados.0.smithi029.stdout:737: finishing rollback tid 0 to smithi02979277-46 2024-04-12T11:17:25.583 INFO:tasks.rados.rados.0.smithi029.stdout:736: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:17:25.583 INFO:tasks.rados.rados.0.smithi029.stdout:736: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:17:25.583 INFO:tasks.rados.rados.0.smithi029.stdout:737: finishing rollback tid 1 to smithi02979277-46 2024-04-12T11:17:25.583 INFO:tasks.rados.rados.0.smithi029.stdout:736: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:17:25.583 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 313 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-04-12T11:17:25.583 INFO:tasks.rados.rados.0.smithi029.stdout:737: finishing rollback tid 2 to smithi02979277-46 2024-04-12T11:17:25.583 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 164 (ObjNum 261 snap 82 seq_num 261) dirty exists 2024-04-12T11:17:25.584 INFO:tasks.rados.rados.0.smithi029.stdout:738: expect (ObjNum 253 snap 79 seq_num 253) 2024-04-12T11:17:25.679 INFO:tasks.rados.rados.0.smithi029.stdout:735: finishing write tid 3 to smithi02979277-33 2024-04-12T11:17:25.680 INFO:tasks.rados.rados.0.smithi029.stdout:735: finishing write tid 4 to smithi02979277-33 2024-04-12T11:17:25.680 INFO:tasks.rados.rados.0.smithi029.stdout:735: finishing write tid 5 to smithi02979277-33 2024-04-12T11:17:25.680 INFO:tasks.rados.rados.0.smithi029.stdout:735: oid 33 updating version 0 to 276 2024-04-12T11:17:25.680 INFO:tasks.rados.rados.0.smithi029.stdout:735: oid 33 version 276 is already newer than 275 2024-04-12T11:17:25.680 INFO:tasks.rados.rados.0.smithi029.stdout:735: oid 33 version 276 is already newer than 274 2024-04-12T11:17:25.680 INFO:tasks.rados.rados.0.smithi029.stdout:735: oid 33 version 276 is already newer than 273 2024-04-12T11:17:25.680 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 276 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-04-12T11:17:25.680 INFO:tasks.rados.rados.0.smithi029.stdout:735: left oid 33 (ObjNum 311 snap 94 seq_num 311) 2024-04-12T11:17:25.680 INFO:tasks.rados.rados.0.smithi029.stdout:739: expect (ObjNum 287 snap 89 seq_num 287) 2024-04-12T11:17:25.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:25 smithi087 ceph-mon[32917]: osdmap e251: 8 total, 8 up, 8 in 2024-04-12T11:17:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:25 smithi029 ceph-mon[93650]: osdmap e251: 8 total, 8 up, 8 in 2024-04-12T11:17:25.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:25 smithi029 ceph-mon[39999]: osdmap e251: 8 total, 8 up, 8 in 2024-04-12T11:17:26.441 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:17:26.525 INFO:tasks.rados.rados.0.smithi029.stdout:735: done (5 left) 2024-04-12T11:17:26.525 INFO:tasks.rados.rados.0.smithi029.stdout:736: done (4 left) 2024-04-12T11:17:26.525 INFO:tasks.rados.rados.0.smithi029.stdout:737: done (3 left) 2024-04-12T11:17:26.525 INFO:tasks.rados.rados.0.smithi029.stdout:738: done (2 left) 2024-04-12T11:17:26.526 INFO:tasks.rados.rados.0.smithi029.stdout:739: done (1 left) 2024-04-12T11:17:26.526 INFO:tasks.rados.rados.0.smithi029.stdout:740: done (0 left) 2024-04-12T11:17:26.526 INFO:tasks.rados.rados.0.smithi029.stdout:741: snap_create 2024-04-12T11:17:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[39999]: pgmap v8: 89 pgs: 89 active+clean; 237 MiB data, 727 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[39999]: osdmap e252: 8 total, 8 up, 8 in 2024-04-12T11:17:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[39999]: mgrmap e36: y(active, since 4s), standbys: x 2024-04-12T11:17:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[93650]: pgmap v8: 89 pgs: 89 active+clean; 237 MiB data, 727 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[93650]: osdmap e252: 8 total, 8 up, 8 in 2024-04-12T11:17:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[93650]: mgrmap e36: y(active, since 4s), standbys: x 2024-04-12T11:17:26.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:26.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:26.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:26 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:26 smithi087 ceph-mon[32917]: pgmap v8: 89 pgs: 89 active+clean; 237 MiB data, 727 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:26 smithi087 ceph-mon[32917]: osdmap e252: 8 total, 8 up, 8 in 2024-04-12T11:17:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:26 smithi087 ceph-mon[32917]: mgrmap e36: y(active, since 4s), standbys: x 2024-04-12T11:17:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:26 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:26 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:26 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:27.494 INFO:tasks.rados.rados.0.smithi029.stdout:741: done (0 left) 2024-04-12T11:17:27.494 INFO:tasks.rados.rados.0.smithi029.stdout:742: rollback oid 17 current snap is 96 2024-04-12T11:17:27.494 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 95 2024-04-12T11:17:27.494 INFO:tasks.rados.rados.0.smithi029.stdout:743: write oid 42 current snap is 96 2024-04-12T11:17:27.494 INFO:tasks.rados.rados.0.smithi029.stdout:743: seq_num 312 ranges {443784=648161,1657676=659775,2818209=425575} 2024-04-12T11:17:27.500 INFO:tasks.rados.rados.0.smithi029.stdout:743: writing smithi02979277-42 from 443784 to 1091945 tid 1 2024-04-12T11:17:27.507 INFO:tasks.rados.rados.0.smithi029.stdout:743: writing smithi02979277-42 from 1657676 to 2317451 tid 2 2024-04-12T11:17:27.513 INFO:tasks.rados.rados.0.smithi029.stdout:743: writing smithi02979277-42 from 2818209 to 3243784 tid 3 2024-04-12T11:17:27.513 INFO:tasks.rados.rados.0.smithi029.stdout:744: rollback oid 47 current snap is 96 2024-04-12T11:17:27.513 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 86 2024-04-12T11:17:27.513 INFO:tasks.rados.rados.0.smithi029.stdout:742: finishing rollback tid 0 to smithi02979277-17 2024-04-12T11:17:27.513 INFO:tasks.rados.rados.0.smithi029.stdout:742: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:17:27.513 INFO:tasks.rados.rados.0.smithi029.stdout:742: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:17:27.514 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 409 (ObjNum 253 snap 79 seq_num 253) dirty exists 2024-04-12T11:17:27.514 INFO:tasks.rados.rados.0.smithi029.stdout:743: finishing write tid 1 to smithi02979277-42 2024-04-12T11:17:27.514 INFO:tasks.rados.rados.0.smithi029.stdout:743: finishing write tid 2 to smithi02979277-42 2024-04-12T11:17:27.514 INFO:tasks.rados.rados.0.smithi029.stdout:742: done (2 left) 2024-04-12T11:17:27.514 INFO:tasks.rados.rados.0.smithi029.stdout:745: write oid 4 current snap is 96 2024-04-12T11:17:27.514 INFO:tasks.rados.rados.0.smithi029.stdout:745: seq_num 313 ranges {460591=723410,1716158=403433,2729279=440140} 2024-04-12T11:17:27.519 INFO:tasks.rados.rados.0.smithi029.stdout:745: writing smithi02979277-4 from 460591 to 1184001 tid 1 2024-04-12T11:17:27.525 INFO:tasks.rados.rados.0.smithi029.stdout:745: writing smithi02979277-4 from 1716158 to 2119591 tid 2 2024-04-12T11:17:27.531 INFO:tasks.rados.rados.0.smithi029.stdout:745: writing smithi02979277-4 from 2729279 to 3169419 tid 3 2024-04-12T11:17:27.531 INFO:tasks.rados.rados.0.smithi029.stdout:746: write oid 36 current snap is 96 2024-04-12T11:17:27.531 INFO:tasks.rados.rados.0.smithi029.stdout:743: finishing write tid 3 to smithi02979277-42 2024-04-12T11:17:27.531 INFO:tasks.rados.rados.0.smithi029.stdout:746: seq_num 314 ranges {477398=798659,1774640=663444,2873055=404343} 2024-04-12T11:17:27.538 INFO:tasks.rados.rados.0.smithi029.stdout:746: writing smithi02979277-36 from 477398 to 1276057 tid 1 2024-04-12T11:17:27.544 INFO:tasks.rados.rados.0.smithi029.stdout:746: writing smithi02979277-36 from 1774640 to 2438084 tid 2 2024-04-12T11:17:27.550 INFO:tasks.rados.rados.0.smithi029.stdout:746: writing smithi02979277-36 from 2873055 to 3277398 tid 3 2024-04-12T11:17:27.550 INFO:tasks.rados.rados.0.smithi029.stdout:747: read oid 38 snap -1 2024-04-12T11:17:27.550 INFO:tasks.rados.rados.0.smithi029.stdout:747: expect (ObjNum 220 snap 63 seq_num 220) 2024-04-12T11:17:27.550 INFO:tasks.rados.rados.0.smithi029.stdout:748: delete oid 31 current snap is 96 2024-04-12T11:17:27.550 INFO:tasks.rados.rados.0.smithi029.stdout:744: finishing rollback tid 0 to smithi02979277-47 2024-04-12T11:17:27.550 INFO:tasks.rados.rados.0.smithi029.stdout:743: finishing write tid 4 to smithi02979277-42 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:744: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:743: finishing write tid 5 to smithi02979277-42 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:743: oid 42 updating version 0 to 109 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:743: oid 42 version 109 is already newer than 106 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:743: oid 42 version 109 is already newer than 107 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:743: oid 42 version 109 is already newer than 108 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 109 (ObjNum 312 snap 96 seq_num 312) dirty exists 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:743: left oid 42 (ObjNum 312 snap 96 seq_num 312) 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:744: finishing rollback tid 2 to smithi02979277-47 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 403 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:745: finishing write tid 1 to smithi02979277-4 2024-04-12T11:17:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:745: finishing write tid 2 to smithi02979277-4 2024-04-12T11:17:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:745: finishing write tid 3 to smithi02979277-4 2024-04-12T11:17:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:745: finishing write tid 4 to smithi02979277-4 2024-04-12T11:17:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:745: finishing write tid 5 to smithi02979277-4 2024-04-12T11:17:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:745: oid 4 updating version 0 to 457 2024-04-12T11:17:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:745: oid 4 version 457 is already newer than 455 2024-04-12T11:17:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:745: oid 4 version 457 is already newer than 454 2024-04-12T11:17:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:745: oid 4 version 457 is already newer than 456 2024-04-12T11:17:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 457 (ObjNum 313 snap 96 seq_num 313) dirty exists 2024-04-12T11:17:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:745: left oid 4 (ObjNum 313 snap 96 seq_num 313) 2024-04-12T11:17:27.552 INFO:tasks.rados.rados.0.smithi029.stdout:746: finishing write tid 1 to smithi02979277-36 2024-04-12T11:17:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:746: finishing write tid 2 to smithi02979277-36 2024-04-12T11:17:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:743: done (5 left) 2024-04-12T11:17:27.553 INFO:tasks.rados.rados.0.smithi029.stdout:744: done (4 left) 2024-04-12T11:17:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:745: done (3 left) 2024-04-12T11:17:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:748: done (2 left) 2024-04-12T11:17:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:749: snap_create 2024-04-12T11:17:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:746: finishing write tid 3 to smithi02979277-36 2024-04-12T11:17:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:746: finishing write tid 4 to smithi02979277-36 2024-04-12T11:17:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:746: finishing write tid 5 to smithi02979277-36 2024-04-12T11:17:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:746: oid 36 updating version 0 to 169 2024-04-12T11:17:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:746: oid 36 version 169 is already newer than 168 2024-04-12T11:17:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:746: oid 36 updating version 169 to 170 2024-04-12T11:17:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:746: oid 36 updating version 170 to 171 2024-04-12T11:17:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 171 (ObjNum 314 snap 96 seq_num 314) dirty exists 2024-04-12T11:17:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:746: left oid 36 (ObjNum 314 snap 96 seq_num 314) 2024-04-12T11:17:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:747: expect (ObjNum 220 snap 63 seq_num 220) 2024-04-12T11:17:27.625 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'sleep 60' 2024-04-12T11:17:27.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[39999]: from='client.25253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:17:27.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[39999]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:17:27.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[39999]: osdmap e253: 8 total, 8 up, 8 in 2024-04-12T11:17:27.814 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:27.814 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:27.814 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[39999]: osdmap e254: 8 total, 8 up, 8 in 2024-04-12T11:17:27.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[93650]: from='client.25253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:17:27.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[93650]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:17:27.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[93650]: osdmap e253: 8 total, 8 up, 8 in 2024-04-12T11:17:27.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:27.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:27.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:27 smithi029 ceph-mon[93650]: osdmap e254: 8 total, 8 up, 8 in 2024-04-12T11:17:27.831 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:27 smithi087 ceph-mon[32917]: from='client.25253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:17:27.831 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:27 smithi087 ceph-mon[32917]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:17:27.832 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:27 smithi087 ceph-mon[32917]: osdmap e253: 8 total, 8 up, 8 in 2024-04-12T11:17:27.832 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:27 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:27.832 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:27 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:27.832 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:27 smithi087 ceph-mon[32917]: osdmap e254: 8 total, 8 up, 8 in 2024-04-12T11:17:28.214 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:17:28.528 INFO:tasks.rados.rados.0.smithi029.stdout:746: done (2 left) 2024-04-12T11:17:28.528 INFO:tasks.rados.rados.0.smithi029.stdout:747: done (1 left) 2024-04-12T11:17:28.528 INFO:tasks.rados.rados.0.smithi029.stdout:749: done (0 left) 2024-04-12T11:17:28.528 INFO:tasks.rados.rados.0.smithi029.stdout:750: rollback oid 17 current snap is 97 2024-04-12T11:17:28.528 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 84 2024-04-12T11:17:28.528 INFO:tasks.rados.rados.0.smithi029.stdout:751: snap_remove snap 85 2024-04-12T11:17:29.156 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:28 smithi087 ceph-mon[32917]: pgmap v11: 89 pgs: 89 active+clean; 237 MiB data, 727 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:29.156 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:28 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:29.156 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:28 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:29.156 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:28 smithi087 ceph-mon[32917]: osdmap e255: 8 total, 8 up, 8 in 2024-04-12T11:17:29.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:28 smithi029 ceph-mon[39999]: pgmap v11: 89 pgs: 89 active+clean; 237 MiB data, 727 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:29.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:28 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:29.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:28 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:29.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:28 smithi029 ceph-mon[39999]: osdmap e255: 8 total, 8 up, 8 in 2024-04-12T11:17:29.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:28 smithi029 ceph-mon[93650]: pgmap v11: 89 pgs: 89 active+clean; 237 MiB data, 727 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:29.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:28 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:29.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:28 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:29.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:28 smithi029 ceph-mon[93650]: osdmap e255: 8 total, 8 up, 8 in 2024-04-12T11:17:29.531 INFO:tasks.rados.rados.0.smithi029.stdout:751: done (1 left) 2024-04-12T11:17:29.531 INFO:tasks.rados.rados.0.smithi029.stdout:752: rollback oid 3 current snap is 97 2024-04-12T11:17:29.531 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 84 2024-04-12T11:17:29.532 INFO:tasks.rados.rados.0.smithi029.stdout:750: finishing rollback tid 0 to smithi02979277-17 2024-04-12T11:17:29.532 INFO:tasks.rados.rados.0.smithi029.stdout:750: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:17:29.532 INFO:tasks.rados.rados.0.smithi029.stdout:750: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:17:29.532 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 413 (ObjNum 253 snap 79 seq_num 253) dirty exists 2024-04-12T11:17:29.532 INFO:tasks.rados.rados.0.smithi029.stdout:750: done (1 left) 2024-04-12T11:17:29.532 INFO:tasks.rados.rados.0.smithi029.stdout:753: delete oid 13 current snap is 97 2024-04-12T11:17:29.536 INFO:tasks.rados.rados.0.smithi029.stdout:752: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:17:29.536 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-04-12T11:17:29.537 INFO:tasks.rados.rados.0.smithi029.stdout:752: done (1 left) 2024-04-12T11:17:29.537 INFO:tasks.rados.rados.0.smithi029.stdout:753: done (0 left) 2024-04-12T11:17:29.537 INFO:tasks.rados.rados.0.smithi029.stdout:754: delete oid 17 current snap is 97 2024-04-12T11:17:29.539 INFO:tasks.rados.rados.0.smithi029.stdout:754: done (0 left) 2024-04-12T11:17:29.540 INFO:tasks.rados.rados.0.smithi029.stdout:755: write oid 27 current snap is 97 2024-04-12T11:17:29.540 INFO:tasks.rados.rados.0.smithi029.stdout:755: seq_num 318 ranges {544626=699655,1892215=554429,3150041=194585} 2024-04-12T11:17:29.546 INFO:tasks.rados.rados.0.smithi029.stdout:755: writing smithi02979277-27 from 544626 to 1244281 tid 1 2024-04-12T11:17:29.553 INFO:tasks.rados.rados.0.smithi029.stdout:755: writing smithi02979277-27 from 1892215 to 2446644 tid 2 2024-04-12T11:17:29.558 INFO:tasks.rados.rados.0.smithi029.stdout:755: writing smithi02979277-27 from 3150041 to 3344626 tid 3 2024-04-12T11:17:29.558 INFO:tasks.rados.rados.0.smithi029.stdout:756: delete oid 39 current snap is 97 2024-04-12T11:17:29.558 INFO:tasks.rados.rados.0.smithi029.stdout:755: finishing write tid 1 to smithi02979277-27 2024-04-12T11:17:29.559 INFO:tasks.rados.rados.0.smithi029.stdout:755: finishing write tid 2 to smithi02979277-27 2024-04-12T11:17:29.560 INFO:tasks.rados.rados.0.smithi029.stdout:755: finishing write tid 3 to smithi02979277-27 2024-04-12T11:17:29.560 INFO:tasks.rados.rados.0.smithi029.stdout:756: done (1 left) 2024-04-12T11:17:29.561 INFO:tasks.rados.rados.0.smithi029.stdout:757: snap_create 2024-04-12T11:17:29.561 INFO:tasks.rados.rados.0.smithi029.stdout:755: finishing write tid 4 to smithi02979277-27 2024-04-12T11:17:29.561 INFO:tasks.rados.rados.0.smithi029.stdout:755: finishing write tid 5 to smithi02979277-27 2024-04-12T11:17:29.561 INFO:tasks.rados.rados.0.smithi029.stdout:755: oid 27 updating version 0 to 330 2024-04-12T11:17:29.561 INFO:tasks.rados.rados.0.smithi029.stdout:755: oid 27 version 330 is already newer than 329 2024-04-12T11:17:29.561 INFO:tasks.rados.rados.0.smithi029.stdout:755: oid 27 updating version 330 to 331 2024-04-12T11:17:29.561 INFO:tasks.rados.rados.0.smithi029.stdout:755: oid 27 version 331 is already newer than 328 2024-04-12T11:17:29.561 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 331 (ObjNum 318 snap 97 seq_num 318) dirty exists 2024-04-12T11:17:29.561 INFO:tasks.rados.rados.0.smithi029.stdout:755: left oid 27 (ObjNum 318 snap 97 seq_num 318) 2024-04-12T11:17:30.531 INFO:tasks.rados.rados.0.smithi029.stdout:755: done (1 left) 2024-04-12T11:17:30.531 INFO:tasks.rados.rados.0.smithi029.stdout:757: done (0 left) 2024-04-12T11:17:30.532 INFO:tasks.rados.rados.0.smithi029.stdout:758: delete oid 21 current snap is 98 2024-04-12T11:17:30.537 INFO:tasks.rados.rados.0.smithi029.stdout:758: done (0 left) 2024-04-12T11:17:30.538 INFO:tasks.rados.rados.0.smithi029.stdout:759: snap_remove snap 91 2024-04-12T11:17:30.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:30 smithi087 ceph-mon[32917]: pgmap v14: 89 pgs: 89 active+clean; 179 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 58 op/s 2024-04-12T11:17:30.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:30 smithi087 ceph-mon[32917]: osdmap e256: 8 total, 8 up, 8 in 2024-04-12T11:17:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:30 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:30 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:30 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:30 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:30 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:30 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:17:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[39999]: pgmap v14: 89 pgs: 89 active+clean; 179 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 58 op/s 2024-04-12T11:17:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[39999]: osdmap e256: 8 total, 8 up, 8 in 2024-04-12T11:17:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:17:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[93650]: pgmap v14: 89 pgs: 89 active+clean; 179 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 58 op/s 2024-04-12T11:17:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[93650]: osdmap e256: 8 total, 8 up, 8 in 2024-04-12T11:17:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:17:31.543 INFO:tasks.rados.rados.0.smithi029.stdout:759: done (0 left) 2024-04-12T11:17:31.544 INFO:tasks.rados.rados.0.smithi029.stdout:760: read oid 41 snap 80 2024-04-12T11:17:31.544 INFO:tasks.rados.rados.0.smithi029.stdout:760: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:17:31.544 INFO:tasks.rados.rados.0.smithi029.stdout:761: write oid 14 current snap is 98 2024-04-12T11:17:31.544 INFO:tasks.rados.rados.0.smithi029.stdout:761: seq_num 321 ranges {595047=641755,1667661=418109,2548663=763022} 2024-04-12T11:17:31.550 INFO:tasks.rados.rados.0.smithi029.stdout:761: writing smithi02979277-14 from 595047 to 1236802 tid 1 2024-04-12T11:17:31.556 INFO:tasks.rados.rados.0.smithi029.stdout:761: writing smithi02979277-14 from 1667661 to 2085770 tid 2 2024-04-12T11:17:31.563 INFO:tasks.rados.rados.0.smithi029.stdout:761: writing smithi02979277-14 from 2548663 to 3311685 tid 3 2024-04-12T11:17:31.563 INFO:tasks.rados.rados.0.smithi029.stdout:762: read oid 10 snap 94 2024-04-12T11:17:31.563 INFO:tasks.rados.rados.0.smithi029.stdout:762: expect (ObjNum 291 snap 93 seq_num 291) 2024-04-12T11:17:31.563 INFO:tasks.rados.rados.0.smithi029.stdout:763: read oid 6 snap -1 2024-04-12T11:17:31.563 INFO:tasks.rados.rados.0.smithi029.stdout:763: expect (ObjNum 294 snap 93 seq_num 294) 2024-04-12T11:17:31.563 INFO:tasks.rados.rados.0.smithi029.stdout:764: snap_create 2024-04-12T11:17:31.563 INFO:tasks.rados.rados.0.smithi029.stdout:761: finishing write tid 1 to smithi02979277-14 2024-04-12T11:17:31.563 INFO:tasks.rados.rados.0.smithi029.stdout:760: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:17:31.706 INFO:tasks.rados.rados.0.smithi029.stdout:761: finishing write tid 2 to smithi02979277-14 2024-04-12T11:17:31.706 INFO:tasks.rados.rados.0.smithi029.stdout:761: finishing write tid 3 to smithi02979277-14 2024-04-12T11:17:31.706 INFO:tasks.rados.rados.0.smithi029.stdout:761: finishing write tid 4 to smithi02979277-14 2024-04-12T11:17:31.707 INFO:tasks.rados.rados.0.smithi029.stdout:761: finishing write tid 5 to smithi02979277-14 2024-04-12T11:17:31.707 INFO:tasks.rados.rados.0.smithi029.stdout:761: oid 14 updating version 0 to 466 2024-04-12T11:17:31.707 INFO:tasks.rados.rados.0.smithi029.stdout:761: oid 14 version 466 is already newer than 463 2024-04-12T11:17:31.707 INFO:tasks.rados.rados.0.smithi029.stdout:761: oid 14 version 466 is already newer than 464 2024-04-12T11:17:31.707 INFO:tasks.rados.rados.0.smithi029.stdout:761: oid 14 version 466 is already newer than 465 2024-04-12T11:17:31.707 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 466 (ObjNum 321 snap 98 seq_num 321) dirty exists 2024-04-12T11:17:31.707 INFO:tasks.rados.rados.0.smithi029.stdout:761: left oid 14 (ObjNum 321 snap 98 seq_num 321) 2024-04-12T11:17:31.707 INFO:tasks.rados.rados.0.smithi029.stdout:762: expect (ObjNum 291 snap 93 seq_num 291) 2024-04-12T11:17:31.803 INFO:tasks.rados.rados.0.smithi029.stdout:763: expect (ObjNum 294 snap 93 seq_num 294) 2024-04-12T11:17:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: osdmap e257: 8 total, 8 up, 8 in 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:31 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:17:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:17:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: osdmap e257: 8 total, 8 up, 8 in 2024-04-12T11:17:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:17:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:17:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:17:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:17:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: Updating smithi029:/etc/ceph/ceph.conf 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: osdmap e257: 8 total, 8 up, 8 in 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.conf 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:17:32.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-12T11:17:32.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:31 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.495 INFO:tasks.rados.rados.0.smithi029.stdout:760: done (4 left) 2024-04-12T11:17:32.496 INFO:tasks.rados.rados.0.smithi029.stdout:761: done (3 left) 2024-04-12T11:17:32.496 INFO:tasks.rados.rados.0.smithi029.stdout:762: done (2 left) 2024-04-12T11:17:32.496 INFO:tasks.rados.rados.0.smithi029.stdout:763: done (1 left) 2024-04-12T11:17:32.496 INFO:tasks.rados.rados.0.smithi029.stdout:764: done (0 left) 2024-04-12T11:17:32.496 INFO:tasks.rados.rados.0.smithi029.stdout:765: snap_create 2024-04-12T11:17:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:17:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:17:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: pgmap v17: 89 pgs: 89 active+clean; 179 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 58 op/s 2024-04-12T11:17:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:17:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:17:32.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: osdmap e258: 8 total, 8 up, 8 in 2024-04-12T11:17:32.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:17:32.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:32.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:17:32.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:32.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:32 smithi087 ceph-mon[32917]: osdmap e259: 8 total, 8 up, 8 in 2024-04-12T11:17:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: pgmap v17: 89 pgs: 89 active+clean; 179 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 58 op/s 2024-04-12T11:17:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:17:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:17:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: osdmap e258: 8 total, 8 up, 8 in 2024-04-12T11:17:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:17:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[93650]: osdmap e259: 8 total, 8 up, 8 in 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: pgmap v17: 89 pgs: 89 active+clean; 179 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 58 op/s 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: Updating smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: Updating smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: osdmap e258: 8 total, 8 up, 8 in 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:17:33.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:33.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:33.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:17:33.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:17:33.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:17:33.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:32 smithi029 ceph-mon[39999]: osdmap e259: 8 total, 8 up, 8 in 2024-04-12T11:17:33.499 INFO:tasks.rados.rados.0.smithi029.stdout:765: done (0 left) 2024-04-12T11:17:33.499 INFO:tasks.rados.rados.0.smithi029.stdout:766: write oid 42 current snap is 100 2024-04-12T11:17:33.500 INFO:tasks.rados.rados.0.smithi029.stdout:766: seq_num 322 ranges {611854=717004,2126143=561767,3259733=152121} 2024-04-12T11:17:33.505 INFO:tasks.rados.rados.0.smithi029.stdout:766: writing smithi02979277-42 from 611854 to 1328858 tid 1 2024-04-12T11:17:33.513 INFO:tasks.rados.rados.0.smithi029.stdout:766: writing smithi02979277-42 from 2126143 to 2687910 tid 2 2024-04-12T11:17:33.518 INFO:tasks.rados.rados.0.smithi029.stdout:766: writing smithi02979277-42 from 3259733 to 3411854 tid 3 2024-04-12T11:17:33.518 INFO:tasks.rados.rados.0.smithi029.stdout:767: read oid 4 snap 89 2024-04-12T11:17:33.518 INFO:tasks.rados.rados.0.smithi029.stdout:767: expect deleted 2024-04-12T11:17:33.518 INFO:tasks.rados.rados.0.smithi029.stdout:768: rollback oid 35 current snap is 100 2024-04-12T11:17:33.518 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 35 to 97 2024-04-12T11:17:33.518 INFO:tasks.rados.rados.0.smithi029.stdout:766: finishing write tid 1 to smithi02979277-42 2024-04-12T11:17:33.518 INFO:tasks.rados.rados.0.smithi029.stdout:766: finishing write tid 2 to smithi02979277-42 2024-04-12T11:17:33.518 INFO:tasks.rados.rados.0.smithi029.stdout:769: snap_create 2024-04-12T11:17:33.519 INFO:tasks.rados.rados.0.smithi029.stdout:766: finishing write tid 3 to smithi02979277-42 2024-04-12T11:17:33.519 INFO:tasks.rados.rados.0.smithi029.stdout:766: finishing write tid 4 to smithi02979277-42 2024-04-12T11:17:33.520 INFO:tasks.rados.rados.0.smithi029.stdout:766: finishing write tid 5 to smithi02979277-42 2024-04-12T11:17:33.520 INFO:tasks.rados.rados.0.smithi029.stdout:766: oid 42 updating version 0 to 117 2024-04-12T11:17:33.520 INFO:tasks.rados.rados.0.smithi029.stdout:766: oid 42 version 117 is already newer than 115 2024-04-12T11:17:33.520 INFO:tasks.rados.rados.0.smithi029.stdout:766: oid 42 version 117 is already newer than 116 2024-04-12T11:17:33.520 INFO:tasks.rados.rados.0.smithi029.stdout:766: oid 42 updating version 117 to 118 2024-04-12T11:17:33.520 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 118 (ObjNum 322 snap 100 seq_num 322) dirty exists 2024-04-12T11:17:33.520 INFO:tasks.rados.rados.0.smithi029.stdout:766: left oid 42 (ObjNum 322 snap 100 seq_num 322) 2024-04-12T11:17:33.521 INFO:tasks.rados.rados.0.smithi029.stdout:768: finishing rollback tid 0 to smithi02979277-35 2024-04-12T11:17:33.522 INFO:tasks.rados.rados.0.smithi029.stdout:768: finishing rollback tid 1 to smithi02979277-35 2024-04-12T11:17:33.522 INFO:tasks.rados.rados.0.smithi029.stdout:768: finishing rollback tid 2 to smithi02979277-35 2024-04-12T11:17:33.522 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 472 (ObjNum 187 snap 49 seq_num 187) dirty exists 2024-04-12T11:17:33.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:17:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:17:33.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:17:34.504 INFO:tasks.rados.rados.0.smithi029.stdout:766: done (3 left) 2024-04-12T11:17:34.504 INFO:tasks.rados.rados.0.smithi029.stdout:767: done (2 left) 2024-04-12T11:17:34.504 INFO:tasks.rados.rados.0.smithi029.stdout:768: done (1 left) 2024-04-12T11:17:34.504 INFO:tasks.rados.rados.0.smithi029.stdout:769: done (0 left) 2024-04-12T11:17:34.504 INFO:tasks.rados.rados.0.smithi029.stdout:770: write oid 19 current snap is 101 2024-04-12T11:17:34.504 INFO:tasks.rados.rados.0.smithi029.stdout:770: seq_num 323 ranges {628661=792253,2184625=421778,3403509=25152} 2024-04-12T11:17:34.511 INFO:tasks.rados.rados.0.smithi029.stdout:770: writing smithi02979277-19 from 628661 to 1420914 tid 1 2024-04-12T11:17:34.518 INFO:tasks.rados.rados.0.smithi029.stdout:770: writing smithi02979277-19 from 2184625 to 2606403 tid 2 2024-04-12T11:17:34.523 INFO:tasks.rados.rados.0.smithi029.stdout:770: writing smithi02979277-19 from 3403509 to 3428661 tid 3 2024-04-12T11:17:34.523 INFO:tasks.rados.rados.0.smithi029.stdout:771: write oid 44 current snap is 101 2024-04-12T11:17:34.524 INFO:tasks.rados.rados.0.smithi029.stdout:770: finishing write tid 1 to smithi02979277-19 2024-04-12T11:17:34.524 INFO:tasks.rados.rados.0.smithi029.stdout:770: finishing write tid 2 to smithi02979277-19 2024-04-12T11:17:34.524 INFO:tasks.rados.rados.0.smithi029.stdout:771: seq_num 324 ranges {645468=467502,1726754=565436,2798226=596838} 2024-04-12T11:17:34.529 INFO:tasks.rados.rados.0.smithi029.stdout:771: writing smithi02979277-44 from 645468 to 1112970 tid 1 2024-04-12T11:17:34.536 INFO:tasks.rados.rados.0.smithi029.stdout:771: writing smithi02979277-44 from 1726754 to 2292190 tid 2 2024-04-12T11:17:34.542 INFO:tasks.rados.rados.0.smithi029.stdout:771: writing smithi02979277-44 from 2798226 to 3395064 tid 3 2024-04-12T11:17:34.542 INFO:tasks.rados.rados.0.smithi029.stdout:772: snap_remove snap 94 2024-04-12T11:17:34.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:34 smithi029 ceph-mon[93650]: pgmap v20: 89 pgs: 89 active+clean; 191 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-04-12T11:17:34.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:34 smithi029 ceph-mon[93650]: osdmap e260: 8 total, 8 up, 8 in 2024-04-12T11:17:34.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:34 smithi029 ceph-mon[39999]: pgmap v20: 89 pgs: 89 active+clean; 191 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-04-12T11:17:34.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:34 smithi029 ceph-mon[39999]: osdmap e260: 8 total, 8 up, 8 in 2024-04-12T11:17:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:34 smithi087 ceph-mon[32917]: pgmap v20: 89 pgs: 89 active+clean; 191 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-04-12T11:17:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:34 smithi087 ceph-mon[32917]: osdmap e260: 8 total, 8 up, 8 in 2024-04-12T11:17:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:17:34] "GET /metrics HTTP/1.1" 200 33532 "" "Prometheus/2.43.0" 2024-04-12T11:17:35.509 INFO:tasks.rados.rados.0.smithi029.stdout:772: done (2 left) 2024-04-12T11:17:35.509 INFO:tasks.rados.rados.0.smithi029.stdout:773: delete oid 41 current snap is 101 2024-04-12T11:17:35.509 INFO:tasks.rados.rados.0.smithi029.stdout:770: finishing write tid 3 to smithi02979277-19 2024-04-12T11:17:35.509 INFO:tasks.rados.rados.0.smithi029.stdout:770: finishing write tid 4 to smithi02979277-19 2024-04-12T11:17:35.509 INFO:tasks.rados.rados.0.smithi029.stdout:770: finishing write tid 5 to smithi02979277-19 2024-04-12T11:17:35.509 INFO:tasks.rados.rados.0.smithi029.stdout:770: oid 19 updating version 0 to 468 2024-04-12T11:17:35.509 INFO:tasks.rados.rados.0.smithi029.stdout:770: oid 19 updating version 468 to 470 2024-04-12T11:17:35.509 INFO:tasks.rados.rados.0.smithi029.stdout:770: oid 19 version 470 is already newer than 469 2024-04-12T11:17:35.509 INFO:tasks.rados.rados.0.smithi029.stdout:770: oid 19 version 470 is already newer than 467 2024-04-12T11:17:35.509 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 470 (ObjNum 323 snap 101 seq_num 323) dirty exists 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:770: left oid 19 (ObjNum 323 snap 101 seq_num 323) 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:771: finishing write tid 1 to smithi02979277-44 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:771: finishing write tid 2 to smithi02979277-44 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:771: finishing write tid 3 to smithi02979277-44 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:771: finishing write tid 4 to smithi02979277-44 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:771: finishing write tid 5 to smithi02979277-44 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:771: oid 44 updating version 0 to 375 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:771: oid 44 updating version 375 to 378 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:771: oid 44 version 378 is already newer than 377 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:771: oid 44 version 378 is already newer than 376 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 378 (ObjNum 324 snap 101 seq_num 324) dirty exists 2024-04-12T11:17:35.510 INFO:tasks.rados.rados.0.smithi029.stdout:771: left oid 44 (ObjNum 324 snap 101 seq_num 324) 2024-04-12T11:17:35.516 INFO:tasks.rados.rados.0.smithi029.stdout:770: done (2 left) 2024-04-12T11:17:35.516 INFO:tasks.rados.rados.0.smithi029.stdout:771: done (1 left) 2024-04-12T11:17:35.516 INFO:tasks.rados.rados.0.smithi029.stdout:773: done (0 left) 2024-04-12T11:17:35.516 INFO:tasks.rados.rados.0.smithi029.stdout:774: snap_remove snap 96 2024-04-12T11:17:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:35 smithi029 ceph-mon[93650]: osdmap e261: 8 total, 8 up, 8 in 2024-04-12T11:17:35.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:35 smithi029 ceph-mon[39999]: osdmap e261: 8 total, 8 up, 8 in 2024-04-12T11:17:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:35 smithi087 ceph-mon[32917]: osdmap e261: 8 total, 8 up, 8 in 2024-04-12T11:17:36.510 INFO:tasks.rados.rados.0.smithi029.stdout:774: done (0 left) 2024-04-12T11:17:36.510 INFO:tasks.rados.rados.0.smithi029.stdout:775: write oid 24 current snap is 101 2024-04-12T11:17:36.510 INFO:tasks.rados.rados.0.smithi029.stdout:775: seq_num 326 ranges {679082=618000,1843718=569105,2853072=626010} 2024-04-12T11:17:36.518 INFO:tasks.rados.rados.0.smithi029.stdout:775: writing smithi02979277-24 from 679082 to 1297082 tid 1 2024-04-12T11:17:36.524 INFO:tasks.rados.rados.0.smithi029.stdout:775: writing smithi02979277-24 from 1843718 to 2412823 tid 2 2024-04-12T11:17:36.531 INFO:tasks.rados.rados.0.smithi029.stdout:775: writing smithi02979277-24 from 2853072 to 3479082 tid 3 2024-04-12T11:17:36.531 INFO:tasks.rados.rados.0.smithi029.stdout:776: read oid 19 snap 90 2024-04-12T11:17:36.531 INFO:tasks.rados.rados.0.smithi029.stdout:776: expect deleted 2024-04-12T11:17:36.531 INFO:tasks.rados.rados.0.smithi029.stdout:777: read oid 3 snap 99 2024-04-12T11:17:36.531 INFO:tasks.rados.rados.0.smithi029.stdout:777: expect deleted 2024-04-12T11:17:36.531 INFO:tasks.rados.rados.0.smithi029.stdout:778: read oid 43 snap -1 2024-04-12T11:17:36.532 INFO:tasks.rados.rados.0.smithi029.stdout:778: expect (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:17:36.532 INFO:tasks.rados.rados.0.smithi029.stdout:779: read oid 46 snap -1 2024-04-12T11:17:36.532 INFO:tasks.rados.rados.0.smithi029.stdout:779: expect (ObjNum 261 snap 82 seq_num 261) 2024-04-12T11:17:36.532 INFO:tasks.rados.rados.0.smithi029.stdout:780: rollback oid 4 current snap is 101 2024-04-12T11:17:36.532 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 4 to 80 2024-04-12T11:17:36.532 INFO:tasks.rados.rados.0.smithi029.stdout:775: finishing write tid 1 to smithi02979277-24 2024-04-12T11:17:36.532 INFO:tasks.rados.rados.0.smithi029.stdout:775: finishing write tid 2 to smithi02979277-24 2024-04-12T11:17:36.532 INFO:tasks.rados.rados.0.smithi029.stdout:781: write oid 18 current snap is 101 2024-04-12T11:17:36.532 INFO:tasks.rados.rados.0.smithi029.stdout:781: seq_num 327 ranges {695889=409602,1618553=712763,2996848=499041} 2024-04-12T11:17:36.537 INFO:tasks.rados.rados.0.smithi029.stdout:781: writing smithi02979277-18 from 695889 to 1105491 tid 1 2024-04-12T11:17:36.543 INFO:tasks.rados.rados.0.smithi029.stdout:781: writing smithi02979277-18 from 1618553 to 2331316 tid 2 2024-04-12T11:17:36.550 INFO:tasks.rados.rados.0.smithi029.stdout:781: writing smithi02979277-18 from 2996848 to 3495889 tid 3 2024-04-12T11:17:36.550 INFO:tasks.rados.rados.0.smithi029.stdout:782: snap_create 2024-04-12T11:17:36.550 INFO:tasks.rados.rados.0.smithi029.stdout:779: expect (ObjNum 261 snap 82 seq_num 261) 2024-04-12T11:17:36.661 INFO:tasks.rados.rados.0.smithi029.stdout:780: finishing rollback tid 0 to smithi02979277-4 2024-04-12T11:17:36.661 INFO:tasks.rados.rados.0.smithi029.stdout:780: finishing rollback tid 1 to smithi02979277-4 2024-04-12T11:17:36.661 INFO:tasks.rados.rados.0.smithi029.stdout:780: finishing rollback tid 2 to smithi02979277-4 2024-04-12T11:17:36.661 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 478 (ObjNum 246 snap 75 seq_num 246) dirty exists 2024-04-12T11:17:36.661 INFO:tasks.rados.rados.0.smithi029.stdout:778: expect (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:17:36.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:36 smithi029 ceph-mon[39999]: pgmap v23: 89 pgs: 89 active+clean; 191 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-04-12T11:17:36.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:36 smithi029 ceph-mon[39999]: osdmap e262: 8 total, 8 up, 8 in 2024-04-12T11:17:36.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:36 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:36 smithi029 ceph-mon[93650]: pgmap v23: 89 pgs: 89 active+clean; 191 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-04-12T11:17:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:36 smithi029 ceph-mon[93650]: osdmap e262: 8 total, 8 up, 8 in 2024-04-12T11:17:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:36.865 INFO:tasks.rados.rados.0.smithi029.stdout:775: finishing write tid 3 to smithi02979277-24 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:775: finishing write tid 4 to smithi02979277-24 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:775: finishing write tid 5 to smithi02979277-24 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:775: oid 24 updating version 0 to 480 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:775: oid 24 version 480 is already newer than 478 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:775: oid 24 updating version 480 to 481 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:775: oid 24 version 481 is already newer than 479 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 481 (ObjNum 326 snap 101 seq_num 326) dirty exists 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:775: left oid 24 (ObjNum 326 snap 101 seq_num 326) 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:781: finishing write tid 1 to smithi02979277-18 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:781: finishing write tid 2 to smithi02979277-18 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:781: finishing write tid 3 to smithi02979277-18 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:781: finishing write tid 4 to smithi02979277-18 2024-04-12T11:17:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:781: finishing write tid 5 to smithi02979277-18 2024-04-12T11:17:36.867 INFO:tasks.rados.rados.0.smithi029.stdout:781: oid 18 updating version 0 to 441 2024-04-12T11:17:36.867 INFO:tasks.rados.rados.0.smithi029.stdout:781: oid 18 version 441 is already newer than 439 2024-04-12T11:17:36.867 INFO:tasks.rados.rados.0.smithi029.stdout:781: oid 18 version 441 is already newer than 440 2024-04-12T11:17:36.867 INFO:tasks.rados.rados.0.smithi029.stdout:781: oid 18 version 441 is already newer than 438 2024-04-12T11:17:36.867 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 441 (ObjNum 327 snap 101 seq_num 327) dirty exists 2024-04-12T11:17:36.867 INFO:tasks.rados.rados.0.smithi029.stdout:781: left oid 18 (ObjNum 327 snap 101 seq_num 327) 2024-04-12T11:17:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:36 smithi087 ceph-mon[32917]: pgmap v23: 89 pgs: 89 active+clean; 191 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-04-12T11:17:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:36 smithi087 ceph-mon[32917]: osdmap e262: 8 total, 8 up, 8 in 2024-04-12T11:17:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:36 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:37.523 INFO:tasks.rados.rados.0.smithi029.stdout:775: done (7 left) 2024-04-12T11:17:37.523 INFO:tasks.rados.rados.0.smithi029.stdout:776: done (6 left) 2024-04-12T11:17:37.523 INFO:tasks.rados.rados.0.smithi029.stdout:777: done (5 left) 2024-04-12T11:17:37.523 INFO:tasks.rados.rados.0.smithi029.stdout:778: done (4 left) 2024-04-12T11:17:37.523 INFO:tasks.rados.rados.0.smithi029.stdout:779: done (3 left) 2024-04-12T11:17:37.523 INFO:tasks.rados.rados.0.smithi029.stdout:780: done (2 left) 2024-04-12T11:17:37.523 INFO:tasks.rados.rados.0.smithi029.stdout:781: done (1 left) 2024-04-12T11:17:37.523 INFO:tasks.rados.rados.0.smithi029.stdout:782: done (0 left) 2024-04-12T11:17:37.523 INFO:tasks.rados.rados.0.smithi029.stdout:783: snap_remove snap 78 2024-04-12T11:17:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:37 smithi087 ceph-mon[32917]: osdmap e263: 8 total, 8 up, 8 in 2024-04-12T11:17:38.028 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:37 smithi029 ceph-mon[93650]: osdmap e263: 8 total, 8 up, 8 in 2024-04-12T11:17:38.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:37 smithi029 ceph-mon[39999]: osdmap e263: 8 total, 8 up, 8 in 2024-04-12T11:17:38.525 INFO:tasks.rados.rados.0.smithi029.stdout:783: done (0 left) 2024-04-12T11:17:38.525 INFO:tasks.rados.rados.0.smithi029.stdout:784: read oid 47 snap 97 2024-04-12T11:17:38.525 INFO:tasks.rados.rados.0.smithi029.stdout:784: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:17:38.525 INFO:tasks.rados.rados.0.smithi029.stdout:785: read oid 28 snap 86 2024-04-12T11:17:38.525 INFO:tasks.rados.rados.0.smithi029.stdout:785: expect (ObjNum 173 snap 46 seq_num 173) 2024-04-12T11:17:38.525 INFO:tasks.rados.rados.0.smithi029.stdout:786: snap_remove snap 88 2024-04-12T11:17:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:38 smithi087 ceph-mon[32917]: pgmap v26: 89 pgs: 89 active+clean; 191 MiB data, 631 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:38 smithi087 ceph-mon[32917]: osdmap e264: 8 total, 8 up, 8 in 2024-04-12T11:17:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:38 smithi029 ceph-mon[39999]: pgmap v26: 89 pgs: 89 active+clean; 191 MiB data, 631 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:38 smithi029 ceph-mon[39999]: osdmap e264: 8 total, 8 up, 8 in 2024-04-12T11:17:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:38 smithi029 ceph-mon[93650]: pgmap v26: 89 pgs: 89 active+clean; 191 MiB data, 631 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:17:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:38 smithi029 ceph-mon[93650]: osdmap e264: 8 total, 8 up, 8 in 2024-04-12T11:17:39.538 INFO:tasks.rados.rados.0.smithi029.stdout:786: done (2 left) 2024-04-12T11:17:39.538 INFO:tasks.rados.rados.0.smithi029.stdout:787: write oid 36 current snap is 102 2024-04-12T11:17:39.539 INFO:tasks.rados.rados.0.smithi029.stdout:787: seq_num 328 ranges {712696=484851,1960682=572774,3307918=204778} 2024-04-12T11:17:39.544 INFO:tasks.rados.rados.0.smithi029.stdout:787: writing smithi02979277-36 from 712696 to 1197547 tid 1 2024-04-12T11:17:39.552 INFO:tasks.rados.rados.0.smithi029.stdout:787: writing smithi02979277-36 from 1960682 to 2533456 tid 2 2024-04-12T11:17:39.558 INFO:tasks.rados.rados.0.smithi029.stdout:787: writing smithi02979277-36 from 3307918 to 3512696 tid 3 2024-04-12T11:17:39.558 INFO:tasks.rados.rados.0.smithi029.stdout:788: snap_create 2024-04-12T11:17:39.558 INFO:tasks.rados.rados.0.smithi029.stdout:784: expect (ObjNum 79 snap 10 seq_num 79) 2024-04-12T11:17:39.713 INFO:tasks.rados.rados.0.smithi029.stdout:785: expect (ObjNum 173 snap 46 seq_num 173) 2024-04-12T11:17:39.844 INFO:tasks.rados.rados.0.smithi029.stdout:787: finishing write tid 1 to smithi02979277-36 2024-04-12T11:17:39.844 INFO:tasks.rados.rados.0.smithi029.stdout:787: finishing write tid 2 to smithi02979277-36 2024-04-12T11:17:39.844 INFO:tasks.rados.rados.0.smithi029.stdout:787: finishing write tid 3 to smithi02979277-36 2024-04-12T11:17:39.845 INFO:tasks.rados.rados.0.smithi029.stdout:787: finishing write tid 4 to smithi02979277-36 2024-04-12T11:17:39.845 INFO:tasks.rados.rados.0.smithi029.stdout:787: finishing write tid 5 to smithi02979277-36 2024-04-12T11:17:39.845 INFO:tasks.rados.rados.0.smithi029.stdout:787: oid 36 updating version 0 to 185 2024-04-12T11:17:39.846 INFO:tasks.rados.rados.0.smithi029.stdout:787: oid 36 updating version 185 to 186 2024-04-12T11:17:39.846 INFO:tasks.rados.rados.0.smithi029.stdout:787: oid 36 version 186 is already newer than 183 2024-04-12T11:17:39.846 INFO:tasks.rados.rados.0.smithi029.stdout:787: oid 36 version 186 is already newer than 184 2024-04-12T11:17:39.846 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 186 (ObjNum 328 snap 102 seq_num 328) dirty exists 2024-04-12T11:17:39.847 INFO:tasks.rados.rados.0.smithi029.stdout:787: left oid 36 (ObjNum 328 snap 102 seq_num 328) 2024-04-12T11:17:39.878 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:39 smithi087 ceph-mon[32917]: osdmap e265: 8 total, 8 up, 8 in 2024-04-12T11:17:40.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:39 smithi029 ceph-mon[93650]: osdmap e265: 8 total, 8 up, 8 in 2024-04-12T11:17:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:39 smithi029 ceph-mon[39999]: osdmap e265: 8 total, 8 up, 8 in 2024-04-12T11:17:40.543 INFO:tasks.rados.rados.0.smithi029.stdout:784: done (3 left) 2024-04-12T11:17:40.543 INFO:tasks.rados.rados.0.smithi029.stdout:785: done (2 left) 2024-04-12T11:17:40.543 INFO:tasks.rados.rados.0.smithi029.stdout:787: done (1 left) 2024-04-12T11:17:40.543 INFO:tasks.rados.rados.0.smithi029.stdout:788: done (0 left) 2024-04-12T11:17:40.544 INFO:tasks.rados.rados.0.smithi029.stdout:789: delete oid 3 current snap is 103 2024-04-12T11:17:40.545 INFO:tasks.rados.rados.0.smithi029.stdout:789: done (0 left) 2024-04-12T11:17:40.545 INFO:tasks.rados.rados.0.smithi029.stdout:790: delete oid 4 current snap is 103 2024-04-12T11:17:40.549 INFO:tasks.rados.rados.0.smithi029.stdout:790: done (0 left) 2024-04-12T11:17:40.549 INFO:tasks.rados.rados.0.smithi029.stdout:791: read oid 46 snap 82 2024-04-12T11:17:40.549 INFO:tasks.rados.rados.0.smithi029.stdout:791: expect (ObjNum 261 snap 82 seq_num 261) 2024-04-12T11:17:40.549 INFO:tasks.rados.rados.0.smithi029.stdout:792: delete oid 9 current snap is 103 2024-04-12T11:17:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:792: done (1 left) 2024-04-12T11:17:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:793: read oid 3 snap 101 2024-04-12T11:17:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:793: expect deleted 2024-04-12T11:17:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:794: read oid 39 snap -1 2024-04-12T11:17:40.552 INFO:tasks.rados.rados.0.smithi029.stdout:794: expect deleted 2024-04-12T11:17:40.552 INFO:tasks.rados.rados.0.smithi029.stdout:795: snap_remove snap 80 2024-04-12T11:17:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:40 smithi087 ceph-mon[32917]: pgmap v29: 89 pgs: 89 active+clean; 203 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-04-12T11:17:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:40 smithi087 ceph-mon[32917]: osdmap e266: 8 total, 8 up, 8 in 2024-04-12T11:17:41.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:40 smithi029 ceph-mon[93650]: pgmap v29: 89 pgs: 89 active+clean; 203 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-04-12T11:17:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:40 smithi029 ceph-mon[93650]: osdmap e266: 8 total, 8 up, 8 in 2024-04-12T11:17:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:40 smithi029 ceph-mon[39999]: pgmap v29: 89 pgs: 89 active+clean; 203 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-04-12T11:17:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:40 smithi029 ceph-mon[39999]: osdmap e266: 8 total, 8 up, 8 in 2024-04-12T11:17:41.550 INFO:tasks.rados.rados.0.smithi029.stdout:795: done (3 left) 2024-04-12T11:17:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:796: rollback oid 26 current snap is 103 2024-04-12T11:17:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 86 2024-04-12T11:17:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:797: write oid 2 current snap is 103 2024-04-12T11:17:41.551 INFO:tasks.rados.rados.0.smithi029.stdout:791: expect (ObjNum 261 snap 82 seq_num 261) 2024-04-12T11:17:41.664 INFO:tasks.rados.rados.0.smithi029.stdout:796: finishing rollback tid 0 to smithi02979277-26 2024-04-12T11:17:41.664 INFO:tasks.rados.rados.0.smithi029.stdout:796: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:17:41.664 INFO:tasks.rados.rados.0.smithi029.stdout:797: seq_num 332 ranges {779924=785847,2078257=580112,3301257=278667} 2024-04-12T11:17:41.672 INFO:tasks.rados.rados.0.smithi029.stdout:797: writing smithi02979277-2 from 779924 to 1565771 tid 1 2024-04-12T11:17:41.678 INFO:tasks.rados.rados.0.smithi029.stdout:797: writing smithi02979277-2 from 2078257 to 2658369 tid 2 2024-04-12T11:17:41.683 INFO:tasks.rados.rados.0.smithi029.stdout:797: writing smithi02979277-2 from 3301257 to 3579924 tid 3 2024-04-12T11:17:41.683 INFO:tasks.rados.rados.0.smithi029.stdout:791: done (4 left) 2024-04-12T11:17:41.683 INFO:tasks.rados.rados.0.smithi029.stdout:793: done (3 left) 2024-04-12T11:17:41.684 INFO:tasks.rados.rados.0.smithi029.stdout:794: done (2 left) 2024-04-12T11:17:41.684 INFO:tasks.rados.rados.0.smithi029.stdout:798: snap_remove snap 97 2024-04-12T11:17:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:41 smithi087 ceph-mon[32917]: osdmap e267: 8 total, 8 up, 8 in 2024-04-12T11:17:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:41 smithi029 ceph-mon[93650]: osdmap e267: 8 total, 8 up, 8 in 2024-04-12T11:17:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:41 smithi029 ceph-mon[39999]: osdmap e267: 8 total, 8 up, 8 in 2024-04-12T11:17:42.499 INFO:tasks.rados.rados.0.smithi029.stdout:798: done (2 left) 2024-04-12T11:17:42.499 INFO:tasks.rados.rados.0.smithi029.stdout:799: write oid 19 current snap is 103 2024-04-12T11:17:42.499 INFO:tasks.rados.rados.0.smithi029.stdout:796: finishing rollback tid 2 to smithi02979277-26 2024-04-12T11:17:42.499 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 494 (ObjNum 135 snap 28 seq_num 135) dirty exists 2024-04-12T11:17:42.499 INFO:tasks.rados.rados.0.smithi029.stdout:799: seq_num 333 ranges {796731=461096,1736739=723770,2928680=668051} 2024-04-12T11:17:42.506 INFO:tasks.rados.rados.0.smithi029.stdout:799: writing smithi02979277-19 from 796731 to 1257827 tid 1 2024-04-12T11:17:42.514 INFO:tasks.rados.rados.0.smithi029.stdout:799: writing smithi02979277-19 from 1736739 to 2460509 tid 2 2024-04-12T11:17:42.520 INFO:tasks.rados.rados.0.smithi029.stdout:799: writing smithi02979277-19 from 2928680 to 3596731 tid 3 2024-04-12T11:17:42.521 INFO:tasks.rados.rados.0.smithi029.stdout:796: done (2 left) 2024-04-12T11:17:42.521 INFO:tasks.rados.rados.0.smithi029.stdout:800: read oid 20 snap 99 2024-04-12T11:17:42.521 INFO:tasks.rados.rados.0.smithi029.stdout:800: expect deleted 2024-04-12T11:17:42.521 INFO:tasks.rados.rados.0.smithi029.stdout:801: snap_remove snap 90 2024-04-12T11:17:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:17:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:17:42.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:17:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:42 smithi087 ceph-mon[32917]: pgmap v32: 89 pgs: 89 active+clean; 203 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-04-12T11:17:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:42 smithi087 ceph-mon[32917]: osdmap e268: 8 total, 8 up, 8 in 2024-04-12T11:17:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:42 smithi087 ceph-mon[32917]: osdmap e269: 8 total, 8 up, 8 in 2024-04-12T11:17:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:42 smithi029 ceph-mon[93650]: pgmap v32: 89 pgs: 89 active+clean; 203 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-04-12T11:17:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:42 smithi029 ceph-mon[93650]: osdmap e268: 8 total, 8 up, 8 in 2024-04-12T11:17:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:42 smithi029 ceph-mon[93650]: osdmap e269: 8 total, 8 up, 8 in 2024-04-12T11:17:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:42 smithi029 ceph-mon[39999]: pgmap v32: 89 pgs: 89 active+clean; 203 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-04-12T11:17:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:42 smithi029 ceph-mon[39999]: osdmap e268: 8 total, 8 up, 8 in 2024-04-12T11:17:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:42 smithi029 ceph-mon[39999]: osdmap e269: 8 total, 8 up, 8 in 2024-04-12T11:17:43.501 INFO:tasks.rados.rados.0.smithi029.stdout:801: done (3 left) 2024-04-12T11:17:43.501 INFO:tasks.rados.rados.0.smithi029.stdout:802: write oid 46 current snap is 103 2024-04-12T11:17:43.501 INFO:tasks.rados.rados.0.smithi029.stdout:802: seq_num 334 ranges {413538=536345,1395221=583781,2556103=658617} 2024-04-12T11:17:43.506 INFO:tasks.rados.rados.0.smithi029.stdout:802: writing smithi02979277-46 from 413538 to 949883 tid 1 2024-04-12T11:17:43.512 INFO:tasks.rados.rados.0.smithi029.stdout:802: writing smithi02979277-46 from 1395221 to 1979002 tid 2 2024-04-12T11:17:43.519 INFO:tasks.rados.rados.0.smithi029.stdout:802: writing smithi02979277-46 from 2556103 to 3214720 tid 3 2024-04-12T11:17:43.519 INFO:tasks.rados.rados.0.smithi029.stdout:803: write oid 10 current snap is 103 2024-04-12T11:17:43.520 INFO:tasks.rados.rados.0.smithi029.stdout:803: seq_num 335 ranges {430345=727947,1570056=727439,2699879=469889} 2024-04-12T11:17:43.526 INFO:tasks.rados.rados.0.smithi029.stdout:803: writing smithi02979277-10 from 430345 to 1158292 tid 1 2024-04-12T11:17:43.532 INFO:tasks.rados.rados.0.smithi029.stdout:803: writing smithi02979277-10 from 1570056 to 2297495 tid 2 2024-04-12T11:17:43.538 INFO:tasks.rados.rados.0.smithi029.stdout:803: writing smithi02979277-10 from 2699879 to 3169768 tid 3 2024-04-12T11:17:43.538 INFO:tasks.rados.rados.0.smithi029.stdout:804: snap_create 2024-04-12T11:17:43.538 INFO:tasks.rados.rados.0.smithi029.stdout:797: finishing write tid 1 to smithi02979277-2 2024-04-12T11:17:43.538 INFO:tasks.rados.rados.0.smithi029.stdout:797: finishing write tid 2 to smithi02979277-2 2024-04-12T11:17:43.538 INFO:tasks.rados.rados.0.smithi029.stdout:797: finishing write tid 3 to smithi02979277-2 2024-04-12T11:17:43.538 INFO:tasks.rados.rados.0.smithi029.stdout:797: finishing write tid 4 to smithi02979277-2 2024-04-12T11:17:43.539 INFO:tasks.rados.rados.0.smithi029.stdout:797: finishing write tid 5 to smithi02979277-2 2024-04-12T11:17:43.539 INFO:tasks.rados.rados.0.smithi029.stdout:797: oid 2 updating version 0 to 19 2024-04-12T11:17:43.539 INFO:tasks.rados.rados.0.smithi029.stdout:797: oid 2 version 19 is already newer than 16 2024-04-12T11:17:43.539 INFO:tasks.rados.rados.0.smithi029.stdout:797: oid 2 version 19 is already newer than 17 2024-04-12T11:17:43.539 INFO:tasks.rados.rados.0.smithi029.stdout:797: oid 2 version 19 is already newer than 18 2024-04-12T11:17:43.539 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 19 (ObjNum 332 snap 103 seq_num 332) dirty exists 2024-04-12T11:17:43.539 INFO:tasks.rados.rados.0.smithi029.stdout:797: left oid 2 (ObjNum 332 snap 103 seq_num 332) 2024-04-12T11:17:43.539 INFO:tasks.rados.rados.0.smithi029.stdout:799: finishing write tid 1 to smithi02979277-19 2024-04-12T11:17:43.540 INFO:tasks.rados.rados.0.smithi029.stdout:799: finishing write tid 2 to smithi02979277-19 2024-04-12T11:17:43.540 INFO:tasks.rados.rados.0.smithi029.stdout:799: finishing write tid 3 to smithi02979277-19 2024-04-12T11:17:43.540 INFO:tasks.rados.rados.0.smithi029.stdout:799: finishing write tid 4 to smithi02979277-19 2024-04-12T11:17:43.540 INFO:tasks.rados.rados.0.smithi029.stdout:799: finishing write tid 5 to smithi02979277-19 2024-04-12T11:17:43.540 INFO:tasks.rados.rados.0.smithi029.stdout:799: oid 19 updating version 0 to 503 2024-04-12T11:17:43.540 INFO:tasks.rados.rados.0.smithi029.stdout:799: oid 19 version 503 is already newer than 501 2024-04-12T11:17:43.540 INFO:tasks.rados.rados.0.smithi029.stdout:799: oid 19 version 503 is already newer than 500 2024-04-12T11:17:43.540 INFO:tasks.rados.rados.0.smithi029.stdout:799: oid 19 version 503 is already newer than 502 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 503 (ObjNum 333 snap 103 seq_num 333) dirty exists 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:799: left oid 19 (ObjNum 333 snap 103 seq_num 333) 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:802: finishing write tid 1 to smithi02979277-46 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:802: finishing write tid 2 to smithi02979277-46 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:802: finishing write tid 3 to smithi02979277-46 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:802: finishing write tid 4 to smithi02979277-46 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:802: finishing write tid 5 to smithi02979277-46 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:802: oid 46 updating version 0 to 181 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:802: oid 46 version 181 is already newer than 179 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:802: oid 46 version 181 is already newer than 176 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:802: oid 46 version 181 is already newer than 180 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 181 (ObjNum 334 snap 103 seq_num 334) dirty exists 2024-04-12T11:17:43.541 INFO:tasks.rados.rados.0.smithi029.stdout:802: left oid 46 (ObjNum 334 snap 103 seq_num 334) 2024-04-12T11:17:43.542 INFO:tasks.rados.rados.0.smithi029.stdout:803: finishing write tid 1 to smithi02979277-10 2024-04-12T11:17:43.542 INFO:tasks.rados.rados.0.smithi029.stdout:803: finishing write tid 2 to smithi02979277-10 2024-04-12T11:17:43.542 INFO:tasks.rados.rados.0.smithi029.stdout:803: finishing write tid 3 to smithi02979277-10 2024-04-12T11:17:43.543 INFO:tasks.rados.rados.0.smithi029.stdout:803: finishing write tid 4 to smithi02979277-10 2024-04-12T11:17:43.543 INFO:tasks.rados.rados.0.smithi029.stdout:803: finishing write tid 5 to smithi02979277-10 2024-04-12T11:17:43.543 INFO:tasks.rados.rados.0.smithi029.stdout:803: oid 10 updating version 0 to 324 2024-04-12T11:17:43.543 INFO:tasks.rados.rados.0.smithi029.stdout:803: oid 10 version 324 is already newer than 322 2024-04-12T11:17:43.543 INFO:tasks.rados.rados.0.smithi029.stdout:803: oid 10 version 324 is already newer than 323 2024-04-12T11:17:43.543 INFO:tasks.rados.rados.0.smithi029.stdout:803: oid 10 version 324 is already newer than 321 2024-04-12T11:17:43.543 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 324 (ObjNum 335 snap 103 seq_num 335) dirty exists 2024-04-12T11:17:43.543 INFO:tasks.rados.rados.0.smithi029.stdout:803: left oid 10 (ObjNum 335 snap 103 seq_num 335) 2024-04-12T11:17:43.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:17:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:17:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:17:44.502 INFO:tasks.rados.rados.0.smithi029.stdout:797: done (5 left) 2024-04-12T11:17:44.502 INFO:tasks.rados.rados.0.smithi029.stdout:799: done (4 left) 2024-04-12T11:17:44.502 INFO:tasks.rados.rados.0.smithi029.stdout:800: done (3 left) 2024-04-12T11:17:44.502 INFO:tasks.rados.rados.0.smithi029.stdout:802: done (2 left) 2024-04-12T11:17:44.502 INFO:tasks.rados.rados.0.smithi029.stdout:803: done (1 left) 2024-04-12T11:17:44.502 INFO:tasks.rados.rados.0.smithi029.stdout:804: done (0 left) 2024-04-12T11:17:44.502 INFO:tasks.rados.rados.0.smithi029.stdout:805: delete oid 35 current snap is 104 2024-04-12T11:17:44.507 INFO:tasks.rados.rados.0.smithi029.stdout:805: done (0 left) 2024-04-12T11:17:44.507 INFO:tasks.rados.rados.0.smithi029.stdout:806: write oid 17 current snap is 104 2024-04-12T11:17:44.507 INFO:tasks.rados.rados.0.smithi029.stdout:806: seq_num 337 ranges {463959=478445,1570667=731108,3038372=492433} 2024-04-12T11:17:44.512 INFO:tasks.rados.rados.0.smithi029.stdout:806: writing smithi02979277-17 from 463959 to 942404 tid 1 2024-04-12T11:17:44.519 INFO:tasks.rados.rados.0.smithi029.stdout:806: writing smithi02979277-17 from 1570667 to 2301775 tid 2 2024-04-12T11:17:44.526 INFO:tasks.rados.rados.0.smithi029.stdout:806: writing smithi02979277-17 from 3038372 to 3530805 tid 3 2024-04-12T11:17:44.526 INFO:tasks.rados.rados.0.smithi029.stdout:807: read oid 50 snap -1 2024-04-12T11:17:44.527 INFO:tasks.rados.rados.0.smithi029.stdout:807: expect deleted 2024-04-12T11:17:44.527 INFO:tasks.rados.rados.0.smithi029.stdout:808: write oid 1 current snap is 104 2024-04-12T11:17:44.527 INFO:tasks.rados.rados.0.smithi029.stdout:806: finishing write tid 1 to smithi02979277-17 2024-04-12T11:17:44.527 INFO:tasks.rados.rados.0.smithi029.stdout:806: finishing write tid 2 to smithi02979277-17 2024-04-12T11:17:44.527 INFO:tasks.rados.rados.0.smithi029.stdout:808: seq_num 338 ranges {480766=553694,1629149=591119,2782148=420058} 2024-04-12T11:17:44.532 INFO:tasks.rados.rados.0.smithi029.stdout:808: writing smithi02979277-1 from 480766 to 1034460 tid 1 2024-04-12T11:17:44.539 INFO:tasks.rados.rados.0.smithi029.stdout:808: writing smithi02979277-1 from 1629149 to 2220268 tid 2 2024-04-12T11:17:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:808: writing smithi02979277-1 from 2782148 to 3202206 tid 3 2024-04-12T11:17:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:809: snap_remove snap 82 2024-04-12T11:17:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:44 smithi029 ceph-mon[93650]: pgmap v35: 89 pgs: 89 active+clean; 184 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 710 KiB/s wr, 10 op/s 2024-04-12T11:17:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:44 smithi029 ceph-mon[93650]: osdmap e270: 8 total, 8 up, 8 in 2024-04-12T11:17:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:44 smithi029 ceph-mon[39999]: pgmap v35: 89 pgs: 89 active+clean; 184 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 710 KiB/s wr, 10 op/s 2024-04-12T11:17:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:44 smithi029 ceph-mon[39999]: osdmap e270: 8 total, 8 up, 8 in 2024-04-12T11:17:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:44 smithi087 ceph-mon[32917]: pgmap v35: 89 pgs: 89 active+clean; 184 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 710 KiB/s wr, 10 op/s 2024-04-12T11:17:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:44 smithi087 ceph-mon[32917]: osdmap e270: 8 total, 8 up, 8 in 2024-04-12T11:17:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:17:44] "GET /metrics HTTP/1.1" 200 36246 "" "Prometheus/2.43.0" 2024-04-12T11:17:45.515 INFO:tasks.rados.rados.0.smithi029.stdout:809: done (3 left) 2024-04-12T11:17:45.515 INFO:tasks.rados.rados.0.smithi029.stdout:810: read oid 14 snap -1 2024-04-12T11:17:45.515 INFO:tasks.rados.rados.0.smithi029.stdout:810: expect (ObjNum 321 snap 98 seq_num 321) 2024-04-12T11:17:45.515 INFO:tasks.rados.rados.0.smithi029.stdout:811: snap_remove snap 84 2024-04-12T11:17:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:45 smithi029 ceph-mon[93650]: osdmap e271: 8 total, 8 up, 8 in 2024-04-12T11:17:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:45 smithi029 ceph-mon[39999]: osdmap e271: 8 total, 8 up, 8 in 2024-04-12T11:17:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:45 smithi087 ceph-mon[32917]: osdmap e271: 8 total, 8 up, 8 in 2024-04-12T11:17:46.522 INFO:tasks.rados.rados.0.smithi029.stdout:811: done (4 left) 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:812: rollback oid 7 current snap is 104 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 89 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:806: finishing write tid 3 to smithi02979277-17 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:806: finishing write tid 4 to smithi02979277-17 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:806: finishing write tid 5 to smithi02979277-17 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:806: oid 17 updating version 0 to 473 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:806: oid 17 updating version 473 to 474 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:806: oid 17 updating version 474 to 475 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:806: oid 17 version 475 is already newer than 472 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 475 (ObjNum 337 snap 104 seq_num 337) dirty exists 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:806: left oid 17 (ObjNum 337 snap 104 seq_num 337) 2024-04-12T11:17:46.523 INFO:tasks.rados.rados.0.smithi029.stdout:808: finishing write tid 1 to smithi02979277-1 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:808: finishing write tid 2 to smithi02979277-1 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:808: finishing write tid 3 to smithi02979277-1 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:808: finishing write tid 4 to smithi02979277-1 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:808: finishing write tid 5 to smithi02979277-1 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:808: oid 1 updating version 0 to 419 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:808: oid 1 version 419 is already newer than 417 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:808: oid 1 version 419 is already newer than 418 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:808: oid 1 updating version 419 to 420 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 420 (ObjNum 338 snap 104 seq_num 338) dirty exists 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:808: left oid 1 (ObjNum 338 snap 104 seq_num 338) 2024-04-12T11:17:46.524 INFO:tasks.rados.rados.0.smithi029.stdout:810: expect (ObjNum 321 snap 98 seq_num 321) 2024-04-12T11:17:46.682 INFO:tasks.rados.rados.0.smithi029.stdout:812: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 339 (ObjNum 529 snap 0 seq_num 1800835232) dirty dne 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:806: done (4 left) 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:807: done (3 left) 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:808: done (2 left) 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:810: done (1 left) 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:812: done (0 left) 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:813: read oid 19 snap 92 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:813: expect deleted 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:814: rollback oid 47 current snap is 104 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 89 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:815: rollback oid 9 current snap is 104 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 9 to 102 2024-04-12T11:17:46.683 INFO:tasks.rados.rados.0.smithi029.stdout:816: snap_remove snap 95 2024-04-12T11:17:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:46 smithi029 ceph-mon[93650]: pgmap v38: 89 pgs: 89 active+clean; 184 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 710 KiB/s wr, 10 op/s 2024-04-12T11:17:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:46 smithi029 ceph-mon[93650]: osdmap e272: 8 total, 8 up, 8 in 2024-04-12T11:17:46.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:46 smithi029 ceph-mon[39999]: pgmap v38: 89 pgs: 89 active+clean; 184 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 710 KiB/s wr, 10 op/s 2024-04-12T11:17:46.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:46 smithi029 ceph-mon[39999]: osdmap e272: 8 total, 8 up, 8 in 2024-04-12T11:17:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:46 smithi087 ceph-mon[32917]: pgmap v38: 89 pgs: 89 active+clean; 184 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 710 KiB/s wr, 10 op/s 2024-04-12T11:17:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:46 smithi087 ceph-mon[32917]: osdmap e272: 8 total, 8 up, 8 in 2024-04-12T11:17:47.528 INFO:tasks.rados.rados.0.smithi029.stdout:815: finishing rollback tid 1 to smithi02979277-9 2024-04-12T11:17:47.528 INFO:tasks.rados.rados.0.smithi029.stdout:815: finishing rollback tid 2 to smithi02979277-9 2024-04-12T11:17:47.528 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 347 (ObjNum 268 snap 86 seq_num 268) dirty exists 2024-04-12T11:17:47.528 INFO:tasks.rados.rados.0.smithi029.stdout:814: finishing rollback tid 0 to smithi02979277-47 2024-04-12T11:17:47.528 INFO:tasks.rados.rados.0.smithi029.stdout:814: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:17:47.528 INFO:tasks.rados.rados.0.smithi029.stdout:814: finishing rollback tid 2 to smithi02979277-47 2024-04-12T11:17:47.528 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 452 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-04-12T11:17:47.528 INFO:tasks.rados.rados.0.smithi029.stdout:813: done (3 left) 2024-04-12T11:17:47.528 INFO:tasks.rados.rados.0.smithi029.stdout:814: done (2 left) 2024-04-12T11:17:47.528 INFO:tasks.rados.rados.0.smithi029.stdout:815: done (1 left) 2024-04-12T11:17:47.529 INFO:tasks.rados.rados.0.smithi029.stdout:816: done (0 left) 2024-04-12T11:17:47.529 INFO:tasks.rados.rados.0.smithi029.stdout:817: snap_remove snap 89 2024-04-12T11:17:47.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:47 smithi087 ceph-mon[32917]: osdmap e273: 8 total, 8 up, 8 in 2024-04-12T11:17:48.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:47 smithi029 ceph-mon[93650]: osdmap e273: 8 total, 8 up, 8 in 2024-04-12T11:17:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:47 smithi029 ceph-mon[39999]: osdmap e273: 8 total, 8 up, 8 in 2024-04-12T11:17:48.531 INFO:tasks.rados.rados.0.smithi029.stdout:817: done (0 left) 2024-04-12T11:17:48.531 INFO:tasks.rados.rados.0.smithi029.stdout:818: delete oid 50 current snap is 104 2024-04-12T11:17:48.541 INFO:tasks.rados.rados.0.smithi029.stdout:818: done (0 left) 2024-04-12T11:17:48.541 INFO:tasks.rados.rados.0.smithi029.stdout:819: read oid 39 snap 103 2024-04-12T11:17:48.541 INFO:tasks.rados.rados.0.smithi029.stdout:819: expect deleted 2024-04-12T11:17:48.541 INFO:tasks.rados.rados.0.smithi029.stdout:820: read oid 43 snap 92 2024-04-12T11:17:48.541 INFO:tasks.rados.rados.0.smithi029.stdout:820: expect (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:17:48.541 INFO:tasks.rados.rados.0.smithi029.stdout:821: snap_create 2024-04-12T11:17:48.545 INFO:tasks.rados.rados.0.smithi029.stdout:820: expect (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:17:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:48 smithi087 ceph-mon[32917]: pgmap v41: 89 pgs: 89 active+clean; 190 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-04-12T11:17:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:48 smithi087 ceph-mon[32917]: osdmap e274: 8 total, 8 up, 8 in 2024-04-12T11:17:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:48 smithi029 ceph-mon[93650]: pgmap v41: 89 pgs: 89 active+clean; 190 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-04-12T11:17:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:48 smithi029 ceph-mon[93650]: osdmap e274: 8 total, 8 up, 8 in 2024-04-12T11:17:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:48 smithi029 ceph-mon[39999]: pgmap v41: 89 pgs: 89 active+clean; 190 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-04-12T11:17:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:48 smithi029 ceph-mon[39999]: osdmap e274: 8 total, 8 up, 8 in 2024-04-12T11:17:49.542 INFO:tasks.rados.rados.0.smithi029.stdout:819: done (2 left) 2024-04-12T11:17:49.542 INFO:tasks.rados.rados.0.smithi029.stdout:820: done (1 left) 2024-04-12T11:17:49.542 INFO:tasks.rados.rados.0.smithi029.stdout:821: done (0 left) 2024-04-12T11:17:49.542 INFO:tasks.rados.rados.0.smithi029.stdout:822: write oid 35 current snap is 105 2024-04-12T11:17:49.542 INFO:tasks.rados.rados.0.smithi029.stdout:822: seq_num 340 ranges {514380=704192,1629760=478435,2604288=442602} 2024-04-12T11:17:49.548 INFO:tasks.rados.rados.0.smithi029.stdout:822: writing smithi02979277-35 from 514380 to 1218572 tid 1 2024-04-12T11:17:49.553 INFO:tasks.rados.rados.0.smithi029.stdout:822: writing smithi02979277-35 from 1629760 to 2108195 tid 2 2024-04-12T11:17:49.559 INFO:tasks.rados.rados.0.smithi029.stdout:822: writing smithi02979277-35 from 2604288 to 3046890 tid 3 2024-04-12T11:17:49.559 INFO:tasks.rados.rados.0.smithi029.stdout:823: write oid 48 current snap is 105 2024-04-12T11:17:49.559 INFO:tasks.rados.rados.0.smithi029.stdout:822: finishing write tid 1 to smithi02979277-35 2024-04-12T11:17:49.559 INFO:tasks.rados.rados.0.smithi029.stdout:822: finishing write tid 2 to smithi02979277-35 2024-04-12T11:17:49.559 INFO:tasks.rados.rados.0.smithi029.stdout:823: seq_num 341 ranges {531187=779441,2088242=738446,3431711=299476} 2024-04-12T11:17:49.566 INFO:tasks.rados.rados.0.smithi029.stdout:823: writing smithi02979277-48 from 531187 to 1310628 tid 1 2024-04-12T11:17:49.574 INFO:tasks.rados.rados.0.smithi029.stdout:823: writing smithi02979277-48 from 2088242 to 2826688 tid 2 2024-04-12T11:17:49.580 INFO:tasks.rados.rados.0.smithi029.stdout:823: writing smithi02979277-48 from 3431711 to 3731187 tid 3 2024-04-12T11:17:49.580 INFO:tasks.rados.rados.0.smithi029.stdout:824: write oid 49 current snap is 105 2024-04-12T11:17:49.580 INFO:tasks.rados.rados.0.smithi029.stdout:822: finishing write tid 3 to smithi02979277-35 2024-04-12T11:17:49.580 INFO:tasks.rados.rados.0.smithi029.stdout:822: finishing write tid 4 to smithi02979277-35 2024-04-12T11:17:49.580 INFO:tasks.rados.rados.0.smithi029.stdout:822: finishing write tid 5 to smithi02979277-35 2024-04-12T11:17:49.580 INFO:tasks.rados.rados.0.smithi029.stdout:822: oid 35 updating version 0 to 546 2024-04-12T11:17:49.580 INFO:tasks.rados.rados.0.smithi029.stdout:822: oid 35 version 546 is already newer than 545 2024-04-12T11:17:49.580 INFO:tasks.rados.rados.0.smithi029.stdout:822: oid 35 version 546 is already newer than 544 2024-04-12T11:17:49.580 INFO:tasks.rados.rados.0.smithi029.stdout:822: oid 35 version 546 is already newer than 543 2024-04-12T11:17:49.580 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 546 (ObjNum 340 snap 105 seq_num 340) dirty exists 2024-04-12T11:17:49.581 INFO:tasks.rados.rados.0.smithi029.stdout:822: left oid 35 (ObjNum 340 snap 105 seq_num 340) 2024-04-12T11:17:49.581 INFO:tasks.rados.rados.0.smithi029.stdout:823: finishing write tid 1 to smithi02979277-48 2024-04-12T11:17:49.581 INFO:tasks.rados.rados.0.smithi029.stdout:823: finishing write tid 2 to smithi02979277-48 2024-04-12T11:17:49.581 INFO:tasks.rados.rados.0.smithi029.stdout:824: seq_num 342 ranges {547994=454690,1746724=482104,2659134=465146,3605976=142018} 2024-04-12T11:17:49.585 INFO:tasks.rados.rados.0.smithi029.stdout:824: writing smithi02979277-49 from 547994 to 1002684 tid 1 2024-04-12T11:17:49.592 INFO:tasks.rados.rados.0.smithi029.stdout:824: writing smithi02979277-49 from 1746724 to 2228828 tid 2 2024-04-12T11:17:49.597 INFO:tasks.rados.rados.0.smithi029.stdout:824: writing smithi02979277-49 from 2659134 to 3124280 tid 3 2024-04-12T11:17:49.601 INFO:tasks.rados.rados.0.smithi029.stdout:824: writing smithi02979277-49 from 3605976 to 3747994 tid 4 2024-04-12T11:17:49.601 INFO:tasks.rados.rados.0.smithi029.stdout:822: done (2 left) 2024-04-12T11:17:49.601 INFO:tasks.rados.rados.0.smithi029.stdout:825: read oid 10 snap -1 2024-04-12T11:17:49.601 INFO:tasks.rados.rados.0.smithi029.stdout:825: expect (ObjNum 335 snap 103 seq_num 335) 2024-04-12T11:17:49.601 INFO:tasks.rados.rados.0.smithi029.stdout:826: delete oid 12 current snap is 105 2024-04-12T11:17:49.601 INFO:tasks.rados.rados.0.smithi029.stdout:823: finishing write tid 3 to smithi02979277-48 2024-04-12T11:17:49.601 INFO:tasks.rados.rados.0.smithi029.stdout:823: finishing write tid 4 to smithi02979277-48 2024-04-12T11:17:49.601 INFO:tasks.rados.rados.0.smithi029.stdout:823: finishing write tid 5 to smithi02979277-48 2024-04-12T11:17:49.601 INFO:tasks.rados.rados.0.smithi029.stdout:823: oid 48 updating version 0 to 343 2024-04-12T11:17:49.602 INFO:tasks.rados.rados.0.smithi029.stdout:823: oid 48 version 343 is already newer than 340 2024-04-12T11:17:49.602 INFO:tasks.rados.rados.0.smithi029.stdout:823: oid 48 version 343 is already newer than 341 2024-04-12T11:17:49.602 INFO:tasks.rados.rados.0.smithi029.stdout:823: oid 48 version 343 is already newer than 342 2024-04-12T11:17:49.602 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 343 (ObjNum 341 snap 105 seq_num 341) dirty exists 2024-04-12T11:17:49.602 INFO:tasks.rados.rados.0.smithi029.stdout:823: left oid 48 (ObjNum 341 snap 105 seq_num 341) 2024-04-12T11:17:49.602 INFO:tasks.rados.rados.0.smithi029.stdout:824: finishing write tid 1 to smithi02979277-49 2024-04-12T11:17:49.602 INFO:tasks.rados.rados.0.smithi029.stdout:824: finishing write tid 2 to smithi02979277-49 2024-04-12T11:17:49.602 INFO:tasks.rados.rados.0.smithi029.stdout:824: finishing write tid 3 to smithi02979277-49 2024-04-12T11:17:49.603 INFO:tasks.rados.rados.0.smithi029.stdout:823: done (3 left) 2024-04-12T11:17:49.603 INFO:tasks.rados.rados.0.smithi029.stdout:826: done (2 left) 2024-04-12T11:17:49.603 INFO:tasks.rados.rados.0.smithi029.stdout:827: write oid 22 current snap is 105 2024-04-12T11:17:49.603 INFO:tasks.rados.rados.0.smithi029.stdout:824: finishing write tid 4 to smithi02979277-49 2024-04-12T11:17:49.603 INFO:tasks.rados.rados.0.smithi029.stdout:827: seq_num 344 ranges {581608=721541,1863688=485773,3113980=604043} 2024-04-12T11:17:49.610 INFO:tasks.rados.rados.0.smithi029.stdout:827: writing smithi02979277-22 from 581608 to 1303149 tid 1 2024-04-12T11:17:49.616 INFO:tasks.rados.rados.0.smithi029.stdout:827: writing smithi02979277-22 from 1863688 to 2349461 tid 2 2024-04-12T11:17:49.623 INFO:tasks.rados.rados.0.smithi029.stdout:827: writing smithi02979277-22 from 3113980 to 3718023 tid 3 2024-04-12T11:17:49.623 INFO:tasks.rados.rados.0.smithi029.stdout:828: write oid 39 current snap is 105 2024-04-12T11:17:49.624 INFO:tasks.rados.rados.0.smithi029.stdout:824: finishing write tid 5 to smithi02979277-49 2024-04-12T11:17:49.624 INFO:tasks.rados.rados.0.smithi029.stdout:828: seq_num 345 ranges {598415=796790,1922170=745784,3141403=415315} 2024-04-12T11:17:49.631 INFO:tasks.rados.rados.0.smithi029.stdout:828: writing smithi02979277-39 from 598415 to 1395205 tid 1 2024-04-12T11:17:49.638 INFO:tasks.rados.rados.0.smithi029.stdout:828: writing smithi02979277-39 from 1922170 to 2667954 tid 2 2024-04-12T11:17:49.643 INFO:tasks.rados.rados.0.smithi029.stdout:828: writing smithi02979277-39 from 3141403 to 3556718 tid 3 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:829: read oid 37 snap 104 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:829: expect (ObjNum 36 snap 0 seq_num 36) 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:830: rollback oid 1 current snap is 105 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:824: finishing write tid 6 to smithi02979277-49 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:824: oid 49 updating version 0 to 357 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:824: oid 49 updating version 357 to 359 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:824: oid 49 version 359 is already newer than 355 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:824: oid 49 version 359 is already newer than 358 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:824: oid 49 version 359 is already newer than 356 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 359 (ObjNum 342 snap 105 seq_num 342) dirty exists 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:824: left oid 49 (ObjNum 342 snap 105 seq_num 342) 2024-04-12T11:17:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:825: expect (ObjNum 335 snap 103 seq_num 335) 2024-04-12T11:17:49.812 INFO:tasks.rados.rados.0.smithi029.stdout:827: finishing write tid 1 to smithi02979277-22 2024-04-12T11:17:49.812 INFO:tasks.rados.rados.0.smithi029.stdout:827: finishing write tid 2 to smithi02979277-22 2024-04-12T11:17:49.812 INFO:tasks.rados.rados.0.smithi029.stdout:827: finishing write tid 3 to smithi02979277-22 2024-04-12T11:17:49.812 INFO:tasks.rados.rados.0.smithi029.stdout:827: finishing write tid 4 to smithi02979277-22 2024-04-12T11:17:49.812 INFO:tasks.rados.rados.0.smithi029.stdout:827: finishing write tid 5 to smithi02979277-22 2024-04-12T11:17:49.812 INFO:tasks.rados.rados.0.smithi029.stdout:827: oid 22 updating version 0 to 309 2024-04-12T11:17:49.812 INFO:tasks.rados.rados.0.smithi029.stdout:827: oid 22 version 309 is already newer than 308 2024-04-12T11:17:49.812 INFO:tasks.rados.rados.0.smithi029.stdout:827: oid 22 updating version 309 to 310 2024-04-12T11:17:49.812 INFO:tasks.rados.rados.0.smithi029.stdout:827: oid 22 version 310 is already newer than 307 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 310 (ObjNum 344 snap 105 seq_num 344) dirty exists 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:827: left oid 22 (ObjNum 344 snap 105 seq_num 344) 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:828: finishing write tid 1 to smithi02979277-39 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:828: finishing write tid 2 to smithi02979277-39 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 1 to 99 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:828: finishing write tid 3 to smithi02979277-39 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:828: finishing write tid 4 to smithi02979277-39 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:828: finishing write tid 5 to smithi02979277-39 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:828: oid 39 updating version 0 to 36 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:828: oid 39 updating version 36 to 38 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:828: oid 39 version 38 is already newer than 35 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:828: oid 39 version 38 is already newer than 37 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 38 (ObjNum 345 snap 105 seq_num 345) dirty exists 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:828: left oid 39 (ObjNum 345 snap 105 seq_num 345) 2024-04-12T11:17:49.813 INFO:tasks.rados.rados.0.smithi029.stdout:829: expect (ObjNum 36 snap 0 seq_num 36) 2024-04-12T11:17:49.868 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:49 smithi087 ceph-mon[32917]: osdmap e275: 8 total, 8 up, 8 in 2024-04-12T11:17:49.895 INFO:tasks.rados.rados.0.smithi029.stdout:830: finishing rollback tid 0 to smithi02979277-1 2024-04-12T11:17:49.896 INFO:tasks.rados.rados.0.smithi029.stdout:830: finishing rollback tid 1 to smithi02979277-1 2024-04-12T11:17:49.896 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 447 (ObjNum 129 snap 0 seq_num 1801111592) dirty dne 2024-04-12T11:17:49.896 INFO:tasks.rados.rados.0.smithi029.stdout:824: done (5 left) 2024-04-12T11:17:49.896 INFO:tasks.rados.rados.0.smithi029.stdout:825: done (4 left) 2024-04-12T11:17:49.896 INFO:tasks.rados.rados.0.smithi029.stdout:827: done (3 left) 2024-04-12T11:17:49.896 INFO:tasks.rados.rados.0.smithi029.stdout:828: done (2 left) 2024-04-12T11:17:49.896 INFO:tasks.rados.rados.0.smithi029.stdout:829: done (1 left) 2024-04-12T11:17:49.896 INFO:tasks.rados.rados.0.smithi029.stdout:830: done (0 left) 2024-04-12T11:17:49.896 INFO:tasks.rados.rados.0.smithi029.stdout:831: delete oid 1 current snap is 105 2024-04-12T11:17:49.897 INFO:tasks.rados.rados.0.smithi029.stdout:831: done (0 left) 2024-04-12T11:17:49.897 INFO:tasks.rados.rados.0.smithi029.stdout:832: read oid 43 snap -1 2024-04-12T11:17:49.897 INFO:tasks.rados.rados.0.smithi029.stdout:832: expect (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:17:49.897 INFO:tasks.rados.rados.0.smithi029.stdout:833: snap_create 2024-04-12T11:17:49.900 INFO:tasks.rados.rados.0.smithi029.stdout:832: expect (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:17:49.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:49 smithi029 ceph-mon[39999]: osdmap e275: 8 total, 8 up, 8 in 2024-04-12T11:17:49.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:49 smithi029 ceph-mon[93650]: osdmap e275: 8 total, 8 up, 8 in 2024-04-12T11:17:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:832: done (1 left) 2024-04-12T11:17:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:833: done (0 left) 2024-04-12T11:17:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:834: rollback oid 18 current snap is 106 2024-04-12T11:17:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 18 to 104 2024-04-12T11:17:50.547 INFO:tasks.rados.rados.0.smithi029.stdout:835: snap_create 2024-04-12T11:17:50.552 INFO:tasks.rados.rados.0.smithi029.stdout:834: finishing rollback tid 0 to smithi02979277-18 2024-04-12T11:17:50.552 INFO:tasks.rados.rados.0.smithi029.stdout:834: finishing rollback tid 1 to smithi02979277-18 2024-04-12T11:17:50.552 INFO:tasks.rados.rados.0.smithi029.stdout:834: finishing rollback tid 2 to smithi02979277-18 2024-04-12T11:17:50.552 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 505 (ObjNum 327 snap 101 seq_num 327) dirty exists 2024-04-12T11:17:50.888 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:50 smithi087 ceph-mon[32917]: pgmap v44: 89 pgs: 89 active+clean; 189 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-04-12T11:17:50.888 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:50 smithi087 ceph-mon[32917]: osdmap e276: 8 total, 8 up, 8 in 2024-04-12T11:17:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:50 smithi029 ceph-mon[93650]: pgmap v44: 89 pgs: 89 active+clean; 189 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-04-12T11:17:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:50 smithi029 ceph-mon[93650]: osdmap e276: 8 total, 8 up, 8 in 2024-04-12T11:17:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:50 smithi029 ceph-mon[39999]: pgmap v44: 89 pgs: 89 active+clean; 189 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-04-12T11:17:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:50 smithi029 ceph-mon[39999]: osdmap e276: 8 total, 8 up, 8 in 2024-04-12T11:17:51.552 INFO:tasks.rados.rados.0.smithi029.stdout:834: done (1 left) 2024-04-12T11:17:51.552 INFO:tasks.rados.rados.0.smithi029.stdout:835: done (0 left) 2024-04-12T11:17:51.552 INFO:tasks.rados.rados.0.smithi029.stdout:836: rollback oid 38 current snap is 107 2024-04-12T11:17:51.552 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 38 to 103 2024-04-12T11:17:51.552 INFO:tasks.rados.rados.0.smithi029.stdout:837: read oid 40 snap 104 2024-04-12T11:17:51.552 INFO:tasks.rados.rados.0.smithi029.stdout:837: expect (ObjNum 282 snap 88 seq_num 282) 2024-04-12T11:17:51.552 INFO:tasks.rados.rados.0.smithi029.stdout:838: write oid 39 current snap is 107 2024-04-12T11:17:51.552 INFO:tasks.rados.rados.0.smithi029.stdout:838: seq_num 347 ranges {632029=547288,1639134=749453,2796249=437859,3653524=178505} 2024-04-12T11:17:51.558 INFO:tasks.rados.rados.0.smithi029.stdout:838: writing smithi02979277-39 from 632029 to 1179317 tid 1 2024-04-12T11:17:51.565 INFO:tasks.rados.rados.0.smithi029.stdout:838: writing smithi02979277-39 from 1639134 to 2388587 tid 2 2024-04-12T11:17:51.570 INFO:tasks.rados.rados.0.smithi029.stdout:838: writing smithi02979277-39 from 2796249 to 3234108 tid 3 2024-04-12T11:17:51.573 INFO:tasks.rados.rados.0.smithi029.stdout:838: writing smithi02979277-39 from 3653524 to 3832029 tid 4 2024-04-12T11:17:51.573 INFO:tasks.rados.rados.0.smithi029.stdout:839: write oid 36 current snap is 107 2024-04-12T11:17:51.573 INFO:tasks.rados.rados.0.smithi029.stdout:839: seq_num 348 ranges {648836=622537,1981263=493111,3107319=741517} 2024-04-12T11:17:51.580 INFO:tasks.rados.rados.0.smithi029.stdout:839: writing smithi02979277-36 from 648836 to 1271373 tid 1 2024-04-12T11:17:51.587 INFO:tasks.rados.rados.0.smithi029.stdout:839: writing smithi02979277-36 from 1981263 to 2474374 tid 2 2024-04-12T11:17:51.594 INFO:tasks.rados.rados.0.smithi029.stdout:839: writing smithi02979277-36 from 3107319 to 3848836 tid 3 2024-04-12T11:17:51.594 INFO:tasks.rados.rados.0.smithi029.stdout:840: snap_remove snap 92 2024-04-12T11:17:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:51 smithi087 ceph-mon[32917]: osdmap e277: 8 total, 8 up, 8 in 2024-04-12T11:17:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:51 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:51 smithi029 ceph-mon[93650]: osdmap e277: 8 total, 8 up, 8 in 2024-04-12T11:17:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:51 smithi029 ceph-mon[39999]: osdmap e277: 8 total, 8 up, 8 in 2024-04-12T11:17:52.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:51 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:17:52.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:17:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:17:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:17:52.503 INFO:tasks.rados.rados.0.smithi029.stdout:840: done (4 left) 2024-04-12T11:17:52.503 INFO:tasks.rados.rados.0.smithi029.stdout:841: write oid 47 current snap is 107 2024-04-12T11:17:52.503 INFO:tasks.rados.rados.0.smithi029.stdout:841: seq_num 349 ranges {665643=697786,2039745=753122,3534742=330901} 2024-04-12T11:17:52.512 INFO:tasks.rados.rados.0.smithi029.stdout:841: writing smithi02979277-47 from 665643 to 1363429 tid 1 2024-04-12T11:17:52.520 INFO:tasks.rados.rados.0.smithi029.stdout:841: writing smithi02979277-47 from 2039745 to 2792867 tid 2 2024-04-12T11:17:52.527 INFO:tasks.rados.rados.0.smithi029.stdout:841: writing smithi02979277-47 from 3534742 to 3865643 tid 3 2024-04-12T11:17:52.527 INFO:tasks.rados.rados.0.smithi029.stdout:842: write oid 16 current snap is 107 2024-04-12T11:17:52.527 INFO:tasks.rados.rados.0.smithi029.stdout:842: seq_num 350 ranges {682450=489388,1814580=496780,2878518=788028} 2024-04-12T11:17:52.533 INFO:tasks.rados.rados.0.smithi029.stdout:842: writing smithi02979277-16 from 682450 to 1171838 tid 1 2024-04-12T11:17:52.539 INFO:tasks.rados.rados.0.smithi029.stdout:842: writing smithi02979277-16 from 1814580 to 2311360 tid 2 2024-04-12T11:17:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:842: writing smithi02979277-16 from 2878518 to 3666546 tid 3 2024-04-12T11:17:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:843: write oid 49 current snap is 107 2024-04-12T11:17:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:843: seq_num 351 ranges {699257=564637,1873062=640438,3305941=593316} 2024-04-12T11:17:52.554 INFO:tasks.rados.rados.0.smithi029.stdout:843: writing smithi02979277-49 from 699257 to 1263894 tid 1 2024-04-12T11:17:52.562 INFO:tasks.rados.rados.0.smithi029.stdout:843: writing smithi02979277-49 from 1873062 to 2513500 tid 2 2024-04-12T11:17:52.568 INFO:tasks.rados.rados.0.smithi029.stdout:843: writing smithi02979277-49 from 3305941 to 3899257 tid 3 2024-04-12T11:17:52.569 INFO:tasks.rados.rados.0.smithi029.stdout:844: snap_create 2024-04-12T11:17:52.569 INFO:tasks.rados.rados.0.smithi029.stdout:836: finishing rollback tid 0 to smithi02979277-38 2024-04-12T11:17:52.569 INFO:tasks.rados.rados.0.smithi029.stdout:837: expect (ObjNum 282 snap 88 seq_num 282) 2024-04-12T11:17:52.696 INFO:tasks.rados.rados.0.smithi029.stdout:836: finishing rollback tid 1 to smithi02979277-38 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:836: finishing rollback tid 2 to smithi02979277-38 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 193 (ObjNum 220 snap 63 seq_num 220) dirty exists 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: finishing write tid 1 to smithi02979277-39 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: finishing write tid 2 to smithi02979277-39 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: finishing write tid 3 to smithi02979277-39 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: finishing write tid 4 to smithi02979277-39 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: finishing write tid 5 to smithi02979277-39 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: finishing write tid 6 to smithi02979277-39 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: oid 39 updating version 0 to 43 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: oid 39 version 43 is already newer than 42 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: oid 39 version 43 is already newer than 40 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: oid 39 version 43 is already newer than 41 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:838: oid 39 updating version 43 to 44 2024-04-12T11:17:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 44 (ObjNum 347 snap 107 seq_num 347) dirty exists 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:838: left oid 39 (ObjNum 347 snap 107 seq_num 347) 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:839: finishing write tid 1 to smithi02979277-36 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:839: finishing write tid 2 to smithi02979277-36 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:839: finishing write tid 3 to smithi02979277-36 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:839: finishing write tid 4 to smithi02979277-36 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:839: finishing write tid 5 to smithi02979277-36 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:839: oid 36 updating version 0 to 202 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:839: oid 36 updating version 202 to 205 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:839: oid 36 version 205 is already newer than 204 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:839: oid 36 version 205 is already newer than 203 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 205 (ObjNum 348 snap 107 seq_num 348) dirty exists 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:839: left oid 36 (ObjNum 348 snap 107 seq_num 348) 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:841: finishing write tid 1 to smithi02979277-47 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:841: finishing write tid 2 to smithi02979277-47 2024-04-12T11:17:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:841: finishing write tid 3 to smithi02979277-47 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:841: finishing write tid 4 to smithi02979277-47 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:841: finishing write tid 5 to smithi02979277-47 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:841: oid 47 updating version 0 to 468 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:841: oid 47 updating version 468 to 471 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:841: oid 47 version 471 is already newer than 470 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:841: oid 47 version 471 is already newer than 469 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 471 (ObjNum 349 snap 107 seq_num 349) dirty exists 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:841: left oid 47 (ObjNum 349 snap 107 seq_num 349) 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:842: finishing write tid 1 to smithi02979277-16 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:842: finishing write tid 2 to smithi02979277-16 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:842: finishing write tid 3 to smithi02979277-16 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:842: finishing write tid 4 to smithi02979277-16 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:842: finishing write tid 5 to smithi02979277-16 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:842: oid 16 updating version 0 to 518 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:842: oid 16 updating version 518 to 519 2024-04-12T11:17:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:842: oid 16 updating version 519 to 520 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:842: oid 16 version 520 is already newer than 517 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 520 (ObjNum 350 snap 107 seq_num 350) dirty exists 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:842: left oid 16 (ObjNum 350 snap 107 seq_num 350) 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:843: finishing write tid 1 to smithi02979277-49 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:843: finishing write tid 2 to smithi02979277-49 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:843: finishing write tid 3 to smithi02979277-49 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:843: finishing write tid 4 to smithi02979277-49 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:843: finishing write tid 5 to smithi02979277-49 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:843: oid 49 updating version 0 to 370 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:843: oid 49 version 370 is already newer than 368 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:843: oid 49 version 370 is already newer than 367 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:843: oid 49 version 370 is already newer than 369 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 370 (ObjNum 351 snap 107 seq_num 351) dirty exists 2024-04-12T11:17:52.700 INFO:tasks.rados.rados.0.smithi029.stdout:843: left oid 49 (ObjNum 351 snap 107 seq_num 351) 2024-04-12T11:17:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:52 smithi087 ceph-mon[32917]: pgmap v47: 89 pgs: 89 active+clean; 189 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 470 KiB/s wr, 1 op/s 2024-04-12T11:17:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:52 smithi087 ceph-mon[32917]: osdmap e278: 8 total, 8 up, 8 in 2024-04-12T11:17:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:52 smithi087 ceph-mon[32917]: osdmap e279: 8 total, 8 up, 8 in 2024-04-12T11:17:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:52 smithi029 ceph-mon[39999]: pgmap v47: 89 pgs: 89 active+clean; 189 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 470 KiB/s wr, 1 op/s 2024-04-12T11:17:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:52 smithi029 ceph-mon[39999]: osdmap e278: 8 total, 8 up, 8 in 2024-04-12T11:17:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:52 smithi029 ceph-mon[39999]: osdmap e279: 8 total, 8 up, 8 in 2024-04-12T11:17:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:52 smithi029 ceph-mon[93650]: pgmap v47: 89 pgs: 89 active+clean; 189 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 470 KiB/s wr, 1 op/s 2024-04-12T11:17:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:52 smithi029 ceph-mon[93650]: osdmap e278: 8 total, 8 up, 8 in 2024-04-12T11:17:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:52 smithi029 ceph-mon[93650]: osdmap e279: 8 total, 8 up, 8 in 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:836: done (7 left) 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:837: done (6 left) 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:838: done (5 left) 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:839: done (4 left) 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:841: done (3 left) 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:842: done (2 left) 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:843: done (1 left) 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:844: done (0 left) 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:845: rollback oid 4 current snap is 108 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 4 to 100 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:846: write oid 5 current snap is 108 2024-04-12T11:17:53.559 INFO:tasks.rados.rados.0.smithi029.stdout:846: seq_num 352 ranges {716064=639886,1815191=500449,2817011=526925} 2024-04-12T11:17:53.566 INFO:tasks.rados.rados.0.smithi029.stdout:846: writing smithi02979277-5 from 716064 to 1355950 tid 1 2024-04-12T11:17:53.572 INFO:tasks.rados.rados.0.smithi029.stdout:846: writing smithi02979277-5 from 1815191 to 2315640 tid 2 2024-04-12T11:17:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:846: writing smithi02979277-5 from 2817011 to 3343936 tid 3 2024-04-12T11:17:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:847: snap_create 2024-04-12T11:17:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:845: finishing rollback tid 1 to smithi02979277-4 2024-04-12T11:17:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:845: finishing rollback tid 2 to smithi02979277-4 2024-04-12T11:17:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 522 (ObjNum 313 snap 96 seq_num 313) dirty exists 2024-04-12T11:17:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:846: finishing write tid 1 to smithi02979277-5 2024-04-12T11:17:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:846: finishing write tid 2 to smithi02979277-5 2024-04-12T11:17:53.581 INFO:tasks.rados.rados.0.smithi029.stdout:846: finishing write tid 3 to smithi02979277-5 2024-04-12T11:17:53.581 INFO:tasks.rados.rados.0.smithi029.stdout:846: finishing write tid 4 to smithi02979277-5 2024-04-12T11:17:53.581 INFO:tasks.rados.rados.0.smithi029.stdout:846: finishing write tid 5 to smithi02979277-5 2024-04-12T11:17:53.581 INFO:tasks.rados.rados.0.smithi029.stdout:846: oid 5 updating version 0 to 524 2024-04-12T11:17:53.581 INFO:tasks.rados.rados.0.smithi029.stdout:846: oid 5 updating version 524 to 527 2024-04-12T11:17:53.582 INFO:tasks.rados.rados.0.smithi029.stdout:846: oid 5 version 527 is already newer than 526 2024-04-12T11:17:53.582 INFO:tasks.rados.rados.0.smithi029.stdout:846: oid 5 version 527 is already newer than 525 2024-04-12T11:17:53.582 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 527 (ObjNum 352 snap 108 seq_num 352) dirty exists 2024-04-12T11:17:53.582 INFO:tasks.rados.rados.0.smithi029.stdout:846: left oid 5 (ObjNum 352 snap 108 seq_num 352) 2024-04-12T11:17:53.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:17:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:17:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:17:54.561 INFO:tasks.rados.rados.0.smithi029.stdout:845: done (2 left) 2024-04-12T11:17:54.561 INFO:tasks.rados.rados.0.smithi029.stdout:846: done (1 left) 2024-04-12T11:17:54.561 INFO:tasks.rados.rados.0.smithi029.stdout:847: done (0 left) 2024-04-12T11:17:54.562 INFO:tasks.rados.rados.0.smithi029.stdout:848: read oid 12 snap -1 2024-04-12T11:17:54.562 INFO:tasks.rados.rados.0.smithi029.stdout:848: expect deleted 2024-04-12T11:17:54.562 INFO:tasks.rados.rados.0.smithi029.stdout:849: rollback oid 9 current snap is 109 2024-04-12T11:17:54.562 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 9 to 104 2024-04-12T11:17:54.562 INFO:tasks.rados.rados.0.smithi029.stdout:850: snap_remove snap 108 2024-04-12T11:17:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:54 smithi029 ceph-mon[93650]: pgmap v50: 89 pgs: 89 active+clean; 198 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 23 op/s 2024-04-12T11:17:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:54 smithi029 ceph-mon[93650]: osdmap e280: 8 total, 8 up, 8 in 2024-04-12T11:17:54.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:54 smithi029 ceph-mon[39999]: pgmap v50: 89 pgs: 89 active+clean; 198 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 23 op/s 2024-04-12T11:17:54.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:54 smithi029 ceph-mon[39999]: osdmap e280: 8 total, 8 up, 8 in 2024-04-12T11:17:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:54 smithi087 ceph-mon[32917]: pgmap v50: 89 pgs: 89 active+clean; 198 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 23 op/s 2024-04-12T11:17:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:54 smithi087 ceph-mon[32917]: osdmap e280: 8 total, 8 up, 8 in 2024-04-12T11:17:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:17:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:17:54] "GET /metrics HTTP/1.1" 200 36247 "" "Prometheus/2.43.0" 2024-04-12T11:17:55.573 INFO:tasks.rados.rados.0.smithi029.stdout:850: done (2 left) 2024-04-12T11:17:55.573 INFO:tasks.rados.rados.0.smithi029.stdout:851: write oid 25 current snap is 109 2024-04-12T11:17:55.573 INFO:tasks.rados.rados.0.smithi029.stdout:851: seq_num 353 ranges {732871=715135,1873673=644107,3244434=688437} 2024-04-12T11:17:55.580 INFO:tasks.rados.rados.0.smithi029.stdout:851: writing smithi02979277-25 from 732871 to 1448006 tid 1 2024-04-12T11:17:55.586 INFO:tasks.rados.rados.0.smithi029.stdout:851: writing smithi02979277-25 from 1873673 to 2517780 tid 2 2024-04-12T11:17:55.595 INFO:tasks.rados.rados.0.smithi029.stdout:851: writing smithi02979277-25 from 3244434 to 3932871 tid 3 2024-04-12T11:17:55.595 INFO:tasks.rados.rados.0.smithi029.stdout:852: delete oid 34 current snap is 109 2024-04-12T11:17:55.595 INFO:tasks.rados.rados.0.smithi029.stdout:849: finishing rollback tid 0 to smithi02979277-9 2024-04-12T11:17:55.595 INFO:tasks.rados.rados.0.smithi029.stdout:849: finishing rollback tid 1 to smithi02979277-9 2024-04-12T11:17:55.595 INFO:tasks.rados.rados.0.smithi029.stdout:849: finishing rollback tid 2 to smithi02979277-9 2024-04-12T11:17:55.595 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 374 (ObjNum 268 snap 86 seq_num 268) dirty exists 2024-04-12T11:17:55.595 INFO:tasks.rados.rados.0.smithi029.stdout:851: finishing write tid 1 to smithi02979277-25 2024-04-12T11:17:55.595 INFO:tasks.rados.rados.0.smithi029.stdout:851: finishing write tid 2 to smithi02979277-25 2024-04-12T11:17:55.597 INFO:tasks.rados.rados.0.smithi029.stdout:848: done (3 left) 2024-04-12T11:17:55.597 INFO:tasks.rados.rados.0.smithi029.stdout:849: done (2 left) 2024-04-12T11:17:55.597 INFO:tasks.rados.rados.0.smithi029.stdout:852: done (1 left) 2024-04-12T11:17:55.597 INFO:tasks.rados.rados.0.smithi029.stdout:853: read oid 5 snap -1 2024-04-12T11:17:55.597 INFO:tasks.rados.rados.0.smithi029.stdout:853: expect (ObjNum 352 snap 108 seq_num 352) 2024-04-12T11:17:55.597 INFO:tasks.rados.rados.0.smithi029.stdout:854: rollback oid 27 current snap is 109 2024-04-12T11:17:55.597 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 107 2024-04-12T11:17:55.597 INFO:tasks.rados.rados.0.smithi029.stdout:855: snap_create 2024-04-12T11:17:55.598 INFO:tasks.rados.rados.0.smithi029.stdout:851: finishing write tid 3 to smithi02979277-25 2024-04-12T11:17:55.598 INFO:tasks.rados.rados.0.smithi029.stdout:851: finishing write tid 4 to smithi02979277-25 2024-04-12T11:17:55.598 INFO:tasks.rados.rados.0.smithi029.stdout:851: finishing write tid 5 to smithi02979277-25 2024-04-12T11:17:55.598 INFO:tasks.rados.rados.0.smithi029.stdout:851: oid 25 updating version 0 to 514 2024-04-12T11:17:55.598 INFO:tasks.rados.rados.0.smithi029.stdout:851: oid 25 version 514 is already newer than 513 2024-04-12T11:17:55.598 INFO:tasks.rados.rados.0.smithi029.stdout:851: oid 25 version 514 is already newer than 512 2024-04-12T11:17:55.599 INFO:tasks.rados.rados.0.smithi029.stdout:851: oid 25 updating version 514 to 515 2024-04-12T11:17:55.599 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 515 (ObjNum 353 snap 109 seq_num 353) dirty exists 2024-04-12T11:17:55.599 INFO:tasks.rados.rados.0.smithi029.stdout:851: left oid 25 (ObjNum 353 snap 109 seq_num 353) 2024-04-12T11:17:55.601 INFO:tasks.rados.rados.0.smithi029.stdout:854: finishing rollback tid 0 to smithi02979277-27 2024-04-12T11:17:55.601 INFO:tasks.rados.rados.0.smithi029.stdout:854: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:17:55.601 INFO:tasks.rados.rados.0.smithi029.stdout:854: finishing rollback tid 2 to smithi02979277-27 2024-04-12T11:17:55.601 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 377 (ObjNum 318 snap 97 seq_num 318) dirty exists 2024-04-12T11:17:55.614 INFO:tasks.rados.rados.0.smithi029.stdout:853: expect (ObjNum 352 snap 108 seq_num 352) 2024-04-12T11:17:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:55 smithi087 ceph-mon[32917]: osdmap e281: 8 total, 8 up, 8 in 2024-04-12T11:17:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:55 smithi029 ceph-mon[93650]: osdmap e281: 8 total, 8 up, 8 in 2024-04-12T11:17:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:55 smithi029 ceph-mon[39999]: osdmap e281: 8 total, 8 up, 8 in 2024-04-12T11:17:56.578 INFO:tasks.rados.rados.0.smithi029.stdout:851: done (3 left) 2024-04-12T11:17:56.578 INFO:tasks.rados.rados.0.smithi029.stdout:853: done (2 left) 2024-04-12T11:17:56.579 INFO:tasks.rados.rados.0.smithi029.stdout:854: done (1 left) 2024-04-12T11:17:56.579 INFO:tasks.rados.rados.0.smithi029.stdout:855: done (0 left) 2024-04-12T11:17:56.579 INFO:tasks.rados.rados.0.smithi029.stdout:856: read oid 25 snap -1 2024-04-12T11:17:56.579 INFO:tasks.rados.rados.0.smithi029.stdout:856: expect (ObjNum 353 snap 109 seq_num 353) 2024-04-12T11:17:56.579 INFO:tasks.rados.rados.0.smithi029.stdout:857: snap_remove snap 98 2024-04-12T11:17:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:56 smithi087 ceph-mon[32917]: pgmap v53: 89 pgs: 89 active+clean; 198 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 23 op/s 2024-04-12T11:17:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:56 smithi087 ceph-mon[32917]: osdmap e282: 8 total, 8 up, 8 in 2024-04-12T11:17:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:56 smithi029 ceph-mon[39999]: pgmap v53: 89 pgs: 89 active+clean; 198 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 23 op/s 2024-04-12T11:17:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:56 smithi029 ceph-mon[39999]: osdmap e282: 8 total, 8 up, 8 in 2024-04-12T11:17:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:56 smithi029 ceph-mon[93650]: pgmap v53: 89 pgs: 89 active+clean; 198 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.0 MiB/s wr, 23 op/s 2024-04-12T11:17:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:56 smithi029 ceph-mon[93650]: osdmap e282: 8 total, 8 up, 8 in 2024-04-12T11:17:57.585 INFO:tasks.rados.rados.0.smithi029.stdout:857: done (1 left) 2024-04-12T11:17:57.585 INFO:tasks.rados.rados.0.smithi029.stdout:858: write oid 12 current snap is 110 2024-04-12T11:17:57.585 INFO:tasks.rados.rados.0.smithi029.stdout:858: seq_num 355 ranges {766485=465633,1990637=647776,3299280=667205} 2024-04-12T11:17:57.592 INFO:tasks.rados.rados.0.smithi029.stdout:858: writing smithi02979277-12 from 766485 to 1232118 tid 1 2024-04-12T11:17:57.601 INFO:tasks.rados.rados.0.smithi029.stdout:858: writing smithi02979277-12 from 1990637 to 2638413 tid 2 2024-04-12T11:17:57.608 INFO:tasks.rados.rados.0.smithi029.stdout:858: writing smithi02979277-12 from 3299280 to 3966485 tid 3 2024-04-12T11:17:57.608 INFO:tasks.rados.rados.0.smithi029.stdout:859: delete oid 11 current snap is 110 2024-04-12T11:17:57.608 INFO:tasks.rados.rados.0.smithi029.stdout:856: expect (ObjNum 353 snap 109 seq_num 353) 2024-04-12T11:17:57.737 INFO:tasks.rados.rados.0.smithi029.stdout:858: finishing write tid 1 to smithi02979277-12 2024-04-12T11:17:57.737 INFO:tasks.rados.rados.0.smithi029.stdout:858: finishing write tid 2 to smithi02979277-12 2024-04-12T11:17:57.737 INFO:tasks.rados.rados.0.smithi029.stdout:856: done (2 left) 2024-04-12T11:17:57.737 INFO:tasks.rados.rados.0.smithi029.stdout:859: done (1 left) 2024-04-12T11:17:57.737 INFO:tasks.rados.rados.0.smithi029.stdout:860: read oid 26 snap -1 2024-04-12T11:17:57.737 INFO:tasks.rados.rados.0.smithi029.stdout:860: expect (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:17:57.737 INFO:tasks.rados.rados.0.smithi029.stdout:861: read oid 9 snap -1 2024-04-12T11:17:57.737 INFO:tasks.rados.rados.0.smithi029.stdout:861: expect (ObjNum 268 snap 86 seq_num 268) 2024-04-12T11:17:57.737 INFO:tasks.rados.rados.0.smithi029.stdout:862: snap_remove snap 103 2024-04-12T11:17:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:57 smithi087 ceph-mon[32917]: osdmap e283: 8 total, 8 up, 8 in 2024-04-12T11:17:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:57 smithi029 ceph-mon[39999]: osdmap e283: 8 total, 8 up, 8 in 2024-04-12T11:17:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:57 smithi029 ceph-mon[93650]: osdmap e283: 8 total, 8 up, 8 in 2024-04-12T11:17:58.589 INFO:tasks.rados.rados.0.smithi029.stdout:862: done (3 left) 2024-04-12T11:17:58.589 INFO:tasks.rados.rados.0.smithi029.stdout:863: delete oid 40 current snap is 110 2024-04-12T11:17:58.589 INFO:tasks.rados.rados.0.smithi029.stdout:858: finishing write tid 3 to smithi02979277-12 2024-04-12T11:17:58.589 INFO:tasks.rados.rados.0.smithi029.stdout:858: finishing write tid 4 to smithi02979277-12 2024-04-12T11:17:58.589 INFO:tasks.rados.rados.0.smithi029.stdout:858: finishing write tid 5 to smithi02979277-12 2024-04-12T11:17:58.589 INFO:tasks.rados.rados.0.smithi029.stdout:858: oid 12 updating version 0 to 476 2024-04-12T11:17:58.590 INFO:tasks.rados.rados.0.smithi029.stdout:858: oid 12 updating version 476 to 478 2024-04-12T11:17:58.590 INFO:tasks.rados.rados.0.smithi029.stdout:858: oid 12 updating version 478 to 479 2024-04-12T11:17:58.590 INFO:tasks.rados.rados.0.smithi029.stdout:858: oid 12 version 479 is already newer than 477 2024-04-12T11:17:58.590 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 479 (ObjNum 355 snap 110 seq_num 355) dirty exists 2024-04-12T11:17:58.590 INFO:tasks.rados.rados.0.smithi029.stdout:858: left oid 12 (ObjNum 355 snap 110 seq_num 355) 2024-04-12T11:17:58.590 INFO:tasks.rados.rados.0.smithi029.stdout:860: expect (ObjNum 135 snap 28 seq_num 135) 2024-04-12T11:17:58.708 INFO:tasks.rados.rados.0.smithi029.stdout:861: expect (ObjNum 268 snap 86 seq_num 268) 2024-04-12T11:17:58.866 INFO:tasks.rados.rados.0.smithi029.stdout:858: done (3 left) 2024-04-12T11:17:58.866 INFO:tasks.rados.rados.0.smithi029.stdout:860: done (2 left) 2024-04-12T11:17:58.867 INFO:tasks.rados.rados.0.smithi029.stdout:861: done (1 left) 2024-04-12T11:17:58.867 INFO:tasks.rados.rados.0.smithi029.stdout:863: done (0 left) 2024-04-12T11:17:58.867 INFO:tasks.rados.rados.0.smithi029.stdout:864: read oid 11 snap 105 2024-04-12T11:17:58.867 INFO:tasks.rados.rados.0.smithi029.stdout:864: expect deleted 2024-04-12T11:17:58.867 INFO:tasks.rados.rados.0.smithi029.stdout:865: rollback oid 3 current snap is 110 2024-04-12T11:17:58.867 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 105 2024-04-12T11:17:58.867 INFO:tasks.rados.rados.0.smithi029.stdout:866: rollback oid 41 current snap is 110 2024-04-12T11:17:58.867 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 86 2024-04-12T11:17:58.867 INFO:tasks.rados.rados.0.smithi029.stdout:867: snap_remove snap 99 2024-04-12T11:17:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:58 smithi087 ceph-mon[32917]: pgmap v56: 89 pgs: 89 active+clean; 207 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-04-12T11:17:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:58 smithi087 ceph-mon[32917]: osdmap e284: 8 total, 8 up, 8 in 2024-04-12T11:17:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:58 smithi029 ceph-mon[39999]: pgmap v56: 89 pgs: 89 active+clean; 207 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-04-12T11:17:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:58 smithi029 ceph-mon[39999]: osdmap e284: 8 total, 8 up, 8 in 2024-04-12T11:17:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:58 smithi029 ceph-mon[93650]: pgmap v56: 89 pgs: 89 active+clean; 207 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-04-12T11:17:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:58 smithi029 ceph-mon[93650]: osdmap e284: 8 total, 8 up, 8 in 2024-04-12T11:17:59.595 INFO:tasks.rados.rados.0.smithi029.stdout:867: done (3 left) 2024-04-12T11:17:59.595 INFO:tasks.rados.rados.0.smithi029.stdout:868: snap_remove snap 101 2024-04-12T11:17:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:17:59 smithi087 ceph-mon[32917]: osdmap e285: 8 total, 8 up, 8 in 2024-04-12T11:18:00.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:17:59 smithi029 ceph-mon[93650]: osdmap e285: 8 total, 8 up, 8 in 2024-04-12T11:18:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:17:59 smithi029 ceph-mon[39999]: osdmap e285: 8 total, 8 up, 8 in 2024-04-12T11:18:00.596 INFO:tasks.rados.rados.0.smithi029.stdout:868: done (3 left) 2024-04-12T11:18:00.596 INFO:tasks.rados.rados.0.smithi029.stdout:869: delete oid 8 current snap is 110 2024-04-12T11:18:00.596 INFO:tasks.rados.rados.0.smithi029.stdout:865: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:18:00.597 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 0 (ObjNum 33 snap 0 seq_num 1801094896) dirty dne 2024-04-12T11:18:00.597 INFO:tasks.rados.rados.0.smithi029.stdout:866: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:18:00.597 INFO:tasks.rados.rados.0.smithi029.stdout:866: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:18:00.597 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 460 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-04-12T11:18:00.602 INFO:tasks.rados.rados.0.smithi029.stdout:864: done (3 left) 2024-04-12T11:18:00.602 INFO:tasks.rados.rados.0.smithi029.stdout:865: done (2 left) 2024-04-12T11:18:00.603 INFO:tasks.rados.rados.0.smithi029.stdout:866: done (1 left) 2024-04-12T11:18:00.603 INFO:tasks.rados.rados.0.smithi029.stdout:869: done (0 left) 2024-04-12T11:18:00.603 INFO:tasks.rados.rados.0.smithi029.stdout:870: read oid 1 snap -1 2024-04-12T11:18:00.603 INFO:tasks.rados.rados.0.smithi029.stdout:870: expect deleted 2024-04-12T11:18:00.603 INFO:tasks.rados.rados.0.smithi029.stdout:871: read oid 7 snap 105 2024-04-12T11:18:00.603 INFO:tasks.rados.rados.0.smithi029.stdout:871: expect deleted 2024-04-12T11:18:00.603 INFO:tasks.rados.rados.0.smithi029.stdout:872: write oid 28 current snap is 110 2024-04-12T11:18:00.603 INFO:tasks.rados.rados.0.smithi029.stdout:872: seq_num 359 ranges {433713=482982,1424565=609148} 2024-04-12T11:18:00.608 INFO:tasks.rados.rados.0.smithi029.stdout:872: writing smithi02979277-28 from 433713 to 916695 tid 1 2024-04-12T11:18:00.615 INFO:tasks.rados.rados.0.smithi029.stdout:872: writing smithi02979277-28 from 1424565 to 2033713 tid 2 2024-04-12T11:18:00.615 INFO:tasks.rados.rados.0.smithi029.stdout:873: read oid 4 snap -1 2024-04-12T11:18:00.616 INFO:tasks.rados.rados.0.smithi029.stdout:873: expect (ObjNum 313 snap 96 seq_num 313) 2024-04-12T11:18:00.616 INFO:tasks.rados.rados.0.smithi029.stdout:874: read oid 18 snap 104 2024-04-12T11:18:00.616 INFO:tasks.rados.rados.0.smithi029.stdout:874: expect (ObjNum 327 snap 101 seq_num 327) 2024-04-12T11:18:00.616 INFO:tasks.rados.rados.0.smithi029.stdout:875: write oid 39 current snap is 110 2024-04-12T11:18:00.616 INFO:tasks.rados.rados.0.smithi029.stdout:875: seq_num 360 ranges {450520=558231,1483047=515125} 2024-04-12T11:18:00.621 INFO:tasks.rados.rados.0.smithi029.stdout:875: writing smithi02979277-39 from 450520 to 1008751 tid 1 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:875: writing smithi02979277-39 from 1483047 to 1998172 tid 2 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:876: snap_create 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:872: finishing write tid 1 to smithi02979277-28 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:872: finishing write tid 2 to smithi02979277-28 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:872: finishing write tid 3 to smithi02979277-28 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:872: finishing write tid 4 to smithi02979277-28 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:872: oid 28 updating version 0 to 365 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:872: oid 28 version 365 is already newer than 364 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:872: oid 28 version 365 is already newer than 363 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 365 (ObjNum 359 snap 110 seq_num 359) dirty exists 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:872: left oid 28 (ObjNum 359 snap 110 seq_num 359) 2024-04-12T11:18:00.627 INFO:tasks.rados.rados.0.smithi029.stdout:873: expect (ObjNum 313 snap 96 seq_num 313) 2024-04-12T11:18:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:875: finishing write tid 1 to smithi02979277-39 2024-04-12T11:18:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:875: finishing write tid 2 to smithi02979277-39 2024-04-12T11:18:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:874: expect (ObjNum 327 snap 101 seq_num 327) 2024-04-12T11:18:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:00 smithi087 ceph-mon[32917]: pgmap v59: 89 pgs: 89 active+clean; 205 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-04-12T11:18:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:00 smithi087 ceph-mon[32917]: osdmap e286: 8 total, 8 up, 8 in 2024-04-12T11:18:00.993 INFO:tasks.rados.rados.0.smithi029.stdout:875: finishing write tid 3 to smithi02979277-39 2024-04-12T11:18:00.993 INFO:tasks.rados.rados.0.smithi029.stdout:875: finishing write tid 4 to smithi02979277-39 2024-04-12T11:18:00.993 INFO:tasks.rados.rados.0.smithi029.stdout:875: oid 39 updating version 0 to 47 2024-04-12T11:18:00.993 INFO:tasks.rados.rados.0.smithi029.stdout:875: oid 39 updating version 47 to 48 2024-04-12T11:18:00.993 INFO:tasks.rados.rados.0.smithi029.stdout:875: oid 39 version 48 is already newer than 46 2024-04-12T11:18:00.993 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 48 (ObjNum 360 snap 110 seq_num 360) dirty exists 2024-04-12T11:18:00.993 INFO:tasks.rados.rados.0.smithi029.stdout:875: left oid 39 (ObjNum 360 snap 110 seq_num 360) 2024-04-12T11:18:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:00 smithi029 ceph-mon[39999]: pgmap v59: 89 pgs: 89 active+clean; 205 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-04-12T11:18:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:00 smithi029 ceph-mon[39999]: osdmap e286: 8 total, 8 up, 8 in 2024-04-12T11:18:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:00 smithi029 ceph-mon[93650]: pgmap v59: 89 pgs: 89 active+clean; 205 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-04-12T11:18:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:00 smithi029 ceph-mon[93650]: osdmap e286: 8 total, 8 up, 8 in 2024-04-12T11:18:01.605 INFO:tasks.rados.rados.0.smithi029.stdout:870: done (6 left) 2024-04-12T11:18:01.605 INFO:tasks.rados.rados.0.smithi029.stdout:871: done (5 left) 2024-04-12T11:18:01.606 INFO:tasks.rados.rados.0.smithi029.stdout:872: done (4 left) 2024-04-12T11:18:01.606 INFO:tasks.rados.rados.0.smithi029.stdout:873: done (3 left) 2024-04-12T11:18:01.606 INFO:tasks.rados.rados.0.smithi029.stdout:874: done (2 left) 2024-04-12T11:18:01.606 INFO:tasks.rados.rados.0.smithi029.stdout:875: done (1 left) 2024-04-12T11:18:01.606 INFO:tasks.rados.rados.0.smithi029.stdout:876: done (0 left) 2024-04-12T11:18:01.606 INFO:tasks.rados.rados.0.smithi029.stdout:877: write oid 45 current snap is 111 2024-04-12T11:18:01.606 INFO:tasks.rados.rados.0.smithi029.stdout:877: seq_num 361 ranges {467327=633480,1825176=242151} 2024-04-12T11:18:01.611 INFO:tasks.rados.rados.0.smithi029.stdout:877: writing smithi02979277-45 from 467327 to 1100807 tid 1 2024-04-12T11:18:01.617 INFO:tasks.rados.rados.0.smithi029.stdout:877: writing smithi02979277-45 from 1825176 to 2067327 tid 2 2024-04-12T11:18:01.617 INFO:tasks.rados.rados.0.smithi029.stdout:878: rollback oid 30 current snap is 111 2024-04-12T11:18:01.617 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 106 2024-04-12T11:18:01.617 INFO:tasks.rados.rados.0.smithi029.stdout:879: read oid 50 snap -1 2024-04-12T11:18:01.617 INFO:tasks.rados.rados.0.smithi029.stdout:879: expect deleted 2024-04-12T11:18:01.617 INFO:tasks.rados.rados.0.smithi029.stdout:880: delete oid 22 current snap is 111 2024-04-12T11:18:01.618 INFO:tasks.rados.rados.0.smithi029.stdout:877: finishing write tid 1 to smithi02979277-45 2024-04-12T11:18:01.619 INFO:tasks.rados.rados.0.smithi029.stdout:877: finishing write tid 2 to smithi02979277-45 2024-04-12T11:18:01.619 INFO:tasks.rados.rados.0.smithi029.stdout:877: finishing write tid 3 to smithi02979277-45 2024-04-12T11:18:01.619 INFO:tasks.rados.rados.0.smithi029.stdout:878: finishing rollback tid 0 to smithi02979277-30 2024-04-12T11:18:01.620 INFO:tasks.rados.rados.0.smithi029.stdout:878: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:18:01.620 INFO:tasks.rados.rados.0.smithi029.stdout:877: finishing write tid 4 to smithi02979277-45 2024-04-12T11:18:01.620 INFO:tasks.rados.rados.0.smithi029.stdout:877: oid 45 updating version 0 to 494 2024-04-12T11:18:01.620 INFO:tasks.rados.rados.0.smithi029.stdout:877: oid 45 version 494 is already newer than 492 2024-04-12T11:18:01.620 INFO:tasks.rados.rados.0.smithi029.stdout:877: oid 45 version 494 is already newer than 493 2024-04-12T11:18:01.620 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 494 (ObjNum 361 snap 111 seq_num 361) dirty exists 2024-04-12T11:18:01.620 INFO:tasks.rados.rados.0.smithi029.stdout:877: left oid 45 (ObjNum 361 snap 111 seq_num 361) 2024-04-12T11:18:01.620 INFO:tasks.rados.rados.0.smithi029.stdout:878: finishing rollback tid 2 to smithi02979277-30 2024-04-12T11:18:01.620 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 389 (ObjNum 286 snap 89 seq_num 286) dirty exists 2024-04-12T11:18:01.621 INFO:tasks.rados.rados.0.smithi029.stdout:877: done (3 left) 2024-04-12T11:18:01.621 INFO:tasks.rados.rados.0.smithi029.stdout:878: done (2 left) 2024-04-12T11:18:01.621 INFO:tasks.rados.rados.0.smithi029.stdout:879: done (1 left) 2024-04-12T11:18:01.622 INFO:tasks.rados.rados.0.smithi029.stdout:880: done (0 left) 2024-04-12T11:18:01.622 INFO:tasks.rados.rados.0.smithi029.stdout:881: snap_create 2024-04-12T11:18:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:01 smithi087 ceph-mon[32917]: osdmap e287: 8 total, 8 up, 8 in 2024-04-12T11:18:02.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:01 smithi029 ceph-mon[93650]: osdmap e287: 8 total, 8 up, 8 in 2024-04-12T11:18:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:01 smithi029 ceph-mon[39999]: osdmap e287: 8 total, 8 up, 8 in 2024-04-12T11:18:02.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:02.504 INFO:tasks.rados.rados.0.smithi029.stdout:881: done (0 left) 2024-04-12T11:18:02.504 INFO:tasks.rados.rados.0.smithi029.stdout:882: snap_create 2024-04-12T11:18:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:02 smithi087 ceph-mon[32917]: pgmap v62: 89 pgs: 89 active+clean; 205 MiB data, 679 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:02 smithi087 ceph-mon[32917]: osdmap e288: 8 total, 8 up, 8 in 2024-04-12T11:18:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:02 smithi087 ceph-mon[32917]: osdmap e289: 8 total, 8 up, 8 in 2024-04-12T11:18:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:02 smithi029 ceph-mon[93650]: pgmap v62: 89 pgs: 89 active+clean; 205 MiB data, 679 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:02 smithi029 ceph-mon[93650]: osdmap e288: 8 total, 8 up, 8 in 2024-04-12T11:18:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:02 smithi029 ceph-mon[93650]: osdmap e289: 8 total, 8 up, 8 in 2024-04-12T11:18:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:02 smithi029 ceph-mon[39999]: pgmap v62: 89 pgs: 89 active+clean; 205 MiB data, 679 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:02 smithi029 ceph-mon[39999]: osdmap e288: 8 total, 8 up, 8 in 2024-04-12T11:18:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:02 smithi029 ceph-mon[39999]: osdmap e289: 8 total, 8 up, 8 in 2024-04-12T11:18:03.505 INFO:tasks.rados.rados.0.smithi029.stdout:882: done (0 left) 2024-04-12T11:18:03.505 INFO:tasks.rados.rados.0.smithi029.stdout:883: snap_create 2024-04-12T11:18:03.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:04.508 INFO:tasks.rados.rados.0.smithi029.stdout:883: done (0 left) 2024-04-12T11:18:04.508 INFO:tasks.rados.rados.0.smithi029.stdout:884: delete oid 27 current snap is 114 2024-04-12T11:18:04.513 INFO:tasks.rados.rados.0.smithi029.stdout:884: done (0 left) 2024-04-12T11:18:04.514 INFO:tasks.rados.rados.0.smithi029.stdout:885: read oid 34 snap -1 2024-04-12T11:18:04.514 INFO:tasks.rados.rados.0.smithi029.stdout:885: expect deleted 2024-04-12T11:18:04.514 INFO:tasks.rados.rados.0.smithi029.stdout:886: delete oid 17 current snap is 114 2024-04-12T11:18:04.519 INFO:tasks.rados.rados.0.smithi029.stdout:885: done (1 left) 2024-04-12T11:18:04.519 INFO:tasks.rados.rados.0.smithi029.stdout:886: done (0 left) 2024-04-12T11:18:04.519 INFO:tasks.rados.rados.0.smithi029.stdout:887: snap_remove snap 110 2024-04-12T11:18:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:04 smithi029 ceph-mon[93650]: pgmap v65: 89 pgs: 89 active+clean; 215 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-04-12T11:18:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:04 smithi029 ceph-mon[93650]: osdmap e290: 8 total, 8 up, 8 in 2024-04-12T11:18:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:04 smithi029 ceph-mon[39999]: pgmap v65: 89 pgs: 89 active+clean; 215 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-04-12T11:18:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:04 smithi029 ceph-mon[39999]: osdmap e290: 8 total, 8 up, 8 in 2024-04-12T11:18:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:04 smithi087 ceph-mon[32917]: pgmap v65: 89 pgs: 89 active+clean; 215 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-04-12T11:18:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:04 smithi087 ceph-mon[32917]: osdmap e290: 8 total, 8 up, 8 in 2024-04-12T11:18:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:18:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:18:04] "GET /metrics HTTP/1.1" 200 36247 "" "Prometheus/2.43.0" 2024-04-12T11:18:05.518 INFO:tasks.rados.rados.0.smithi029.stdout:887: done (0 left) 2024-04-12T11:18:05.518 INFO:tasks.rados.rados.0.smithi029.stdout:888: read oid 21 snap 107 2024-04-12T11:18:05.518 INFO:tasks.rados.rados.0.smithi029.stdout:888: expect deleted 2024-04-12T11:18:05.519 INFO:tasks.rados.rados.0.smithi029.stdout:889: snap_create 2024-04-12T11:18:05.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:05 smithi029 ceph-mon[93650]: osdmap e291: 8 total, 8 up, 8 in 2024-04-12T11:18:05.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:05 smithi029 ceph-mon[39999]: osdmap e291: 8 total, 8 up, 8 in 2024-04-12T11:18:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:05 smithi087 ceph-mon[32917]: osdmap e291: 8 total, 8 up, 8 in 2024-04-12T11:18:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:888: done (1 left) 2024-04-12T11:18:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:889: done (0 left) 2024-04-12T11:18:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:890: read oid 5 snap -1 2024-04-12T11:18:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:890: expect (ObjNum 352 snap 108 seq_num 352) 2024-04-12T11:18:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:891: read oid 36 snap -1 2024-04-12T11:18:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:891: expect (ObjNum 348 snap 107 seq_num 348) 2024-04-12T11:18:06.520 INFO:tasks.rados.rados.0.smithi029.stdout:892: snap_remove snap 111 2024-04-12T11:18:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:06 smithi087 ceph-mon[32917]: pgmap v68: 89 pgs: 89 active+clean; 215 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-04-12T11:18:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:06 smithi087 ceph-mon[32917]: osdmap e292: 8 total, 8 up, 8 in 2024-04-12T11:18:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:06 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:18:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:06 smithi029 ceph-mon[93650]: pgmap v68: 89 pgs: 89 active+clean; 215 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-04-12T11:18:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:06 smithi029 ceph-mon[93650]: osdmap e292: 8 total, 8 up, 8 in 2024-04-12T11:18:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:18:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:06 smithi029 ceph-mon[39999]: pgmap v68: 89 pgs: 89 active+clean; 215 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-04-12T11:18:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:06 smithi029 ceph-mon[39999]: osdmap e292: 8 total, 8 up, 8 in 2024-04-12T11:18:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:06 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:18:07.532 INFO:tasks.rados.rados.0.smithi029.stdout:892: done (2 left) 2024-04-12T11:18:07.532 INFO:tasks.rados.rados.0.smithi029.stdout:893: snap_create 2024-04-12T11:18:07.532 INFO:tasks.rados.rados.0.smithi029.stdout:890: expect (ObjNum 352 snap 108 seq_num 352) 2024-04-12T11:18:07.796 INFO:tasks.rados.rados.0.smithi029.stdout:891: expect (ObjNum 348 snap 107 seq_num 348) 2024-04-12T11:18:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:07 smithi087 ceph-mon[32917]: osdmap e293: 8 total, 8 up, 8 in 2024-04-12T11:18:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:07 smithi029 ceph-mon[93650]: osdmap e293: 8 total, 8 up, 8 in 2024-04-12T11:18:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:07 smithi029 ceph-mon[39999]: osdmap e293: 8 total, 8 up, 8 in 2024-04-12T11:18:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:890: done (2 left) 2024-04-12T11:18:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:891: done (1 left) 2024-04-12T11:18:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:893: done (0 left) 2024-04-12T11:18:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:894: write oid 35 current snap is 116 2024-04-12T11:18:08.538 INFO:tasks.rados.rados.0.smithi029.stdout:894: seq_num 365 ranges {534555=650829,1659104=475451} 2024-04-12T11:18:08.544 INFO:tasks.rados.rados.0.smithi029.stdout:894: writing smithi02979277-35 from 534555 to 1185384 tid 1 2024-04-12T11:18:08.549 INFO:tasks.rados.rados.0.smithi029.stdout:894: writing smithi02979277-35 from 1659104 to 2134555 tid 2 2024-04-12T11:18:08.550 INFO:tasks.rados.rados.0.smithi029.stdout:895: read oid 45 snap -1 2024-04-12T11:18:08.550 INFO:tasks.rados.rados.0.smithi029.stdout:895: expect (ObjNum 361 snap 111 seq_num 361) 2024-04-12T11:18:08.550 INFO:tasks.rados.rados.0.smithi029.stdout:896: snap_remove snap 102 2024-04-12T11:18:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:08 smithi087 ceph-mon[32917]: pgmap v71: 89 pgs: 89 active+clean; 215 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:18:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:08 smithi087 ceph-mon[32917]: osdmap e294: 8 total, 8 up, 8 in 2024-04-12T11:18:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:08 smithi029 ceph-mon[93650]: pgmap v71: 89 pgs: 89 active+clean; 215 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:18:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:08 smithi029 ceph-mon[93650]: osdmap e294: 8 total, 8 up, 8 in 2024-04-12T11:18:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:08 smithi029 ceph-mon[39999]: pgmap v71: 89 pgs: 89 active+clean; 215 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:18:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:08 smithi029 ceph-mon[39999]: osdmap e294: 8 total, 8 up, 8 in 2024-04-12T11:18:09.545 INFO:tasks.rados.rados.0.smithi029.stdout:896: done (2 left) 2024-04-12T11:18:09.545 INFO:tasks.rados.rados.0.smithi029.stdout:897: write oid 18 current snap is 116 2024-04-12T11:18:09.545 INFO:tasks.rados.rados.0.smithi029.stdout:897: seq_num 366 ranges {551362=726078,1717586=409779} 2024-04-12T11:18:09.552 INFO:tasks.rados.rados.0.smithi029.stdout:897: writing smithi02979277-18 from 551362 to 1277440 tid 1 2024-04-12T11:18:09.557 INFO:tasks.rados.rados.0.smithi029.stdout:897: writing smithi02979277-18 from 1717586 to 2127365 tid 2 2024-04-12T11:18:09.557 INFO:tasks.rados.rados.0.smithi029.stdout:898: snap_remove snap 109 2024-04-12T11:18:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:09 smithi087 ceph-mon[32917]: osdmap e295: 8 total, 8 up, 8 in 2024-04-12T11:18:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:09 smithi029 ceph-mon[93650]: osdmap e295: 8 total, 8 up, 8 in 2024-04-12T11:18:10.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:09 smithi029 ceph-mon[39999]: osdmap e295: 8 total, 8 up, 8 in 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:898: done (3 left) 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:899: rollback oid 15 current snap is 116 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 15 to 115 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:894: finishing write tid 1 to smithi02979277-35 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:894: finishing write tid 2 to smithi02979277-35 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:894: finishing write tid 3 to smithi02979277-35 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:894: finishing write tid 4 to smithi02979277-35 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:894: oid 35 updating version 0 to 580 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:894: oid 35 updating version 580 to 581 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:894: oid 35 updating version 581 to 582 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 582 (ObjNum 365 snap 116 seq_num 365) dirty exists 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:894: left oid 35 (ObjNum 365 snap 116 seq_num 365) 2024-04-12T11:18:10.549 INFO:tasks.rados.rados.0.smithi029.stdout:895: expect (ObjNum 361 snap 111 seq_num 361) 2024-04-12T11:18:10.620 INFO:tasks.rados.rados.0.smithi029.stdout:897: finishing write tid 1 to smithi02979277-18 2024-04-12T11:18:10.620 INFO:tasks.rados.rados.0.smithi029.stdout:897: finishing write tid 2 to smithi02979277-18 2024-04-12T11:18:10.620 INFO:tasks.rados.rados.0.smithi029.stdout:897: finishing write tid 3 to smithi02979277-18 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:897: finishing write tid 4 to smithi02979277-18 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:897: oid 18 updating version 0 to 535 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:897: oid 18 version 535 is already newer than 531 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:897: oid 18 version 535 is already newer than 534 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 535 (ObjNum 366 snap 116 seq_num 366) dirty exists 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:897: left oid 18 (ObjNum 366 snap 116 seq_num 366) 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:894: done (3 left) 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:895: done (2 left) 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:897: done (1 left) 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:900: read oid 33 snap -1 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:900: expect (ObjNum 311 snap 94 seq_num 311) 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:901: snap_create 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:899: finishing rollback tid 0 to smithi02979277-15 2024-04-12T11:18:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:899: finishing rollback tid 1 to smithi02979277-15 2024-04-12T11:18:10.622 INFO:tasks.rados.rados.0.smithi029.stdout:899: finishing rollback tid 2 to smithi02979277-15 2024-04-12T11:18:10.622 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 405 (ObjNum 288 snap 90 seq_num 288) dirty exists 2024-04-12T11:18:10.633 INFO:tasks.rados.rados.0.smithi029.stdout:900: expect (ObjNum 311 snap 94 seq_num 311) 2024-04-12T11:18:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:10 smithi087 ceph-mon[32917]: pgmap v74: 89 pgs: 89 active+clean; 215 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:18:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:10 smithi087 ceph-mon[32917]: osdmap e296: 8 total, 8 up, 8 in 2024-04-12T11:18:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:10 smithi029 ceph-mon[39999]: pgmap v74: 89 pgs: 89 active+clean; 215 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:18:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:10 smithi029 ceph-mon[39999]: osdmap e296: 8 total, 8 up, 8 in 2024-04-12T11:18:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:10 smithi029 ceph-mon[93650]: pgmap v74: 89 pgs: 89 active+clean; 215 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 0 B/s wr, 4 op/s 2024-04-12T11:18:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:10 smithi029 ceph-mon[93650]: osdmap e296: 8 total, 8 up, 8 in 2024-04-12T11:18:11.554 INFO:tasks.rados.rados.0.smithi029.stdout:899: done (2 left) 2024-04-12T11:18:11.554 INFO:tasks.rados.rados.0.smithi029.stdout:900: done (1 left) 2024-04-12T11:18:11.554 INFO:tasks.rados.rados.0.smithi029.stdout:901: done (0 left) 2024-04-12T11:18:11.554 INFO:tasks.rados.rados.0.smithi029.stdout:902: write oid 12 current snap is 117 2024-04-12T11:18:11.555 INFO:tasks.rados.rados.0.smithi029.stdout:902: seq_num 367 ranges {568169=401327,1376068=669790} 2024-04-12T11:18:11.559 INFO:tasks.rados.rados.0.smithi029.stdout:902: writing smithi02979277-12 from 568169 to 969496 tid 1 2024-04-12T11:18:11.565 INFO:tasks.rados.rados.0.smithi029.stdout:902: writing smithi02979277-12 from 1376068 to 2045858 tid 2 2024-04-12T11:18:11.565 INFO:tasks.rados.rados.0.smithi029.stdout:903: delete oid 36 current snap is 117 2024-04-12T11:18:11.566 INFO:tasks.rados.rados.0.smithi029.stdout:902: finishing write tid 1 to smithi02979277-12 2024-04-12T11:18:11.568 INFO:tasks.rados.rados.0.smithi029.stdout:903: done (1 left) 2024-04-12T11:18:11.568 INFO:tasks.rados.rados.0.smithi029.stdout:904: read oid 15 snap -1 2024-04-12T11:18:11.568 INFO:tasks.rados.rados.0.smithi029.stdout:904: expect (ObjNum 288 snap 90 seq_num 288) 2024-04-12T11:18:11.568 INFO:tasks.rados.rados.0.smithi029.stdout:905: delete oid 17 current snap is 117 2024-04-12T11:18:11.569 INFO:tasks.rados.rados.0.smithi029.stdout:902: finishing write tid 2 to smithi02979277-12 2024-04-12T11:18:11.570 INFO:tasks.rados.rados.0.smithi029.stdout:902: finishing write tid 3 to smithi02979277-12 2024-04-12T11:18:11.572 INFO:tasks.rados.rados.0.smithi029.stdout:902: finishing write tid 4 to smithi02979277-12 2024-04-12T11:18:11.572 INFO:tasks.rados.rados.0.smithi029.stdout:902: oid 12 updating version 0 to 501 2024-04-12T11:18:11.572 INFO:tasks.rados.rados.0.smithi029.stdout:902: oid 12 updating version 501 to 502 2024-04-12T11:18:11.572 INFO:tasks.rados.rados.0.smithi029.stdout:902: oid 12 version 502 is already newer than 500 2024-04-12T11:18:11.572 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 502 (ObjNum 367 snap 117 seq_num 367) dirty exists 2024-04-12T11:18:11.572 INFO:tasks.rados.rados.0.smithi029.stdout:902: left oid 12 (ObjNum 367 snap 117 seq_num 367) 2024-04-12T11:18:11.572 INFO:tasks.rados.rados.0.smithi029.stdout:904: expect (ObjNum 288 snap 90 seq_num 288) 2024-04-12T11:18:11.765 INFO:tasks.rados.rados.0.smithi029.stdout:902: done (2 left) 2024-04-12T11:18:11.766 INFO:tasks.rados.rados.0.smithi029.stdout:904: done (1 left) 2024-04-12T11:18:11.766 INFO:tasks.rados.rados.0.smithi029.stdout:905: done (0 left) 2024-04-12T11:18:11.766 INFO:tasks.rados.rados.0.smithi029.stdout:906: snap_create 2024-04-12T11:18:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:11 smithi087 ceph-mon[32917]: osdmap e297: 8 total, 8 up, 8 in 2024-04-12T11:18:12.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:11 smithi029 ceph-mon[93650]: osdmap e297: 8 total, 8 up, 8 in 2024-04-12T11:18:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:11 smithi029 ceph-mon[39999]: osdmap e297: 8 total, 8 up, 8 in 2024-04-12T11:18:12.507 INFO:tasks.rados.rados.0.smithi029.stdout:906: done (0 left) 2024-04-12T11:18:12.507 INFO:tasks.rados.rados.0.smithi029.stdout:907: delete oid 23 current snap is 118 2024-04-12T11:18:12.512 INFO:tasks.rados.rados.0.smithi029.stdout:907: done (0 left) 2024-04-12T11:18:12.512 INFO:tasks.rados.rados.0.smithi029.stdout:908: delete oid 30 current snap is 118 2024-04-12T11:18:12.515 INFO:tasks.rados.rados.0.smithi029.stdout:908: done (0 left) 2024-04-12T11:18:12.515 INFO:tasks.rados.rados.0.smithi029.stdout:909: write oid 39 current snap is 118 2024-04-12T11:18:12.515 INFO:tasks.rados.rados.0.smithi029.stdout:909: seq_num 372 ranges {652204=777572,1952125=300079} 2024-04-12T11:18:12.523 INFO:tasks.rados.rados.0.smithi029.stdout:909: writing smithi02979277-39 from 652204 to 1429776 tid 1 2024-04-12T11:18:12.528 INFO:tasks.rados.rados.0.smithi029.stdout:909: writing smithi02979277-39 from 1952125 to 2252204 tid 2 2024-04-12T11:18:12.528 INFO:tasks.rados.rados.0.smithi029.stdout:910: snap_create 2024-04-12T11:18:12.530 INFO:tasks.rados.rados.0.smithi029.stdout:909: finishing write tid 1 to smithi02979277-39 2024-04-12T11:18:12.530 INFO:tasks.rados.rados.0.smithi029.stdout:909: finishing write tid 2 to smithi02979277-39 2024-04-12T11:18:12.530 INFO:tasks.rados.rados.0.smithi029.stdout:909: finishing write tid 3 to smithi02979277-39 2024-04-12T11:18:12.531 INFO:tasks.rados.rados.0.smithi029.stdout:909: finishing write tid 4 to smithi02979277-39 2024-04-12T11:18:12.531 INFO:tasks.rados.rados.0.smithi029.stdout:909: oid 39 updating version 0 to 52 2024-04-12T11:18:12.531 INFO:tasks.rados.rados.0.smithi029.stdout:909: oid 39 updating version 52 to 53 2024-04-12T11:18:12.531 INFO:tasks.rados.rados.0.smithi029.stdout:909: oid 39 updating version 53 to 54 2024-04-12T11:18:12.531 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 54 (ObjNum 372 snap 118 seq_num 372) dirty exists 2024-04-12T11:18:12.531 INFO:tasks.rados.rados.0.smithi029.stdout:909: left oid 39 (ObjNum 372 snap 118 seq_num 372) 2024-04-12T11:18:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:12 smithi087 ceph-mon[32917]: pgmap v77: 89 pgs: 89 active+clean; 215 MiB data, 677 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:12 smithi087 ceph-mon[32917]: osdmap e298: 8 total, 8 up, 8 in 2024-04-12T11:18:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:12 smithi087 ceph-mon[32917]: osdmap e299: 8 total, 8 up, 8 in 2024-04-12T11:18:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:12 smithi029 ceph-mon[39999]: pgmap v77: 89 pgs: 89 active+clean; 215 MiB data, 677 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:12 smithi029 ceph-mon[39999]: osdmap e298: 8 total, 8 up, 8 in 2024-04-12T11:18:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:12 smithi029 ceph-mon[39999]: osdmap e299: 8 total, 8 up, 8 in 2024-04-12T11:18:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:12 smithi029 ceph-mon[93650]: pgmap v77: 89 pgs: 89 active+clean; 215 MiB data, 677 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:12 smithi029 ceph-mon[93650]: osdmap e298: 8 total, 8 up, 8 in 2024-04-12T11:18:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:12 smithi029 ceph-mon[93650]: osdmap e299: 8 total, 8 up, 8 in 2024-04-12T11:18:13.508 INFO:tasks.rados.rados.0.smithi029.stdout:909: done (1 left) 2024-04-12T11:18:13.509 INFO:tasks.rados.rados.0.smithi029.stdout:910: done (0 left) 2024-04-12T11:18:13.509 INFO:tasks.rados.rados.0.smithi029.stdout:911: delete oid 43 current snap is 119 2024-04-12T11:18:13.513 INFO:tasks.rados.rados.0.smithi029.stdout:911: done (0 left) 2024-04-12T11:18:13.513 INFO:tasks.rados.rados.0.smithi029.stdout:912: delete oid 5 current snap is 119 2024-04-12T11:18:13.515 INFO:tasks.rados.rados.0.smithi029.stdout:912: done (0 left) 2024-04-12T11:18:13.515 INFO:tasks.rados.rados.0.smithi029.stdout:913: read oid 3 snap -1 2024-04-12T11:18:13.515 INFO:tasks.rados.rados.0.smithi029.stdout:913: expect deleted 2024-04-12T11:18:13.515 INFO:tasks.rados.rados.0.smithi029.stdout:914: rollback oid 37 current snap is 119 2024-04-12T11:18:13.515 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 115 2024-04-12T11:18:13.515 INFO:tasks.rados.rados.0.smithi029.stdout:915: snap_remove snap 104 2024-04-12T11:18:13.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:14.515 INFO:tasks.rados.rados.0.smithi029.stdout:915: done (2 left) 2024-04-12T11:18:14.515 INFO:tasks.rados.rados.0.smithi029.stdout:916: rollback oid 40 current snap is 119 2024-04-12T11:18:14.515 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 40 to 115 2024-04-12T11:18:14.515 INFO:tasks.rados.rados.0.smithi029.stdout:917: write oid 2 current snap is 119 2024-04-12T11:18:14.515 INFO:tasks.rados.rados.0.smithi029.stdout:917: seq_num 375 ranges {702625=719672,2127571=175054} 2024-04-12T11:18:14.524 INFO:tasks.rados.rados.0.smithi029.stdout:917: writing smithi02979277-2 from 702625 to 1422297 tid 1 2024-04-12T11:18:14.529 INFO:tasks.rados.rados.0.smithi029.stdout:917: writing smithi02979277-2 from 2127571 to 2302625 tid 2 2024-04-12T11:18:14.529 INFO:tasks.rados.rados.0.smithi029.stdout:918: delete oid 25 current snap is 119 2024-04-12T11:18:14.530 INFO:tasks.rados.rados.0.smithi029.stdout:914: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:18:14.530 INFO:tasks.rados.rados.0.smithi029.stdout:914: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:18:14.530 INFO:tasks.rados.rados.0.smithi029.stdout:914: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:18:14.530 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 398 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-04-12T11:18:14.530 INFO:tasks.rados.rados.0.smithi029.stdout:916: finishing rollback tid 1 to smithi02979277-40 2024-04-12T11:18:14.530 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 561 (ObjNum 97 snap 0 seq_num 862035191) dirty dne 2024-04-12T11:18:14.530 INFO:tasks.rados.rados.0.smithi029.stdout:917: finishing write tid 1 to smithi02979277-2 2024-04-12T11:18:14.532 INFO:tasks.rados.rados.0.smithi029.stdout:917: finishing write tid 2 to smithi02979277-2 2024-04-12T11:18:14.532 INFO:tasks.rados.rados.0.smithi029.stdout:917: finishing write tid 3 to smithi02979277-2 2024-04-12T11:18:14.532 INFO:tasks.rados.rados.0.smithi029.stdout:913: done (4 left) 2024-04-12T11:18:14.532 INFO:tasks.rados.rados.0.smithi029.stdout:914: done (3 left) 2024-04-12T11:18:14.532 INFO:tasks.rados.rados.0.smithi029.stdout:916: done (2 left) 2024-04-12T11:18:14.532 INFO:tasks.rados.rados.0.smithi029.stdout:918: done (1 left) 2024-04-12T11:18:14.532 INFO:tasks.rados.rados.0.smithi029.stdout:919: rollback oid 27 current snap is 119 2024-04-12T11:18:14.532 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 105 2024-04-12T11:18:14.532 INFO:tasks.rados.rados.0.smithi029.stdout:920: read oid 38 snap -1 2024-04-12T11:18:14.533 INFO:tasks.rados.rados.0.smithi029.stdout:920: expect (ObjNum 220 snap 63 seq_num 220) 2024-04-12T11:18:14.533 INFO:tasks.rados.rados.0.smithi029.stdout:921: read oid 33 snap -1 2024-04-12T11:18:14.533 INFO:tasks.rados.rados.0.smithi029.stdout:921: expect (ObjNum 311 snap 94 seq_num 311) 2024-04-12T11:18:14.533 INFO:tasks.rados.rados.0.smithi029.stdout:922: rollback oid 7 current snap is 119 2024-04-12T11:18:14.533 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 118 2024-04-12T11:18:14.533 INFO:tasks.rados.rados.0.smithi029.stdout:923: snap_remove snap 112 2024-04-12T11:18:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:14 smithi029 ceph-mon[93650]: pgmap v80: 89 pgs: 89 active+clean; 214 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 815 KiB/s wr, 10 op/s 2024-04-12T11:18:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:14 smithi029 ceph-mon[93650]: osdmap e300: 8 total, 8 up, 8 in 2024-04-12T11:18:14.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:14 smithi029 ceph-mon[39999]: pgmap v80: 89 pgs: 89 active+clean; 214 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 815 KiB/s wr, 10 op/s 2024-04-12T11:18:14.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:14 smithi029 ceph-mon[39999]: osdmap e300: 8 total, 8 up, 8 in 2024-04-12T11:18:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:14 smithi087 ceph-mon[32917]: pgmap v80: 89 pgs: 89 active+clean; 214 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 815 KiB/s wr, 10 op/s 2024-04-12T11:18:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:14 smithi087 ceph-mon[32917]: osdmap e300: 8 total, 8 up, 8 in 2024-04-12T11:18:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:18:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:18:14] "GET /metrics HTTP/1.1" 200 36249 "" "Prometheus/2.43.0" 2024-04-12T11:18:15.519 INFO:tasks.rados.rados.0.smithi029.stdout:917: finishing write tid 4 to smithi02979277-2 2024-04-12T11:18:15.520 INFO:tasks.rados.rados.0.smithi029.stdout:917: oid 2 updating version 0 to 45 2024-04-12T11:18:15.520 INFO:tasks.rados.rados.0.smithi029.stdout:917: oid 2 version 45 is already newer than 43 2024-04-12T11:18:15.520 INFO:tasks.rados.rados.0.smithi029.stdout:917: oid 2 version 45 is already newer than 44 2024-04-12T11:18:15.520 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 45 (ObjNum 375 snap 119 seq_num 375) dirty exists 2024-04-12T11:18:15.520 INFO:tasks.rados.rados.0.smithi029.stdout:917: left oid 2 (ObjNum 375 snap 119 seq_num 375) 2024-04-12T11:18:15.520 INFO:tasks.rados.rados.0.smithi029.stdout:920: expect (ObjNum 220 snap 63 seq_num 220) 2024-04-12T11:18:15.697 INFO:tasks.rados.rados.0.smithi029.stdout:922: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:18:15.698 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 339 (ObjNum 129 snap 0 seq_num 1800911264) dirty dne 2024-04-12T11:18:15.698 INFO:tasks.rados.rados.0.smithi029.stdout:919: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:18:15.698 INFO:tasks.rados.rados.0.smithi029.stdout:919: finishing rollback tid 2 to smithi02979277-27 2024-04-12T11:18:15.698 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 419 (ObjNum 318 snap 97 seq_num 318) dirty exists 2024-04-12T11:18:15.698 INFO:tasks.rados.rados.0.smithi029.stdout:921: expect (ObjNum 311 snap 94 seq_num 311) 2024-04-12T11:18:15.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:15 smithi029 ceph-mon[39999]: osdmap e301: 8 total, 8 up, 8 in 2024-04-12T11:18:15.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:15 smithi029 ceph-mon[93650]: osdmap e301: 8 total, 8 up, 8 in 2024-04-12T11:18:15.840 INFO:tasks.rados.rados.0.smithi029.stdout:917: done (5 left) 2024-04-12T11:18:15.840 INFO:tasks.rados.rados.0.smithi029.stdout:919: done (4 left) 2024-04-12T11:18:15.840 INFO:tasks.rados.rados.0.smithi029.stdout:920: done (3 left) 2024-04-12T11:18:15.841 INFO:tasks.rados.rados.0.smithi029.stdout:921: done (2 left) 2024-04-12T11:18:15.841 INFO:tasks.rados.rados.0.smithi029.stdout:922: done (1 left) 2024-04-12T11:18:15.841 INFO:tasks.rados.rados.0.smithi029.stdout:923: done (0 left) 2024-04-12T11:18:15.841 INFO:tasks.rados.rados.0.smithi029.stdout:924: rollback oid 41 current snap is 119 2024-04-12T11:18:15.841 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 86 2024-04-12T11:18:15.841 INFO:tasks.rados.rados.0.smithi029.stdout:925: snap_remove snap 115 2024-04-12T11:18:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:15 smithi087 ceph-mon[32917]: osdmap e301: 8 total, 8 up, 8 in 2024-04-12T11:18:16.525 INFO:tasks.rados.rados.0.smithi029.stdout:925: done (1 left) 2024-04-12T11:18:16.525 INFO:tasks.rados.rados.0.smithi029.stdout:924: finishing rollback tid 0 to smithi02979277-41 2024-04-12T11:18:16.526 INFO:tasks.rados.rados.0.smithi029.stdout:924: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:18:16.526 INFO:tasks.rados.rados.0.smithi029.stdout:924: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:18:16.526 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 470 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-04-12T11:18:16.526 INFO:tasks.rados.rados.0.smithi029.stdout:926: read oid 27 snap -1 2024-04-12T11:18:16.526 INFO:tasks.rados.rados.0.smithi029.stdout:926: expect (ObjNum 318 snap 97 seq_num 318) 2024-04-12T11:18:16.526 INFO:tasks.rados.rados.0.smithi029.stdout:924: done (1 left) 2024-04-12T11:18:16.526 INFO:tasks.rados.rados.0.smithi029.stdout:927: write oid 35 current snap is 119 2024-04-12T11:18:16.526 INFO:tasks.rados.rados.0.smithi029.stdout:927: seq_num 377 ranges {736239=470170,1728182=571782} 2024-04-12T11:18:16.532 INFO:tasks.rados.rados.0.smithi029.stdout:927: writing smithi02979277-35 from 736239 to 1206409 tid 1 2024-04-12T11:18:16.539 INFO:tasks.rados.rados.0.smithi029.stdout:927: writing smithi02979277-35 from 1728182 to 2299964 tid 2 2024-04-12T11:18:16.539 INFO:tasks.rados.rados.0.smithi029.stdout:928: write oid 17 current snap is 119 2024-04-12T11:18:16.540 INFO:tasks.rados.rados.0.smithi029.stdout:928: seq_num 378 ranges {753046=545419,1786664=431793} 2024-04-12T11:18:16.546 INFO:tasks.rados.rados.0.smithi029.stdout:928: writing smithi02979277-17 from 753046 to 1298465 tid 1 2024-04-12T11:18:16.552 INFO:tasks.rados.rados.0.smithi029.stdout:928: writing smithi02979277-17 from 1786664 to 2218457 tid 2 2024-04-12T11:18:16.552 INFO:tasks.rados.rados.0.smithi029.stdout:929: read oid 31 snap -1 2024-04-12T11:18:16.552 INFO:tasks.rados.rados.0.smithi029.stdout:929: expect deleted 2024-04-12T11:18:16.552 INFO:tasks.rados.rados.0.smithi029.stdout:930: snap_remove snap 116 2024-04-12T11:18:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:16 smithi029 ceph-mon[93650]: pgmap v83: 89 pgs: 89 active+clean; 214 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 815 KiB/s wr, 10 op/s 2024-04-12T11:18:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:16 smithi029 ceph-mon[93650]: osdmap e302: 8 total, 8 up, 8 in 2024-04-12T11:18:16.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:16 smithi029 ceph-mon[39999]: pgmap v83: 89 pgs: 89 active+clean; 214 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 815 KiB/s wr, 10 op/s 2024-04-12T11:18:16.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:16 smithi029 ceph-mon[39999]: osdmap e302: 8 total, 8 up, 8 in 2024-04-12T11:18:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:16 smithi087 ceph-mon[32917]: pgmap v83: 89 pgs: 89 active+clean; 214 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 815 KiB/s wr, 10 op/s 2024-04-12T11:18:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:16 smithi087 ceph-mon[32917]: osdmap e302: 8 total, 8 up, 8 in 2024-04-12T11:18:17.530 INFO:tasks.rados.rados.0.smithi029.stdout:926: expect (ObjNum 318 snap 97 seq_num 318) 2024-04-12T11:18:17.700 INFO:tasks.rados.rados.0.smithi029.stdout:927: finishing write tid 1 to smithi02979277-35 2024-04-12T11:18:17.700 INFO:tasks.rados.rados.0.smithi029.stdout:927: finishing write tid 2 to smithi02979277-35 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:927: finishing write tid 3 to smithi02979277-35 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:927: finishing write tid 4 to smithi02979277-35 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:927: oid 35 updating version 0 to 602 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:927: oid 35 version 602 is already newer than 601 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:927: oid 35 updating version 602 to 603 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 603 (ObjNum 377 snap 119 seq_num 377) dirty exists 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:927: left oid 35 (ObjNum 377 snap 119 seq_num 377) 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:928: finishing write tid 1 to smithi02979277-17 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:926: done (4 left) 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:927: done (3 left) 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:930: done (2 left) 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:931: read oid 2 snap -1 2024-04-12T11:18:17.701 INFO:tasks.rados.rados.0.smithi029.stdout:931: expect (ObjNum 375 snap 119 seq_num 375) 2024-04-12T11:18:17.702 INFO:tasks.rados.rados.0.smithi029.stdout:932: snap_create 2024-04-12T11:18:17.702 INFO:tasks.rados.rados.0.smithi029.stdout:928: finishing write tid 2 to smithi02979277-17 2024-04-12T11:18:17.702 INFO:tasks.rados.rados.0.smithi029.stdout:928: finishing write tid 3 to smithi02979277-17 2024-04-12T11:18:17.702 INFO:tasks.rados.rados.0.smithi029.stdout:928: finishing write tid 4 to smithi02979277-17 2024-04-12T11:18:17.702 INFO:tasks.rados.rados.0.smithi029.stdout:928: oid 17 updating version 0 to 557 2024-04-12T11:18:17.702 INFO:tasks.rados.rados.0.smithi029.stdout:928: oid 17 updating version 557 to 559 2024-04-12T11:18:17.702 INFO:tasks.rados.rados.0.smithi029.stdout:928: oid 17 version 559 is already newer than 558 2024-04-12T11:18:17.702 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 559 (ObjNum 378 snap 119 seq_num 378) dirty exists 2024-04-12T11:18:17.702 INFO:tasks.rados.rados.0.smithi029.stdout:928: left oid 17 (ObjNum 378 snap 119 seq_num 378) 2024-04-12T11:18:17.705 INFO:tasks.rados.rados.0.smithi029.stdout:931: expect (ObjNum 375 snap 119 seq_num 375) 2024-04-12T11:18:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:17 smithi087 ceph-mon[32917]: osdmap e303: 8 total, 8 up, 8 in 2024-04-12T11:18:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:17 smithi029 ceph-mon[93650]: osdmap e303: 8 total, 8 up, 8 in 2024-04-12T11:18:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:17 smithi029 ceph-mon[39999]: osdmap e303: 8 total, 8 up, 8 in 2024-04-12T11:18:18.531 INFO:tasks.rados.rados.0.smithi029.stdout:928: done (3 left) 2024-04-12T11:18:18.531 INFO:tasks.rados.rados.0.smithi029.stdout:929: done (2 left) 2024-04-12T11:18:18.531 INFO:tasks.rados.rados.0.smithi029.stdout:931: done (1 left) 2024-04-12T11:18:18.531 INFO:tasks.rados.rados.0.smithi029.stdout:932: done (0 left) 2024-04-12T11:18:18.531 INFO:tasks.rados.rados.0.smithi029.stdout:933: rollback oid 28 current snap is 120 2024-04-12T11:18:18.531 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 28 to 117 2024-04-12T11:18:18.532 INFO:tasks.rados.rados.0.smithi029.stdout:934: read oid 29 snap -1 2024-04-12T11:18:18.532 INFO:tasks.rados.rados.0.smithi029.stdout:934: expect deleted 2024-04-12T11:18:18.532 INFO:tasks.rados.rados.0.smithi029.stdout:935: rollback oid 42 current snap is 120 2024-04-12T11:18:18.532 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 119 2024-04-12T11:18:18.532 INFO:tasks.rados.rados.0.smithi029.stdout:936: read oid 46 snap -1 2024-04-12T11:18:18.532 INFO:tasks.rados.rados.0.smithi029.stdout:936: expect (ObjNum 334 snap 103 seq_num 334) 2024-04-12T11:18:18.532 INFO:tasks.rados.rados.0.smithi029.stdout:937: write oid 1 current snap is 120 2024-04-12T11:18:18.532 INFO:tasks.rados.rados.0.smithi029.stdout:937: seq_num 379 ranges {769853=620668,1845146=524707} 2024-04-12T11:18:18.538 INFO:tasks.rados.rados.0.smithi029.stdout:937: writing smithi02979277-1 from 769853 to 1390521 tid 1 2024-04-12T11:18:18.545 INFO:tasks.rados.rados.0.smithi029.stdout:937: writing smithi02979277-1 from 1845146 to 2369853 tid 2 2024-04-12T11:18:18.545 INFO:tasks.rados.rados.0.smithi029.stdout:938: read oid 30 snap -1 2024-04-12T11:18:18.545 INFO:tasks.rados.rados.0.smithi029.stdout:938: expect deleted 2024-04-12T11:18:18.546 INFO:tasks.rados.rados.0.smithi029.stdout:939: delete oid 48 current snap is 120 2024-04-12T11:18:18.546 INFO:tasks.rados.rados.0.smithi029.stdout:935: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:18:18.546 INFO:tasks.rados.rados.0.smithi029.stdout:935: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:18:18.546 INFO:tasks.rados.rados.0.smithi029.stdout:935: finishing rollback tid 2 to smithi02979277-42 2024-04-12T11:18:18.546 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 150 (ObjNum 322 snap 100 seq_num 322) dirty exists 2024-04-12T11:18:18.546 INFO:tasks.rados.rados.0.smithi029.stdout:937: finishing write tid 1 to smithi02979277-1 2024-04-12T11:18:18.546 INFO:tasks.rados.rados.0.smithi029.stdout:933: finishing rollback tid 0 to smithi02979277-28 2024-04-12T11:18:18.546 INFO:tasks.rados.rados.0.smithi029.stdout:933: finishing rollback tid 1 to smithi02979277-28 2024-04-12T11:18:18.548 INFO:tasks.rados.rados.0.smithi029.stdout:933: finishing rollback tid 2 to smithi02979277-28 2024-04-12T11:18:18.549 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 381 (ObjNum 359 snap 110 seq_num 359) dirty exists 2024-04-12T11:18:18.549 INFO:tasks.rados.rados.0.smithi029.stdout:937: finishing write tid 2 to smithi02979277-1 2024-04-12T11:18:18.549 INFO:tasks.rados.rados.0.smithi029.stdout:937: finishing write tid 3 to smithi02979277-1 2024-04-12T11:18:18.549 INFO:tasks.rados.rados.0.smithi029.stdout:936: expect (ObjNum 334 snap 103 seq_num 334) 2024-04-12T11:18:18.744 INFO:tasks.rados.rados.0.smithi029.stdout:937: finishing write tid 4 to smithi02979277-1 2024-04-12T11:18:18.744 INFO:tasks.rados.rados.0.smithi029.stdout:937: oid 1 updating version 0 to 477 2024-04-12T11:18:18.744 INFO:tasks.rados.rados.0.smithi029.stdout:937: oid 1 version 477 is already newer than 476 2024-04-12T11:18:18.744 INFO:tasks.rados.rados.0.smithi029.stdout:937: oid 1 version 477 is already newer than 475 2024-04-12T11:18:18.744 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 477 (ObjNum 379 snap 120 seq_num 379) dirty exists 2024-04-12T11:18:18.744 INFO:tasks.rados.rados.0.smithi029.stdout:937: left oid 1 (ObjNum 379 snap 120 seq_num 379) 2024-04-12T11:18:18.745 INFO:tasks.rados.rados.0.smithi029.stdout:933: done (6 left) 2024-04-12T11:18:18.745 INFO:tasks.rados.rados.0.smithi029.stdout:934: done (5 left) 2024-04-12T11:18:18.745 INFO:tasks.rados.rados.0.smithi029.stdout:935: done (4 left) 2024-04-12T11:18:18.745 INFO:tasks.rados.rados.0.smithi029.stdout:936: done (3 left) 2024-04-12T11:18:18.745 INFO:tasks.rados.rados.0.smithi029.stdout:937: done (2 left) 2024-04-12T11:18:18.745 INFO:tasks.rados.rados.0.smithi029.stdout:938: done (1 left) 2024-04-12T11:18:18.745 INFO:tasks.rados.rados.0.smithi029.stdout:939: done (0 left) 2024-04-12T11:18:18.745 INFO:tasks.rados.rados.0.smithi029.stdout:940: snap_remove snap 114 2024-04-12T11:18:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:18 smithi087 ceph-mon[32917]: pgmap v86: 89 pgs: 89 active+clean; 222 MiB data, 690 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 975 KiB/s wr, 14 op/s 2024-04-12T11:18:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:18 smithi087 ceph-mon[32917]: osdmap e304: 8 total, 8 up, 8 in 2024-04-12T11:18:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:18 smithi029 ceph-mon[93650]: pgmap v86: 89 pgs: 89 active+clean; 222 MiB data, 690 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 975 KiB/s wr, 14 op/s 2024-04-12T11:18:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:18 smithi029 ceph-mon[93650]: osdmap e304: 8 total, 8 up, 8 in 2024-04-12T11:18:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:18 smithi029 ceph-mon[39999]: pgmap v86: 89 pgs: 89 active+clean; 222 MiB data, 690 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 975 KiB/s wr, 14 op/s 2024-04-12T11:18:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:18 smithi029 ceph-mon[39999]: osdmap e304: 8 total, 8 up, 8 in 2024-04-12T11:18:19.544 INFO:tasks.rados.rados.0.smithi029.stdout:940: done (0 left) 2024-04-12T11:18:19.545 INFO:tasks.rados.rados.0.smithi029.stdout:941: read oid 5 snap -1 2024-04-12T11:18:19.545 INFO:tasks.rados.rados.0.smithi029.stdout:941: expect deleted 2024-04-12T11:18:19.545 INFO:tasks.rados.rados.0.smithi029.stdout:942: delete oid 37 current snap is 120 2024-04-12T11:18:19.548 INFO:tasks.rados.rados.0.smithi029.stdout:941: done (1 left) 2024-04-12T11:18:19.548 INFO:tasks.rados.rados.0.smithi029.stdout:942: done (0 left) 2024-04-12T11:18:19.548 INFO:tasks.rados.rados.0.smithi029.stdout:943: write oid 9 current snap is 120 2024-04-12T11:18:19.548 INFO:tasks.rados.rados.0.smithi029.stdout:943: seq_num 382 ranges {420274=562768,1620592=439131} 2024-04-12T11:18:19.554 INFO:tasks.rados.rados.0.smithi029.stdout:943: writing smithi02979277-9 from 420274 to 983042 tid 1 2024-04-12T11:18:19.560 INFO:tasks.rados.rados.0.smithi029.stdout:943: writing smithi02979277-9 from 1620592 to 2059723 tid 2 2024-04-12T11:18:19.560 INFO:tasks.rados.rados.0.smithi029.stdout:944: read oid 42 snap -1 2024-04-12T11:18:19.560 INFO:tasks.rados.rados.0.smithi029.stdout:944: expect (ObjNum 322 snap 100 seq_num 322) 2024-04-12T11:18:19.560 INFO:tasks.rados.rados.0.smithi029.stdout:945: snap_remove snap 86 2024-04-12T11:18:19.867 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:19 smithi087 ceph-mon[32917]: osdmap e305: 8 total, 8 up, 8 in 2024-04-12T11:18:19.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:19 smithi029 ceph-mon[39999]: osdmap e305: 8 total, 8 up, 8 in 2024-04-12T11:18:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:19 smithi029 ceph-mon[93650]: osdmap e305: 8 total, 8 up, 8 in 2024-04-12T11:18:20.546 INFO:tasks.rados.rados.0.smithi029.stdout:943: finishing write tid 1 to smithi02979277-9 2024-04-12T11:18:20.546 INFO:tasks.rados.rados.0.smithi029.stdout:943: finishing write tid 2 to smithi02979277-9 2024-04-12T11:18:20.546 INFO:tasks.rados.rados.0.smithi029.stdout:943: finishing write tid 3 to smithi02979277-9 2024-04-12T11:18:20.546 INFO:tasks.rados.rados.0.smithi029.stdout:943: finishing write tid 4 to smithi02979277-9 2024-04-12T11:18:20.547 INFO:tasks.rados.rados.0.smithi029.stdout:943: oid 9 updating version 0 to 418 2024-04-12T11:18:20.547 INFO:tasks.rados.rados.0.smithi029.stdout:943: oid 9 version 418 is already newer than 416 2024-04-12T11:18:20.547 INFO:tasks.rados.rados.0.smithi029.stdout:943: oid 9 version 418 is already newer than 417 2024-04-12T11:18:20.547 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 418 (ObjNum 382 snap 120 seq_num 382) dirty exists 2024-04-12T11:18:20.547 INFO:tasks.rados.rados.0.smithi029.stdout:943: left oid 9 (ObjNum 382 snap 120 seq_num 382) 2024-04-12T11:18:20.547 INFO:tasks.rados.rados.0.smithi029.stdout:944: expect (ObjNum 322 snap 100 seq_num 322) 2024-04-12T11:18:20.787 INFO:tasks.rados.rados.0.smithi029.stdout:943: done (2 left) 2024-04-12T11:18:20.788 INFO:tasks.rados.rados.0.smithi029.stdout:944: done (1 left) 2024-04-12T11:18:20.788 INFO:tasks.rados.rados.0.smithi029.stdout:945: done (0 left) 2024-04-12T11:18:20.788 INFO:tasks.rados.rados.0.smithi029.stdout:946: write oid 5 current snap is 120 2024-04-12T11:18:20.788 INFO:tasks.rados.rados.0.smithi029.stdout:946: seq_num 383 ranges {437081=638017,1679074=582789} 2024-04-12T11:18:20.793 INFO:tasks.rados.rados.0.smithi029.stdout:946: writing smithi02979277-5 from 437081 to 1075098 tid 1 2024-04-12T11:18:20.800 INFO:tasks.rados.rados.0.smithi029.stdout:946: writing smithi02979277-5 from 1679074 to 2261863 tid 2 2024-04-12T11:18:20.800 INFO:tasks.rados.rados.0.smithi029.stdout:947: write oid 38 current snap is 120 2024-04-12T11:18:20.800 INFO:tasks.rados.rados.0.smithi029.stdout:947: seq_num 384 ranges {453888=713266,1737556=442800} 2024-04-12T11:18:20.806 INFO:tasks.rados.rados.0.smithi029.stdout:947: writing smithi02979277-38 from 453888 to 1167154 tid 1 2024-04-12T11:18:20.812 INFO:tasks.rados.rados.0.smithi029.stdout:947: writing smithi02979277-38 from 1737556 to 2180356 tid 2 2024-04-12T11:18:20.812 INFO:tasks.rados.rados.0.smithi029.stdout:948: rollback oid 24 current snap is 120 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 24 to 117 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:946: finishing write tid 1 to smithi02979277-5 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:946: finishing write tid 2 to smithi02979277-5 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:946: finishing write tid 3 to smithi02979277-5 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:946: finishing write tid 4 to smithi02979277-5 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:946: oid 5 updating version 0 to 580 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:946: oid 5 version 580 is already newer than 579 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:946: oid 5 version 580 is already newer than 578 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 580 (ObjNum 383 snap 120 seq_num 383) dirty exists 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:946: left oid 5 (ObjNum 383 snap 120 seq_num 383) 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:946: done (2 left) 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:949: write oid 6 current snap is 120 2024-04-12T11:18:20.813 INFO:tasks.rados.rados.0.smithi029.stdout:947: finishing write tid 1 to smithi02979277-38 2024-04-12T11:18:20.814 INFO:tasks.rados.rados.0.smithi029.stdout:949: seq_num 385 ranges {470695=788515,1679685=586458} 2024-04-12T11:18:20.819 INFO:tasks.rados.rados.0.smithi029.stdout:949: writing smithi02979277-6 from 470695 to 1259210 tid 1 2024-04-12T11:18:20.825 INFO:tasks.rados.rados.0.smithi029.stdout:949: writing smithi02979277-6 from 1679685 to 2266143 tid 2 2024-04-12T11:18:20.825 INFO:tasks.rados.rados.0.smithi029.stdout:950: snap_remove snap 106 2024-04-12T11:18:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:20 smithi087 ceph-mon[32917]: pgmap v89: 89 pgs: 89 active+clean; 222 MiB data, 690 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 975 KiB/s wr, 14 op/s 2024-04-12T11:18:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:20 smithi087 ceph-mon[32917]: osdmap e306: 8 total, 8 up, 8 in 2024-04-12T11:18:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:20 smithi029 ceph-mon[93650]: pgmap v89: 89 pgs: 89 active+clean; 222 MiB data, 690 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 975 KiB/s wr, 14 op/s 2024-04-12T11:18:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:20 smithi029 ceph-mon[93650]: osdmap e306: 8 total, 8 up, 8 in 2024-04-12T11:18:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:20 smithi029 ceph-mon[39999]: pgmap v89: 89 pgs: 89 active+clean; 222 MiB data, 690 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 975 KiB/s wr, 14 op/s 2024-04-12T11:18:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:20 smithi029 ceph-mon[39999]: osdmap e306: 8 total, 8 up, 8 in 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:950: done (3 left) 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:951: snap_create 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:947: finishing write tid 2 to smithi02979277-38 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:948: finishing rollback tid 0 to smithi02979277-24 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:947: finishing write tid 3 to smithi02979277-38 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:947: finishing write tid 4 to smithi02979277-38 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:947: oid 38 updating version 0 to 223 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:947: oid 38 updating version 223 to 224 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:947: oid 38 updating version 224 to 225 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 225 (ObjNum 384 snap 120 seq_num 384) dirty exists 2024-04-12T11:18:21.551 INFO:tasks.rados.rados.0.smithi029.stdout:947: left oid 38 (ObjNum 384 snap 120 seq_num 384) 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:948: finishing rollback tid 1 to smithi02979277-24 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:948: finishing rollback tid 2 to smithi02979277-24 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 619 (ObjNum 326 snap 101 seq_num 326) dirty exists 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:949: finishing write tid 1 to smithi02979277-6 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:949: finishing write tid 2 to smithi02979277-6 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:949: finishing write tid 3 to smithi02979277-6 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:949: finishing write tid 4 to smithi02979277-6 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:949: oid 6 updating version 0 to 233 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:949: oid 6 updating version 233 to 235 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:949: oid 6 version 235 is already newer than 234 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 235 (ObjNum 385 snap 120 seq_num 385) dirty exists 2024-04-12T11:18:21.552 INFO:tasks.rados.rados.0.smithi029.stdout:949: left oid 6 (ObjNum 385 snap 120 seq_num 385) 2024-04-12T11:18:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:21 smithi087 ceph-mon[32917]: osdmap e307: 8 total, 8 up, 8 in 2024-04-12T11:18:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:18:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:21 smithi029 ceph-mon[93650]: osdmap e307: 8 total, 8 up, 8 in 2024-04-12T11:18:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:18:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:21 smithi029 ceph-mon[39999]: osdmap e307: 8 total, 8 up, 8 in 2024-04-12T11:18:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:21 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:18:22.509 INFO:tasks.rados.rados.0.smithi029.stdout:947: done (3 left) 2024-04-12T11:18:22.509 INFO:tasks.rados.rados.0.smithi029.stdout:948: done (2 left) 2024-04-12T11:18:22.509 INFO:tasks.rados.rados.0.smithi029.stdout:949: done (1 left) 2024-04-12T11:18:22.509 INFO:tasks.rados.rados.0.smithi029.stdout:951: done (0 left) 2024-04-12T11:18:22.510 INFO:tasks.rados.rados.0.smithi029.stdout:952: snap_remove snap 117 2024-04-12T11:18:22.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:22 smithi087 ceph-mon[32917]: pgmap v92: 89 pgs: 89 active+clean; 222 MiB data, 690 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:22 smithi087 ceph-mon[32917]: osdmap e308: 8 total, 8 up, 8 in 2024-04-12T11:18:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:22 smithi087 ceph-mon[32917]: osdmap e309: 8 total, 8 up, 8 in 2024-04-12T11:18:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:22 smithi029 ceph-mon[93650]: pgmap v92: 89 pgs: 89 active+clean; 222 MiB data, 690 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:22 smithi029 ceph-mon[93650]: osdmap e308: 8 total, 8 up, 8 in 2024-04-12T11:18:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:22 smithi029 ceph-mon[93650]: osdmap e309: 8 total, 8 up, 8 in 2024-04-12T11:18:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:22 smithi029 ceph-mon[39999]: pgmap v92: 89 pgs: 89 active+clean; 222 MiB data, 690 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:22 smithi029 ceph-mon[39999]: osdmap e308: 8 total, 8 up, 8 in 2024-04-12T11:18:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:22 smithi029 ceph-mon[39999]: osdmap e309: 8 total, 8 up, 8 in 2024-04-12T11:18:23.511 INFO:tasks.rados.rados.0.smithi029.stdout:952: done (0 left) 2024-04-12T11:18:23.511 INFO:tasks.rados.rados.0.smithi029.stdout:953: delete oid 50 current snap is 121 2024-04-12T11:18:23.518 INFO:tasks.rados.rados.0.smithi029.stdout:953: done (0 left) 2024-04-12T11:18:23.518 INFO:tasks.rados.rados.0.smithi029.stdout:954: delete oid 37 current snap is 121 2024-04-12T11:18:23.522 INFO:tasks.rados.rados.0.smithi029.stdout:954: done (0 left) 2024-04-12T11:18:23.522 INFO:tasks.rados.rados.0.smithi029.stdout:955: write oid 39 current snap is 121 2024-04-12T11:18:23.522 INFO:tasks.rados.rados.0.smithi029.stdout:955: seq_num 388 ranges {521116=730615,1971484=549632} 2024-04-12T11:18:23.529 INFO:tasks.rados.rados.0.smithi029.stdout:955: writing smithi02979277-39 from 521116 to 1251731 tid 1 2024-04-12T11:18:23.536 INFO:tasks.rados.rados.0.smithi029.stdout:955: writing smithi02979277-39 from 1971484 to 2521116 tid 2 2024-04-12T11:18:23.536 INFO:tasks.rados.rados.0.smithi029.stdout:956: rollback oid 32 current snap is 121 2024-04-12T11:18:23.537 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 32 to 120 2024-04-12T11:18:23.537 INFO:tasks.rados.rados.0.smithi029.stdout:955: finishing write tid 1 to smithi02979277-39 2024-04-12T11:18:23.537 INFO:tasks.rados.rados.0.smithi029.stdout:957: snap_remove snap 118 2024-04-12T11:18:23.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:24.516 INFO:tasks.rados.rados.0.smithi029.stdout:957: done (2 left) 2024-04-12T11:18:24.516 INFO:tasks.rados.rados.0.smithi029.stdout:958: write oid 19 current snap is 121 2024-04-12T11:18:24.517 INFO:tasks.rados.rados.0.smithi029.stdout:955: finishing write tid 2 to smithi02979277-39 2024-04-12T11:18:24.517 INFO:tasks.rados.rados.0.smithi029.stdout:958: seq_num 389 ranges {537923=405864,1513613=593796} 2024-04-12T11:18:24.522 INFO:tasks.rados.rados.0.smithi029.stdout:958: writing smithi02979277-19 from 537923 to 943787 tid 1 2024-04-12T11:18:24.529 INFO:tasks.rados.rados.0.smithi029.stdout:958: writing smithi02979277-19 from 1513613 to 2107409 tid 2 2024-04-12T11:18:24.530 INFO:tasks.rados.rados.0.smithi029.stdout:959: snap_remove snap 119 2024-04-12T11:18:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:24 smithi029 ceph-mon[39999]: pgmap v95: 89 pgs: 89 active+clean; 186 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-04-12T11:18:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:24 smithi029 ceph-mon[39999]: osdmap e310: 8 total, 8 up, 8 in 2024-04-12T11:18:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:24 smithi029 ceph-mon[93650]: pgmap v95: 89 pgs: 89 active+clean; 186 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-04-12T11:18:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:24 smithi029 ceph-mon[93650]: osdmap e310: 8 total, 8 up, 8 in 2024-04-12T11:18:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:24 smithi087 ceph-mon[32917]: pgmap v95: 89 pgs: 89 active+clean; 186 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-04-12T11:18:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:24 smithi087 ceph-mon[32917]: osdmap e310: 8 total, 8 up, 8 in 2024-04-12T11:18:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:18:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:18:24] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.43.0" 2024-04-12T11:18:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:959: done (3 left) 2024-04-12T11:18:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:960: rollback oid 28 current snap is 121 2024-04-12T11:18:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 28 to 100 2024-04-12T11:18:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:955: finishing write tid 3 to smithi02979277-39 2024-04-12T11:18:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:955: finishing write tid 4 to smithi02979277-39 2024-04-12T11:18:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:955: oid 39 updating version 0 to 69 2024-04-12T11:18:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:955: oid 39 updating version 69 to 70 2024-04-12T11:18:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:955: oid 39 version 70 is already newer than 68 2024-04-12T11:18:25.523 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 70 (ObjNum 388 snap 121 seq_num 388) dirty exists 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:955: left oid 39 (ObjNum 388 snap 121 seq_num 388) 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:956: finishing rollback tid 0 to smithi02979277-32 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:956: finishing rollback tid 1 to smithi02979277-32 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:956: finishing rollback tid 2 to smithi02979277-32 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 361 (ObjNum 306 snap 94 seq_num 306) dirty exists 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:958: finishing write tid 1 to smithi02979277-19 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:958: finishing write tid 2 to smithi02979277-19 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:958: finishing write tid 3 to smithi02979277-19 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:958: finishing write tid 4 to smithi02979277-19 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:958: oid 19 updating version 0 to 592 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:958: oid 19 version 592 is already newer than 591 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:958: oid 19 version 592 is already newer than 590 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 592 (ObjNum 389 snap 121 seq_num 389) dirty exists 2024-04-12T11:18:25.524 INFO:tasks.rados.rados.0.smithi029.stdout:958: left oid 19 (ObjNum 389 snap 121 seq_num 389) 2024-04-12T11:18:25.525 INFO:tasks.rados.rados.0.smithi029.stdout:955: done (3 left) 2024-04-12T11:18:25.525 INFO:tasks.rados.rados.0.smithi029.stdout:956: done (2 left) 2024-04-12T11:18:25.525 INFO:tasks.rados.rados.0.smithi029.stdout:958: done (1 left) 2024-04-12T11:18:25.525 INFO:tasks.rados.rados.0.smithi029.stdout:961: write oid 17 current snap is 121 2024-04-12T11:18:25.525 INFO:tasks.rados.rados.0.smithi029.stdout:961: seq_num 390 ranges {554730=481113,1572095=737454} 2024-04-12T11:18:25.528 INFO:tasks.rados.rados.0.smithi029.stdout:961: writing smithi02979277-17 from 554730 to 1035843 tid 1 2024-04-12T11:18:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:961: writing smithi02979277-17 from 1572095 to 2309549 tid 2 2024-04-12T11:18:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:962: delete oid 5 current snap is 121 2024-04-12T11:18:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:960: finishing rollback tid 0 to smithi02979277-28 2024-04-12T11:18:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:960: finishing rollback tid 1 to smithi02979277-28 2024-04-12T11:18:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:960: finishing rollback tid 2 to smithi02979277-28 2024-04-12T11:18:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 409 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-04-12T11:18:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:961: finishing write tid 1 to smithi02979277-17 2024-04-12T11:18:25.539 INFO:tasks.rados.rados.0.smithi029.stdout:960: done (2 left) 2024-04-12T11:18:25.539 INFO:tasks.rados.rados.0.smithi029.stdout:962: done (1 left) 2024-04-12T11:18:25.539 INFO:tasks.rados.rados.0.smithi029.stdout:963: write oid 11 current snap is 121 2024-04-12T11:18:25.539 INFO:tasks.rados.rados.0.smithi029.stdout:963: seq_num 392 ranges {588344=631611,1689059=741123} 2024-04-12T11:18:25.545 INFO:tasks.rados.rados.0.smithi029.stdout:963: writing smithi02979277-11 from 588344 to 1219955 tid 1 2024-04-12T11:18:25.552 INFO:tasks.rados.rados.0.smithi029.stdout:963: writing smithi02979277-11 from 1689059 to 2430182 tid 2 2024-04-12T11:18:25.552 INFO:tasks.rados.rados.0.smithi029.stdout:964: snap_remove snap 120 2024-04-12T11:18:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:25 smithi087 ceph-mon[32917]: osdmap e311: 8 total, 8 up, 8 in 2024-04-12T11:18:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:25 smithi029 ceph-mon[93650]: osdmap e311: 8 total, 8 up, 8 in 2024-04-12T11:18:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:25 smithi029 ceph-mon[39999]: osdmap e311: 8 total, 8 up, 8 in 2024-04-12T11:18:26.527 INFO:tasks.rados.rados.0.smithi029.stdout:964: done (2 left) 2024-04-12T11:18:26.527 INFO:tasks.rados.rados.0.smithi029.stdout:965: snap_create 2024-04-12T11:18:26.527 INFO:tasks.rados.rados.0.smithi029.stdout:961: finishing write tid 2 to smithi02979277-17 2024-04-12T11:18:26.527 INFO:tasks.rados.rados.0.smithi029.stdout:961: finishing write tid 3 to smithi02979277-17 2024-04-12T11:18:26.527 INFO:tasks.rados.rados.0.smithi029.stdout:961: finishing write tid 4 to smithi02979277-17 2024-04-12T11:18:26.527 INFO:tasks.rados.rados.0.smithi029.stdout:961: oid 17 updating version 0 to 582 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:961: oid 17 updating version 582 to 583 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:961: oid 17 version 583 is already newer than 581 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 583 (ObjNum 390 snap 121 seq_num 390) dirty exists 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:961: left oid 17 (ObjNum 390 snap 121 seq_num 390) 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:963: finishing write tid 1 to smithi02979277-11 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:963: finishing write tid 2 to smithi02979277-11 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:963: finishing write tid 3 to smithi02979277-11 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:963: finishing write tid 4 to smithi02979277-11 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:963: oid 11 updating version 0 to 637 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:963: oid 11 version 637 is already newer than 636 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:963: oid 11 updating version 637 to 638 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 638 (ObjNum 392 snap 121 seq_num 392) dirty exists 2024-04-12T11:18:26.528 INFO:tasks.rados.rados.0.smithi029.stdout:963: left oid 11 (ObjNum 392 snap 121 seq_num 392) 2024-04-12T11:18:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:26 smithi087 ceph-mon[32917]: pgmap v98: 89 pgs: 89 active+clean; 186 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-04-12T11:18:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:26 smithi087 ceph-mon[32917]: osdmap e312: 8 total, 8 up, 8 in 2024-04-12T11:18:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:26 smithi029 ceph-mon[93650]: pgmap v98: 89 pgs: 89 active+clean; 186 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-04-12T11:18:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:26 smithi029 ceph-mon[93650]: osdmap e312: 8 total, 8 up, 8 in 2024-04-12T11:18:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:26 smithi029 ceph-mon[39999]: pgmap v98: 89 pgs: 89 active+clean; 186 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-04-12T11:18:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:26 smithi029 ceph-mon[39999]: osdmap e312: 8 total, 8 up, 8 in 2024-04-12T11:18:27.511 INFO:tasks.rados.rados.0.smithi029.stdout:961: done (2 left) 2024-04-12T11:18:27.511 INFO:tasks.rados.rados.0.smithi029.stdout:963: done (1 left) 2024-04-12T11:18:27.511 INFO:tasks.rados.rados.0.smithi029.stdout:965: done (0 left) 2024-04-12T11:18:27.511 INFO:tasks.rados.rados.0.smithi029.stdout:966: snap_create 2024-04-12T11:18:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:27 smithi087 ceph-mon[32917]: osdmap e313: 8 total, 8 up, 8 in 2024-04-12T11:18:27.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:27 smithi087 ceph-mon[32917]: osdmap e314: 8 total, 8 up, 8 in 2024-04-12T11:18:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:27 smithi029 ceph-mon[93650]: osdmap e313: 8 total, 8 up, 8 in 2024-04-12T11:18:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:27 smithi029 ceph-mon[93650]: osdmap e314: 8 total, 8 up, 8 in 2024-04-12T11:18:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:27 smithi029 ceph-mon[39999]: osdmap e313: 8 total, 8 up, 8 in 2024-04-12T11:18:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:27 smithi029 ceph-mon[39999]: osdmap e314: 8 total, 8 up, 8 in 2024-04-12T11:18:28.513 INFO:tasks.rados.rados.0.smithi029.stdout:966: done (0 left) 2024-04-12T11:18:28.514 INFO:tasks.rados.rados.0.smithi029.stdout:967: write oid 31 current snap is 123 2024-04-12T11:18:28.514 INFO:tasks.rados.rados.0.smithi029.stdout:967: seq_num 393 ranges {605151=706860,2031188=573963} 2024-04-12T11:18:28.520 INFO:tasks.rados.rados.0.smithi029.stdout:967: writing smithi02979277-31 from 605151 to 1312011 tid 1 2024-04-12T11:18:28.528 INFO:tasks.rados.rados.0.smithi029.stdout:967: writing smithi02979277-31 from 2031188 to 2605151 tid 2 2024-04-12T11:18:28.528 INFO:tasks.rados.rados.0.smithi029.stdout:968: write oid 7 current snap is 123 2024-04-12T11:18:28.528 INFO:tasks.rados.rados.0.smithi029.stdout:967: finishing write tid 1 to smithi02979277-31 2024-04-12T11:18:28.528 INFO:tasks.rados.rados.0.smithi029.stdout:968: seq_num 394 ranges {621958=782109,2089670=532288} 2024-04-12T11:18:28.535 INFO:tasks.rados.rados.0.smithi029.stdout:968: writing smithi02979277-7 from 621958 to 1404067 tid 1 2024-04-12T11:18:28.543 INFO:tasks.rados.rados.0.smithi029.stdout:968: writing smithi02979277-7 from 2089670 to 2621958 tid 2 2024-04-12T11:18:28.543 INFO:tasks.rados.rados.0.smithi029.stdout:969: write oid 45 current snap is 123 2024-04-12T11:18:28.543 INFO:tasks.rados.rados.0.smithi029.stdout:967: finishing write tid 2 to smithi02979277-31 2024-04-12T11:18:28.543 INFO:tasks.rados.rados.0.smithi029.stdout:969: seq_num 395 ranges {638765=457358,1748152=604803} 2024-04-12T11:18:28.548 INFO:tasks.rados.rados.0.smithi029.stdout:969: writing smithi02979277-45 from 638765 to 1096123 tid 1 2024-04-12T11:18:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:969: writing smithi02979277-45 from 1748152 to 2352955 tid 2 2024-04-12T11:18:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:970: read oid 9 snap -1 2024-04-12T11:18:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:970: expect (ObjNum 382 snap 120 seq_num 382) 2024-04-12T11:18:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:971: write oid 48 current snap is 123 2024-04-12T11:18:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:967: finishing write tid 3 to smithi02979277-31 2024-04-12T11:18:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:967: finishing write tid 4 to smithi02979277-31 2024-04-12T11:18:28.556 INFO:tasks.rados.rados.0.smithi029.stdout:967: oid 31 updating version 0 to 416 2024-04-12T11:18:28.557 INFO:tasks.rados.rados.0.smithi029.stdout:967: oid 31 updating version 416 to 417 2024-04-12T11:18:28.557 INFO:tasks.rados.rados.0.smithi029.stdout:967: oid 31 updating version 417 to 418 2024-04-12T11:18:28.557 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 418 (ObjNum 393 snap 123 seq_num 393) dirty exists 2024-04-12T11:18:28.557 INFO:tasks.rados.rados.0.smithi029.stdout:967: left oid 31 (ObjNum 393 snap 123 seq_num 393) 2024-04-12T11:18:28.557 INFO:tasks.rados.rados.0.smithi029.stdout:968: finishing write tid 1 to smithi02979277-7 2024-04-12T11:18:28.557 INFO:tasks.rados.rados.0.smithi029.stdout:968: finishing write tid 2 to smithi02979277-7 2024-04-12T11:18:28.557 INFO:tasks.rados.rados.0.smithi029.stdout:968: finishing write tid 3 to smithi02979277-7 2024-04-12T11:18:28.557 INFO:tasks.rados.rados.0.smithi029.stdout:971: seq_num 396 ranges {655572=648960,1922987=732585} 2024-04-12T11:18:28.563 INFO:tasks.rados.rados.0.smithi029.stdout:971: writing smithi02979277-48 from 655572 to 1304532 tid 1 2024-04-12T11:18:28.571 INFO:tasks.rados.rados.0.smithi029.stdout:971: writing smithi02979277-48 from 1922987 to 2655572 tid 2 2024-04-12T11:18:28.571 INFO:tasks.rados.rados.0.smithi029.stdout:967: done (4 left) 2024-04-12T11:18:28.571 INFO:tasks.rados.rados.0.smithi029.stdout:972: read oid 24 snap -1 2024-04-12T11:18:28.571 INFO:tasks.rados.rados.0.smithi029.stdout:972: expect (ObjNum 326 snap 101 seq_num 326) 2024-04-12T11:18:28.571 INFO:tasks.rados.rados.0.smithi029.stdout:973: read oid 23 snap -1 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:973: expect deleted 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:974: rollback oid 3 current snap is 123 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 100 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:968: finishing write tid 4 to smithi02979277-7 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:968: oid 7 updating version 0 to 526 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:968: oid 7 version 526 is already newer than 525 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:968: oid 7 version 526 is already newer than 524 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 526 (ObjNum 394 snap 123 seq_num 394) dirty exists 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:968: left oid 7 (ObjNum 394 snap 123 seq_num 394) 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:969: finishing write tid 1 to smithi02979277-45 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:969: finishing write tid 2 to smithi02979277-45 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:969: finishing write tid 3 to smithi02979277-45 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:969: finishing write tid 4 to smithi02979277-45 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:969: oid 45 updating version 0 to 529 2024-04-12T11:18:28.572 INFO:tasks.rados.rados.0.smithi029.stdout:969: oid 45 version 529 is already newer than 528 2024-04-12T11:18:28.573 INFO:tasks.rados.rados.0.smithi029.stdout:969: oid 45 updating version 529 to 530 2024-04-12T11:18:28.573 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 530 (ObjNum 395 snap 123 seq_num 395) dirty exists 2024-04-12T11:18:28.573 INFO:tasks.rados.rados.0.smithi029.stdout:969: left oid 45 (ObjNum 395 snap 123 seq_num 395) 2024-04-12T11:18:28.573 INFO:tasks.rados.rados.0.smithi029.stdout:970: expect (ObjNum 382 snap 120 seq_num 382) 2024-04-12T11:18:28.726 INFO:tasks.rados.rados.0.smithi029.stdout:971: finishing write tid 1 to smithi02979277-48 2024-04-12T11:18:28.727 INFO:tasks.rados.rados.0.smithi029.stdout:971: finishing write tid 2 to smithi02979277-48 2024-04-12T11:18:28.727 INFO:tasks.rados.rados.0.smithi029.stdout:971: finishing write tid 3 to smithi02979277-48 2024-04-12T11:18:28.727 INFO:tasks.rados.rados.0.smithi029.stdout:971: finishing write tid 4 to smithi02979277-48 2024-04-12T11:18:28.727 INFO:tasks.rados.rados.0.smithi029.stdout:971: oid 48 updating version 0 to 420 2024-04-12T11:18:28.727 INFO:tasks.rados.rados.0.smithi029.stdout:971: oid 48 updating version 420 to 421 2024-04-12T11:18:28.727 INFO:tasks.rados.rados.0.smithi029.stdout:971: oid 48 version 421 is already newer than 419 2024-04-12T11:18:28.727 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 421 (ObjNum 396 snap 123 seq_num 396) dirty exists 2024-04-12T11:18:28.727 INFO:tasks.rados.rados.0.smithi029.stdout:971: left oid 48 (ObjNum 396 snap 123 seq_num 396) 2024-04-12T11:18:28.727 INFO:tasks.rados.rados.0.smithi029.stdout:972: expect (ObjNum 326 snap 101 seq_num 326) 2024-04-12T11:18:28.911 INFO:tasks.rados.rados.0.smithi029.stdout:974: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:18:28.911 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 0 (ObjNum 129 snap 0 seq_num 1800753408) dirty dne 2024-04-12T11:18:28.911 INFO:tasks.rados.rados.0.smithi029.stdout:968: done (6 left) 2024-04-12T11:18:28.911 INFO:tasks.rados.rados.0.smithi029.stdout:969: done (5 left) 2024-04-12T11:18:28.911 INFO:tasks.rados.rados.0.smithi029.stdout:970: done (4 left) 2024-04-12T11:18:28.912 INFO:tasks.rados.rados.0.smithi029.stdout:971: done (3 left) 2024-04-12T11:18:28.912 INFO:tasks.rados.rados.0.smithi029.stdout:972: done (2 left) 2024-04-12T11:18:28.912 INFO:tasks.rados.rados.0.smithi029.stdout:973: done (1 left) 2024-04-12T11:18:28.912 INFO:tasks.rados.rados.0.smithi029.stdout:974: done (0 left) 2024-04-12T11:18:28.912 INFO:tasks.rados.rados.0.smithi029.stdout:975: delete oid 9 current snap is 123 2024-04-12T11:18:28.913 INFO:tasks.rados.rados.0.smithi029.stdout:975: done (0 left) 2024-04-12T11:18:28.913 INFO:tasks.rados.rados.0.smithi029.stdout:976: read oid 32 snap -1 2024-04-12T11:18:28.913 INFO:tasks.rados.rados.0.smithi029.stdout:976: expect (ObjNum 306 snap 94 seq_num 306) 2024-04-12T11:18:28.914 INFO:tasks.rados.rados.0.smithi029.stdout:977: rollback oid 16 current snap is 123 2024-04-12T11:18:28.914 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 16 to 100 2024-04-12T11:18:28.914 INFO:tasks.rados.rados.0.smithi029.stdout:978: snap_remove snap 113 2024-04-12T11:18:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:28 smithi087 ceph-mon[32917]: pgmap v101: 89 pgs: 89 active+clean; 189 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-04-12T11:18:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:28 smithi087 ceph-mon[32917]: osdmap e315: 8 total, 8 up, 8 in 2024-04-12T11:18:29.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:28 smithi029 ceph-mon[93650]: pgmap v101: 89 pgs: 89 active+clean; 189 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-04-12T11:18:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:28 smithi029 ceph-mon[93650]: osdmap e315: 8 total, 8 up, 8 in 2024-04-12T11:18:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:28 smithi029 ceph-mon[39999]: pgmap v101: 89 pgs: 89 active+clean; 189 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-04-12T11:18:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:28 smithi029 ceph-mon[39999]: osdmap e315: 8 total, 8 up, 8 in 2024-04-12T11:18:29.542 INFO:tasks.rados.rados.0.smithi029.stdout:978: done (2 left) 2024-04-12T11:18:29.542 INFO:tasks.rados.rados.0.smithi029.stdout:979: snap_remove snap 121 2024-04-12T11:18:30.545 INFO:tasks.rados.rados.0.smithi029.stdout:979: done (2 left) 2024-04-12T11:18:30.546 INFO:tasks.rados.rados.0.smithi029.stdout:980: write oid 46 current snap is 123 2024-04-12T11:18:30.546 INFO:tasks.rados.rados.0.smithi029.stdout:980: seq_num 398 ranges {689186=799458,1923598=752130} 2024-04-12T11:18:30.554 INFO:tasks.rados.rados.0.smithi029.stdout:980: writing smithi02979277-46 from 689186 to 1488644 tid 1 2024-04-12T11:18:30.562 INFO:tasks.rados.rados.0.smithi029.stdout:980: writing smithi02979277-46 from 1923598 to 2675728 tid 2 2024-04-12T11:18:30.562 INFO:tasks.rados.rados.0.smithi029.stdout:981: snap_remove snap 122 2024-04-12T11:18:30.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch upgrade resume' 2024-04-12T11:18:30.819 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:18:30.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:30 smithi029 ceph-mon[93650]: pgmap v104: 89 pgs: 89 active+clean; 189 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-04-12T11:18:30.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:30 smithi029 ceph-mon[93650]: osdmap e316: 8 total, 8 up, 8 in 2024-04-12T11:18:30.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:30 smithi029 ceph-mon[39999]: pgmap v104: 89 pgs: 89 active+clean; 189 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-04-12T11:18:30.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:30 smithi029 ceph-mon[39999]: osdmap e316: 8 total, 8 up, 8 in 2024-04-12T11:18:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:30 smithi087 ceph-mon[32917]: pgmap v104: 89 pgs: 89 active+clean; 189 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-04-12T11:18:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:30 smithi087 ceph-mon[32917]: osdmap e316: 8 total, 8 up, 8 in 2024-04-12T11:18:31.550 INFO:tasks.rados.rados.0.smithi029.stdout:981: done (3 left) 2024-04-12T11:18:31.550 INFO:tasks.rados.rados.0.smithi029.stdout:982: snap_remove snap 105 2024-04-12T11:18:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:31 smithi087 ceph-mon[32917]: osdmap e317: 8 total, 8 up, 8 in 2024-04-12T11:18:32.007 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:31 smithi029 ceph-mon[39999]: osdmap e317: 8 total, 8 up, 8 in 2024-04-12T11:18:32.007 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:31 smithi029 ceph-mon[93650]: osdmap e317: 8 total, 8 up, 8 in 2024-04-12T11:18:32.557 INFO:tasks.rados.rados.0.smithi029.stdout:982: done (3 left) 2024-04-12T11:18:32.557 INFO:tasks.rados.rados.0.smithi029.stdout:983: read oid 35 snap -1 2024-04-12T11:18:32.557 INFO:tasks.rados.rados.0.smithi029.stdout:983: expect (ObjNum 377 snap 119 seq_num 377) 2024-04-12T11:18:32.557 INFO:tasks.rados.rados.0.smithi029.stdout:984: rollback oid 26 current snap is 123 2024-04-12T11:18:32.558 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 100 2024-04-12T11:18:32.558 INFO:tasks.rados.rados.0.smithi029.stdout:985: read oid 50 snap 107 2024-04-12T11:18:32.558 INFO:tasks.rados.rados.0.smithi029.stdout:985: expect deleted 2024-04-12T11:18:32.558 INFO:tasks.rados.rados.0.smithi029.stdout:986: read oid 9 snap 100 2024-04-12T11:18:32.558 INFO:tasks.rados.rados.0.smithi029.stdout:986: expect (ObjNum 268 snap 86 seq_num 268) 2024-04-12T11:18:32.558 INFO:tasks.rados.rados.0.smithi029.stdout:987: write oid 8 current snap is 123 2024-04-12T11:18:32.558 INFO:tasks.rados.rados.0.smithi029.stdout:987: seq_num 399 ranges {705993=474707,1582080=612141} 2024-04-12T11:18:32.564 INFO:tasks.rados.rados.0.smithi029.stdout:987: writing smithi02979277-8 from 705993 to 1180700 tid 1 2024-04-12T11:18:32.572 INFO:tasks.rados.rados.0.smithi029.stdout:987: writing smithi02979277-8 from 1582080 to 2194221 tid 2 2024-04-12T11:18:32.572 INFO:tasks.rados.rados.0.smithi029.stdout:988: read oid 13 snap -1 2024-04-12T11:18:32.572 INFO:tasks.rados.rados.0.smithi029.stdout:988: expect deleted 2024-04-12T11:18:32.572 INFO:tasks.rados.rados.0.smithi029.stdout:977: finishing rollback tid 0 to smithi02979277-16 2024-04-12T11:18:32.573 INFO:tasks.rados.rados.0.smithi029.stdout:977: finishing rollback tid 1 to smithi02979277-16 2024-04-12T11:18:32.573 INFO:tasks.rados.rados.0.smithi029.stdout:977: finishing rollback tid 2 to smithi02979277-16 2024-04-12T11:18:32.573 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 604 (ObjNum 240 snap 75 seq_num 240) dirty exists 2024-04-12T11:18:32.573 INFO:tasks.rados.rados.0.smithi029.stdout:976: expect (ObjNum 306 snap 94 seq_num 306) 2024-04-12T11:18:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:32 smithi087 ceph-mon[32917]: pgmap v107: 89 pgs: 89 active+clean; 189 MiB data, 644 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:32 smithi087 ceph-mon[32917]: osdmap e318: 8 total, 8 up, 8 in 2024-04-12T11:18:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:32 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:18:32.730 INFO:tasks.rados.rados.0.smithi029.stdout:976: done (8 left) 2024-04-12T11:18:32.730 INFO:tasks.rados.rados.0.smithi029.stdout:977: done (7 left) 2024-04-12T11:18:32.730 INFO:tasks.rados.rados.0.smithi029.stdout:989: read oid 24 snap -1 2024-04-12T11:18:32.730 INFO:tasks.rados.rados.0.smithi029.stdout:989: expect (ObjNum 326 snap 101 seq_num 326) 2024-04-12T11:18:32.730 INFO:tasks.rados.rados.0.smithi029.stdout:990: rollback oid 33 current snap is 123 2024-04-12T11:18:32.730 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 107 2024-04-12T11:18:32.730 INFO:tasks.rados.rados.0.smithi029.stdout:991: write oid 25 current snap is 123 2024-04-12T11:18:32.731 INFO:tasks.rados.rados.0.smithi029.stdout:991: seq_num 400 ranges {722800=549956,2040562=682238} 2024-04-12T11:18:32.737 INFO:tasks.rados.rados.0.smithi029.stdout:991: writing smithi02979277-25 from 722800 to 1272756 tid 1 2024-04-12T11:18:32.745 INFO:tasks.rados.rados.0.smithi029.stdout:991: writing smithi02979277-25 from 2040562 to 2722800 tid 2 2024-04-12T11:18:32.745 INFO:tasks.rados.rados.0.smithi029.stdout:992: write oid 2 current snap is 123 2024-04-12T11:18:32.746 INFO:tasks.rados.rados.0.smithi029.stdout:980: finishing write tid 1 to smithi02979277-46 2024-04-12T11:18:32.746 INFO:tasks.rados.rados.0.smithi029.stdout:980: finishing write tid 2 to smithi02979277-46 2024-04-12T11:18:32.746 INFO:tasks.rados.rados.0.smithi029.stdout:992: seq_num 401 ranges {739607=625205,1982691=499457} 2024-04-12T11:18:32.753 INFO:tasks.rados.rados.0.smithi029.stdout:992: writing smithi02979277-2 from 739607 to 1364812 tid 1 2024-04-12T11:18:32.760 INFO:tasks.rados.rados.0.smithi029.stdout:992: writing smithi02979277-2 from 1982691 to 2482148 tid 2 2024-04-12T11:18:32.760 INFO:tasks.rados.rados.0.smithi029.stdout:993: read oid 39 snap -1 2024-04-12T11:18:32.760 INFO:tasks.rados.rados.0.smithi029.stdout:993: expect (ObjNum 388 snap 121 seq_num 388) 2024-04-12T11:18:32.760 INFO:tasks.rados.rados.0.smithi029.stdout:994: write oid 18 current snap is 123 2024-04-12T11:18:32.760 INFO:tasks.rados.rados.0.smithi029.stdout:980: finishing write tid 3 to smithi02979277-46 2024-04-12T11:18:32.760 INFO:tasks.rados.rados.0.smithi029.stdout:980: finishing write tid 4 to smithi02979277-46 2024-04-12T11:18:32.760 INFO:tasks.rados.rados.0.smithi029.stdout:980: oid 46 updating version 0 to 237 2024-04-12T11:18:32.760 INFO:tasks.rados.rados.0.smithi029.stdout:980: oid 46 updating version 237 to 238 2024-04-12T11:18:32.761 INFO:tasks.rados.rados.0.smithi029.stdout:980: oid 46 updating version 238 to 239 2024-04-12T11:18:32.761 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 239 (ObjNum 398 snap 123 seq_num 398) dirty exists 2024-04-12T11:18:32.761 INFO:tasks.rados.rados.0.smithi029.stdout:980: left oid 46 (ObjNum 398 snap 123 seq_num 398) 2024-04-12T11:18:32.761 INFO:tasks.rados.rados.0.smithi029.stdout:986: expect (ObjNum 268 snap 86 seq_num 268) 2024-04-12T11:18:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:32 smithi029 ceph-mon[39999]: pgmap v107: 89 pgs: 89 active+clean; 189 MiB data, 644 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:32 smithi029 ceph-mon[39999]: osdmap e318: 8 total, 8 up, 8 in 2024-04-12T11:18:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:32 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:18:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:32 smithi029 ceph-mon[93650]: pgmap v107: 89 pgs: 89 active+clean; 189 MiB data, 644 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:32 smithi029 ceph-mon[93650]: osdmap e318: 8 total, 8 up, 8 in 2024-04-12T11:18:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:18:32.851 INFO:teuthology.orchestra.run.smithi029.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:18:32.924 INFO:tasks.rados.rados.0.smithi029.stdout:987: finishing write tid 1 to smithi02979277-8 2024-04-12T11:18:32.924 INFO:tasks.rados.rados.0.smithi029.stdout:983: expect (ObjNum 377 snap 119 seq_num 377) 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:984: finishing rollback tid 0 to smithi02979277-26 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:984: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:984: finishing rollback tid 2 to smithi02979277-26 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 647 (ObjNum 135 snap 28 seq_num 135) dirty exists 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:987: finishing write tid 2 to smithi02979277-8 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:987: finishing write tid 3 to smithi02979277-8 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:987: finishing write tid 4 to smithi02979277-8 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:987: oid 8 updating version 0 to 74 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:987: oid 8 version 74 is already newer than 73 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:987: oid 8 updating version 74 to 75 2024-04-12T11:18:33.064 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 75 (ObjNum 399 snap 123 seq_num 399) dirty exists 2024-04-12T11:18:33.065 INFO:tasks.rados.rados.0.smithi029.stdout:987: left oid 8 (ObjNum 399 snap 123 seq_num 399) 2024-04-12T11:18:33.065 INFO:tasks.rados.rados.0.smithi029.stdout:994: seq_num 402 ranges {756414=700454,2041173=715241} 2024-04-12T11:18:33.071 INFO:tasks.rados.rados.0.smithi029.stdout:994: writing smithi02979277-18 from 756414 to 1456868 tid 1 2024-04-12T11:18:33.079 INFO:tasks.rados.rados.0.smithi029.stdout:994: writing smithi02979277-18 from 2041173 to 2756414 tid 2 2024-04-12T11:18:33.079 INFO:tasks.rados.rados.0.smithi029.stdout:980: done (12 left) 2024-04-12T11:18:33.079 INFO:tasks.rados.rados.0.smithi029.stdout:983: done (11 left) 2024-04-12T11:18:33.079 INFO:tasks.rados.rados.0.smithi029.stdout:984: done (10 left) 2024-04-12T11:18:33.079 INFO:tasks.rados.rados.0.smithi029.stdout:985: done (9 left) 2024-04-12T11:18:33.079 INFO:tasks.rados.rados.0.smithi029.stdout:986: done (8 left) 2024-04-12T11:18:33.080 INFO:tasks.rados.rados.0.smithi029.stdout:987: done (7 left) 2024-04-12T11:18:33.080 INFO:tasks.rados.rados.0.smithi029.stdout:988: done (6 left) 2024-04-12T11:18:33.080 INFO:tasks.rados.rados.0.smithi029.stdout:995: read oid 4 snap 100 2024-04-12T11:18:33.080 INFO:tasks.rados.rados.0.smithi029.stdout:995: expect (ObjNum 313 snap 96 seq_num 313) 2024-04-12T11:18:33.080 INFO:tasks.rados.rados.0.smithi029.stdout:996: snap_create 2024-04-12T11:18:33.080 INFO:tasks.rados.rados.0.smithi029.stdout:990: finishing rollback tid 0 to smithi02979277-33 2024-04-12T11:18:33.080 INFO:tasks.rados.rados.0.smithi029.stdout:990: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:18:33.080 INFO:tasks.rados.rados.0.smithi029.stdout:990: finishing rollback tid 2 to smithi02979277-33 2024-04-12T11:18:33.080 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 367 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-04-12T11:18:33.080 INFO:tasks.rados.rados.0.smithi029.stdout:989: expect (ObjNum 326 snap 101 seq_num 326) 2024-04-12T11:18:33.269 INFO:tasks.rados.rados.0.smithi029.stdout:991: finishing write tid 1 to smithi02979277-25 2024-04-12T11:18:33.270 INFO:tasks.rados.rados.0.smithi029.stdout:991: finishing write tid 2 to smithi02979277-25 2024-04-12T11:18:33.270 INFO:tasks.rados.rados.0.smithi029.stdout:991: finishing write tid 3 to smithi02979277-25 2024-04-12T11:18:33.270 INFO:tasks.rados.rados.0.smithi029.stdout:991: finishing write tid 4 to smithi02979277-25 2024-04-12T11:18:33.270 INFO:tasks.rados.rados.0.smithi029.stdout:991: oid 25 updating version 0 to 596 2024-04-12T11:18:33.270 INFO:tasks.rados.rados.0.smithi029.stdout:991: oid 25 updating version 596 to 598 2024-04-12T11:18:33.270 INFO:tasks.rados.rados.0.smithi029.stdout:991: oid 25 version 598 is already newer than 597 2024-04-12T11:18:33.270 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 598 (ObjNum 400 snap 123 seq_num 400) dirty exists 2024-04-12T11:18:33.270 INFO:tasks.rados.rados.0.smithi029.stdout:991: left oid 25 (ObjNum 400 snap 123 seq_num 400) 2024-04-12T11:18:33.270 INFO:tasks.rados.rados.0.smithi029.stdout:992: finishing write tid 1 to smithi02979277-2 2024-04-12T11:18:33.270 INFO:tasks.rados.rados.0.smithi029.stdout:993: expect (ObjNum 388 snap 121 seq_num 388) 2024-04-12T11:18:33.406 INFO:tasks.rados.rados.0.smithi029.stdout:992: finishing write tid 2 to smithi02979277-2 2024-04-12T11:18:33.406 INFO:tasks.rados.rados.0.smithi029.stdout:992: finishing write tid 3 to smithi02979277-2 2024-04-12T11:18:33.406 INFO:tasks.rados.rados.0.smithi029.stdout:992: finishing write tid 4 to smithi02979277-2 2024-04-12T11:18:33.406 INFO:tasks.rados.rados.0.smithi029.stdout:992: oid 2 updating version 0 to 67 2024-04-12T11:18:33.406 INFO:tasks.rados.rados.0.smithi029.stdout:992: oid 2 updating version 67 to 68 2024-04-12T11:18:33.406 INFO:tasks.rados.rados.0.smithi029.stdout:992: oid 2 version 68 is already newer than 66 2024-04-12T11:18:33.406 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 68 (ObjNum 401 snap 123 seq_num 401) dirty exists 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:992: left oid 2 (ObjNum 401 snap 123 seq_num 401) 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:994: finishing write tid 1 to smithi02979277-18 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:994: finishing write tid 2 to smithi02979277-18 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:994: finishing write tid 3 to smithi02979277-18 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:994: finishing write tid 4 to smithi02979277-18 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:994: oid 18 updating version 0 to 600 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:994: oid 18 updating version 600 to 601 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:994: oid 18 version 601 is already newer than 599 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 601 (ObjNum 402 snap 123 seq_num 402) dirty exists 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:994: left oid 18 (ObjNum 402 snap 123 seq_num 402) 2024-04-12T11:18:33.407 INFO:tasks.rados.rados.0.smithi029.stdout:995: expect (ObjNum 313 snap 96 seq_num 313) 2024-04-12T11:18:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:33 smithi029 ceph-mon[39999]: osdmap e319: 8 total, 8 up, 8 in 2024-04-12T11:18:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:33 smithi029 ceph-mon[39999]: from='client.25259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:18:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:33 smithi029 ceph-mon[39999]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:18:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:33 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:33 smithi029 ceph-mon[93650]: osdmap e319: 8 total, 8 up, 8 in 2024-04-12T11:18:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:33 smithi029 ceph-mon[93650]: from='client.25259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:18:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:33 smithi029 ceph-mon[93650]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:18:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:33 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:33.825 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:33.503Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:33.825 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:33 smithi087 ceph-mon[32917]: osdmap e319: 8 total, 8 up, 8 in 2024-04-12T11:18:33.826 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:33 smithi087 ceph-mon[32917]: from='client.25259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:18:33.826 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:33 smithi087 ceph-mon[32917]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:18:33.826 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:33 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:33.855 INFO:tasks.rados.rados.0.smithi029.stdout:989: done (7 left) 2024-04-12T11:18:33.855 INFO:tasks.rados.rados.0.smithi029.stdout:990: done (6 left) 2024-04-12T11:18:33.855 INFO:tasks.rados.rados.0.smithi029.stdout:991: done (5 left) 2024-04-12T11:18:33.855 INFO:tasks.rados.rados.0.smithi029.stdout:992: done (4 left) 2024-04-12T11:18:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:993: done (3 left) 2024-04-12T11:18:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:994: done (2 left) 2024-04-12T11:18:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:995: done (1 left) 2024-04-12T11:18:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:996: done (0 left) 2024-04-12T11:18:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:997: snap_create 2024-04-12T11:18:34.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-12T11:18:34.856 INFO:tasks.rados.rados.0.smithi029.stdout:997: done (0 left) 2024-04-12T11:18:34.857 INFO:tasks.rados.rados.0.smithi029.stdout:998: rollback oid 43 current snap is 125 2024-04-12T11:18:34.857 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 124 2024-04-12T11:18:34.857 INFO:tasks.rados.rados.0.smithi029.stdout:999: rollback oid 11 current snap is 125 2024-04-12T11:18:34.857 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 11 to 123 2024-04-12T11:18:34.857 INFO:tasks.rados.rados.0.smithi029.stdout:1000: snap_create 2024-04-12T11:18:34.864 INFO:tasks.rados.rados.0.smithi029.stdout:998: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:18:34.864 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 333 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-04-12T11:18:34.864 INFO:tasks.rados.rados.0.smithi029.stdout:999: finishing rollback tid 0 to smithi02979277-11 2024-04-12T11:18:34.864 INFO:tasks.rados.rados.0.smithi029.stdout:999: finishing rollback tid 1 to smithi02979277-11 2024-04-12T11:18:34.864 INFO:tasks.rados.rados.0.smithi029.stdout:999: finishing rollback tid 2 to smithi02979277-11 2024-04-12T11:18:34.864 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 657 (ObjNum 392 snap 121 seq_num 392) dirty exists 2024-04-12T11:18:35.011 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:18:35.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:34 smithi029 ceph-mon[93650]: pgmap v110: 89 pgs: 89 active+clean; 185 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-04-12T11:18:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:34 smithi029 ceph-mon[93650]: osdmap e320: 8 total, 8 up, 8 in 2024-04-12T11:18:35.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:34 smithi029 ceph-mon[39999]: pgmap v110: 89 pgs: 89 active+clean; 185 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-04-12T11:18:35.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:34 smithi029 ceph-mon[39999]: osdmap e320: 8 total, 8 up, 8 in 2024-04-12T11:18:35.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:18:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:18:34] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.43.0" 2024-04-12T11:18:35.122 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:34 smithi087 ceph-mon[32917]: pgmap v110: 89 pgs: 89 active+clean; 185 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-04-12T11:18:35.122 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:34 smithi087 ceph-mon[32917]: osdmap e320: 8 total, 8 up, 8 in 2024-04-12T11:18:35.865 INFO:tasks.rados.rados.0.smithi029.stdout:998: done (2 left) 2024-04-12T11:18:35.866 INFO:tasks.rados.rados.0.smithi029.stdout:999: done (1 left) 2024-04-12T11:18:35.866 INFO:tasks.rados.rados.0.smithi029.stdout:1000: done (0 left) 2024-04-12T11:18:35.866 INFO:tasks.rados.rados.0.smithi029.stdout:1001: delete oid 21 current snap is 126 2024-04-12T11:18:35.868 INFO:tasks.rados.rados.0.smithi029.stdout:1001: done (0 left) 2024-04-12T11:18:35.868 INFO:tasks.rados.rados.0.smithi029.stdout:1002: snap_remove snap 125 2024-04-12T11:18:36.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:35 smithi087 ceph-mon[32917]: osdmap e321: 8 total, 8 up, 8 in 2024-04-12T11:18:36.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:35 smithi029 ceph-mon[93650]: osdmap e321: 8 total, 8 up, 8 in 2024-04-12T11:18:36.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:35 smithi029 ceph-mon[39999]: osdmap e321: 8 total, 8 up, 8 in 2024-04-12T11:18:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:1002: done (0 left) 2024-04-12T11:18:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:1003: read oid 31 snap -1 2024-04-12T11:18:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:1003: expect (ObjNum 393 snap 123 seq_num 393) 2024-04-12T11:18:36.866 INFO:tasks.rados.rados.0.smithi029.stdout:1004: snap_create 2024-04-12T11:18:36.872 INFO:tasks.rados.rados.0.smithi029.stdout:1003: expect (ObjNum 393 snap 123 seq_num 393) 2024-04-12T11:18:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:36 smithi087 ceph-mon[32917]: pgmap v113: 89 pgs: 89 active+clean; 185 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-04-12T11:18:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:36 smithi087 ceph-mon[32917]: osdmap e322: 8 total, 8 up, 8 in 2024-04-12T11:18:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:36 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:18:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:36 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:36 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:36 smithi029 ceph-mon[39999]: pgmap v113: 89 pgs: 89 active+clean; 185 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-04-12T11:18:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:36 smithi029 ceph-mon[39999]: osdmap e322: 8 total, 8 up, 8 in 2024-04-12T11:18:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:36 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:18:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:36 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:36 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:36 smithi029 ceph-mon[93650]: pgmap v113: 89 pgs: 89 active+clean; 185 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-04-12T11:18:37.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:36 smithi029 ceph-mon[93650]: osdmap e322: 8 total, 8 up, 8 in 2024-04-12T11:18:37.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:18:37.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:37.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:37.487 INFO:teuthology.orchestra.run.smithi029.stdout:wait for majority of mons to upgrade 2024-04-12T11:18:37.512 INFO:tasks.rados.rados.0.smithi029.stdout:1003: done (1 left) 2024-04-12T11:18:37.512 INFO:tasks.rados.rados.0.smithi029.stdout:1004: done (0 left) 2024-04-12T11:18:37.512 INFO:tasks.rados.rados.0.smithi029.stdout:1005: read oid 29 snap 124 2024-04-12T11:18:37.512 INFO:tasks.rados.rados.0.smithi029.stdout:1005: expect deleted 2024-04-12T11:18:37.512 INFO:tasks.rados.rados.0.smithi029.stdout:1006: delete oid 9 current snap is 127 2024-04-12T11:18:37.514 INFO:tasks.rados.rados.0.smithi029.stdout:1005: done (1 left) 2024-04-12T11:18:37.514 INFO:tasks.rados.rados.0.smithi029.stdout:1006: done (0 left) 2024-04-12T11:18:37.514 INFO:tasks.rados.rados.0.smithi029.stdout:1007: read oid 39 snap 124 2024-04-12T11:18:37.514 INFO:tasks.rados.rados.0.smithi029.stdout:1007: expect (ObjNum 388 snap 121 seq_num 388) 2024-04-12T11:18:37.514 INFO:tasks.rados.rados.0.smithi029.stdout:1008: read oid 16 snap -1 2024-04-12T11:18:37.514 INFO:tasks.rados.rados.0.smithi029.stdout:1008: expect (ObjNum 240 snap 75 seq_num 240) 2024-04-12T11:18:37.514 INFO:tasks.rados.rados.0.smithi029.stdout:1009: rollback oid 14 current snap is 127 2024-04-12T11:18:37.514 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 124 2024-04-12T11:18:37.514 INFO:tasks.rados.rados.0.smithi029.stdout:1010: delete oid 48 current snap is 127 2024-04-12T11:18:37.517 INFO:tasks.rados.rados.0.smithi029.stdout:1007: expect (ObjNum 388 snap 121 seq_num 388) 2024-04-12T11:18:37.655 INFO:tasks.rados.rados.0.smithi029.stdout:1008: expect (ObjNum 240 snap 75 seq_num 240) 2024-04-12T11:18:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:1009: finishing rollback tid 0 to smithi02979277-14 2024-04-12T11:18:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:1009: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:18:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:1009: finishing rollback tid 2 to smithi02979277-14 2024-04-12T11:18:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 624 (ObjNum 321 snap 98 seq_num 321) dirty exists 2024-04-12T11:18:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:1007: done (3 left) 2024-04-12T11:18:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:1008: done (2 left) 2024-04-12T11:18:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:1009: done (1 left) 2024-04-12T11:18:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:1010: done (0 left) 2024-04-12T11:18:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:1011: read oid 40 snap -1 2024-04-12T11:18:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:1011: expect deleted 2024-04-12T11:18:37.724 INFO:tasks.rados.rados.0.smithi029.stdout:1012: write oid 16 current snap is 127 2024-04-12T11:18:37.724 INFO:tasks.rados.rados.0.smithi029.stdout:1012: seq_num 406 ranges {423642=717803,1875101=766806} 2024-04-12T11:18:37.728 INFO:tasks.rados.rados.0.smithi029.stdout:1012: writing smithi02979277-16 from 423642 to 1141445 tid 1 2024-04-12T11:18:37.738 INFO:tasks.rados.rados.0.smithi029.stdout:1012: writing smithi02979277-16 from 1875101 to 2641907 tid 2 2024-04-12T11:18:37.738 INFO:tasks.rados.rados.0.smithi029.stdout:1013: rollback oid 34 current snap is 127 2024-04-12T11:18:37.738 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 100 2024-04-12T11:18:37.738 INFO:tasks.rados.rados.0.smithi029.stdout:1012: finishing write tid 1 to smithi02979277-16 2024-04-12T11:18:37.738 INFO:tasks.rados.rados.0.smithi029.stdout:1011: done (2 left) 2024-04-12T11:18:37.738 INFO:tasks.rados.rados.0.smithi029.stdout:1014: read oid 2 snap 123 2024-04-12T11:18:37.738 INFO:tasks.rados.rados.0.smithi029.stdout:1014: expect (ObjNum 401 snap 123 seq_num 401) 2024-04-12T11:18:37.739 INFO:tasks.rados.rados.0.smithi029.stdout:1015: delete oid 49 current snap is 127 2024-04-12T11:18:37.740 INFO:tasks.rados.rados.0.smithi029.stdout:1013: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:18:37.740 INFO:tasks.rados.rados.0.smithi029.stdout:1015: done (3 left) 2024-04-12T11:18:37.740 INFO:tasks.rados.rados.0.smithi029.stdout:1016: rollback oid 1 current snap is 127 2024-04-12T11:18:37.740 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 1 to 107 2024-04-12T11:18:37.740 INFO:tasks.rados.rados.0.smithi029.stdout:1013: finishing rollback tid 2 to smithi02979277-34 2024-04-12T11:18:37.740 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 382 (ObjNum 310 snap 94 seq_num 310) dirty exists 2024-04-12T11:18:37.740 INFO:tasks.rados.rados.0.smithi029.stdout:1013: done (3 left) 2024-04-12T11:18:37.740 INFO:tasks.rados.rados.0.smithi029.stdout:1017: rollback oid 34 current snap is 127 2024-04-12T11:18:37.740 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 100 2024-04-12T11:18:37.741 INFO:tasks.rados.rados.0.smithi029.stdout:1018: write oid 44 current snap is 127 2024-04-12T11:18:37.741 INFO:tasks.rados.rados.0.smithi029.stdout:1018: seq_num 408 ranges {457256=468301,1592065=770475,2854581=2675} 2024-04-12T11:18:37.745 INFO:tasks.rados.rados.0.smithi029.stdout:1018: writing smithi02979277-44 from 457256 to 925557 tid 1 2024-04-12T11:18:37.754 INFO:tasks.rados.rados.0.smithi029.stdout:1018: writing smithi02979277-44 from 1592065 to 2362540 tid 2 2024-04-12T11:18:37.757 INFO:tasks.rados.rados.0.smithi029.stdout:1018: writing smithi02979277-44 from 2854581 to 2857256 tid 3 2024-04-12T11:18:37.757 INFO:tasks.rados.rados.0.smithi029.stdout:1019: read oid 8 snap -1 2024-04-12T11:18:37.757 INFO:tasks.rados.rados.0.smithi029.stdout:1019: expect (ObjNum 399 snap 123 seq_num 399) 2024-04-12T11:18:37.757 INFO:tasks.rados.rados.0.smithi029.stdout:1020: snap_remove snap 126 2024-04-12T11:18:38.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:37 smithi029 ceph-mon[39999]: osdmap e323: 8 total, 8 up, 8 in 2024-04-12T11:18:38.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:37 smithi029 ceph-mon[39999]: osdmap e324: 8 total, 8 up, 8 in 2024-04-12T11:18:38.164 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:37 smithi029 ceph-mon[93650]: osdmap e323: 8 total, 8 up, 8 in 2024-04-12T11:18:38.164 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:37 smithi029 ceph-mon[93650]: osdmap e324: 8 total, 8 up, 8 in 2024-04-12T11:18:38.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-12T11:18:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:37 smithi087 ceph-mon[32917]: osdmap e323: 8 total, 8 up, 8 in 2024-04-12T11:18:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:37 smithi087 ceph-mon[32917]: osdmap e324: 8 total, 8 up, 8 in 2024-04-12T11:18:38.515 INFO:tasks.rados.rados.0.smithi029.stdout:1020: done (6 left) 2024-04-12T11:18:38.515 INFO:tasks.rados.rados.0.smithi029.stdout:1021: snap_create 2024-04-12T11:18:38.515 INFO:tasks.rados.rados.0.smithi029.stdout:1016: finishing rollback tid 0 to smithi02979277-1 2024-04-12T11:18:38.515 INFO:tasks.rados.rados.0.smithi029.stdout:1016: finishing rollback tid 1 to smithi02979277-1 2024-04-12T11:18:38.515 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 495 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-04-12T11:18:38.515 INFO:tasks.rados.rados.0.smithi029.stdout:1014: expect (ObjNum 401 snap 123 seq_num 401) 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:1017: finishing rollback tid 0 to smithi02979277-34 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:1012: finishing write tid 2 to smithi02979277-16 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:1017: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:1012: finishing write tid 3 to smithi02979277-16 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:1012: finishing write tid 4 to smithi02979277-16 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:1012: oid 16 updating version 0 to 626 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:1012: oid 16 updating version 626 to 628 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:1012: oid 16 version 628 is already newer than 627 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 628 (ObjNum 406 snap 127 seq_num 406) dirty exists 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:1012: left oid 16 (ObjNum 406 snap 127 seq_num 406) 2024-04-12T11:18:38.705 INFO:tasks.rados.rados.0.smithi029.stdout:1017: finishing rollback tid 2 to smithi02979277-34 2024-04-12T11:18:38.706 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 385 (ObjNum 310 snap 94 seq_num 310) dirty exists 2024-04-12T11:18:38.706 INFO:tasks.rados.rados.0.smithi029.stdout:1018: finishing write tid 1 to smithi02979277-44 2024-04-12T11:18:38.706 INFO:tasks.rados.rados.0.smithi029.stdout:1018: finishing write tid 2 to smithi02979277-44 2024-04-12T11:18:38.706 INFO:tasks.rados.rados.0.smithi029.stdout:1018: finishing write tid 3 to smithi02979277-44 2024-04-12T11:18:38.706 INFO:tasks.rados.rados.0.smithi029.stdout:1018: finishing write tid 4 to smithi02979277-44 2024-04-12T11:18:38.706 INFO:tasks.rados.rados.0.smithi029.stdout:1018: finishing write tid 5 to smithi02979277-44 2024-04-12T11:18:38.707 INFO:tasks.rados.rados.0.smithi029.stdout:1018: oid 44 updating version 0 to 497 2024-04-12T11:18:38.707 INFO:tasks.rados.rados.0.smithi029.stdout:1018: oid 44 updating version 497 to 500 2024-04-12T11:18:38.707 INFO:tasks.rados.rados.0.smithi029.stdout:1018: oid 44 version 500 is already newer than 499 2024-04-12T11:18:38.707 INFO:tasks.rados.rados.0.smithi029.stdout:1018: oid 44 version 500 is already newer than 498 2024-04-12T11:18:38.707 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 500 (ObjNum 408 snap 127 seq_num 408) dirty exists 2024-04-12T11:18:38.707 INFO:tasks.rados.rados.0.smithi029.stdout:1018: left oid 44 (ObjNum 408 snap 127 seq_num 408) 2024-04-12T11:18:38.707 INFO:tasks.rados.rados.0.smithi029.stdout:1019: expect (ObjNum 399 snap 123 seq_num 399) 2024-04-12T11:18:39.042 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:18:39.165 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:38 smithi029 ceph-mon[39999]: pgmap v116: 89 pgs: 89 active+clean; 186 MiB data, 635 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-04-12T11:18:39.165 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:38 smithi029 ceph-mon[39999]: osdmap e325: 8 total, 8 up, 8 in 2024-04-12T11:18:39.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:38 smithi029 ceph-mon[93650]: pgmap v116: 89 pgs: 89 active+clean; 186 MiB data, 635 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-04-12T11:18:39.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:38 smithi029 ceph-mon[93650]: osdmap e325: 8 total, 8 up, 8 in 2024-04-12T11:18:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:38 smithi087 ceph-mon[32917]: pgmap v116: 89 pgs: 89 active+clean; 186 MiB data, 635 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-04-12T11:18:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:38 smithi087 ceph-mon[32917]: osdmap e325: 8 total, 8 up, 8 in 2024-04-12T11:18:39.515 INFO:tasks.rados.rados.0.smithi029.stdout:1012: done (6 left) 2024-04-12T11:18:39.516 INFO:tasks.rados.rados.0.smithi029.stdout:1014: done (5 left) 2024-04-12T11:18:39.516 INFO:tasks.rados.rados.0.smithi029.stdout:1016: done (4 left) 2024-04-12T11:18:39.516 INFO:tasks.rados.rados.0.smithi029.stdout:1017: done (3 left) 2024-04-12T11:18:39.516 INFO:tasks.rados.rados.0.smithi029.stdout:1018: done (2 left) 2024-04-12T11:18:39.516 INFO:tasks.rados.rados.0.smithi029.stdout:1019: done (1 left) 2024-04-12T11:18:39.516 INFO:tasks.rados.rados.0.smithi029.stdout:1021: done (0 left) 2024-04-12T11:18:39.516 INFO:tasks.rados.rados.0.smithi029.stdout:1022: write oid 4 current snap is 128 2024-04-12T11:18:39.516 INFO:tasks.rados.rados.0.smithi029.stdout:1022: seq_num 409 ranges {474063=543550,1650547=514133,2765651=108412} 2024-04-12T11:18:39.521 INFO:tasks.rados.rados.0.smithi029.stdout:1022: writing smithi02979277-4 from 474063 to 1017613 tid 1 2024-04-12T11:18:39.527 INFO:tasks.rados.rados.0.smithi029.stdout:1022: writing smithi02979277-4 from 1650547 to 2164680 tid 2 2024-04-12T11:18:39.532 INFO:tasks.rados.rados.0.smithi029.stdout:1022: writing smithi02979277-4 from 2765651 to 2874063 tid 3 2024-04-12T11:18:39.532 INFO:tasks.rados.rados.0.smithi029.stdout:1023: delete oid 48 current snap is 128 2024-04-12T11:18:39.532 INFO:tasks.rados.rados.0.smithi029.stdout:1022: finishing write tid 1 to smithi02979277-4 2024-04-12T11:18:39.532 INFO:tasks.rados.rados.0.smithi029.stdout:1022: finishing write tid 2 to smithi02979277-4 2024-04-12T11:18:39.534 INFO:tasks.rados.rados.0.smithi029.stdout:1022: finishing write tid 3 to smithi02979277-4 2024-04-12T11:18:39.534 INFO:tasks.rados.rados.0.smithi029.stdout:1022: finishing write tid 4 to smithi02979277-4 2024-04-12T11:18:39.535 INFO:tasks.rados.rados.0.smithi029.stdout:1022: finishing write tid 5 to smithi02979277-4 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1022: oid 4 updating version 0 to 636 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1022: oid 4 version 636 is already newer than 635 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1022: oid 4 updating version 636 to 637 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1022: oid 4 version 637 is already newer than 634 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 637 (ObjNum 409 snap 128 seq_num 409) dirty exists 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1022: left oid 4 (ObjNum 409 snap 128 seq_num 409) 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1022: done (1 left) 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1023: done (0 left) 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1024: read oid 14 snap 100 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1024: expect (ObjNum 321 snap 98 seq_num 321) 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1025: write oid 15 current snap is 128 2024-04-12T11:18:39.536 INFO:tasks.rados.rados.0.smithi029.stdout:1025: seq_num 411 ranges {507677=410401,1367511=517802,2420497=487180} 2024-04-12T11:18:39.540 INFO:tasks.rados.rados.0.smithi029.stdout:1025: writing smithi02979277-15 from 507677 to 918078 tid 1 2024-04-12T11:18:39.546 INFO:tasks.rados.rados.0.smithi029.stdout:1025: writing smithi02979277-15 from 1367511 to 1885313 tid 2 2024-04-12T11:18:39.552 INFO:tasks.rados.rados.0.smithi029.stdout:1025: writing smithi02979277-15 from 2420497 to 2907677 tid 3 2024-04-12T11:18:39.552 INFO:tasks.rados.rados.0.smithi029.stdout:1026: read oid 9 snap -1 2024-04-12T11:18:39.552 INFO:tasks.rados.rados.0.smithi029.stdout:1026: expect deleted 2024-04-12T11:18:39.552 INFO:tasks.rados.rados.0.smithi029.stdout:1027: rollback oid 27 current snap is 128 2024-04-12T11:18:39.552 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 123 2024-04-12T11:18:39.552 INFO:tasks.rados.rados.0.smithi029.stdout:1025: finishing write tid 1 to smithi02979277-15 2024-04-12T11:18:39.552 INFO:tasks.rados.rados.0.smithi029.stdout:1025: finishing write tid 2 to smithi02979277-15 2024-04-12T11:18:39.552 INFO:tasks.rados.rados.0.smithi029.stdout:1024: expect (ObjNum 321 snap 98 seq_num 321) 2024-04-12T11:18:39.711 INFO:tasks.rados.rados.0.smithi029.stdout:1025: finishing write tid 3 to smithi02979277-15 2024-04-12T11:18:39.711 INFO:tasks.rados.rados.0.smithi029.stdout:1025: finishing write tid 4 to smithi02979277-15 2024-04-12T11:18:39.711 INFO:tasks.rados.rados.0.smithi029.stdout:1025: finishing write tid 5 to smithi02979277-15 2024-04-12T11:18:39.711 INFO:tasks.rados.rados.0.smithi029.stdout:1025: oid 15 updating version 0 to 464 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1025: oid 15 updating version 464 to 466 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1025: oid 15 version 466 is already newer than 465 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1025: oid 15 version 466 is already newer than 463 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 466 (ObjNum 411 snap 128 seq_num 411) dirty exists 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1025: left oid 15 (ObjNum 411 snap 128 seq_num 411) 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1027: finishing rollback tid 0 to smithi02979277-27 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1027: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1027: finishing rollback tid 2 to smithi02979277-27 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 470 (ObjNum 318 snap 97 seq_num 318) dirty exists 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1024: done (3 left) 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1025: done (2 left) 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1026: done (1 left) 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1027: done (0 left) 2024-04-12T11:18:39.712 INFO:tasks.rados.rados.0.smithi029.stdout:1028: snap_create 2024-04-12T11:18:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: pgmap v119: 89 pgs: 89 active+clean; 186 MiB data, 635 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-04-12T11:18:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: osdmap e326: 8 total, 8 up, 8 in 2024-04-12T11:18:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:18:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:18:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: pgmap v119: 89 pgs: 89 active+clean; 186 MiB data, 635 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: osdmap e326: 8 total, 8 up, 8 in 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:40.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:18:40.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:18:40.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T11:18:40.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:40 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-12T11:18:40.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: pgmap v119: 89 pgs: 89 active+clean; 186 MiB data, 635 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-04-12T11:18:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: osdmap e326: 8 total, 8 up, 8 in 2024-04-12T11:18:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:18:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:18:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:18:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:18:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T11:18:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:40 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-12T11:18:40.517 INFO:tasks.rados.rados.0.smithi029.stdout:1028: done (0 left) 2024-04-12T11:18:40.517 INFO:tasks.rados.rados.0.smithi029.stdout:1029: read oid 1 snap 127 2024-04-12T11:18:40.517 INFO:tasks.rados.rados.0.smithi029.stdout:1029: expect deleted 2024-04-12T11:18:40.517 INFO:tasks.rados.rados.0.smithi029.stdout:1030: write oid 28 current snap is 129 2024-04-12T11:18:40.517 INFO:tasks.rados.rados.0.smithi029.stdout:1030: seq_num 412 ranges {524484=485650,1425993=661460,2847920=76564} 2024-04-12T11:18:40.522 INFO:tasks.rados.rados.0.smithi029.stdout:1030: writing smithi02979277-28 from 524484 to 1010134 tid 1 2024-04-12T11:18:40.528 INFO:tasks.rados.rados.0.smithi029.stdout:1030: writing smithi02979277-28 from 1425993 to 2087453 tid 2 2024-04-12T11:18:40.533 INFO:tasks.rados.rados.0.smithi029.stdout:1030: writing smithi02979277-28 from 2847920 to 2924484 tid 3 2024-04-12T11:18:40.533 INFO:tasks.rados.rados.0.smithi029.stdout:1031: write oid 26 current snap is 129 2024-04-12T11:18:40.533 INFO:tasks.rados.rados.0.smithi029.stdout:1031: seq_num 413 ranges {541291=560899,1884475=521471} 2024-04-12T11:18:40.539 INFO:tasks.rados.rados.0.smithi029.stdout:1031: writing smithi02979277-26 from 541291 to 1102190 tid 1 2024-04-12T11:18:40.546 INFO:tasks.rados.rados.0.smithi029.stdout:1031: writing smithi02979277-26 from 1884475 to 2405946 tid 2 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1032: snap_create 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1030: finishing write tid 1 to smithi02979277-28 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1030: finishing write tid 2 to smithi02979277-28 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1030: finishing write tid 3 to smithi02979277-28 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1030: finishing write tid 4 to smithi02979277-28 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1030: finishing write tid 5 to smithi02979277-28 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1030: oid 28 updating version 0 to 437 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1030: oid 28 version 437 is already newer than 436 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1030: oid 28 updating version 437 to 439 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1030: oid 28 version 439 is already newer than 438 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 439 (ObjNum 412 snap 129 seq_num 412) dirty exists 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1030: left oid 28 (ObjNum 412 snap 129 seq_num 412) 2024-04-12T11:18:40.547 INFO:tasks.rados.rados.0.smithi029.stdout:1031: finishing write tid 1 to smithi02979277-26 2024-04-12T11:18:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1031: finishing write tid 2 to smithi02979277-26 2024-04-12T11:18:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1031: finishing write tid 3 to smithi02979277-26 2024-04-12T11:18:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1031: finishing write tid 4 to smithi02979277-26 2024-04-12T11:18:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1031: oid 26 updating version 0 to 659 2024-04-12T11:18:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1031: oid 26 updating version 659 to 660 2024-04-12T11:18:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1031: oid 26 updating version 660 to 661 2024-04-12T11:18:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 661 (ObjNum 413 snap 129 seq_num 413) dirty exists 2024-04-12T11:18:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1031: left oid 26 (ObjNum 413 snap 129 seq_num 413) 2024-04-12T11:18:41.522 INFO:tasks.rados.rados.0.smithi029.stdout:1029: done (3 left) 2024-04-12T11:18:41.522 INFO:tasks.rados.rados.0.smithi029.stdout:1030: done (2 left) 2024-04-12T11:18:41.522 INFO:tasks.rados.rados.0.smithi029.stdout:1031: done (1 left) 2024-04-12T11:18:41.522 INFO:tasks.rados.rados.0.smithi029.stdout:1032: done (0 left) 2024-04-12T11:18:41.522 INFO:tasks.rados.rados.0.smithi029.stdout:1033: delete oid 26 current snap is 130 2024-04-12T11:18:41.525 INFO:tasks.rados.rados.0.smithi029.stdout:1033: done (0 left) 2024-04-12T11:18:41.525 INFO:tasks.rados.rados.0.smithi029.stdout:1034: write oid 47 current snap is 130 2024-04-12T11:18:41.525 INFO:tasks.rados.rados.0.smithi029.stdout:1034: seq_num 415 ranges {574905=711397,1885086=525140,2930189=44716} 2024-04-12T11:18:41.532 INFO:tasks.rados.rados.0.smithi029.stdout:1034: writing smithi02979277-47 from 574905 to 1286302 tid 1 2024-04-12T11:18:41.538 INFO:tasks.rados.rados.0.smithi029.stdout:1034: writing smithi02979277-47 from 1885086 to 2410226 tid 2 2024-04-12T11:18:41.542 INFO:tasks.rados.rados.0.smithi029.stdout:1034: writing smithi02979277-47 from 2930189 to 2974905 tid 3 2024-04-12T11:18:41.542 INFO:tasks.rados.rados.0.smithi029.stdout:1035: write oid 20 current snap is 130 2024-04-12T11:18:41.542 INFO:tasks.rados.rados.0.smithi029.stdout:1034: finishing write tid 1 to smithi02979277-47 2024-04-12T11:18:41.542 INFO:tasks.rados.rados.0.smithi029.stdout:1035: seq_num 416 ranges {591712=786646,1943568=668798} 2024-04-12T11:18:41.549 INFO:tasks.rados.rados.0.smithi029.stdout:1035: writing smithi02979277-20 from 591712 to 1378358 tid 1 2024-04-12T11:18:41.557 INFO:tasks.rados.rados.0.smithi029.stdout:1035: writing smithi02979277-20 from 1943568 to 2612366 tid 2 2024-04-12T11:18:41.557 INFO:tasks.rados.rados.0.smithi029.stdout:1036: snap_create 2024-04-12T11:18:41.557 INFO:tasks.rados.rados.0.smithi029.stdout:1034: finishing write tid 2 to smithi02979277-47 2024-04-12T11:18:41.557 INFO:tasks.rados.rados.0.smithi029.stdout:1034: finishing write tid 3 to smithi02979277-47 2024-04-12T11:18:41.557 INFO:tasks.rados.rados.0.smithi029.stdout:1034: finishing write tid 4 to smithi02979277-47 2024-04-12T11:18:41.557 INFO:tasks.rados.rados.0.smithi029.stdout:1034: finishing write tid 5 to smithi02979277-47 2024-04-12T11:18:41.557 INFO:tasks.rados.rados.0.smithi029.stdout:1034: oid 47 updating version 0 to 542 2024-04-12T11:18:41.558 INFO:tasks.rados.rados.0.smithi029.stdout:1034: oid 47 updating version 542 to 543 2024-04-12T11:18:41.558 INFO:tasks.rados.rados.0.smithi029.stdout:1034: oid 47 updating version 543 to 545 2024-04-12T11:18:41.558 INFO:tasks.rados.rados.0.smithi029.stdout:1034: oid 47 version 545 is already newer than 544 2024-04-12T11:18:41.558 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 545 (ObjNum 415 snap 130 seq_num 415) dirty exists 2024-04-12T11:18:41.558 INFO:tasks.rados.rados.0.smithi029.stdout:1034: left oid 47 (ObjNum 415 snap 130 seq_num 415) 2024-04-12T11:18:41.558 INFO:tasks.rados.rados.0.smithi029.stdout:1035: finishing write tid 1 to smithi02979277-20 2024-04-12T11:18:41.560 INFO:tasks.rados.rados.0.smithi029.stdout:1035: finishing write tid 2 to smithi02979277-20 2024-04-12T11:18:41.560 INFO:tasks.rados.rados.0.smithi029.stdout:1035: finishing write tid 3 to smithi02979277-20 2024-04-12T11:18:41.561 INFO:tasks.rados.rados.0.smithi029.stdout:1035: finishing write tid 4 to smithi02979277-20 2024-04-12T11:18:41.561 INFO:tasks.rados.rados.0.smithi029.stdout:1035: oid 20 updating version 0 to 247 2024-04-12T11:18:41.561 INFO:tasks.rados.rados.0.smithi029.stdout:1035: oid 20 updating version 247 to 248 2024-04-12T11:18:41.561 INFO:tasks.rados.rados.0.smithi029.stdout:1035: oid 20 version 248 is already newer than 246 2024-04-12T11:18:41.561 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 248 (ObjNum 416 snap 130 seq_num 416) dirty exists 2024-04-12T11:18:41.561 INFO:tasks.rados.rados.0.smithi029.stdout:1035: left oid 20 (ObjNum 416 snap 130 seq_num 416) 2024-04-12T11:18:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:41 smithi029 ceph-mon[39999]: Upgrade: It appears safe to stop mon.c 2024-04-12T11:18:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:41 smithi029 ceph-mon[39999]: osdmap e327: 8 total, 8 up, 8 in 2024-04-12T11:18:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:41 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/1412302785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:18:41.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:41 smithi029 ceph-mon[93650]: Upgrade: It appears safe to stop mon.c 2024-04-12T11:18:41.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:41 smithi029 ceph-mon[93650]: osdmap e327: 8 total, 8 up, 8 in 2024-04-12T11:18:41.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:41 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1412302785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:18:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:41 smithi087 ceph-mon[32917]: Upgrade: It appears safe to stop mon.c 2024-04-12T11:18:41.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:41 smithi087 ceph-mon[32917]: osdmap e327: 8 total, 8 up, 8 in 2024-04-12T11:18:41.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:41 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/1412302785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:18:42.514 INFO:tasks.rados.rados.0.smithi029.stdout:1034: done (2 left) 2024-04-12T11:18:42.514 INFO:tasks.rados.rados.0.smithi029.stdout:1035: done (1 left) 2024-04-12T11:18:42.514 INFO:tasks.rados.rados.0.smithi029.stdout:1036: done (0 left) 2024-04-12T11:18:42.514 INFO:tasks.rados.rados.0.smithi029.stdout:1037: read oid 36 snap 129 2024-04-12T11:18:42.514 INFO:tasks.rados.rados.0.smithi029.stdout:1037: expect deleted 2024-04-12T11:18:42.514 INFO:tasks.rados.rados.0.smithi029.stdout:1038: write oid 8 current snap is 131 2024-04-12T11:18:42.514 INFO:tasks.rados.rados.0.smithi029.stdout:1038: seq_num 417 ranges {608519=461895,1602050=528809,2585035=423484} 2024-04-12T11:18:42.519 INFO:tasks.rados.rados.0.smithi029.stdout:1038: writing smithi02979277-8 from 608519 to 1070414 tid 1 2024-04-12T11:18:42.525 INFO:tasks.rados.rados.0.smithi029.stdout:1038: writing smithi02979277-8 from 1602050 to 2130859 tid 2 2024-04-12T11:18:42.531 INFO:tasks.rados.rados.0.smithi029.stdout:1038: writing smithi02979277-8 from 2585035 to 3008519 tid 3 2024-04-12T11:18:42.531 INFO:tasks.rados.rados.0.smithi029.stdout:1039: write oid 16 current snap is 131 2024-04-12T11:18:42.531 INFO:tasks.rados.rados.0.smithi029.stdout:1039: seq_num 418 ranges {625326=537144,1944179=672467} 2024-04-12T11:18:42.536 INFO:tasks.rados.rados.0.smithi029.stdout:1039: writing smithi02979277-16 from 625326 to 1162470 tid 1 2024-04-12T11:18:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:1039: writing smithi02979277-16 from 1944179 to 2616646 tid 2 2024-04-12T11:18:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:1040: rollback oid 9 current snap is 131 2024-04-12T11:18:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 9 to 100 2024-04-12T11:18:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:1038: finishing write tid 1 to smithi02979277-8 2024-04-12T11:18:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:1038: finishing write tid 2 to smithi02979277-8 2024-04-12T11:18:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:1038: finishing write tid 3 to smithi02979277-8 2024-04-12T11:18:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:1038: finishing write tid 4 to smithi02979277-8 2024-04-12T11:18:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:1038: finishing write tid 5 to smithi02979277-8 2024-04-12T11:18:42.546 INFO:tasks.rados.rados.0.smithi029.stdout:1038: oid 8 updating version 0 to 79 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1038: oid 8 updating version 79 to 80 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1038: oid 8 version 80 is already newer than 77 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1038: oid 8 version 80 is already newer than 78 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 80 (ObjNum 417 snap 131 seq_num 417) dirty exists 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1038: left oid 8 (ObjNum 417 snap 131 seq_num 417) 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1039: finishing write tid 1 to smithi02979277-16 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1037: done (3 left) 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1038: done (2 left) 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1041: read oid 21 snap -1 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1041: expect deleted 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1042: write oid 2 current snap is 131 2024-04-12T11:18:42.547 INFO:tasks.rados.rados.0.smithi029.stdout:1042: seq_num 419 ranges {642133=728746,2119014=532478} 2024-04-12T11:18:42.552 INFO:tasks.rados.rados.0.smithi029.stdout:1042: writing smithi02979277-2 from 642133 to 1370879 tid 1 2024-04-12T11:18:42.560 INFO:tasks.rados.rados.0.smithi029.stdout:1042: writing smithi02979277-2 from 2119014 to 2651492 tid 2 2024-04-12T11:18:42.560 INFO:tasks.rados.rados.0.smithi029.stdout:1043: snap_create 2024-04-12T11:18:42.560 INFO:tasks.rados.rados.0.smithi029.stdout:1040: finishing rollback tid 1 to smithi02979277-9 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:1040: finishing rollback tid 2 to smithi02979277-9 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 466 (ObjNum 268 snap 86 seq_num 268) dirty exists 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:1039: finishing write tid 2 to smithi02979277-16 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:1039: finishing write tid 3 to smithi02979277-16 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:1039: finishing write tid 4 to smithi02979277-16 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:1039: oid 16 updating version 0 to 641 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:1039: oid 16 version 641 is already newer than 639 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:1039: oid 16 version 641 is already newer than 640 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 641 (ObjNum 418 snap 131 seq_num 418) dirty exists 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:1039: left oid 16 (ObjNum 418 snap 131 seq_num 418) 2024-04-12T11:18:42.561 INFO:tasks.rados.rados.0.smithi029.stdout:1042: finishing write tid 1 to smithi02979277-2 2024-04-12T11:18:42.562 INFO:tasks.rados.rados.0.smithi029.stdout:1042: finishing write tid 2 to smithi02979277-2 2024-04-12T11:18:42.563 INFO:tasks.rados.rados.0.smithi029.stdout:1042: finishing write tid 3 to smithi02979277-2 2024-04-12T11:18:42.563 INFO:tasks.rados.rados.0.smithi029.stdout:1042: finishing write tid 4 to smithi02979277-2 2024-04-12T11:18:42.563 INFO:tasks.rados.rados.0.smithi029.stdout:1042: oid 2 updating version 0 to 71 2024-04-12T11:18:42.563 INFO:tasks.rados.rados.0.smithi029.stdout:1042: oid 2 updating version 71 to 72 2024-04-12T11:18:42.563 INFO:tasks.rados.rados.0.smithi029.stdout:1042: oid 2 version 72 is already newer than 70 2024-04-12T11:18:42.563 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 72 (ObjNum 419 snap 131 seq_num 419) dirty exists 2024-04-12T11:18:42.563 INFO:tasks.rados.rados.0.smithi029.stdout:1042: left oid 2 (ObjNum 419 snap 131 seq_num 419) 2024-04-12T11:18:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:42 smithi087 ceph-mon[32917]: pgmap v122: 89 pgs: 89 active+clean; 186 MiB data, 635 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:42 smithi087 ceph-mon[32917]: osdmap e328: 8 total, 8 up, 8 in 2024-04-12T11:18:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:42 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:42 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:18:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:42 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:18:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:42 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:18:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:42 smithi087 ceph-mon[32917]: osdmap e329: 8 total, 8 up, 8 in 2024-04-12T11:18:42.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[39999]: pgmap v122: 89 pgs: 89 active+clean; 186 MiB data, 635 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:42.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[39999]: osdmap e328: 8 total, 8 up, 8 in 2024-04-12T11:18:42.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:42.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:18:42.807 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:18:42.807 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[39999]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:18:42.807 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[39999]: osdmap e329: 8 total, 8 up, 8 in 2024-04-12T11:18:42.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[93650]: pgmap v122: 89 pgs: 89 active+clean; 186 MiB data, 635 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:18:42.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[93650]: osdmap e328: 8 total, 8 up, 8 in 2024-04-12T11:18:42.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:42.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:18:42.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:18:42.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:18:42.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:42 smithi029 ceph-mon[93650]: osdmap e329: 8 total, 8 up, 8 in 2024-04-12T11:18:43.528 INFO:tasks.rados.rados.0.smithi029.stdout:1039: done (4 left) 2024-04-12T11:18:43.529 INFO:tasks.rados.rados.0.smithi029.stdout:1040: done (3 left) 2024-04-12T11:18:43.529 INFO:tasks.rados.rados.0.smithi029.stdout:1041: done (2 left) 2024-04-12T11:18:43.529 INFO:tasks.rados.rados.0.smithi029.stdout:1042: done (1 left) 2024-04-12T11:18:43.529 INFO:tasks.rados.rados.0.smithi029.stdout:1043: done (0 left) 2024-04-12T11:18:43.529 INFO:tasks.rados.rados.0.smithi029.stdout:1044: rollback oid 27 current snap is 132 2024-04-12T11:18:43.529 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 100 2024-04-12T11:18:43.529 INFO:tasks.rados.rados.0.smithi029.stdout:1045: write oid 39 current snap is 132 2024-04-12T11:18:43.529 INFO:tasks.rados.rados.0.smithi029.stdout:1045: seq_num 420 ranges {658940=403995,1777496=676136,2950951=107989} 2024-04-12T11:18:43.534 INFO:tasks.rados.rados.0.smithi029.stdout:1045: writing smithi02979277-39 from 658940 to 1062935 tid 1 2024-04-12T11:18:43.542 INFO:tasks.rados.rados.0.smithi029.stdout:1045: writing smithi02979277-39 from 1777496 to 2453632 tid 2 2024-04-12T11:18:43.545 INFO:tasks.rados.rados.0.smithi029.stdout:1045: writing smithi02979277-39 from 2950951 to 3058940 tid 3 2024-04-12T11:18:43.546 INFO:tasks.rados.rados.0.smithi029.stdout:1046: write oid 38 current snap is 132 2024-04-12T11:18:43.546 INFO:tasks.rados.rados.0.smithi029.stdout:1044: finishing rollback tid 0 to smithi02979277-27 2024-04-12T11:18:43.546 INFO:tasks.rados.rados.0.smithi029.stdout:1044: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:18:43.546 INFO:tasks.rados.rados.0.smithi029.stdout:1044: finishing rollback tid 2 to smithi02979277-27 2024-04-12T11:18:43.546 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 474 (ObjNum 318 snap 97 seq_num 318) dirty exists 2024-04-12T11:18:43.546 INFO:tasks.rados.rados.0.smithi029.stdout:1045: finishing write tid 1 to smithi02979277-39 2024-04-12T11:18:43.546 INFO:tasks.rados.rados.0.smithi029.stdout:1045: finishing write tid 2 to smithi02979277-39 2024-04-12T11:18:43.546 INFO:tasks.rados.rados.0.smithi029.stdout:1046: seq_num 421 ranges {675747=479244,1835978=536147} 2024-04-12T11:18:43.551 INFO:tasks.rados.rados.0.smithi029.stdout:1046: writing smithi02979277-38 from 675747 to 1154991 tid 1 2024-04-12T11:18:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1046: writing smithi02979277-38 from 1835978 to 2372125 tid 2 2024-04-12T11:18:43.559 INFO:tasks.rados.rados.0.smithi029.stdout:1044: done (2 left) 2024-04-12T11:18:43.559 INFO:tasks.rados.rados.0.smithi029.stdout:1047: write oid 32 current snap is 132 2024-04-12T11:18:43.559 INFO:tasks.rados.rados.0.smithi029.stdout:1045: finishing write tid 3 to smithi02979277-39 2024-04-12T11:18:43.559 INFO:tasks.rados.rados.0.smithi029.stdout:1047: seq_num 422 ranges {692554=554493,1778107=679805,2889444=203110} 2024-04-12T11:18:43.565 INFO:tasks.rados.rados.0.smithi029.stdout:1047: writing smithi02979277-32 from 692554 to 1247047 tid 1 2024-04-12T11:18:43.572 INFO:tasks.rados.rados.0.smithi029.stdout:1047: writing smithi02979277-32 from 1778107 to 2457912 tid 2 2024-04-12T11:18:43.577 INFO:tasks.rados.rados.0.smithi029.stdout:1047: writing smithi02979277-32 from 2889444 to 3092554 tid 3 2024-04-12T11:18:43.577 INFO:tasks.rados.rados.0.smithi029.stdout:1048: read oid 16 snap 131 2024-04-12T11:18:43.577 INFO:tasks.rados.rados.0.smithi029.stdout:1048: expect (ObjNum 418 snap 131 seq_num 418) 2024-04-12T11:18:43.577 INFO:tasks.rados.rados.0.smithi029.stdout:1049: rollback oid 8 current snap is 132 2024-04-12T11:18:43.577 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 128 2024-04-12T11:18:43.577 INFO:tasks.rados.rados.0.smithi029.stdout:1045: finishing write tid 4 to smithi02979277-39 2024-04-12T11:18:43.577 INFO:tasks.rados.rados.0.smithi029.stdout:1045: finishing write tid 5 to smithi02979277-39 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1045: oid 39 updating version 0 to 82 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1045: oid 39 updating version 82 to 85 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1045: oid 39 version 85 is already newer than 83 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1045: oid 39 version 85 is already newer than 84 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 85 (ObjNum 420 snap 132 seq_num 420) dirty exists 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1045: left oid 39 (ObjNum 420 snap 132 seq_num 420) 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1046: finishing write tid 1 to smithi02979277-38 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1046: finishing write tid 2 to smithi02979277-38 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1046: finishing write tid 3 to smithi02979277-38 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1046: finishing write tid 4 to smithi02979277-38 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1046: oid 38 updating version 0 to 250 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1046: oid 38 updating version 250 to 252 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:1046: oid 38 version 252 is already newer than 251 2024-04-12T11:18:43.578 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 252 (ObjNum 421 snap 132 seq_num 421) dirty exists 2024-04-12T11:18:43.579 INFO:tasks.rados.rados.0.smithi029.stdout:1046: left oid 38 (ObjNum 421 snap 132 seq_num 421) 2024-04-12T11:18:43.579 INFO:tasks.rados.rados.0.smithi029.stdout:1047: finishing write tid 1 to smithi02979277-32 2024-04-12T11:18:43.579 INFO:tasks.rados.rados.0.smithi029.stdout:1047: finishing write tid 2 to smithi02979277-32 2024-04-12T11:18:43.579 INFO:tasks.rados.rados.0.smithi029.stdout:1045: done (4 left) 2024-04-12T11:18:43.579 INFO:tasks.rados.rados.0.smithi029.stdout:1046: done (3 left) 2024-04-12T11:18:43.579 INFO:tasks.rados.rados.0.smithi029.stdout:1050: snap_create 2024-04-12T11:18:43.580 INFO:tasks.rados.rados.0.smithi029.stdout:1047: finishing write tid 3 to smithi02979277-32 2024-04-12T11:18:43.580 INFO:tasks.rados.rados.0.smithi029.stdout:1047: finishing write tid 4 to smithi02979277-32 2024-04-12T11:18:43.580 INFO:tasks.rados.rados.0.smithi029.stdout:1047: finishing write tid 5 to smithi02979277-32 2024-04-12T11:18:43.580 INFO:tasks.rados.rados.0.smithi029.stdout:1047: oid 32 updating version 0 to 389 2024-04-12T11:18:43.580 INFO:tasks.rados.rados.0.smithi029.stdout:1047: oid 32 version 389 is already newer than 388 2024-04-12T11:18:43.580 INFO:tasks.rados.rados.0.smithi029.stdout:1047: oid 32 version 389 is already newer than 387 2024-04-12T11:18:43.580 INFO:tasks.rados.rados.0.smithi029.stdout:1047: oid 32 updating version 389 to 390 2024-04-12T11:18:43.580 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 390 (ObjNum 422 snap 132 seq_num 422) dirty exists 2024-04-12T11:18:43.580 INFO:tasks.rados.rados.0.smithi029.stdout:1047: left oid 32 (ObjNum 422 snap 132 seq_num 422) 2024-04-12T11:18:43.586 INFO:tasks.rados.rados.0.smithi029.stdout:1048: expect (ObjNum 418 snap 131 seq_num 418) 2024-04-12T11:18:43.771 INFO:tasks.rados.rados.0.smithi029.stdout:1049: finishing rollback tid 0 to smithi02979277-8 2024-04-12T11:18:43.771 INFO:tasks.rados.rados.0.smithi029.stdout:1049: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:18:43.771 INFO:tasks.rados.rados.0.smithi029.stdout:1049: finishing rollback tid 2 to smithi02979277-8 2024-04-12T11:18:43.771 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 84 (ObjNum 399 snap 123 seq_num 399) dirty exists 2024-04-12T11:18:43.809 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:43 smithi029 ceph-mon[39999]: Upgrade: Updating mon.c 2024-04-12T11:18:43.809 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:43 smithi029 ceph-mon[39999]: Deploying daemon mon.c on smithi029 2024-04-12T11:18:43.809 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:43 smithi029 ceph-mon[93650]: Upgrade: Updating mon.c 2024-04-12T11:18:43.809 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:43 smithi029 ceph-mon[93650]: Deploying daemon mon.c on smithi029 2024-04-12T11:18:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:43 smithi087 ceph-mon[32917]: Upgrade: Updating mon.c 2024-04-12T11:18:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:43 smithi087 ceph-mon[32917]: Deploying daemon mon.c on smithi029 2024-04-12T11:18:43.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:44.535 INFO:tasks.rados.rados.0.smithi029.stdout:1047: done (3 left) 2024-04-12T11:18:44.535 INFO:tasks.rados.rados.0.smithi029.stdout:1048: done (2 left) 2024-04-12T11:18:44.535 INFO:tasks.rados.rados.0.smithi029.stdout:1049: done (1 left) 2024-04-12T11:18:44.535 INFO:tasks.rados.rados.0.smithi029.stdout:1050: done (0 left) 2024-04-12T11:18:44.536 INFO:tasks.rados.rados.0.smithi029.stdout:1051: snap_remove snap 132 2024-04-12T11:18:44.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:44 smithi029 systemd[1]: Stopping Ceph mon.c for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:18:44.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:44 smithi029 ceph-mon[39999]: pgmap v125: 89 pgs: 89 active+clean; 201 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 25 op/s 2024-04-12T11:18:44.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:44 smithi029 ceph-mon[39999]: osdmap e330: 8 total, 8 up, 8 in 2024-04-12T11:18:44.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:44 smithi029 ceph-mon[39999]: from='client.? 172.21.15.29:0/942275072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:18:44.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:44 smithi029 ceph-mon[93650]: pgmap v125: 89 pgs: 89 active+clean; 201 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 25 op/s 2024-04-12T11:18:44.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:44 smithi029 ceph-mon[93650]: osdmap e330: 8 total, 8 up, 8 in 2024-04-12T11:18:44.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:44 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/942275072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:18:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:44 smithi087 ceph-mon[32917]: pgmap v125: 89 pgs: 89 active+clean; 201 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 25 op/s 2024-04-12T11:18:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:44 smithi087 ceph-mon[32917]: osdmap e330: 8 total, 8 up, 8 in 2024-04-12T11:18:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:44 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/942275072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:18:45.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:18:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:18:44] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.43.0" 2024-04-12T11:18:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c[39995]: 2024-04-12T11:18:44.736+0000 7f710507a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:18:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c[39995]: 2024-04-12T11:18:44.736+0000 7f710507a700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-12T11:18:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:44 smithi029 podman[97595]: 2024-04-12 11:18:44.91094906 +0000 UTC m=+0.275666767 container died 903a61eba1ba470c97553c87f35a543e56032a5838a7ac748beadbde0977b75e (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, RELEASE=reef-d540eba, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-12T11:18:45.474 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:45 smithi029 podman[97595]: 2024-04-12 11:18:45.459281011 +0000 UTC m=+0.823998781 container cleanup 903a61eba1ba470c97553c87f35a543e56032a5838a7ac748beadbde0977b75e (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=reef-d540eba, ceph=True) 2024-04-12T11:18:45.474 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:45 smithi029 bash[97595]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c 2024-04-12T11:18:46.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:45 smithi029 podman[97609]: 2024-04-12 11:18:45.789185188 +0000 UTC m=+0.877552740 container remove 903a61eba1ba470c97553c87f35a543e56032a5838a7ac748beadbde0977b75e (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=reef-d540eba, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403) 2024-04-12T11:18:46.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:45 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-12T11:18:46.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:45 smithi029 systemd[1]: Stopped Ceph mon.c for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:18:46.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:45 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.c.service: Consumed 8.220s CPU time. 2024-04-12T11:18:46.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 systemd[1]: Starting Ceph mon.c for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:18:46.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 podman[97768]: 2024-04-12 11:18:46.271774644 +0000 UTC m=+0.018103844 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:18:46.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 podman[97768]: 2024-04-12 11:18:46.383576024 +0000 UTC m=+0.129905289 container create 24b3f58e1d6692c78136ded59902d8d19032a359698a90ca0fac8168aa230e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, RELEASE=wip-rf-comb1-2cee185, ceph=True, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-12T11:18:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 podman[97768]: 2024-04-12 11:18:46.657027741 +0000 UTC m=+0.403356939 container init 24b3f58e1d6692c78136ded59902d8d19032a359698a90ca0fac8168aa230e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-rf-comb1-2cee185, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409) 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 podman[97768]: 2024-04-12 11:18:46.660301352 +0000 UTC m=+0.406630545 container start 24b3f58e1d6692c78136ded59902d8d19032a359698a90ca0fac8168aa230e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-rf-comb1-2cee185, ceph=True) 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev), process ceph-mon, pid 2 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: pidfile_write: ignore empty --pid-file 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: load: jerasure load: lrc 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: RocksDB version: 7.9.2 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Git sha 0 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Compile date 2024-04-11 21:23:14 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: DB SUMMARY 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: DB Session ID: 4SUADZO3X6TB794ERC9D 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: CURRENT file: CURRENT 2024-04-12T11:18:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1160 Bytes 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000024.sst 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000022.log size: 10102499 ; 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.error_if_exists: 0 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.create_if_missing: 0 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.paranoid_checks: 1 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.env: 0x5631c9573cc0 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T11:18:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.info_log: 0x5631c9ed9bc0 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.statistics: (nil) 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.use_fsync: 0 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_log_file_size: 0 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.allow_fallocate: 1 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.use_direct_reads: 0 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T11:18:47.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.db_log_dir: 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.wal_dir: 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.write_buffer_manager: 0x5631c9eaa1e0 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T11:18:47.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.unordered_write: 0 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.row_cache: None 2024-04-12T11:18:47.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.wal_filter: None 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.two_write_queues: 0 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.wal_compression: 0 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.atomic_flush: 0 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.log_readahead_size: 0 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T11:18:47.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_background_jobs: 2 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_background_compactions: -1 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_subcompactions: 1 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T11:18:47.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_open_files: -1 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_background_flushes: -1 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Compression algorithms supported: 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: kZSTD supported: 0 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: kXpressCompression supported: 0 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: kZlibCompression supported: 1 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: kSnappyCompression supported: 1 2024-04-12T11:18:47.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: kLZ4Compression supported: 1 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: kBZip2Compression supported: 0 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.merge_operator: 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_filter: None 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5631c9ed9da0) 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks: 1 2024-04-12T11:18:47.028 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_top_level_index_and_filter: 1 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_type: 0 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_index_type: 0 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_shortening: 1 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: checksum: 4 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: no_block_cache: 0 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache: 0x5631c9edd090 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_name: BinnedLRUCache 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_options: 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: capacity : 536870912 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: num_shard_bits : 4 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: strict_capacity_limit : 0 2024-04-12T11:18:47.029 INFO:journalctl@ceph.mon.c.smithi029.stdout: high_pri_pool_ratio: 0.000 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_compressed: (nil) 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: persistent_cache: (nil) 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size: 4096 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size_deviation: 10 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_restart_interval: 16 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_block_restart_interval: 1 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: metadata_block_size: 4096 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: partition_filters: 0 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: use_delta_encoding: 1 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: filter_policy: bloomfilter 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: whole_key_filtering: 1 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: verify_compression: 0 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: read_amp_bytes_per_bit: 0 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: format_version: 5 2024-04-12T11:18:47.030 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_index_compression: 1 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_align: 0 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout: max_auto_readahead_size: 262144 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout: prepopulate_block_cache: 0 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout: initial_auto_readahead_size: 8192 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compression: NoCompression 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.num_levels: 7 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T11:18:47.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T11:18:47.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T11:18:47.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T11:18:47.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.inplace_update_support: 0 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.bloom_locality: 0 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.max_successive_merges: 0 2024-04-12T11:18:47.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.ttl: 2592000 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.enable_blob_files: false 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.min_blob_size: 0 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T11:18:47.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 26, last_sequence is 13360, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 22 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: afee1398-990a-44be-958f-7bc352c06785 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920726689093, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920726711504, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 6305670, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13365, "largest_seqno": 17321, "table_properties": {"data_size": 6287842, "index_size": 12621, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4165, "raw_key_size": 37297, "raw_average_key_size": 22, "raw_value_size": 6252184, "raw_average_value_size": 3773, "num_data_blocks": 546, "num_entries": 1657, "num_filter_entries": 1657, "num_deletions": 304, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712920726, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "afee1398-990a-44be-958f-7bc352c06785", "db_session_id": "4SUADZO3X6TB794ERC9D", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920726711625, "job": 1, "event": "recovery_finished"} 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T11:18:47.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5631c9fcc000 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: DB pointer 0x5631c9fb6000 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** DB Stats ** 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2024-04-12T11:18:47.038 INFO:journalctl@ceph.mon.c.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: L0 1/0 6.01 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 664.0 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: L6 1/0 7.49 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: Sum 2/0 13.50 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 664.0 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 664.0 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 664.0 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T11:18:47.039 INFO:journalctl@ceph.mon.c.smithi029.stdout: Flush(GB): cumulative 0.006, interval 0.006 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative compaction: 0.01 GB write, 238.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval compaction: 0.01 GB write, 238.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: Block cache BinnedLRUCache@0x5631c9edd090#2 capacity: 512.00 MB usage: 17.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,13.08 KB,0.00249445%) Misc(1,0.00 KB,0%) 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: starting mon.c rank 2 at public addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] at bind addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: mon.c@-1(???) e3 preinit fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: mon.c@-1(???).mds e9 new map 2024-04-12T11:18:47.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: mon.c@-1(???).mds e9 print_map 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: e9 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: legacy client fscid: 1 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: Filesystem 'foo' (1) 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: fs_name foo 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: epoch 8 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: created 2024-04-12T11:12:57.763670+0000 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: modified 2024-04-12T11:13:16.631701+0000 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: tableserver 0 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: root 0 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: session_timeout 60 2024-04-12T11:18:47.041 INFO:journalctl@ceph.mon.c.smithi029.stdout: session_autoclose 300 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: max_file_size 1099511627776 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: max_xattr_size 65536 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: required_client_features {} 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: last_failure 0 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: last_failure_osd_epoch 0 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: max_mds 1 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: in 0 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: up {0=14661} 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: failed 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: damaged 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: stopped 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_pools [4] 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: metadata_pool 3 2024-04-12T11:18:47.042 INFO:journalctl@ceph.mon.c.smithi029.stdout: inline_data disabled 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout: balancer 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout: bal_rank_mask -1 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout: standby_count_wanted 1 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout: qdb_cluster leader: 0 members: 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout: [mds.foo.smithi029.scbuux{0:14661} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout: Standby daemons: 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout: [mds.foo.smithi087.mbflzu{-1:24502} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.87:6832/2304329976,v1:172.21.15.87:6833/2304329976] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: mon.c@-1(???).osd e331 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: mon.c@-1(???).osd e331 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: mon.c@-1(???).osd e331 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: mon.c@-1(???).osd e331 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 ceph-mon[97787]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-12T11:18:47.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 bash[97768]: 24b3f58e1d6692c78136ded59902d8d19032a359698a90ca0fac8168aa230e35 2024-04-12T11:18:47.044 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:46 smithi029 systemd[1]: Started Ceph mon.c for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:18:52.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:52.957 INFO:tasks.rados.rados.0.smithi029.stdout:1051: done (0 left) 2024-04-12T11:18:52.957 INFO:tasks.rados.rados.0.smithi029.stdout:1052: snap_create 2024-04-12T11:18:52.992 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: mon.c calling monitor election 2024-04-12T11:18:52.992 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: mon.b calling monitor election 2024-04-12T11:18:52.992 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: mon.a calling monitor election 2024-04-12T11:18:52.992 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:18:52.992 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: monmap epoch 3 2024-04-12T11:18:52.992 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:18:52.992 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: last_changed 2024-04-12T11:07:50.917802+0000 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: created 2024-04-12T11:06:03.531247+0000 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: min_mon_release 18 (reef) 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: election_strategy: 1 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: 1: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.b 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: osdmap e331: 8 total, 8 up, 8 in 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: mgrmap e36: y(active, since 91s), standbys: x 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: Cluster is now healthy 2024-04-12T11:18:52.993 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:52 smithi087 ceph-mon[32917]: overall HEALTH_OK 2024-04-12T11:18:53.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: mon.c calling monitor election 2024-04-12T11:18:53.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: mon.b calling monitor election 2024-04-12T11:18:53.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: mon.a calling monitor election 2024-04-12T11:18:53.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:18:53.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: monmap epoch 3 2024-04-12T11:18:53.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:18:53.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: last_changed 2024-04-12T11:07:50.917802+0000 2024-04-12T11:18:53.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: created 2024-04-12T11:06:03.531247+0000 2024-04-12T11:18:53.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: min_mon_release 18 (reef) 2024-04-12T11:18:53.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: election_strategy: 1 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: 1: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.b 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: osdmap e331: 8 total, 8 up, 8 in 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: mgrmap e36: y(active, since 91s), standbys: x 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[93650]: overall HEALTH_OK 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: mon.c calling monitor election 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: mon.b calling monitor election 2024-04-12T11:18:53.022 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: mon.a calling monitor election 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: monmap epoch 3 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: last_changed 2024-04-12T11:07:50.917802+0000 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: created 2024-04-12T11:06:03.531247+0000 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: min_mon_release 18 (reef) 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: election_strategy: 1 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: 1: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.b 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: osdmap e331: 8 total, 8 up, 8 in 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: mgrmap e36: y(active, since 91s), standbys: x 2024-04-12T11:18:53.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-12T11:18:53.024 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:18:53.024 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:52 smithi029 ceph-mon[97787]: overall HEALTH_OK 2024-04-12T11:18:53.909 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:18:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:18:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:18:53.967 INFO:tasks.rados.rados.0.smithi029.stdout:1052: done (0 left) 2024-04-12T11:18:53.967 INFO:tasks.rados.rados.0.smithi029.stdout:1053: snap_remove snap 128 2024-04-12T11:18:54.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:53 smithi087 ceph-mon[32917]: osdmap e332: 8 total, 8 up, 8 in 2024-04-12T11:18:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:53 smithi029 ceph-mon[93650]: osdmap e332: 8 total, 8 up, 8 in 2024-04-12T11:18:54.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:53 smithi029 ceph-mon[97787]: osdmap e332: 8 total, 8 up, 8 in 2024-04-12T11:18:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev)": 2 2024-04-12T11:18:54.964 INFO:tasks.rados.rados.0.smithi029.stdout:1053: done (0 left) 2024-04-12T11:18:54.964 INFO:tasks.rados.rados.0.smithi029.stdout:1054: delete oid 17 current snap is 134 2024-04-12T11:18:54.965 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:18:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:18:54] "GET /metrics HTTP/1.1" 200 36243 "" "Prometheus/2.43.0" 2024-04-12T11:18:54.969 INFO:tasks.rados.rados.0.smithi029.stdout:1054: done (0 left) 2024-04-12T11:18:54.969 INFO:tasks.rados.rados.0.smithi029.stdout:1055: read oid 16 snap -1 2024-04-12T11:18:54.969 INFO:tasks.rados.rados.0.smithi029.stdout:1055: expect (ObjNum 418 snap 131 seq_num 418) 2024-04-12T11:18:54.970 INFO:tasks.rados.rados.0.smithi029.stdout:1056: read oid 42 snap -1 2024-04-12T11:18:54.970 INFO:tasks.rados.rados.0.smithi029.stdout:1056: expect (ObjNum 322 snap 100 seq_num 322) 2024-04-12T11:18:54.970 INFO:tasks.rados.rados.0.smithi029.stdout:1057: snap_remove snap 127 2024-04-12T11:18:55.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:54 smithi087 ceph-mon[32917]: pgmap v133: 89 pgs: 89 active+clean; 217 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 850 KiB/s wr, 5 op/s 2024-04-12T11:18:55.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:54 smithi087 ceph-mon[32917]: osdmap e333: 8 total, 8 up, 8 in 2024-04-12T11:18:55.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:54 smithi087 ceph-mon[32917]: mgrmap e37: y(active, since 92s), standbys: x 2024-04-12T11:18:55.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:54 smithi087 ceph-mon[32917]: from='client.? 172.21.15.29:0/2482762887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:18:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:54 smithi029 ceph-mon[93650]: pgmap v133: 89 pgs: 89 active+clean; 217 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 850 KiB/s wr, 5 op/s 2024-04-12T11:18:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:54 smithi029 ceph-mon[93650]: osdmap e333: 8 total, 8 up, 8 in 2024-04-12T11:18:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:54 smithi029 ceph-mon[93650]: mgrmap e37: y(active, since 92s), standbys: x 2024-04-12T11:18:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:54 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2482762887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:18:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:54 smithi029 ceph-mon[97787]: pgmap v133: 89 pgs: 89 active+clean; 217 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 850 KiB/s wr, 5 op/s 2024-04-12T11:18:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:54 smithi029 ceph-mon[97787]: osdmap e333: 8 total, 8 up, 8 in 2024-04-12T11:18:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:54 smithi029 ceph-mon[97787]: mgrmap e37: y(active, since 92s), standbys: x 2024-04-12T11:18:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:54 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2482762887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:18:55.546 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch ps' 2024-04-12T11:18:55.973 INFO:tasks.rados.rados.0.smithi029.stdout:1057: done (2 left) 2024-04-12T11:18:55.974 INFO:tasks.rados.rados.0.smithi029.stdout:1058: snap_create 2024-04-12T11:18:55.974 INFO:tasks.rados.rados.0.smithi029.stdout:1056: expect (ObjNum 322 snap 100 seq_num 322) 2024-04-12T11:18:56.201 INFO:tasks.rados.rados.0.smithi029.stdout:1055: expect (ObjNum 418 snap 131 seq_num 418) 2024-04-12T11:18:56.248 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:18:56.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:55 smithi029 ceph-mon[93650]: osdmap e334: 8 total, 8 up, 8 in 2024-04-12T11:18:56.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:55 smithi029 ceph-mon[97787]: osdmap e334: 8 total, 8 up, 8 in 2024-04-12T11:18:56.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:55 smithi087 ceph-mon[32917]: osdmap e334: 8 total, 8 up, 8 in 2024-04-12T11:18:56.975 INFO:tasks.rados.rados.0.smithi029.stdout:1055: done (2 left) 2024-04-12T11:18:56.975 INFO:tasks.rados.rados.0.smithi029.stdout:1056: done (1 left) 2024-04-12T11:18:56.975 INFO:tasks.rados.rados.0.smithi029.stdout:1058: done (0 left) 2024-04-12T11:18:56.975 INFO:tasks.rados.rados.0.smithi029.stdout:1059: read oid 27 snap -1 2024-04-12T11:18:56.975 INFO:tasks.rados.rados.0.smithi029.stdout:1059: expect (ObjNum 318 snap 97 seq_num 318) 2024-04-12T11:18:56.975 INFO:tasks.rados.rados.0.smithi029.stdout:1060: read oid 41 snap -1 2024-04-12T11:18:56.975 INFO:tasks.rados.rados.0.smithi029.stdout:1060: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:18:56.975 INFO:tasks.rados.rados.0.smithi029.stdout:1061: write oid 6 current snap is 135 2024-04-12T11:18:56.976 INFO:tasks.rados.rados.0.smithi029.stdout:1061: seq_num 424 ranges {726168=704991,1895071=683474} 2024-04-12T11:18:56.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:56 smithi029 ceph-mon[97787]: pgmap v136: 89 pgs: 89 active+clean; 217 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 0 op/s 2024-04-12T11:18:56.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:56 smithi029 ceph-mon[97787]: osdmap e335: 8 total, 8 up, 8 in 2024-04-12T11:18:56.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:56 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:56.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:56 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:56.979 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:56 smithi029 ceph-mon[93650]: pgmap v136: 89 pgs: 89 active+clean; 217 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 0 op/s 2024-04-12T11:18:56.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:56 smithi029 ceph-mon[93650]: osdmap e335: 8 total, 8 up, 8 in 2024-04-12T11:18:56.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:56 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:56.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:56 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:56.980 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:18:56 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:18:56.719+0000 7f4c1f740700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-12T11:18:56.984 INFO:tasks.rados.rados.0.smithi029.stdout:1061: writing smithi02979277-6 from 726168 to 1431159 tid 1 2024-04-12T11:18:56.990 INFO:tasks.rados.rados.0.smithi029.stdout:1061: writing smithi02979277-6 from 1895071 to 2578545 tid 2 2024-04-12T11:18:56.990 INFO:tasks.rados.rados.0.smithi029.stdout:1062: delete oid 23 current snap is 135 2024-04-12T11:18:56.990 INFO:tasks.rados.rados.0.smithi029.stdout:1059: expect (ObjNum 318 snap 97 seq_num 318) 2024-04-12T11:18:57.163 INFO:tasks.rados.rados.0.smithi029.stdout:1060: expect (ObjNum 73 snap 9 seq_num 73) 2024-04-12T11:18:57.306 INFO:tasks.rados.rados.0.smithi029.stdout:1061: finishing write tid 1 to smithi02979277-6 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1061: finishing write tid 2 to smithi02979277-6 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1061: finishing write tid 3 to smithi02979277-6 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1061: finishing write tid 4 to smithi02979277-6 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1061: oid 6 updating version 0 to 256 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1061: oid 6 updating version 256 to 257 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1061: oid 6 version 257 is already newer than 255 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 257 (ObjNum 424 snap 135 seq_num 424) dirty exists 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1061: left oid 6 (ObjNum 424 snap 135 seq_num 424) 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1059: done (3 left) 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1060: done (2 left) 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1061: done (1 left) 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1062: done (0 left) 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1063: read oid 37 snap 133 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1063: expect deleted 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1064: read oid 32 snap -1 2024-04-12T11:18:57.307 INFO:tasks.rados.rados.0.smithi029.stdout:1064: expect (ObjNum 422 snap 132 seq_num 422) 2024-04-12T11:18:57.308 INFO:tasks.rados.rados.0.smithi029.stdout:1065: read oid 45 snap -1 2024-04-12T11:18:57.308 INFO:tasks.rados.rados.0.smithi029.stdout:1065: expect (ObjNum 395 snap 123 seq_num 395) 2024-04-12T11:18:57.308 INFO:tasks.rados.rados.0.smithi029.stdout:1066: write oid 24 current snap is 135 2024-04-12T11:18:57.308 INFO:tasks.rados.rados.0.smithi029.stdout:1066: seq_num 426 ranges {759782=571842,2012035=687143} 2024-04-12T11:18:57.313 INFO:tasks.rados.rados.0.smithi029.stdout:1066: writing smithi02979277-24 from 759782 to 1331624 tid 1 2024-04-12T11:18:57.321 INFO:tasks.rados.rados.0.smithi029.stdout:1066: writing smithi02979277-24 from 2012035 to 2699178 tid 2 2024-04-12T11:18:57.321 INFO:tasks.rados.rados.0.smithi029.stdout:1067: read oid 48 snap -1 2024-04-12T11:18:57.321 INFO:tasks.rados.rados.0.smithi029.stdout:1067: expect deleted 2024-04-12T11:18:57.321 INFO:tasks.rados.rados.0.smithi029.stdout:1068: rollback oid 22 current snap is 135 2024-04-12T11:18:57.321 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 22 to 123 2024-04-12T11:18:57.322 INFO:tasks.rados.rados.0.smithi029.stdout:1065: expect (ObjNum 395 snap 123 seq_num 395) 2024-04-12T11:18:57.448 INFO:tasks.rados.rados.0.smithi029.stdout:1064: expect (ObjNum 422 snap 132 seq_num 422) 2024-04-12T11:18:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:56 smithi087 ceph-mon[32917]: pgmap v136: 89 pgs: 89 active+clean; 217 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 0 op/s 2024-04-12T11:18:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:56 smithi087 ceph-mon[32917]: osdmap e335: 8 total, 8 up, 8 in 2024-04-12T11:18:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:56 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:56 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:57.622 INFO:tasks.rados.rados.0.smithi029.stdout:1066: finishing write tid 1 to smithi02979277-24 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:1068: finishing rollback tid 1 to smithi02979277-22 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 326 (ObjNum 33 snap 0 seq_num 1801121360) dirty dne 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:1066: finishing write tid 2 to smithi02979277-24 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:1066: finishing write tid 3 to smithi02979277-24 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:1066: finishing write tid 4 to smithi02979277-24 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:1066: oid 24 updating version 0 to 670 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:1066: oid 24 version 670 is already newer than 669 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:1066: oid 24 updating version 670 to 671 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 671 (ObjNum 426 snap 135 seq_num 426) dirty exists 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:1066: left oid 24 (ObjNum 426 snap 135 seq_num 426) 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:1063: done (5 left) 2024-04-12T11:18:57.623 INFO:tasks.rados.rados.0.smithi029.stdout:1064: done (4 left) 2024-04-12T11:18:57.624 INFO:tasks.rados.rados.0.smithi029.stdout:1065: done (3 left) 2024-04-12T11:18:57.624 INFO:tasks.rados.rados.0.smithi029.stdout:1066: done (2 left) 2024-04-12T11:18:57.624 INFO:tasks.rados.rados.0.smithi029.stdout:1067: done (1 left) 2024-04-12T11:18:57.624 INFO:tasks.rados.rados.0.smithi029.stdout:1068: done (0 left) 2024-04-12T11:18:57.624 INFO:tasks.rados.rados.0.smithi029.stdout:1069: read oid 28 snap -1 2024-04-12T11:18:57.624 INFO:tasks.rados.rados.0.smithi029.stdout:1069: expect (ObjNum 412 snap 129 seq_num 412) 2024-04-12T11:18:57.624 INFO:tasks.rados.rados.0.smithi029.stdout:1070: rollback oid 11 current snap is 135 2024-04-12T11:18:57.624 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 11 to 100 2024-04-12T11:18:57.624 INFO:tasks.rados.rados.0.smithi029.stdout:1071: write oid 43 current snap is 135 2024-04-12T11:18:57.624 INFO:tasks.rados.rados.0.smithi029.stdout:1071: seq_num 427 ranges {776589=647091,2070517=430801,3026559=150030} 2024-04-12T11:18:57.630 INFO:tasks.rados.rados.0.smithi029.stdout:1071: writing smithi02979277-43 from 776589 to 1423680 tid 1 2024-04-12T11:18:57.636 INFO:tasks.rados.rados.0.smithi029.stdout:1071: writing smithi02979277-43 from 2070517 to 2501318 tid 2 2024-04-12T11:18:57.640 INFO:tasks.rados.rados.0.smithi029.stdout:1071: writing smithi02979277-43 from 3026559 to 3176589 tid 3 2024-04-12T11:18:57.640 INFO:tasks.rados.rados.0.smithi029.stdout:1072: snap_create 2024-04-12T11:18:57.640 INFO:tasks.rados.rados.0.smithi029.stdout:1070: finishing rollback tid 0 to smithi02979277-11 2024-04-12T11:18:57.640 INFO:tasks.rados.rados.0.smithi029.stdout:1070: finishing rollback tid 1 to smithi02979277-11 2024-04-12T11:18:57.640 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 674 (ObjNum 49 snap 0 seq_num 0) dirty dne 2024-04-12T11:18:57.640 INFO:tasks.rados.rados.0.smithi029.stdout:1069: expect (ObjNum 412 snap 129 seq_num 412) 2024-04-12T11:18:57.813 INFO:tasks.rados.rados.0.smithi029.stdout:1071: finishing write tid 1 to smithi02979277-43 2024-04-12T11:18:57.814 INFO:tasks.rados.rados.0.smithi029.stdout:1071: finishing write tid 2 to smithi02979277-43 2024-04-12T11:18:57.814 INFO:tasks.rados.rados.0.smithi029.stdout:1071: finishing write tid 3 to smithi02979277-43 2024-04-12T11:18:57.814 INFO:tasks.rados.rados.0.smithi029.stdout:1071: finishing write tid 4 to smithi02979277-43 2024-04-12T11:18:57.814 INFO:tasks.rados.rados.0.smithi029.stdout:1071: finishing write tid 5 to smithi02979277-43 2024-04-12T11:18:57.814 INFO:tasks.rados.rados.0.smithi029.stdout:1071: oid 43 updating version 0 to 396 2024-04-12T11:18:57.814 INFO:tasks.rados.rados.0.smithi029.stdout:1071: oid 43 updating version 396 to 397 2024-04-12T11:18:57.814 INFO:tasks.rados.rados.0.smithi029.stdout:1071: oid 43 version 397 is already newer than 395 2024-04-12T11:18:57.814 INFO:tasks.rados.rados.0.smithi029.stdout:1071: oid 43 updating version 397 to 398 2024-04-12T11:18:57.814 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 398 (ObjNum 427 snap 135 seq_num 427) dirty exists 2024-04-12T11:18:57.814 INFO:tasks.rados.rados.0.smithi029.stdout:1071: left oid 43 (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:18:58.063 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:57 smithi029 ceph-mon[97787]: osdmap e336: 8 total, 8 up, 8 in 2024-04-12T11:18:58.063 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:57 smithi029 ceph-mon[97787]: osdmap e337: 8 total, 8 up, 8 in 2024-04-12T11:18:58.063 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:57 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:58.063 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:57 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:58.063 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:57 smithi029 ceph-mon[93650]: osdmap e336: 8 total, 8 up, 8 in 2024-04-12T11:18:58.063 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:57 smithi029 ceph-mon[93650]: osdmap e337: 8 total, 8 up, 8 in 2024-04-12T11:18:58.064 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:57 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:58.064 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:57 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:58.303 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T11:18:58.303 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (4m) 1s ago 6m 26.3M - 0.25.0 c8568f914cd2 5f36b9c37cec 2024-04-12T11:18:58.303 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi087 *:3000 running (6m) 21s ago 6m 89.2M - 9.4.7 954c08fa6188 d4ba4d587fdb 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.scbuux smithi029 running (5m) 1s ago 5m 17.0M - 18.2.2-1187-gd540ebac bb2c5eac26a4 a659ce0bccaf 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi087.mbflzu smithi087 running (5m) 21s ago 5m 16.9M - 18.2.2-1187-gd540ebac bb2c5eac26a4 64bafd1e6a4c 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi087 *:8443,9283,8765 running (2m) 21s ago 10m 450M - 19.0.0-2874-g2cee1857 4a0e59462345 5a46135dfdbc 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (2m) 1s ago 12m 499M - 19.0.0-2874-g2cee1857 4a0e59462345 05c187489262 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (106s) 1s ago 12m 42.9M 2048M 19.0.0-2874-g2cee1857 4a0e59462345 d1506def0998 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi087 running (11m) 21s ago 11m 59.0M 2048M 18.2.2-1187-gd540ebac bb2c5eac26a4 adfeefa5d84f 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (12s) 1s ago 11m 25.6M 2048M 19.0.0-2874-g2cee1857 4a0e59462345 24b3f58e1d66 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (6m) 1s ago 6m 14.9M - 1.5.0 0da6a335fe13 32124106e912 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi087 *:9100 running (6m) 21s ago 6m 15.7M - 1.5.0 0da6a335fe13 78910b2d30c6 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (10m) 1s ago 10m 97.3M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 f65c324b0084 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (10m) 1s ago 10m 75.7M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 bb552e04f578 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (9m) 1s ago 9m 94.1M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 dbb7abb8911b 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (9m) 1s ago 9m 69.2M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 a9b5c72d15da 2024-04-12T11:18:58.304 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi087 running (8m) 21s ago 8m 87.5M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 ab3a8e864afd 2024-04-12T11:18:58.305 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi087 running (8m) 21s ago 8m 89.6M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 573e2f90e4da 2024-04-12T11:18:58.305 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi087 running (8m) 21s ago 8m 69.1M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 2b67d3f477dd 2024-04-12T11:18:58.305 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi087 running (7m) 21s ago 7m 85.5M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 9c484b9f0790 2024-04-12T11:18:58.305 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi087 *:9095 running (2m) 21s ago 7m 50.4M - 2.43.0 a07b618ecd1d 569140fc5a53 2024-04-12T11:18:58.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:57 smithi087 ceph-mon[32917]: osdmap e336: 8 total, 8 up, 8 in 2024-04-12T11:18:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:57 smithi087 ceph-mon[32917]: osdmap e337: 8 total, 8 up, 8 in 2024-04-12T11:18:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:57 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:57 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:18:58.517 INFO:tasks.rados.rados.0.smithi029.stdout:1069: done (3 left) 2024-04-12T11:18:58.518 INFO:tasks.rados.rados.0.smithi029.stdout:1070: done (2 left) 2024-04-12T11:18:58.518 INFO:tasks.rados.rados.0.smithi029.stdout:1071: done (1 left) 2024-04-12T11:18:58.518 INFO:tasks.rados.rados.0.smithi029.stdout:1072: done (0 left) 2024-04-12T11:18:58.518 INFO:tasks.rados.rados.0.smithi029.stdout:1073: snap_create 2024-04-12T11:18:59.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch upgrade pause' 2024-04-12T11:18:59.259 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:58 smithi029 ceph-mon[97787]: pgmap v139: 89 pgs: 89 active+clean; 218 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 339 KiB/s wr, 10 op/s 2024-04-12T11:18:59.259 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:58 smithi029 ceph-mon[97787]: osdmap e338: 8 total, 8 up, 8 in 2024-04-12T11:18:59.260 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:58 smithi029 ceph-mon[93650]: pgmap v139: 89 pgs: 89 active+clean; 218 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 339 KiB/s wr, 10 op/s 2024-04-12T11:18:59.260 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:58 smithi029 ceph-mon[93650]: osdmap e338: 8 total, 8 up, 8 in 2024-04-12T11:18:59.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:58 smithi087 ceph-mon[32917]: pgmap v139: 89 pgs: 89 active+clean; 218 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 339 KiB/s wr, 10 op/s 2024-04-12T11:18:59.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:58 smithi087 ceph-mon[32917]: osdmap e338: 8 total, 8 up, 8 in 2024-04-12T11:18:59.519 INFO:tasks.rados.rados.0.smithi029.stdout:1073: done (0 left) 2024-04-12T11:18:59.519 INFO:tasks.rados.rados.0.smithi029.stdout:1074: read oid 47 snap -1 2024-04-12T11:18:59.519 INFO:tasks.rados.rados.0.smithi029.stdout:1074: expect (ObjNum 415 snap 130 seq_num 415) 2024-04-12T11:18:59.519 INFO:tasks.rados.rados.0.smithi029.stdout:1075: write oid 26 current snap is 137 2024-04-12T11:18:59.519 INFO:tasks.rados.rados.0.smithi029.stdout:1075: seq_num 428 ranges {793396=722340,2128999=690812} 2024-04-12T11:18:59.527 INFO:tasks.rados.rados.0.smithi029.stdout:1075: writing smithi02979277-26 from 793396 to 1515736 tid 1 2024-04-12T11:18:59.535 INFO:tasks.rados.rados.0.smithi029.stdout:1075: writing smithi02979277-26 from 2128999 to 2819811 tid 2 2024-04-12T11:18:59.535 INFO:tasks.rados.rados.0.smithi029.stdout:1076: write oid 49 current snap is 137 2024-04-12T11:18:59.535 INFO:tasks.rados.rados.0.smithi029.stdout:1076: seq_num 429 ranges {410203=797589,1787481=434470,2681405=528798} 2024-04-12T11:18:59.541 INFO:tasks.rados.rados.0.smithi029.stdout:1076: writing smithi02979277-49 from 410203 to 1207792 tid 1 2024-04-12T11:18:59.547 INFO:tasks.rados.rados.0.smithi029.stdout:1076: writing smithi02979277-49 from 1787481 to 2221951 tid 2 2024-04-12T11:18:59.552 INFO:tasks.rados.rados.0.smithi029.stdout:1076: writing smithi02979277-49 from 2681405 to 3210203 tid 3 2024-04-12T11:18:59.552 INFO:tasks.rados.rados.0.smithi029.stdout:1077: write oid 2 current snap is 137 2024-04-12T11:18:59.553 INFO:tasks.rados.rados.0.smithi029.stdout:1077: seq_num 430 ranges {427010=472838,1329610=694481,2708828=518182} 2024-04-12T11:18:59.557 INFO:tasks.rados.rados.0.smithi029.stdout:1077: writing smithi02979277-2 from 427010 to 899848 tid 1 2024-04-12T11:18:59.564 INFO:tasks.rados.rados.0.smithi029.stdout:1077: writing smithi02979277-2 from 1329610 to 2024091 tid 2 2024-04-12T11:18:59.571 INFO:tasks.rados.rados.0.smithi029.stdout:1077: writing smithi02979277-2 from 2708828 to 3227010 tid 3 2024-04-12T11:18:59.571 INFO:tasks.rados.rados.0.smithi029.stdout:1078: read oid 36 snap -1 2024-04-12T11:18:59.571 INFO:tasks.rados.rados.0.smithi029.stdout:1078: expect deleted 2024-04-12T11:18:59.571 INFO:tasks.rados.rados.0.smithi029.stdout:1079: read oid 16 snap -1 2024-04-12T11:18:59.571 INFO:tasks.rados.rados.0.smithi029.stdout:1079: expect (ObjNum 418 snap 131 seq_num 418) 2024-04-12T11:18:59.571 INFO:tasks.rados.rados.0.smithi029.stdout:1080: rollback oid 4 current snap is 137 2024-04-12T11:18:59.571 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 4 to 134 2024-04-12T11:18:59.572 INFO:tasks.rados.rados.0.smithi029.stdout:1074: expect (ObjNum 415 snap 130 seq_num 415) 2024-04-12T11:18:59.787 INFO:tasks.rados.rados.0.smithi029.stdout:1075: finishing write tid 1 to smithi02979277-26 2024-04-12T11:18:59.787 INFO:tasks.rados.rados.0.smithi029.stdout:1075: finishing write tid 2 to smithi02979277-26 2024-04-12T11:18:59.787 INFO:tasks.rados.rados.0.smithi029.stdout:1075: finishing write tid 3 to smithi02979277-26 2024-04-12T11:18:59.787 INFO:tasks.rados.rados.0.smithi029.stdout:1075: finishing write tid 4 to smithi02979277-26 2024-04-12T11:18:59.787 INFO:tasks.rados.rados.0.smithi029.stdout:1075: oid 26 updating version 0 to 677 2024-04-12T11:18:59.787 INFO:tasks.rados.rados.0.smithi029.stdout:1075: oid 26 version 677 is already newer than 676 2024-04-12T11:18:59.787 INFO:tasks.rados.rados.0.smithi029.stdout:1075: oid 26 version 677 is already newer than 675 2024-04-12T11:18:59.787 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 677 (ObjNum 428 snap 137 seq_num 428) dirty exists 2024-04-12T11:18:59.788 INFO:tasks.rados.rados.0.smithi029.stdout:1075: left oid 26 (ObjNum 428 snap 137 seq_num 428) 2024-04-12T11:18:59.788 INFO:tasks.rados.rados.0.smithi029.stdout:1076: finishing write tid 1 to smithi02979277-49 2024-04-12T11:18:59.788 INFO:tasks.rados.rados.0.smithi029.stdout:1076: finishing write tid 2 to smithi02979277-49 2024-04-12T11:18:59.788 INFO:tasks.rados.rados.0.smithi029.stdout:1074: done (6 left) 2024-04-12T11:18:59.788 INFO:tasks.rados.rados.0.smithi029.stdout:1075: done (5 left) 2024-04-12T11:18:59.788 INFO:tasks.rados.rados.0.smithi029.stdout:1081: write oid 21 current snap is 137 2024-04-12T11:18:59.788 INFO:tasks.rados.rados.0.smithi029.stdout:1076: finishing write tid 3 to smithi02979277-49 2024-04-12T11:18:59.788 INFO:tasks.rados.rados.0.smithi029.stdout:1081: seq_num 431 ranges {443817=548087,1788092=438139,3019898=223919} 2024-04-12T11:18:59.792 INFO:tasks.rados.rados.0.smithi029.stdout:1081: writing smithi02979277-21 from 443817 to 991904 tid 1 2024-04-12T11:18:59.799 INFO:tasks.rados.rados.0.smithi029.stdout:1081: writing smithi02979277-21 from 1788092 to 2226231 tid 2 2024-04-12T11:18:59.805 INFO:tasks.rados.rados.0.smithi029.stdout:1081: writing smithi02979277-21 from 3019898 to 3243817 tid 3 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1082: snap_create 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1076: finishing write tid 4 to smithi02979277-49 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1076: finishing write tid 5 to smithi02979277-49 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1076: oid 49 updating version 0 to 470 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1076: oid 49 version 470 is already newer than 467 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1076: oid 49 version 470 is already newer than 469 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1076: oid 49 version 470 is already newer than 468 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 470 (ObjNum 429 snap 137 seq_num 429) dirty exists 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1076: left oid 49 (ObjNum 429 snap 137 seq_num 429) 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1077: finishing write tid 1 to smithi02979277-2 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1077: finishing write tid 2 to smithi02979277-2 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1077: finishing write tid 3 to smithi02979277-2 2024-04-12T11:18:59.806 INFO:tasks.rados.rados.0.smithi029.stdout:1077: finishing write tid 4 to smithi02979277-2 2024-04-12T11:18:59.807 INFO:tasks.rados.rados.0.smithi029.stdout:1077: finishing write tid 5 to smithi02979277-2 2024-04-12T11:18:59.807 INFO:tasks.rados.rados.0.smithi029.stdout:1077: oid 2 updating version 0 to 78 2024-04-12T11:18:59.807 INFO:tasks.rados.rados.0.smithi029.stdout:1077: oid 2 updating version 78 to 80 2024-04-12T11:18:59.807 INFO:tasks.rados.rados.0.smithi029.stdout:1077: oid 2 version 80 is already newer than 79 2024-04-12T11:18:59.807 INFO:tasks.rados.rados.0.smithi029.stdout:1077: oid 2 updating version 80 to 81 2024-04-12T11:18:59.807 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 81 (ObjNum 430 snap 137 seq_num 430) dirty exists 2024-04-12T11:18:59.807 INFO:tasks.rados.rados.0.smithi029.stdout:1077: left oid 2 (ObjNum 430 snap 137 seq_num 430) 2024-04-12T11:18:59.807 INFO:tasks.rados.rados.0.smithi029.stdout:1079: expect (ObjNum 418 snap 131 seq_num 418) 2024-04-12T11:18:59.908 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:1080: finishing rollback tid 0 to smithi02979277-4 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:1080: finishing rollback tid 1 to smithi02979277-4 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:1080: finishing rollback tid 2 to smithi02979277-4 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 651 (ObjNum 409 snap 128 seq_num 409) dirty exists 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:1081: finishing write tid 1 to smithi02979277-21 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:1081: finishing write tid 2 to smithi02979277-21 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:1081: finishing write tid 3 to smithi02979277-21 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:1081: finishing write tid 4 to smithi02979277-21 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:1081: finishing write tid 5 to smithi02979277-21 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:1081: oid 21 updating version 0 to 508 2024-04-12T11:19:00.000 INFO:tasks.rados.rados.0.smithi029.stdout:1081: oid 21 version 508 is already newer than 507 2024-04-12T11:19:00.001 INFO:tasks.rados.rados.0.smithi029.stdout:1081: oid 21 version 508 is already newer than 506 2024-04-12T11:19:00.001 INFO:tasks.rados.rados.0.smithi029.stdout:1081: oid 21 version 508 is already newer than 505 2024-04-12T11:19:00.001 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 508 (ObjNum 431 snap 137 seq_num 431) dirty exists 2024-04-12T11:19:00.001 INFO:tasks.rados.rados.0.smithi029.stdout:1081: left oid 21 (ObjNum 431 snap 137 seq_num 431) 2024-04-12T11:19:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:59 smithi029 ceph-mon[93650]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:19:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:18:59 smithi029 ceph-mon[93650]: osdmap e339: 8 total, 8 up, 8 in 2024-04-12T11:19:00.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:59 smithi029 ceph-mon[97787]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:19:00.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:18:59 smithi029 ceph-mon[97787]: osdmap e339: 8 total, 8 up, 8 in 2024-04-12T11:19:00.471 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:59 smithi087 ceph-mon[32917]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:19:00.471 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:18:59 smithi087 ceph-mon[32917]: osdmap e339: 8 total, 8 up, 8 in 2024-04-12T11:19:00.525 INFO:tasks.rados.rados.0.smithi029.stdout:1076: done (6 left) 2024-04-12T11:19:00.525 INFO:tasks.rados.rados.0.smithi029.stdout:1077: done (5 left) 2024-04-12T11:19:00.525 INFO:tasks.rados.rados.0.smithi029.stdout:1078: done (4 left) 2024-04-12T11:19:00.525 INFO:tasks.rados.rados.0.smithi029.stdout:1079: done (3 left) 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1080: done (2 left) 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1081: done (1 left) 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1082: done (0 left) 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1083: rollback oid 17 current snap is 138 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 130 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1084: rollback oid 23 current snap is 138 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 23 to 131 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1085: read oid 24 snap 131 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1085: expect (ObjNum 326 snap 101 seq_num 326) 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1086: read oid 20 snap -1 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1086: expect (ObjNum 416 snap 130 seq_num 416) 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1087: read oid 21 snap -1 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1087: expect (ObjNum 431 snap 137 seq_num 431) 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1088: read oid 34 snap -1 2024-04-12T11:19:00.526 INFO:tasks.rados.rados.0.smithi029.stdout:1088: expect (ObjNum 310 snap 94 seq_num 310) 2024-04-12T11:19:00.527 INFO:tasks.rados.rados.0.smithi029.stdout:1089: read oid 1 snap -1 2024-04-12T11:19:00.527 INFO:tasks.rados.rados.0.smithi029.stdout:1089: expect deleted 2024-04-12T11:19:00.527 INFO:tasks.rados.rados.0.smithi029.stdout:1090: write oid 48 current snap is 138 2024-04-12T11:19:00.527 INFO:tasks.rados.rados.0.smithi029.stdout:1090: seq_num 432 ranges {460624=623336,1846574=698150,3163674=96950} 2024-04-12T11:19:00.528 INFO:tasks.rados.rados.0.smithi029.stdout:1090: writing smithi02979277-48 from 460624 to 1083960 tid 1 2024-04-12T11:19:00.536 INFO:tasks.rados.rados.0.smithi029.stdout:1090: writing smithi02979277-48 from 1846574 to 2544724 tid 2 2024-04-12T11:19:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:1090: writing smithi02979277-48 from 3163674 to 3260624 tid 3 2024-04-12T11:19:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:1091: rollback oid 2 current snap is 138 2024-04-12T11:19:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 2 to 124 2024-04-12T11:19:00.540 INFO:tasks.rados.rados.0.smithi029.stdout:1085: expect (ObjNum 326 snap 101 seq_num 326) 2024-04-12T11:19:00.731 INFO:tasks.rados.rados.0.smithi029.stdout:1083: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:19:00.731 INFO:tasks.rados.rados.0.smithi029.stdout:1084: finishing rollback tid 1 to smithi02979277-23 2024-04-12T11:19:00.731 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 334 (ObjNum 33 snap 0 seq_num 1777354544) dirty dne 2024-04-12T11:19:00.731 INFO:tasks.rados.rados.0.smithi029.stdout:1083: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:19:00.731 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 607 (ObjNum 390 snap 121 seq_num 390) dirty exists 2024-04-12T11:19:00.731 INFO:tasks.rados.rados.0.smithi029.stdout:1087: expect (ObjNum 431 snap 137 seq_num 431) 2024-04-12T11:19:00.841 INFO:tasks.rados.rados.0.smithi029.stdout:1086: expect (ObjNum 416 snap 130 seq_num 416) 2024-04-12T11:19:00.940 INFO:tasks.rados.rados.0.smithi029.stdout:1088: expect (ObjNum 310 snap 94 seq_num 310) 2024-04-12T11:19:01.117 INFO:tasks.rados.rados.0.smithi029.stdout:1090: finishing write tid 1 to smithi02979277-48 2024-04-12T11:19:01.117 INFO:tasks.rados.rados.0.smithi029.stdout:1090: finishing write tid 2 to smithi02979277-48 2024-04-12T11:19:01.117 INFO:tasks.rados.rados.0.smithi029.stdout:1090: finishing write tid 3 to smithi02979277-48 2024-04-12T11:19:01.117 INFO:tasks.rados.rados.0.smithi029.stdout:1090: finishing write tid 4 to smithi02979277-48 2024-04-12T11:19:01.117 INFO:tasks.rados.rados.0.smithi029.stdout:1090: finishing write tid 5 to smithi02979277-48 2024-04-12T11:19:01.117 INFO:tasks.rados.rados.0.smithi029.stdout:1090: oid 48 updating version 0 to 445 2024-04-12T11:19:01.117 INFO:tasks.rados.rados.0.smithi029.stdout:1090: oid 48 version 445 is already newer than 444 2024-04-12T11:19:01.117 INFO:tasks.rados.rados.0.smithi029.stdout:1090: oid 48 updating version 445 to 446 2024-04-12T11:19:01.117 INFO:tasks.rados.rados.0.smithi029.stdout:1090: oid 48 updating version 446 to 447 2024-04-12T11:19:01.117 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 447 (ObjNum 432 snap 138 seq_num 432) dirty exists 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1090: left oid 48 (ObjNum 432 snap 138 seq_num 432) 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1091: finishing rollback tid 0 to smithi02979277-2 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1091: finishing rollback tid 1 to smithi02979277-2 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1091: finishing rollback tid 2 to smithi02979277-2 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 85 (ObjNum 401 snap 123 seq_num 401) dirty exists 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1083: done (8 left) 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1084: done (7 left) 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1085: done (6 left) 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1086: done (5 left) 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1087: done (4 left) 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1088: done (3 left) 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1089: done (2 left) 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1090: done (1 left) 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1091: done (0 left) 2024-04-12T11:19:01.118 INFO:tasks.rados.rados.0.smithi029.stdout:1092: delete oid 10 current snap is 138 2024-04-12T11:19:01.119 INFO:tasks.rados.rados.0.smithi029.stdout:1092: done (0 left) 2024-04-12T11:19:01.119 INFO:tasks.rados.rados.0.smithi029.stdout:1093: rollback oid 7 current snap is 138 2024-04-12T11:19:01.119 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 130 2024-04-12T11:19:01.119 INFO:tasks.rados.rados.0.smithi029.stdout:1094: snap_create 2024-04-12T11:19:01.123 INFO:tasks.rados.rados.0.smithi029.stdout:1093: finishing rollback tid 0 to smithi02979277-7 2024-04-12T11:19:01.123 INFO:tasks.rados.rados.0.smithi029.stdout:1093: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:19:01.123 INFO:tasks.rados.rados.0.smithi029.stdout:1093: finishing rollback tid 2 to smithi02979277-7 2024-04-12T11:19:01.123 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 553 (ObjNum 394 snap 123 seq_num 394) dirty exists 2024-04-12T11:19:01.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:00 smithi029 ceph-mon[97787]: pgmap v142: 89 pgs: 89 active+clean; 218 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 339 KiB/s wr, 10 op/s 2024-04-12T11:19:01.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:00 smithi029 ceph-mon[97787]: osdmap e340: 8 total, 8 up, 8 in 2024-04-12T11:19:01.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:00 smithi029 ceph-mon[93650]: pgmap v142: 89 pgs: 89 active+clean; 218 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 339 KiB/s wr, 10 op/s 2024-04-12T11:19:01.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:00 smithi029 ceph-mon[93650]: osdmap e340: 8 total, 8 up, 8 in 2024-04-12T11:19:01.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:00 smithi087 ceph-mon[32917]: pgmap v142: 89 pgs: 89 active+clean; 218 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 339 KiB/s wr, 10 op/s 2024-04-12T11:19:01.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:00 smithi087 ceph-mon[32917]: osdmap e340: 8 total, 8 up, 8 in 2024-04-12T11:19:01.997 INFO:tasks.rados.rados.0.smithi029.stdout:1093: done (1 left) 2024-04-12T11:19:01.998 INFO:tasks.rados.rados.0.smithi029.stdout:1094: done (0 left) 2024-04-12T11:19:01.998 INFO:tasks.rados.rados.0.smithi029.stdout:1095: rollback oid 11 current snap is 139 2024-04-12T11:19:01.998 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 11 to 107 2024-04-12T11:19:01.998 INFO:tasks.rados.rados.0.smithi029.stdout:1096: snap_remove snap 131 2024-04-12T11:19:02.621 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:19:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:03.004 INFO:tasks.rados.rados.0.smithi029.stdout:1096: done (1 left) 2024-04-12T11:19:03.004 INFO:tasks.rados.rados.0.smithi029.stdout:1097: write oid 26 current snap is 139 2024-04-12T11:19:03.004 INFO:tasks.rados.rados.0.smithi029.stdout:1095: finishing rollback tid 1 to smithi02979277-11 2024-04-12T11:19:03.004 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 674 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-04-12T11:19:03.004 INFO:tasks.rados.rados.0.smithi029.stdout:1097: seq_num 434 ranges {494238=490187,1563538=701819,2818520=475718} 2024-04-12T11:19:03.010 INFO:tasks.rados.rados.0.smithi029.stdout:1097: writing smithi02979277-26 from 494238 to 984425 tid 1 2024-04-12T11:19:03.018 INFO:tasks.rados.rados.0.smithi029.stdout:1097: writing smithi02979277-26 from 1563538 to 2265357 tid 2 2024-04-12T11:19:03.024 INFO:tasks.rados.rados.0.smithi029.stdout:1097: writing smithi02979277-26 from 2818520 to 3294238 tid 3 2024-04-12T11:19:03.024 INFO:tasks.rados.rados.0.smithi029.stdout:1095: done (1 left) 2024-04-12T11:19:03.024 INFO:tasks.rados.rados.0.smithi029.stdout:1098: read oid 30 snap -1 2024-04-12T11:19:03.024 INFO:tasks.rados.rados.0.smithi029.stdout:1098: expect deleted 2024-04-12T11:19:03.024 INFO:tasks.rados.rados.0.smithi029.stdout:1099: write oid 10 current snap is 139 2024-04-12T11:19:03.025 INFO:tasks.rados.rados.0.smithi029.stdout:1097: finishing write tid 1 to smithi02979277-26 2024-04-12T11:19:03.025 INFO:tasks.rados.rados.0.smithi029.stdout:1099: seq_num 435 ranges {511045=565436,1622020=445477,2729590=556855} 2024-04-12T11:19:03.030 INFO:tasks.rados.rados.0.smithi029.stdout:1099: writing smithi02979277-10 from 511045 to 1076481 tid 1 2024-04-12T11:19:03.036 INFO:tasks.rados.rados.0.smithi029.stdout:1099: writing smithi02979277-10 from 1622020 to 2067497 tid 2 2024-04-12T11:19:03.044 INFO:tasks.rados.rados.0.smithi029.stdout:1099: writing smithi02979277-10 from 2729590 to 3286445 tid 3 2024-04-12T11:19:03.044 INFO:tasks.rados.rados.0.smithi029.stdout:1100: write oid 5 current snap is 139 2024-04-12T11:19:03.044 INFO:tasks.rados.rados.0.smithi029.stdout:1097: finishing write tid 2 to smithi02979277-26 2024-04-12T11:19:03.044 INFO:tasks.rados.rados.0.smithi029.stdout:1100: seq_num 436 ranges {527852=640685,1680502=589135,2757013=570839} 2024-04-12T11:19:03.050 INFO:tasks.rados.rados.0.smithi029.stdout:1100: writing smithi02979277-5 from 527852 to 1168537 tid 1 2024-04-12T11:19:03.056 INFO:tasks.rados.rados.0.smithi029.stdout:1100: writing smithi02979277-5 from 1680502 to 2269637 tid 2 2024-04-12T11:19:03.063 INFO:tasks.rados.rados.0.smithi029.stdout:1100: writing smithi02979277-5 from 2757013 to 3327852 tid 3 2024-04-12T11:19:03.063 INFO:tasks.rados.rados.0.smithi029.stdout:1101: read oid 7 snap -1 2024-04-12T11:19:03.063 INFO:tasks.rados.rados.0.smithi029.stdout:1101: expect (ObjNum 394 snap 123 seq_num 394) 2024-04-12T11:19:03.063 INFO:tasks.rados.rados.0.smithi029.stdout:1102: rollback oid 24 current snap is 139 2024-04-12T11:19:03.063 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 24 to 124 2024-04-12T11:19:03.063 INFO:tasks.rados.rados.0.smithi029.stdout:1097: finishing write tid 3 to smithi02979277-26 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1097: finishing write tid 4 to smithi02979277-26 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1097: finishing write tid 5 to smithi02979277-26 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1097: oid 26 updating version 0 to 686 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1097: oid 26 updating version 686 to 687 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1097: oid 26 version 687 is already newer than 680 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1097: oid 26 version 687 is already newer than 685 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 687 (ObjNum 434 snap 139 seq_num 434) dirty exists 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1097: left oid 26 (ObjNum 434 snap 139 seq_num 434) 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1097: done (5 left) 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1098: done (4 left) 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1103: delete oid 14 current snap is 139 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1099: finishing write tid 1 to smithi02979277-10 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1099: finishing write tid 2 to smithi02979277-10 2024-04-12T11:19:03.064 INFO:tasks.rados.rados.0.smithi029.stdout:1099: finishing write tid 3 to smithi02979277-10 2024-04-12T11:19:03.065 INFO:tasks.rados.rados.0.smithi029.stdout:1099: finishing write tid 4 to smithi02979277-10 2024-04-12T11:19:03.065 INFO:tasks.rados.rados.0.smithi029.stdout:1099: finishing write tid 5 to smithi02979277-10 2024-04-12T11:19:03.065 INFO:tasks.rados.rados.0.smithi029.stdout:1099: oid 10 updating version 0 to 455 2024-04-12T11:19:03.065 INFO:tasks.rados.rados.0.smithi029.stdout:1099: oid 10 version 455 is already newer than 452 2024-04-12T11:19:03.065 INFO:tasks.rados.rados.0.smithi029.stdout:1099: oid 10 version 455 is already newer than 453 2024-04-12T11:19:03.065 INFO:tasks.rados.rados.0.smithi029.stdout:1099: oid 10 version 455 is already newer than 454 2024-04-12T11:19:03.065 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 455 (ObjNum 435 snap 139 seq_num 435) dirty exists 2024-04-12T11:19:03.065 INFO:tasks.rados.rados.0.smithi029.stdout:1099: left oid 10 (ObjNum 435 snap 139 seq_num 435) 2024-04-12T11:19:03.065 INFO:tasks.rados.rados.0.smithi029.stdout:1100: finishing write tid 1 to smithi02979277-5 2024-04-12T11:19:03.065 INFO:tasks.rados.rados.0.smithi029.stdout:1100: finishing write tid 2 to smithi02979277-5 2024-04-12T11:19:03.067 INFO:tasks.rados.rados.0.smithi029.stdout:1102: finishing rollback tid 0 to smithi02979277-24 2024-04-12T11:19:03.067 INFO:tasks.rados.rados.0.smithi029.stdout:1102: finishing rollback tid 1 to smithi02979277-24 2024-04-12T11:19:03.067 INFO:tasks.rados.rados.0.smithi029.stdout:1102: finishing rollback tid 2 to smithi02979277-24 2024-04-12T11:19:03.067 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 691 (ObjNum 326 snap 101 seq_num 326) dirty exists 2024-04-12T11:19:03.068 INFO:tasks.rados.rados.0.smithi029.stdout:1100: finishing write tid 3 to smithi02979277-5 2024-04-12T11:19:03.068 INFO:tasks.rados.rados.0.smithi029.stdout:1100: finishing write tid 4 to smithi02979277-5 2024-04-12T11:19:03.068 INFO:tasks.rados.rados.0.smithi029.stdout:1100: finishing write tid 5 to smithi02979277-5 2024-04-12T11:19:03.068 INFO:tasks.rados.rados.0.smithi029.stdout:1100: oid 5 updating version 0 to 656 2024-04-12T11:19:03.068 INFO:tasks.rados.rados.0.smithi029.stdout:1100: oid 5 version 656 is already newer than 655 2024-04-12T11:19:03.068 INFO:tasks.rados.rados.0.smithi029.stdout:1100: oid 5 updating version 656 to 657 2024-04-12T11:19:03.068 INFO:tasks.rados.rados.0.smithi029.stdout:1100: oid 5 version 657 is already newer than 654 2024-04-12T11:19:03.068 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 657 (ObjNum 436 snap 139 seq_num 436) dirty exists 2024-04-12T11:19:03.069 INFO:tasks.rados.rados.0.smithi029.stdout:1100: left oid 5 (ObjNum 436 snap 139 seq_num 436) 2024-04-12T11:19:03.069 INFO:tasks.rados.rados.0.smithi029.stdout:1099: done (4 left) 2024-04-12T11:19:03.069 INFO:tasks.rados.rados.0.smithi029.stdout:1100: done (3 left) 2024-04-12T11:19:03.069 INFO:tasks.rados.rados.0.smithi029.stdout:1102: done (2 left) 2024-04-12T11:19:03.069 INFO:tasks.rados.rados.0.smithi029.stdout:1103: done (1 left) 2024-04-12T11:19:03.069 INFO:tasks.rados.rados.0.smithi029.stdout:1104: write oid 15 current snap is 139 2024-04-12T11:19:03.069 INFO:tasks.rados.rados.0.smithi029.stdout:1104: seq_num 438 ranges {561466=791183,2081113=592804,3095506=265960} 2024-04-12T11:19:03.075 INFO:tasks.rados.rados.0.smithi029.stdout:1104: writing smithi02979277-15 from 561466 to 1352649 tid 1 2024-04-12T11:19:03.082 INFO:tasks.rados.rados.0.smithi029.stdout:1104: writing smithi02979277-15 from 2081113 to 2673917 tid 2 2024-04-12T11:19:03.086 INFO:tasks.rados.rados.0.smithi029.stdout:1104: writing smithi02979277-15 from 3095506 to 3361466 tid 3 2024-04-12T11:19:03.086 INFO:tasks.rados.rados.0.smithi029.stdout:1105: write oid 44 current snap is 139 2024-04-12T11:19:03.086 INFO:tasks.rados.rados.0.smithi029.stdout:1101: expect (ObjNum 394 snap 123 seq_num 394) 2024-04-12T11:19:03.173 INFO:tasks.rados.rados.0.smithi029.stdout:1104: finishing write tid 1 to smithi02979277-15 2024-04-12T11:19:03.173 INFO:tasks.rados.rados.0.smithi029.stdout:1104: finishing write tid 2 to smithi02979277-15 2024-04-12T11:19:03.173 INFO:tasks.rados.rados.0.smithi029.stdout:1104: finishing write tid 3 to smithi02979277-15 2024-04-12T11:19:03.174 INFO:tasks.rados.rados.0.smithi029.stdout:1104: finishing write tid 4 to smithi02979277-15 2024-04-12T11:19:03.174 INFO:tasks.rados.rados.0.smithi029.stdout:1104: finishing write tid 5 to smithi02979277-15 2024-04-12T11:19:03.174 INFO:tasks.rados.rados.0.smithi029.stdout:1104: oid 15 updating version 0 to 485 2024-04-12T11:19:03.174 INFO:tasks.rados.rados.0.smithi029.stdout:1104: oid 15 updating version 485 to 486 2024-04-12T11:19:03.174 INFO:tasks.rados.rados.0.smithi029.stdout:1104: oid 15 version 486 is already newer than 484 2024-04-12T11:19:03.174 INFO:tasks.rados.rados.0.smithi029.stdout:1104: oid 15 updating version 486 to 487 2024-04-12T11:19:03.174 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 487 (ObjNum 438 snap 139 seq_num 438) dirty exists 2024-04-12T11:19:03.174 INFO:tasks.rados.rados.0.smithi029.stdout:1104: left oid 15 (ObjNum 438 snap 139 seq_num 438) 2024-04-12T11:19:03.174 INFO:tasks.rados.rados.0.smithi029.stdout:1105: seq_num 439 ranges {578273=466432,1739595=452815,2722929=655344} 2024-04-12T11:19:03.179 INFO:tasks.rados.rados.0.smithi029.stdout:1105: writing smithi02979277-44 from 578273 to 1044705 tid 1 2024-04-12T11:19:03.186 INFO:tasks.rados.rados.0.smithi029.stdout:1105: writing smithi02979277-44 from 1739595 to 2192410 tid 2 2024-04-12T11:19:03.194 INFO:tasks.rados.rados.0.smithi029.stdout:1105: writing smithi02979277-44 from 2722929 to 3378273 tid 3 2024-04-12T11:19:03.194 INFO:tasks.rados.rados.0.smithi029.stdout:1101: done (2 left) 2024-04-12T11:19:03.194 INFO:tasks.rados.rados.0.smithi029.stdout:1104: done (1 left) 2024-04-12T11:19:03.194 INFO:tasks.rados.rados.0.smithi029.stdout:1106: snap_create 2024-04-12T11:19:03.194 INFO:tasks.rados.rados.0.smithi029.stdout:1105: finishing write tid 1 to smithi02979277-44 2024-04-12T11:19:03.194 INFO:tasks.rados.rados.0.smithi029.stdout:1105: finishing write tid 2 to smithi02979277-44 2024-04-12T11:19:03.197 INFO:tasks.rados.rados.0.smithi029.stdout:1105: finishing write tid 3 to smithi02979277-44 2024-04-12T11:19:03.197 INFO:tasks.rados.rados.0.smithi029.stdout:1105: finishing write tid 4 to smithi02979277-44 2024-04-12T11:19:03.197 INFO:tasks.rados.rados.0.smithi029.stdout:1105: finishing write tid 5 to smithi02979277-44 2024-04-12T11:19:03.197 INFO:tasks.rados.rados.0.smithi029.stdout:1105: oid 44 updating version 0 to 513 2024-04-12T11:19:03.197 INFO:tasks.rados.rados.0.smithi029.stdout:1105: oid 44 version 513 is already newer than 511 2024-04-12T11:19:03.197 INFO:tasks.rados.rados.0.smithi029.stdout:1105: oid 44 version 513 is already newer than 510 2024-04-12T11:19:03.198 INFO:tasks.rados.rados.0.smithi029.stdout:1105: oid 44 version 513 is already newer than 512 2024-04-12T11:19:03.198 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 513 (ObjNum 439 snap 139 seq_num 439) dirty exists 2024-04-12T11:19:03.198 INFO:tasks.rados.rados.0.smithi029.stdout:1105: left oid 44 (ObjNum 439 snap 139 seq_num 439) 2024-04-12T11:19:03.250 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:02 smithi029 ceph-mon[93650]: pgmap v145: 89 pgs: 89 active+clean; 218 MiB data, 689 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:19:03.250 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:02 smithi029 ceph-mon[93650]: osdmap e341: 8 total, 8 up, 8 in 2024-04-12T11:19:03.250 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:02 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:03.251 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:02 smithi029 ceph-mon[97787]: pgmap v145: 89 pgs: 89 active+clean; 218 MiB data, 689 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:19:03.251 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:02 smithi029 ceph-mon[97787]: osdmap e341: 8 total, 8 up, 8 in 2024-04-12T11:19:03.251 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:02 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:03.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:02 smithi087 ceph-mon[32917]: pgmap v145: 89 pgs: 89 active+clean; 218 MiB data, 689 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:19:03.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:02 smithi087 ceph-mon[32917]: osdmap e341: 8 total, 8 up, 8 in 2024-04-12T11:19:03.409 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:02 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:03.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:03.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:03.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'sleep 60' 2024-04-12T11:19:04.006 INFO:tasks.rados.rados.0.smithi029.stdout:1105: done (1 left) 2024-04-12T11:19:04.006 INFO:tasks.rados.rados.0.smithi029.stdout:1106: done (0 left) 2024-04-12T11:19:04.006 INFO:tasks.rados.rados.0.smithi029.stdout:1107: read oid 45 snap -1 2024-04-12T11:19:04.006 INFO:tasks.rados.rados.0.smithi029.stdout:1107: expect (ObjNum 395 snap 123 seq_num 395) 2024-04-12T11:19:04.006 INFO:tasks.rados.rados.0.smithi029.stdout:1108: snap_create 2024-04-12T11:19:04.011 INFO:tasks.rados.rados.0.smithi029.stdout:1107: expect (ObjNum 395 snap 123 seq_num 395) 2024-04-12T11:19:04.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:03 smithi029 ceph-mon[93650]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:19:04.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:03 smithi029 ceph-mon[93650]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:19:04.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:03 smithi029 ceph-mon[93650]: osdmap e342: 8 total, 8 up, 8 in 2024-04-12T11:19:04.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:03 smithi029 ceph-mon[93650]: pgmap v148: 89 pgs: 89 active+clean; 241 MiB data, 725 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-04-12T11:19:04.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:03 smithi029 ceph-mon[97787]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:19:04.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:03 smithi029 ceph-mon[97787]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:19:04.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:03 smithi029 ceph-mon[97787]: osdmap e342: 8 total, 8 up, 8 in 2024-04-12T11:19:04.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:03 smithi029 ceph-mon[97787]: pgmap v148: 89 pgs: 89 active+clean; 241 MiB data, 725 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-04-12T11:19:04.362 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:19:04.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:03 smithi087 ceph-mon[32917]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:19:04.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:03 smithi087 ceph-mon[32917]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:19:04.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:03 smithi087 ceph-mon[32917]: osdmap e342: 8 total, 8 up, 8 in 2024-04-12T11:19:04.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:03 smithi087 ceph-mon[32917]: pgmap v148: 89 pgs: 89 active+clean; 241 MiB data, 725 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-04-12T11:19:05.008 INFO:tasks.rados.rados.0.smithi029.stdout:1107: done (1 left) 2024-04-12T11:19:05.008 INFO:tasks.rados.rados.0.smithi029.stdout:1108: done (0 left) 2024-04-12T11:19:05.008 INFO:tasks.rados.rados.0.smithi029.stdout:1109: read oid 37 snap -1 2024-04-12T11:19:05.009 INFO:tasks.rados.rados.0.smithi029.stdout:1109: expect deleted 2024-04-12T11:19:05.009 INFO:tasks.rados.rados.0.smithi029.stdout:1110: rollback oid 38 current snap is 141 2024-04-12T11:19:05.009 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 38 to 138 2024-04-12T11:19:05.009 INFO:tasks.rados.rados.0.smithi029.stdout:1111: write oid 1 current snap is 141 2024-04-12T11:19:05.009 INFO:tasks.rados.rados.0.smithi029.stdout:1111: seq_num 440 ranges {595080=541681,1798077=596473,3150352=244728} 2024-04-12T11:19:05.014 INFO:tasks.rados.rados.0.smithi029.stdout:1111: writing smithi02979277-1 from 595080 to 1136761 tid 1 2024-04-12T11:19:05.017 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:19:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:19:04] "GET /metrics HTTP/1.1" 200 36243 "" "Prometheus/2.43.0" 2024-04-12T11:19:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[97787]: osdmap e343: 8 total, 8 up, 8 in 2024-04-12T11:19:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:19:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:19:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:19:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:19:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:19:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[93650]: osdmap e343: 8 total, 8 up, 8 in 2024-04-12T11:19:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:19:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:19:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:19:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:19:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:19:05.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.021 INFO:tasks.rados.rados.0.smithi029.stdout:1111: writing smithi02979277-1 from 1798077 to 2394550 tid 2 2024-04-12T11:19:05.027 INFO:tasks.rados.rados.0.smithi029.stdout:1111: writing smithi02979277-1 from 3150352 to 3395080 tid 3 2024-04-12T11:19:05.027 INFO:tasks.rados.rados.0.smithi029.stdout:1112: delete oid 7 current snap is 141 2024-04-12T11:19:05.027 INFO:tasks.rados.rados.0.smithi029.stdout:1110: finishing rollback tid 0 to smithi02979277-38 2024-04-12T11:19:05.028 INFO:tasks.rados.rados.0.smithi029.stdout:1110: finishing rollback tid 1 to smithi02979277-38 2024-04-12T11:19:05.028 INFO:tasks.rados.rados.0.smithi029.stdout:1110: finishing rollback tid 2 to smithi02979277-38 2024-04-12T11:19:05.028 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 262 (ObjNum 421 snap 132 seq_num 421) dirty exists 2024-04-12T11:19:05.028 INFO:tasks.rados.rados.0.smithi029.stdout:1111: finishing write tid 1 to smithi02979277-1 2024-04-12T11:19:05.028 INFO:tasks.rados.rados.0.smithi029.stdout:1111: finishing write tid 2 to smithi02979277-1 2024-04-12T11:19:05.029 INFO:tasks.rados.rados.0.smithi029.stdout:1109: done (3 left) 2024-04-12T11:19:05.029 INFO:tasks.rados.rados.0.smithi029.stdout:1110: done (2 left) 2024-04-12T11:19:05.029 INFO:tasks.rados.rados.0.smithi029.stdout:1112: done (1 left) 2024-04-12T11:19:05.030 INFO:tasks.rados.rados.0.smithi029.stdout:1113: rollback oid 48 current snap is 141 2024-04-12T11:19:05.030 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 107 2024-04-12T11:19:05.030 INFO:tasks.rados.rados.0.smithi029.stdout:1114: write oid 5 current snap is 141 2024-04-12T11:19:05.030 INFO:tasks.rados.rados.0.smithi029.stdout:1114: seq_num 442 ranges {628694=408532,1515041=600142,2805198=552112} 2024-04-12T11:19:05.035 INFO:tasks.rados.rados.0.smithi029.stdout:1114: writing smithi02979277-5 from 628694 to 1037226 tid 1 2024-04-12T11:19:05.041 INFO:tasks.rados.rados.0.smithi029.stdout:1114: writing smithi02979277-5 from 1515041 to 2115183 tid 2 2024-04-12T11:19:05.048 INFO:tasks.rados.rados.0.smithi029.stdout:1114: writing smithi02979277-5 from 2805198 to 3357310 tid 3 2024-04-12T11:19:05.048 INFO:tasks.rados.rados.0.smithi029.stdout:1115: write oid 29 current snap is 141 2024-04-12T11:19:05.048 INFO:tasks.rados.rados.0.smithi029.stdout:1111: finishing write tid 3 to smithi02979277-1 2024-04-12T11:19:05.048 INFO:tasks.rados.rados.0.smithi029.stdout:1115: seq_num 443 ranges {645501=483781,1573523=460153,2548974=763384} 2024-04-12T11:19:05.054 INFO:tasks.rados.rados.0.smithi029.stdout:1115: writing smithi02979277-29 from 645501 to 1129282 tid 1 2024-04-12T11:19:05.059 INFO:tasks.rados.rados.0.smithi029.stdout:1115: writing smithi02979277-29 from 1573523 to 2033676 tid 2 2024-04-12T11:19:05.067 INFO:tasks.rados.rados.0.smithi029.stdout:1115: writing smithi02979277-29 from 2548974 to 3312358 tid 3 2024-04-12T11:19:05.067 INFO:tasks.rados.rados.0.smithi029.stdout:1116: read oid 15 snap -1 2024-04-12T11:19:05.067 INFO:tasks.rados.rados.0.smithi029.stdout:1116: expect (ObjNum 438 snap 139 seq_num 438) 2024-04-12T11:19:05.067 INFO:tasks.rados.rados.0.smithi029.stdout:1117: read oid 16 snap -1 2024-04-12T11:19:05.067 INFO:tasks.rados.rados.0.smithi029.stdout:1117: expect (ObjNum 418 snap 131 seq_num 418) 2024-04-12T11:19:05.067 INFO:tasks.rados.rados.0.smithi029.stdout:1118: snap_create 2024-04-12T11:19:05.067 INFO:tasks.rados.rados.0.smithi029.stdout:1111: finishing write tid 4 to smithi02979277-1 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1111: finishing write tid 5 to smithi02979277-1 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1111: oid 1 updating version 0 to 514 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1111: oid 1 updating version 514 to 517 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1111: oid 1 version 517 is already newer than 516 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1111: oid 1 version 517 is already newer than 515 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 517 (ObjNum 440 snap 141 seq_num 440) dirty exists 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1111: left oid 1 (ObjNum 440 snap 141 seq_num 440) 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1113: finishing rollback tid 0 to smithi02979277-48 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1113: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1113: finishing rollback tid 2 to smithi02979277-48 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 459 (ObjNum 341 snap 105 seq_num 341) dirty exists 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1114: finishing write tid 1 to smithi02979277-5 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1114: finishing write tid 2 to smithi02979277-5 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1114: finishing write tid 3 to smithi02979277-5 2024-04-12T11:19:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:1114: finishing write tid 4 to smithi02979277-5 2024-04-12T11:19:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:1114: finishing write tid 5 to smithi02979277-5 2024-04-12T11:19:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:1114: oid 5 updating version 0 to 663 2024-04-12T11:19:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:1114: oid 5 updating version 663 to 664 2024-04-12T11:19:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:1114: oid 5 version 664 is already newer than 662 2024-04-12T11:19:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:1114: oid 5 version 664 is already newer than 661 2024-04-12T11:19:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 664 (ObjNum 442 snap 141 seq_num 442) dirty exists 2024-04-12T11:19:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:1114: left oid 5 (ObjNum 442 snap 141 seq_num 442) 2024-04-12T11:19:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:1115: finishing write tid 1 to smithi02979277-29 2024-04-12T11:19:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:1115: finishing write tid 2 to smithi02979277-29 2024-04-12T11:19:05.070 INFO:tasks.rados.rados.0.smithi029.stdout:1115: finishing write tid 3 to smithi02979277-29 2024-04-12T11:19:05.071 INFO:tasks.rados.rados.0.smithi029.stdout:1115: finishing write tid 4 to smithi02979277-29 2024-04-12T11:19:05.072 INFO:tasks.rados.rados.0.smithi029.stdout:1115: finishing write tid 5 to smithi02979277-29 2024-04-12T11:19:05.072 INFO:tasks.rados.rados.0.smithi029.stdout:1115: oid 29 updating version 0 to 401 2024-04-12T11:19:05.072 INFO:tasks.rados.rados.0.smithi029.stdout:1115: oid 29 updating version 401 to 404 2024-04-12T11:19:05.072 INFO:tasks.rados.rados.0.smithi029.stdout:1115: oid 29 version 404 is already newer than 402 2024-04-12T11:19:05.072 INFO:tasks.rados.rados.0.smithi029.stdout:1115: oid 29 version 404 is already newer than 403 2024-04-12T11:19:05.072 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 404 (ObjNum 443 snap 141 seq_num 443) dirty exists 2024-04-12T11:19:05.072 INFO:tasks.rados.rados.0.smithi029.stdout:1115: left oid 29 (ObjNum 443 snap 141 seq_num 443) 2024-04-12T11:19:05.072 INFO:tasks.rados.rados.0.smithi029.stdout:1116: expect (ObjNum 438 snap 139 seq_num 438) 2024-04-12T11:19:05.274 INFO:tasks.rados.rados.0.smithi029.stdout:1117: expect (ObjNum 418 snap 131 seq_num 418) 2024-04-12T11:19:05.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:05 smithi087 ceph-mon[32917]: osdmap e343: 8 total, 8 up, 8 in 2024-04-12T11:19:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:05 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:05 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:05 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:19:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:05 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:19:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:05 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:05 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:19:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:05 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:19:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:05 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:19:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:05 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:19:06.017 INFO:tasks.rados.rados.0.smithi029.stdout:1111: done (6 left) 2024-04-12T11:19:06.018 INFO:tasks.rados.rados.0.smithi029.stdout:1113: done (5 left) 2024-04-12T11:19:06.018 INFO:tasks.rados.rados.0.smithi029.stdout:1114: done (4 left) 2024-04-12T11:19:06.018 INFO:tasks.rados.rados.0.smithi029.stdout:1115: done (3 left) 2024-04-12T11:19:06.018 INFO:tasks.rados.rados.0.smithi029.stdout:1116: done (2 left) 2024-04-12T11:19:06.018 INFO:tasks.rados.rados.0.smithi029.stdout:1117: done (1 left) 2024-04-12T11:19:06.018 INFO:tasks.rados.rados.0.smithi029.stdout:1118: done (0 left) 2024-04-12T11:19:06.018 INFO:tasks.rados.rados.0.smithi029.stdout:1119: rollback oid 34 current snap is 142 2024-04-12T11:19:06.018 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 141 2024-04-12T11:19:06.018 INFO:tasks.rados.rados.0.smithi029.stdout:1120: snap_create 2024-04-12T11:19:06.021 INFO:tasks.rados.rados.0.smithi029.stdout:1119: finishing rollback tid 0 to smithi02979277-34 2024-04-12T11:19:06.021 INFO:tasks.rados.rados.0.smithi029.stdout:1119: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:19:06.021 INFO:tasks.rados.rados.0.smithi029.stdout:1119: finishing rollback tid 2 to smithi02979277-34 2024-04-12T11:19:06.021 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 408 (ObjNum 310 snap 94 seq_num 310) dirty exists 2024-04-12T11:19:06.219 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:06 smithi029 ceph-mon[97787]: osdmap e344: 8 total, 8 up, 8 in 2024-04-12T11:19:06.219 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:06 smithi029 ceph-mon[97787]: pgmap v151: 89 pgs: 89 active+clean; 241 MiB data, 725 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-04-12T11:19:06.219 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:06 smithi029 ceph-mon[93650]: osdmap e344: 8 total, 8 up, 8 in 2024-04-12T11:19:06.220 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:06 smithi029 ceph-mon[93650]: pgmap v151: 89 pgs: 89 active+clean; 241 MiB data, 725 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-04-12T11:19:06.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:06 smithi087 ceph-mon[32917]: osdmap e344: 8 total, 8 up, 8 in 2024-04-12T11:19:06.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:06 smithi087 ceph-mon[32917]: pgmap v151: 89 pgs: 89 active+clean; 241 MiB data, 725 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-04-12T11:19:07.019 INFO:tasks.rados.rados.0.smithi029.stdout:1119: done (1 left) 2024-04-12T11:19:07.019 INFO:tasks.rados.rados.0.smithi029.stdout:1120: done (0 left) 2024-04-12T11:19:07.019 INFO:tasks.rados.rados.0.smithi029.stdout:1121: snap_create 2024-04-12T11:19:07.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:07 smithi087 ceph-mon[32917]: osdmap e345: 8 total, 8 up, 8 in 2024-04-12T11:19:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:07 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:07.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:07 smithi029 ceph-mon[97787]: osdmap e345: 8 total, 8 up, 8 in 2024-04-12T11:19:07.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:07.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:07 smithi029 ceph-mon[93650]: osdmap e345: 8 total, 8 up, 8 in 2024-04-12T11:19:07.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:07.519 INFO:tasks.rados.rados.0.smithi029.stdout:1121: done (0 left) 2024-04-12T11:19:07.519 INFO:tasks.rados.rados.0.smithi029.stdout:1122: write oid 16 current snap is 144 2024-04-12T11:19:07.520 INFO:tasks.rados.rados.0.smithi029.stdout:1122: seq_num 444 ranges {662308=559030,1632005=603811,2860044=602264} 2024-04-12T11:19:07.526 INFO:tasks.rados.rados.0.smithi029.stdout:1122: writing smithi02979277-16 from 662308 to 1221338 tid 1 2024-04-12T11:19:07.532 INFO:tasks.rados.rados.0.smithi029.stdout:1122: writing smithi02979277-16 from 1632005 to 2235816 tid 2 2024-04-12T11:19:07.539 INFO:tasks.rados.rados.0.smithi029.stdout:1122: writing smithi02979277-16 from 2860044 to 3462308 tid 3 2024-04-12T11:19:07.539 INFO:tasks.rados.rados.0.smithi029.stdout:1123: read oid 3 snap -1 2024-04-12T11:19:07.539 INFO:tasks.rados.rados.0.smithi029.stdout:1123: expect deleted 2024-04-12T11:19:07.539 INFO:tasks.rados.rados.0.smithi029.stdout:1122: finishing write tid 1 to smithi02979277-16 2024-04-12T11:19:07.539 INFO:tasks.rados.rados.0.smithi029.stdout:1122: finishing write tid 2 to smithi02979277-16 2024-04-12T11:19:07.539 INFO:tasks.rados.rados.0.smithi029.stdout:1124: snap_remove snap 138 2024-04-12T11:19:08.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:08 smithi087 ceph-mon[32917]: osdmap e346: 8 total, 8 up, 8 in 2024-04-12T11:19:08.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:08 smithi087 ceph-mon[32917]: pgmap v154: 89 pgs: 89 active+clean; 264 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-04-12T11:19:08.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:08 smithi087 ceph-mon[32917]: osdmap e347: 8 total, 8 up, 8 in 2024-04-12T11:19:08.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:08 smithi029 ceph-mon[93650]: osdmap e346: 8 total, 8 up, 8 in 2024-04-12T11:19:08.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:08 smithi029 ceph-mon[93650]: pgmap v154: 89 pgs: 89 active+clean; 264 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-04-12T11:19:08.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:08 smithi029 ceph-mon[93650]: osdmap e347: 8 total, 8 up, 8 in 2024-04-12T11:19:08.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:08 smithi029 ceph-mon[97787]: osdmap e346: 8 total, 8 up, 8 in 2024-04-12T11:19:08.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:08 smithi029 ceph-mon[97787]: pgmap v154: 89 pgs: 89 active+clean; 264 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-04-12T11:19:08.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:08 smithi029 ceph-mon[97787]: osdmap e347: 8 total, 8 up, 8 in 2024-04-12T11:19:08.523 INFO:tasks.rados.rados.0.smithi029.stdout:1122: finishing write tid 3 to smithi02979277-16 2024-04-12T11:19:08.523 INFO:tasks.rados.rados.0.smithi029.stdout:1122: finishing write tid 4 to smithi02979277-16 2024-04-12T11:19:08.523 INFO:tasks.rados.rados.0.smithi029.stdout:1123: done (2 left) 2024-04-12T11:19:08.523 INFO:tasks.rados.rados.0.smithi029.stdout:1124: done (1 left) 2024-04-12T11:19:08.523 INFO:tasks.rados.rados.0.smithi029.stdout:1125: read oid 11 snap -1 2024-04-12T11:19:08.523 INFO:tasks.rados.rados.0.smithi029.stdout:1125: expect deleted 2024-04-12T11:19:08.523 INFO:tasks.rados.rados.0.smithi029.stdout:1126: delete oid 23 current snap is 144 2024-04-12T11:19:08.523 INFO:tasks.rados.rados.0.smithi029.stdout:1122: finishing write tid 5 to smithi02979277-16 2024-04-12T11:19:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:1122: oid 16 updating version 0 to 668 2024-04-12T11:19:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:1122: oid 16 updating version 668 to 669 2024-04-12T11:19:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:1122: oid 16 version 669 is already newer than 667 2024-04-12T11:19:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:1122: oid 16 version 669 is already newer than 666 2024-04-12T11:19:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 669 (ObjNum 444 snap 144 seq_num 444) dirty exists 2024-04-12T11:19:08.524 INFO:tasks.rados.rados.0.smithi029.stdout:1122: left oid 16 (ObjNum 444 snap 144 seq_num 444) 2024-04-12T11:19:08.530 INFO:tasks.rados.rados.0.smithi029.stdout:1122: done (2 left) 2024-04-12T11:19:08.530 INFO:tasks.rados.rados.0.smithi029.stdout:1125: done (1 left) 2024-04-12T11:19:08.530 INFO:tasks.rados.rados.0.smithi029.stdout:1126: done (0 left) 2024-04-12T11:19:08.530 INFO:tasks.rados.rados.0.smithi029.stdout:1127: snap_create 2024-04-12T11:19:09.526 INFO:tasks.rados.rados.0.smithi029.stdout:1127: done (0 left) 2024-04-12T11:19:09.527 INFO:tasks.rados.rados.0.smithi029.stdout:1128: delete oid 24 current snap is 145 2024-04-12T11:19:09.534 INFO:tasks.rados.rados.0.smithi029.stdout:1128: done (0 left) 2024-04-12T11:19:09.535 INFO:tasks.rados.rados.0.smithi029.stdout:1129: read oid 43 snap 100 2024-04-12T11:19:09.535 INFO:tasks.rados.rados.0.smithi029.stdout:1129: expect (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:19:09.535 INFO:tasks.rados.rados.0.smithi029.stdout:1130: delete oid 39 current snap is 145 2024-04-12T11:19:09.538 INFO:tasks.rados.rados.0.smithi029.stdout:1130: done (1 left) 2024-04-12T11:19:09.538 INFO:tasks.rados.rados.0.smithi029.stdout:1131: snap_create 2024-04-12T11:19:09.541 INFO:tasks.rados.rados.0.smithi029.stdout:1129: expect (ObjNum 205 snap 58 seq_num 205) 2024-04-12T11:19:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:09 smithi087 ceph-mon[32917]: osdmap e348: 8 total, 8 up, 8 in 2024-04-12T11:19:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:09 smithi029 ceph-mon[93650]: osdmap e348: 8 total, 8 up, 8 in 2024-04-12T11:19:10.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:09 smithi029 ceph-mon[97787]: osdmap e348: 8 total, 8 up, 8 in 2024-04-12T11:19:10.532 INFO:tasks.rados.rados.0.smithi029.stdout:1129: done (1 left) 2024-04-12T11:19:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:1131: done (0 left) 2024-04-12T11:19:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:1132: read oid 6 snap -1 2024-04-12T11:19:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:1132: expect (ObjNum 424 snap 135 seq_num 424) 2024-04-12T11:19:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:1133: read oid 21 snap -1 2024-04-12T11:19:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:1133: expect (ObjNum 431 snap 137 seq_num 431) 2024-04-12T11:19:10.533 INFO:tasks.rados.rados.0.smithi029.stdout:1134: snap_create 2024-04-12T11:19:10.546 INFO:tasks.rados.rados.0.smithi029.stdout:1133: expect (ObjNum 431 snap 137 seq_num 431) 2024-04-12T11:19:10.659 INFO:tasks.rados.rados.0.smithi029.stdout:1132: expect (ObjNum 424 snap 135 seq_num 424) 2024-04-12T11:19:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:10 smithi087 ceph-mon[32917]: pgmap v157: 89 pgs: 89 active+clean; 264 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-04-12T11:19:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:10 smithi087 ceph-mon[32917]: osdmap e349: 8 total, 8 up, 8 in 2024-04-12T11:19:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:10 smithi029 ceph-mon[93650]: pgmap v157: 89 pgs: 89 active+clean; 264 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-04-12T11:19:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:10 smithi029 ceph-mon[93650]: osdmap e349: 8 total, 8 up, 8 in 2024-04-12T11:19:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:10 smithi029 ceph-mon[97787]: pgmap v157: 89 pgs: 89 active+clean; 264 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-04-12T11:19:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:10 smithi029 ceph-mon[97787]: osdmap e349: 8 total, 8 up, 8 in 2024-04-12T11:19:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:1132: done (2 left) 2024-04-12T11:19:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:1133: done (1 left) 2024-04-12T11:19:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:1134: done (0 left) 2024-04-12T11:19:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:1135: write oid 49 current snap is 147 2024-04-12T11:19:11.538 INFO:tasks.rados.rados.0.smithi029.stdout:1135: seq_num 448 ranges {729536=460026,1749580=611149,2853383=452450} 2024-04-12T11:19:11.544 INFO:tasks.rados.rados.0.smithi029.stdout:1135: writing smithi02979277-49 from 729536 to 1189562 tid 1 2024-04-12T11:19:11.551 INFO:tasks.rados.rados.0.smithi029.stdout:1135: writing smithi02979277-49 from 1749580 to 2360729 tid 2 2024-04-12T11:19:11.556 INFO:tasks.rados.rados.0.smithi029.stdout:1135: writing smithi02979277-49 from 2853383 to 3305833 tid 3 2024-04-12T11:19:11.556 INFO:tasks.rados.rados.0.smithi029.stdout:1136: snap_remove snap 136 2024-04-12T11:19:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:11 smithi087 ceph-mon[32917]: osdmap e350: 8 total, 8 up, 8 in 2024-04-12T11:19:12.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:11 smithi029 ceph-mon[93650]: osdmap e350: 8 total, 8 up, 8 in 2024-04-12T11:19:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:11 smithi029 ceph-mon[97787]: osdmap e350: 8 total, 8 up, 8 in 2024-04-12T11:19:12.543 INFO:tasks.rados.rados.0.smithi029.stdout:1136: done (1 left) 2024-04-12T11:19:12.543 INFO:tasks.rados.rados.0.smithi029.stdout:1137: read oid 1 snap -1 2024-04-12T11:19:12.544 INFO:tasks.rados.rados.0.smithi029.stdout:1137: expect (ObjNum 440 snap 141 seq_num 440) 2024-04-12T11:19:12.544 INFO:tasks.rados.rados.0.smithi029.stdout:1138: read oid 40 snap 129 2024-04-12T11:19:12.544 INFO:tasks.rados.rados.0.smithi029.stdout:1138: expect deleted 2024-04-12T11:19:12.544 INFO:tasks.rados.rados.0.smithi029.stdout:1139: snap_remove snap 142 2024-04-12T11:19:12.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:12 smithi087 ceph-mon[32917]: pgmap v160: 89 pgs: 89 active+clean; 264 MiB data, 758 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:19:12.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:12 smithi087 ceph-mon[32917]: osdmap e351: 8 total, 8 up, 8 in 2024-04-12T11:19:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:12 smithi029 ceph-mon[97787]: pgmap v160: 89 pgs: 89 active+clean; 264 MiB data, 758 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:19:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:12 smithi029 ceph-mon[97787]: osdmap e351: 8 total, 8 up, 8 in 2024-04-12T11:19:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:12 smithi029 ceph-mon[93650]: pgmap v160: 89 pgs: 89 active+clean; 264 MiB data, 758 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:19:13.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:12 smithi029 ceph-mon[93650]: osdmap e351: 8 total, 8 up, 8 in 2024-04-12T11:19:13.548 INFO:tasks.rados.rados.0.smithi029.stdout:1139: done (3 left) 2024-04-12T11:19:13.549 INFO:tasks.rados.rados.0.smithi029.stdout:1140: read oid 37 snap 140 2024-04-12T11:19:13.549 INFO:tasks.rados.rados.0.smithi029.stdout:1140: expect deleted 2024-04-12T11:19:13.549 INFO:tasks.rados.rados.0.smithi029.stdout:1141: read oid 4 snap -1 2024-04-12T11:19:13.549 INFO:tasks.rados.rados.0.smithi029.stdout:1141: expect (ObjNum 409 snap 128 seq_num 409) 2024-04-12T11:19:13.549 INFO:tasks.rados.rados.0.smithi029.stdout:1142: read oid 20 snap -1 2024-04-12T11:19:13.549 INFO:tasks.rados.rados.0.smithi029.stdout:1142: expect (ObjNum 416 snap 130 seq_num 416) 2024-04-12T11:19:13.549 INFO:tasks.rados.rados.0.smithi029.stdout:1143: write oid 35 current snap is 147 2024-04-12T11:19:13.549 INFO:tasks.rados.rados.0.smithi029.stdout:1143: seq_num 449 ranges {746343=651628,1924415=754807,3397159=149184} 2024-04-12T11:19:13.556 INFO:tasks.rados.rados.0.smithi029.stdout:1143: writing smithi02979277-35 from 746343 to 1397971 tid 1 2024-04-12T11:19:13.563 INFO:tasks.rados.rados.0.smithi029.stdout:1143: writing smithi02979277-35 from 1924415 to 2679222 tid 2 2024-04-12T11:19:13.569 INFO:tasks.rados.rados.0.smithi029.stdout:1143: writing smithi02979277-35 from 3397159 to 3546343 tid 3 2024-04-12T11:19:13.569 INFO:tasks.rados.rados.0.smithi029.stdout:1135: finishing write tid 1 to smithi02979277-49 2024-04-12T11:19:13.569 INFO:tasks.rados.rados.0.smithi029.stdout:1135: finishing write tid 2 to smithi02979277-49 2024-04-12T11:19:13.569 INFO:tasks.rados.rados.0.smithi029.stdout:1135: finishing write tid 3 to smithi02979277-49 2024-04-12T11:19:13.569 INFO:tasks.rados.rados.0.smithi029.stdout:1135: finishing write tid 4 to smithi02979277-49 2024-04-12T11:19:13.569 INFO:tasks.rados.rados.0.smithi029.stdout:1135: finishing write tid 5 to smithi02979277-49 2024-04-12T11:19:13.569 INFO:tasks.rados.rados.0.smithi029.stdout:1135: oid 49 updating version 0 to 475 2024-04-12T11:19:13.570 INFO:tasks.rados.rados.0.smithi029.stdout:1135: oid 49 version 475 is already newer than 474 2024-04-12T11:19:13.570 INFO:tasks.rados.rados.0.smithi029.stdout:1135: oid 49 version 475 is already newer than 472 2024-04-12T11:19:13.570 INFO:tasks.rados.rados.0.smithi029.stdout:1135: oid 49 version 475 is already newer than 473 2024-04-12T11:19:13.570 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 475 (ObjNum 448 snap 147 seq_num 448) dirty exists 2024-04-12T11:19:13.570 INFO:tasks.rados.rados.0.smithi029.stdout:1135: left oid 49 (ObjNum 448 snap 147 seq_num 448) 2024-04-12T11:19:13.570 INFO:tasks.rados.rados.0.smithi029.stdout:1137: expect (ObjNum 440 snap 141 seq_num 440) 2024-04-12T11:19:13.684 INFO:tasks.rados.rados.0.smithi029.stdout:1142: expect (ObjNum 416 snap 130 seq_num 416) 2024-04-12T11:19:13.784 INFO:tasks.rados.rados.0.smithi029.stdout:1143: finishing write tid 1 to smithi02979277-35 2024-04-12T11:19:13.784 INFO:tasks.rados.rados.0.smithi029.stdout:1141: expect (ObjNum 409 snap 128 seq_num 409) 2024-04-12T11:19:13.924 INFO:tasks.rados.rados.0.smithi029.stdout:1143: finishing write tid 2 to smithi02979277-35 2024-04-12T11:19:13.924 INFO:tasks.rados.rados.0.smithi029.stdout:1143: finishing write tid 3 to smithi02979277-35 2024-04-12T11:19:13.924 INFO:tasks.rados.rados.0.smithi029.stdout:1143: finishing write tid 4 to smithi02979277-35 2024-04-12T11:19:13.924 INFO:tasks.rados.rados.0.smithi029.stdout:1143: finishing write tid 5 to smithi02979277-35 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:1143: oid 35 updating version 0 to 706 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:1143: oid 35 version 706 is already newer than 705 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:1143: oid 35 version 706 is already newer than 704 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:1143: oid 35 version 706 is already newer than 701 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 706 (ObjNum 449 snap 147 seq_num 449) dirty exists 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:1143: left oid 35 (ObjNum 449 snap 147 seq_num 449) 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:1144: read oid 18 snap -1 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:1144: expect (ObjNum 402 snap 123 seq_num 402) 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:1135: done (7 left) 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:1137: done (6 left) 2024-04-12T11:19:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:1138: done (5 left) 2024-04-12T11:19:13.926 INFO:tasks.rados.rados.0.smithi029.stdout:1140: done (4 left) 2024-04-12T11:19:13.926 INFO:tasks.rados.rados.0.smithi029.stdout:1141: done (3 left) 2024-04-12T11:19:13.926 INFO:tasks.rados.rados.0.smithi029.stdout:1142: done (2 left) 2024-04-12T11:19:13.926 INFO:tasks.rados.rados.0.smithi029.stdout:1143: done (1 left) 2024-04-12T11:19:13.926 INFO:tasks.rados.rados.0.smithi029.stdout:1145: snap_remove snap 124 2024-04-12T11:19:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:13 smithi087 ceph-mon[32917]: osdmap e352: 8 total, 8 up, 8 in 2024-04-12T11:19:13.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:13 smithi029 ceph-mon[93650]: osdmap e352: 8 total, 8 up, 8 in 2024-04-12T11:19:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:13 smithi029 ceph-mon[97787]: osdmap e352: 8 total, 8 up, 8 in 2024-04-12T11:19:14.550 INFO:tasks.rados.rados.0.smithi029.stdout:1144: expect (ObjNum 402 snap 123 seq_num 402) 2024-04-12T11:19:14.704 INFO:tasks.rados.rados.0.smithi029.stdout:1144: done (1 left) 2024-04-12T11:19:14.704 INFO:tasks.rados.rados.0.smithi029.stdout:1145: done (0 left) 2024-04-12T11:19:14.704 INFO:tasks.rados.rados.0.smithi029.stdout:1146: read oid 17 snap -1 2024-04-12T11:19:14.705 INFO:tasks.rados.rados.0.smithi029.stdout:1146: expect (ObjNum 390 snap 121 seq_num 390) 2024-04-12T11:19:14.705 INFO:tasks.rados.rados.0.smithi029.stdout:1147: read oid 28 snap -1 2024-04-12T11:19:14.705 INFO:tasks.rados.rados.0.smithi029.stdout:1147: expect (ObjNum 412 snap 129 seq_num 412) 2024-04-12T11:19:14.705 INFO:tasks.rados.rados.0.smithi029.stdout:1148: delete oid 24 current snap is 147 2024-04-12T11:19:14.706 INFO:tasks.rados.rados.0.smithi029.stdout:1148: done (2 left) 2024-04-12T11:19:14.706 INFO:tasks.rados.rados.0.smithi029.stdout:1149: snap_create 2024-04-12T11:19:14.706 INFO:tasks.rados.rados.0.smithi029.stdout:1147: expect (ObjNum 412 snap 129 seq_num 412) 2024-04-12T11:19:14.877 INFO:tasks.rados.rados.0.smithi029.stdout:1146: expect (ObjNum 390 snap 121 seq_num 390) 2024-04-12T11:19:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:14 smithi029 ceph-mon[97787]: pgmap v163: 89 pgs: 89 active+clean; 267 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 803 KiB/s wr, 10 op/s 2024-04-12T11:19:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:14 smithi029 ceph-mon[97787]: osdmap e353: 8 total, 8 up, 8 in 2024-04-12T11:19:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:14 smithi029 ceph-mon[93650]: pgmap v163: 89 pgs: 89 active+clean; 267 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 803 KiB/s wr, 10 op/s 2024-04-12T11:19:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:14 smithi029 ceph-mon[93650]: osdmap e353: 8 total, 8 up, 8 in 2024-04-12T11:19:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:14 smithi087 ceph-mon[32917]: pgmap v163: 89 pgs: 89 active+clean; 267 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 803 KiB/s wr, 10 op/s 2024-04-12T11:19:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:14 smithi087 ceph-mon[32917]: osdmap e353: 8 total, 8 up, 8 in 2024-04-12T11:19:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:19:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:19:14] "GET /metrics HTTP/1.1" 200 36243 "" "Prometheus/2.43.0" 2024-04-12T11:19:15.561 INFO:tasks.rados.rados.0.smithi029.stdout:1146: done (2 left) 2024-04-12T11:19:15.561 INFO:tasks.rados.rados.0.smithi029.stdout:1147: done (1 left) 2024-04-12T11:19:15.562 INFO:tasks.rados.rados.0.smithi029.stdout:1149: done (0 left) 2024-04-12T11:19:15.562 INFO:tasks.rados.rados.0.smithi029.stdout:1150: snap_remove snap 146 2024-04-12T11:19:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:15 smithi087 ceph-mon[32917]: osdmap e354: 8 total, 8 up, 8 in 2024-04-12T11:19:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:15 smithi029 ceph-mon[97787]: osdmap e354: 8 total, 8 up, 8 in 2024-04-12T11:19:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:15 smithi029 ceph-mon[93650]: osdmap e354: 8 total, 8 up, 8 in 2024-04-12T11:19:16.563 INFO:tasks.rados.rados.0.smithi029.stdout:1150: done (0 left) 2024-04-12T11:19:16.563 INFO:tasks.rados.rados.0.smithi029.stdout:1151: rollback oid 42 current snap is 148 2024-04-12T11:19:16.563 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 143 2024-04-12T11:19:16.563 INFO:tasks.rados.rados.0.smithi029.stdout:1152: read oid 44 snap -1 2024-04-12T11:19:16.563 INFO:tasks.rados.rados.0.smithi029.stdout:1152: expect (ObjNum 439 snap 139 seq_num 439) 2024-04-12T11:19:16.563 INFO:tasks.rados.rados.0.smithi029.stdout:1153: read oid 43 snap -1 2024-04-12T11:19:16.563 INFO:tasks.rados.rados.0.smithi029.stdout:1153: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:19:16.563 INFO:tasks.rados.rados.0.smithi029.stdout:1154: write oid 22 current snap is 148 2024-04-12T11:19:16.564 INFO:tasks.rados.rados.0.smithi029.stdout:1154: seq_num 451 ranges {779957=402126,1925026=758476,3335652=244305} 2024-04-12T11:19:16.569 INFO:tasks.rados.rados.0.smithi029.stdout:1154: writing smithi02979277-22 from 779957 to 1182083 tid 1 2024-04-12T11:19:16.578 INFO:tasks.rados.rados.0.smithi029.stdout:1154: writing smithi02979277-22 from 1925026 to 2683502 tid 2 2024-04-12T11:19:16.583 INFO:tasks.rados.rados.0.smithi029.stdout:1154: writing smithi02979277-22 from 3335652 to 3579957 tid 3 2024-04-12T11:19:16.583 INFO:tasks.rados.rados.0.smithi029.stdout:1155: read oid 21 snap -1 2024-04-12T11:19:16.583 INFO:tasks.rados.rados.0.smithi029.stdout:1155: expect (ObjNum 431 snap 137 seq_num 431) 2024-04-12T11:19:16.584 INFO:tasks.rados.rados.0.smithi029.stdout:1156: write oid 48 current snap is 148 2024-04-12T11:19:16.584 INFO:tasks.rados.rados.0.smithi029.stdout:1156: seq_num 452 ranges {796764=477375,1983508=618487,3363075=233689} 2024-04-12T11:19:16.590 INFO:tasks.rados.rados.0.smithi029.stdout:1156: writing smithi02979277-48 from 796764 to 1274139 tid 1 2024-04-12T11:19:16.597 INFO:tasks.rados.rados.0.smithi029.stdout:1156: writing smithi02979277-48 from 1983508 to 2601995 tid 2 2024-04-12T11:19:16.603 INFO:tasks.rados.rados.0.smithi029.stdout:1156: writing smithi02979277-48 from 3363075 to 3596764 tid 3 2024-04-12T11:19:16.603 INFO:tasks.rados.rados.0.smithi029.stdout:1157: snap_create 2024-04-12T11:19:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:1153: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1151: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1151: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1151: finishing rollback tid 2 to smithi02979277-42 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 170 (ObjNum 322 snap 100 seq_num 322) dirty exists 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1154: finishing write tid 1 to smithi02979277-22 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1154: finishing write tid 2 to smithi02979277-22 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1154: finishing write tid 3 to smithi02979277-22 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1154: finishing write tid 4 to smithi02979277-22 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1154: finishing write tid 5 to smithi02979277-22 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1154: oid 22 updating version 0 to 374 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1154: oid 22 updating version 374 to 375 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1154: oid 22 version 375 is already newer than 373 2024-04-12T11:19:16.712 INFO:tasks.rados.rados.0.smithi029.stdout:1154: oid 22 version 375 is already newer than 372 2024-04-12T11:19:16.713 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 375 (ObjNum 451 snap 148 seq_num 451) dirty exists 2024-04-12T11:19:16.713 INFO:tasks.rados.rados.0.smithi029.stdout:1154: left oid 22 (ObjNum 451 snap 148 seq_num 451) 2024-04-12T11:19:16.713 INFO:tasks.rados.rados.0.smithi029.stdout:1152: expect (ObjNum 439 snap 139 seq_num 439) 2024-04-12T11:19:16.953 INFO:tasks.rados.rados.0.smithi029.stdout:1156: finishing write tid 1 to smithi02979277-48 2024-04-12T11:19:16.954 INFO:tasks.rados.rados.0.smithi029.stdout:1155: expect (ObjNum 431 snap 137 seq_num 431) 2024-04-12T11:19:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:16 smithi087 ceph-mon[32917]: pgmap v166: 89 pgs: 89 active+clean; 267 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 803 KiB/s wr, 10 op/s 2024-04-12T11:19:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:16 smithi087 ceph-mon[32917]: osdmap e355: 8 total, 8 up, 8 in 2024-04-12T11:19:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:16 smithi029 ceph-mon[97787]: pgmap v166: 89 pgs: 89 active+clean; 267 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 803 KiB/s wr, 10 op/s 2024-04-12T11:19:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:16 smithi029 ceph-mon[97787]: osdmap e355: 8 total, 8 up, 8 in 2024-04-12T11:19:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:16 smithi029 ceph-mon[93650]: pgmap v166: 89 pgs: 89 active+clean; 267 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 803 KiB/s wr, 10 op/s 2024-04-12T11:19:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:16 smithi029 ceph-mon[93650]: osdmap e355: 8 total, 8 up, 8 in 2024-04-12T11:19:17.065 INFO:tasks.rados.rados.0.smithi029.stdout:1156: finishing write tid 2 to smithi02979277-48 2024-04-12T11:19:17.065 INFO:tasks.rados.rados.0.smithi029.stdout:1156: finishing write tid 3 to smithi02979277-48 2024-04-12T11:19:17.065 INFO:tasks.rados.rados.0.smithi029.stdout:1156: finishing write tid 4 to smithi02979277-48 2024-04-12T11:19:17.065 INFO:tasks.rados.rados.0.smithi029.stdout:1156: finishing write tid 5 to smithi02979277-48 2024-04-12T11:19:17.065 INFO:tasks.rados.rados.0.smithi029.stdout:1156: oid 48 updating version 0 to 472 2024-04-12T11:19:17.065 INFO:tasks.rados.rados.0.smithi029.stdout:1156: oid 48 version 472 is already newer than 471 2024-04-12T11:19:17.066 INFO:tasks.rados.rados.0.smithi029.stdout:1156: oid 48 updating version 472 to 473 2024-04-12T11:19:17.066 INFO:tasks.rados.rados.0.smithi029.stdout:1156: oid 48 updating version 473 to 474 2024-04-12T11:19:17.066 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 474 (ObjNum 452 snap 148 seq_num 452) dirty exists 2024-04-12T11:19:17.066 INFO:tasks.rados.rados.0.smithi029.stdout:1156: left oid 48 (ObjNum 452 snap 148 seq_num 452) 2024-04-12T11:19:17.522 INFO:tasks.rados.rados.0.smithi029.stdout:1151: done (6 left) 2024-04-12T11:19:17.522 INFO:tasks.rados.rados.0.smithi029.stdout:1152: done (5 left) 2024-04-12T11:19:17.522 INFO:tasks.rados.rados.0.smithi029.stdout:1153: done (4 left) 2024-04-12T11:19:17.522 INFO:tasks.rados.rados.0.smithi029.stdout:1154: done (3 left) 2024-04-12T11:19:17.522 INFO:tasks.rados.rados.0.smithi029.stdout:1155: done (2 left) 2024-04-12T11:19:17.522 INFO:tasks.rados.rados.0.smithi029.stdout:1156: done (1 left) 2024-04-12T11:19:17.522 INFO:tasks.rados.rados.0.smithi029.stdout:1157: done (0 left) 2024-04-12T11:19:17.522 INFO:tasks.rados.rados.0.smithi029.stdout:1158: write oid 7 current snap is 149 2024-04-12T11:19:17.523 INFO:tasks.rados.rados.0.smithi029.stdout:1158: seq_num 453 ranges {413571=552624,1641990=762145,2990498=425163} 2024-04-12T11:19:17.527 INFO:tasks.rados.rados.0.smithi029.stdout:1158: writing smithi02979277-7 from 413571 to 966195 tid 1 2024-04-12T11:19:17.535 INFO:tasks.rados.rados.0.smithi029.stdout:1158: writing smithi02979277-7 from 1641990 to 2404135 tid 2 2024-04-12T11:19:17.541 INFO:tasks.rados.rados.0.smithi029.stdout:1158: writing smithi02979277-7 from 2990498 to 3415661 tid 3 2024-04-12T11:19:17.541 INFO:tasks.rados.rados.0.smithi029.stdout:1159: write oid 19 current snap is 149 2024-04-12T11:19:17.541 INFO:tasks.rados.rados.0.smithi029.stdout:1158: finishing write tid 1 to smithi02979277-7 2024-04-12T11:19:17.541 INFO:tasks.rados.rados.0.smithi029.stdout:1158: finishing write tid 2 to smithi02979277-7 2024-04-12T11:19:17.541 INFO:tasks.rados.rados.0.smithi029.stdout:1159: seq_num 454 ranges {430378=627873,1700472=622156,3017921=612457} 2024-04-12T11:19:17.546 INFO:tasks.rados.rados.0.smithi029.stdout:1159: writing smithi02979277-19 from 430378 to 1058251 tid 1 2024-04-12T11:19:17.553 INFO:tasks.rados.rados.0.smithi029.stdout:1159: writing smithi02979277-19 from 1700472 to 2322628 tid 2 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:1159: writing smithi02979277-19 from 3017921 to 3630378 tid 3 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:1160: snap_create 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:1158: finishing write tid 3 to smithi02979277-7 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:1158: finishing write tid 4 to smithi02979277-7 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:1158: finishing write tid 5 to smithi02979277-7 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:1158: oid 7 updating version 0 to 566 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:1158: oid 7 updating version 566 to 568 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:1158: oid 7 updating version 568 to 569 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:1158: oid 7 version 569 is already newer than 567 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 569 (ObjNum 453 snap 149 seq_num 453) dirty exists 2024-04-12T11:19:17.561 INFO:tasks.rados.rados.0.smithi029.stdout:1158: left oid 7 (ObjNum 453 snap 149 seq_num 453) 2024-04-12T11:19:17.562 INFO:tasks.rados.rados.0.smithi029.stdout:1159: finishing write tid 1 to smithi02979277-19 2024-04-12T11:19:17.562 INFO:tasks.rados.rados.0.smithi029.stdout:1159: finishing write tid 2 to smithi02979277-19 2024-04-12T11:19:17.565 INFO:tasks.rados.rados.0.smithi029.stdout:1159: finishing write tid 3 to smithi02979277-19 2024-04-12T11:19:17.565 INFO:tasks.rados.rados.0.smithi029.stdout:1159: finishing write tid 4 to smithi02979277-19 2024-04-12T11:19:17.565 INFO:tasks.rados.rados.0.smithi029.stdout:1159: finishing write tid 5 to smithi02979277-19 2024-04-12T11:19:17.565 INFO:tasks.rados.rados.0.smithi029.stdout:1159: oid 19 updating version 0 to 692 2024-04-12T11:19:17.565 INFO:tasks.rados.rados.0.smithi029.stdout:1159: oid 19 version 692 is already newer than 689 2024-04-12T11:19:17.565 INFO:tasks.rados.rados.0.smithi029.stdout:1159: oid 19 version 692 is already newer than 690 2024-04-12T11:19:17.565 INFO:tasks.rados.rados.0.smithi029.stdout:1159: oid 19 version 692 is already newer than 691 2024-04-12T11:19:17.565 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 692 (ObjNum 454 snap 149 seq_num 454) dirty exists 2024-04-12T11:19:17.565 INFO:tasks.rados.rados.0.smithi029.stdout:1159: left oid 19 (ObjNum 454 snap 149 seq_num 454) 2024-04-12T11:19:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:17 smithi087 ceph-mon[32917]: osdmap e356: 8 total, 8 up, 8 in 2024-04-12T11:19:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:17 smithi087 ceph-mon[32917]: osdmap e357: 8 total, 8 up, 8 in 2024-04-12T11:19:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:17 smithi029 ceph-mon[93650]: osdmap e356: 8 total, 8 up, 8 in 2024-04-12T11:19:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:17 smithi029 ceph-mon[93650]: osdmap e357: 8 total, 8 up, 8 in 2024-04-12T11:19:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:17 smithi029 ceph-mon[97787]: osdmap e356: 8 total, 8 up, 8 in 2024-04-12T11:19:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:17 smithi029 ceph-mon[97787]: osdmap e357: 8 total, 8 up, 8 in 2024-04-12T11:19:18.524 INFO:tasks.rados.rados.0.smithi029.stdout:1158: done (2 left) 2024-04-12T11:19:18.524 INFO:tasks.rados.rados.0.smithi029.stdout:1159: done (1 left) 2024-04-12T11:19:18.524 INFO:tasks.rados.rados.0.smithi029.stdout:1160: done (0 left) 2024-04-12T11:19:18.524 INFO:tasks.rados.rados.0.smithi029.stdout:1161: read oid 47 snap 147 2024-04-12T11:19:18.524 INFO:tasks.rados.rados.0.smithi029.stdout:1161: expect (ObjNum 415 snap 130 seq_num 415) 2024-04-12T11:19:18.524 INFO:tasks.rados.rados.0.smithi029.stdout:1162: snap_remove snap 100 2024-04-12T11:19:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:18 smithi087 ceph-mon[32917]: pgmap v169: 89 pgs: 89 active+clean; 274 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.0 MiB/s wr, 22 op/s 2024-04-12T11:19:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:18 smithi087 ceph-mon[32917]: osdmap e358: 8 total, 8 up, 8 in 2024-04-12T11:19:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:18 smithi029 ceph-mon[93650]: pgmap v169: 89 pgs: 89 active+clean; 274 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.0 MiB/s wr, 22 op/s 2024-04-12T11:19:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:18 smithi029 ceph-mon[93650]: osdmap e358: 8 total, 8 up, 8 in 2024-04-12T11:19:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:18 smithi029 ceph-mon[97787]: pgmap v169: 89 pgs: 89 active+clean; 274 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.0 MiB/s wr, 22 op/s 2024-04-12T11:19:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:18 smithi029 ceph-mon[97787]: osdmap e358: 8 total, 8 up, 8 in 2024-04-12T11:19:19.526 INFO:tasks.rados.rados.0.smithi029.stdout:1162: done (1 left) 2024-04-12T11:19:19.527 INFO:tasks.rados.rados.0.smithi029.stdout:1163: snap_remove snap 144 2024-04-12T11:19:20.532 INFO:tasks.rados.rados.0.smithi029.stdout:1163: done (1 left) 2024-04-12T11:19:20.532 INFO:tasks.rados.rados.0.smithi029.stdout:1164: delete oid 21 current snap is 150 2024-04-12T11:19:20.532 INFO:tasks.rados.rados.0.smithi029.stdout:1161: expect (ObjNum 415 snap 130 seq_num 415) 2024-04-12T11:19:20.749 INFO:tasks.rados.rados.0.smithi029.stdout:1161: done (1 left) 2024-04-12T11:19:20.749 INFO:tasks.rados.rados.0.smithi029.stdout:1164: done (0 left) 2024-04-12T11:19:20.749 INFO:tasks.rados.rados.0.smithi029.stdout:1165: rollback oid 19 current snap is 150 2024-04-12T11:19:20.749 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 19 to 123 2024-04-12T11:19:20.749 INFO:tasks.rados.rados.0.smithi029.stdout:1166: read oid 5 snap -1 2024-04-12T11:19:20.750 INFO:tasks.rados.rados.0.smithi029.stdout:1166: expect (ObjNum 442 snap 141 seq_num 442) 2024-04-12T11:19:20.750 INFO:tasks.rados.rados.0.smithi029.stdout:1167: write oid 1 current snap is 150 2024-04-12T11:19:20.750 INFO:tasks.rados.rados.0.smithi029.stdout:1167: seq_num 456 ranges {463992=778371,1701083=625825,2956414=658979} 2024-04-12T11:19:20.757 INFO:tasks.rados.rados.0.smithi029.stdout:1167: writing smithi02979277-1 from 463992 to 1242363 tid 1 2024-04-12T11:19:20.766 INFO:tasks.rados.rados.0.smithi029.stdout:1167: writing smithi02979277-1 from 1701083 to 2326908 tid 2 2024-04-12T11:19:20.775 INFO:tasks.rados.rados.0.smithi029.stdout:1167: writing smithi02979277-1 from 2956414 to 3615393 tid 3 2024-04-12T11:19:20.775 INFO:tasks.rados.rados.0.smithi029.stdout:1168: rollback oid 10 current snap is 150 2024-04-12T11:19:20.775 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 10 to 143 2024-04-12T11:19:20.775 INFO:tasks.rados.rados.0.smithi029.stdout:1165: finishing rollback tid 0 to smithi02979277-19 2024-04-12T11:19:20.775 INFO:tasks.rados.rados.0.smithi029.stdout:1165: finishing rollback tid 1 to smithi02979277-19 2024-04-12T11:19:20.776 INFO:tasks.rados.rados.0.smithi029.stdout:1165: finishing rollback tid 2 to smithi02979277-19 2024-04-12T11:19:20.776 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 706 (ObjNum 389 snap 121 seq_num 389) dirty exists 2024-04-12T11:19:20.776 INFO:tasks.rados.rados.0.smithi029.stdout:1165: done (3 left) 2024-04-12T11:19:20.776 INFO:tasks.rados.rados.0.smithi029.stdout:1169: read oid 2 snap -1 2024-04-12T11:19:20.776 INFO:tasks.rados.rados.0.smithi029.stdout:1169: expect (ObjNum 401 snap 123 seq_num 401) 2024-04-12T11:19:20.776 INFO:tasks.rados.rados.0.smithi029.stdout:1170: write oid 30 current snap is 150 2024-04-12T11:19:20.776 INFO:tasks.rados.rados.0.smithi029.stdout:1167: finishing write tid 1 to smithi02979277-1 2024-04-12T11:19:20.776 INFO:tasks.rados.rados.0.smithi029.stdout:1166: expect (ObjNum 442 snap 141 seq_num 442) 2024-04-12T11:19:20.939 INFO:tasks.rados.rados.0.smithi029.stdout:1167: finishing write tid 2 to smithi02979277-1 2024-04-12T11:19:20.939 INFO:tasks.rados.rados.0.smithi029.stdout:1168: finishing rollback tid 0 to smithi02979277-10 2024-04-12T11:19:20.939 INFO:tasks.rados.rados.0.smithi029.stdout:1168: finishing rollback tid 1 to smithi02979277-10 2024-04-12T11:19:20.939 INFO:tasks.rados.rados.0.smithi029.stdout:1168: finishing rollback tid 2 to smithi02979277-10 2024-04-12T11:19:20.939 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 486 (ObjNum 435 snap 139 seq_num 435) dirty exists 2024-04-12T11:19:20.939 INFO:tasks.rados.rados.0.smithi029.stdout:1170: seq_num 457 ranges {480799=569973,1475918=769483,2700190=586604} 2024-04-12T11:19:20.945 INFO:tasks.rados.rados.0.smithi029.stdout:1170: writing smithi02979277-30 from 480799 to 1050772 tid 1 2024-04-12T11:19:20.951 INFO:tasks.rados.rados.0.smithi029.stdout:1170: writing smithi02979277-30 from 1475918 to 2245401 tid 2 2024-04-12T11:19:20.957 INFO:tasks.rados.rados.0.smithi029.stdout:1170: writing smithi02979277-30 from 2700190 to 3286794 tid 3 2024-04-12T11:19:20.957 INFO:tasks.rados.rados.0.smithi029.stdout:1166: done (4 left) 2024-04-12T11:19:20.957 INFO:tasks.rados.rados.0.smithi029.stdout:1168: done (3 left) 2024-04-12T11:19:20.957 INFO:tasks.rados.rados.0.smithi029.stdout:1171: write oid 40 current snap is 150 2024-04-12T11:19:20.957 INFO:tasks.rados.rados.0.smithi029.stdout:1167: finishing write tid 3 to smithi02979277-1 2024-04-12T11:19:20.957 INFO:tasks.rados.rados.0.smithi029.stdout:1167: finishing write tid 4 to smithi02979277-1 2024-04-12T11:19:20.957 INFO:tasks.rados.rados.0.smithi029.stdout:1169: expect (ObjNum 401 snap 123 seq_num 401) 2024-04-12T11:19:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:20 smithi087 ceph-mon[32917]: pgmap v172: 89 pgs: 89 active+clean; 274 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.0 MiB/s wr, 22 op/s 2024-04-12T11:19:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:20 smithi087 ceph-mon[32917]: osdmap e359: 8 total, 8 up, 8 in 2024-04-12T11:19:21.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:20 smithi029 ceph-mon[93650]: pgmap v172: 89 pgs: 89 active+clean; 274 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.0 MiB/s wr, 22 op/s 2024-04-12T11:19:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:20 smithi029 ceph-mon[93650]: osdmap e359: 8 total, 8 up, 8 in 2024-04-12T11:19:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:20 smithi029 ceph-mon[97787]: pgmap v172: 89 pgs: 89 active+clean; 274 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.0 MiB/s wr, 22 op/s 2024-04-12T11:19:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:20 smithi029 ceph-mon[97787]: osdmap e359: 8 total, 8 up, 8 in 2024-04-12T11:19:21.142 INFO:tasks.rados.rados.0.smithi029.stdout:1167: finishing write tid 5 to smithi02979277-1 2024-04-12T11:19:21.142 INFO:tasks.rados.rados.0.smithi029.stdout:1167: oid 1 updating version 0 to 534 2024-04-12T11:19:21.142 INFO:tasks.rados.rados.0.smithi029.stdout:1167: oid 1 updating version 534 to 536 2024-04-12T11:19:21.142 INFO:tasks.rados.rados.0.smithi029.stdout:1167: oid 1 version 536 is already newer than 533 2024-04-12T11:19:21.143 INFO:tasks.rados.rados.0.smithi029.stdout:1167: oid 1 version 536 is already newer than 535 2024-04-12T11:19:21.143 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 536 (ObjNum 456 snap 150 seq_num 456) dirty exists 2024-04-12T11:19:21.143 INFO:tasks.rados.rados.0.smithi029.stdout:1167: left oid 1 (ObjNum 456 snap 150 seq_num 456) 2024-04-12T11:19:21.143 INFO:tasks.rados.rados.0.smithi029.stdout:1170: finishing write tid 1 to smithi02979277-30 2024-04-12T11:19:21.143 INFO:tasks.rados.rados.0.smithi029.stdout:1171: seq_num 458 ranges {497606=645222,1934400=629494,3243966=453640} 2024-04-12T11:19:21.148 INFO:tasks.rados.rados.0.smithi029.stdout:1171: writing smithi02979277-40 from 497606 to 1142828 tid 1 2024-04-12T11:19:21.155 INFO:tasks.rados.rados.0.smithi029.stdout:1171: writing smithi02979277-40 from 1934400 to 2563894 tid 2 2024-04-12T11:19:21.162 INFO:tasks.rados.rados.0.smithi029.stdout:1171: writing smithi02979277-40 from 3243966 to 3697606 tid 3 2024-04-12T11:19:21.162 INFO:tasks.rados.rados.0.smithi029.stdout:1167: done (3 left) 2024-04-12T11:19:21.162 INFO:tasks.rados.rados.0.smithi029.stdout:1169: done (2 left) 2024-04-12T11:19:21.162 INFO:tasks.rados.rados.0.smithi029.stdout:1172: write oid 17 current snap is 150 2024-04-12T11:19:21.163 INFO:tasks.rados.rados.0.smithi029.stdout:1172: seq_num 459 ranges {514413=720471,1876529=773152,3438683=275730} 2024-04-12T11:19:21.168 INFO:tasks.rados.rados.0.smithi029.stdout:1172: writing smithi02979277-17 from 514413 to 1234884 tid 1 2024-04-12T11:19:21.176 INFO:tasks.rados.rados.0.smithi029.stdout:1172: writing smithi02979277-17 from 1876529 to 2649681 tid 2 2024-04-12T11:19:21.184 INFO:tasks.rados.rados.0.smithi029.stdout:1172: writing smithi02979277-17 from 3438683 to 3714413 tid 3 2024-04-12T11:19:21.184 INFO:tasks.rados.rados.0.smithi029.stdout:1173: rollback oid 6 current snap is 150 2024-04-12T11:19:21.184 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 6 to 107 2024-04-12T11:19:21.184 INFO:tasks.rados.rados.0.smithi029.stdout:1170: finishing write tid 2 to smithi02979277-30 2024-04-12T11:19:21.184 INFO:tasks.rados.rados.0.smithi029.stdout:1170: finishing write tid 3 to smithi02979277-30 2024-04-12T11:19:21.184 INFO:tasks.rados.rados.0.smithi029.stdout:1170: finishing write tid 4 to smithi02979277-30 2024-04-12T11:19:21.184 INFO:tasks.rados.rados.0.smithi029.stdout:1170: finishing write tid 5 to smithi02979277-30 2024-04-12T11:19:21.184 INFO:tasks.rados.rados.0.smithi029.stdout:1170: oid 30 updating version 0 to 502 2024-04-12T11:19:21.185 INFO:tasks.rados.rados.0.smithi029.stdout:1170: oid 30 updating version 502 to 503 2024-04-12T11:19:21.185 INFO:tasks.rados.rados.0.smithi029.stdout:1170: oid 30 updating version 503 to 505 2024-04-12T11:19:21.185 INFO:tasks.rados.rados.0.smithi029.stdout:1170: oid 30 version 505 is already newer than 504 2024-04-12T11:19:21.185 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 505 (ObjNum 457 snap 150 seq_num 457) dirty exists 2024-04-12T11:19:21.185 INFO:tasks.rados.rados.0.smithi029.stdout:1170: left oid 30 (ObjNum 457 snap 150 seq_num 457) 2024-04-12T11:19:21.185 INFO:tasks.rados.rados.0.smithi029.stdout:1171: finishing write tid 1 to smithi02979277-40 2024-04-12T11:19:21.185 INFO:tasks.rados.rados.0.smithi029.stdout:1171: finishing write tid 2 to smithi02979277-40 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1171: finishing write tid 3 to smithi02979277-40 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1170: done (3 left) 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1174: write oid 39 current snap is 150 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1171: finishing write tid 4 to smithi02979277-40 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1171: finishing write tid 5 to smithi02979277-40 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1171: oid 40 updating version 0 to 733 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1171: oid 40 version 733 is already newer than 732 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1171: oid 40 version 733 is already newer than 730 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1171: oid 40 version 733 is already newer than 731 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 733 (ObjNum 458 snap 150 seq_num 458) dirty exists 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1171: left oid 40 (ObjNum 458 snap 150 seq_num 458) 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1172: finishing write tid 1 to smithi02979277-17 2024-04-12T11:19:21.186 INFO:tasks.rados.rados.0.smithi029.stdout:1172: finishing write tid 2 to smithi02979277-17 2024-04-12T11:19:21.187 INFO:tasks.rados.rados.0.smithi029.stdout:1174: seq_num 460 ranges {531220=795720,1935011=516810,3066106=420420} 2024-04-12T11:19:21.189 INFO:tasks.rados.rados.0.smithi029.stdout:1174: writing smithi02979277-39 from 531220 to 1326940 tid 1 2024-04-12T11:19:21.196 INFO:tasks.rados.rados.0.smithi029.stdout:1174: writing smithi02979277-39 from 1935011 to 2451821 tid 2 2024-04-12T11:19:21.202 INFO:tasks.rados.rados.0.smithi029.stdout:1174: writing smithi02979277-39 from 3066106 to 3486526 tid 3 2024-04-12T11:19:21.202 INFO:tasks.rados.rados.0.smithi029.stdout:1171: done (3 left) 2024-04-12T11:19:21.202 INFO:tasks.rados.rados.0.smithi029.stdout:1175: snap_create 2024-04-12T11:19:21.202 INFO:tasks.rados.rados.0.smithi029.stdout:1172: finishing write tid 3 to smithi02979277-17 2024-04-12T11:19:21.202 INFO:tasks.rados.rados.0.smithi029.stdout:1172: finishing write tid 4 to smithi02979277-17 2024-04-12T11:19:21.202 INFO:tasks.rados.rados.0.smithi029.stdout:1173: finishing rollback tid 0 to smithi02979277-6 2024-04-12T11:19:21.202 INFO:tasks.rados.rados.0.smithi029.stdout:1173: finishing rollback tid 1 to smithi02979277-6 2024-04-12T11:19:21.202 INFO:tasks.rados.rados.0.smithi029.stdout:1172: finishing write tid 5 to smithi02979277-17 2024-04-12T11:19:21.202 INFO:tasks.rados.rados.0.smithi029.stdout:1172: oid 17 updating version 0 to 618 2024-04-12T11:19:21.203 INFO:tasks.rados.rados.0.smithi029.stdout:1172: oid 17 version 618 is already newer than 616 2024-04-12T11:19:21.203 INFO:tasks.rados.rados.0.smithi029.stdout:1172: oid 17 version 618 is already newer than 617 2024-04-12T11:19:21.203 INFO:tasks.rados.rados.0.smithi029.stdout:1172: oid 17 version 618 is already newer than 615 2024-04-12T11:19:21.203 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 618 (ObjNum 459 snap 150 seq_num 459) dirty exists 2024-04-12T11:19:21.203 INFO:tasks.rados.rados.0.smithi029.stdout:1172: left oid 17 (ObjNum 459 snap 150 seq_num 459) 2024-04-12T11:19:21.203 INFO:tasks.rados.rados.0.smithi029.stdout:1173: finishing rollback tid 2 to smithi02979277-6 2024-04-12T11:19:21.203 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 269 (ObjNum 294 snap 93 seq_num 294) dirty exists 2024-04-12T11:19:21.203 INFO:tasks.rados.rados.0.smithi029.stdout:1174: finishing write tid 1 to smithi02979277-39 2024-04-12T11:19:21.203 INFO:tasks.rados.rados.0.smithi029.stdout:1174: finishing write tid 2 to smithi02979277-39 2024-04-12T11:19:21.204 INFO:tasks.rados.rados.0.smithi029.stdout:1174: finishing write tid 3 to smithi02979277-39 2024-04-12T11:19:21.204 INFO:tasks.rados.rados.0.smithi029.stdout:1174: finishing write tid 4 to smithi02979277-39 2024-04-12T11:19:21.204 INFO:tasks.rados.rados.0.smithi029.stdout:1174: finishing write tid 5 to smithi02979277-39 2024-04-12T11:19:21.204 INFO:tasks.rados.rados.0.smithi029.stdout:1174: oid 39 updating version 0 to 102 2024-04-12T11:19:21.204 INFO:tasks.rados.rados.0.smithi029.stdout:1174: oid 39 updating version 102 to 105 2024-04-12T11:19:21.204 INFO:tasks.rados.rados.0.smithi029.stdout:1174: oid 39 version 105 is already newer than 104 2024-04-12T11:19:21.204 INFO:tasks.rados.rados.0.smithi029.stdout:1174: oid 39 version 105 is already newer than 103 2024-04-12T11:19:21.204 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 105 (ObjNum 460 snap 150 seq_num 460) dirty exists 2024-04-12T11:19:21.204 INFO:tasks.rados.rados.0.smithi029.stdout:1174: left oid 39 (ObjNum 460 snap 150 seq_num 460) 2024-04-12T11:19:21.536 INFO:tasks.rados.rados.0.smithi029.stdout:1172: done (3 left) 2024-04-12T11:19:21.537 INFO:tasks.rados.rados.0.smithi029.stdout:1173: done (2 left) 2024-04-12T11:19:21.537 INFO:tasks.rados.rados.0.smithi029.stdout:1174: done (1 left) 2024-04-12T11:19:21.537 INFO:tasks.rados.rados.0.smithi029.stdout:1175: done (0 left) 2024-04-12T11:19:21.537 INFO:tasks.rados.rados.0.smithi029.stdout:1176: rollback oid 13 current snap is 151 2024-04-12T11:19:21.537 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 13 to 139 2024-04-12T11:19:21.537 INFO:tasks.rados.rados.0.smithi029.stdout:1177: read oid 18 snap 123 2024-04-12T11:19:21.537 INFO:tasks.rados.rados.0.smithi029.stdout:1177: expect (ObjNum 402 snap 123 seq_num 402) 2024-04-12T11:19:21.537 INFO:tasks.rados.rados.0.smithi029.stdout:1178: delete oid 24 current snap is 151 2024-04-12T11:19:21.538 INFO:tasks.rados.rados.0.smithi029.stdout:1178: done (2 left) 2024-04-12T11:19:21.538 INFO:tasks.rados.rados.0.smithi029.stdout:1179: snap_create 2024-04-12T11:19:21.539 INFO:tasks.rados.rados.0.smithi029.stdout:1176: finishing rollback tid 1 to smithi02979277-13 2024-04-12T11:19:21.539 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 0 (ObjNum 33 snap 0 seq_num 1777354544) dirty dne 2024-04-12T11:19:21.549 INFO:tasks.rados.rados.0.smithi029.stdout:1177: expect (ObjNum 402 snap 123 seq_num 402) 2024-04-12T11:19:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:21 smithi087 ceph-mon[32917]: osdmap e360: 8 total, 8 up, 8 in 2024-04-12T11:19:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:21 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:21 smithi029 ceph-mon[93650]: osdmap e360: 8 total, 8 up, 8 in 2024-04-12T11:19:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:21 smithi029 ceph-mon[97787]: osdmap e360: 8 total, 8 up, 8 in 2024-04-12T11:19:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:22.524 INFO:tasks.rados.rados.0.smithi029.stdout:1176: done (2 left) 2024-04-12T11:19:22.524 INFO:tasks.rados.rados.0.smithi029.stdout:1177: done (1 left) 2024-04-12T11:19:22.524 INFO:tasks.rados.rados.0.smithi029.stdout:1179: done (0 left) 2024-04-12T11:19:22.524 INFO:tasks.rados.rados.0.smithi029.stdout:1180: snap_remove snap 150 2024-04-12T11:19:22.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:22 smithi087 ceph-mon[32917]: pgmap v175: 89 pgs: 89 active+clean; 274 MiB data, 778 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:19:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:22 smithi087 ceph-mon[32917]: osdmap e361: 8 total, 8 up, 8 in 2024-04-12T11:19:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:22 smithi087 ceph-mon[32917]: osdmap e362: 8 total, 8 up, 8 in 2024-04-12T11:19:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:22.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:22 smithi029 ceph-mon[93650]: pgmap v175: 89 pgs: 89 active+clean; 274 MiB data, 778 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:19:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:22 smithi029 ceph-mon[93650]: osdmap e361: 8 total, 8 up, 8 in 2024-04-12T11:19:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:22 smithi029 ceph-mon[93650]: osdmap e362: 8 total, 8 up, 8 in 2024-04-12T11:19:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:22 smithi029 ceph-mon[97787]: pgmap v175: 89 pgs: 89 active+clean; 274 MiB data, 778 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:19:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:22 smithi029 ceph-mon[97787]: osdmap e361: 8 total, 8 up, 8 in 2024-04-12T11:19:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:22 smithi029 ceph-mon[97787]: osdmap e362: 8 total, 8 up, 8 in 2024-04-12T11:19:23.525 INFO:tasks.rados.rados.0.smithi029.stdout:1180: done (0 left) 2024-04-12T11:19:23.525 INFO:tasks.rados.rados.0.smithi029.stdout:1181: snap_remove snap 135 2024-04-12T11:19:23.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:23.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:24.532 INFO:tasks.rados.rados.0.smithi029.stdout:1181: done (0 left) 2024-04-12T11:19:24.532 INFO:tasks.rados.rados.0.smithi029.stdout:1182: rollback oid 2 current snap is 152 2024-04-12T11:19:24.532 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 2 to 151 2024-04-12T11:19:24.532 INFO:tasks.rados.rados.0.smithi029.stdout:1183: write oid 26 current snap is 152 2024-04-12T11:19:24.532 INFO:tasks.rados.rados.0.smithi029.stdout:1183: seq_num 462 ranges {564834=546218,1651975=520479,2720952=559317} 2024-04-12T11:19:24.538 INFO:tasks.rados.rados.0.smithi029.stdout:1183: writing smithi02979277-26 from 564834 to 1111052 tid 1 2024-04-12T11:19:24.545 INFO:tasks.rados.rados.0.smithi029.stdout:1183: writing smithi02979277-26 from 1651975 to 2172454 tid 2 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:1183: writing smithi02979277-26 from 2720952 to 3280269 tid 3 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:1184: rollback oid 44 current snap is 152 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 44 to 129 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:1182: finishing rollback tid 0 to smithi02979277-2 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:1182: finishing rollback tid 1 to smithi02979277-2 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:1182: finishing rollback tid 2 to smithi02979277-2 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 97 (ObjNum 401 snap 123 seq_num 401) dirty exists 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:1183: finishing write tid 1 to smithi02979277-26 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:1183: finishing write tid 2 to smithi02979277-26 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:1182: done (2 left) 2024-04-12T11:19:24.552 INFO:tasks.rados.rados.0.smithi029.stdout:1185: rollback oid 14 current snap is 152 2024-04-12T11:19:24.553 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 140 2024-04-12T11:19:24.553 INFO:tasks.rados.rados.0.smithi029.stdout:1186: snap_remove snap 137 2024-04-12T11:19:24.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:24 smithi029 ceph-mon[97787]: pgmap v178: 89 pgs: 89 active+clean; 269 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 21 op/s 2024-04-12T11:19:24.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:24 smithi029 ceph-mon[97787]: osdmap e363: 8 total, 8 up, 8 in 2024-04-12T11:19:24.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:24 smithi029 ceph-mon[93650]: pgmap v178: 89 pgs: 89 active+clean; 269 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 21 op/s 2024-04-12T11:19:24.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:24 smithi029 ceph-mon[93650]: osdmap e363: 8 total, 8 up, 8 in 2024-04-12T11:19:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:24 smithi087 ceph-mon[32917]: pgmap v178: 89 pgs: 89 active+clean; 269 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 21 op/s 2024-04-12T11:19:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:24 smithi087 ceph-mon[32917]: osdmap e363: 8 total, 8 up, 8 in 2024-04-12T11:19:25.100 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:19:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:19:24] "GET /metrics HTTP/1.1" 200 36241 "" "Prometheus/2.43.0" 2024-04-12T11:19:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:1186: done (3 left) 2024-04-12T11:19:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:1187: write oid 29 current snap is 152 2024-04-12T11:19:25.537 INFO:tasks.rados.rados.0.smithi029.stdout:1187: seq_num 463 ranges {581641=621467,1994104=780490,3432022=349619} 2024-04-12T11:19:25.544 INFO:tasks.rados.rados.0.smithi029.stdout:1187: writing smithi02979277-29 from 581641 to 1203108 tid 1 2024-04-12T11:19:25.554 INFO:tasks.rados.rados.0.smithi029.stdout:1187: writing smithi02979277-29 from 1994104 to 2774594 tid 2 2024-04-12T11:19:25.559 INFO:tasks.rados.rados.0.smithi029.stdout:1187: writing smithi02979277-29 from 3432022 to 3781641 tid 3 2024-04-12T11:19:25.559 INFO:tasks.rados.rados.0.smithi029.stdout:1188: snap_remove snap 147 2024-04-12T11:19:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:25 smithi087 ceph-mon[32917]: osdmap e364: 8 total, 8 up, 8 in 2024-04-12T11:19:26.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:25 smithi029 ceph-mon[93650]: osdmap e364: 8 total, 8 up, 8 in 2024-04-12T11:19:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:25 smithi029 ceph-mon[97787]: osdmap e364: 8 total, 8 up, 8 in 2024-04-12T11:19:26.543 INFO:tasks.rados.rados.0.smithi029.stdout:1188: done (4 left) 2024-04-12T11:19:26.543 INFO:tasks.rados.rados.0.smithi029.stdout:1189: rollback oid 41 current snap is 152 2024-04-12T11:19:26.543 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 149 2024-04-12T11:19:26.543 INFO:tasks.rados.rados.0.smithi029.stdout:1185: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:19:26.543 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 659 (ObjNum 33 snap 0 seq_num 4231417884) dirty dne 2024-04-12T11:19:26.543 INFO:tasks.rados.rados.0.smithi029.stdout:1184: finishing rollback tid 0 to smithi02979277-44 2024-04-12T11:19:26.543 INFO:tasks.rados.rados.0.smithi029.stdout:1183: finishing write tid 3 to smithi02979277-26 2024-04-12T11:19:26.543 INFO:tasks.rados.rados.0.smithi029.stdout:1184: finishing rollback tid 1 to smithi02979277-44 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:1183: finishing write tid 4 to smithi02979277-26 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:1184: finishing rollback tid 2 to smithi02979277-44 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 542 (ObjNum 408 snap 127 seq_num 408) dirty exists 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:1183: finishing write tid 5 to smithi02979277-26 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:1183: oid 26 updating version 0 to 736 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:1183: oid 26 updating version 736 to 741 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:1183: oid 26 updating version 741 to 742 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:1183: oid 26 updating version 742 to 743 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 743 (ObjNum 462 snap 152 seq_num 462) dirty exists 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:1183: left oid 26 (ObjNum 462 snap 152 seq_num 462) 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:1187: finishing write tid 1 to smithi02979277-29 2024-04-12T11:19:26.544 INFO:tasks.rados.rados.0.smithi029.stdout:1187: finishing write tid 2 to smithi02979277-29 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1187: finishing write tid 3 to smithi02979277-29 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1187: finishing write tid 4 to smithi02979277-29 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1187: finishing write tid 5 to smithi02979277-29 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1187: oid 29 updating version 0 to 427 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1187: oid 29 updating version 427 to 429 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1187: oid 29 version 429 is already newer than 424 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1187: oid 29 version 429 is already newer than 428 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 429 (ObjNum 463 snap 152 seq_num 463) dirty exists 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1187: left oid 29 (ObjNum 463 snap 152 seq_num 463) 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1183: done (4 left) 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1184: done (3 left) 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1185: done (2 left) 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1187: done (1 left) 2024-04-12T11:19:26.545 INFO:tasks.rados.rados.0.smithi029.stdout:1190: delete oid 2 current snap is 152 2024-04-12T11:19:26.549 INFO:tasks.rados.rados.0.smithi029.stdout:1190: done (1 left) 2024-04-12T11:19:26.549 INFO:tasks.rados.rados.0.smithi029.stdout:1191: rollback oid 22 current snap is 152 2024-04-12T11:19:26.549 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 22 to 139 2024-04-12T11:19:26.549 INFO:tasks.rados.rados.0.smithi029.stdout:1192: snap_create 2024-04-12T11:19:26.550 INFO:tasks.rados.rados.0.smithi029.stdout:1189: finishing rollback tid 0 to smithi02979277-41 2024-04-12T11:19:26.550 INFO:tasks.rados.rados.0.smithi029.stdout:1189: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:19:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:1189: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:19:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 550 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-04-12T11:19:26.551 INFO:tasks.rados.rados.0.smithi029.stdout:1191: finishing rollback tid 0 to smithi02979277-22 2024-04-12T11:19:26.553 INFO:tasks.rados.rados.0.smithi029.stdout:1191: finishing rollback tid 1 to smithi02979277-22 2024-04-12T11:19:26.553 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 384 (ObjNum 49 snap 0 seq_num 862924110) dirty dne 2024-04-12T11:19:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:26 smithi087 ceph-mon[32917]: pgmap v181: 89 pgs: 89 active+clean; 269 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 21 op/s 2024-04-12T11:19:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:26 smithi087 ceph-mon[32917]: osdmap e365: 8 total, 8 up, 8 in 2024-04-12T11:19:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:26 smithi029 ceph-mon[93650]: pgmap v181: 89 pgs: 89 active+clean; 269 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 21 op/s 2024-04-12T11:19:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:26 smithi029 ceph-mon[93650]: osdmap e365: 8 total, 8 up, 8 in 2024-04-12T11:19:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:26 smithi029 ceph-mon[97787]: pgmap v181: 89 pgs: 89 active+clean; 269 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 21 op/s 2024-04-12T11:19:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:26 smithi029 ceph-mon[97787]: osdmap e365: 8 total, 8 up, 8 in 2024-04-12T11:19:27.525 INFO:tasks.rados.rados.0.smithi029.stdout:1189: done (2 left) 2024-04-12T11:19:27.525 INFO:tasks.rados.rados.0.smithi029.stdout:1191: done (1 left) 2024-04-12T11:19:27.525 INFO:tasks.rados.rados.0.smithi029.stdout:1192: done (0 left) 2024-04-12T11:19:27.525 INFO:tasks.rados.rados.0.smithi029.stdout:1193: delete oid 11 current snap is 153 2024-04-12T11:19:27.527 INFO:tasks.rados.rados.0.smithi029.stdout:1193: done (0 left) 2024-04-12T11:19:27.527 INFO:tasks.rados.rados.0.smithi029.stdout:1194: delete oid 37 current snap is 153 2024-04-12T11:19:27.530 INFO:tasks.rados.rados.0.smithi029.stdout:1194: done (0 left) 2024-04-12T11:19:27.530 INFO:tasks.rados.rados.0.smithi029.stdout:1195: read oid 46 snap -1 2024-04-12T11:19:27.530 INFO:tasks.rados.rados.0.smithi029.stdout:1195: expect (ObjNum 398 snap 123 seq_num 398) 2024-04-12T11:19:27.530 INFO:tasks.rados.rados.0.smithi029.stdout:1196: rollback oid 6 current snap is 153 2024-04-12T11:19:27.530 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 6 to 140 2024-04-12T11:19:27.530 INFO:tasks.rados.rados.0.smithi029.stdout:1197: snap_create 2024-04-12T11:19:27.533 INFO:tasks.rados.rados.0.smithi029.stdout:1196: finishing rollback tid 0 to smithi02979277-6 2024-04-12T11:19:27.534 INFO:tasks.rados.rados.0.smithi029.stdout:1196: finishing rollback tid 1 to smithi02979277-6 2024-04-12T11:19:27.534 INFO:tasks.rados.rados.0.smithi029.stdout:1196: finishing rollback tid 2 to smithi02979277-6 2024-04-12T11:19:27.534 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 279 (ObjNum 424 snap 135 seq_num 424) dirty exists 2024-04-12T11:19:27.534 INFO:tasks.rados.rados.0.smithi029.stdout:1195: expect (ObjNum 398 snap 123 seq_num 398) 2024-04-12T11:19:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:27 smithi087 ceph-mon[32917]: osdmap e366: 8 total, 8 up, 8 in 2024-04-12T11:19:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:27 smithi087 ceph-mon[32917]: osdmap e367: 8 total, 8 up, 8 in 2024-04-12T11:19:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:27 smithi029 ceph-mon[97787]: osdmap e366: 8 total, 8 up, 8 in 2024-04-12T11:19:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:27 smithi029 ceph-mon[97787]: osdmap e367: 8 total, 8 up, 8 in 2024-04-12T11:19:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:27 smithi029 ceph-mon[93650]: osdmap e366: 8 total, 8 up, 8 in 2024-04-12T11:19:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:27 smithi029 ceph-mon[93650]: osdmap e367: 8 total, 8 up, 8 in 2024-04-12T11:19:28.526 INFO:tasks.rados.rados.0.smithi029.stdout:1195: done (2 left) 2024-04-12T11:19:28.526 INFO:tasks.rados.rados.0.smithi029.stdout:1196: done (1 left) 2024-04-12T11:19:28.526 INFO:tasks.rados.rados.0.smithi029.stdout:1197: done (0 left) 2024-04-12T11:19:28.526 INFO:tasks.rados.rados.0.smithi029.stdout:1198: write oid 29 current snap is 154 2024-04-12T11:19:28.526 INFO:tasks.rados.rados.0.smithi029.stdout:1198: seq_num 467 ranges {648869=638816,1828032=787828,3141714=532030} 2024-04-12T11:19:28.532 INFO:tasks.rados.rados.0.smithi029.stdout:1198: writing smithi02979277-29 from 648869 to 1287685 tid 1 2024-04-12T11:19:28.540 INFO:tasks.rados.rados.0.smithi029.stdout:1198: writing smithi02979277-29 from 1828032 to 2615860 tid 2 2024-04-12T11:19:28.546 INFO:tasks.rados.rados.0.smithi029.stdout:1198: writing smithi02979277-29 from 3141714 to 3673744 tid 3 2024-04-12T11:19:28.546 INFO:tasks.rados.rados.0.smithi029.stdout:1199: write oid 41 current snap is 154 2024-04-12T11:19:28.546 INFO:tasks.rados.rados.0.smithi029.stdout:1198: finishing write tid 1 to smithi02979277-29 2024-04-12T11:19:28.546 INFO:tasks.rados.rados.0.smithi029.stdout:1199: seq_num 468 ranges {665676=714065,1886514=531486,3169137=696539} 2024-04-12T11:19:28.554 INFO:tasks.rados.rados.0.smithi029.stdout:1199: writing smithi02979277-41 from 665676 to 1379741 tid 1 2024-04-12T11:19:28.559 INFO:tasks.rados.rados.0.smithi029.stdout:1199: writing smithi02979277-41 from 1886514 to 2418000 tid 2 2024-04-12T11:19:28.568 INFO:tasks.rados.rados.0.smithi029.stdout:1199: writing smithi02979277-41 from 3169137 to 3865676 tid 3 2024-04-12T11:19:28.568 INFO:tasks.rados.rados.0.smithi029.stdout:1200: read oid 24 snap -1 2024-04-12T11:19:28.568 INFO:tasks.rados.rados.0.smithi029.stdout:1200: expect deleted 2024-04-12T11:19:28.568 INFO:tasks.rados.rados.0.smithi029.stdout:1201: snap_remove snap 143 2024-04-12T11:19:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:28 smithi087 ceph-mon[32917]: pgmap v184: 89 pgs: 89 active+clean; 270 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 825 KiB/s wr, 5 op/s 2024-04-12T11:19:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:28 smithi087 ceph-mon[32917]: osdmap e368: 8 total, 8 up, 8 in 2024-04-12T11:19:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:28 smithi029 ceph-mon[93650]: pgmap v184: 89 pgs: 89 active+clean; 270 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 825 KiB/s wr, 5 op/s 2024-04-12T11:19:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:28 smithi029 ceph-mon[93650]: osdmap e368: 8 total, 8 up, 8 in 2024-04-12T11:19:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:28 smithi029 ceph-mon[97787]: pgmap v184: 89 pgs: 89 active+clean; 270 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 825 KiB/s wr, 5 op/s 2024-04-12T11:19:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:28 smithi029 ceph-mon[97787]: osdmap e368: 8 total, 8 up, 8 in 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:1201: done (3 left) 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:1202: snap_create 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:1198: finishing write tid 2 to smithi02979277-29 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:1198: finishing write tid 3 to smithi02979277-29 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:1198: finishing write tid 4 to smithi02979277-29 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:1198: finishing write tid 5 to smithi02979277-29 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:1198: oid 29 updating version 0 to 433 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:1198: oid 29 updating version 433 to 434 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:1198: oid 29 updating version 434 to 435 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:1198: oid 29 updating version 435 to 436 2024-04-12T11:19:29.555 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 436 (ObjNum 467 snap 154 seq_num 467) dirty exists 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1198: left oid 29 (ObjNum 467 snap 154 seq_num 467) 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: finishing write tid 1 to smithi02979277-41 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: finishing write tid 2 to smithi02979277-41 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: finishing write tid 3 to smithi02979277-41 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: finishing write tid 4 to smithi02979277-41 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: finishing write tid 5 to smithi02979277-41 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: oid 41 updating version 0 to 561 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: oid 41 version 561 is already newer than 558 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: oid 41 version 561 is already newer than 560 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: oid 41 version 561 is already newer than 559 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 561 (ObjNum 468 snap 154 seq_num 468) dirty exists 2024-04-12T11:19:29.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: left oid 41 (ObjNum 468 snap 154 seq_num 468) 2024-04-12T11:19:30.556 INFO:tasks.rados.rados.0.smithi029.stdout:1198: done (3 left) 2024-04-12T11:19:30.556 INFO:tasks.rados.rados.0.smithi029.stdout:1199: done (2 left) 2024-04-12T11:19:30.557 INFO:tasks.rados.rados.0.smithi029.stdout:1200: done (1 left) 2024-04-12T11:19:30.557 INFO:tasks.rados.rados.0.smithi029.stdout:1202: done (0 left) 2024-04-12T11:19:30.557 INFO:tasks.rados.rados.0.smithi029.stdout:1203: read oid 37 snap -1 2024-04-12T11:19:30.557 INFO:tasks.rados.rados.0.smithi029.stdout:1203: expect deleted 2024-04-12T11:19:30.557 INFO:tasks.rados.rados.0.smithi029.stdout:1204: snap_remove snap 148 2024-04-12T11:19:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:30 smithi087 ceph-mon[32917]: pgmap v187: 89 pgs: 89 active+clean; 270 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 825 KiB/s wr, 5 op/s 2024-04-12T11:19:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:30 smithi087 ceph-mon[32917]: osdmap e369: 8 total, 8 up, 8 in 2024-04-12T11:19:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:30 smithi029 ceph-mon[97787]: pgmap v187: 89 pgs: 89 active+clean; 270 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 825 KiB/s wr, 5 op/s 2024-04-12T11:19:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:30 smithi029 ceph-mon[97787]: osdmap e369: 8 total, 8 up, 8 in 2024-04-12T11:19:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:30 smithi029 ceph-mon[93650]: pgmap v187: 89 pgs: 89 active+clean; 270 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 825 KiB/s wr, 5 op/s 2024-04-12T11:19:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:30 smithi029 ceph-mon[93650]: osdmap e369: 8 total, 8 up, 8 in 2024-04-12T11:19:31.558 INFO:tasks.rados.rados.0.smithi029.stdout:1204: done (1 left) 2024-04-12T11:19:31.559 INFO:tasks.rados.rados.0.smithi029.stdout:1205: rollback oid 41 current snap is 155 2024-04-12T11:19:31.559 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 149 2024-04-12T11:19:31.559 INFO:tasks.rados.rados.0.smithi029.stdout:1203: done (1 left) 2024-04-12T11:19:31.559 INFO:tasks.rados.rados.0.smithi029.stdout:1206: rollback oid 10 current snap is 155 2024-04-12T11:19:31.559 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 10 to 129 2024-04-12T11:19:31.559 INFO:tasks.rados.rados.0.smithi029.stdout:1207: snap_remove snap 107 2024-04-12T11:19:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:31 smithi087 ceph-mon[32917]: osdmap e370: 8 total, 8 up, 8 in 2024-04-12T11:19:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:31 smithi029 ceph-mon[93650]: osdmap e370: 8 total, 8 up, 8 in 2024-04-12T11:19:32.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:31 smithi029 ceph-mon[97787]: osdmap e370: 8 total, 8 up, 8 in 2024-04-12T11:19:32.561 INFO:tasks.rados.rados.0.smithi029.stdout:1205: finishing rollback tid 0 to smithi02979277-41 2024-04-12T11:19:32.561 INFO:tasks.rados.rados.0.smithi029.stdout:1205: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:1205: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 573 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:1206: finishing rollback tid 0 to smithi02979277-10 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:1206: finishing rollback tid 1 to smithi02979277-10 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:1206: finishing rollback tid 2 to smithi02979277-10 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 502 (ObjNum 335 snap 103 seq_num 335) dirty exists 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:1205: done (2 left) 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:1206: done (1 left) 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:1207: done (0 left) 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:1208: write oid 39 current snap is 155 2024-04-12T11:19:32.562 INFO:tasks.rados.rados.0.smithi029.stdout:1208: seq_num 469 ranges {682483=789314,1944996=791497,3196560=554574} 2024-04-12T11:19:32.566 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:32.570 INFO:tasks.rados.rados.0.smithi029.stdout:1208: writing smithi02979277-39 from 682483 to 1471797 tid 1 2024-04-12T11:19:32.579 INFO:tasks.rados.rados.0.smithi029.stdout:1208: writing smithi02979277-39 from 1944996 to 2736493 tid 2 2024-04-12T11:19:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:1208: writing smithi02979277-39 from 3196560 to 3751134 tid 3 2024-04-12T11:19:32.585 INFO:tasks.rados.rados.0.smithi029.stdout:1209: snap_create 2024-04-12T11:19:32.585 INFO:tasks.rados.rados.0.smithi029.stdout:1208: finishing write tid 1 to smithi02979277-39 2024-04-12T11:19:32.585 INFO:tasks.rados.rados.0.smithi029.stdout:1208: finishing write tid 2 to smithi02979277-39 2024-04-12T11:19:32.587 INFO:tasks.rados.rados.0.smithi029.stdout:1208: finishing write tid 3 to smithi02979277-39 2024-04-12T11:19:32.587 INFO:tasks.rados.rados.0.smithi029.stdout:1208: finishing write tid 4 to smithi02979277-39 2024-04-12T11:19:32.587 INFO:tasks.rados.rados.0.smithi029.stdout:1208: finishing write tid 5 to smithi02979277-39 2024-04-12T11:19:32.588 INFO:tasks.rados.rados.0.smithi029.stdout:1208: oid 39 updating version 0 to 117 2024-04-12T11:19:32.588 INFO:tasks.rados.rados.0.smithi029.stdout:1208: oid 39 version 117 is already newer than 115 2024-04-12T11:19:32.588 INFO:tasks.rados.rados.0.smithi029.stdout:1208: oid 39 version 117 is already newer than 116 2024-04-12T11:19:32.588 INFO:tasks.rados.rados.0.smithi029.stdout:1208: oid 39 updating version 117 to 118 2024-04-12T11:19:32.588 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 118 (ObjNum 469 snap 155 seq_num 469) dirty exists 2024-04-12T11:19:32.588 INFO:tasks.rados.rados.0.smithi029.stdout:1208: left oid 39 (ObjNum 469 snap 155 seq_num 469) 2024-04-12T11:19:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:32 smithi087 ceph-mon[32917]: pgmap v190: 89 pgs: 89 active+clean; 270 MiB data, 792 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:19:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:32 smithi087 ceph-mon[32917]: osdmap e371: 8 total, 8 up, 8 in 2024-04-12T11:19:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:32 smithi087 ceph-mon[32917]: osdmap e372: 8 total, 8 up, 8 in 2024-04-12T11:19:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:32 smithi029 ceph-mon[93650]: pgmap v190: 89 pgs: 89 active+clean; 270 MiB data, 792 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:19:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:32 smithi029 ceph-mon[93650]: osdmap e371: 8 total, 8 up, 8 in 2024-04-12T11:19:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:32 smithi029 ceph-mon[93650]: osdmap e372: 8 total, 8 up, 8 in 2024-04-12T11:19:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:32 smithi029 ceph-mon[97787]: pgmap v190: 89 pgs: 89 active+clean; 270 MiB data, 792 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:19:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:32 smithi029 ceph-mon[97787]: osdmap e371: 8 total, 8 up, 8 in 2024-04-12T11:19:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:32 smithi029 ceph-mon[97787]: osdmap e372: 8 total, 8 up, 8 in 2024-04-12T11:19:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:1208: done (1 left) 2024-04-12T11:19:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:1209: done (0 left) 2024-04-12T11:19:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:1210: rollback oid 29 current snap is 156 2024-04-12T11:19:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 29 to 145 2024-04-12T11:19:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:1211: rollback oid 34 current snap is 156 2024-04-12T11:19:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 141 2024-04-12T11:19:33.575 INFO:tasks.rados.rados.0.smithi029.stdout:1212: delete oid 2 current snap is 156 2024-04-12T11:19:33.576 INFO:tasks.rados.rados.0.smithi029.stdout:1212: done (2 left) 2024-04-12T11:19:33.577 INFO:tasks.rados.rados.0.smithi029.stdout:1213: write oid 14 current snap is 156 2024-04-12T11:19:33.577 INFO:tasks.rados.rados.0.smithi029.stdout:1213: seq_num 471 ranges {716097=539812,1945607=795166,3535053=381044} 2024-04-12T11:19:33.583 INFO:tasks.rados.rados.0.smithi029.stdout:1213: writing smithi02979277-14 from 716097 to 1255909 tid 1 2024-04-12T11:19:33.592 INFO:tasks.rados.rados.0.smithi029.stdout:1213: writing smithi02979277-14 from 1945607 to 2740773 tid 2 2024-04-12T11:19:33.598 INFO:tasks.rados.rados.0.smithi029.stdout:1213: writing smithi02979277-14 from 3535053 to 3916097 tid 3 2024-04-12T11:19:33.598 INFO:tasks.rados.rados.0.smithi029.stdout:1214: snap_remove snap 140 2024-04-12T11:19:33.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:33.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:34.583 INFO:tasks.rados.rados.0.smithi029.stdout:1210: finishing rollback tid 0 to smithi02979277-29 2024-04-12T11:19:34.583 INFO:tasks.rados.rados.0.smithi029.stdout:1210: finishing rollback tid 1 to smithi02979277-29 2024-04-12T11:19:34.583 INFO:tasks.rados.rados.0.smithi029.stdout:1210: finishing rollback tid 2 to smithi02979277-29 2024-04-12T11:19:34.583 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 450 (ObjNum 443 snap 141 seq_num 443) dirty exists 2024-04-12T11:19:34.583 INFO:tasks.rados.rados.0.smithi029.stdout:1211: finishing rollback tid 0 to smithi02979277-34 2024-04-12T11:19:34.583 INFO:tasks.rados.rados.0.smithi029.stdout:1211: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:19:34.583 INFO:tasks.rados.rados.0.smithi029.stdout:1211: finishing rollback tid 2 to smithi02979277-34 2024-04-12T11:19:34.583 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 454 (ObjNum 310 snap 94 seq_num 310) dirty exists 2024-04-12T11:19:34.583 INFO:tasks.rados.rados.0.smithi029.stdout:1213: finishing write tid 1 to smithi02979277-14 2024-04-12T11:19:34.583 INFO:tasks.rados.rados.0.smithi029.stdout:1213: finishing write tid 2 to smithi02979277-14 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1213: finishing write tid 3 to smithi02979277-14 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1213: finishing write tid 4 to smithi02979277-14 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1213: finishing write tid 5 to smithi02979277-14 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1213: oid 14 updating version 0 to 739 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1213: oid 14 updating version 739 to 740 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1213: oid 14 version 740 is already newer than 738 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1213: oid 14 updating version 740 to 741 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 741 (ObjNum 471 snap 156 seq_num 471) dirty exists 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1213: left oid 14 (ObjNum 471 snap 156 seq_num 471) 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1210: done (3 left) 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1211: done (2 left) 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1213: done (1 left) 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1214: done (0 left) 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1215: write oid 44 current snap is 156 2024-04-12T11:19:34.584 INFO:tasks.rados.rados.0.smithi029.stdout:1215: seq_num 472 ranges {732904=731414,2120442=538824,3278829=504743} 2024-04-12T11:19:34.591 INFO:tasks.rados.rados.0.smithi029.stdout:1215: writing smithi02979277-44 from 732904 to 1464318 tid 1 2024-04-12T11:19:34.598 INFO:tasks.rados.rados.0.smithi029.stdout:1215: writing smithi02979277-44 from 2120442 to 2659266 tid 2 2024-04-12T11:19:34.605 INFO:tasks.rados.rados.0.smithi029.stdout:1215: writing smithi02979277-44 from 3278829 to 3783572 tid 3 2024-04-12T11:19:34.605 INFO:tasks.rados.rados.0.smithi029.stdout:1216: rollback oid 42 current snap is 156 2024-04-12T11:19:34.605 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 151 2024-04-12T11:19:34.605 INFO:tasks.rados.rados.0.smithi029.stdout:1215: finishing write tid 1 to smithi02979277-44 2024-04-12T11:19:34.605 INFO:tasks.rados.rados.0.smithi029.stdout:1215: finishing write tid 2 to smithi02979277-44 2024-04-12T11:19:34.605 INFO:tasks.rados.rados.0.smithi029.stdout:1217: snap_remove snap 152 2024-04-12T11:19:34.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:34 smithi029 ceph-mon[97787]: pgmap v193: 89 pgs: 89 active+clean; 280 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 953 KiB/s wr, 6 op/s 2024-04-12T11:19:34.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:34 smithi029 ceph-mon[97787]: osdmap e373: 8 total, 8 up, 8 in 2024-04-12T11:19:34.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:34 smithi029 ceph-mon[93650]: pgmap v193: 89 pgs: 89 active+clean; 280 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 953 KiB/s wr, 6 op/s 2024-04-12T11:19:34.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:34 smithi029 ceph-mon[93650]: osdmap e373: 8 total, 8 up, 8 in 2024-04-12T11:19:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:34 smithi087 ceph-mon[32917]: pgmap v193: 89 pgs: 89 active+clean; 280 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 953 KiB/s wr, 6 op/s 2024-04-12T11:19:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:34 smithi087 ceph-mon[32917]: osdmap e373: 8 total, 8 up, 8 in 2024-04-12T11:19:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:19:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:19:34] "GET /metrics HTTP/1.1" 200 36241 "" "Prometheus/2.43.0" 2024-04-12T11:19:35.589 INFO:tasks.rados.rados.0.smithi029.stdout:1215: finishing write tid 3 to smithi02979277-44 2024-04-12T11:19:35.589 INFO:tasks.rados.rados.0.smithi029.stdout:1217: done (2 left) 2024-04-12T11:19:35.589 INFO:tasks.rados.rados.0.smithi029.stdout:1218: delete oid 45 current snap is 156 2024-04-12T11:19:35.589 INFO:tasks.rados.rados.0.smithi029.stdout:1216: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:19:35.589 INFO:tasks.rados.rados.0.smithi029.stdout:1215: finishing write tid 4 to smithi02979277-44 2024-04-12T11:19:35.589 INFO:tasks.rados.rados.0.smithi029.stdout:1216: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:19:35.590 INFO:tasks.rados.rados.0.smithi029.stdout:1216: finishing rollback tid 2 to smithi02979277-42 2024-04-12T11:19:35.590 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 190 (ObjNum 322 snap 100 seq_num 322) dirty exists 2024-04-12T11:19:35.590 INFO:tasks.rados.rados.0.smithi029.stdout:1215: finishing write tid 5 to smithi02979277-44 2024-04-12T11:19:35.590 INFO:tasks.rados.rados.0.smithi029.stdout:1215: oid 44 updating version 0 to 594 2024-04-12T11:19:35.590 INFO:tasks.rados.rados.0.smithi029.stdout:1215: oid 44 version 594 is already newer than 592 2024-04-12T11:19:35.590 INFO:tasks.rados.rados.0.smithi029.stdout:1215: oid 44 version 594 is already newer than 589 2024-04-12T11:19:35.590 INFO:tasks.rados.rados.0.smithi029.stdout:1215: oid 44 version 594 is already newer than 593 2024-04-12T11:19:35.590 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 594 (ObjNum 472 snap 156 seq_num 472) dirty exists 2024-04-12T11:19:35.590 INFO:tasks.rados.rados.0.smithi029.stdout:1215: left oid 44 (ObjNum 472 snap 156 seq_num 472) 2024-04-12T11:19:35.594 INFO:tasks.rados.rados.0.smithi029.stdout:1215: done (2 left) 2024-04-12T11:19:35.594 INFO:tasks.rados.rados.0.smithi029.stdout:1216: done (1 left) 2024-04-12T11:19:35.594 INFO:tasks.rados.rados.0.smithi029.stdout:1218: done (0 left) 2024-04-12T11:19:35.594 INFO:tasks.rados.rados.0.smithi029.stdout:1219: write oid 50 current snap is 156 2024-04-12T11:19:35.594 INFO:tasks.rados.rados.0.smithi029.stdout:1219: seq_num 474 ranges {766518=481912,1837406=542493,2933675=527287,3876584=89934} 2024-04-12T11:19:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:1219: writing smithi02979277-50 from 766518 to 1248430 tid 1 2024-04-12T11:19:35.607 INFO:tasks.rados.rados.0.smithi029.stdout:1219: writing smithi02979277-50 from 1837406 to 2379899 tid 2 2024-04-12T11:19:35.613 INFO:tasks.rados.rados.0.smithi029.stdout:1219: writing smithi02979277-50 from 2933675 to 3460962 tid 3 2024-04-12T11:19:35.616 INFO:tasks.rados.rados.0.smithi029.stdout:1219: writing smithi02979277-50 from 3876584 to 3966518 tid 4 2024-04-12T11:19:35.616 INFO:tasks.rados.rados.0.smithi029.stdout:1220: read oid 31 snap 134 2024-04-12T11:19:35.616 INFO:tasks.rados.rados.0.smithi029.stdout:1220: expect (ObjNum 393 snap 123 seq_num 393) 2024-04-12T11:19:35.616 INFO:tasks.rados.rados.0.smithi029.stdout:1221: write oid 34 current snap is 156 2024-04-12T11:19:35.616 INFO:tasks.rados.rados.0.smithi029.stdout:1219: finishing write tid 1 to smithi02979277-50 2024-04-12T11:19:35.616 INFO:tasks.rados.rados.0.smithi029.stdout:1221: seq_num 475 ranges {783325=557161,1779535=686151,3244745=454912} 2024-04-12T11:19:35.623 INFO:tasks.rados.rados.0.smithi029.stdout:1221: writing smithi02979277-34 from 783325 to 1340486 tid 1 2024-04-12T11:19:35.629 INFO:tasks.rados.rados.0.smithi029.stdout:1221: writing smithi02979277-34 from 1779535 to 2465686 tid 2 2024-04-12T11:19:35.636 INFO:tasks.rados.rados.0.smithi029.stdout:1221: writing smithi02979277-34 from 3244745 to 3699657 tid 3 2024-04-12T11:19:35.636 INFO:tasks.rados.rados.0.smithi029.stdout:1222: write oid 10 current snap is 156 2024-04-12T11:19:35.636 INFO:tasks.rados.rados.0.smithi029.stdout:1219: finishing write tid 2 to smithi02979277-50 2024-04-12T11:19:35.636 INFO:tasks.rados.rados.0.smithi029.stdout:1219: finishing write tid 3 to smithi02979277-50 2024-04-12T11:19:35.637 INFO:tasks.rados.rados.0.smithi029.stdout:1219: finishing write tid 4 to smithi02979277-50 2024-04-12T11:19:35.637 INFO:tasks.rados.rados.0.smithi029.stdout:1222: seq_num 476 ranges {400132=632410,1438017=546162} 2024-04-12T11:19:35.642 INFO:tasks.rados.rados.0.smithi029.stdout:1222: writing smithi02979277-10 from 400132 to 1032542 tid 1 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1222: writing smithi02979277-10 from 1438017 to 1984179 tid 2 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1223: write oid 42 current snap is 156 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1219: finishing write tid 5 to smithi02979277-50 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1219: finishing write tid 6 to smithi02979277-50 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1219: oid 50 updating version 0 to 397 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1219: oid 50 version 397 is already newer than 396 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1219: oid 50 version 397 is already newer than 394 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1219: oid 50 version 397 is already newer than 393 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1219: oid 50 version 397 is already newer than 395 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 397 (ObjNum 474 snap 156 seq_num 474) dirty exists 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1219: left oid 50 (ObjNum 474 snap 156 seq_num 474) 2024-04-12T11:19:35.648 INFO:tasks.rados.rados.0.smithi029.stdout:1220: expect (ObjNum 393 snap 123 seq_num 393) 2024-04-12T11:19:35.733 INFO:tasks.rados.rados.0.smithi029.stdout:1221: finishing write tid 1 to smithi02979277-34 2024-04-12T11:19:35.734 INFO:tasks.rados.rados.0.smithi029.stdout:1221: finishing write tid 2 to smithi02979277-34 2024-04-12T11:19:35.734 INFO:tasks.rados.rados.0.smithi029.stdout:1223: seq_num 477 ranges {416939=707659,1896499=120440} 2024-04-12T11:19:35.740 INFO:tasks.rados.rados.0.smithi029.stdout:1223: writing smithi02979277-42 from 416939 to 1124598 tid 1 2024-04-12T11:19:35.745 INFO:tasks.rados.rados.0.smithi029.stdout:1223: writing smithi02979277-42 from 1896499 to 2016939 tid 2 2024-04-12T11:19:35.745 INFO:tasks.rados.rados.0.smithi029.stdout:1219: done (4 left) 2024-04-12T11:19:35.745 INFO:tasks.rados.rados.0.smithi029.stdout:1220: done (3 left) 2024-04-12T11:19:35.745 INFO:tasks.rados.rados.0.smithi029.stdout:1224: snap_remove snap 155 2024-04-12T11:19:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:35 smithi087 ceph-mon[32917]: osdmap e374: 8 total, 8 up, 8 in 2024-04-12T11:19:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:35 smithi029 ceph-mon[93650]: osdmap e374: 8 total, 8 up, 8 in 2024-04-12T11:19:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:35 smithi029 ceph-mon[97787]: osdmap e374: 8 total, 8 up, 8 in 2024-04-12T11:19:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1224: done (3 left) 2024-04-12T11:19:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1225: read oid 46 snap -1 2024-04-12T11:19:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1225: expect (ObjNum 398 snap 123 seq_num 398) 2024-04-12T11:19:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1226: delete oid 17 current snap is 156 2024-04-12T11:19:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1221: finishing write tid 3 to smithi02979277-34 2024-04-12T11:19:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1221: finishing write tid 4 to smithi02979277-34 2024-04-12T11:19:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1221: finishing write tid 5 to smithi02979277-34 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1221: oid 34 updating version 0 to 462 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1221: oid 34 version 462 is already newer than 461 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1221: oid 34 updating version 462 to 463 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1221: oid 34 updating version 463 to 464 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 464 (ObjNum 475 snap 156 seq_num 475) dirty exists 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1221: left oid 34 (ObjNum 475 snap 156 seq_num 475) 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1222: finishing write tid 1 to smithi02979277-10 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1222: finishing write tid 2 to smithi02979277-10 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1222: finishing write tid 3 to smithi02979277-10 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1222: finishing write tid 4 to smithi02979277-10 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1222: oid 10 updating version 0 to 519 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1222: oid 10 version 519 is already newer than 518 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1222: oid 10 updating version 519 to 520 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 520 (ObjNum 476 snap 156 seq_num 476) dirty exists 2024-04-12T11:19:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1222: left oid 10 (ObjNum 476 snap 156 seq_num 476) 2024-04-12T11:19:36.592 INFO:tasks.rados.rados.0.smithi029.stdout:1223: finishing write tid 1 to smithi02979277-42 2024-04-12T11:19:36.592 INFO:tasks.rados.rados.0.smithi029.stdout:1223: finishing write tid 2 to smithi02979277-42 2024-04-12T11:19:36.592 INFO:tasks.rados.rados.0.smithi029.stdout:1223: finishing write tid 3 to smithi02979277-42 2024-04-12T11:19:36.592 INFO:tasks.rados.rados.0.smithi029.stdout:1223: finishing write tid 4 to smithi02979277-42 2024-04-12T11:19:36.592 INFO:tasks.rados.rados.0.smithi029.stdout:1223: oid 42 updating version 0 to 194 2024-04-12T11:19:36.592 INFO:tasks.rados.rados.0.smithi029.stdout:1223: oid 42 version 194 is already newer than 193 2024-04-12T11:19:36.592 INFO:tasks.rados.rados.0.smithi029.stdout:1223: oid 42 updating version 194 to 195 2024-04-12T11:19:36.592 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 195 (ObjNum 477 snap 156 seq_num 477) dirty exists 2024-04-12T11:19:36.592 INFO:tasks.rados.rados.0.smithi029.stdout:1223: left oid 42 (ObjNum 477 snap 156 seq_num 477) 2024-04-12T11:19:36.600 INFO:tasks.rados.rados.0.smithi029.stdout:1225: expect (ObjNum 398 snap 123 seq_num 398) 2024-04-12T11:19:36.721 INFO:tasks.rados.rados.0.smithi029.stdout:1221: done (4 left) 2024-04-12T11:19:36.722 INFO:tasks.rados.rados.0.smithi029.stdout:1222: done (3 left) 2024-04-12T11:19:36.722 INFO:tasks.rados.rados.0.smithi029.stdout:1223: done (2 left) 2024-04-12T11:19:36.722 INFO:tasks.rados.rados.0.smithi029.stdout:1225: done (1 left) 2024-04-12T11:19:36.722 INFO:tasks.rados.rados.0.smithi029.stdout:1226: done (0 left) 2024-04-12T11:19:36.722 INFO:tasks.rados.rados.0.smithi029.stdout:1227: write oid 2 current snap is 156 2024-04-12T11:19:36.722 INFO:tasks.rados.rados.0.smithi029.stdout:1227: seq_num 479 ranges {450553=574510,1613463=437090} 2024-04-12T11:19:36.727 INFO:tasks.rados.rados.0.smithi029.stdout:1227: writing smithi02979277-2 from 450553 to 1025063 tid 1 2024-04-12T11:19:36.732 INFO:tasks.rados.rados.0.smithi029.stdout:1227: writing smithi02979277-2 from 1613463 to 2050553 tid 2 2024-04-12T11:19:36.733 INFO:tasks.rados.rados.0.smithi029.stdout:1228: snap_remove snap 139 2024-04-12T11:19:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:36 smithi087 ceph-mon[32917]: pgmap v196: 89 pgs: 89 active+clean; 280 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 953 KiB/s wr, 6 op/s 2024-04-12T11:19:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:36 smithi087 ceph-mon[32917]: osdmap e375: 8 total, 8 up, 8 in 2024-04-12T11:19:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:36 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:36 smithi029 ceph-mon[93650]: pgmap v196: 89 pgs: 89 active+clean; 280 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 953 KiB/s wr, 6 op/s 2024-04-12T11:19:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:36 smithi029 ceph-mon[93650]: osdmap e375: 8 total, 8 up, 8 in 2024-04-12T11:19:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:36 smithi029 ceph-mon[97787]: pgmap v196: 89 pgs: 89 active+clean; 280 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 953 KiB/s wr, 6 op/s 2024-04-12T11:19:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:36 smithi029 ceph-mon[97787]: osdmap e375: 8 total, 8 up, 8 in 2024-04-12T11:19:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:37.528 INFO:tasks.rados.rados.0.smithi029.stdout:1228: done (1 left) 2024-04-12T11:19:37.528 INFO:tasks.rados.rados.0.smithi029.stdout:1229: rollback oid 34 current snap is 156 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 153 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1227: finishing write tid 1 to smithi02979277-2 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1227: finishing write tid 2 to smithi02979277-2 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1227: finishing write tid 3 to smithi02979277-2 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1227: finishing write tid 4 to smithi02979277-2 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1227: oid 2 updating version 0 to 115 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1227: oid 2 version 115 is already newer than 114 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1227: oid 2 updating version 115 to 116 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 116 (ObjNum 479 snap 156 seq_num 479) dirty exists 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1227: left oid 2 (ObjNum 479 snap 156 seq_num 479) 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1227: done (1 left) 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1230: read oid 15 snap -1 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1230: expect (ObjNum 438 snap 139 seq_num 438) 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:1231: rollback oid 31 current snap is 156 2024-04-12T11:19:37.529 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 31 to 153 2024-04-12T11:19:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:1232: read oid 25 snap -1 2024-04-12T11:19:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:1232: expect (ObjNum 400 snap 123 seq_num 400) 2024-04-12T11:19:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:1233: rollback oid 17 current snap is 156 2024-04-12T11:19:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 151 2024-04-12T11:19:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:1234: write oid 46 current snap is 156 2024-04-12T11:19:37.530 INFO:tasks.rados.rados.0.smithi029.stdout:1234: seq_num 480 ranges {467360=649759,1671945=395415} 2024-04-12T11:19:37.535 INFO:tasks.rados.rados.0.smithi029.stdout:1234: writing smithi02979277-46 from 467360 to 1117119 tid 1 2024-04-12T11:19:37.542 INFO:tasks.rados.rados.0.smithi029.stdout:1234: writing smithi02979277-46 from 1671945 to 2067360 tid 2 2024-04-12T11:19:37.542 INFO:tasks.rados.rados.0.smithi029.stdout:1235: read oid 27 snap -1 2024-04-12T11:19:37.542 INFO:tasks.rados.rados.0.smithi029.stdout:1235: expect (ObjNum 318 snap 97 seq_num 318) 2024-04-12T11:19:37.542 INFO:tasks.rados.rados.0.smithi029.stdout:1236: write oid 10 current snap is 156 2024-04-12T11:19:37.542 INFO:tasks.rados.rados.0.smithi029.stdout:1236: seq_num 481 ranges {484167=725008,1730427=353740} 2024-04-12T11:19:37.549 INFO:tasks.rados.rados.0.smithi029.stdout:1236: writing smithi02979277-10 from 484167 to 1209175 tid 1 2024-04-12T11:19:37.554 INFO:tasks.rados.rados.0.smithi029.stdout:1236: writing smithi02979277-10 from 1730427 to 2084167 tid 2 2024-04-12T11:19:37.554 INFO:tasks.rados.rados.0.smithi029.stdout:1237: write oid 29 current snap is 156 2024-04-12T11:19:37.554 INFO:tasks.rados.rados.0.smithi029.stdout:1237: seq_num 482 ranges {500974=400257,1388909=557169} 2024-04-12T11:19:37.559 INFO:tasks.rados.rados.0.smithi029.stdout:1237: writing smithi02979277-29 from 500974 to 901231 tid 1 2024-04-12T11:19:37.565 INFO:tasks.rados.rados.0.smithi029.stdout:1237: writing smithi02979277-29 from 1388909 to 1946078 tid 2 2024-04-12T11:19:37.565 INFO:tasks.rados.rados.0.smithi029.stdout:1238: write oid 36 current snap is 156 2024-04-12T11:19:37.565 INFO:tasks.rados.rados.0.smithi029.stdout:1238: seq_num 483 ranges {517781=475506,1731038=386743} 2024-04-12T11:19:37.570 INFO:tasks.rados.rados.0.smithi029.stdout:1238: writing smithi02979277-36 from 517781 to 993287 tid 1 2024-04-12T11:19:37.576 INFO:tasks.rados.rados.0.smithi029.stdout:1238: writing smithi02979277-36 from 1731038 to 2117781 tid 2 2024-04-12T11:19:37.577 INFO:tasks.rados.rados.0.smithi029.stdout:1239: snap_create 2024-04-12T11:19:37.577 INFO:tasks.rados.rados.0.smithi029.stdout:1231: finishing rollback tid 0 to smithi02979277-31 2024-04-12T11:19:37.577 INFO:tasks.rados.rados.0.smithi029.stdout:1231: finishing rollback tid 1 to smithi02979277-31 2024-04-12T11:19:37.577 INFO:tasks.rados.rados.0.smithi029.stdout:1231: finishing rollback tid 2 to smithi02979277-31 2024-04-12T11:19:37.577 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 530 (ObjNum 393 snap 123 seq_num 393) dirty exists 2024-04-12T11:19:37.577 INFO:tasks.rados.rados.0.smithi029.stdout:1230: expect (ObjNum 438 snap 139 seq_num 438) 2024-04-12T11:19:37.779 INFO:tasks.rados.rados.0.smithi029.stdout:1229: finishing rollback tid 0 to smithi02979277-34 2024-04-12T11:19:37.779 INFO:tasks.rados.rados.0.smithi029.stdout:1229: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:19:37.779 INFO:tasks.rados.rados.0.smithi029.stdout:1229: finishing rollback tid 2 to smithi02979277-34 2024-04-12T11:19:37.779 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 471 (ObjNum 310 snap 94 seq_num 310) dirty exists 2024-04-12T11:19:37.779 INFO:tasks.rados.rados.0.smithi029.stdout:1232: expect (ObjNum 400 snap 123 seq_num 400) 2024-04-12T11:19:37.870 INFO:tasks.rados.rados.0.smithi029.stdout:1233: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:19:37.870 INFO:tasks.rados.rados.0.smithi029.stdout:1233: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:19:37.870 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 634 (ObjNum 459 snap 150 seq_num 459) dirty exists 2024-04-12T11:19:37.870 INFO:tasks.rados.rados.0.smithi029.stdout:1234: finishing write tid 1 to smithi02979277-46 2024-04-12T11:19:37.870 INFO:tasks.rados.rados.0.smithi029.stdout:1234: finishing write tid 2 to smithi02979277-46 2024-04-12T11:19:37.870 INFO:tasks.rados.rados.0.smithi029.stdout:1234: finishing write tid 3 to smithi02979277-46 2024-04-12T11:19:37.870 INFO:tasks.rados.rados.0.smithi029.stdout:1234: finishing write tid 4 to smithi02979277-46 2024-04-12T11:19:37.870 INFO:tasks.rados.rados.0.smithi029.stdout:1234: oid 46 updating version 0 to 286 2024-04-12T11:19:37.871 INFO:tasks.rados.rados.0.smithi029.stdout:1234: oid 46 updating version 286 to 287 2024-04-12T11:19:37.871 INFO:tasks.rados.rados.0.smithi029.stdout:1234: oid 46 updating version 287 to 288 2024-04-12T11:19:37.871 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 288 (ObjNum 480 snap 156 seq_num 480) dirty exists 2024-04-12T11:19:37.871 INFO:tasks.rados.rados.0.smithi029.stdout:1234: left oid 46 (ObjNum 480 snap 156 seq_num 480) 2024-04-12T11:19:37.871 INFO:tasks.rados.rados.0.smithi029.stdout:1235: expect (ObjNum 318 snap 97 seq_num 318) 2024-04-12T11:19:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:37 smithi087 ceph-mon[32917]: osdmap e376: 8 total, 8 up, 8 in 2024-04-12T11:19:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:37 smithi087 ceph-mon[32917]: osdmap e377: 8 total, 8 up, 8 in 2024-04-12T11:19:38.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:37 smithi029 ceph-mon[93650]: osdmap e376: 8 total, 8 up, 8 in 2024-04-12T11:19:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:37 smithi029 ceph-mon[93650]: osdmap e377: 8 total, 8 up, 8 in 2024-04-12T11:19:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:37 smithi029 ceph-mon[97787]: osdmap e376: 8 total, 8 up, 8 in 2024-04-12T11:19:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:37 smithi029 ceph-mon[97787]: osdmap e377: 8 total, 8 up, 8 in 2024-04-12T11:19:38.039 INFO:tasks.rados.rados.0.smithi029.stdout:1236: finishing write tid 1 to smithi02979277-10 2024-04-12T11:19:38.039 INFO:tasks.rados.rados.0.smithi029.stdout:1236: finishing write tid 2 to smithi02979277-10 2024-04-12T11:19:38.039 INFO:tasks.rados.rados.0.smithi029.stdout:1236: finishing write tid 3 to smithi02979277-10 2024-04-12T11:19:38.039 INFO:tasks.rados.rados.0.smithi029.stdout:1236: finishing write tid 4 to smithi02979277-10 2024-04-12T11:19:38.039 INFO:tasks.rados.rados.0.smithi029.stdout:1236: oid 10 updating version 0 to 532 2024-04-12T11:19:38.039 INFO:tasks.rados.rados.0.smithi029.stdout:1236: oid 10 updating version 532 to 533 2024-04-12T11:19:38.039 INFO:tasks.rados.rados.0.smithi029.stdout:1236: oid 10 version 533 is already newer than 531 2024-04-12T11:19:38.039 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 533 (ObjNum 481 snap 156 seq_num 481) dirty exists 2024-04-12T11:19:38.039 INFO:tasks.rados.rados.0.smithi029.stdout:1236: left oid 10 (ObjNum 481 snap 156 seq_num 481) 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1237: finishing write tid 1 to smithi02979277-29 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1237: finishing write tid 2 to smithi02979277-29 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1237: finishing write tid 3 to smithi02979277-29 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1237: finishing write tid 4 to smithi02979277-29 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1237: oid 29 updating version 0 to 475 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1237: oid 29 version 475 is already newer than 474 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1237: oid 29 updating version 475 to 476 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 476 (ObjNum 482 snap 156 seq_num 482) dirty exists 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1237: left oid 29 (ObjNum 482 snap 156 seq_num 482) 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1238: finishing write tid 1 to smithi02979277-36 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1238: finishing write tid 2 to smithi02979277-36 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1238: finishing write tid 3 to smithi02979277-36 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1238: finishing write tid 4 to smithi02979277-36 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1238: oid 36 updating version 0 to 294 2024-04-12T11:19:38.040 INFO:tasks.rados.rados.0.smithi029.stdout:1238: oid 36 updating version 294 to 296 2024-04-12T11:19:38.041 INFO:tasks.rados.rados.0.smithi029.stdout:1238: oid 36 version 296 is already newer than 295 2024-04-12T11:19:38.041 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 296 (ObjNum 483 snap 156 seq_num 483) dirty exists 2024-04-12T11:19:38.041 INFO:tasks.rados.rados.0.smithi029.stdout:1238: left oid 36 (ObjNum 483 snap 156 seq_num 483) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1229: done (10 left) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1230: done (9 left) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1231: done (8 left) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1232: done (7 left) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1233: done (6 left) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1234: done (5 left) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1235: done (4 left) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1236: done (3 left) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1237: done (2 left) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1238: done (1 left) 2024-04-12T11:19:38.529 INFO:tasks.rados.rados.0.smithi029.stdout:1239: done (0 left) 2024-04-12T11:19:38.530 INFO:tasks.rados.rados.0.smithi029.stdout:1240: delete oid 32 current snap is 157 2024-04-12T11:19:38.531 INFO:tasks.rados.rados.0.smithi029.stdout:1240: done (0 left) 2024-04-12T11:19:38.531 INFO:tasks.rados.rados.0.smithi029.stdout:1241: delete oid 41 current snap is 157 2024-04-12T11:19:38.534 INFO:tasks.rados.rados.0.smithi029.stdout:1241: done (0 left) 2024-04-12T11:19:38.535 INFO:tasks.rados.rados.0.smithi029.stdout:1242: snap_create 2024-04-12T11:19:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:38 smithi087 ceph-mon[32917]: pgmap v199: 89 pgs: 89 active+clean; 234 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-04-12T11:19:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:38 smithi087 ceph-mon[32917]: osdmap e378: 8 total, 8 up, 8 in 2024-04-12T11:19:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:38 smithi029 ceph-mon[93650]: pgmap v199: 89 pgs: 89 active+clean; 234 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-04-12T11:19:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:38 smithi029 ceph-mon[93650]: osdmap e378: 8 total, 8 up, 8 in 2024-04-12T11:19:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:38 smithi029 ceph-mon[97787]: pgmap v199: 89 pgs: 89 active+clean; 234 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-04-12T11:19:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:38 smithi029 ceph-mon[97787]: osdmap e378: 8 total, 8 up, 8 in 2024-04-12T11:19:39.530 INFO:tasks.rados.rados.0.smithi029.stdout:1242: done (0 left) 2024-04-12T11:19:39.530 INFO:tasks.rados.rados.0.smithi029.stdout:1243: delete oid 34 current snap is 158 2024-04-12T11:19:39.532 INFO:tasks.rados.rados.0.smithi029.stdout:1243: done (0 left) 2024-04-12T11:19:39.532 INFO:tasks.rados.rados.0.smithi029.stdout:1244: write oid 46 current snap is 158 2024-04-12T11:19:39.532 INFO:tasks.rados.rados.0.smithi029.stdout:1244: seq_num 487 ranges {585009=492855,1564966=620043} 2024-04-12T11:19:39.538 INFO:tasks.rados.rados.0.smithi029.stdout:1244: writing smithi02979277-46 from 585009 to 1077864 tid 1 2024-04-12T11:19:39.545 INFO:tasks.rados.rados.0.smithi029.stdout:1244: writing smithi02979277-46 from 1564966 to 2185009 tid 2 2024-04-12T11:19:39.545 INFO:tasks.rados.rados.0.smithi029.stdout:1245: write oid 48 current snap is 158 2024-04-12T11:19:39.545 INFO:tasks.rados.rados.0.smithi029.stdout:1244: finishing write tid 1 to smithi02979277-46 2024-04-12T11:19:39.545 INFO:tasks.rados.rados.0.smithi029.stdout:1245: seq_num 488 ranges {601816=568104,1623448=451823} 2024-04-12T11:19:39.551 INFO:tasks.rados.rados.0.smithi029.stdout:1245: writing smithi02979277-48 from 601816 to 1169920 tid 1 2024-04-12T11:19:39.557 INFO:tasks.rados.rados.0.smithi029.stdout:1245: writing smithi02979277-48 from 1623448 to 2075271 tid 2 2024-04-12T11:19:39.557 INFO:tasks.rados.rados.0.smithi029.stdout:1246: write oid 18 current snap is 158 2024-04-12T11:19:39.557 INFO:tasks.rados.rados.0.smithi029.stdout:1244: finishing write tid 2 to smithi02979277-46 2024-04-12T11:19:39.557 INFO:tasks.rados.rados.0.smithi029.stdout:1244: finishing write tid 3 to smithi02979277-46 2024-04-12T11:19:39.557 INFO:tasks.rados.rados.0.smithi029.stdout:1244: finishing write tid 4 to smithi02979277-46 2024-04-12T11:19:39.557 INFO:tasks.rados.rados.0.smithi029.stdout:1244: oid 46 updating version 0 to 290 2024-04-12T11:19:39.557 INFO:tasks.rados.rados.0.smithi029.stdout:1244: oid 46 updating version 290 to 291 2024-04-12T11:19:39.557 INFO:tasks.rados.rados.0.smithi029.stdout:1244: oid 46 updating version 291 to 292 2024-04-12T11:19:39.558 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 292 (ObjNum 487 snap 158 seq_num 487) dirty exists 2024-04-12T11:19:39.558 INFO:tasks.rados.rados.0.smithi029.stdout:1244: left oid 46 (ObjNum 487 snap 158 seq_num 487) 2024-04-12T11:19:39.558 INFO:tasks.rados.rados.0.smithi029.stdout:1246: seq_num 489 ranges {618623=643353,1681930=536693} 2024-04-12T11:19:39.564 INFO:tasks.rados.rados.0.smithi029.stdout:1246: writing smithi02979277-18 from 618623 to 1261976 tid 1 2024-04-12T11:19:39.569 INFO:tasks.rados.rados.0.smithi029.stdout:1246: writing smithi02979277-18 from 1681930 to 2218623 tid 2 2024-04-12T11:19:39.570 INFO:tasks.rados.rados.0.smithi029.stdout:1244: done (2 left) 2024-04-12T11:19:39.570 INFO:tasks.rados.rados.0.smithi029.stdout:1247: snap_remove snap 123 2024-04-12T11:19:40.536 INFO:tasks.rados.rados.0.smithi029.stdout:1247: done (2 left) 2024-04-12T11:19:40.536 INFO:tasks.rados.rados.0.smithi029.stdout:1248: write oid 20 current snap is 158 2024-04-12T11:19:40.536 INFO:tasks.rados.rados.0.smithi029.stdout:1245: finishing write tid 1 to smithi02979277-48 2024-04-12T11:19:40.536 INFO:tasks.rados.rados.0.smithi029.stdout:1248: seq_num 490 ranges {635430=718602,2140412=95018} 2024-04-12T11:19:40.544 INFO:tasks.rados.rados.0.smithi029.stdout:1248: writing smithi02979277-20 from 635430 to 1354032 tid 1 2024-04-12T11:19:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1248: writing smithi02979277-20 from 2140412 to 2235430 tid 2 2024-04-12T11:19:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1249: snap_create 2024-04-12T11:19:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1245: finishing write tid 2 to smithi02979277-48 2024-04-12T11:19:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1245: finishing write tid 3 to smithi02979277-48 2024-04-12T11:19:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1245: finishing write tid 4 to smithi02979277-48 2024-04-12T11:19:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1245: oid 48 updating version 0 to 537 2024-04-12T11:19:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1245: oid 48 version 537 is already newer than 536 2024-04-12T11:19:40.550 INFO:tasks.rados.rados.0.smithi029.stdout:1245: oid 48 version 537 is already newer than 535 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 537 (ObjNum 488 snap 158 seq_num 488) dirty exists 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:1245: left oid 48 (ObjNum 488 snap 158 seq_num 488) 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:1246: finishing write tid 1 to smithi02979277-18 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:1246: finishing write tid 2 to smithi02979277-18 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:1246: finishing write tid 3 to smithi02979277-18 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:1246: finishing write tid 4 to smithi02979277-18 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:1246: oid 18 updating version 0 to 639 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:1246: oid 18 version 639 is already newer than 638 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:1246: oid 18 updating version 639 to 640 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 640 (ObjNum 489 snap 158 seq_num 489) dirty exists 2024-04-12T11:19:40.551 INFO:tasks.rados.rados.0.smithi029.stdout:1246: left oid 18 (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:19:40.552 INFO:tasks.rados.rados.0.smithi029.stdout:1248: finishing write tid 1 to smithi02979277-20 2024-04-12T11:19:40.552 INFO:tasks.rados.rados.0.smithi029.stdout:1248: finishing write tid 2 to smithi02979277-20 2024-04-12T11:19:40.552 INFO:tasks.rados.rados.0.smithi029.stdout:1248: finishing write tid 3 to smithi02979277-20 2024-04-12T11:19:40.552 INFO:tasks.rados.rados.0.smithi029.stdout:1248: finishing write tid 4 to smithi02979277-20 2024-04-12T11:19:40.552 INFO:tasks.rados.rados.0.smithi029.stdout:1248: oid 20 updating version 0 to 300 2024-04-12T11:19:40.553 INFO:tasks.rados.rados.0.smithi029.stdout:1248: oid 20 version 300 is already newer than 298 2024-04-12T11:19:40.553 INFO:tasks.rados.rados.0.smithi029.stdout:1248: oid 20 version 300 is already newer than 299 2024-04-12T11:19:40.553 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 300 (ObjNum 490 snap 158 seq_num 490) dirty exists 2024-04-12T11:19:40.553 INFO:tasks.rados.rados.0.smithi029.stdout:1248: left oid 20 (ObjNum 490 snap 158 seq_num 490) 2024-04-12T11:19:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:40 smithi087 ceph-mon[32917]: pgmap v202: 89 pgs: 89 active+clean; 234 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-04-12T11:19:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:40 smithi087 ceph-mon[32917]: osdmap e379: 8 total, 8 up, 8 in 2024-04-12T11:19:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:40 smithi029 ceph-mon[97787]: pgmap v202: 89 pgs: 89 active+clean; 234 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-04-12T11:19:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:40 smithi029 ceph-mon[97787]: osdmap e379: 8 total, 8 up, 8 in 2024-04-12T11:19:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:40 smithi029 ceph-mon[93650]: pgmap v202: 89 pgs: 89 active+clean; 234 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-04-12T11:19:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:40 smithi029 ceph-mon[93650]: osdmap e379: 8 total, 8 up, 8 in 2024-04-12T11:19:41.541 INFO:tasks.rados.rados.0.smithi029.stdout:1245: done (3 left) 2024-04-12T11:19:41.541 INFO:tasks.rados.rados.0.smithi029.stdout:1246: done (2 left) 2024-04-12T11:19:41.542 INFO:tasks.rados.rados.0.smithi029.stdout:1248: done (1 left) 2024-04-12T11:19:41.542 INFO:tasks.rados.rados.0.smithi029.stdout:1249: done (0 left) 2024-04-12T11:19:41.542 INFO:tasks.rados.rados.0.smithi029.stdout:1250: snap_remove snap 154 2024-04-12T11:19:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:41 smithi087 ceph-mon[32917]: osdmap e380: 8 total, 8 up, 8 in 2024-04-12T11:19:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:41 smithi029 ceph-mon[97787]: osdmap e380: 8 total, 8 up, 8 in 2024-04-12T11:19:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:41 smithi029 ceph-mon[93650]: osdmap e380: 8 total, 8 up, 8 in 2024-04-12T11:19:42.529 INFO:tasks.rados.rados.0.smithi029.stdout:1250: done (0 left) 2024-04-12T11:19:42.529 INFO:tasks.rados.rados.0.smithi029.stdout:1251: read oid 18 snap -1 2024-04-12T11:19:42.530 INFO:tasks.rados.rados.0.smithi029.stdout:1251: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:19:42.530 INFO:tasks.rados.rados.0.smithi029.stdout:1252: snap_remove snap 134 2024-04-12T11:19:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:42 smithi087 ceph-mon[32917]: pgmap v205: 89 pgs: 89 active+clean; 234 MiB data, 732 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:19:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:42 smithi087 ceph-mon[32917]: osdmap e381: 8 total, 8 up, 8 in 2024-04-12T11:19:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:42 smithi087 ceph-mon[32917]: osdmap e382: 8 total, 8 up, 8 in 2024-04-12T11:19:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:42 smithi029 ceph-mon[93650]: pgmap v205: 89 pgs: 89 active+clean; 234 MiB data, 732 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:19:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:42 smithi029 ceph-mon[93650]: osdmap e381: 8 total, 8 up, 8 in 2024-04-12T11:19:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:42 smithi029 ceph-mon[93650]: osdmap e382: 8 total, 8 up, 8 in 2024-04-12T11:19:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:42 smithi029 ceph-mon[97787]: pgmap v205: 89 pgs: 89 active+clean; 234 MiB data, 732 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:19:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:42 smithi029 ceph-mon[97787]: osdmap e381: 8 total, 8 up, 8 in 2024-04-12T11:19:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:42 smithi029 ceph-mon[97787]: osdmap e382: 8 total, 8 up, 8 in 2024-04-12T11:19:43.531 INFO:tasks.rados.rados.0.smithi029.stdout:1252: done (1 left) 2024-04-12T11:19:43.531 INFO:tasks.rados.rados.0.smithi029.stdout:1253: rollback oid 32 current snap is 159 2024-04-12T11:19:43.532 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 32 to 157 2024-04-12T11:19:43.532 INFO:tasks.rados.rados.0.smithi029.stdout:1254: write oid 23 current snap is 159 2024-04-12T11:19:43.532 INFO:tasks.rados.rados.0.smithi029.stdout:1254: seq_num 491 ranges {652237=793851,2198894=53343} 2024-04-12T11:19:43.540 INFO:tasks.rados.rados.0.smithi029.stdout:1254: writing smithi02979277-23 from 652237 to 1446088 tid 1 2024-04-12T11:19:43.545 INFO:tasks.rados.rados.0.smithi029.stdout:1254: writing smithi02979277-23 from 2198894 to 2252237 tid 2 2024-04-12T11:19:43.545 INFO:tasks.rados.rados.0.smithi029.stdout:1255: write oid 17 current snap is 159 2024-04-12T11:19:43.545 INFO:tasks.rados.rados.0.smithi029.stdout:1255: seq_num 492 ranges {669044=469100,1741023=459161} 2024-04-12T11:19:43.551 INFO:tasks.rados.rados.0.smithi029.stdout:1255: writing smithi02979277-17 from 669044 to 1138144 tid 1 2024-04-12T11:19:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1255: writing smithi02979277-17 from 1741023 to 2200184 tid 2 2024-04-12T11:19:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1256: delete oid 5 current snap is 159 2024-04-12T11:19:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1251: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:19:43.695 INFO:tasks.rados.rados.0.smithi029.stdout:1253: finishing rollback tid 1 to smithi02979277-32 2024-04-12T11:19:43.695 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 478 (ObjNum 945 snap 0 seq_num 1800972896) dirty dne 2024-04-12T11:19:43.695 INFO:tasks.rados.rados.0.smithi029.stdout:1254: finishing write tid 1 to smithi02979277-23 2024-04-12T11:19:43.695 INFO:tasks.rados.rados.0.smithi029.stdout:1254: finishing write tid 2 to smithi02979277-23 2024-04-12T11:19:43.695 INFO:tasks.rados.rados.0.smithi029.stdout:1254: finishing write tid 3 to smithi02979277-23 2024-04-12T11:19:43.695 INFO:tasks.rados.rados.0.smithi029.stdout:1254: finishing write tid 4 to smithi02979277-23 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1254: oid 23 updating version 0 to 400 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1254: oid 23 version 400 is already newer than 398 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1254: oid 23 version 400 is already newer than 399 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 400 (ObjNum 491 snap 159 seq_num 491) dirty exists 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1254: left oid 23 (ObjNum 491 snap 159 seq_num 491) 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1255: finishing write tid 1 to smithi02979277-17 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1255: finishing write tid 2 to smithi02979277-17 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1255: finishing write tid 3 to smithi02979277-17 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1251: done (4 left) 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1253: done (3 left) 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1254: done (2 left) 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1256: done (1 left) 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:1257: rollback oid 26 current snap is 159 2024-04-12T11:19:43.696 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 133 2024-04-12T11:19:43.697 INFO:tasks.rados.rados.0.smithi029.stdout:1255: finishing write tid 4 to smithi02979277-17 2024-04-12T11:19:43.697 INFO:tasks.rados.rados.0.smithi029.stdout:1255: oid 17 updating version 0 to 654 2024-04-12T11:19:43.697 INFO:tasks.rados.rados.0.smithi029.stdout:1255: oid 17 version 654 is already newer than 653 2024-04-12T11:19:43.697 INFO:tasks.rados.rados.0.smithi029.stdout:1255: oid 17 version 654 is already newer than 652 2024-04-12T11:19:43.697 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 654 (ObjNum 492 snap 159 seq_num 492) dirty exists 2024-04-12T11:19:43.697 INFO:tasks.rados.rados.0.smithi029.stdout:1255: left oid 17 (ObjNum 492 snap 159 seq_num 492) 2024-04-12T11:19:43.697 INFO:tasks.rados.rados.0.smithi029.stdout:1255: done (1 left) 2024-04-12T11:19:43.697 INFO:tasks.rados.rados.0.smithi029.stdout:1258: read oid 15 snap -1 2024-04-12T11:19:43.697 INFO:tasks.rados.rados.0.smithi029.stdout:1258: expect (ObjNum 438 snap 139 seq_num 438) 2024-04-12T11:19:43.697 INFO:tasks.rados.rados.0.smithi029.stdout:1259: delete oid 41 current snap is 159 2024-04-12T11:19:43.698 INFO:tasks.rados.rados.0.smithi029.stdout:1259: done (2 left) 2024-04-12T11:19:43.698 INFO:tasks.rados.rados.0.smithi029.stdout:1260: write oid 22 current snap is 159 2024-04-12T11:19:43.698 INFO:tasks.rados.rados.0.smithi029.stdout:1260: seq_num 495 ranges {719465=411200,1632822=686643} 2024-04-12T11:19:43.703 INFO:tasks.rados.rados.0.smithi029.stdout:1260: writing smithi02979277-22 from 719465 to 1130665 tid 1 2024-04-12T11:19:43.709 INFO:tasks.rados.rados.0.smithi029.stdout:1260: writing smithi02979277-22 from 1632822 to 2319465 tid 2 2024-04-12T11:19:43.709 INFO:tasks.rados.rados.0.smithi029.stdout:1261: snap_create 2024-04-12T11:19:43.709 INFO:tasks.rados.rados.0.smithi029.stdout:1258: expect (ObjNum 438 snap 139 seq_num 438) 2024-04-12T11:19:43.909 INFO:tasks.rados.rados.0.smithi029.stdout:1257: finishing rollback tid 0 to smithi02979277-26 2024-04-12T11:19:43.909 INFO:tasks.rados.rados.0.smithi029.stdout:1257: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:19:43.909 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 797 (ObjNum 81 snap 0 seq_num 862860015) dirty dne 2024-04-12T11:19:43.909 INFO:tasks.rados.rados.0.smithi029.stdout:1260: finishing write tid 1 to smithi02979277-22 2024-04-12T11:19:43.909 INFO:tasks.rados.rados.0.smithi029.stdout:1260: finishing write tid 2 to smithi02979277-22 2024-04-12T11:19:43.910 INFO:tasks.rados.rados.0.smithi029.stdout:1260: finishing write tid 3 to smithi02979277-22 2024-04-12T11:19:43.910 INFO:tasks.rados.rados.0.smithi029.stdout:1260: finishing write tid 4 to smithi02979277-22 2024-04-12T11:19:43.910 INFO:tasks.rados.rados.0.smithi029.stdout:1260: oid 22 updating version 0 to 403 2024-04-12T11:19:43.910 INFO:tasks.rados.rados.0.smithi029.stdout:1260: oid 22 version 403 is already newer than 401 2024-04-12T11:19:43.910 INFO:tasks.rados.rados.0.smithi029.stdout:1260: oid 22 version 403 is already newer than 402 2024-04-12T11:19:43.910 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 403 (ObjNum 495 snap 159 seq_num 495) dirty exists 2024-04-12T11:19:43.910 INFO:tasks.rados.rados.0.smithi029.stdout:1260: left oid 22 (ObjNum 495 snap 159 seq_num 495) 2024-04-12T11:19:43.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:44.546 INFO:tasks.rados.rados.0.smithi029.stdout:1257: done (3 left) 2024-04-12T11:19:44.546 INFO:tasks.rados.rados.0.smithi029.stdout:1258: done (2 left) 2024-04-12T11:19:44.546 INFO:tasks.rados.rados.0.smithi029.stdout:1260: done (1 left) 2024-04-12T11:19:44.546 INFO:tasks.rados.rados.0.smithi029.stdout:1261: done (0 left) 2024-04-12T11:19:44.547 INFO:tasks.rados.rados.0.smithi029.stdout:1262: write oid 44 current snap is 160 2024-04-12T11:19:44.547 INFO:tasks.rados.rados.0.smithi029.stdout:1262: seq_num 496 ranges {736272=486449,1974951=361321} 2024-04-12T11:19:44.552 INFO:tasks.rados.rados.0.smithi029.stdout:1262: writing smithi02979277-44 from 736272 to 1222721 tid 1 2024-04-12T11:19:44.559 INFO:tasks.rados.rados.0.smithi029.stdout:1262: writing smithi02979277-44 from 1974951 to 2336272 tid 2 2024-04-12T11:19:44.560 INFO:tasks.rados.rados.0.smithi029.stdout:1263: read oid 49 snap 151 2024-04-12T11:19:44.560 INFO:tasks.rados.rados.0.smithi029.stdout:1263: expect (ObjNum 448 snap 147 seq_num 448) 2024-04-12T11:19:44.560 INFO:tasks.rados.rados.0.smithi029.stdout:1264: rollback oid 32 current snap is 160 2024-04-12T11:19:44.560 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 32 to 159 2024-04-12T11:19:44.560 INFO:tasks.rados.rados.0.smithi029.stdout:1262: finishing write tid 1 to smithi02979277-44 2024-04-12T11:19:44.560 INFO:tasks.rados.rados.0.smithi029.stdout:1265: read oid 5 snap -1 2024-04-12T11:19:44.560 INFO:tasks.rados.rados.0.smithi029.stdout:1265: expect deleted 2024-04-12T11:19:44.560 INFO:tasks.rados.rados.0.smithi029.stdout:1266: snap_remove snap 153 2024-04-12T11:19:44.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:44 smithi029 ceph-mon[97787]: pgmap v208: 89 pgs: 89 active+clean; 223 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-04-12T11:19:44.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:44 smithi029 ceph-mon[97787]: osdmap e383: 8 total, 8 up, 8 in 2024-04-12T11:19:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:44 smithi029 ceph-mon[93650]: pgmap v208: 89 pgs: 89 active+clean; 223 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-04-12T11:19:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:44 smithi029 ceph-mon[93650]: osdmap e383: 8 total, 8 up, 8 in 2024-04-12T11:19:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:44 smithi087 ceph-mon[32917]: pgmap v208: 89 pgs: 89 active+clean; 223 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-04-12T11:19:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:44 smithi087 ceph-mon[32917]: osdmap e383: 8 total, 8 up, 8 in 2024-04-12T11:19:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:19:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:19:44] "GET /metrics HTTP/1.1" 200 36252 "" "Prometheus/2.43.0" 2024-04-12T11:19:45.552 INFO:tasks.rados.rados.0.smithi029.stdout:1266: done (4 left) 2024-04-12T11:19:45.552 INFO:tasks.rados.rados.0.smithi029.stdout:1267: delete oid 37 current snap is 160 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:1262: finishing write tid 2 to smithi02979277-44 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:1262: finishing write tid 3 to smithi02979277-44 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:1262: finishing write tid 4 to smithi02979277-44 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:1262: oid 44 updating version 0 to 626 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:1262: oid 44 updating version 626 to 627 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:1262: oid 44 version 627 is already newer than 625 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 627 (ObjNum 496 snap 160 seq_num 496) dirty exists 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:1262: left oid 44 (ObjNum 496 snap 160 seq_num 496) 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:1264: finishing rollback tid 1 to smithi02979277-32 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 478 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-04-12T11:19:45.553 INFO:tasks.rados.rados.0.smithi029.stdout:1263: expect (ObjNum 448 snap 147 seq_num 448) 2024-04-12T11:19:45.722 INFO:tasks.rados.rados.0.smithi029.stdout:1262: done (4 left) 2024-04-12T11:19:45.722 INFO:tasks.rados.rados.0.smithi029.stdout:1263: done (3 left) 2024-04-12T11:19:45.722 INFO:tasks.rados.rados.0.smithi029.stdout:1264: done (2 left) 2024-04-12T11:19:45.722 INFO:tasks.rados.rados.0.smithi029.stdout:1265: done (1 left) 2024-04-12T11:19:45.722 INFO:tasks.rados.rados.0.smithi029.stdout:1267: done (0 left) 2024-04-12T11:19:45.722 INFO:tasks.rados.rados.0.smithi029.stdout:1268: snap_remove snap 151 2024-04-12T11:19:45.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:45 smithi087 ceph-mon[32917]: osdmap e384: 8 total, 8 up, 8 in 2024-04-12T11:19:46.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:45 smithi029 ceph-mon[93650]: osdmap e384: 8 total, 8 up, 8 in 2024-04-12T11:19:46.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:45 smithi029 ceph-mon[97787]: osdmap e384: 8 total, 8 up, 8 in 2024-04-12T11:19:46.557 INFO:tasks.rados.rados.0.smithi029.stdout:1268: done (0 left) 2024-04-12T11:19:46.557 INFO:tasks.rados.rados.0.smithi029.stdout:1269: snap_create 2024-04-12T11:19:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:46 smithi087 ceph-mon[32917]: pgmap v211: 89 pgs: 89 active+clean; 223 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-04-12T11:19:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:46 smithi087 ceph-mon[32917]: osdmap e385: 8 total, 8 up, 8 in 2024-04-12T11:19:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:46 smithi029 ceph-mon[97787]: pgmap v211: 89 pgs: 89 active+clean; 223 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-04-12T11:19:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:46 smithi029 ceph-mon[97787]: osdmap e385: 8 total, 8 up, 8 in 2024-04-12T11:19:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:46 smithi029 ceph-mon[93650]: pgmap v211: 89 pgs: 89 active+clean; 223 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-04-12T11:19:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:46 smithi029 ceph-mon[93650]: osdmap e385: 8 total, 8 up, 8 in 2024-04-12T11:19:47.562 INFO:tasks.rados.rados.0.smithi029.stdout:1269: done (0 left) 2024-04-12T11:19:47.563 INFO:tasks.rados.rados.0.smithi029.stdout:1270: delete oid 27 current snap is 161 2024-04-12T11:19:47.568 INFO:tasks.rados.rados.0.smithi029.stdout:1270: done (0 left) 2024-04-12T11:19:47.568 INFO:tasks.rados.rados.0.smithi029.stdout:1271: read oid 6 snap 158 2024-04-12T11:19:47.568 INFO:tasks.rados.rados.0.smithi029.stdout:1271: expect (ObjNum 424 snap 135 seq_num 424) 2024-04-12T11:19:47.568 INFO:tasks.rados.rados.0.smithi029.stdout:1272: write oid 24 current snap is 161 2024-04-12T11:19:47.568 INFO:tasks.rados.rados.0.smithi029.stdout:1272: seq_num 499 ranges {786693=712196,2150397=236296} 2024-04-12T11:19:47.575 INFO:tasks.rados.rados.0.smithi029.stdout:1272: writing smithi02979277-24 from 786693 to 1498889 tid 1 2024-04-12T11:19:47.581 INFO:tasks.rados.rados.0.smithi029.stdout:1272: writing smithi02979277-24 from 2150397 to 2386693 tid 2 2024-04-12T11:19:47.581 INFO:tasks.rados.rados.0.smithi029.stdout:1273: write oid 7 current snap is 161 2024-04-12T11:19:47.581 INFO:tasks.rados.rados.0.smithi029.stdout:1273: seq_num 500 ranges {403500=787445,1692526=473837} 2024-04-12T11:19:47.588 INFO:tasks.rados.rados.0.smithi029.stdout:1273: writing smithi02979277-7 from 403500 to 1190945 tid 1 2024-04-12T11:19:47.594 INFO:tasks.rados.rados.0.smithi029.stdout:1273: writing smithi02979277-7 from 1692526 to 2166363 tid 2 2024-04-12T11:19:47.594 INFO:tasks.rados.rados.0.smithi029.stdout:1274: snap_create 2024-04-12T11:19:47.594 INFO:tasks.rados.rados.0.smithi029.stdout:1271: expect (ObjNum 424 snap 135 seq_num 424) 2024-04-12T11:19:47.761 INFO:tasks.rados.rados.0.smithi029.stdout:1272: finishing write tid 1 to smithi02979277-24 2024-04-12T11:19:47.761 INFO:tasks.rados.rados.0.smithi029.stdout:1272: finishing write tid 2 to smithi02979277-24 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1272: finishing write tid 3 to smithi02979277-24 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1272: finishing write tid 4 to smithi02979277-24 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1272: oid 24 updating version 0 to 802 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1272: oid 24 updating version 802 to 804 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1272: oid 24 version 804 is already newer than 803 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 804 (ObjNum 499 snap 161 seq_num 499) dirty exists 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1272: left oid 24 (ObjNum 499 snap 161 seq_num 499) 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1273: finishing write tid 1 to smithi02979277-7 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1273: finishing write tid 2 to smithi02979277-7 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1273: finishing write tid 3 to smithi02979277-7 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1273: finishing write tid 4 to smithi02979277-7 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1273: oid 7 updating version 0 to 607 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1273: oid 7 updating version 607 to 609 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:1273: oid 7 version 609 is already newer than 608 2024-04-12T11:19:47.762 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 609 (ObjNum 500 snap 161 seq_num 500) dirty exists 2024-04-12T11:19:47.763 INFO:tasks.rados.rados.0.smithi029.stdout:1273: left oid 7 (ObjNum 500 snap 161 seq_num 500) 2024-04-12T11:19:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:47 smithi087 ceph-mon[32917]: osdmap e386: 8 total, 8 up, 8 in 2024-04-12T11:19:48.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:47 smithi029 ceph-mon[97787]: osdmap e386: 8 total, 8 up, 8 in 2024-04-12T11:19:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:47 smithi029 ceph-mon[93650]: osdmap e386: 8 total, 8 up, 8 in 2024-04-12T11:19:48.567 INFO:tasks.rados.rados.0.smithi029.stdout:1271: done (3 left) 2024-04-12T11:19:48.567 INFO:tasks.rados.rados.0.smithi029.stdout:1272: done (2 left) 2024-04-12T11:19:48.567 INFO:tasks.rados.rados.0.smithi029.stdout:1273: done (1 left) 2024-04-12T11:19:48.567 INFO:tasks.rados.rados.0.smithi029.stdout:1274: done (0 left) 2024-04-12T11:19:48.567 INFO:tasks.rados.rados.0.smithi029.stdout:1275: rollback oid 42 current snap is 162 2024-04-12T11:19:48.567 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 161 2024-04-12T11:19:48.568 INFO:tasks.rados.rados.0.smithi029.stdout:1276: rollback oid 39 current snap is 162 2024-04-12T11:19:48.568 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 130 2024-04-12T11:19:48.568 INFO:tasks.rados.rados.0.smithi029.stdout:1277: write oid 21 current snap is 162 2024-04-12T11:19:48.568 INFO:tasks.rados.rados.0.smithi029.stdout:1277: seq_num 501 ranges {420307=462694,1351008=617495} 2024-04-12T11:19:48.571 INFO:tasks.rados.rados.0.smithi029.stdout:1277: writing smithi02979277-21 from 420307 to 883001 tid 1 2024-04-12T11:19:48.578 INFO:tasks.rados.rados.0.smithi029.stdout:1277: writing smithi02979277-21 from 1351008 to 1968503 tid 2 2024-04-12T11:19:48.578 INFO:tasks.rados.rados.0.smithi029.stdout:1278: read oid 50 snap -1 2024-04-12T11:19:48.578 INFO:tasks.rados.rados.0.smithi029.stdout:1278: expect (ObjNum 474 snap 156 seq_num 474) 2024-04-12T11:19:48.578 INFO:tasks.rados.rados.0.smithi029.stdout:1279: write oid 38 current snap is 162 2024-04-12T11:19:48.579 INFO:tasks.rados.rados.0.smithi029.stdout:1275: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:19:48.579 INFO:tasks.rados.rados.0.smithi029.stdout:1275: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:19:48.579 INFO:tasks.rados.rados.0.smithi029.stdout:1279: seq_num 502 ranges {437114=654296,1525843=477506} 2024-04-12T11:19:48.584 INFO:tasks.rados.rados.0.smithi029.stdout:1279: writing smithi02979277-38 from 437114 to 1091410 tid 1 2024-04-12T11:19:48.591 INFO:tasks.rados.rados.0.smithi029.stdout:1279: writing smithi02979277-38 from 1525843 to 2003349 tid 2 2024-04-12T11:19:48.591 INFO:tasks.rados.rados.0.smithi029.stdout:1275: finishing rollback tid 2 to smithi02979277-42 2024-04-12T11:19:48.591 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 213 (ObjNum 477 snap 156 seq_num 477) dirty exists 2024-04-12T11:19:48.591 INFO:tasks.rados.rados.0.smithi029.stdout:1276: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:19:48.591 INFO:tasks.rados.rados.0.smithi029.stdout:1276: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:19:48.591 INFO:tasks.rados.rados.0.smithi029.stdout:1276: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:19:48.591 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 138 (ObjNum 388 snap 121 seq_num 388) dirty exists 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1277: finishing write tid 1 to smithi02979277-21 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1280: read oid 29 snap -1 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1280: expect (ObjNum 482 snap 156 seq_num 482) 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1275: done (5 left) 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1276: done (4 left) 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1281: write oid 19 current snap is 162 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1277: finishing write tid 2 to smithi02979277-21 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1277: finishing write tid 3 to smithi02979277-21 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1277: finishing write tid 4 to smithi02979277-21 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1277: oid 21 updating version 0 to 638 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1277: oid 21 version 638 is already newer than 637 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1277: oid 21 version 638 is already newer than 636 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 638 (ObjNum 501 snap 162 seq_num 501) dirty exists 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1277: left oid 21 (ObjNum 501 snap 162 seq_num 501) 2024-04-12T11:19:48.592 INFO:tasks.rados.rados.0.smithi029.stdout:1279: finishing write tid 1 to smithi02979277-38 2024-04-12T11:19:48.593 INFO:tasks.rados.rados.0.smithi029.stdout:1281: seq_num 503 ranges {453921=729545,1584325=621164} 2024-04-12T11:19:48.597 INFO:tasks.rados.rados.0.smithi029.stdout:1281: writing smithi02979277-19 from 453921 to 1183466 tid 1 2024-04-12T11:19:48.603 INFO:tasks.rados.rados.0.smithi029.stdout:1281: writing smithi02979277-19 from 1584325 to 2205489 tid 2 2024-04-12T11:19:48.603 INFO:tasks.rados.rados.0.smithi029.stdout:1277: done (4 left) 2024-04-12T11:19:48.603 INFO:tasks.rados.rados.0.smithi029.stdout:1282: write oid 30 current snap is 162 2024-04-12T11:19:48.603 INFO:tasks.rados.rados.0.smithi029.stdout:1282: seq_num 504 ranges {470728=404794,1526454=481175} 2024-04-12T11:19:48.608 INFO:tasks.rados.rados.0.smithi029.stdout:1282: writing smithi02979277-30 from 470728 to 875522 tid 1 2024-04-12T11:19:48.614 INFO:tasks.rados.rados.0.smithi029.stdout:1282: writing smithi02979277-30 from 1526454 to 2007629 tid 2 2024-04-12T11:19:48.615 INFO:tasks.rados.rados.0.smithi029.stdout:1283: read oid 18 snap 158 2024-04-12T11:19:48.615 INFO:tasks.rados.rados.0.smithi029.stdout:1283: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:19:48.615 INFO:tasks.rados.rados.0.smithi029.stdout:1284: snap_create 2024-04-12T11:19:48.615 INFO:tasks.rados.rados.0.smithi029.stdout:1280: expect (ObjNum 482 snap 156 seq_num 482) 2024-04-12T11:19:48.714 INFO:tasks.rados.rados.0.smithi029.stdout:1279: finishing write tid 2 to smithi02979277-38 2024-04-12T11:19:48.715 INFO:tasks.rados.rados.0.smithi029.stdout:1279: finishing write tid 3 to smithi02979277-38 2024-04-12T11:19:48.715 INFO:tasks.rados.rados.0.smithi029.stdout:1279: finishing write tid 4 to smithi02979277-38 2024-04-12T11:19:48.715 INFO:tasks.rados.rados.0.smithi029.stdout:1279: oid 38 updating version 0 to 321 2024-04-12T11:19:48.715 INFO:tasks.rados.rados.0.smithi029.stdout:1279: oid 38 version 321 is already newer than 320 2024-04-12T11:19:48.715 INFO:tasks.rados.rados.0.smithi029.stdout:1279: oid 38 updating version 321 to 322 2024-04-12T11:19:48.715 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 322 (ObjNum 502 snap 162 seq_num 502) dirty exists 2024-04-12T11:19:48.715 INFO:tasks.rados.rados.0.smithi029.stdout:1279: left oid 38 (ObjNum 502 snap 162 seq_num 502) 2024-04-12T11:19:48.715 INFO:tasks.rados.rados.0.smithi029.stdout:1278: expect (ObjNum 474 snap 156 seq_num 474) 2024-04-12T11:19:48.848 INFO:tasks.rados.rados.0.smithi029.stdout:1281: finishing write tid 1 to smithi02979277-19 2024-04-12T11:19:48.848 INFO:tasks.rados.rados.0.smithi029.stdout:1281: finishing write tid 2 to smithi02979277-19 2024-04-12T11:19:48.848 INFO:tasks.rados.rados.0.smithi029.stdout:1281: finishing write tid 3 to smithi02979277-19 2024-04-12T11:19:48.848 INFO:tasks.rados.rados.0.smithi029.stdout:1281: finishing write tid 4 to smithi02979277-19 2024-04-12T11:19:48.848 INFO:tasks.rados.rados.0.smithi029.stdout:1281: oid 19 updating version 0 to 777 2024-04-12T11:19:48.848 INFO:tasks.rados.rados.0.smithi029.stdout:1281: oid 19 updating version 777 to 779 2024-04-12T11:19:48.848 INFO:tasks.rados.rados.0.smithi029.stdout:1281: oid 19 version 779 is already newer than 778 2024-04-12T11:19:48.848 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 779 (ObjNum 503 snap 162 seq_num 503) dirty exists 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:1281: left oid 19 (ObjNum 503 snap 162 seq_num 503) 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:1282: finishing write tid 1 to smithi02979277-30 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:1282: finishing write tid 2 to smithi02979277-30 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:1282: finishing write tid 3 to smithi02979277-30 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:1282: finishing write tid 4 to smithi02979277-30 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:1282: oid 30 updating version 0 to 525 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:1282: oid 30 updating version 525 to 527 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:1282: oid 30 version 527 is already newer than 526 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 527 (ObjNum 504 snap 162 seq_num 504) dirty exists 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:1282: left oid 30 (ObjNum 504 snap 162 seq_num 504) 2024-04-12T11:19:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:1283: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:19:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:48 smithi087 ceph-mon[32917]: pgmap v214: 89 pgs: 89 active+clean; 216 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 910 KiB/s wr, 12 op/s 2024-04-12T11:19:48.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:48 smithi087 ceph-mon[32917]: osdmap e387: 8 total, 8 up, 8 in 2024-04-12T11:19:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:48 smithi029 ceph-mon[93650]: pgmap v214: 89 pgs: 89 active+clean; 216 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 910 KiB/s wr, 12 op/s 2024-04-12T11:19:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:48 smithi029 ceph-mon[93650]: osdmap e387: 8 total, 8 up, 8 in 2024-04-12T11:19:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:48 smithi029 ceph-mon[97787]: pgmap v214: 89 pgs: 89 active+clean; 216 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 910 KiB/s wr, 12 op/s 2024-04-12T11:19:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:48 smithi029 ceph-mon[97787]: osdmap e387: 8 total, 8 up, 8 in 2024-04-12T11:19:49.572 INFO:tasks.rados.rados.0.smithi029.stdout:1278: done (6 left) 2024-04-12T11:19:49.572 INFO:tasks.rados.rados.0.smithi029.stdout:1279: done (5 left) 2024-04-12T11:19:49.572 INFO:tasks.rados.rados.0.smithi029.stdout:1280: done (4 left) 2024-04-12T11:19:49.572 INFO:tasks.rados.rados.0.smithi029.stdout:1281: done (3 left) 2024-04-12T11:19:49.572 INFO:tasks.rados.rados.0.smithi029.stdout:1282: done (2 left) 2024-04-12T11:19:49.572 INFO:tasks.rados.rados.0.smithi029.stdout:1283: done (1 left) 2024-04-12T11:19:49.573 INFO:tasks.rados.rados.0.smithi029.stdout:1284: done (0 left) 2024-04-12T11:19:49.573 INFO:tasks.rados.rados.0.smithi029.stdout:1285: snap_create 2024-04-12T11:19:49.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:49 smithi087 ceph-mon[32917]: osdmap e388: 8 total, 8 up, 8 in 2024-04-12T11:19:50.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:49 smithi029 ceph-mon[93650]: osdmap e388: 8 total, 8 up, 8 in 2024-04-12T11:19:50.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:49 smithi029 ceph-mon[97787]: osdmap e388: 8 total, 8 up, 8 in 2024-04-12T11:19:50.573 INFO:tasks.rados.rados.0.smithi029.stdout:1285: done (0 left) 2024-04-12T11:19:50.573 INFO:tasks.rados.rados.0.smithi029.stdout:1286: read oid 13 snap -1 2024-04-12T11:19:50.574 INFO:tasks.rados.rados.0.smithi029.stdout:1286: expect deleted 2024-04-12T11:19:50.574 INFO:tasks.rados.rados.0.smithi029.stdout:1287: read oid 44 snap -1 2024-04-12T11:19:50.574 INFO:tasks.rados.rados.0.smithi029.stdout:1287: expect (ObjNum 496 snap 160 seq_num 496) 2024-04-12T11:19:50.574 INFO:tasks.rados.rados.0.smithi029.stdout:1288: snap_remove snap 159 2024-04-12T11:19:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:50 smithi087 ceph-mon[32917]: pgmap v217: 89 pgs: 89 active+clean; 216 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 910 KiB/s wr, 12 op/s 2024-04-12T11:19:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:50 smithi087 ceph-mon[32917]: osdmap e389: 8 total, 8 up, 8 in 2024-04-12T11:19:51.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:50 smithi029 ceph-mon[97787]: pgmap v217: 89 pgs: 89 active+clean; 216 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 910 KiB/s wr, 12 op/s 2024-04-12T11:19:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:50 smithi029 ceph-mon[97787]: osdmap e389: 8 total, 8 up, 8 in 2024-04-12T11:19:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:50 smithi029 ceph-mon[93650]: pgmap v217: 89 pgs: 89 active+clean; 216 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 910 KiB/s wr, 12 op/s 2024-04-12T11:19:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:50 smithi029 ceph-mon[93650]: osdmap e389: 8 total, 8 up, 8 in 2024-04-12T11:19:51.584 INFO:tasks.rados.rados.0.smithi029.stdout:1288: done (2 left) 2024-04-12T11:19:51.584 INFO:tasks.rados.rados.0.smithi029.stdout:1289: read oid 35 snap -1 2024-04-12T11:19:51.584 INFO:tasks.rados.rados.0.smithi029.stdout:1289: expect (ObjNum 449 snap 147 seq_num 449) 2024-04-12T11:19:51.584 INFO:tasks.rados.rados.0.smithi029.stdout:1290: rollback oid 29 current snap is 164 2024-04-12T11:19:51.584 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 29 to 129 2024-04-12T11:19:51.584 INFO:tasks.rados.rados.0.smithi029.stdout:1286: done (3 left) 2024-04-12T11:19:51.584 INFO:tasks.rados.rados.0.smithi029.stdout:1291: write oid 39 current snap is 164 2024-04-12T11:19:51.585 INFO:tasks.rados.rados.0.smithi029.stdout:1287: expect (ObjNum 496 snap 160 seq_num 496) 2024-04-12T11:19:51.745 INFO:tasks.rados.rados.0.smithi029.stdout:1290: finishing rollback tid 0 to smithi02979277-29 2024-04-12T11:19:51.745 INFO:tasks.rados.rados.0.smithi029.stdout:1290: finishing rollback tid 1 to smithi02979277-29 2024-04-12T11:19:51.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 509 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-04-12T11:19:51.745 INFO:tasks.rados.rados.0.smithi029.stdout:1289: expect (ObjNum 449 snap 147 seq_num 449) 2024-04-12T11:19:51.956 INFO:tasks.rados.rados.0.smithi029.stdout:1291: seq_num 505 ranges {487535=480043,1584936=624833} 2024-04-12T11:19:51.961 INFO:tasks.rados.rados.0.smithi029.stdout:1291: writing smithi02979277-39 from 487535 to 967578 tid 1 2024-04-12T11:19:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:51 smithi087 ceph-mon[32917]: osdmap e390: 8 total, 8 up, 8 in 2024-04-12T11:19:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:51 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:51.968 INFO:tasks.rados.rados.0.smithi029.stdout:1291: writing smithi02979277-39 from 1584936 to 2209769 tid 2 2024-04-12T11:19:51.968 INFO:tasks.rados.rados.0.smithi029.stdout:1287: done (3 left) 2024-04-12T11:19:51.968 INFO:tasks.rados.rados.0.smithi029.stdout:1289: done (2 left) 2024-04-12T11:19:51.968 INFO:tasks.rados.rados.0.smithi029.stdout:1290: done (1 left) 2024-04-12T11:19:51.968 INFO:tasks.rados.rados.0.smithi029.stdout:1292: write oid 32 current snap is 164 2024-04-12T11:19:51.968 INFO:tasks.rados.rados.0.smithi029.stdout:1291: finishing write tid 1 to smithi02979277-39 2024-04-12T11:19:51.968 INFO:tasks.rados.rados.0.smithi029.stdout:1292: seq_num 506 ranges {504342=555292,1643418=484844} 2024-04-12T11:19:51.973 INFO:tasks.rados.rados.0.smithi029.stdout:1292: writing smithi02979277-32 from 504342 to 1059634 tid 1 2024-04-12T11:19:51.980 INFO:tasks.rados.rados.0.smithi029.stdout:1292: writing smithi02979277-32 from 1643418 to 2128262 tid 2 2024-04-12T11:19:51.980 INFO:tasks.rados.rados.0.smithi029.stdout:1293: write oid 10 current snap is 164 2024-04-12T11:19:51.980 INFO:tasks.rados.rados.0.smithi029.stdout:1291: finishing write tid 2 to smithi02979277-39 2024-04-12T11:19:51.980 INFO:tasks.rados.rados.0.smithi029.stdout:1291: finishing write tid 3 to smithi02979277-39 2024-04-12T11:19:51.980 INFO:tasks.rados.rados.0.smithi029.stdout:1293: seq_num 507 ranges {521149=630541,1701900=628502} 2024-04-12T11:19:51.986 INFO:tasks.rados.rados.0.smithi029.stdout:1293: writing smithi02979277-10 from 521149 to 1151690 tid 1 2024-04-12T11:19:51.994 INFO:tasks.rados.rados.0.smithi029.stdout:1293: writing smithi02979277-10 from 1701900 to 2330402 tid 2 2024-04-12T11:19:51.994 INFO:tasks.rados.rados.0.smithi029.stdout:1294: read oid 18 snap -1 2024-04-12T11:19:51.994 INFO:tasks.rados.rados.0.smithi029.stdout:1294: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:19:51.994 INFO:tasks.rados.rados.0.smithi029.stdout:1295: rollback oid 48 current snap is 164 2024-04-12T11:19:51.994 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 133 2024-04-12T11:19:51.994 INFO:tasks.rados.rados.0.smithi029.stdout:1291: finishing write tid 4 to smithi02979277-39 2024-04-12T11:19:51.994 INFO:tasks.rados.rados.0.smithi029.stdout:1291: oid 39 updating version 0 to 143 2024-04-12T11:19:51.994 INFO:tasks.rados.rados.0.smithi029.stdout:1291: oid 39 version 143 is already newer than 142 2024-04-12T11:19:51.994 INFO:tasks.rados.rados.0.smithi029.stdout:1291: oid 39 updating version 143 to 144 2024-04-12T11:19:51.994 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 144 (ObjNum 505 snap 164 seq_num 505) dirty exists 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1291: left oid 39 (ObjNum 505 snap 164 seq_num 505) 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1292: finishing write tid 1 to smithi02979277-32 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1291: done (4 left) 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1296: rollback oid 46 current snap is 164 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 46 to 163 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1292: finishing write tid 2 to smithi02979277-32 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1292: finishing write tid 3 to smithi02979277-32 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1292: finishing write tid 4 to smithi02979277-32 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1292: oid 32 updating version 0 to 511 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1292: oid 32 updating version 511 to 512 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1292: oid 32 version 512 is already newer than 510 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 512 (ObjNum 506 snap 164 seq_num 506) dirty exists 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1292: left oid 32 (ObjNum 506 snap 164 seq_num 506) 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1293: finishing write tid 1 to smithi02979277-10 2024-04-12T11:19:51.995 INFO:tasks.rados.rados.0.smithi029.stdout:1292: done (4 left) 2024-04-12T11:19:51.996 INFO:tasks.rados.rados.0.smithi029.stdout:1297: read oid 36 snap 141 2024-04-12T11:19:51.996 INFO:tasks.rados.rados.0.smithi029.stdout:1297: expect deleted 2024-04-12T11:19:51.996 INFO:tasks.rados.rados.0.smithi029.stdout:1298: read oid 14 snap -1 2024-04-12T11:19:51.996 INFO:tasks.rados.rados.0.smithi029.stdout:1298: expect (ObjNum 471 snap 156 seq_num 471) 2024-04-12T11:19:51.996 INFO:tasks.rados.rados.0.smithi029.stdout:1299: write oid 45 current snap is 164 2024-04-12T11:19:51.996 INFO:tasks.rados.rados.0.smithi029.stdout:1299: seq_num 508 ranges {537956=705790,1644029=772160} 2024-04-12T11:19:52.000 INFO:tasks.rados.rados.0.smithi029.stdout:1299: writing smithi02979277-45 from 537956 to 1243746 tid 1 2024-04-12T11:19:52.007 INFO:tasks.rados.rados.0.smithi029.stdout:1299: writing smithi02979277-45 from 1644029 to 2416189 tid 2 2024-04-12T11:19:52.007 INFO:tasks.rados.rados.0.smithi029.stdout:1300: read oid 20 snap -1 2024-04-12T11:19:52.007 INFO:tasks.rados.rados.0.smithi029.stdout:1300: expect (ObjNum 490 snap 158 seq_num 490) 2024-04-12T11:19:52.007 INFO:tasks.rados.rados.0.smithi029.stdout:1301: read oid 12 snap 133 2024-04-12T11:19:52.007 INFO:tasks.rados.rados.0.smithi029.stdout:1301: expect (ObjNum 367 snap 117 seq_num 367) 2024-04-12T11:19:52.008 INFO:tasks.rados.rados.0.smithi029.stdout:1302: snap_remove snap 145 2024-04-12T11:19:52.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:51 smithi029 ceph-mon[93650]: osdmap e390: 8 total, 8 up, 8 in 2024-04-12T11:19:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:51 smithi029 ceph-mon[97787]: osdmap e390: 8 total, 8 up, 8 in 2024-04-12T11:19:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:19:52.531 INFO:tasks.rados.rados.0.smithi029.stdout:1302: done (9 left) 2024-04-12T11:19:52.531 INFO:tasks.rados.rados.0.smithi029.stdout:1303: write oid 1 current snap is 164 2024-04-12T11:19:52.531 INFO:tasks.rados.rados.0.smithi029.stdout:1303: seq_num 509 ranges {554763=781039,2102511=452252} 2024-04-12T11:19:52.539 INFO:tasks.rados.rados.0.smithi029.stdout:1303: writing smithi02979277-1 from 554763 to 1335802 tid 1 2024-04-12T11:19:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:1303: writing smithi02979277-1 from 2102511 to 2554763 tid 2 2024-04-12T11:19:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:1304: snap_create 2024-04-12T11:19:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:1293: finishing write tid 2 to smithi02979277-10 2024-04-12T11:19:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:1293: finishing write tid 3 to smithi02979277-10 2024-04-12T11:19:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:1293: finishing write tid 4 to smithi02979277-10 2024-04-12T11:19:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:1293: oid 10 updating version 0 to 571 2024-04-12T11:19:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:1293: oid 10 version 571 is already newer than 570 2024-04-12T11:19:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:1293: oid 10 version 571 is already newer than 569 2024-04-12T11:19:52.547 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 571 (ObjNum 507 snap 164 seq_num 507) dirty exists 2024-04-12T11:19:52.548 INFO:tasks.rados.rados.0.smithi029.stdout:1293: left oid 10 (ObjNum 507 snap 164 seq_num 507) 2024-04-12T11:19:52.548 INFO:tasks.rados.rados.0.smithi029.stdout:1296: finishing rollback tid 0 to smithi02979277-46 2024-04-12T11:19:52.548 INFO:tasks.rados.rados.0.smithi029.stdout:1296: finishing rollback tid 1 to smithi02979277-46 2024-04-12T11:19:52.548 INFO:tasks.rados.rados.0.smithi029.stdout:1296: finishing rollback tid 2 to smithi02979277-46 2024-04-12T11:19:52.548 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 328 (ObjNum 487 snap 158 seq_num 487) dirty exists 2024-04-12T11:19:52.548 INFO:tasks.rados.rados.0.smithi029.stdout:1295: finishing rollback tid 0 to smithi02979277-48 2024-04-12T11:19:52.548 INFO:tasks.rados.rados.0.smithi029.stdout:1295: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:19:52.548 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 574 (ObjNum 945 snap 0 seq_num 2951547216) dirty dne 2024-04-12T11:19:52.548 INFO:tasks.rados.rados.0.smithi029.stdout:1294: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:19:52.580 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:52.683 INFO:tasks.rados.rados.0.smithi029.stdout:1298: expect (ObjNum 471 snap 156 seq_num 471) 2024-04-12T11:19:52.814 INFO:tasks.rados.rados.0.smithi029.stdout:1299: finishing write tid 1 to smithi02979277-45 2024-04-12T11:19:52.814 INFO:tasks.rados.rados.0.smithi029.stdout:1300: expect (ObjNum 490 snap 158 seq_num 490) 2024-04-12T11:19:52.922 INFO:tasks.rados.rados.0.smithi029.stdout:1299: finishing write tid 2 to smithi02979277-45 2024-04-12T11:19:52.922 INFO:tasks.rados.rados.0.smithi029.stdout:1299: finishing write tid 3 to smithi02979277-45 2024-04-12T11:19:52.923 INFO:tasks.rados.rados.0.smithi029.stdout:1299: finishing write tid 4 to smithi02979277-45 2024-04-12T11:19:52.923 INFO:tasks.rados.rados.0.smithi029.stdout:1299: oid 45 updating version 0 to 612 2024-04-12T11:19:52.923 INFO:tasks.rados.rados.0.smithi029.stdout:1299: oid 45 updating version 612 to 613 2024-04-12T11:19:52.923 INFO:tasks.rados.rados.0.smithi029.stdout:1299: oid 45 updating version 613 to 614 2024-04-12T11:19:52.923 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 614 (ObjNum 508 snap 164 seq_num 508) dirty exists 2024-04-12T11:19:52.923 INFO:tasks.rados.rados.0.smithi029.stdout:1299: left oid 45 (ObjNum 508 snap 164 seq_num 508) 2024-04-12T11:19:52.923 INFO:tasks.rados.rados.0.smithi029.stdout:1301: expect (ObjNum 367 snap 117 seq_num 367) 2024-04-12T11:19:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:52 smithi087 ceph-mon[32917]: pgmap v220: 89 pgs: 89 active+clean; 216 MiB data, 700 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:19:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:52 smithi087 ceph-mon[32917]: osdmap e391: 8 total, 8 up, 8 in 2024-04-12T11:19:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:52 smithi087 ceph-mon[32917]: osdmap e392: 8 total, 8 up, 8 in 2024-04-12T11:19:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:52 smithi029 ceph-mon[97787]: pgmap v220: 89 pgs: 89 active+clean; 216 MiB data, 700 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:19:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:52 smithi029 ceph-mon[97787]: osdmap e391: 8 total, 8 up, 8 in 2024-04-12T11:19:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:52 smithi029 ceph-mon[97787]: osdmap e392: 8 total, 8 up, 8 in 2024-04-12T11:19:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:52 smithi029 ceph-mon[93650]: pgmap v220: 89 pgs: 89 active+clean; 216 MiB data, 700 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:19:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:52 smithi029 ceph-mon[93650]: osdmap e391: 8 total, 8 up, 8 in 2024-04-12T11:19:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:52 smithi029 ceph-mon[93650]: osdmap e392: 8 total, 8 up, 8 in 2024-04-12T11:19:53.024 INFO:tasks.rados.rados.0.smithi029.stdout:1303: finishing write tid 1 to smithi02979277-1 2024-04-12T11:19:53.024 INFO:tasks.rados.rados.0.smithi029.stdout:1303: finishing write tid 2 to smithi02979277-1 2024-04-12T11:19:53.024 INFO:tasks.rados.rados.0.smithi029.stdout:1303: finishing write tid 3 to smithi02979277-1 2024-04-12T11:19:53.024 INFO:tasks.rados.rados.0.smithi029.stdout:1303: finishing write tid 4 to smithi02979277-1 2024-04-12T11:19:53.024 INFO:tasks.rados.rados.0.smithi029.stdout:1303: oid 1 updating version 0 to 652 2024-04-12T11:19:53.024 INFO:tasks.rados.rados.0.smithi029.stdout:1303: oid 1 version 652 is already newer than 646 2024-04-12T11:19:53.024 INFO:tasks.rados.rados.0.smithi029.stdout:1303: oid 1 version 652 is already newer than 651 2024-04-12T11:19:53.024 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 652 (ObjNum 509 snap 164 seq_num 509) dirty exists 2024-04-12T11:19:53.024 INFO:tasks.rados.rados.0.smithi029.stdout:1303: left oid 1 (ObjNum 509 snap 164 seq_num 509) 2024-04-12T11:19:53.533 INFO:tasks.rados.rados.0.smithi029.stdout:1293: done (10 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1294: done (9 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1295: done (8 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1296: done (7 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1297: done (6 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1298: done (5 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1299: done (4 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1300: done (3 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1301: done (2 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1303: done (1 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1304: done (0 left) 2024-04-12T11:19:53.534 INFO:tasks.rados.rados.0.smithi029.stdout:1305: write oid 25 current snap is 165 2024-04-12T11:19:53.535 INFO:tasks.rados.rados.0.smithi029.stdout:1305: seq_num 510 ranges {571570=572641,1877346=694224} 2024-04-12T11:19:53.539 INFO:tasks.rados.rados.0.smithi029.stdout:1305: writing smithi02979277-25 from 571570 to 1144211 tid 1 2024-04-12T11:19:53.548 INFO:tasks.rados.rados.0.smithi029.stdout:1305: writing smithi02979277-25 from 1877346 to 2571570 tid 2 2024-04-12T11:19:53.548 INFO:tasks.rados.rados.0.smithi029.stdout:1306: read oid 21 snap -1 2024-04-12T11:19:53.548 INFO:tasks.rados.rados.0.smithi029.stdout:1306: expect (ObjNum 501 snap 162 seq_num 501) 2024-04-12T11:19:53.548 INFO:tasks.rados.rados.0.smithi029.stdout:1307: rollback oid 5 current snap is 165 2024-04-12T11:19:53.549 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 5 to 149 2024-04-12T11:19:53.549 INFO:tasks.rados.rados.0.smithi029.stdout:1305: finishing write tid 1 to smithi02979277-25 2024-04-12T11:19:53.549 INFO:tasks.rados.rados.0.smithi029.stdout:1308: write oid 26 current snap is 165 2024-04-12T11:19:53.549 INFO:tasks.rados.rados.0.smithi029.stdout:1308: seq_num 511 ranges {588377=647890,1935828=635840} 2024-04-12T11:19:53.555 INFO:tasks.rados.rados.0.smithi029.stdout:1308: writing smithi02979277-26 from 588377 to 1236267 tid 1 2024-04-12T11:19:53.563 INFO:tasks.rados.rados.0.smithi029.stdout:1308: writing smithi02979277-26 from 1935828 to 2571668 tid 2 2024-04-12T11:19:53.563 INFO:tasks.rados.rados.0.smithi029.stdout:1309: write oid 42 current snap is 165 2024-04-12T11:19:53.563 INFO:tasks.rados.rados.0.smithi029.stdout:1307: finishing rollback tid 1 to smithi02979277-5 2024-04-12T11:19:53.563 INFO:tasks.rados.rados.0.smithi029.stdout:1309: seq_num 512 ranges {605184=723139,1877957=727227} 2024-04-12T11:19:53.570 INFO:tasks.rados.rados.0.smithi029.stdout:1309: writing smithi02979277-42 from 605184 to 1328323 tid 1 2024-04-12T11:19:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:1309: writing smithi02979277-42 from 1877957 to 2605184 tid 2 2024-04-12T11:19:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:1310: snap_create 2024-04-12T11:19:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:1307: finishing rollback tid 2 to smithi02979277-5 2024-04-12T11:19:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 787 (ObjNum 442 snap 141 seq_num 442) dirty exists 2024-04-12T11:19:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:1305: finishing write tid 2 to smithi02979277-25 2024-04-12T11:19:53.578 INFO:tasks.rados.rados.0.smithi029.stdout:1305: finishing write tid 3 to smithi02979277-25 2024-04-12T11:19:53.579 INFO:tasks.rados.rados.0.smithi029.stdout:1305: finishing write tid 4 to smithi02979277-25 2024-04-12T11:19:53.579 INFO:tasks.rados.rados.0.smithi029.stdout:1305: oid 25 updating version 0 to 669 2024-04-12T11:19:53.579 INFO:tasks.rados.rados.0.smithi029.stdout:1305: oid 25 updating version 669 to 670 2024-04-12T11:19:53.579 INFO:tasks.rados.rados.0.smithi029.stdout:1305: oid 25 version 670 is already newer than 668 2024-04-12T11:19:53.579 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 670 (ObjNum 510 snap 165 seq_num 510) dirty exists 2024-04-12T11:19:53.579 INFO:tasks.rados.rados.0.smithi029.stdout:1305: left oid 25 (ObjNum 510 snap 165 seq_num 510) 2024-04-12T11:19:53.579 INFO:tasks.rados.rados.0.smithi029.stdout:1306: expect (ObjNum 501 snap 162 seq_num 501) 2024-04-12T11:19:53.659 INFO:tasks.rados.rados.0.smithi029.stdout:1308: finishing write tid 1 to smithi02979277-26 2024-04-12T11:19:53.659 INFO:tasks.rados.rados.0.smithi029.stdout:1308: finishing write tid 2 to smithi02979277-26 2024-04-12T11:19:53.659 INFO:tasks.rados.rados.0.smithi029.stdout:1308: finishing write tid 3 to smithi02979277-26 2024-04-12T11:19:53.659 INFO:tasks.rados.rados.0.smithi029.stdout:1308: finishing write tid 4 to smithi02979277-26 2024-04-12T11:19:53.659 INFO:tasks.rados.rados.0.smithi029.stdout:1308: oid 26 updating version 0 to 809 2024-04-12T11:19:53.659 INFO:tasks.rados.rados.0.smithi029.stdout:1308: oid 26 updating version 809 to 810 2024-04-12T11:19:53.659 INFO:tasks.rados.rados.0.smithi029.stdout:1308: oid 26 updating version 810 to 811 2024-04-12T11:19:53.659 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 811 (ObjNum 511 snap 165 seq_num 511) dirty exists 2024-04-12T11:19:53.659 INFO:tasks.rados.rados.0.smithi029.stdout:1308: left oid 26 (ObjNum 511 snap 165 seq_num 511) 2024-04-12T11:19:53.660 INFO:tasks.rados.rados.0.smithi029.stdout:1309: finishing write tid 1 to smithi02979277-42 2024-04-12T11:19:53.660 INFO:tasks.rados.rados.0.smithi029.stdout:1309: finishing write tid 2 to smithi02979277-42 2024-04-12T11:19:53.660 INFO:tasks.rados.rados.0.smithi029.stdout:1309: finishing write tid 3 to smithi02979277-42 2024-04-12T11:19:53.660 INFO:tasks.rados.rados.0.smithi029.stdout:1309: finishing write tid 4 to smithi02979277-42 2024-04-12T11:19:53.660 INFO:tasks.rados.rados.0.smithi029.stdout:1309: oid 42 updating version 0 to 220 2024-04-12T11:19:53.660 INFO:tasks.rados.rados.0.smithi029.stdout:1309: oid 42 updating version 220 to 221 2024-04-12T11:19:53.660 INFO:tasks.rados.rados.0.smithi029.stdout:1309: oid 42 version 221 is already newer than 219 2024-04-12T11:19:53.660 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 221 (ObjNum 512 snap 165 seq_num 512) dirty exists 2024-04-12T11:19:53.660 INFO:tasks.rados.rados.0.smithi029.stdout:1309: left oid 42 (ObjNum 512 snap 165 seq_num 512) 2024-04-12T11:19:53.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:19:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:19:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:19:54.540 INFO:tasks.rados.rados.0.smithi029.stdout:1305: done (5 left) 2024-04-12T11:19:54.540 INFO:tasks.rados.rados.0.smithi029.stdout:1306: done (4 left) 2024-04-12T11:19:54.540 INFO:tasks.rados.rados.0.smithi029.stdout:1307: done (3 left) 2024-04-12T11:19:54.540 INFO:tasks.rados.rados.0.smithi029.stdout:1308: done (2 left) 2024-04-12T11:19:54.540 INFO:tasks.rados.rados.0.smithi029.stdout:1309: done (1 left) 2024-04-12T11:19:54.540 INFO:tasks.rados.rados.0.smithi029.stdout:1310: done (0 left) 2024-04-12T11:19:54.540 INFO:tasks.rados.rados.0.smithi029.stdout:1311: read oid 24 snap -1 2024-04-12T11:19:54.540 INFO:tasks.rados.rados.0.smithi029.stdout:1311: expect (ObjNum 499 snap 161 seq_num 499) 2024-04-12T11:19:54.540 INFO:tasks.rados.rados.0.smithi029.stdout:1312: delete oid 31 current snap is 166 2024-04-12T11:19:54.546 INFO:tasks.rados.rados.0.smithi029.stdout:1312: done (1 left) 2024-04-12T11:19:54.546 INFO:tasks.rados.rados.0.smithi029.stdout:1313: delete oid 19 current snap is 166 2024-04-12T11:19:54.549 INFO:tasks.rados.rados.0.smithi029.stdout:1311: expect (ObjNum 499 snap 161 seq_num 499) 2024-04-12T11:19:54.631 INFO:tasks.rados.rados.0.smithi029.stdout:1311: done (1 left) 2024-04-12T11:19:54.631 INFO:tasks.rados.rados.0.smithi029.stdout:1313: done (0 left) 2024-04-12T11:19:54.631 INFO:tasks.rados.rados.0.smithi029.stdout:1314: snap_remove snap 149 2024-04-12T11:19:54.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:54 smithi029 ceph-mon[97787]: pgmap v223: 89 pgs: 89 active+clean; 235 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-04-12T11:19:54.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:54 smithi029 ceph-mon[97787]: osdmap e393: 8 total, 8 up, 8 in 2024-04-12T11:19:54.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:54 smithi029 ceph-mon[93650]: pgmap v223: 89 pgs: 89 active+clean; 235 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-04-12T11:19:54.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:54 smithi029 ceph-mon[93650]: osdmap e393: 8 total, 8 up, 8 in 2024-04-12T11:19:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:54 smithi087 ceph-mon[32917]: pgmap v223: 89 pgs: 89 active+clean; 235 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-04-12T11:19:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:54 smithi087 ceph-mon[32917]: osdmap e393: 8 total, 8 up, 8 in 2024-04-12T11:19:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:19:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:19:54] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.43.0" 2024-04-12T11:19:55.546 INFO:tasks.rados.rados.0.smithi029.stdout:1314: done (0 left) 2024-04-12T11:19:55.546 INFO:tasks.rados.rados.0.smithi029.stdout:1315: snap_create 2024-04-12T11:19:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:55 smithi087 ceph-mon[32917]: osdmap e394: 8 total, 8 up, 8 in 2024-04-12T11:19:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:55 smithi029 ceph-mon[97787]: osdmap e394: 8 total, 8 up, 8 in 2024-04-12T11:19:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:55 smithi029 ceph-mon[93650]: osdmap e394: 8 total, 8 up, 8 in 2024-04-12T11:19:56.551 INFO:tasks.rados.rados.0.smithi029.stdout:1315: done (0 left) 2024-04-12T11:19:56.551 INFO:tasks.rados.rados.0.smithi029.stdout:1316: read oid 32 snap -1 2024-04-12T11:19:56.551 INFO:tasks.rados.rados.0.smithi029.stdout:1316: expect (ObjNum 506 snap 164 seq_num 506) 2024-04-12T11:19:56.551 INFO:tasks.rados.rados.0.smithi029.stdout:1317: rollback oid 49 current snap is 167 2024-04-12T11:19:56.551 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 49 to 164 2024-04-12T11:19:56.551 INFO:tasks.rados.rados.0.smithi029.stdout:1318: delete oid 50 current snap is 167 2024-04-12T11:19:56.557 INFO:tasks.rados.rados.0.smithi029.stdout:1316: expect (ObjNum 506 snap 164 seq_num 506) 2024-04-12T11:19:56.642 INFO:tasks.rados.rados.0.smithi029.stdout:1317: finishing rollback tid 0 to smithi02979277-49 2024-04-12T11:19:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:1317: finishing rollback tid 1 to smithi02979277-49 2024-04-12T11:19:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:1317: finishing rollback tid 2 to smithi02979277-49 2024-04-12T11:19:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 513 (ObjNum 448 snap 147 seq_num 448) dirty exists 2024-04-12T11:19:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:1316: done (2 left) 2024-04-12T11:19:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:1317: done (1 left) 2024-04-12T11:19:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:1318: done (0 left) 2024-04-12T11:19:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:1319: read oid 45 snap 157 2024-04-12T11:19:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:1319: expect deleted 2024-04-12T11:19:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:1320: delete oid 34 current snap is 167 2024-04-12T11:19:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:1319: done (1 left) 2024-04-12T11:19:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:1320: done (0 left) 2024-04-12T11:19:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:1321: read oid 3 snap 133 2024-04-12T11:19:56.645 INFO:tasks.rados.rados.0.smithi029.stdout:1321: expect deleted 2024-04-12T11:19:56.645 INFO:tasks.rados.rados.0.smithi029.stdout:1322: rollback oid 19 current snap is 167 2024-04-12T11:19:56.645 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 19 to 158 2024-04-12T11:19:56.645 INFO:tasks.rados.rados.0.smithi029.stdout:1323: write oid 23 current snap is 167 2024-04-12T11:19:56.645 INFO:tasks.rados.rados.0.smithi029.stdout:1323: seq_num 517 ranges {689219=415737,1770367=646847} 2024-04-12T11:19:56.650 INFO:tasks.rados.rados.0.smithi029.stdout:1323: writing smithi02979277-23 from 689219 to 1104956 tid 1 2024-04-12T11:19:56.657 INFO:tasks.rados.rados.0.smithi029.stdout:1323: writing smithi02979277-23 from 1770367 to 2417214 tid 2 2024-04-12T11:19:56.657 INFO:tasks.rados.rados.0.smithi029.stdout:1324: snap_create 2024-04-12T11:19:56.658 INFO:tasks.rados.rados.0.smithi029.stdout:1322: finishing rollback tid 1 to smithi02979277-19 2024-04-12T11:19:56.658 INFO:tasks.rados.rados.0.smithi029.stdout:1322: finishing rollback tid 2 to smithi02979277-19 2024-04-12T11:19:56.658 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 795 (ObjNum 389 snap 121 seq_num 389) dirty exists 2024-04-12T11:19:56.658 INFO:tasks.rados.rados.0.smithi029.stdout:1323: finishing write tid 1 to smithi02979277-23 2024-04-12T11:19:56.661 INFO:tasks.rados.rados.0.smithi029.stdout:1323: finishing write tid 2 to smithi02979277-23 2024-04-12T11:19:56.661 INFO:tasks.rados.rados.0.smithi029.stdout:1323: finishing write tid 3 to smithi02979277-23 2024-04-12T11:19:56.662 INFO:tasks.rados.rados.0.smithi029.stdout:1323: finishing write tid 4 to smithi02979277-23 2024-04-12T11:19:56.662 INFO:tasks.rados.rados.0.smithi029.stdout:1323: oid 23 updating version 0 to 412 2024-04-12T11:19:56.662 INFO:tasks.rados.rados.0.smithi029.stdout:1323: oid 23 version 412 is already newer than 411 2024-04-12T11:19:56.662 INFO:tasks.rados.rados.0.smithi029.stdout:1323: oid 23 updating version 412 to 413 2024-04-12T11:19:56.662 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 413 (ObjNum 517 snap 167 seq_num 517) dirty exists 2024-04-12T11:19:56.662 INFO:tasks.rados.rados.0.smithi029.stdout:1323: left oid 23 (ObjNum 517 snap 167 seq_num 517) 2024-04-12T11:19:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:56 smithi087 ceph-mon[32917]: pgmap v226: 89 pgs: 89 active+clean; 235 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-04-12T11:19:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:56 smithi087 ceph-mon[32917]: osdmap e395: 8 total, 8 up, 8 in 2024-04-12T11:19:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:56 smithi029 ceph-mon[97787]: pgmap v226: 89 pgs: 89 active+clean; 235 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-04-12T11:19:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:56 smithi029 ceph-mon[97787]: osdmap e395: 8 total, 8 up, 8 in 2024-04-12T11:19:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:56 smithi029 ceph-mon[93650]: pgmap v226: 89 pgs: 89 active+clean; 235 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-04-12T11:19:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:56 smithi029 ceph-mon[93650]: osdmap e395: 8 total, 8 up, 8 in 2024-04-12T11:19:57.556 INFO:tasks.rados.rados.0.smithi029.stdout:1321: done (3 left) 2024-04-12T11:19:57.556 INFO:tasks.rados.rados.0.smithi029.stdout:1322: done (2 left) 2024-04-12T11:19:57.556 INFO:tasks.rados.rados.0.smithi029.stdout:1323: done (1 left) 2024-04-12T11:19:57.556 INFO:tasks.rados.rados.0.smithi029.stdout:1324: done (0 left) 2024-04-12T11:19:57.556 INFO:tasks.rados.rados.0.smithi029.stdout:1325: delete oid 41 current snap is 168 2024-04-12T11:19:57.560 INFO:tasks.rados.rados.0.smithi029.stdout:1325: done (0 left) 2024-04-12T11:19:57.560 INFO:tasks.rados.rados.0.smithi029.stdout:1326: delete oid 45 current snap is 168 2024-04-12T11:19:57.563 INFO:tasks.rados.rados.0.smithi029.stdout:1326: done (0 left) 2024-04-12T11:19:57.563 INFO:tasks.rados.rados.0.smithi029.stdout:1327: snap_create 2024-04-12T11:19:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:57 smithi087 ceph-mon[32917]: osdmap e396: 8 total, 8 up, 8 in 2024-04-12T11:19:58.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:57 smithi029 ceph-mon[93650]: osdmap e396: 8 total, 8 up, 8 in 2024-04-12T11:19:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:57 smithi029 ceph-mon[97787]: osdmap e396: 8 total, 8 up, 8 in 2024-04-12T11:19:58.558 INFO:tasks.rados.rados.0.smithi029.stdout:1327: done (0 left) 2024-04-12T11:19:58.558 INFO:tasks.rados.rados.0.smithi029.stdout:1328: write oid 34 current snap is 169 2024-04-12T11:19:58.558 INFO:tasks.rados.rados.0.smithi029.stdout:1328: seq_num 520 ranges {739640=641484,1829460=794174} 2024-04-12T11:19:58.565 INFO:tasks.rados.rados.0.smithi029.stdout:1328: writing smithi02979277-34 from 739640 to 1381124 tid 1 2024-04-12T11:19:58.573 INFO:tasks.rados.rados.0.smithi029.stdout:1328: writing smithi02979277-34 from 1829460 to 2623634 tid 2 2024-04-12T11:19:58.573 INFO:tasks.rados.rados.0.smithi029.stdout:1329: read oid 50 snap -1 2024-04-12T11:19:58.573 INFO:tasks.rados.rados.0.smithi029.stdout:1329: expect deleted 2024-04-12T11:19:58.573 INFO:tasks.rados.rados.0.smithi029.stdout:1330: read oid 19 snap -1 2024-04-12T11:19:58.573 INFO:tasks.rados.rados.0.smithi029.stdout:1330: expect (ObjNum 389 snap 121 seq_num 389) 2024-04-12T11:19:58.573 INFO:tasks.rados.rados.0.smithi029.stdout:1331: snap_remove snap 158 2024-04-12T11:19:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:58 smithi087 ceph-mon[32917]: pgmap v229: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-04-12T11:19:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:58 smithi087 ceph-mon[32917]: osdmap e397: 8 total, 8 up, 8 in 2024-04-12T11:19:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:58 smithi029 ceph-mon[93650]: pgmap v229: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-04-12T11:19:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:58 smithi029 ceph-mon[93650]: osdmap e397: 8 total, 8 up, 8 in 2024-04-12T11:19:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:58 smithi029 ceph-mon[97787]: pgmap v229: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-04-12T11:19:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:58 smithi029 ceph-mon[97787]: osdmap e397: 8 total, 8 up, 8 in 2024-04-12T11:19:59.570 INFO:tasks.rados.rados.0.smithi029.stdout:1331: done (3 left) 2024-04-12T11:19:59.570 INFO:tasks.rados.rados.0.smithi029.stdout:1332: write oid 8 current snap is 169 2024-04-12T11:19:59.570 INFO:tasks.rados.rados.0.smithi029.stdout:1332: seq_num 521 ranges {756447=716733,1887942=537832} 2024-04-12T11:19:59.579 INFO:tasks.rados.rados.0.smithi029.stdout:1332: writing smithi02979277-8 from 756447 to 1473180 tid 1 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1332: writing smithi02979277-8 from 1887942 to 2425774 tid 2 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1333: read oid 1 snap -1 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1333: expect (ObjNum 509 snap 164 seq_num 509) 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1334: read oid 43 snap -1 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1334: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1335: read oid 21 snap -1 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1335: expect (ObjNum 501 snap 162 seq_num 501) 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1336: snap_create 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1328: finishing write tid 1 to smithi02979277-34 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1328: finishing write tid 2 to smithi02979277-34 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1328: finishing write tid 3 to smithi02979277-34 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1328: finishing write tid 4 to smithi02979277-34 2024-04-12T11:19:59.585 INFO:tasks.rados.rados.0.smithi029.stdout:1328: oid 34 updating version 0 to 528 2024-04-12T11:19:59.586 INFO:tasks.rados.rados.0.smithi029.stdout:1328: oid 34 version 528 is already newer than 526 2024-04-12T11:19:59.586 INFO:tasks.rados.rados.0.smithi029.stdout:1328: oid 34 version 528 is already newer than 527 2024-04-12T11:19:59.586 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 528 (ObjNum 520 snap 169 seq_num 520) dirty exists 2024-04-12T11:19:59.586 INFO:tasks.rados.rados.0.smithi029.stdout:1328: left oid 34 (ObjNum 520 snap 169 seq_num 520) 2024-04-12T11:19:59.586 INFO:tasks.rados.rados.0.smithi029.stdout:1330: expect (ObjNum 389 snap 121 seq_num 389) 2024-04-12T11:19:59.745 INFO:tasks.rados.rados.0.smithi029.stdout:1332: finishing write tid 1 to smithi02979277-8 2024-04-12T11:19:59.745 INFO:tasks.rados.rados.0.smithi029.stdout:1332: finishing write tid 2 to smithi02979277-8 2024-04-12T11:19:59.745 INFO:tasks.rados.rados.0.smithi029.stdout:1332: finishing write tid 3 to smithi02979277-8 2024-04-12T11:19:59.745 INFO:tasks.rados.rados.0.smithi029.stdout:1332: finishing write tid 4 to smithi02979277-8 2024-04-12T11:19:59.745 INFO:tasks.rados.rados.0.smithi029.stdout:1332: oid 8 updating version 0 to 96 2024-04-12T11:19:59.745 INFO:tasks.rados.rados.0.smithi029.stdout:1332: oid 8 updating version 96 to 98 2024-04-12T11:19:59.745 INFO:tasks.rados.rados.0.smithi029.stdout:1332: oid 8 version 98 is already newer than 97 2024-04-12T11:19:59.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 98 (ObjNum 521 snap 169 seq_num 521) dirty exists 2024-04-12T11:19:59.746 INFO:tasks.rados.rados.0.smithi029.stdout:1332: left oid 8 (ObjNum 521 snap 169 seq_num 521) 2024-04-12T11:19:59.746 INFO:tasks.rados.rados.0.smithi029.stdout:1334: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:19:59.853 INFO:tasks.rados.rados.0.smithi029.stdout:1333: expect (ObjNum 509 snap 164 seq_num 509) 2024-04-12T11:19:59.966 INFO:tasks.rados.rados.0.smithi029.stdout:1335: expect (ObjNum 501 snap 162 seq_num 501) 2024-04-12T11:19:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:19:59 smithi087 ceph-mon[32917]: osdmap e398: 8 total, 8 up, 8 in 2024-04-12T11:20:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:19:59 smithi029 ceph-mon[97787]: osdmap e398: 8 total, 8 up, 8 in 2024-04-12T11:20:00.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:19:59 smithi029 ceph-mon[93650]: osdmap e398: 8 total, 8 up, 8 in 2024-04-12T11:20:00.572 INFO:tasks.rados.rados.0.smithi029.stdout:1328: done (7 left) 2024-04-12T11:20:00.572 INFO:tasks.rados.rados.0.smithi029.stdout:1329: done (6 left) 2024-04-12T11:20:00.572 INFO:tasks.rados.rados.0.smithi029.stdout:1330: done (5 left) 2024-04-12T11:20:00.572 INFO:tasks.rados.rados.0.smithi029.stdout:1332: done (4 left) 2024-04-12T11:20:00.572 INFO:tasks.rados.rados.0.smithi029.stdout:1333: done (3 left) 2024-04-12T11:20:00.572 INFO:tasks.rados.rados.0.smithi029.stdout:1334: done (2 left) 2024-04-12T11:20:00.572 INFO:tasks.rados.rados.0.smithi029.stdout:1335: done (1 left) 2024-04-12T11:20:00.572 INFO:tasks.rados.rados.0.smithi029.stdout:1336: done (0 left) 2024-04-12T11:20:00.572 INFO:tasks.rados.rados.0.smithi029.stdout:1337: write oid 19 current snap is 170 2024-04-12T11:20:00.572 INFO:tasks.rados.rados.0.smithi029.stdout:1337: seq_num 522 ranges {773254=791982,2346424=426830} 2024-04-12T11:20:00.580 INFO:tasks.rados.rados.0.smithi029.stdout:1337: writing smithi02979277-19 from 773254 to 1565236 tid 1 2024-04-12T11:20:00.587 INFO:tasks.rados.rados.0.smithi029.stdout:1337: writing smithi02979277-19 from 2346424 to 2773254 tid 2 2024-04-12T11:20:00.587 INFO:tasks.rados.rados.0.smithi029.stdout:1338: rollback oid 15 current snap is 170 2024-04-12T11:20:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 15 to 156 2024-04-12T11:20:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:1339: read oid 38 snap -1 2024-04-12T11:20:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:1339: expect (ObjNum 502 snap 162 seq_num 502) 2024-04-12T11:20:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:1340: read oid 2 snap -1 2024-04-12T11:20:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:1340: expect (ObjNum 479 snap 156 seq_num 479) 2024-04-12T11:20:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:1341: delete oid 5 current snap is 170 2024-04-12T11:20:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:1337: finishing write tid 1 to smithi02979277-19 2024-04-12T11:20:00.591 INFO:tasks.rados.rados.0.smithi029.stdout:1338: finishing rollback tid 0 to smithi02979277-15 2024-04-12T11:20:00.591 INFO:tasks.rados.rados.0.smithi029.stdout:1337: finishing write tid 2 to smithi02979277-19 2024-04-12T11:20:00.592 INFO:tasks.rados.rados.0.smithi029.stdout:1337: finishing write tid 3 to smithi02979277-19 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1337: finishing write tid 4 to smithi02979277-19 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1337: oid 19 updating version 0 to 802 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1337: oid 19 updating version 802 to 803 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1337: oid 19 version 803 is already newer than 801 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 803 (ObjNum 522 snap 170 seq_num 522) dirty exists 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1337: left oid 19 (ObjNum 522 snap 170 seq_num 522) 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1337: done (4 left) 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1341: done (3 left) 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1342: read oid 8 snap 161 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1342: expect (ObjNum 399 snap 123 seq_num 399) 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1343: read oid 29 snap -1 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1343: expect deleted 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1344: rollback oid 18 current snap is 170 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 18 to 167 2024-04-12T11:20:00.593 INFO:tasks.rados.rados.0.smithi029.stdout:1345: rollback oid 33 current snap is 170 2024-04-12T11:20:00.594 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 165 2024-04-12T11:20:00.594 INFO:tasks.rados.rados.0.smithi029.stdout:1346: snap_remove snap 130 2024-04-12T11:20:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:00 smithi087 ceph-mon[32917]: pgmap v232: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-04-12T11:20:00.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:00 smithi087 ceph-mon[32917]: osdmap e399: 8 total, 8 up, 8 in 2024-04-12T11:20:00.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:00 smithi087 ceph-mon[32917]: overall HEALTH_OK 2024-04-12T11:20:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:00 smithi029 ceph-mon[93650]: pgmap v232: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-04-12T11:20:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:00 smithi029 ceph-mon[93650]: osdmap e399: 8 total, 8 up, 8 in 2024-04-12T11:20:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:00 smithi029 ceph-mon[93650]: overall HEALTH_OK 2024-04-12T11:20:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:00 smithi029 ceph-mon[97787]: pgmap v232: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-04-12T11:20:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:00 smithi029 ceph-mon[97787]: osdmap e399: 8 total, 8 up, 8 in 2024-04-12T11:20:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:00 smithi029 ceph-mon[97787]: overall HEALTH_OK 2024-04-12T11:20:01.578 INFO:tasks.rados.rados.0.smithi029.stdout:1346: done (7 left) 2024-04-12T11:20:01.578 INFO:tasks.rados.rados.0.smithi029.stdout:1347: delete oid 27 current snap is 170 2024-04-12T11:20:01.579 INFO:tasks.rados.rados.0.smithi029.stdout:1340: expect (ObjNum 479 snap 156 seq_num 479) 2024-04-12T11:20:01.650 INFO:tasks.rados.rados.0.smithi029.stdout:1338: finishing rollback tid 1 to smithi02979277-15 2024-04-12T11:20:01.650 INFO:tasks.rados.rados.0.smithi029.stdout:1338: finishing rollback tid 2 to smithi02979277-15 2024-04-12T11:20:01.650 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 543 (ObjNum 438 snap 139 seq_num 438) dirty exists 2024-04-12T11:20:01.650 INFO:tasks.rados.rados.0.smithi029.stdout:1339: expect (ObjNum 502 snap 162 seq_num 502) 2024-04-12T11:20:01.822 INFO:tasks.rados.rados.0.smithi029.stdout:1342: expect (ObjNum 399 snap 123 seq_num 399) 2024-04-12T11:20:01.914 INFO:tasks.rados.rados.0.smithi029.stdout:1345: finishing rollback tid 0 to smithi02979277-33 2024-04-12T11:20:01.914 INFO:tasks.rados.rados.0.smithi029.stdout:1344: finishing rollback tid 0 to smithi02979277-18 2024-04-12T11:20:01.914 INFO:tasks.rados.rados.0.smithi029.stdout:1345: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:20:01.914 INFO:tasks.rados.rados.0.smithi029.stdout:1344: finishing rollback tid 1 to smithi02979277-18 2024-04-12T11:20:01.914 INFO:tasks.rados.rados.0.smithi029.stdout:1345: finishing rollback tid 2 to smithi02979277-33 2024-04-12T11:20:01.914 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 419 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-04-12T11:20:01.914 INFO:tasks.rados.rados.0.smithi029.stdout:1344: finishing rollback tid 2 to smithi02979277-18 2024-04-12T11:20:01.915 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 684 (ObjNum 489 snap 158 seq_num 489) dirty exists 2024-04-12T11:20:01.916 INFO:tasks.rados.rados.0.smithi029.stdout:1338: done (7 left) 2024-04-12T11:20:01.916 INFO:tasks.rados.rados.0.smithi029.stdout:1339: done (6 left) 2024-04-12T11:20:01.916 INFO:tasks.rados.rados.0.smithi029.stdout:1340: done (5 left) 2024-04-12T11:20:01.916 INFO:tasks.rados.rados.0.smithi029.stdout:1342: done (4 left) 2024-04-12T11:20:01.916 INFO:tasks.rados.rados.0.smithi029.stdout:1343: done (3 left) 2024-04-12T11:20:01.916 INFO:tasks.rados.rados.0.smithi029.stdout:1344: done (2 left) 2024-04-12T11:20:01.916 INFO:tasks.rados.rados.0.smithi029.stdout:1345: done (1 left) 2024-04-12T11:20:01.916 INFO:tasks.rados.rados.0.smithi029.stdout:1347: done (0 left) 2024-04-12T11:20:01.916 INFO:tasks.rados.rados.0.smithi029.stdout:1348: snap_remove snap 168 2024-04-12T11:20:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:01 smithi087 ceph-mon[32917]: osdmap e400: 8 total, 8 up, 8 in 2024-04-12T11:20:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:01 smithi029 ceph-mon[93650]: osdmap e400: 8 total, 8 up, 8 in 2024-04-12T11:20:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:01 smithi029 ceph-mon[97787]: osdmap e400: 8 total, 8 up, 8 in 2024-04-12T11:20:02.533 INFO:tasks.rados.rados.0.smithi029.stdout:1348: done (0 left) 2024-04-12T11:20:02.533 INFO:tasks.rados.rados.0.smithi029.stdout:1349: read oid 19 snap 157 2024-04-12T11:20:02.534 INFO:tasks.rados.rados.0.smithi029.stdout:1349: expect (ObjNum 389 snap 121 seq_num 389) 2024-04-12T11:20:02.534 INFO:tasks.rados.rados.0.smithi029.stdout:1350: write oid 21 current snap is 170 2024-04-12T11:20:02.534 INFO:tasks.rados.rados.0.smithi029.stdout:1350: seq_num 525 ranges {423675=734082,1721870=545170,2717777=105898} 2024-04-12T11:20:02.540 INFO:tasks.rados.rados.0.smithi029.stdout:1350: writing smithi02979277-21 from 423675 to 1157757 tid 1 2024-04-12T11:20:02.547 INFO:tasks.rados.rados.0.smithi029.stdout:1350: writing smithi02979277-21 from 1721870 to 2267040 tid 2 2024-04-12T11:20:02.550 INFO:tasks.rados.rados.0.smithi029.stdout:1350: writing smithi02979277-21 from 2717777 to 2823675 tid 3 2024-04-12T11:20:02.550 INFO:tasks.rados.rados.0.smithi029.stdout:1351: write oid 44 current snap is 170 2024-04-12T11:20:02.550 INFO:tasks.rados.rados.0.smithi029.stdout:1351: seq_num 526 ranges {440482=409331,1380352=405181,2461553=378929} 2024-04-12T11:20:02.554 INFO:tasks.rados.rados.0.smithi029.stdout:1351: writing smithi02979277-44 from 440482 to 849813 tid 1 2024-04-12T11:20:02.560 INFO:tasks.rados.rados.0.smithi029.stdout:1351: writing smithi02979277-44 from 1380352 to 1785533 tid 2 2024-04-12T11:20:02.566 INFO:tasks.rados.rados.0.smithi029.stdout:1351: writing smithi02979277-44 from 2461553 to 2840482 tid 3 2024-04-12T11:20:02.566 INFO:tasks.rados.rados.0.smithi029.stdout:1352: read oid 35 snap -1 2024-04-12T11:20:02.566 INFO:tasks.rados.rados.0.smithi029.stdout:1352: expect (ObjNum 449 snap 147 seq_num 449) 2024-04-12T11:20:02.566 INFO:tasks.rados.rados.0.smithi029.stdout:1353: read oid 50 snap -1 2024-04-12T11:20:02.566 INFO:tasks.rados.rados.0.smithi029.stdout:1353: expect deleted 2024-04-12T11:20:02.566 INFO:tasks.rados.rados.0.smithi029.stdout:1354: write oid 26 current snap is 170 2024-04-12T11:20:02.566 INFO:tasks.rados.rados.0.smithi029.stdout:1354: seq_num 527 ranges {457289=484580,1438834=548839,2772623=84666} 2024-04-12T11:20:02.571 INFO:tasks.rados.rados.0.smithi029.stdout:1354: writing smithi02979277-26 from 457289 to 941869 tid 1 2024-04-12T11:20:02.577 INFO:tasks.rados.rados.0.smithi029.stdout:1354: writing smithi02979277-26 from 1438834 to 1987673 tid 2 2024-04-12T11:20:02.578 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:02.582 INFO:tasks.rados.rados.0.smithi029.stdout:1354: writing smithi02979277-26 from 2772623 to 2857289 tid 3 2024-04-12T11:20:02.582 INFO:tasks.rados.rados.0.smithi029.stdout:1355: snap_remove snap 162 2024-04-12T11:20:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:02 smithi087 ceph-mon[32917]: pgmap v235: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:02 smithi087 ceph-mon[32917]: osdmap e401: 8 total, 8 up, 8 in 2024-04-12T11:20:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:02 smithi087 ceph-mon[32917]: osdmap e402: 8 total, 8 up, 8 in 2024-04-12T11:20:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:02 smithi029 ceph-mon[93650]: pgmap v235: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:02 smithi029 ceph-mon[93650]: osdmap e401: 8 total, 8 up, 8 in 2024-04-12T11:20:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:02 smithi029 ceph-mon[93650]: osdmap e402: 8 total, 8 up, 8 in 2024-04-12T11:20:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:02 smithi029 ceph-mon[97787]: pgmap v235: 89 pgs: 89 active+clean; 242 MiB data, 737 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:02 smithi029 ceph-mon[97787]: osdmap e401: 8 total, 8 up, 8 in 2024-04-12T11:20:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:02 smithi029 ceph-mon[97787]: osdmap e402: 8 total, 8 up, 8 in 2024-04-12T11:20:03.586 INFO:tasks.rados.rados.0.smithi029.stdout:1355: done (6 left) 2024-04-12T11:20:03.586 INFO:tasks.rados.rados.0.smithi029.stdout:1356: rollback oid 29 current snap is 170 2024-04-12T11:20:03.586 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 29 to 129 2024-04-12T11:20:03.586 INFO:tasks.rados.rados.0.smithi029.stdout:1349: expect (ObjNum 389 snap 121 seq_num 389) 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:1350: finishing write tid 1 to smithi02979277-21 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:1350: finishing write tid 2 to smithi02979277-21 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:1350: finishing write tid 3 to smithi02979277-21 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:1350: finishing write tid 4 to smithi02979277-21 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:1350: finishing write tid 5 to smithi02979277-21 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:1350: oid 21 updating version 0 to 674 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:1350: oid 21 version 674 is already newer than 671 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:1350: oid 21 version 674 is already newer than 673 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:1350: oid 21 version 674 is already newer than 672 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 674 (ObjNum 525 snap 170 seq_num 525) dirty exists 2024-04-12T11:20:03.748 INFO:tasks.rados.rados.0.smithi029.stdout:1350: left oid 21 (ObjNum 525 snap 170 seq_num 525) 2024-04-12T11:20:03.749 INFO:tasks.rados.rados.0.smithi029.stdout:1351: finishing write tid 1 to smithi02979277-44 2024-04-12T11:20:03.749 INFO:tasks.rados.rados.0.smithi029.stdout:1349: done (6 left) 2024-04-12T11:20:03.749 INFO:tasks.rados.rados.0.smithi029.stdout:1350: done (5 left) 2024-04-12T11:20:03.749 INFO:tasks.rados.rados.0.smithi029.stdout:1357: read oid 10 snap -1 2024-04-12T11:20:03.749 INFO:tasks.rados.rados.0.smithi029.stdout:1357: expect (ObjNum 507 snap 164 seq_num 507) 2024-04-12T11:20:03.749 INFO:tasks.rados.rados.0.smithi029.stdout:1358: write oid 48 current snap is 170 2024-04-12T11:20:03.749 INFO:tasks.rados.rados.0.smithi029.stdout:1351: finishing write tid 2 to smithi02979277-44 2024-04-12T11:20:03.749 INFO:tasks.rados.rados.0.smithi029.stdout:1358: seq_num 528 ranges {474096=559829,1780963=408850,2800046=74050} 2024-04-12T11:20:03.753 INFO:tasks.rados.rados.0.smithi029.stdout:1358: writing smithi02979277-48 from 474096 to 1033925 tid 1 2024-04-12T11:20:03.759 INFO:tasks.rados.rados.0.smithi029.stdout:1358: writing smithi02979277-48 from 1780963 to 2189813 tid 2 2024-04-12T11:20:03.763 INFO:tasks.rados.rados.0.smithi029.stdout:1358: writing smithi02979277-48 from 2800046 to 2874096 tid 3 2024-04-12T11:20:03.763 INFO:tasks.rados.rados.0.smithi029.stdout:1359: write oid 20 current snap is 170 2024-04-12T11:20:03.763 INFO:tasks.rados.rados.0.smithi029.stdout:1351: finishing write tid 3 to smithi02979277-44 2024-04-12T11:20:03.764 INFO:tasks.rados.rados.0.smithi029.stdout:1351: finishing write tid 4 to smithi02979277-44 2024-04-12T11:20:03.764 INFO:tasks.rados.rados.0.smithi029.stdout:1351: finishing write tid 5 to smithi02979277-44 2024-04-12T11:20:03.764 INFO:tasks.rados.rados.0.smithi029.stdout:1351: oid 44 updating version 0 to 677 2024-04-12T11:20:03.764 INFO:tasks.rados.rados.0.smithi029.stdout:1351: oid 44 updating version 677 to 679 2024-04-12T11:20:03.764 INFO:tasks.rados.rados.0.smithi029.stdout:1351: oid 44 version 679 is already newer than 676 2024-04-12T11:20:03.764 INFO:tasks.rados.rados.0.smithi029.stdout:1351: oid 44 version 679 is already newer than 678 2024-04-12T11:20:03.764 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 679 (ObjNum 526 snap 170 seq_num 526) dirty exists 2024-04-12T11:20:03.764 INFO:tasks.rados.rados.0.smithi029.stdout:1351: left oid 44 (ObjNum 526 snap 170 seq_num 526) 2024-04-12T11:20:03.764 INFO:tasks.rados.rados.0.smithi029.stdout:1352: expect (ObjNum 449 snap 147 seq_num 449) 2024-04-12T11:20:03.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:1354: finishing write tid 1 to smithi02979277-26 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:1354: finishing write tid 2 to smithi02979277-26 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:1354: finishing write tid 3 to smithi02979277-26 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:1354: finishing write tid 4 to smithi02979277-26 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:1354: finishing write tid 5 to smithi02979277-26 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:1354: oid 26 updating version 0 to 825 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:1354: oid 26 version 825 is already newer than 823 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:1354: oid 26 version 825 is already newer than 824 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:1354: oid 26 updating version 825 to 826 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 826 (ObjNum 527 snap 170 seq_num 527) dirty exists 2024-04-12T11:20:03.975 INFO:tasks.rados.rados.0.smithi029.stdout:1354: left oid 26 (ObjNum 527 snap 170 seq_num 527) 2024-04-12T11:20:03.976 INFO:tasks.rados.rados.0.smithi029.stdout:1356: finishing rollback tid 1 to smithi02979277-29 2024-04-12T11:20:03.976 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 509 (ObjNum 49 snap 0 seq_num 1777349616) dirty dne 2024-04-12T11:20:03.976 INFO:tasks.rados.rados.0.smithi029.stdout:1359: seq_num 529 ranges {490903=635078,1839445=552508} 2024-04-12T11:20:03.981 INFO:tasks.rados.rados.0.smithi029.stdout:1359: writing smithi02979277-20 from 490903 to 1125981 tid 1 2024-04-12T11:20:03.988 INFO:tasks.rados.rados.0.smithi029.stdout:1359: writing smithi02979277-20 from 1839445 to 2391953 tid 2 2024-04-12T11:20:03.988 INFO:tasks.rados.rados.0.smithi029.stdout:1351: done (7 left) 2024-04-12T11:20:03.988 INFO:tasks.rados.rados.0.smithi029.stdout:1352: done (6 left) 2024-04-12T11:20:03.988 INFO:tasks.rados.rados.0.smithi029.stdout:1353: done (5 left) 2024-04-12T11:20:03.988 INFO:tasks.rados.rados.0.smithi029.stdout:1354: done (4 left) 2024-04-12T11:20:03.988 INFO:tasks.rados.rados.0.smithi029.stdout:1356: done (3 left) 2024-04-12T11:20:03.988 INFO:tasks.rados.rados.0.smithi029.stdout:1360: delete oid 19 current snap is 170 2024-04-12T11:20:03.988 INFO:tasks.rados.rados.0.smithi029.stdout:1357: expect (ObjNum 507 snap 164 seq_num 507) 2024-04-12T11:20:04.151 INFO:tasks.rados.rados.0.smithi029.stdout:1358: finishing write tid 1 to smithi02979277-48 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1358: finishing write tid 2 to smithi02979277-48 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1358: finishing write tid 3 to smithi02979277-48 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1358: finishing write tid 4 to smithi02979277-48 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1358: finishing write tid 5 to smithi02979277-48 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1358: oid 48 updating version 0 to 607 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1358: oid 48 updating version 607 to 608 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1358: oid 48 version 608 is already newer than 605 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1358: oid 48 version 608 is already newer than 606 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 608 (ObjNum 528 snap 170 seq_num 528) dirty exists 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1358: left oid 48 (ObjNum 528 snap 170 seq_num 528) 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1359: finishing write tid 1 to smithi02979277-20 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1359: finishing write tid 2 to smithi02979277-20 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1359: finishing write tid 3 to smithi02979277-20 2024-04-12T11:20:04.152 INFO:tasks.rados.rados.0.smithi029.stdout:1359: finishing write tid 4 to smithi02979277-20 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1359: oid 20 updating version 0 to 356 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1359: oid 20 version 356 is already newer than 354 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1359: oid 20 version 356 is already newer than 355 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 356 (ObjNum 529 snap 170 seq_num 529) dirty exists 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1359: left oid 20 (ObjNum 529 snap 170 seq_num 529) 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1357: done (3 left) 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1358: done (2 left) 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1359: done (1 left) 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1360: done (0 left) 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1361: read oid 14 snap -1 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1361: expect (ObjNum 471 snap 156 seq_num 471) 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1362: read oid 22 snap 164 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1362: expect (ObjNum 495 snap 159 seq_num 495) 2024-04-12T11:20:04.153 INFO:tasks.rados.rados.0.smithi029.stdout:1363: snap_create 2024-04-12T11:20:04.161 INFO:tasks.rados.rados.0.smithi029.stdout:1362: expect (ObjNum 495 snap 159 seq_num 495) 2024-04-12T11:20:04.239 INFO:tasks.rados.rados.0.smithi029.stdout:1361: expect (ObjNum 471 snap 156 seq_num 471) 2024-04-12T11:20:04.591 INFO:tasks.rados.rados.0.smithi029.stdout:1361: done (2 left) 2024-04-12T11:20:04.592 INFO:tasks.rados.rados.0.smithi029.stdout:1362: done (1 left) 2024-04-12T11:20:04.592 INFO:tasks.rados.rados.0.smithi029.stdout:1363: done (0 left) 2024-04-12T11:20:04.592 INFO:tasks.rados.rados.0.smithi029.stdout:1364: read oid 17 snap -1 2024-04-12T11:20:04.592 INFO:tasks.rados.rados.0.smithi029.stdout:1364: expect (ObjNum 492 snap 159 seq_num 492) 2024-04-12T11:20:04.592 INFO:tasks.rados.rados.0.smithi029.stdout:1365: write oid 41 current snap is 171 2024-04-12T11:20:04.592 INFO:tasks.rados.rados.0.smithi029.stdout:1365: seq_num 531 ranges {524517=785576,1956409=556177} 2024-04-12T11:20:04.598 INFO:tasks.rados.rados.0.smithi029.stdout:1365: writing smithi02979277-41 from 524517 to 1310093 tid 1 2024-04-12T11:20:04.605 INFO:tasks.rados.rados.0.smithi029.stdout:1365: writing smithi02979277-41 from 1956409 to 2512586 tid 2 2024-04-12T11:20:04.605 INFO:tasks.rados.rados.0.smithi029.stdout:1366: snap_create 2024-04-12T11:20:04.605 INFO:tasks.rados.rados.0.smithi029.stdout:1364: expect (ObjNum 492 snap 159 seq_num 492) 2024-04-12T11:20:04.763 INFO:tasks.rados.rados.0.smithi029.stdout:1365: finishing write tid 1 to smithi02979277-41 2024-04-12T11:20:04.763 INFO:tasks.rados.rados.0.smithi029.stdout:1365: finishing write tid 2 to smithi02979277-41 2024-04-12T11:20:04.763 INFO:tasks.rados.rados.0.smithi029.stdout:1365: finishing write tid 3 to smithi02979277-41 2024-04-12T11:20:04.763 INFO:tasks.rados.rados.0.smithi029.stdout:1365: finishing write tid 4 to smithi02979277-41 2024-04-12T11:20:04.764 INFO:tasks.rados.rados.0.smithi029.stdout:1365: oid 41 updating version 0 to 687 2024-04-12T11:20:04.764 INFO:tasks.rados.rados.0.smithi029.stdout:1365: oid 41 updating version 687 to 688 2024-04-12T11:20:04.764 INFO:tasks.rados.rados.0.smithi029.stdout:1365: oid 41 version 688 is already newer than 686 2024-04-12T11:20:04.764 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 688 (ObjNum 531 snap 171 seq_num 531) dirty exists 2024-04-12T11:20:04.764 INFO:tasks.rados.rados.0.smithi029.stdout:1365: left oid 41 (ObjNum 531 snap 171 seq_num 531) 2024-04-12T11:20:04.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:04 smithi029 ceph-mon[97787]: pgmap v238: 89 pgs: 89 active+clean; 247 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 955 KiB/s wr, 21 op/s 2024-04-12T11:20:04.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:04 smithi029 ceph-mon[97787]: osdmap e403: 8 total, 8 up, 8 in 2024-04-12T11:20:04.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:04 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:04 smithi029 ceph-mon[93650]: pgmap v238: 89 pgs: 89 active+clean; 247 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 955 KiB/s wr, 21 op/s 2024-04-12T11:20:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:04 smithi029 ceph-mon[93650]: osdmap e403: 8 total, 8 up, 8 in 2024-04-12T11:20:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:04 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:04 smithi087 ceph-mon[32917]: pgmap v238: 89 pgs: 89 active+clean; 247 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 955 KiB/s wr, 21 op/s 2024-04-12T11:20:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:04 smithi087 ceph-mon[32917]: osdmap e403: 8 total, 8 up, 8 in 2024-04-12T11:20:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:04 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:05.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:20:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:20:04] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.43.0" 2024-04-12T11:20:05.596 INFO:tasks.rados.rados.0.smithi029.stdout:1364: done (2 left) 2024-04-12T11:20:05.597 INFO:tasks.rados.rados.0.smithi029.stdout:1365: done (1 left) 2024-04-12T11:20:05.597 INFO:tasks.rados.rados.0.smithi029.stdout:1366: done (0 left) 2024-04-12T11:20:05.597 INFO:tasks.rados.rados.0.smithi029.stdout:1367: write oid 10 current snap is 172 2024-04-12T11:20:05.597 INFO:tasks.rados.rados.0.smithi029.stdout:1367: seq_num 532 ranges {541324=460825,1498538=416188,2393385=547939} 2024-04-12T11:20:05.601 INFO:tasks.rados.rados.0.smithi029.stdout:1367: writing smithi02979277-10 from 541324 to 1002149 tid 1 2024-04-12T11:20:05.607 INFO:tasks.rados.rados.0.smithi029.stdout:1367: writing smithi02979277-10 from 1498538 to 1914726 tid 2 2024-04-12T11:20:05.614 INFO:tasks.rados.rados.0.smithi029.stdout:1367: writing smithi02979277-10 from 2393385 to 2941324 tid 3 2024-04-12T11:20:05.614 INFO:tasks.rados.rados.0.smithi029.stdout:1368: delete oid 29 current snap is 172 2024-04-12T11:20:05.614 INFO:tasks.rados.rados.0.smithi029.stdout:1367: finishing write tid 1 to smithi02979277-10 2024-04-12T11:20:05.614 INFO:tasks.rados.rados.0.smithi029.stdout:1367: finishing write tid 2 to smithi02979277-10 2024-04-12T11:20:05.616 INFO:tasks.rados.rados.0.smithi029.stdout:1367: finishing write tid 3 to smithi02979277-10 2024-04-12T11:20:05.616 INFO:tasks.rados.rados.0.smithi029.stdout:1367: finishing write tid 4 to smithi02979277-10 2024-04-12T11:20:05.616 INFO:tasks.rados.rados.0.smithi029.stdout:1367: finishing write tid 5 to smithi02979277-10 2024-04-12T11:20:05.616 INFO:tasks.rados.rados.0.smithi029.stdout:1367: oid 10 updating version 0 to 613 2024-04-12T11:20:05.616 INFO:tasks.rados.rados.0.smithi029.stdout:1367: oid 10 version 613 is already newer than 612 2024-04-12T11:20:05.616 INFO:tasks.rados.rados.0.smithi029.stdout:1367: oid 10 version 613 is already newer than 610 2024-04-12T11:20:05.616 INFO:tasks.rados.rados.0.smithi029.stdout:1367: oid 10 version 613 is already newer than 611 2024-04-12T11:20:05.616 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 613 (ObjNum 532 snap 172 seq_num 532) dirty exists 2024-04-12T11:20:05.616 INFO:tasks.rados.rados.0.smithi029.stdout:1367: left oid 10 (ObjNum 532 snap 172 seq_num 532) 2024-04-12T11:20:05.617 INFO:tasks.rados.rados.0.smithi029.stdout:1367: done (1 left) 2024-04-12T11:20:05.617 INFO:tasks.rados.rados.0.smithi029.stdout:1368: done (0 left) 2024-04-12T11:20:05.617 INFO:tasks.rados.rados.0.smithi029.stdout:1369: read oid 27 snap -1 2024-04-12T11:20:05.617 INFO:tasks.rados.rados.0.smithi029.stdout:1369: expect deleted 2024-04-12T11:20:05.617 INFO:tasks.rados.rados.0.smithi029.stdout:1370: snap_create 2024-04-12T11:20:05.911 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:05 smithi087 ceph-mon[32917]: osdmap e404: 8 total, 8 up, 8 in 2024-04-12T11:20:06.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:05 smithi029 ceph-mon[93650]: osdmap e404: 8 total, 8 up, 8 in 2024-04-12T11:20:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:05 smithi029 ceph-mon[97787]: osdmap e404: 8 total, 8 up, 8 in 2024-04-12T11:20:06.599 INFO:tasks.rados.rados.0.smithi029.stdout:1369: done (1 left) 2024-04-12T11:20:06.600 INFO:tasks.rados.rados.0.smithi029.stdout:1370: done (0 left) 2024-04-12T11:20:06.600 INFO:tasks.rados.rados.0.smithi029.stdout:1371: read oid 41 snap -1 2024-04-12T11:20:06.600 INFO:tasks.rados.rados.0.smithi029.stdout:1371: expect (ObjNum 531 snap 171 seq_num 531) 2024-04-12T11:20:06.600 INFO:tasks.rados.rados.0.smithi029.stdout:1372: delete oid 20 current snap is 173 2024-04-12T11:20:06.602 INFO:tasks.rados.rados.0.smithi029.stdout:1372: done (1 left) 2024-04-12T11:20:06.603 INFO:tasks.rados.rados.0.smithi029.stdout:1373: write oid 17 current snap is 173 2024-04-12T11:20:06.603 INFO:tasks.rados.rados.0.smithi029.stdout:1373: seq_num 535 ranges {591745=402925,1790337=563515} 2024-04-12T11:20:06.608 INFO:tasks.rados.rados.0.smithi029.stdout:1373: writing smithi02979277-17 from 591745 to 994670 tid 1 2024-04-12T11:20:06.615 INFO:tasks.rados.rados.0.smithi029.stdout:1373: writing smithi02979277-17 from 1790337 to 2353852 tid 2 2024-04-12T11:20:06.615 INFO:tasks.rados.rados.0.smithi029.stdout:1374: write oid 8 current snap is 173 2024-04-12T11:20:06.616 INFO:tasks.rados.rados.0.smithi029.stdout:1374: seq_num 536 ranges {608552=478174,1848819=707173} 2024-04-12T11:20:06.621 INFO:tasks.rados.rados.0.smithi029.stdout:1374: writing smithi02979277-8 from 608552 to 1086726 tid 1 2024-04-12T11:20:06.629 INFO:tasks.rados.rados.0.smithi029.stdout:1374: writing smithi02979277-8 from 1848819 to 2555992 tid 2 2024-04-12T11:20:06.630 INFO:tasks.rados.rados.0.smithi029.stdout:1375: rollback oid 40 current snap is 173 2024-04-12T11:20:06.630 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 40 to 166 2024-04-12T11:20:06.630 INFO:tasks.rados.rados.0.smithi029.stdout:1371: expect (ObjNum 531 snap 171 seq_num 531) 2024-04-12T11:20:06.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:06 smithi087 ceph-mon[32917]: pgmap v241: 89 pgs: 89 active+clean; 247 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 955 KiB/s wr, 21 op/s 2024-04-12T11:20:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:06 smithi087 ceph-mon[32917]: osdmap e405: 8 total, 8 up, 8 in 2024-04-12T11:20:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:06 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:06.727 INFO:tasks.rados.rados.0.smithi029.stdout:1373: finishing write tid 1 to smithi02979277-17 2024-04-12T11:20:06.727 INFO:tasks.rados.rados.0.smithi029.stdout:1373: finishing write tid 2 to smithi02979277-17 2024-04-12T11:20:06.727 INFO:tasks.rados.rados.0.smithi029.stdout:1371: done (3 left) 2024-04-12T11:20:06.727 INFO:tasks.rados.rados.0.smithi029.stdout:1376: write oid 45 current snap is 173 2024-04-12T11:20:06.727 INFO:tasks.rados.rados.0.smithi029.stdout:1373: finishing write tid 3 to smithi02979277-17 2024-04-12T11:20:06.727 INFO:tasks.rados.rados.0.smithi029.stdout:1376: seq_num 537 ranges {625359=553423,1790948=567184,2930500=94859} 2024-04-12T11:20:06.733 INFO:tasks.rados.rados.0.smithi029.stdout:1376: writing smithi02979277-45 from 625359 to 1178782 tid 1 2024-04-12T11:20:06.739 INFO:tasks.rados.rados.0.smithi029.stdout:1376: writing smithi02979277-45 from 1790948 to 2358132 tid 2 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1376: writing smithi02979277-45 from 2930500 to 3025359 tid 3 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1377: snap_create 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1373: finishing write tid 4 to smithi02979277-17 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1373: oid 17 updating version 0 to 698 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1373: oid 17 updating version 698 to 699 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1373: oid 17 updating version 699 to 700 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 700 (ObjNum 535 snap 173 seq_num 535) dirty exists 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1373: left oid 17 (ObjNum 535 snap 173 seq_num 535) 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1374: finishing write tid 1 to smithi02979277-8 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1375: finishing rollback tid 0 to smithi02979277-40 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1375: finishing rollback tid 1 to smithi02979277-40 2024-04-12T11:20:06.744 INFO:tasks.rados.rados.0.smithi029.stdout:1375: finishing rollback tid 2 to smithi02979277-40 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 830 (ObjNum 458 snap 150 seq_num 458) dirty exists 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:1374: finishing write tid 2 to smithi02979277-8 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:1374: finishing write tid 3 to smithi02979277-8 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:1374: finishing write tid 4 to smithi02979277-8 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:1374: oid 8 updating version 0 to 108 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:1374: oid 8 version 108 is already newer than 107 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:1374: oid 8 version 108 is already newer than 106 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 108 (ObjNum 536 snap 173 seq_num 536) dirty exists 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:1374: left oid 8 (ObjNum 536 snap 173 seq_num 536) 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:1376: finishing write tid 1 to smithi02979277-45 2024-04-12T11:20:06.745 INFO:tasks.rados.rados.0.smithi029.stdout:1376: finishing write tid 2 to smithi02979277-45 2024-04-12T11:20:06.746 INFO:tasks.rados.rados.0.smithi029.stdout:1376: finishing write tid 3 to smithi02979277-45 2024-04-12T11:20:06.746 INFO:tasks.rados.rados.0.smithi029.stdout:1376: finishing write tid 4 to smithi02979277-45 2024-04-12T11:20:06.746 INFO:tasks.rados.rados.0.smithi029.stdout:1376: finishing write tid 5 to smithi02979277-45 2024-04-12T11:20:06.746 INFO:tasks.rados.rados.0.smithi029.stdout:1376: oid 45 updating version 0 to 630 2024-04-12T11:20:06.746 INFO:tasks.rados.rados.0.smithi029.stdout:1376: oid 45 updating version 630 to 632 2024-04-12T11:20:06.746 INFO:tasks.rados.rados.0.smithi029.stdout:1376: oid 45 version 632 is already newer than 629 2024-04-12T11:20:06.746 INFO:tasks.rados.rados.0.smithi029.stdout:1376: oid 45 version 632 is already newer than 631 2024-04-12T11:20:06.746 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 632 (ObjNum 537 snap 173 seq_num 537) dirty exists 2024-04-12T11:20:06.747 INFO:tasks.rados.rados.0.smithi029.stdout:1376: left oid 45 (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:20:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:06 smithi029 ceph-mon[93650]: pgmap v241: 89 pgs: 89 active+clean; 247 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 955 KiB/s wr, 21 op/s 2024-04-12T11:20:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:06 smithi029 ceph-mon[93650]: osdmap e405: 8 total, 8 up, 8 in 2024-04-12T11:20:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:06.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:06 smithi029 ceph-mon[97787]: pgmap v241: 89 pgs: 89 active+clean; 247 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 955 KiB/s wr, 21 op/s 2024-04-12T11:20:06.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:06 smithi029 ceph-mon[97787]: osdmap e405: 8 total, 8 up, 8 in 2024-04-12T11:20:06.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:06.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch upgrade resume' 2024-04-12T11:20:07.457 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:20:07.533 INFO:tasks.rados.rados.0.smithi029.stdout:1373: done (4 left) 2024-04-12T11:20:07.533 INFO:tasks.rados.rados.0.smithi029.stdout:1374: done (3 left) 2024-04-12T11:20:07.533 INFO:tasks.rados.rados.0.smithi029.stdout:1375: done (2 left) 2024-04-12T11:20:07.533 INFO:tasks.rados.rados.0.smithi029.stdout:1376: done (1 left) 2024-04-12T11:20:07.533 INFO:tasks.rados.rados.0.smithi029.stdout:1377: done (0 left) 2024-04-12T11:20:07.533 INFO:tasks.rados.rados.0.smithi029.stdout:1378: delete oid 36 current snap is 174 2024-04-12T11:20:07.537 INFO:tasks.rados.rados.0.smithi029.stdout:1378: done (0 left) 2024-04-12T11:20:07.537 INFO:tasks.rados.rados.0.smithi029.stdout:1379: read oid 1 snap 167 2024-04-12T11:20:07.537 INFO:tasks.rados.rados.0.smithi029.stdout:1379: expect (ObjNum 509 snap 164 seq_num 509) 2024-04-12T11:20:07.537 INFO:tasks.rados.rados.0.smithi029.stdout:1380: write oid 35 current snap is 174 2024-04-12T11:20:07.538 INFO:tasks.rados.rados.0.smithi029.stdout:1380: seq_num 539 ranges {658973=703921,1907912=570853,2985346=73627} 2024-04-12T11:20:07.545 INFO:tasks.rados.rados.0.smithi029.stdout:1380: writing smithi02979277-35 from 658973 to 1362894 tid 1 2024-04-12T11:20:07.551 INFO:tasks.rados.rados.0.smithi029.stdout:1380: writing smithi02979277-35 from 1907912 to 2478765 tid 2 2024-04-12T11:20:07.555 INFO:tasks.rados.rados.0.smithi029.stdout:1380: writing smithi02979277-35 from 2985346 to 3058973 tid 3 2024-04-12T11:20:07.555 INFO:tasks.rados.rados.0.smithi029.stdout:1381: read oid 45 snap -1 2024-04-12T11:20:07.555 INFO:tasks.rados.rados.0.smithi029.stdout:1381: expect (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:20:07.555 INFO:tasks.rados.rados.0.smithi029.stdout:1382: write oid 41 current snap is 174 2024-04-12T11:20:07.555 INFO:tasks.rados.rados.0.smithi029.stdout:1382: seq_num 540 ranges {675780=495523,1682747=714511,3012769=63011} 2024-04-12T11:20:07.561 INFO:tasks.rados.rados.0.smithi029.stdout:1382: writing smithi02979277-41 from 675780 to 1171303 tid 1 2024-04-12T11:20:07.568 INFO:tasks.rados.rados.0.smithi029.stdout:1382: writing smithi02979277-41 from 1682747 to 2397258 tid 2 2024-04-12T11:20:07.572 INFO:tasks.rados.rados.0.smithi029.stdout:1382: writing smithi02979277-41 from 3012769 to 3075780 tid 3 2024-04-12T11:20:07.572 INFO:tasks.rados.rados.0.smithi029.stdout:1383: delete oid 24 current snap is 174 2024-04-12T11:20:07.572 INFO:tasks.rados.rados.0.smithi029.stdout:1379: expect (ObjNum 509 snap 164 seq_num 509) 2024-04-12T11:20:07.686 INFO:tasks.rados.rados.0.smithi029.stdout:1380: finishing write tid 1 to smithi02979277-35 2024-04-12T11:20:07.686 INFO:tasks.rados.rados.0.smithi029.stdout:1380: finishing write tid 2 to smithi02979277-35 2024-04-12T11:20:07.686 INFO:tasks.rados.rados.0.smithi029.stdout:1380: finishing write tid 3 to smithi02979277-35 2024-04-12T11:20:07.686 INFO:tasks.rados.rados.0.smithi029.stdout:1380: finishing write tid 4 to smithi02979277-35 2024-04-12T11:20:07.686 INFO:tasks.rados.rados.0.smithi029.stdout:1380: finishing write tid 5 to smithi02979277-35 2024-04-12T11:20:07.686 INFO:tasks.rados.rados.0.smithi029.stdout:1380: oid 35 updating version 0 to 835 2024-04-12T11:20:07.686 INFO:tasks.rados.rados.0.smithi029.stdout:1380: oid 35 version 835 is already newer than 832 2024-04-12T11:20:07.686 INFO:tasks.rados.rados.0.smithi029.stdout:1380: oid 35 version 835 is already newer than 833 2024-04-12T11:20:07.686 INFO:tasks.rados.rados.0.smithi029.stdout:1380: oid 35 version 835 is already newer than 834 2024-04-12T11:20:07.686 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 835 (ObjNum 539 snap 174 seq_num 539) dirty exists 2024-04-12T11:20:07.687 INFO:tasks.rados.rados.0.smithi029.stdout:1380: left oid 35 (ObjNum 539 snap 174 seq_num 539) 2024-04-12T11:20:07.687 INFO:tasks.rados.rados.0.smithi029.stdout:1381: expect (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:20:07.790 INFO:tasks.rados.rados.0.smithi029.stdout:1382: finishing write tid 1 to smithi02979277-41 2024-04-12T11:20:07.790 INFO:tasks.rados.rados.0.smithi029.stdout:1382: finishing write tid 2 to smithi02979277-41 2024-04-12T11:20:07.790 INFO:tasks.rados.rados.0.smithi029.stdout:1382: finishing write tid 3 to smithi02979277-41 2024-04-12T11:20:07.790 INFO:tasks.rados.rados.0.smithi029.stdout:1382: finishing write tid 4 to smithi02979277-41 2024-04-12T11:20:07.790 INFO:tasks.rados.rados.0.smithi029.stdout:1382: finishing write tid 5 to smithi02979277-41 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1382: oid 41 updating version 0 to 693 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1382: oid 41 version 693 is already newer than 691 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1382: oid 41 version 693 is already newer than 692 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1382: oid 41 version 693 is already newer than 690 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 693 (ObjNum 540 snap 174 seq_num 540) dirty exists 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1382: left oid 41 (ObjNum 540 snap 174 seq_num 540) 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1379: done (4 left) 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1380: done (3 left) 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1381: done (2 left) 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1382: done (1 left) 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1383: done (0 left) 2024-04-12T11:20:07.791 INFO:tasks.rados.rados.0.smithi029.stdout:1384: snap_remove snap 169 2024-04-12T11:20:07.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:07 smithi029 ceph-mon[97787]: osdmap e406: 8 total, 8 up, 8 in 2024-04-12T11:20:07.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:07 smithi029 ceph-mon[97787]: osdmap e407: 8 total, 8 up, 8 in 2024-04-12T11:20:07.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:07 smithi029 ceph-mon[93650]: osdmap e406: 8 total, 8 up, 8 in 2024-04-12T11:20:07.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:07 smithi029 ceph-mon[93650]: osdmap e407: 8 total, 8 up, 8 in 2024-04-12T11:20:07.913 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:07 smithi087 ceph-mon[32917]: osdmap e406: 8 total, 8 up, 8 in 2024-04-12T11:20:07.913 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:07 smithi087 ceph-mon[32917]: osdmap e407: 8 total, 8 up, 8 in 2024-04-12T11:20:08.612 INFO:tasks.rados.rados.0.smithi029.stdout:1384: done (0 left) 2024-04-12T11:20:08.613 INFO:tasks.rados.rados.0.smithi029.stdout:1385: snap_create 2024-04-12T11:20:08.887 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:08 smithi029 ceph-mon[97787]: pgmap v244: 89 pgs: 89 active+clean; 265 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 29 op/s 2024-04-12T11:20:08.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:08 smithi029 ceph-mon[93650]: pgmap v244: 89 pgs: 89 active+clean; 265 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 29 op/s 2024-04-12T11:20:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:08 smithi087 ceph-mon[32917]: pgmap v244: 89 pgs: 89 active+clean; 265 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 29 op/s 2024-04-12T11:20:09.613 INFO:tasks.rados.rados.0.smithi029.stdout:1385: done (0 left) 2024-04-12T11:20:09.613 INFO:tasks.rados.rados.0.smithi029.stdout:1386: delete oid 44 current snap is 175 2024-04-12T11:20:09.619 INFO:tasks.rados.rados.0.smithi029.stdout:1386: done (0 left) 2024-04-12T11:20:09.620 INFO:tasks.rados.rados.0.smithi029.stdout:1387: write oid 12 current snap is 175 2024-04-12T11:20:09.620 INFO:tasks.rados.rados.0.smithi029.stdout:1387: seq_num 543 ranges {726201=721270,2141840=578191} 2024-04-12T11:20:09.627 INFO:tasks.rados.rados.0.smithi029.stdout:1387: writing smithi02979277-12 from 726201 to 1447471 tid 1 2024-04-12T11:20:09.635 INFO:tasks.rados.rados.0.smithi029.stdout:1387: writing smithi02979277-12 from 2141840 to 2720031 tid 2 2024-04-12T11:20:09.635 INFO:tasks.rados.rados.0.smithi029.stdout:1388: read oid 40 snap -1 2024-04-12T11:20:09.635 INFO:tasks.rados.rados.0.smithi029.stdout:1388: expect (ObjNum 458 snap 150 seq_num 458) 2024-04-12T11:20:09.635 INFO:tasks.rados.rados.0.smithi029.stdout:1389: rollback oid 2 current snap is 175 2024-04-12T11:20:09.636 INFO:tasks.rados.rados.0.smithi029.stdout:1387: finishing write tid 1 to smithi02979277-12 2024-04-12T11:20:09.636 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 2 to 167 2024-04-12T11:20:09.636 INFO:tasks.rados.rados.0.smithi029.stdout:1390: rollback oid 42 current snap is 175 2024-04-12T11:20:09.636 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 164 2024-04-12T11:20:09.636 INFO:tasks.rados.rados.0.smithi029.stdout:1391: read oid 45 snap -1 2024-04-12T11:20:09.636 INFO:tasks.rados.rados.0.smithi029.stdout:1391: expect (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:20:09.636 INFO:tasks.rados.rados.0.smithi029.stdout:1392: write oid 23 current snap is 175 2024-04-12T11:20:09.636 INFO:tasks.rados.rados.0.smithi029.stdout:1392: seq_num 544 ranges {743008=796519,2200322=721849} 2024-04-12T11:20:09.644 INFO:tasks.rados.rados.0.smithi029.stdout:1392: writing smithi02979277-23 from 743008 to 1539527 tid 1 2024-04-12T11:20:09.652 INFO:tasks.rados.rados.0.smithi029.stdout:1392: writing smithi02979277-23 from 2200322 to 2922171 tid 2 2024-04-12T11:20:09.652 INFO:tasks.rados.rados.0.smithi029.stdout:1393: snap_remove snap 160 2024-04-12T11:20:09.706 INFO:teuthology.orchestra.run.smithi029.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:20:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:09 smithi087 ceph-mon[32917]: osdmap e408: 8 total, 8 up, 8 in 2024-04-12T11:20:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:09 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:09 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:10.003 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:09 smithi029 ceph-mon[93650]: osdmap e408: 8 total, 8 up, 8 in 2024-04-12T11:20:10.003 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:10.003 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:10.003 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:09 smithi029 ceph-mon[97787]: osdmap e408: 8 total, 8 up, 8 in 2024-04-12T11:20:10.003 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:10.003 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:10.452 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'echo wait for all mons to upgrade' 2024-04-12T11:20:10.622 INFO:tasks.rados.rados.0.smithi029.stdout:1393: done (6 left) 2024-04-12T11:20:10.622 INFO:tasks.rados.rados.0.smithi029.stdout:1394: delete oid 19 current snap is 175 2024-04-12T11:20:10.623 INFO:tasks.rados.rados.0.smithi029.stdout:1390: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:20:10.623 INFO:tasks.rados.rados.0.smithi029.stdout:1390: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:20:10.623 INFO:tasks.rados.rados.0.smithi029.stdout:1390: finishing rollback tid 2 to smithi02979277-42 2024-04-12T11:20:10.623 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 233 (ObjNum 477 snap 156 seq_num 477) dirty exists 2024-04-12T11:20:10.623 INFO:tasks.rados.rados.0.smithi029.stdout:1391: expect (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:20:10.729 INFO:tasks.rados.rados.0.smithi029.stdout:1387: finishing write tid 2 to smithi02979277-12 2024-04-12T11:20:10.729 INFO:tasks.rados.rados.0.smithi029.stdout:1387: finishing write tid 3 to smithi02979277-12 2024-04-12T11:20:10.730 INFO:tasks.rados.rados.0.smithi029.stdout:1387: finishing write tid 4 to smithi02979277-12 2024-04-12T11:20:10.730 INFO:tasks.rados.rados.0.smithi029.stdout:1387: oid 12 updating version 0 to 635 2024-04-12T11:20:10.730 INFO:tasks.rados.rados.0.smithi029.stdout:1387: oid 12 updating version 635 to 636 2024-04-12T11:20:10.730 INFO:tasks.rados.rados.0.smithi029.stdout:1387: oid 12 version 636 is already newer than 634 2024-04-12T11:20:10.730 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 636 (ObjNum 543 snap 175 seq_num 543) dirty exists 2024-04-12T11:20:10.730 INFO:tasks.rados.rados.0.smithi029.stdout:1387: left oid 12 (ObjNum 543 snap 175 seq_num 543) 2024-04-12T11:20:10.730 INFO:tasks.rados.rados.0.smithi029.stdout:1388: expect (ObjNum 458 snap 150 seq_num 458) 2024-04-12T11:20:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:10 smithi029 ceph-mon[97787]: pgmap v247: 89 pgs: 89 active+clean; 265 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 29 op/s 2024-04-12T11:20:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:10 smithi029 ceph-mon[97787]: osdmap e409: 8 total, 8 up, 8 in 2024-04-12T11:20:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:10 smithi029 ceph-mon[97787]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:20:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:10 smithi029 ceph-mon[97787]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:20:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:10 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:10 smithi029 ceph-mon[93650]: pgmap v247: 89 pgs: 89 active+clean; 265 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 29 op/s 2024-04-12T11:20:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:10 smithi029 ceph-mon[93650]: osdmap e409: 8 total, 8 up, 8 in 2024-04-12T11:20:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:10 smithi029 ceph-mon[93650]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:20:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:10 smithi029 ceph-mon[93650]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:20:10.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:10 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:10.854 INFO:tasks.rados.rados.0.smithi029.stdout:1389: finishing rollback tid 0 to smithi02979277-2 2024-04-12T11:20:10.854 INFO:tasks.rados.rados.0.smithi029.stdout:1389: finishing rollback tid 1 to smithi02979277-2 2024-04-12T11:20:10.854 INFO:tasks.rados.rados.0.smithi029.stdout:1389: finishing rollback tid 2 to smithi02979277-2 2024-04-12T11:20:10.854 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 134 (ObjNum 479 snap 156 seq_num 479) dirty exists 2024-04-12T11:20:10.854 INFO:tasks.rados.rados.0.smithi029.stdout:1392: finishing write tid 1 to smithi02979277-23 2024-04-12T11:20:10.854 INFO:tasks.rados.rados.0.smithi029.stdout:1392: finishing write tid 2 to smithi02979277-23 2024-04-12T11:20:10.854 INFO:tasks.rados.rados.0.smithi029.stdout:1392: finishing write tid 3 to smithi02979277-23 2024-04-12T11:20:10.854 INFO:tasks.rados.rados.0.smithi029.stdout:1392: finishing write tid 4 to smithi02979277-23 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1392: oid 23 updating version 0 to 435 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1392: oid 23 version 435 is already newer than 434 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1392: oid 23 version 435 is already newer than 433 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 435 (ObjNum 544 snap 175 seq_num 544) dirty exists 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1392: left oid 23 (ObjNum 544 snap 175 seq_num 544) 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1387: done (6 left) 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1388: done (5 left) 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1389: done (4 left) 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1390: done (3 left) 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1391: done (2 left) 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1392: done (1 left) 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1394: done (0 left) 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1395: read oid 9 snap -1 2024-04-12T11:20:10.855 INFO:tasks.rados.rados.0.smithi029.stdout:1395: expect (ObjNum 268 snap 86 seq_num 268) 2024-04-12T11:20:10.856 INFO:tasks.rados.rados.0.smithi029.stdout:1396: read oid 16 snap -1 2024-04-12T11:20:10.856 INFO:tasks.rados.rados.0.smithi029.stdout:1396: expect (ObjNum 444 snap 144 seq_num 444) 2024-04-12T11:20:10.856 INFO:tasks.rados.rados.0.smithi029.stdout:1397: write oid 24 current snap is 175 2024-04-12T11:20:10.856 INFO:tasks.rados.rados.0.smithi029.stdout:1397: seq_num 546 ranges {776622=547017,1800933=725518,2944601=232021} 2024-04-12T11:20:10.861 INFO:tasks.rados.rados.0.smithi029.stdout:1397: writing smithi02979277-24 from 776622 to 1323639 tid 1 2024-04-12T11:20:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:1397: writing smithi02979277-24 from 1800933 to 2526451 tid 2 2024-04-12T11:20:10.872 INFO:tasks.rados.rados.0.smithi029.stdout:1397: writing smithi02979277-24 from 2944601 to 3176622 tid 3 2024-04-12T11:20:10.872 INFO:tasks.rados.rados.0.smithi029.stdout:1398: snap_remove snap 174 2024-04-12T11:20:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:10 smithi087 ceph-mon[32917]: pgmap v247: 89 pgs: 89 active+clean; 265 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.9 MiB/s wr, 29 op/s 2024-04-12T11:20:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:10 smithi087 ceph-mon[32917]: osdmap e409: 8 total, 8 up, 8 in 2024-04-12T11:20:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:10 smithi087 ceph-mon[32917]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:20:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:10 smithi087 ceph-mon[32917]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:20:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:10 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:11.331 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:20:11.629 INFO:tasks.rados.rados.0.smithi029.stdout:1398: done (3 left) 2024-04-12T11:20:11.629 INFO:tasks.rados.rados.0.smithi029.stdout:1399: snap_remove snap 173 2024-04-12T11:20:11.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:11 smithi029 ceph-mon[97787]: osdmap e410: 8 total, 8 up, 8 in 2024-04-12T11:20:11.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:11 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:11.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:11 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:11.741 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:11 smithi029 ceph-mon[93650]: osdmap e410: 8 total, 8 up, 8 in 2024-04-12T11:20:11.742 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:11 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:11.742 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:11 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:11 smithi087 ceph-mon[32917]: osdmap e410: 8 total, 8 up, 8 in 2024-04-12T11:20:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:11 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:11 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:12.627 INFO:tasks.rados.rados.0.smithi029.stdout:1399: done (3 left) 2024-04-12T11:20:12.628 INFO:tasks.rados.rados.0.smithi029.stdout:1400: write oid 36 current snap is 175 2024-04-12T11:20:12.628 INFO:tasks.rados.rados.0.smithi029.stdout:1400: seq_num 547 ranges {793429=622266,1859415=469176,2972024=221405} 2024-04-12T11:20:12.630 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:12.635 INFO:tasks.rados.rados.0.smithi029.stdout:1400: writing smithi02979277-36 from 793429 to 1415695 tid 1 2024-04-12T11:20:12.641 INFO:tasks.rados.rados.0.smithi029.stdout:1400: writing smithi02979277-36 from 1859415 to 2328591 tid 2 2024-04-12T11:20:12.646 INFO:tasks.rados.rados.0.smithi029.stdout:1400: writing smithi02979277-36 from 2972024 to 3193429 tid 3 2024-04-12T11:20:12.646 INFO:tasks.rados.rados.0.smithi029.stdout:1401: snap_create 2024-04-12T11:20:12.646 INFO:tasks.rados.rados.0.smithi029.stdout:1395: expect (ObjNum 268 snap 86 seq_num 268) 2024-04-12T11:20:12.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:12 smithi029 ceph-mon[97787]: pgmap v250: 89 pgs: 89 active+clean; 265 MiB data, 776 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:12.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:12 smithi029 ceph-mon[97787]: osdmap e411: 8 total, 8 up, 8 in 2024-04-12T11:20:12.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:12 smithi029 ceph-mon[93650]: pgmap v250: 89 pgs: 89 active+clean; 265 MiB data, 776 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:12.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:12 smithi029 ceph-mon[93650]: osdmap e411: 8 total, 8 up, 8 in 2024-04-12T11:20:12.812 INFO:tasks.rados.rados.0.smithi029.stdout:1396: expect (ObjNum 444 snap 144 seq_num 444) 2024-04-12T11:20:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:12 smithi087 ceph-mon[32917]: pgmap v250: 89 pgs: 89 active+clean; 265 MiB data, 776 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:12 smithi087 ceph-mon[32917]: osdmap e411: 8 total, 8 up, 8 in 2024-04-12T11:20:13.043 INFO:teuthology.orchestra.run.smithi029.stdout:wait for all mons to upgrade 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1397: finishing write tid 1 to smithi02979277-24 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1397: finishing write tid 2 to smithi02979277-24 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1397: finishing write tid 3 to smithi02979277-24 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1397: finishing write tid 4 to smithi02979277-24 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1397: finishing write tid 5 to smithi02979277-24 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1397: oid 24 updating version 0 to 850 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1397: oid 24 updating version 850 to 853 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1397: oid 24 version 853 is already newer than 851 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1397: oid 24 version 853 is already newer than 852 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 853 (ObjNum 546 snap 175 seq_num 546) dirty exists 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1397: left oid 24 (ObjNum 546 snap 175 seq_num 546) 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1400: finishing write tid 1 to smithi02979277-36 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1400: finishing write tid 2 to smithi02979277-36 2024-04-12T11:20:13.050 INFO:tasks.rados.rados.0.smithi029.stdout:1400: finishing write tid 3 to smithi02979277-36 2024-04-12T11:20:13.051 INFO:tasks.rados.rados.0.smithi029.stdout:1400: finishing write tid 4 to smithi02979277-36 2024-04-12T11:20:13.051 INFO:tasks.rados.rados.0.smithi029.stdout:1400: finishing write tid 5 to smithi02979277-36 2024-04-12T11:20:13.051 INFO:tasks.rados.rados.0.smithi029.stdout:1400: oid 36 updating version 0 to 319 2024-04-12T11:20:13.051 INFO:tasks.rados.rados.0.smithi029.stdout:1400: oid 36 updating version 319 to 320 2024-04-12T11:20:13.051 INFO:tasks.rados.rados.0.smithi029.stdout:1400: oid 36 version 320 is already newer than 318 2024-04-12T11:20:13.051 INFO:tasks.rados.rados.0.smithi029.stdout:1400: oid 36 version 320 is already newer than 315 2024-04-12T11:20:13.051 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 320 (ObjNum 547 snap 175 seq_num 547) dirty exists 2024-04-12T11:20:13.051 INFO:tasks.rados.rados.0.smithi029.stdout:1400: left oid 36 (ObjNum 547 snap 175 seq_num 547) 2024-04-12T11:20:13.638 INFO:tasks.rados.rados.0.smithi029.stdout:1395: done (4 left) 2024-04-12T11:20:13.639 INFO:tasks.rados.rados.0.smithi029.stdout:1396: done (3 left) 2024-04-12T11:20:13.639 INFO:tasks.rados.rados.0.smithi029.stdout:1397: done (2 left) 2024-04-12T11:20:13.639 INFO:tasks.rados.rados.0.smithi029.stdout:1400: done (1 left) 2024-04-12T11:20:13.639 INFO:tasks.rados.rados.0.smithi029.stdout:1401: done (0 left) 2024-04-12T11:20:13.639 INFO:tasks.rados.rados.0.smithi029.stdout:1402: write oid 24 current snap is 176 2024-04-12T11:20:13.639 INFO:tasks.rados.rados.0.smithi029.stdout:1402: seq_num 548 ranges {410236=413868,1234250=729187,2715800=423063} 2024-04-12T11:20:13.642 INFO:tasks.rados.rados.0.smithi029.stdout:1402: writing smithi02979277-24 from 410236 to 824104 tid 1 2024-04-12T11:20:13.649 INFO:tasks.rados.rados.0.smithi029.stdout:1402: writing smithi02979277-24 from 1234250 to 1963437 tid 2 2024-04-12T11:20:13.656 INFO:tasks.rados.rados.0.smithi029.stdout:1402: writing smithi02979277-24 from 2715800 to 3138863 tid 3 2024-04-12T11:20:13.656 INFO:tasks.rados.rados.0.smithi029.stdout:1403: read oid 49 snap -1 2024-04-12T11:20:13.656 INFO:tasks.rados.rados.0.smithi029.stdout:1403: expect (ObjNum 448 snap 147 seq_num 448) 2024-04-12T11:20:13.656 INFO:tasks.rados.rados.0.smithi029.stdout:1404: write oid 9 current snap is 176 2024-04-12T11:20:13.657 INFO:tasks.rados.rados.0.smithi029.stdout:1402: finishing write tid 1 to smithi02979277-24 2024-04-12T11:20:13.657 INFO:tasks.rados.rados.0.smithi029.stdout:1404: seq_num 549 ranges {427043=489117,1576379=472845,2626870=600173} 2024-04-12T11:20:13.661 INFO:tasks.rados.rados.0.smithi029.stdout:1404: writing smithi02979277-9 from 427043 to 916160 tid 1 2024-04-12T11:20:13.667 INFO:tasks.rados.rados.0.smithi029.stdout:1404: writing smithi02979277-9 from 1576379 to 2049224 tid 2 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1404: writing smithi02979277-9 from 2626870 to 3227043 tid 3 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1405: read oid 12 snap 141 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1405: expect (ObjNum 367 snap 117 seq_num 367) 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1406: delete oid 10 current snap is 176 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1402: finishing write tid 2 to smithi02979277-24 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1402: finishing write tid 3 to smithi02979277-24 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1402: finishing write tid 4 to smithi02979277-24 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1402: finishing write tid 5 to smithi02979277-24 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1402: oid 24 updating version 0 to 859 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1402: oid 24 updating version 859 to 862 2024-04-12T11:20:13.675 INFO:tasks.rados.rados.0.smithi029.stdout:1402: oid 24 version 862 is already newer than 861 2024-04-12T11:20:13.676 INFO:tasks.rados.rados.0.smithi029.stdout:1402: oid 24 version 862 is already newer than 860 2024-04-12T11:20:13.676 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 862 (ObjNum 548 snap 176 seq_num 548) dirty exists 2024-04-12T11:20:13.676 INFO:tasks.rados.rados.0.smithi029.stdout:1402: left oid 24 (ObjNum 548 snap 176 seq_num 548) 2024-04-12T11:20:13.676 INFO:tasks.rados.rados.0.smithi029.stdout:1403: expect (ObjNum 448 snap 147 seq_num 448) 2024-04-12T11:20:13.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-12T11:20:13.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:13 smithi087 ceph-mon[32917]: osdmap e412: 8 total, 8 up, 8 in 2024-04-12T11:20:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:13 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:13 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:20:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:13 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:13 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:13 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:20:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:13 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T11:20:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:13 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-12T11:20:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:13 smithi087 ceph-mon[32917]: Upgrade: It appears safe to stop mon.b 2024-04-12T11:20:13.843 INFO:tasks.rados.rados.0.smithi029.stdout:1404: finishing write tid 1 to smithi02979277-9 2024-04-12T11:20:13.843 INFO:tasks.rados.rados.0.smithi029.stdout:1404: finishing write tid 2 to smithi02979277-9 2024-04-12T11:20:13.843 INFO:tasks.rados.rados.0.smithi029.stdout:1402: done (4 left) 2024-04-12T11:20:13.843 INFO:tasks.rados.rados.0.smithi029.stdout:1403: done (3 left) 2024-04-12T11:20:13.843 INFO:tasks.rados.rados.0.smithi029.stdout:1406: done (2 left) 2024-04-12T11:20:13.843 INFO:tasks.rados.rados.0.smithi029.stdout:1407: snap_remove snap 175 2024-04-12T11:20:13.927 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:20:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[93650]: osdmap e412: 8 total, 8 up, 8 in 2024-04-12T11:20:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:20:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:20:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T11:20:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-12T11:20:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[93650]: Upgrade: It appears safe to stop mon.b 2024-04-12T11:20:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[97787]: osdmap e412: 8 total, 8 up, 8 in 2024-04-12T11:20:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:20:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:20:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T11:20:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-12T11:20:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:13 smithi029 ceph-mon[97787]: Upgrade: It appears safe to stop mon.b 2024-04-12T11:20:14.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:14 smithi087 ceph-mon[32917]: pgmap v253: 89 pgs: 89 active+clean; 274 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-04-12T11:20:14.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:14 smithi087 ceph-mon[32917]: osdmap e413: 8 total, 8 up, 8 in 2024-04-12T11:20:14.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:14 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:14.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:14 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:14.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:14 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:14.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:14 smithi087 ceph-mon[32917]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:14.734 INFO:tasks.rados.rados.0.smithi029.stdout:1407: done (2 left) 2024-04-12T11:20:14.734 INFO:tasks.rados.rados.0.smithi029.stdout:1408: write oid 8 current snap is 176 2024-04-12T11:20:14.734 INFO:tasks.rados.rados.0.smithi029.stdout:1404: finishing write tid 3 to smithi02979277-9 2024-04-12T11:20:14.734 INFO:tasks.rados.rados.0.smithi029.stdout:1404: finishing write tid 4 to smithi02979277-9 2024-04-12T11:20:14.735 INFO:tasks.rados.rados.0.smithi029.stdout:1404: finishing write tid 5 to smithi02979277-9 2024-04-12T11:20:14.735 INFO:tasks.rados.rados.0.smithi029.stdout:1404: oid 9 updating version 0 to 523 2024-04-12T11:20:14.735 INFO:tasks.rados.rados.0.smithi029.stdout:1404: oid 9 version 523 is already newer than 521 2024-04-12T11:20:14.735 INFO:tasks.rados.rados.0.smithi029.stdout:1404: oid 9 version 523 is already newer than 522 2024-04-12T11:20:14.735 INFO:tasks.rados.rados.0.smithi029.stdout:1404: oid 9 updating version 523 to 524 2024-04-12T11:20:14.735 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 524 (ObjNum 549 snap 176 seq_num 549) dirty exists 2024-04-12T11:20:14.735 INFO:tasks.rados.rados.0.smithi029.stdout:1404: left oid 9 (ObjNum 549 snap 176 seq_num 549) 2024-04-12T11:20:14.735 INFO:tasks.rados.rados.0.smithi029.stdout:1405: expect (ObjNum 367 snap 117 seq_num 367) 2024-04-12T11:20:14.837 INFO:tasks.rados.rados.0.smithi029.stdout:1408: seq_num 551 ranges {460657=639615,1693343=476514,2681716=578941} 2024-04-12T11:20:14.842 INFO:tasks.rados.rados.0.smithi029.stdout:1408: writing smithi02979277-8 from 460657 to 1100272 tid 1 2024-04-12T11:20:14.848 INFO:tasks.rados.rados.0.smithi029.stdout:1408: writing smithi02979277-8 from 1693343 to 2169857 tid 2 2024-04-12T11:20:14.854 INFO:tasks.rados.rados.0.smithi029.stdout:1408: writing smithi02979277-8 from 2681716 to 3260657 tid 3 2024-04-12T11:20:14.855 INFO:tasks.rados.rados.0.smithi029.stdout:1404: done (2 left) 2024-04-12T11:20:14.855 INFO:tasks.rados.rados.0.smithi029.stdout:1405: done (1 left) 2024-04-12T11:20:14.855 INFO:tasks.rados.rados.0.smithi029.stdout:1409: read oid 24 snap -1 2024-04-12T11:20:14.855 INFO:tasks.rados.rados.0.smithi029.stdout:1409: expect (ObjNum 548 snap 176 seq_num 548) 2024-04-12T11:20:14.855 INFO:tasks.rados.rados.0.smithi029.stdout:1410: delete oid 19 current snap is 176 2024-04-12T11:20:14.855 INFO:tasks.rados.rados.0.smithi029.stdout:1408: finishing write tid 1 to smithi02979277-8 2024-04-12T11:20:14.855 INFO:tasks.rados.rados.0.smithi029.stdout:1408: finishing write tid 2 to smithi02979277-8 2024-04-12T11:20:14.857 INFO:tasks.rados.rados.0.smithi029.stdout:1408: finishing write tid 3 to smithi02979277-8 2024-04-12T11:20:14.857 INFO:tasks.rados.rados.0.smithi029.stdout:1408: finishing write tid 4 to smithi02979277-8 2024-04-12T11:20:14.857 INFO:tasks.rados.rados.0.smithi029.stdout:1408: finishing write tid 5 to smithi02979277-8 2024-04-12T11:20:14.857 INFO:tasks.rados.rados.0.smithi029.stdout:1408: oid 8 updating version 0 to 116 2024-04-12T11:20:14.858 INFO:tasks.rados.rados.0.smithi029.stdout:1408: oid 8 version 116 is already newer than 115 2024-04-12T11:20:14.858 INFO:tasks.rados.rados.0.smithi029.stdout:1408: oid 8 version 116 is already newer than 114 2024-04-12T11:20:14.858 INFO:tasks.rados.rados.0.smithi029.stdout:1408: oid 8 version 116 is already newer than 113 2024-04-12T11:20:14.858 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 116 (ObjNum 551 snap 176 seq_num 551) dirty exists 2024-04-12T11:20:14.858 INFO:tasks.rados.rados.0.smithi029.stdout:1408: left oid 8 (ObjNum 551 snap 176 seq_num 551) 2024-04-12T11:20:14.858 INFO:tasks.rados.rados.0.smithi029.stdout:1408: done (2 left) 2024-04-12T11:20:14.858 INFO:tasks.rados.rados.0.smithi029.stdout:1410: done (1 left) 2024-04-12T11:20:14.858 INFO:tasks.rados.rados.0.smithi029.stdout:1411: delete oid 22 current snap is 176 2024-04-12T11:20:14.860 INFO:tasks.rados.rados.0.smithi029.stdout:1409: expect (ObjNum 548 snap 176 seq_num 548) 2024-04-12T11:20:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[97787]: pgmap v253: 89 pgs: 89 active+clean; 274 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-04-12T11:20:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[97787]: osdmap e413: 8 total, 8 up, 8 in 2024-04-12T11:20:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[93650]: pgmap v253: 89 pgs: 89 active+clean; 274 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-04-12T11:20:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[93650]: osdmap e413: 8 total, 8 up, 8 in 2024-04-12T11:20:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:15.005 INFO:tasks.rados.rados.0.smithi029.stdout:1409: done (1 left) 2024-04-12T11:20:15.005 INFO:tasks.rados.rados.0.smithi029.stdout:1411: done (0 left) 2024-04-12T11:20:15.005 INFO:tasks.rados.rados.0.smithi029.stdout:1412: write oid 34 current snap is 176 2024-04-12T11:20:15.005 INFO:tasks.rados.rados.0.smithi029.stdout:1412: seq_num 554 ranges {511078=465362,1752436=740194,3163985=147093} 2024-04-12T11:20:15.010 INFO:tasks.rados.rados.0.smithi029.stdout:1412: writing smithi02979277-34 from 511078 to 976440 tid 1 2024-04-12T11:20:15.019 INFO:tasks.rados.rados.0.smithi029.stdout:1412: writing smithi02979277-34 from 1752436 to 2492630 tid 2 2024-04-12T11:20:15.023 INFO:tasks.rados.rados.0.smithi029.stdout:1412: writing smithi02979277-34 from 3163985 to 3311078 tid 3 2024-04-12T11:20:15.023 INFO:tasks.rados.rados.0.smithi029.stdout:1413: write oid 8 current snap is 176 2024-04-12T11:20:15.023 INFO:tasks.rados.rados.0.smithi029.stdout:1413: seq_num 555 ranges {527885=656964,1927271=483852,3191408=136477} 2024-04-12T11:20:15.030 INFO:tasks.rados.rados.0.smithi029.stdout:1413: writing smithi02979277-8 from 527885 to 1184849 tid 1 2024-04-12T11:20:15.036 INFO:tasks.rados.rados.0.smithi029.stdout:1413: writing smithi02979277-8 from 1927271 to 2411123 tid 2 2024-04-12T11:20:15.042 INFO:tasks.rados.rados.0.smithi029.stdout:1413: writing smithi02979277-8 from 3191408 to 3327885 tid 3 2024-04-12T11:20:15.042 INFO:tasks.rados.rados.0.smithi029.stdout:1414: read oid 28 snap -1 2024-04-12T11:20:15.042 INFO:tasks.rados.rados.0.smithi029.stdout:1414: expect (ObjNum 412 snap 129 seq_num 412) 2024-04-12T11:20:15.042 INFO:tasks.rados.rados.0.smithi029.stdout:1415: snap_remove snap 156 2024-04-12T11:20:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:20:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:20:14] "GET /metrics HTTP/1.1" 200 36250 "" "Prometheus/2.43.0" 2024-04-12T11:20:15.738 INFO:tasks.rados.rados.0.smithi029.stdout:1415: done (3 left) 2024-04-12T11:20:15.739 INFO:tasks.rados.rados.0.smithi029.stdout:1416: read oid 17 snap -1 2024-04-12T11:20:15.739 INFO:tasks.rados.rados.0.smithi029.stdout:1416: expect (ObjNum 535 snap 173 seq_num 535) 2024-04-12T11:20:15.739 INFO:tasks.rados.rados.0.smithi029.stdout:1417: snap_remove snap 170 2024-04-12T11:20:15.950 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:15 smithi087 ceph-mon[32917]: Upgrade: Updating mon.b 2024-04-12T11:20:15.950 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:15 smithi087 ceph-mon[32917]: Deploying daemon mon.b on smithi087 2024-04-12T11:20:15.950 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:15 smithi087 ceph-mon[32917]: osdmap e414: 8 total, 8 up, 8 in 2024-04-12T11:20:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:15 smithi029 ceph-mon[93650]: Upgrade: Updating mon.b 2024-04-12T11:20:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:15 smithi029 ceph-mon[93650]: Deploying daemon mon.b on smithi087 2024-04-12T11:20:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:15 smithi029 ceph-mon[93650]: osdmap e414: 8 total, 8 up, 8 in 2024-04-12T11:20:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:15 smithi029 ceph-mon[97787]: Upgrade: Updating mon.b 2024-04-12T11:20:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:15 smithi029 ceph-mon[97787]: Deploying daemon mon.b on smithi087 2024-04-12T11:20:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:15 smithi029 ceph-mon[97787]: osdmap e414: 8 total, 8 up, 8 in 2024-04-12T11:20:16.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:16 smithi087 systemd[1]: Stopping Ceph mon.b for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:20:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b[32913]: 2024-04-12T11:20:16.463+0000 7f4bb8a16700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:20:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b[32913]: 2024-04-12T11:20:16.463+0000 7f4bb8a16700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-12T11:20:16.980 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:16 smithi087 podman[88849]: 2024-04-12 11:20:16.718257994 +0000 UTC m=+0.361190388 container died adfeefa5d84f4bfba5eb665911c36e2a466e6e058c3aea99933f68a91e7f71c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, RELEASE=reef-d540eba, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-12T11:20:17.390 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:17 smithi087 podman[88849]: 2024-04-12 11:20:17.071206909 +0000 UTC m=+0.714139310 container cleanup adfeefa5d84f4bfba5eb665911c36e2a466e6e058c3aea99933f68a91e7f71c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=reef-d540eba, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T11:20:17.391 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:17 smithi087 bash[88849]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b 2024-04-12T11:20:17.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:17 smithi087 podman[88863]: 2024-04-12 11:20:17.390722993 +0000 UTC m=+0.673118309 container remove adfeefa5d84f4bfba5eb665911c36e2a466e6e058c3aea99933f68a91e7f71c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=reef-d540eba, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-12T11:20:17.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:17 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-12T11:20:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:17 smithi087 systemd[1]: Stopped Ceph mon.b for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:20:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:17 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.b.service: Consumed 10.575s CPU time. 2024-04-12T11:20:18.097 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:17 smithi087 systemd[1]: Starting Ceph mon.b for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:20:18.097 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:17 smithi087 podman[88995]: 2024-04-12 11:20:17.858982958 +0000 UTC m=+0.017990595 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:20:18.097 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:17 smithi087 podman[88995]: 2024-04-12 11:20:17.973731454 +0000 UTC m=+0.132739125 container create ae22114389c9fe0161e2f8e45d4753a9066723b5b05b3c50b1640e0faec7590f (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-12T11:20:18.470 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 podman[88995]: 2024-04-12 11:20:18.201500568 +0000 UTC m=+0.360508198 container init ae22114389c9fe0161e2f8e45d4753a9066723b5b05b3c50b1640e0faec7590f (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240409, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-12T11:20:18.470 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 podman[88995]: 2024-04-12 11:20:18.204837106 +0000 UTC m=+0.363844741 container start ae22114389c9fe0161e2f8e45d4753a9066723b5b05b3c50b1640e0faec7590f (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-12T11:20:18.473 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T11:20:18.473 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev), process ceph-mon, pid 2 2024-04-12T11:20:18.473 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: pidfile_write: ignore empty --pid-file 2024-04-12T11:20:18.473 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: load: jerasure load: lrc 2024-04-12T11:20:18.473 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: RocksDB version: 7.9.2 2024-04-12T11:20:18.473 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Git sha 0 2024-04-12T11:20:18.473 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Compile date 2024-04-11 21:23:14 2024-04-12T11:20:18.473 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: DB SUMMARY 2024-04-12T11:20:18.473 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: DB Session ID: CGPK4W8AEK3PY8AZNAWD 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: CURRENT file: CURRENT 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1403 Bytes 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 667182 ; 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.error_if_exists: 0 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.create_if_missing: 0 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.paranoid_checks: 1 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T11:20:18.474 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.env: 0x561b59054cc0 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.info_log: 0x561b59de7bc0 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.statistics: (nil) 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.use_fsync: 0 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_log_file_size: 0 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.allow_fallocate: 1 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T11:20:18.475 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.use_direct_reads: 0 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.db_log_dir: 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.wal_dir: 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.write_buffer_manager: 0x561b59db81e0 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T11:20:18.476 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.unordered_write: 0 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.row_cache: None 2024-04-12T11:20:18.477 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.wal_filter: None 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.two_write_queues: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.wal_compression: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.atomic_flush: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.log_readahead_size: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T11:20:18.478 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_background_jobs: 2 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_background_compactions: -1 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_subcompactions: 1 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T11:20:18.479 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_open_files: -1 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_background_flushes: -1 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Compression algorithms supported: 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: kZSTD supported: 0 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: kXpressCompression supported: 0 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T11:20:18.480 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: kZlibCompression supported: 1 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: kSnappyCompression supported: 1 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: kLZ4Compression supported: 1 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: kBZip2Compression supported: 0 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.merge_operator: 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_filter: None 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T11:20:18.481 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561b59de7da0) 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: cache_index_and_filter_blocks: 1 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: pin_top_level_index_and_filter: 1 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: index_type: 0 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: data_block_index_type: 0 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: index_shortening: 1 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: checksum: 4 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: no_block_cache: 0 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: block_cache: 0x561b59deb090 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: block_cache_name: BinnedLRUCache 2024-04-12T11:20:18.482 INFO:journalctl@ceph.mon.b.smithi087.stdout: block_cache_options: 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: capacity : 536870912 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: num_shard_bits : 4 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: strict_capacity_limit : 0 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: high_pri_pool_ratio: 0.000 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: block_cache_compressed: (nil) 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: persistent_cache: (nil) 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: block_size: 4096 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: block_size_deviation: 10 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: block_restart_interval: 16 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: index_block_restart_interval: 1 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: metadata_block_size: 4096 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: partition_filters: 0 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: use_delta_encoding: 1 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: filter_policy: bloomfilter 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: whole_key_filtering: 1 2024-04-12T11:20:18.483 INFO:journalctl@ceph.mon.b.smithi087.stdout: verify_compression: 0 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout: read_amp_bytes_per_bit: 0 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout: format_version: 5 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout: enable_index_compression: 1 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout: block_align: 0 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout: max_auto_readahead_size: 262144 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout: prepopulate_block_cache: 0 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout: initial_auto_readahead_size: 8192 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compression: NoCompression 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T11:20:18.484 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.num_levels: 7 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T11:20:18.485 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T11:20:18.486 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T11:20:18.487 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.inplace_update_support: 0 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T11:20:18.488 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.bloom_locality: 0 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.max_successive_merges: 0 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.ttl: 2592000 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.enable_blob_files: false 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.min_blob_size: 0 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T11:20:18.489 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 19428, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d108fa57-44a3-456d-9a09-5e824988271a 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920818233366, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920818235706, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 441015, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 19433, "largest_seqno": 19781, "table_properties": {"data_size": 438343, "index_size": 1248, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 389, "raw_key_size": 3565, "raw_average_key_size": 24, "raw_value_size": 434680, "raw_average_value_size": 2937, "num_data_blocks": 53, "num_entries": 148, "num_filter_entries": 148, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712920818, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d108fa57-44a3-456d-9a09-5e824988271a", "db_session_id": "CGPK4W8AEK3PY8AZNAWD", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-12T11:20:18.490 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712920818235789, "job": 1, "event": "recovery_finished"} 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561b59eda000 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: DB pointer 0x561b59ec4000 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout: ** DB Stats ** 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T11:20:18.491 INFO:journalctl@ceph.mon.b.smithi087.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: ** Compaction Stats [default] ** 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: L0 1/0 430.68 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 324.8 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: L6 1/0 12.22 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: Sum 2/0 12.64 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 324.8 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 324.8 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: ** Compaction Stats [default] ** 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 324.8 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-12T11:20:18.492 INFO:journalctl@ceph.mon.b.smithi087.stdout: 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: Cumulative compaction: 0.00 GB write, 95.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: Interval compaction: 0.00 GB write, 95.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: Block cache BinnedLRUCache@0x561b59deb090#2 capacity: 512.00 MB usage: 1.77 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.44 KB,8.34465e-05%) IndexBlock(1,1.33 KB,0.00025332%) Misc(1,0.00 KB,0%) 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: 2024-04-12T11:20:18.493 INFO:journalctl@ceph.mon.b.smithi087.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: starting mon.b rank 1 at public addrs [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] at bind addrs [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: mon.b@-1(???) e3 preinit fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: mon.b@-1(???).mds e9 new map 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: mon.b@-1(???).mds e9 print_map 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout: e9 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout: legacy client fscid: 1 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout: 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout: Filesystem 'foo' (1) 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout: fs_name foo 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout: epoch 8 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-12T11:20:18.494 INFO:journalctl@ceph.mon.b.smithi087.stdout: created 2024-04-12T11:12:57.763670+0000 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: modified 2024-04-12T11:13:16.631701+0000 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: tableserver 0 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: root 0 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: session_timeout 60 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: session_autoclose 300 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: max_file_size 1099511627776 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: max_xattr_size 65536 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: required_client_features {} 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: last_failure 0 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: last_failure_osd_epoch 0 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: max_mds 1 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: in 0 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: up {0=14661} 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: failed 2024-04-12T11:20:18.495 INFO:journalctl@ceph.mon.b.smithi087.stdout: damaged 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: stopped 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: data_pools [4] 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: metadata_pool 3 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: inline_data disabled 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: balancer 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: bal_rank_mask -1 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: standby_count_wanted 1 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: qdb_cluster leader: 0 members: 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: [mds.foo.smithi029.scbuux{0:14661} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.29:6834/3928502699,v1:172.21.15.29:6835/3928502699] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: Standby daemons: 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout: [mds.foo.smithi087.mbflzu{-1:24502} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.87:6832/2304329976,v1:172.21.15.87:6833/2304329976] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T11:20:18.496 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: mon.b@-1(???).osd e415 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-12T11:20:18.497 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: mon.b@-1(???).osd e415 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T11:20:18.497 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: mon.b@-1(???).osd e415 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T11:20:18.497 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: mon.b@-1(???).osd e415 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T11:20:18.497 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 ceph-mon[89014]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-04-12T11:20:18.497 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 bash[88995]: ae22114389c9fe0161e2f8e45d4753a9066723b5b05b3c50b1640e0faec7590f 2024-04-12T11:20:18.497 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:18 smithi087 systemd[1]: Started Ceph mon.b for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:20:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:1417: done (4 left) 2024-04-12T11:20:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:1418: snap_create 2024-04-12T11:20:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:1412: finishing write tid 1 to smithi02979277-34 2024-04-12T11:20:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:1412: finishing write tid 2 to smithi02979277-34 2024-04-12T11:20:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:1412: finishing write tid 3 to smithi02979277-34 2024-04-12T11:20:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:1412: finishing write tid 4 to smithi02979277-34 2024-04-12T11:20:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:1412: finishing write tid 5 to smithi02979277-34 2024-04-12T11:20:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:1412: oid 34 updating version 0 to 542 2024-04-12T11:20:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:1412: oid 34 updating version 542 to 545 2024-04-12T11:20:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:1412: oid 34 version 545 is already newer than 544 2024-04-12T11:20:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:1412: oid 34 version 545 is already newer than 543 2024-04-12T11:20:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 545 (ObjNum 554 snap 176 seq_num 554) dirty exists 2024-04-12T11:20:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:1412: left oid 34 (ObjNum 554 snap 176 seq_num 554) 2024-04-12T11:20:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:1413: finishing write tid 1 to smithi02979277-8 2024-04-12T11:20:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:1413: finishing write tid 2 to smithi02979277-8 2024-04-12T11:20:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:1414: expect (ObjNum 412 snap 129 seq_num 412) 2024-04-12T11:20:19.927 INFO:tasks.rados.rados.0.smithi029.stdout:1413: finishing write tid 3 to smithi02979277-8 2024-04-12T11:20:19.927 INFO:tasks.rados.rados.0.smithi029.stdout:1413: finishing write tid 4 to smithi02979277-8 2024-04-12T11:20:19.927 INFO:tasks.rados.rados.0.smithi029.stdout:1413: finishing write tid 5 to smithi02979277-8 2024-04-12T11:20:19.927 INFO:tasks.rados.rados.0.smithi029.stdout:1413: oid 8 updating version 0 to 119 2024-04-12T11:20:19.927 INFO:tasks.rados.rados.0.smithi029.stdout:1413: oid 8 version 119 is already newer than 117 2024-04-12T11:20:19.927 INFO:tasks.rados.rados.0.smithi029.stdout:1413: oid 8 updating version 119 to 120 2024-04-12T11:20:19.927 INFO:tasks.rados.rados.0.smithi029.stdout:1413: oid 8 version 120 is already newer than 118 2024-04-12T11:20:19.928 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 120 (ObjNum 555 snap 176 seq_num 555) dirty exists 2024-04-12T11:20:19.928 INFO:tasks.rados.rados.0.smithi029.stdout:1413: left oid 8 (ObjNum 555 snap 176 seq_num 555) 2024-04-12T11:20:19.928 INFO:tasks.rados.rados.0.smithi029.stdout:1416: expect (ObjNum 535 snap 173 seq_num 535) 2024-04-12T11:20:20.758 INFO:tasks.rados.rados.0.smithi029.stdout:1412: done (4 left) 2024-04-12T11:20:20.758 INFO:tasks.rados.rados.0.smithi029.stdout:1413: done (3 left) 2024-04-12T11:20:20.758 INFO:tasks.rados.rados.0.smithi029.stdout:1414: done (2 left) 2024-04-12T11:20:20.758 INFO:tasks.rados.rados.0.smithi029.stdout:1416: done (1 left) 2024-04-12T11:20:20.758 INFO:tasks.rados.rados.0.smithi029.stdout:1418: done (0 left) 2024-04-12T11:20:20.758 INFO:tasks.rados.rados.0.smithi029.stdout:1419: delete oid 36 current snap is 177 2024-04-12T11:20:20.761 INFO:tasks.rados.rados.0.smithi029.stdout:1419: done (0 left) 2024-04-12T11:20:20.762 INFO:tasks.rados.rados.0.smithi029.stdout:1420: read oid 29 snap 133 2024-04-12T11:20:20.762 INFO:tasks.rados.rados.0.smithi029.stdout:1420: expect deleted 2024-04-12T11:20:20.762 INFO:tasks.rados.rados.0.smithi029.stdout:1421: write oid 27 current snap is 177 2024-04-12T11:20:20.762 INFO:tasks.rados.rados.0.smithi029.stdout:1421: seq_num 557 ranges {561499=407462,1527882=487521,2729901=557217} 2024-04-12T11:20:20.767 INFO:tasks.rados.rados.0.smithi029.stdout:1421: writing smithi02979277-27 from 561499 to 968961 tid 1 2024-04-12T11:20:20.772 INFO:tasks.rados.rados.0.smithi029.stdout:1421: writing smithi02979277-27 from 1527882 to 2015403 tid 2 2024-04-12T11:20:20.780 INFO:tasks.rados.rados.0.smithi029.stdout:1421: writing smithi02979277-27 from 2729901 to 3287118 tid 3 2024-04-12T11:20:20.780 INFO:tasks.rados.rados.0.smithi029.stdout:1422: read oid 43 snap -1 2024-04-12T11:20:20.780 INFO:tasks.rados.rados.0.smithi029.stdout:1422: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:20:20.780 INFO:tasks.rados.rados.0.smithi029.stdout:1423: delete oid 44 current snap is 177 2024-04-12T11:20:20.780 INFO:tasks.rados.rados.0.smithi029.stdout:1421: finishing write tid 1 to smithi02979277-27 2024-04-12T11:20:20.780 INFO:tasks.rados.rados.0.smithi029.stdout:1421: finishing write tid 2 to smithi02979277-27 2024-04-12T11:20:20.782 INFO:tasks.rados.rados.0.smithi029.stdout:1421: finishing write tid 3 to smithi02979277-27 2024-04-12T11:20:20.782 INFO:tasks.rados.rados.0.smithi029.stdout:1421: finishing write tid 4 to smithi02979277-27 2024-04-12T11:20:20.783 INFO:tasks.rados.rados.0.smithi029.stdout:1420: done (3 left) 2024-04-12T11:20:20.783 INFO:tasks.rados.rados.0.smithi029.stdout:1423: done (2 left) 2024-04-12T11:20:20.783 INFO:tasks.rados.rados.0.smithi029.stdout:1424: write oid 14 current snap is 177 2024-04-12T11:20:20.783 INFO:tasks.rados.rados.0.smithi029.stdout:1424: seq_num 559 ranges {595113=557960,1644846=491190,2784747=610366} 2024-04-12T11:20:20.789 INFO:tasks.rados.rados.0.smithi029.stdout:1424: writing smithi02979277-14 from 595113 to 1153073 tid 1 2024-04-12T11:20:20.795 INFO:tasks.rados.rados.0.smithi029.stdout:1424: writing smithi02979277-14 from 1644846 to 2136036 tid 2 2024-04-12T11:20:20.802 INFO:tasks.rados.rados.0.smithi029.stdout:1424: writing smithi02979277-14 from 2784747 to 3395113 tid 3 2024-04-12T11:20:20.802 INFO:tasks.rados.rados.0.smithi029.stdout:1425: snap_remove snap 165 2024-04-12T11:20:20.841 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev)": 3 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: pgmap v259: 89 pgs: 89 active+clean; 280 MiB data, 830 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: mon.b calling monitor election 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: mon.c calling monitor election 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: mon.a calling monitor election 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: monmap epoch 4 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: last_changed 2024-04-12T11:20:19.747279+0000 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: created 2024-04-12T11:06:03.531247+0000 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: min_mon_release 19 (squid) 2024-04-12T11:20:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: election_strategy: 1 2024-04-12T11:20:20.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-04-12T11:20:20.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: 1: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.b 2024-04-12T11:20:20.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-04-12T11:20:20.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:20:20.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: osdmap e416: 8 total, 8 up, 8 in 2024-04-12T11:20:20.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: mgrmap e37: y(active, since 2m), standbys: x 2024-04-12T11:20:20.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: overall HEALTH_OK 2024-04-12T11:20:20.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:20.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:20.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: pgmap v259: 89 pgs: 89 active+clean; 280 MiB data, 830 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-04-12T11:20:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: mon.b calling monitor election 2024-04-12T11:20:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: mon.c calling monitor election 2024-04-12T11:20:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:20:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:20:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: mon.a calling monitor election 2024-04-12T11:20:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: monmap epoch 4 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: last_changed 2024-04-12T11:20:19.747279+0000 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: created 2024-04-12T11:06:03.531247+0000 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: min_mon_release 19 (squid) 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: election_strategy: 1 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: 1: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.b 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: osdmap e416: 8 total, 8 up, 8 in 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: mgrmap e37: y(active, since 2m), standbys: x 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: overall HEALTH_OK 2024-04-12T11:20:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: pgmap v259: 89 pgs: 89 active+clean; 280 MiB data, 830 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: mon.b calling monitor election 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: mon.c calling monitor election 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: mon.a calling monitor election 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: monmap epoch 4 2024-04-12T11:20:21.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: last_changed 2024-04-12T11:20:19.747279+0000 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: created 2024-04-12T11:06:03.531247+0000 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: min_mon_release 19 (squid) 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: election_strategy: 1 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: 1: [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon.b 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: fsmap foo:1 {0=foo.smithi029.scbuux=up:active} 1 up:standby 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: osdmap e416: 8 total, 8 up, 8 in 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: mgrmap e37: y(active, since 2m), standbys: x 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: overall HEALTH_OK 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:21.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:21.553 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch ps' 2024-04-12T11:20:21.761 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:20:21.768 INFO:tasks.rados.rados.0.smithi029.stdout:1425: done (3 left) 2024-04-12T11:20:21.768 INFO:tasks.rados.rados.0.smithi029.stdout:1426: read oid 18 snap -1 2024-04-12T11:20:21.769 INFO:tasks.rados.rados.0.smithi029.stdout:1426: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:20:21.769 INFO:tasks.rados.rados.0.smithi029.stdout:1427: write oid 12 current snap is 177 2024-04-12T11:20:21.769 INFO:tasks.rados.rados.0.smithi029.stdout:1427: seq_num 560 ranges {611920=633209,1703328=634848,2812170=507386} 2024-04-12T11:20:21.776 INFO:tasks.rados.rados.0.smithi029.stdout:1427: writing smithi02979277-12 from 611920 to 1245129 tid 1 2024-04-12T11:20:21.784 INFO:tasks.rados.rados.0.smithi029.stdout:1427: writing smithi02979277-12 from 1703328 to 2338176 tid 2 2024-04-12T11:20:21.790 INFO:tasks.rados.rados.0.smithi029.stdout:1427: writing smithi02979277-12 from 2812170 to 3319556 tid 3 2024-04-12T11:20:21.790 INFO:tasks.rados.rados.0.smithi029.stdout:1428: write oid 34 current snap is 177 2024-04-12T11:20:21.790 INFO:tasks.rados.rados.0.smithi029.stdout:1421: finishing write tid 5 to smithi02979277-27 2024-04-12T11:20:21.790 INFO:tasks.rados.rados.0.smithi029.stdout:1421: oid 27 updating version 0 to 568 2024-04-12T11:20:21.790 INFO:tasks.rados.rados.0.smithi029.stdout:1421: oid 27 version 568 is already newer than 567 2024-04-12T11:20:21.790 INFO:tasks.rados.rados.0.smithi029.stdout:1421: oid 27 updating version 568 to 569 2024-04-12T11:20:21.791 INFO:tasks.rados.rados.0.smithi029.stdout:1421: oid 27 updating version 569 to 570 2024-04-12T11:20:21.791 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 570 (ObjNum 557 snap 177 seq_num 557) dirty exists 2024-04-12T11:20:21.791 INFO:tasks.rados.rados.0.smithi029.stdout:1421: left oid 27 (ObjNum 557 snap 177 seq_num 557) 2024-04-12T11:20:21.791 INFO:tasks.rados.rados.0.smithi029.stdout:1422: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:20:21.891 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:21 smithi029 ceph-mon[93650]: osdmap e417: 8 total, 8 up, 8 in 2024-04-12T11:20:21.891 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:21 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3419972369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:20:21.892 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:21.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:21 smithi029 ceph-mon[97787]: osdmap e417: 8 total, 8 up, 8 in 2024-04-12T11:20:21.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:21 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3419972369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:20:21.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:21.899 INFO:tasks.rados.rados.0.smithi029.stdout:1424: finishing write tid 1 to smithi02979277-14 2024-04-12T11:20:21.899 INFO:tasks.rados.rados.0.smithi029.stdout:1424: finishing write tid 2 to smithi02979277-14 2024-04-12T11:20:21.900 INFO:tasks.rados.rados.0.smithi029.stdout:1424: finishing write tid 3 to smithi02979277-14 2024-04-12T11:20:21.900 INFO:tasks.rados.rados.0.smithi029.stdout:1428: seq_num 561 ranges {628727=708458,2045457=494859,3123240=305487} 2024-04-12T11:20:21.906 INFO:tasks.rados.rados.0.smithi029.stdout:1428: writing smithi02979277-34 from 628727 to 1337185 tid 1 2024-04-12T11:20:21.913 INFO:tasks.rados.rados.0.smithi029.stdout:1428: writing smithi02979277-34 from 2045457 to 2540316 tid 2 2024-04-12T11:20:21.919 INFO:tasks.rados.rados.0.smithi029.stdout:1428: writing smithi02979277-34 from 3123240 to 3428727 tid 3 2024-04-12T11:20:21.919 INFO:tasks.rados.rados.0.smithi029.stdout:1421: done (5 left) 2024-04-12T11:20:21.919 INFO:tasks.rados.rados.0.smithi029.stdout:1422: done (4 left) 2024-04-12T11:20:21.919 INFO:tasks.rados.rados.0.smithi029.stdout:1429: read oid 20 snap -1 2024-04-12T11:20:21.919 INFO:tasks.rados.rados.0.smithi029.stdout:1429: expect deleted 2024-04-12T11:20:21.919 INFO:tasks.rados.rados.0.smithi029.stdout:1430: write oid 3 current snap is 177 2024-04-12T11:20:21.919 INFO:tasks.rados.rados.0.smithi029.stdout:1424: finishing write tid 4 to smithi02979277-14 2024-04-12T11:20:21.919 INFO:tasks.rados.rados.0.smithi029.stdout:1424: finishing write tid 5 to smithi02979277-14 2024-04-12T11:20:21.920 INFO:tasks.rados.rados.0.smithi029.stdout:1424: oid 14 updating version 0 to 834 2024-04-12T11:20:21.920 INFO:tasks.rados.rados.0.smithi029.stdout:1424: oid 14 updating version 834 to 836 2024-04-12T11:20:21.920 INFO:tasks.rados.rados.0.smithi029.stdout:1424: oid 14 version 836 is already newer than 835 2024-04-12T11:20:21.920 INFO:tasks.rados.rados.0.smithi029.stdout:1424: oid 14 updating version 836 to 837 2024-04-12T11:20:21.920 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 837 (ObjNum 559 snap 177 seq_num 559) dirty exists 2024-04-12T11:20:21.920 INFO:tasks.rados.rados.0.smithi029.stdout:1424: left oid 14 (ObjNum 559 snap 177 seq_num 559) 2024-04-12T11:20:21.920 INFO:tasks.rados.rados.0.smithi029.stdout:1430: seq_num 562 ranges {645534=783707,2103939=638517,3150663=294871} 2024-04-12T11:20:21.926 INFO:tasks.rados.rados.0.smithi029.stdout:1430: writing smithi02979277-3 from 645534 to 1429241 tid 1 2024-04-12T11:20:21.933 INFO:tasks.rados.rados.0.smithi029.stdout:1430: writing smithi02979277-3 from 2103939 to 2742456 tid 2 2024-04-12T11:20:21.937 INFO:tasks.rados.rados.0.smithi029.stdout:1430: writing smithi02979277-3 from 3150663 to 3445534 tid 3 2024-04-12T11:20:21.938 INFO:tasks.rados.rados.0.smithi029.stdout:1424: done (5 left) 2024-04-12T11:20:21.938 INFO:tasks.rados.rados.0.smithi029.stdout:1431: rollback oid 5 current snap is 177 2024-04-12T11:20:21.938 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 5 to 133 2024-04-12T11:20:21.938 INFO:tasks.rados.rados.0.smithi029.stdout:1426: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:20:22.079 INFO:tasks.rados.rados.0.smithi029.stdout:1427: finishing write tid 1 to smithi02979277-12 2024-04-12T11:20:22.079 INFO:tasks.rados.rados.0.smithi029.stdout:1427: finishing write tid 2 to smithi02979277-12 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:1427: finishing write tid 3 to smithi02979277-12 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:1427: finishing write tid 4 to smithi02979277-12 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:1427: finishing write tid 5 to smithi02979277-12 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:1427: oid 12 updating version 0 to 659 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:1427: oid 12 version 659 is already newer than 657 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:1427: oid 12 version 659 is already newer than 658 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:1427: oid 12 version 659 is already newer than 656 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 659 (ObjNum 560 snap 177 seq_num 560) dirty exists 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:1427: left oid 12 (ObjNum 560 snap 177 seq_num 560) 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:1428: finishing write tid 1 to smithi02979277-34 2024-04-12T11:20:22.080 INFO:tasks.rados.rados.0.smithi029.stdout:1428: finishing write tid 2 to smithi02979277-34 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1426: done (5 left) 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1427: done (4 left) 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1432: write oid 47 current snap is 177 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1428: finishing write tid 3 to smithi02979277-34 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1428: finishing write tid 4 to smithi02979277-34 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1428: finishing write tid 5 to smithi02979277-34 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1428: oid 34 updating version 0 to 556 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1428: oid 34 version 556 is already newer than 555 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1428: oid 34 updating version 556 to 557 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1428: oid 34 updating version 557 to 558 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 558 (ObjNum 561 snap 177 seq_num 561) dirty exists 2024-04-12T11:20:22.081 INFO:tasks.rados.rados.0.smithi029.stdout:1428: left oid 34 (ObjNum 561 snap 177 seq_num 561) 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:1430: finishing write tid 1 to smithi02979277-3 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:1430: finishing write tid 2 to smithi02979277-3 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:1431: finishing rollback tid 1 to smithi02979277-5 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 805 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:1430: finishing write tid 3 to smithi02979277-3 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:1430: finishing write tid 4 to smithi02979277-3 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:1430: finishing write tid 5 to smithi02979277-3 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:1430: oid 3 updating version 0 to 887 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:1430: oid 3 updating version 887 to 888 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:1430: oid 3 version 888 is already newer than 886 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:1430: oid 3 version 888 is already newer than 885 2024-04-12T11:20:22.082 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 888 (ObjNum 562 snap 177 seq_num 562) dirty exists 2024-04-12T11:20:22.083 INFO:tasks.rados.rados.0.smithi029.stdout:1430: left oid 3 (ObjNum 562 snap 177 seq_num 562) 2024-04-12T11:20:22.083 INFO:tasks.rados.rados.0.smithi029.stdout:1432: seq_num 563 ranges {662341=575309,1878774=498528,2894439=457555} 2024-04-12T11:20:22.085 INFO:tasks.rados.rados.0.smithi029.stdout:1432: writing smithi02979277-47 from 662341 to 1237650 tid 1 2024-04-12T11:20:22.091 INFO:tasks.rados.rados.0.smithi029.stdout:1432: writing smithi02979277-47 from 1878774 to 2377302 tid 2 2024-04-12T11:20:22.097 INFO:tasks.rados.rados.0.smithi029.stdout:1432: writing smithi02979277-47 from 2894439 to 3351994 tid 3 2024-04-12T11:20:22.097 INFO:tasks.rados.rados.0.smithi029.stdout:1428: done (4 left) 2024-04-12T11:20:22.097 INFO:tasks.rados.rados.0.smithi029.stdout:1429: done (3 left) 2024-04-12T11:20:22.097 INFO:tasks.rados.rados.0.smithi029.stdout:1430: done (2 left) 2024-04-12T11:20:22.097 INFO:tasks.rados.rados.0.smithi029.stdout:1431: done (1 left) 2024-04-12T11:20:22.097 INFO:tasks.rados.rados.0.smithi029.stdout:1433: write oid 40 current snap is 177 2024-04-12T11:20:22.097 INFO:tasks.rados.rados.0.smithi029.stdout:1432: finishing write tid 1 to smithi02979277-47 2024-04-12T11:20:22.097 INFO:tasks.rados.rados.0.smithi029.stdout:1433: seq_num 564 ranges {679148=650558,1937256=642186,3321862=157286} 2024-04-12T11:20:22.104 INFO:tasks.rados.rados.0.smithi029.stdout:1433: writing smithi02979277-40 from 679148 to 1329706 tid 1 2024-04-12T11:20:22.110 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:21 smithi087 ceph-mon[89014]: osdmap e417: 8 total, 8 up, 8 in 2024-04-12T11:20:22.110 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:21 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3419972369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-12T11:20:22.110 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:22.111 INFO:tasks.rados.rados.0.smithi029.stdout:1433: writing smithi02979277-40 from 1937256 to 2579442 tid 2 2024-04-12T11:20:22.116 INFO:tasks.rados.rados.0.smithi029.stdout:1433: writing smithi02979277-40 from 3321862 to 3479148 tid 3 2024-04-12T11:20:22.116 INFO:tasks.rados.rados.0.smithi029.stdout:1434: snap_remove snap 167 2024-04-12T11:20:22.362 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:22.537 INFO:tasks.rados.rados.0.smithi029.stdout:1434: done (2 left) 2024-04-12T11:20:22.537 INFO:tasks.rados.rados.0.smithi029.stdout:1435: read oid 42 snap -1 2024-04-12T11:20:22.537 INFO:tasks.rados.rados.0.smithi029.stdout:1435: expect (ObjNum 477 snap 156 seq_num 477) 2024-04-12T11:20:22.537 INFO:tasks.rados.rados.0.smithi029.stdout:1436: rollback oid 13 current snap is 177 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 13 to 171 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1432: finishing write tid 2 to smithi02979277-47 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1432: finishing write tid 3 to smithi02979277-47 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1432: finishing write tid 4 to smithi02979277-47 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1432: finishing write tid 5 to smithi02979277-47 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1432: oid 47 updating version 0 to 661 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1432: oid 47 updating version 661 to 663 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1432: oid 47 version 663 is already newer than 662 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1432: oid 47 updating version 663 to 664 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 664 (ObjNum 563 snap 177 seq_num 563) dirty exists 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1432: left oid 47 (ObjNum 563 snap 177 seq_num 563) 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1433: finishing write tid 1 to smithi02979277-40 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1433: finishing write tid 2 to smithi02979277-40 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1433: finishing write tid 3 to smithi02979277-40 2024-04-12T11:20:22.538 INFO:tasks.rados.rados.0.smithi029.stdout:1433: finishing write tid 4 to smithi02979277-40 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1433: finishing write tid 5 to smithi02979277-40 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1433: oid 40 updating version 0 to 892 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1433: oid 40 updating version 892 to 893 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1433: oid 40 version 893 is already newer than 890 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1433: oid 40 version 893 is already newer than 891 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 893 (ObjNum 564 snap 177 seq_num 564) dirty exists 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1433: left oid 40 (ObjNum 564 snap 177 seq_num 564) 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1432: done (3 left) 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1433: done (2 left) 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1437: rollback oid 47 current snap is 177 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 133 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1438: write oid 10 current snap is 177 2024-04-12T11:20:22.539 INFO:tasks.rados.rados.0.smithi029.stdout:1438: seq_num 565 ranges {695955=725807,1879385=502197,2949285=480099} 2024-04-12T11:20:22.546 INFO:tasks.rados.rados.0.smithi029.stdout:1438: writing smithi02979277-10 from 695955 to 1421762 tid 1 2024-04-12T11:20:22.552 INFO:tasks.rados.rados.0.smithi029.stdout:1438: writing smithi02979277-10 from 1879385 to 2381582 tid 2 2024-04-12T11:20:22.558 INFO:tasks.rados.rados.0.smithi029.stdout:1438: writing smithi02979277-10 from 2949285 to 3429384 tid 3 2024-04-12T11:20:22.558 INFO:tasks.rados.rados.0.smithi029.stdout:1439: rollback oid 25 current snap is 177 2024-04-12T11:20:22.558 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 25 to 164 2024-04-12T11:20:22.558 INFO:tasks.rados.rados.0.smithi029.stdout:1435: expect (ObjNum 477 snap 156 seq_num 477) 2024-04-12T11:20:22.704 INFO:tasks.rados.rados.0.smithi029.stdout:1436: finishing rollback tid 1 to smithi02979277-13 2024-04-12T11:20:22.704 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 0 (ObjNum 289 snap 0 seq_num 2925506032) dirty dne 2024-04-12T11:20:22.704 INFO:tasks.rados.rados.0.smithi029.stdout:1437: finishing rollback tid 0 to smithi02979277-47 2024-04-12T11:20:22.704 INFO:tasks.rados.rados.0.smithi029.stdout:1437: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1435: done (4 left) 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1436: done (3 left) 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1440: rollback oid 50 current snap is 177 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 50 to 164 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1437: finishing rollback tid 2 to smithi02979277-47 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 667 (ObjNum 415 snap 130 seq_num 415) dirty exists 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1438: finishing write tid 1 to smithi02979277-10 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1438: finishing write tid 2 to smithi02979277-10 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1439: finishing rollback tid 0 to smithi02979277-25 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1439: finishing rollback tid 1 to smithi02979277-25 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1438: finishing write tid 3 to smithi02979277-10 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1439: finishing rollback tid 2 to smithi02979277-25 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 731 (ObjNum 400 snap 123 seq_num 400) dirty exists 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1438: finishing write tid 4 to smithi02979277-10 2024-04-12T11:20:22.705 INFO:tasks.rados.rados.0.smithi029.stdout:1438: finishing write tid 5 to smithi02979277-10 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1438: oid 10 updating version 0 to 640 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1438: oid 10 updating version 640 to 643 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1438: oid 10 version 643 is already newer than 642 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1438: oid 10 version 643 is already newer than 641 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 643 (ObjNum 565 snap 177 seq_num 565) dirty exists 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1438: left oid 10 (ObjNum 565 snap 177 seq_num 565) 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1437: done (3 left) 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1438: done (2 left) 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1439: done (1 left) 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1441: read oid 37 snap -1 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1441: expect deleted 2024-04-12T11:20:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:1442: snap_remove snap 172 2024-04-12T11:20:22.990 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:22 smithi029 ceph-mon[93650]: pgmap v262: 89 pgs: 89 active+clean; 280 MiB data, 830 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-04-12T11:20:22.991 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:22 smithi029 ceph-mon[93650]: osdmap e418: 8 total, 8 up, 8 in 2024-04-12T11:20:22.991 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:22 smithi029 ceph-mon[93650]: osdmap e419: 8 total, 8 up, 8 in 2024-04-12T11:20:22.991 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:22 smithi029 ceph-mon[97787]: pgmap v262: 89 pgs: 89 active+clean; 280 MiB data, 830 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-04-12T11:20:22.991 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:22 smithi029 ceph-mon[97787]: osdmap e418: 8 total, 8 up, 8 in 2024-04-12T11:20:22.991 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:22 smithi029 ceph-mon[97787]: osdmap e419: 8 total, 8 up, 8 in 2024-04-12T11:20:23.119 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:22 smithi087 ceph-mon[89014]: pgmap v262: 89 pgs: 89 active+clean; 280 MiB data, 830 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-04-12T11:20:23.120 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:22 smithi087 ceph-mon[89014]: osdmap e418: 8 total, 8 up, 8 in 2024-04-12T11:20:23.120 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:22 smithi087 ceph-mon[89014]: osdmap e419: 8 total, 8 up, 8 in 2024-04-12T11:20:23.508 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:20:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:20:23.243+0000 7f4c1f740700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-12T11:20:23.539 INFO:tasks.rados.rados.0.smithi029.stdout:1442: done (2 left) 2024-04-12T11:20:23.539 INFO:tasks.rados.rados.0.smithi029.stdout:1443: write oid 11 current snap is 177 2024-04-12T11:20:23.539 INFO:tasks.rados.rados.0.smithi029.stdout:1443: seq_num 566 ranges {712762=401056,1537867=645855,2860355=652407} 2024-04-12T11:20:23.545 INFO:tasks.rados.rados.0.smithi029.stdout:1443: writing smithi02979277-11 from 712762 to 1113818 tid 1 2024-04-12T11:20:23.551 INFO:tasks.rados.rados.0.smithi029.stdout:1443: writing smithi02979277-11 from 1537867 to 2183722 tid 2 2024-04-12T11:20:23.559 INFO:tasks.rados.rados.0.smithi029.stdout:1443: writing smithi02979277-11 from 2860355 to 3512762 tid 3 2024-04-12T11:20:23.559 INFO:tasks.rados.rados.0.smithi029.stdout:1444: write oid 25 current snap is 177 2024-04-12T11:20:23.559 INFO:tasks.rados.rados.0.smithi029.stdout:1444: seq_num 567 ranges {729569=476305,1996349=505866,3004131=525438} 2024-04-12T11:20:23.566 INFO:tasks.rados.rados.0.smithi029.stdout:1444: writing smithi02979277-25 from 729569 to 1205874 tid 1 2024-04-12T11:20:23.573 INFO:tasks.rados.rados.0.smithi029.stdout:1444: writing smithi02979277-25 from 1996349 to 2502215 tid 2 2024-04-12T11:20:23.579 INFO:tasks.rados.rados.0.smithi029.stdout:1444: writing smithi02979277-25 from 3004131 to 3529569 tid 3 2024-04-12T11:20:23.579 INFO:tasks.rados.rados.0.smithi029.stdout:1445: write oid 20 current snap is 177 2024-04-12T11:20:23.579 INFO:tasks.rados.rados.0.smithi029.stdout:1445: seq_num 568 ranges {746376=551554,2054831=649524,3315201=231175} 2024-04-12T11:20:23.585 INFO:tasks.rados.rados.0.smithi029.stdout:1445: writing smithi02979277-20 from 746376 to 1297930 tid 1 2024-04-12T11:20:23.593 INFO:tasks.rados.rados.0.smithi029.stdout:1445: writing smithi02979277-20 from 2054831 to 2704355 tid 2 2024-04-12T11:20:23.598 INFO:tasks.rados.rados.0.smithi029.stdout:1445: writing smithi02979277-20 from 3315201 to 3546376 tid 3 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1446: snap_create 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1440: finishing rollback tid 1 to smithi02979277-50 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1440: finishing rollback tid 2 to smithi02979277-50 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 471 (ObjNum 474 snap 156 seq_num 474) dirty exists 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1443: finishing write tid 1 to smithi02979277-11 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1443: finishing write tid 2 to smithi02979277-11 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1443: finishing write tid 3 to smithi02979277-11 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1443: finishing write tid 4 to smithi02979277-11 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1443: finishing write tid 5 to smithi02979277-11 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1443: oid 11 updating version 0 to 908 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1443: oid 11 updating version 908 to 909 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1443: oid 11 updating version 909 to 910 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:1443: oid 11 updating version 910 to 911 2024-04-12T11:20:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 911 (ObjNum 566 snap 177 seq_num 566) dirty exists 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1443: left oid 11 (ObjNum 566 snap 177 seq_num 566) 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1444: finishing write tid 1 to smithi02979277-25 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1444: finishing write tid 2 to smithi02979277-25 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1444: finishing write tid 3 to smithi02979277-25 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1444: finishing write tid 4 to smithi02979277-25 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1444: finishing write tid 5 to smithi02979277-25 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1444: oid 25 updating version 0 to 738 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1444: oid 25 updating version 738 to 739 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1444: oid 25 version 739 is already newer than 737 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1444: oid 25 version 739 is already newer than 736 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 739 (ObjNum 567 snap 177 seq_num 567) dirty exists 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1444: left oid 25 (ObjNum 567 snap 177 seq_num 567) 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1445: finishing write tid 1 to smithi02979277-20 2024-04-12T11:20:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:1445: finishing write tid 2 to smithi02979277-20 2024-04-12T11:20:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:1445: finishing write tid 3 to smithi02979277-20 2024-04-12T11:20:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:1445: finishing write tid 4 to smithi02979277-20 2024-04-12T11:20:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:1445: finishing write tid 5 to smithi02979277-20 2024-04-12T11:20:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:1445: oid 20 updating version 0 to 381 2024-04-12T11:20:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:1445: oid 20 updating version 381 to 382 2024-04-12T11:20:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:1445: oid 20 updating version 382 to 383 2024-04-12T11:20:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:1445: oid 20 updating version 383 to 384 2024-04-12T11:20:23.602 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 384 (ObjNum 568 snap 177 seq_num 568) dirty exists 2024-04-12T11:20:23.602 INFO:tasks.rados.rados.0.smithi029.stdout:1445: left oid 20 (ObjNum 568 snap 177 seq_num 568) 2024-04-12T11:20:23.759 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T11:20:23.759 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (6m) 12s ago 7m 25.7M - 0.25.0 c8568f914cd2 5f36b9c37cec 2024-04-12T11:20:23.759 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi087 *:3000 running (7m) 14s ago 7m 87.9M - 9.4.7 954c08fa6188 d4ba4d587fdb 2024-04-12T11:20:23.759 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.scbuux smithi029 running (7m) 12s ago 7m 17.8M - 18.2.2-1187-gd540ebac bb2c5eac26a4 a659ce0bccaf 2024-04-12T11:20:23.759 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi087.mbflzu smithi087 running (7m) 14s ago 7m 17.2M - 18.2.2-1187-gd540ebac bb2c5eac26a4 64bafd1e6a4c 2024-04-12T11:20:23.759 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi087 *:8443,9283,8765 running (3m) 14s ago 12m 451M - 19.0.0-2874-g2cee1857 4a0e59462345 5a46135dfdbc 2024-04-12T11:20:23.759 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (3m) 12s ago 14m 502M - 19.0.0-2874-g2cee1857 4a0e59462345 05c187489262 2024-04-12T11:20:23.759 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (3m) 12s ago 14m 54.5M 2048M 19.0.0-2874-g2cee1857 4a0e59462345 d1506def0998 2024-04-12T11:20:23.759 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi087 starting - - - 2048M 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (97s) 12s ago 12m 40.6M 2048M 19.0.0-2874-g2cee1857 4a0e59462345 24b3f58e1d66 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (7m) 12s ago 7m 15.5M - 1.5.0 0da6a335fe13 32124106e912 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi087 *:9100 running (7m) 14s ago 7m 15.9M - 1.5.0 0da6a335fe13 78910b2d30c6 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (11m) 12s ago 11m 108M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 f65c324b0084 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (11m) 12s ago 11m 80.2M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 bb552e04f578 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (11m) 12s ago 11m 101M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 dbb7abb8911b 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (10m) 12s ago 10m 73.0M 2481M 18.2.2-1187-gd540ebac bb2c5eac26a4 a9b5c72d15da 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi087 running (10m) 14s ago 10m 91.5M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 ab3a8e864afd 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi087 running (9m) 14s ago 9m 106M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 573e2f90e4da 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi087 running (9m) 14s ago 9m 74.7M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 2b67d3f477dd 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi087 running (9m) 14s ago 9m 94.4M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 9c484b9f0790 2024-04-12T11:20:23.760 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi087 *:9095 running (3m) 14s ago 8m 49.0M - 2.43.0 a07b618ecd1d 569140fc5a53 2024-04-12T11:20:23.769 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:23.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:24.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch upgrade pause' 2024-04-12T11:20:24.543 INFO:tasks.rados.rados.0.smithi029.stdout:1440: done (5 left) 2024-04-12T11:20:24.543 INFO:tasks.rados.rados.0.smithi029.stdout:1441: done (4 left) 2024-04-12T11:20:24.543 INFO:tasks.rados.rados.0.smithi029.stdout:1443: done (3 left) 2024-04-12T11:20:24.543 INFO:tasks.rados.rados.0.smithi029.stdout:1444: done (2 left) 2024-04-12T11:20:24.543 INFO:tasks.rados.rados.0.smithi029.stdout:1445: done (1 left) 2024-04-12T11:20:24.543 INFO:tasks.rados.rados.0.smithi029.stdout:1446: done (0 left) 2024-04-12T11:20:24.544 INFO:tasks.rados.rados.0.smithi029.stdout:1447: write oid 17 current snap is 178 2024-04-12T11:20:24.544 INFO:tasks.rados.rados.0.smithi029.stdout:1447: seq_num 569 ranges {763183=626803,1996960=793182,3226271=336912} 2024-04-12T11:20:24.550 INFO:tasks.rados.rados.0.smithi029.stdout:1447: writing smithi02979277-17 from 763183 to 1389986 tid 1 2024-04-12T11:20:24.558 INFO:tasks.rados.rados.0.smithi029.stdout:1447: writing smithi02979277-17 from 1996960 to 2790142 tid 2 2024-04-12T11:20:24.562 INFO:tasks.rados.rados.0.smithi029.stdout:1447: writing smithi02979277-17 from 3226271 to 3563183 tid 3 2024-04-12T11:20:24.562 INFO:tasks.rados.rados.0.smithi029.stdout:1448: delete oid 4 current snap is 178 2024-04-12T11:20:24.562 INFO:tasks.rados.rados.0.smithi029.stdout:1447: finishing write tid 1 to smithi02979277-17 2024-04-12T11:20:24.563 INFO:tasks.rados.rados.0.smithi029.stdout:1447: finishing write tid 2 to smithi02979277-17 2024-04-12T11:20:24.566 INFO:tasks.rados.rados.0.smithi029.stdout:1448: done (1 left) 2024-04-12T11:20:24.566 INFO:tasks.rados.rados.0.smithi029.stdout:1449: snap_create 2024-04-12T11:20:24.566 INFO:tasks.rados.rados.0.smithi029.stdout:1447: finishing write tid 3 to smithi02979277-17 2024-04-12T11:20:24.567 INFO:tasks.rados.rados.0.smithi029.stdout:1447: finishing write tid 4 to smithi02979277-17 2024-04-12T11:20:24.567 INFO:tasks.rados.rados.0.smithi029.stdout:1447: finishing write tid 5 to smithi02979277-17 2024-04-12T11:20:24.567 INFO:tasks.rados.rados.0.smithi029.stdout:1447: oid 17 updating version 0 to 744 2024-04-12T11:20:24.567 INFO:tasks.rados.rados.0.smithi029.stdout:1447: oid 17 version 744 is already newer than 742 2024-04-12T11:20:24.567 INFO:tasks.rados.rados.0.smithi029.stdout:1447: oid 17 version 744 is already newer than 743 2024-04-12T11:20:24.567 INFO:tasks.rados.rados.0.smithi029.stdout:1447: oid 17 version 744 is already newer than 741 2024-04-12T11:20:24.567 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 744 (ObjNum 569 snap 178 seq_num 569) dirty exists 2024-04-12T11:20:24.567 INFO:tasks.rados.rados.0.smithi029.stdout:1447: left oid 17 (ObjNum 569 snap 178 seq_num 569) 2024-04-12T11:20:24.665 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:20:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-mon[93650]: pgmap v265: 89 pgs: 89 active+clean; 295 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-mon[93650]: osdmap e420: 8 total, 8 up, 8 in 2024-04-12T11:20:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-mon[93650]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:20:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-mon[97787]: pgmap v265: 89 pgs: 89 active+clean; 295 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-mon[97787]: osdmap e420: 8 total, 8 up, 8 in 2024-04-12T11:20:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-mon[97787]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:20:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:24 smithi087 ceph-mon[89014]: pgmap v265: 89 pgs: 89 active+clean; 295 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:24 smithi087 ceph-mon[89014]: osdmap e420: 8 total, 8 up, 8 in 2024-04-12T11:20:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:24 smithi087 ceph-mon[89014]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:20:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:24 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:24 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:25.101 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:20:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:20:24] "GET /metrics HTTP/1.1" 200 36179 "" "Prometheus/2.43.0" 2024-04-12T11:20:25.547 INFO:tasks.rados.rados.0.smithi029.stdout:1447: done (1 left) 2024-04-12T11:20:25.547 INFO:tasks.rados.rados.0.smithi029.stdout:1449: done (0 left) 2024-04-12T11:20:25.547 INFO:tasks.rados.rados.0.smithi029.stdout:1450: snap_create 2024-04-12T11:20:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:25 smithi029 ceph-mon[93650]: osdmap e421: 8 total, 8 up, 8 in 2024-04-12T11:20:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:25.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:25 smithi029 ceph-mon[97787]: osdmap e421: 8 total, 8 up, 8 in 2024-04-12T11:20:25.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:25.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:25.854 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:25 smithi087 ceph-mon[89014]: osdmap e421: 8 total, 8 up, 8 in 2024-04-12T11:20:25.854 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:25.854 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:26.549 INFO:tasks.rados.rados.0.smithi029.stdout:1450: done (0 left) 2024-04-12T11:20:26.549 INFO:tasks.rados.rados.0.smithi029.stdout:1451: rollback oid 17 current snap is 180 2024-04-12T11:20:26.549 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 129 2024-04-12T11:20:26.549 INFO:tasks.rados.rados.0.smithi029.stdout:1452: read oid 29 snap -1 2024-04-12T11:20:26.549 INFO:tasks.rados.rados.0.smithi029.stdout:1452: expect deleted 2024-04-12T11:20:26.549 INFO:tasks.rados.rados.0.smithi029.stdout:1453: snap_create 2024-04-12T11:20:26.553 INFO:tasks.rados.rados.0.smithi029.stdout:1451: finishing rollback tid 0 to smithi02979277-17 2024-04-12T11:20:26.553 INFO:tasks.rados.rados.0.smithi029.stdout:1451: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:20:26.554 INFO:tasks.rados.rados.0.smithi029.stdout:1451: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:20:26.554 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 748 (ObjNum 390 snap 121 seq_num 390) dirty exists 2024-04-12T11:20:26.669 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:26 smithi087 ceph-mon[89014]: pgmap v268: 89 pgs: 89 active+clean; 295 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:26.669 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:26 smithi087 ceph-mon[89014]: osdmap e422: 8 total, 8 up, 8 in 2024-04-12T11:20:26.738 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:20:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:26 smithi029 ceph-mon[93650]: pgmap v268: 89 pgs: 89 active+clean; 295 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:26 smithi029 ceph-mon[93650]: osdmap e422: 8 total, 8 up, 8 in 2024-04-12T11:20:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:26 smithi029 ceph-mon[97787]: pgmap v268: 89 pgs: 89 active+clean; 295 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:26 smithi029 ceph-mon[97787]: osdmap e422: 8 total, 8 up, 8 in 2024-04-12T11:20:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:1451: done (2 left) 2024-04-12T11:20:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:1452: done (1 left) 2024-04-12T11:20:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:1453: done (0 left) 2024-04-12T11:20:27.551 INFO:tasks.rados.rados.0.smithi029.stdout:1454: delete oid 6 current snap is 181 2024-04-12T11:20:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:1454: done (0 left) 2024-04-12T11:20:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:1455: rollback oid 25 current snap is 181 2024-04-12T11:20:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 25 to 180 2024-04-12T11:20:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:1456: read oid 45 snap -1 2024-04-12T11:20:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:1456: expect (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:20:27.554 INFO:tasks.rados.rados.0.smithi029.stdout:1457: read oid 21 snap -1 2024-04-12T11:20:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:1457: expect (ObjNum 525 snap 170 seq_num 525) 2024-04-12T11:20:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:1458: read oid 46 snap -1 2024-04-12T11:20:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:1458: expect (ObjNum 487 snap 158 seq_num 487) 2024-04-12T11:20:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:1459: rollback oid 40 current snap is 181 2024-04-12T11:20:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 40 to 157 2024-04-12T11:20:27.555 INFO:tasks.rados.rados.0.smithi029.stdout:1460: snap_remove snap 161 2024-04-12T11:20:27.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'sleep 60' 2024-04-12T11:20:27.807 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:27 smithi087 ceph-mon[89014]: osdmap e423: 8 total, 8 up, 8 in 2024-04-12T11:20:27.808 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:27 smithi087 ceph-mon[89014]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:20:27.808 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:27 smithi087 ceph-mon[89014]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:20:27.808 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:27 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:27.907 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:20:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:27 smithi029 ceph-mon[93650]: osdmap e423: 8 total, 8 up, 8 in 2024-04-12T11:20:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:27 smithi029 ceph-mon[93650]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:20:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:27 smithi029 ceph-mon[93650]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:20:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:27 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:27 smithi029 ceph-mon[97787]: osdmap e423: 8 total, 8 up, 8 in 2024-04-12T11:20:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:27 smithi029 ceph-mon[97787]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:20:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:27 smithi029 ceph-mon[97787]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:20:28.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:27 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:28.561 INFO:tasks.rados.rados.0.smithi029.stdout:1460: done (5 left) 2024-04-12T11:20:28.562 INFO:tasks.rados.rados.0.smithi029.stdout:1461: read oid 49 snap -1 2024-04-12T11:20:28.562 INFO:tasks.rados.rados.0.smithi029.stdout:1461: expect (ObjNum 448 snap 147 seq_num 448) 2024-04-12T11:20:28.562 INFO:tasks.rados.rados.0.smithi029.stdout:1462: snap_remove snap 179 2024-04-12T11:20:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[93650]: pgmap v271: 89 pgs: 89 active+clean; 311 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-04-12T11:20:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[93650]: osdmap e424: 8 total, 8 up, 8 in 2024-04-12T11:20:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:20:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:28.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[97787]: pgmap v271: 89 pgs: 89 active+clean; 311 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-04-12T11:20:28.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[97787]: osdmap e424: 8 total, 8 up, 8 in 2024-04-12T11:20:28.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:20:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:28 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:28 smithi087 ceph-mon[89014]: pgmap v271: 89 pgs: 89 active+clean; 311 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-04-12T11:20:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:28 smithi087 ceph-mon[89014]: osdmap e424: 8 total, 8 up, 8 in 2024-04-12T11:20:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:28 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:28 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:28 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:28 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:20:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:28 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:29.567 INFO:tasks.rados.rados.0.smithi029.stdout:1462: done (6 left) 2024-04-12T11:20:29.567 INFO:tasks.rados.rados.0.smithi029.stdout:1463: rollback oid 27 current snap is 181 2024-04-12T11:20:29.567 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 164 2024-04-12T11:20:29.567 INFO:tasks.rados.rados.0.smithi029.stdout:1456: expect (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:20:29.589 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[97787]: osdmap e425: 8 total, 8 up, 8 in 2024-04-12T11:20:29.589 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[97787]: Reconfiguring mon.a (monmap changed)... 2024-04-12T11:20:29.589 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:29.589 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:29.589 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:29.589 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[97787]: Reconfiguring daemon mon.a on smithi029 2024-04-12T11:20:29.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[93650]: osdmap e425: 8 total, 8 up, 8 in 2024-04-12T11:20:29.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[93650]: Reconfiguring mon.a (monmap changed)... 2024-04-12T11:20:29.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:29.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:29.590 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:29.590 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:29 smithi029 ceph-mon[93650]: Reconfiguring daemon mon.a on smithi029 2024-04-12T11:20:29.671 INFO:tasks.rados.rados.0.smithi029.stdout:1458: expect (ObjNum 487 snap 158 seq_num 487) 2024-04-12T11:20:29.803 INFO:tasks.rados.rados.0.smithi029.stdout:1459: finishing rollback tid 0 to smithi02979277-40 2024-04-12T11:20:29.803 INFO:tasks.rados.rados.0.smithi029.stdout:1455: finishing rollback tid 0 to smithi02979277-25 2024-04-12T11:20:29.803 INFO:tasks.rados.rados.0.smithi029.stdout:1459: finishing rollback tid 1 to smithi02979277-40 2024-04-12T11:20:29.803 INFO:tasks.rados.rados.0.smithi029.stdout:1459: finishing rollback tid 2 to smithi02979277-40 2024-04-12T11:20:29.803 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 915 (ObjNum 458 snap 150 seq_num 458) dirty exists 2024-04-12T11:20:29.803 INFO:tasks.rados.rados.0.smithi029.stdout:1455: finishing rollback tid 1 to smithi02979277-25 2024-04-12T11:20:29.803 INFO:tasks.rados.rados.0.smithi029.stdout:1455: finishing rollback tid 2 to smithi02979277-25 2024-04-12T11:20:29.803 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 752 (ObjNum 567 snap 177 seq_num 567) dirty exists 2024-04-12T11:20:29.803 INFO:tasks.rados.rados.0.smithi029.stdout:1457: expect (ObjNum 525 snap 170 seq_num 525) 2024-04-12T11:20:29.933 INFO:tasks.rados.rados.0.smithi029.stdout:1461: expect (ObjNum 448 snap 147 seq_num 448) 2024-04-12T11:20:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:29 smithi087 ceph-mon[89014]: osdmap e425: 8 total, 8 up, 8 in 2024-04-12T11:20:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:29 smithi087 ceph-mon[89014]: Reconfiguring mon.a (monmap changed)... 2024-04-12T11:20:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:29 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:29 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:29 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:29 smithi087 ceph-mon[89014]: Reconfiguring daemon mon.a on smithi029 2024-04-12T11:20:30.100 INFO:tasks.rados.rados.0.smithi029.stdout:1463: finishing rollback tid 0 to smithi02979277-27 2024-04-12T11:20:30.100 INFO:tasks.rados.rados.0.smithi029.stdout:1463: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:20:30.100 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 581 (ObjNum 289 snap 0 seq_num 862900838) dirty dne 2024-04-12T11:20:30.100 INFO:tasks.rados.rados.0.smithi029.stdout:1455: done (6 left) 2024-04-12T11:20:30.100 INFO:tasks.rados.rados.0.smithi029.stdout:1456: done (5 left) 2024-04-12T11:20:30.100 INFO:tasks.rados.rados.0.smithi029.stdout:1457: done (4 left) 2024-04-12T11:20:30.100 INFO:tasks.rados.rados.0.smithi029.stdout:1458: done (3 left) 2024-04-12T11:20:30.100 INFO:tasks.rados.rados.0.smithi029.stdout:1459: done (2 left) 2024-04-12T11:20:30.100 INFO:tasks.rados.rados.0.smithi029.stdout:1461: done (1 left) 2024-04-12T11:20:30.101 INFO:tasks.rados.rados.0.smithi029.stdout:1463: done (0 left) 2024-04-12T11:20:30.101 INFO:tasks.rados.rados.0.smithi029.stdout:1464: read oid 18 snap -1 2024-04-12T11:20:30.101 INFO:tasks.rados.rados.0.smithi029.stdout:1464: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:20:30.101 INFO:tasks.rados.rados.0.smithi029.stdout:1465: snap_remove snap 164 2024-04-12T11:20:30.572 INFO:tasks.rados.rados.0.smithi029.stdout:1465: done (1 left) 2024-04-12T11:20:30.572 INFO:tasks.rados.rados.0.smithi029.stdout:1466: delete oid 31 current snap is 181 2024-04-12T11:20:30.572 INFO:tasks.rados.rados.0.smithi029.stdout:1464: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:20:30.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[97787]: pgmap v274: 89 pgs: 89 active+clean; 311 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-04-12T11:20:30.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[97787]: osdmap e426: 8 total, 8 up, 8 in 2024-04-12T11:20:30.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:30.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:30.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:20:30.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:20:30.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:30.664 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[93650]: pgmap v274: 89 pgs: 89 active+clean; 311 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-04-12T11:20:30.664 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[93650]: osdmap e426: 8 total, 8 up, 8 in 2024-04-12T11:20:30.664 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:30.664 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:30.664 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:20:30.664 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:20:30.664 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:30.714 INFO:tasks.rados.rados.0.smithi029.stdout:1464: done (1 left) 2024-04-12T11:20:30.714 INFO:tasks.rados.rados.0.smithi029.stdout:1466: done (0 left) 2024-04-12T11:20:30.715 INFO:tasks.rados.rados.0.smithi029.stdout:1467: write oid 14 current snap is 181 2024-04-12T11:20:30.715 INFO:tasks.rados.rados.0.smithi029.stdout:1467: seq_num 573 ranges {430411=644152,1830888=400520,2935963=402981} 2024-04-12T11:20:30.720 INFO:tasks.rados.rados.0.smithi029.stdout:1467: writing smithi02979277-14 from 430411 to 1074563 tid 1 2024-04-12T11:20:30.727 INFO:tasks.rados.rados.0.smithi029.stdout:1467: writing smithi02979277-14 from 1830888 to 2231408 tid 2 2024-04-12T11:20:30.733 INFO:tasks.rados.rados.0.smithi029.stdout:1467: writing smithi02979277-14 from 2935963 to 3338944 tid 3 2024-04-12T11:20:30.733 INFO:tasks.rados.rados.0.smithi029.stdout:1468: read oid 30 snap -1 2024-04-12T11:20:30.733 INFO:tasks.rados.rados.0.smithi029.stdout:1468: expect (ObjNum 504 snap 162 seq_num 504) 2024-04-12T11:20:30.733 INFO:tasks.rados.rados.0.smithi029.stdout:1469: snap_create 2024-04-12T11:20:30.733 INFO:tasks.rados.rados.0.smithi029.stdout:1467: finishing write tid 1 to smithi02979277-14 2024-04-12T11:20:30.733 INFO:tasks.rados.rados.0.smithi029.stdout:1467: finishing write tid 2 to smithi02979277-14 2024-04-12T11:20:30.737 INFO:tasks.rados.rados.0.smithi029.stdout:1467: finishing write tid 3 to smithi02979277-14 2024-04-12T11:20:30.737 INFO:tasks.rados.rados.0.smithi029.stdout:1467: finishing write tid 4 to smithi02979277-14 2024-04-12T11:20:30.737 INFO:tasks.rados.rados.0.smithi029.stdout:1467: finishing write tid 5 to smithi02979277-14 2024-04-12T11:20:30.737 INFO:tasks.rados.rados.0.smithi029.stdout:1467: oid 14 updating version 0 to 869 2024-04-12T11:20:30.737 INFO:tasks.rados.rados.0.smithi029.stdout:1467: oid 14 updating version 869 to 871 2024-04-12T11:20:30.737 INFO:tasks.rados.rados.0.smithi029.stdout:1467: oid 14 version 871 is already newer than 870 2024-04-12T11:20:30.737 INFO:tasks.rados.rados.0.smithi029.stdout:1467: oid 14 version 871 is already newer than 868 2024-04-12T11:20:30.737 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 871 (ObjNum 573 snap 181 seq_num 573) dirty exists 2024-04-12T11:20:30.737 INFO:tasks.rados.rados.0.smithi029.stdout:1467: left oid 14 (ObjNum 573 snap 181 seq_num 573) 2024-04-12T11:20:30.737 INFO:tasks.rados.rados.0.smithi029.stdout:1468: expect (ObjNum 504 snap 162 seq_num 504) 2024-04-12T11:20:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:30 smithi087 ceph-mon[89014]: pgmap v274: 89 pgs: 89 active+clean; 311 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.9 MiB/s wr, 11 op/s 2024-04-12T11:20:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:30 smithi087 ceph-mon[89014]: osdmap e426: 8 total, 8 up, 8 in 2024-04-12T11:20:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:30 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:30 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:30 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:20:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:30 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:20:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:30 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:31.578 INFO:tasks.rados.rados.0.smithi029.stdout:1467: done (2 left) 2024-04-12T11:20:31.579 INFO:tasks.rados.rados.0.smithi029.stdout:1468: done (1 left) 2024-04-12T11:20:31.579 INFO:tasks.rados.rados.0.smithi029.stdout:1469: done (0 left) 2024-04-12T11:20:31.579 INFO:tasks.rados.rados.0.smithi029.stdout:1470: write oid 32 current snap is 182 2024-04-12T11:20:31.579 INFO:tasks.rados.rados.0.smithi029.stdout:1470: seq_num 574 ranges {447218=719401,1889370=660531,2963386=614253} 2024-04-12T11:20:31.584 INFO:tasks.rados.rados.0.smithi029.stdout:1470: writing smithi02979277-32 from 447218 to 1166619 tid 1 2024-04-12T11:20:31.592 INFO:tasks.rados.rados.0.smithi029.stdout:1470: writing smithi02979277-32 from 1889370 to 2549901 tid 2 2024-04-12T11:20:31.598 INFO:tasks.rados.rados.0.smithi029.stdout:1470: writing smithi02979277-32 from 2963386 to 3577639 tid 3 2024-04-12T11:20:31.598 INFO:tasks.rados.rados.0.smithi029.stdout:1471: snap_create 2024-04-12T11:20:31.598 INFO:tasks.rados.rados.0.smithi029.stdout:1470: finishing write tid 1 to smithi02979277-32 2024-04-12T11:20:31.598 INFO:tasks.rados.rados.0.smithi029.stdout:1470: finishing write tid 2 to smithi02979277-32 2024-04-12T11:20:31.600 INFO:tasks.rados.rados.0.smithi029.stdout:1470: finishing write tid 3 to smithi02979277-32 2024-04-12T11:20:31.600 INFO:tasks.rados.rados.0.smithi029.stdout:1470: finishing write tid 4 to smithi02979277-32 2024-04-12T11:20:31.600 INFO:tasks.rados.rados.0.smithi029.stdout:1470: finishing write tid 5 to smithi02979277-32 2024-04-12T11:20:31.600 INFO:tasks.rados.rados.0.smithi029.stdout:1470: oid 32 updating version 0 to 566 2024-04-12T11:20:31.600 INFO:tasks.rados.rados.0.smithi029.stdout:1470: oid 32 updating version 566 to 567 2024-04-12T11:20:31.601 INFO:tasks.rados.rados.0.smithi029.stdout:1470: oid 32 version 567 is already newer than 565 2024-04-12T11:20:31.601 INFO:tasks.rados.rados.0.smithi029.stdout:1470: oid 32 version 567 is already newer than 564 2024-04-12T11:20:31.601 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 567 (ObjNum 574 snap 182 seq_num 574) dirty exists 2024-04-12T11:20:31.601 INFO:tasks.rados.rados.0.smithi029.stdout:1470: left oid 32 (ObjNum 574 snap 182 seq_num 574) 2024-04-12T11:20:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:31 smithi087 ceph-mon[89014]: Reconfiguring mgr.y (monmap changed)... 2024-04-12T11:20:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:31 smithi087 ceph-mon[89014]: Reconfiguring daemon mgr.y on smithi029 2024-04-12T11:20:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:31 smithi087 ceph-mon[89014]: osdmap e427: 8 total, 8 up, 8 in 2024-04-12T11:20:32.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:31 smithi029 ceph-mon[97787]: Reconfiguring mgr.y (monmap changed)... 2024-04-12T11:20:32.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:31 smithi029 ceph-mon[97787]: Reconfiguring daemon mgr.y on smithi029 2024-04-12T11:20:32.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:31 smithi029 ceph-mon[97787]: osdmap e427: 8 total, 8 up, 8 in 2024-04-12T11:20:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:31 smithi029 ceph-mon[93650]: Reconfiguring mgr.y (monmap changed)... 2024-04-12T11:20:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:31 smithi029 ceph-mon[93650]: Reconfiguring daemon mgr.y on smithi029 2024-04-12T11:20:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:31 smithi029 ceph-mon[93650]: osdmap e427: 8 total, 8 up, 8 in 2024-04-12T11:20:32.538 INFO:tasks.rados.rados.0.smithi029.stdout:1470: done (1 left) 2024-04-12T11:20:32.538 INFO:tasks.rados.rados.0.smithi029.stdout:1471: done (0 left) 2024-04-12T11:20:32.538 INFO:tasks.rados.rados.0.smithi029.stdout:1472: write oid 12 current snap is 183 2024-04-12T11:20:32.538 INFO:tasks.rados.rados.0.smithi029.stdout:1472: seq_num 575 ranges {464025=794650,1947852=404189,2990809=425525} 2024-04-12T11:20:32.545 INFO:tasks.rados.rados.0.smithi029.stdout:1472: writing smithi02979277-12 from 464025 to 1258675 tid 1 2024-04-12T11:20:32.551 INFO:tasks.rados.rados.0.smithi029.stdout:1472: writing smithi02979277-12 from 1947852 to 2352041 tid 2 2024-04-12T11:20:32.557 INFO:tasks.rados.rados.0.smithi029.stdout:1472: writing smithi02979277-12 from 2990809 to 3416334 tid 3 2024-04-12T11:20:32.557 INFO:tasks.rados.rados.0.smithi029.stdout:1473: snap_create 2024-04-12T11:20:32.557 INFO:tasks.rados.rados.0.smithi029.stdout:1472: finishing write tid 1 to smithi02979277-12 2024-04-12T11:20:32.557 INFO:tasks.rados.rados.0.smithi029.stdout:1472: finishing write tid 2 to smithi02979277-12 2024-04-12T11:20:32.560 INFO:tasks.rados.rados.0.smithi029.stdout:1472: finishing write tid 3 to smithi02979277-12 2024-04-12T11:20:32.560 INFO:tasks.rados.rados.0.smithi029.stdout:1472: finishing write tid 4 to smithi02979277-12 2024-04-12T11:20:32.560 INFO:tasks.rados.rados.0.smithi029.stdout:1472: finishing write tid 5 to smithi02979277-12 2024-04-12T11:20:32.560 INFO:tasks.rados.rados.0.smithi029.stdout:1472: oid 12 updating version 0 to 690 2024-04-12T11:20:32.560 INFO:tasks.rados.rados.0.smithi029.stdout:1472: oid 12 updating version 690 to 692 2024-04-12T11:20:32.560 INFO:tasks.rados.rados.0.smithi029.stdout:1472: oid 12 version 692 is already newer than 689 2024-04-12T11:20:32.560 INFO:tasks.rados.rados.0.smithi029.stdout:1472: oid 12 version 692 is already newer than 691 2024-04-12T11:20:32.560 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 692 (ObjNum 575 snap 183 seq_num 575) dirty exists 2024-04-12T11:20:32.561 INFO:tasks.rados.rados.0.smithi029.stdout:1472: left oid 12 (ObjNum 575 snap 183 seq_num 575) 2024-04-12T11:20:32.575 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:32.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[97787]: pgmap v277: 89 pgs: 89 active+clean; 311 MiB data, 880 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:32.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[97787]: osdmap e428: 8 total, 8 up, 8 in 2024-04-12T11:20:32.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:32.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:32.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:32.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:32.826 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:32.826 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[97787]: osdmap e429: 8 total, 8 up, 8 in 2024-04-12T11:20:32.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[93650]: pgmap v277: 89 pgs: 89 active+clean; 311 MiB data, 880 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:32.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[93650]: osdmap e428: 8 total, 8 up, 8 in 2024-04-12T11:20:32.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:32.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:32.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:32.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:32.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:32.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:32 smithi029 ceph-mon[93650]: osdmap e429: 8 total, 8 up, 8 in 2024-04-12T11:20:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:32 smithi087 ceph-mon[89014]: pgmap v277: 89 pgs: 89 active+clean; 311 MiB data, 880 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:32 smithi087 ceph-mon[89014]: osdmap e428: 8 total, 8 up, 8 in 2024-04-12T11:20:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:32 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:32 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:32 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:32 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:32 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:32 smithi087 ceph-mon[89014]: osdmap e429: 8 total, 8 up, 8 in 2024-04-12T11:20:33.540 INFO:tasks.rados.rados.0.smithi029.stdout:1472: done (1 left) 2024-04-12T11:20:33.540 INFO:tasks.rados.rados.0.smithi029.stdout:1473: done (0 left) 2024-04-12T11:20:33.540 INFO:tasks.rados.rados.0.smithi029.stdout:1474: read oid 36 snap -1 2024-04-12T11:20:33.540 INFO:tasks.rados.rados.0.smithi029.stdout:1474: expect deleted 2024-04-12T11:20:33.540 INFO:tasks.rados.rados.0.smithi029.stdout:1475: write oid 10 current snap is 184 2024-04-12T11:20:33.540 INFO:tasks.rados.rados.0.smithi029.stdout:1475: seq_num 576 ranges {480832=469899,1606334=664200,3018232=662600} 2024-04-12T11:20:33.545 INFO:tasks.rados.rados.0.smithi029.stdout:1475: writing smithi02979277-10 from 480832 to 950731 tid 1 2024-04-12T11:20:33.553 INFO:tasks.rados.rados.0.smithi029.stdout:1475: writing smithi02979277-10 from 1606334 to 2270534 tid 2 2024-04-12T11:20:33.561 INFO:tasks.rados.rados.0.smithi029.stdout:1475: writing smithi02979277-10 from 3018232 to 3680832 tid 3 2024-04-12T11:20:33.561 INFO:tasks.rados.rados.0.smithi029.stdout:1476: rollback oid 48 current snap is 184 2024-04-12T11:20:33.561 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 180 2024-04-12T11:20:33.561 INFO:tasks.rados.rados.0.smithi029.stdout:1475: finishing write tid 1 to smithi02979277-10 2024-04-12T11:20:33.561 INFO:tasks.rados.rados.0.smithi029.stdout:1475: finishing write tid 2 to smithi02979277-10 2024-04-12T11:20:33.562 INFO:tasks.rados.rados.0.smithi029.stdout:1474: done (2 left) 2024-04-12T11:20:33.562 INFO:tasks.rados.rados.0.smithi029.stdout:1477: snap_remove snap 178 2024-04-12T11:20:33.806 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:33 smithi029 ceph-mon[93650]: Reconfiguring mon.c (monmap changed)... 2024-04-12T11:20:33.806 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:33 smithi029 ceph-mon[93650]: Reconfiguring daemon mon.c on smithi029 2024-04-12T11:20:33.806 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:33 smithi029 ceph-mon[93650]: osdmap e430: 8 total, 8 up, 8 in 2024-04-12T11:20:33.807 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:33 smithi029 ceph-mon[97787]: Reconfiguring mon.c (monmap changed)... 2024-04-12T11:20:33.807 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:33 smithi029 ceph-mon[97787]: Reconfiguring daemon mon.c on smithi029 2024-04-12T11:20:33.807 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:33 smithi029 ceph-mon[97787]: osdmap e430: 8 total, 8 up, 8 in 2024-04-12T11:20:33.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:33 smithi087 ceph-mon[89014]: Reconfiguring mon.c (monmap changed)... 2024-04-12T11:20:33.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:33 smithi087 ceph-mon[89014]: Reconfiguring daemon mon.c on smithi029 2024-04-12T11:20:33.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:33 smithi087 ceph-mon[89014]: osdmap e430: 8 total, 8 up, 8 in 2024-04-12T11:20:33.967 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:34.542 INFO:tasks.rados.rados.0.smithi029.stdout:1477: done (2 left) 2024-04-12T11:20:34.542 INFO:tasks.rados.rados.0.smithi029.stdout:1478: snap_remove snap 133 2024-04-12T11:20:35.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:20:34] "GET /metrics HTTP/1.1" 200 36179 "" "Prometheus/2.43.0" 2024-04-12T11:20:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[93650]: pgmap v280: 89 pgs: 89 active+clean; 316 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 841 KiB/s wr, 16 op/s 2024-04-12T11:20:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[93650]: Reconfiguring osd.0 (monmap changed)... 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[93650]: Reconfiguring daemon osd.0 on smithi029 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[93650]: osdmap e431: 8 total, 8 up, 8 in 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[97787]: pgmap v280: 89 pgs: 89 active+clean; 316 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 841 KiB/s wr, 16 op/s 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[97787]: Reconfiguring osd.0 (monmap changed)... 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[97787]: Reconfiguring daemon osd.0 on smithi029 2024-04-12T11:20:35.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:34 smithi029 ceph-mon[97787]: osdmap e431: 8 total, 8 up, 8 in 2024-04-12T11:20:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:34 smithi087 ceph-mon[89014]: pgmap v280: 89 pgs: 89 active+clean; 316 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 841 KiB/s wr, 16 op/s 2024-04-12T11:20:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:34 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:34 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:34 smithi087 ceph-mon[89014]: Reconfiguring osd.0 (monmap changed)... 2024-04-12T11:20:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:34 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T11:20:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:34 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:34 smithi087 ceph-mon[89014]: Reconfiguring daemon osd.0 on smithi029 2024-04-12T11:20:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:34 smithi087 ceph-mon[89014]: osdmap e431: 8 total, 8 up, 8 in 2024-04-12T11:20:35.545 INFO:tasks.rados.rados.0.smithi029.stdout:1478: done (2 left) 2024-04-12T11:20:35.545 INFO:tasks.rados.rados.0.smithi029.stdout:1479: read oid 47 snap -1 2024-04-12T11:20:35.545 INFO:tasks.rados.rados.0.smithi029.stdout:1479: expect (ObjNum 415 snap 130 seq_num 415) 2024-04-12T11:20:35.545 INFO:tasks.rados.rados.0.smithi029.stdout:1480: write oid 35 current snap is 184 2024-04-12T11:20:35.545 INFO:tasks.rados.rados.0.smithi029.stdout:1480: seq_num 577 ranges {497639=545148,1548463=407858,2529302=564422,3528143=169496} 2024-04-12T11:20:35.551 INFO:tasks.rados.rados.0.smithi029.stdout:1480: writing smithi02979277-35 from 497639 to 1042787 tid 1 2024-04-12T11:20:35.557 INFO:tasks.rados.rados.0.smithi029.stdout:1480: writing smithi02979277-35 from 1548463 to 1956321 tid 2 2024-04-12T11:20:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:1480: writing smithi02979277-35 from 2529302 to 3093724 tid 3 2024-04-12T11:20:35.569 INFO:tasks.rados.rados.0.smithi029.stdout:1480: writing smithi02979277-35 from 3528143 to 3697639 tid 4 2024-04-12T11:20:35.569 INFO:tasks.rados.rados.0.smithi029.stdout:1481: read oid 43 snap -1 2024-04-12T11:20:35.569 INFO:tasks.rados.rados.0.smithi029.stdout:1481: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:20:35.569 INFO:tasks.rados.rados.0.smithi029.stdout:1482: snap_create 2024-04-12T11:20:35.569 INFO:tasks.rados.rados.0.smithi029.stdout:1475: finishing write tid 3 to smithi02979277-10 2024-04-12T11:20:35.569 INFO:tasks.rados.rados.0.smithi029.stdout:1475: finishing write tid 4 to smithi02979277-10 2024-04-12T11:20:35.569 INFO:tasks.rados.rados.0.smithi029.stdout:1475: finishing write tid 5 to smithi02979277-10 2024-04-12T11:20:35.569 INFO:tasks.rados.rados.0.smithi029.stdout:1475: oid 10 updating version 0 to 659 2024-04-12T11:20:35.569 INFO:tasks.rados.rados.0.smithi029.stdout:1475: oid 10 version 659 is already newer than 658 2024-04-12T11:20:35.569 INFO:tasks.rados.rados.0.smithi029.stdout:1475: oid 10 updating version 659 to 661 2024-04-12T11:20:35.570 INFO:tasks.rados.rados.0.smithi029.stdout:1475: oid 10 version 661 is already newer than 660 2024-04-12T11:20:35.570 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 661 (ObjNum 576 snap 184 seq_num 576) dirty exists 2024-04-12T11:20:35.570 INFO:tasks.rados.rados.0.smithi029.stdout:1475: left oid 10 (ObjNum 576 snap 184 seq_num 576) 2024-04-12T11:20:35.570 INFO:tasks.rados.rados.0.smithi029.stdout:1476: finishing rollback tid 0 to smithi02979277-48 2024-04-12T11:20:35.570 INFO:tasks.rados.rados.0.smithi029.stdout:1476: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:20:35.570 INFO:tasks.rados.rados.0.smithi029.stdout:1476: finishing rollback tid 2 to smithi02979277-48 2024-04-12T11:20:35.570 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 665 (ObjNum 528 snap 170 seq_num 528) dirty exists 2024-04-12T11:20:35.570 INFO:tasks.rados.rados.0.smithi029.stdout:1479: expect (ObjNum 415 snap 130 seq_num 415) 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: finishing write tid 1 to smithi02979277-35 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: finishing write tid 2 to smithi02979277-35 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: finishing write tid 3 to smithi02979277-35 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: finishing write tid 4 to smithi02979277-35 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: finishing write tid 5 to smithi02979277-35 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: finishing write tid 6 to smithi02979277-35 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: oid 35 updating version 0 to 942 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: oid 35 updating version 942 to 943 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: oid 35 version 943 is already newer than 941 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: oid 35 version 943 is already newer than 934 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:1480: oid 35 version 943 is already newer than 933 2024-04-12T11:20:35.796 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 943 (ObjNum 577 snap 184 seq_num 577) dirty exists 2024-04-12T11:20:35.797 INFO:tasks.rados.rados.0.smithi029.stdout:1480: left oid 35 (ObjNum 577 snap 184 seq_num 577) 2024-04-12T11:20:35.797 INFO:tasks.rados.rados.0.smithi029.stdout:1481: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:20:36.548 INFO:tasks.rados.rados.0.smithi029.stdout:1475: done (5 left) 2024-04-12T11:20:36.548 INFO:tasks.rados.rados.0.smithi029.stdout:1476: done (4 left) 2024-04-12T11:20:36.548 INFO:tasks.rados.rados.0.smithi029.stdout:1479: done (3 left) 2024-04-12T11:20:36.548 INFO:tasks.rados.rados.0.smithi029.stdout:1480: done (2 left) 2024-04-12T11:20:36.548 INFO:tasks.rados.rados.0.smithi029.stdout:1481: done (1 left) 2024-04-12T11:20:36.548 INFO:tasks.rados.rados.0.smithi029.stdout:1482: done (0 left) 2024-04-12T11:20:36.548 INFO:tasks.rados.rados.0.smithi029.stdout:1483: write oid 7 current snap is 185 2024-04-12T11:20:36.548 INFO:tasks.rados.rados.0.smithi029.stdout:1483: seq_num 578 ranges {514446=736750,1723298=667869,3073078=641368} 2024-04-12T11:20:36.554 INFO:tasks.rados.rados.0.smithi029.stdout:1483: writing smithi02979277-7 from 514446 to 1251196 tid 1 2024-04-12T11:20:36.561 INFO:tasks.rados.rados.0.smithi029.stdout:1483: writing smithi02979277-7 from 1723298 to 2391167 tid 2 2024-04-12T11:20:36.568 INFO:tasks.rados.rados.0.smithi029.stdout:1483: writing smithi02979277-7 from 3073078 to 3714446 tid 3 2024-04-12T11:20:36.568 INFO:tasks.rados.rados.0.smithi029.stdout:1484: write oid 21 current snap is 185 2024-04-12T11:20:36.568 INFO:tasks.rados.rados.0.smithi029.stdout:1483: finishing write tid 1 to smithi02979277-7 2024-04-12T11:20:36.568 INFO:tasks.rados.rados.0.smithi029.stdout:1483: finishing write tid 2 to smithi02979277-7 2024-04-12T11:20:36.568 INFO:tasks.rados.rados.0.smithi029.stdout:1484: seq_num 579 ranges {531253=411999,1381780=411527,2300501=586966,3376974=354279} 2024-04-12T11:20:36.573 INFO:tasks.rados.rados.0.smithi029.stdout:1484: writing smithi02979277-21 from 531253 to 943252 tid 1 2024-04-12T11:20:36.578 INFO:tasks.rados.rados.0.smithi029.stdout:1484: writing smithi02979277-21 from 1381780 to 1793307 tid 2 2024-04-12T11:20:36.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[97787]: pgmap v283: 89 pgs: 89 active+clean; 316 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 841 KiB/s wr, 16 op/s 2024-04-12T11:20:36.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[97787]: osdmap e432: 8 total, 8 up, 8 in 2024-04-12T11:20:36.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:36.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:36.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[97787]: Reconfiguring osd.1 (monmap changed)... 2024-04-12T11:20:36.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T11:20:36.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:36.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[97787]: Reconfiguring daemon osd.1 on smithi029 2024-04-12T11:20:36.582 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:36.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[93650]: pgmap v283: 89 pgs: 89 active+clean; 316 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 841 KiB/s wr, 16 op/s 2024-04-12T11:20:36.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[93650]: osdmap e432: 8 total, 8 up, 8 in 2024-04-12T11:20:36.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:36.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:36.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[93650]: Reconfiguring osd.1 (monmap changed)... 2024-04-12T11:20:36.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T11:20:36.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:36.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[93650]: Reconfiguring daemon osd.1 on smithi029 2024-04-12T11:20:36.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:36.584 INFO:tasks.rados.rados.0.smithi029.stdout:1484: writing smithi02979277-21 from 2300501 to 2887467 tid 3 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1484: writing smithi02979277-21 from 3376974 to 3731253 tid 4 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1485: write oid 44 current snap is 185 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1483: finishing write tid 3 to smithi02979277-7 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1483: finishing write tid 4 to smithi02979277-7 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1483: finishing write tid 5 to smithi02979277-7 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1483: oid 7 updating version 0 to 705 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1483: oid 7 updating version 705 to 707 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1483: oid 7 version 707 is already newer than 706 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1483: oid 7 version 707 is already newer than 704 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 707 (ObjNum 578 snap 185 seq_num 578) dirty exists 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1483: left oid 7 (ObjNum 578 snap 185 seq_num 578) 2024-04-12T11:20:36.590 INFO:tasks.rados.rados.0.smithi029.stdout:1484: finishing write tid 1 to smithi02979277-21 2024-04-12T11:20:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1484: finishing write tid 2 to smithi02979277-21 2024-04-12T11:20:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1484: finishing write tid 3 to smithi02979277-21 2024-04-12T11:20:36.591 INFO:tasks.rados.rados.0.smithi029.stdout:1485: seq_num 580 ranges {548060=487248,1440262=671538,2844277=798238} 2024-04-12T11:20:36.595 INFO:tasks.rados.rados.0.smithi029.stdout:1485: writing smithi02979277-44 from 548060 to 1035308 tid 1 2024-04-12T11:20:36.601 INFO:tasks.rados.rados.0.smithi029.stdout:1485: writing smithi02979277-44 from 1440262 to 2111800 tid 2 2024-04-12T11:20:36.610 INFO:tasks.rados.rados.0.smithi029.stdout:1485: writing smithi02979277-44 from 2844277 to 3642515 tid 3 2024-04-12T11:20:36.610 INFO:tasks.rados.rados.0.smithi029.stdout:1483: done (2 left) 2024-04-12T11:20:36.610 INFO:tasks.rados.rados.0.smithi029.stdout:1486: delete oid 48 current snap is 185 2024-04-12T11:20:36.610 INFO:tasks.rados.rados.0.smithi029.stdout:1484: finishing write tid 4 to smithi02979277-21 2024-04-12T11:20:36.610 INFO:tasks.rados.rados.0.smithi029.stdout:1484: finishing write tid 5 to smithi02979277-21 2024-04-12T11:20:36.610 INFO:tasks.rados.rados.0.smithi029.stdout:1484: finishing write tid 6 to smithi02979277-21 2024-04-12T11:20:36.610 INFO:tasks.rados.rados.0.smithi029.stdout:1484: oid 21 updating version 0 to 745 2024-04-12T11:20:36.610 INFO:tasks.rados.rados.0.smithi029.stdout:1484: oid 21 version 745 is already newer than 744 2024-04-12T11:20:36.610 INFO:tasks.rados.rados.0.smithi029.stdout:1484: oid 21 updating version 745 to 748 2024-04-12T11:20:36.610 INFO:tasks.rados.rados.0.smithi029.stdout:1484: oid 21 version 748 is already newer than 746 2024-04-12T11:20:36.611 INFO:tasks.rados.rados.0.smithi029.stdout:1484: oid 21 version 748 is already newer than 747 2024-04-12T11:20:36.611 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 748 (ObjNum 579 snap 185 seq_num 579) dirty exists 2024-04-12T11:20:36.611 INFO:tasks.rados.rados.0.smithi029.stdout:1484: left oid 21 (ObjNum 579 snap 185 seq_num 579) 2024-04-12T11:20:36.611 INFO:tasks.rados.rados.0.smithi029.stdout:1485: finishing write tid 1 to smithi02979277-44 2024-04-12T11:20:36.611 INFO:tasks.rados.rados.0.smithi029.stdout:1485: finishing write tid 2 to smithi02979277-44 2024-04-12T11:20:36.613 INFO:tasks.rados.rados.0.smithi029.stdout:1484: done (2 left) 2024-04-12T11:20:36.614 INFO:tasks.rados.rados.0.smithi029.stdout:1486: done (1 left) 2024-04-12T11:20:36.614 INFO:tasks.rados.rados.0.smithi029.stdout:1487: write oid 39 current snap is 185 2024-04-12T11:20:36.614 INFO:tasks.rados.rados.0.smithi029.stdout:1487: seq_num 582 ranges {581674=637746,1840873=558854,3066417=537135} 2024-04-12T11:20:36.620 INFO:tasks.rados.rados.0.smithi029.stdout:1487: writing smithi02979277-39 from 581674 to 1219420 tid 1 2024-04-12T11:20:36.626 INFO:tasks.rados.rados.0.smithi029.stdout:1487: writing smithi02979277-39 from 1840873 to 2399727 tid 2 2024-04-12T11:20:36.633 INFO:tasks.rados.rados.0.smithi029.stdout:1487: writing smithi02979277-39 from 3066417 to 3603552 tid 3 2024-04-12T11:20:36.633 INFO:tasks.rados.rados.0.smithi029.stdout:1488: snap_remove snap 141 2024-04-12T11:20:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:36 smithi087 ceph-mon[89014]: pgmap v283: 89 pgs: 89 active+clean; 316 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 841 KiB/s wr, 16 op/s 2024-04-12T11:20:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:36 smithi087 ceph-mon[89014]: osdmap e432: 8 total, 8 up, 8 in 2024-04-12T11:20:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:36 smithi087 ceph-mon[89014]: Reconfiguring osd.1 (monmap changed)... 2024-04-12T11:20:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T11:20:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:36 smithi087 ceph-mon[89014]: Reconfiguring daemon osd.1 on smithi029 2024-04-12T11:20:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:37.540 INFO:tasks.rados.rados.0.smithi029.stdout:1488: done (2 left) 2024-04-12T11:20:37.540 INFO:tasks.rados.rados.0.smithi029.stdout:1489: read oid 41 snap -1 2024-04-12T11:20:37.540 INFO:tasks.rados.rados.0.smithi029.stdout:1489: expect (ObjNum 540 snap 174 seq_num 540) 2024-04-12T11:20:37.540 INFO:tasks.rados.rados.0.smithi029.stdout:1490: write oid 6 current snap is 185 2024-04-12T11:20:37.540 INFO:tasks.rados.rados.0.smithi029.stdout:1490: seq_num 583 ranges {598481=712995,1899355=418865,3093840=704641} 2024-04-12T11:20:37.548 INFO:tasks.rados.rados.0.smithi029.stdout:1490: writing smithi02979277-6 from 598481 to 1311476 tid 1 2024-04-12T11:20:37.556 INFO:tasks.rados.rados.0.smithi029.stdout:1490: writing smithi02979277-6 from 1899355 to 2318220 tid 2 2024-04-12T11:20:37.567 INFO:tasks.rados.rados.0.smithi029.stdout:1490: writing smithi02979277-6 from 3093840 to 3798481 tid 3 2024-04-12T11:20:37.567 INFO:tasks.rados.rados.0.smithi029.stdout:1491: snap_create 2024-04-12T11:20:37.567 INFO:tasks.rados.rados.0.smithi029.stdout:1485: finishing write tid 3 to smithi02979277-44 2024-04-12T11:20:37.567 INFO:tasks.rados.rados.0.smithi029.stdout:1485: finishing write tid 4 to smithi02979277-44 2024-04-12T11:20:37.567 INFO:tasks.rados.rados.0.smithi029.stdout:1485: finishing write tid 5 to smithi02979277-44 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1485: oid 44 updating version 0 to 752 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1485: oid 44 version 752 is already newer than 751 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1485: oid 44 version 752 is already newer than 749 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1485: oid 44 version 752 is already newer than 750 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 752 (ObjNum 580 snap 185 seq_num 580) dirty exists 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1485: left oid 44 (ObjNum 580 snap 185 seq_num 580) 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1487: finishing write tid 1 to smithi02979277-39 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1487: finishing write tid 2 to smithi02979277-39 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1487: finishing write tid 3 to smithi02979277-39 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1487: finishing write tid 4 to smithi02979277-39 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1487: finishing write tid 5 to smithi02979277-39 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1487: oid 39 updating version 0 to 163 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1487: oid 39 version 163 is already newer than 160 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1487: oid 39 version 163 is already newer than 161 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:1487: oid 39 version 163 is already newer than 162 2024-04-12T11:20:37.568 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 163 (ObjNum 582 snap 185 seq_num 582) dirty exists 2024-04-12T11:20:37.569 INFO:tasks.rados.rados.0.smithi029.stdout:1487: left oid 39 (ObjNum 582 snap 185 seq_num 582) 2024-04-12T11:20:37.569 INFO:tasks.rados.rados.0.smithi029.stdout:1490: finishing write tid 1 to smithi02979277-6 2024-04-12T11:20:37.569 INFO:tasks.rados.rados.0.smithi029.stdout:1490: finishing write tid 2 to smithi02979277-6 2024-04-12T11:20:37.569 INFO:tasks.rados.rados.0.smithi029.stdout:1489: expect (ObjNum 540 snap 174 seq_num 540) 2024-04-12T11:20:37.714 INFO:tasks.rados.rados.0.smithi029.stdout:1490: finishing write tid 3 to smithi02979277-6 2024-04-12T11:20:37.714 INFO:tasks.rados.rados.0.smithi029.stdout:1490: finishing write tid 4 to smithi02979277-6 2024-04-12T11:20:37.714 INFO:tasks.rados.rados.0.smithi029.stdout:1490: finishing write tid 5 to smithi02979277-6 2024-04-12T11:20:37.714 INFO:tasks.rados.rados.0.smithi029.stdout:1490: oid 6 updating version 0 to 351 2024-04-12T11:20:37.714 INFO:tasks.rados.rados.0.smithi029.stdout:1490: oid 6 updating version 351 to 352 2024-04-12T11:20:37.714 INFO:tasks.rados.rados.0.smithi029.stdout:1490: oid 6 updating version 352 to 354 2024-04-12T11:20:37.714 INFO:tasks.rados.rados.0.smithi029.stdout:1490: oid 6 version 354 is already newer than 353 2024-04-12T11:20:37.714 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 354 (ObjNum 583 snap 185 seq_num 583) dirty exists 2024-04-12T11:20:37.714 INFO:tasks.rados.rados.0.smithi029.stdout:1490: left oid 6 (ObjNum 583 snap 185 seq_num 583) 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[93650]: osdmap e433: 8 total, 8 up, 8 in 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[93650]: osdmap e434: 8 total, 8 up, 8 in 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[97787]: osdmap e433: 8 total, 8 up, 8 in 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:37 smithi029 ceph-mon[97787]: osdmap e434: 8 total, 8 up, 8 in 2024-04-12T11:20:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:37 smithi087 ceph-mon[89014]: osdmap e433: 8 total, 8 up, 8 in 2024-04-12T11:20:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T11:20:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:37 smithi087 ceph-mon[89014]: osdmap e434: 8 total, 8 up, 8 in 2024-04-12T11:20:38.566 INFO:tasks.rados.rados.0.smithi029.stdout:1485: done (4 left) 2024-04-12T11:20:38.566 INFO:tasks.rados.rados.0.smithi029.stdout:1487: done (3 left) 2024-04-12T11:20:38.566 INFO:tasks.rados.rados.0.smithi029.stdout:1489: done (2 left) 2024-04-12T11:20:38.566 INFO:tasks.rados.rados.0.smithi029.stdout:1490: done (1 left) 2024-04-12T11:20:38.566 INFO:tasks.rados.rados.0.smithi029.stdout:1491: done (0 left) 2024-04-12T11:20:38.566 INFO:tasks.rados.rados.0.smithi029.stdout:1492: snap_create 2024-04-12T11:20:38.737 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:38 smithi029 ceph-mon[97787]: pgmap v286: 89 pgs: 89 active+clean; 329 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.0 MiB/s wr, 16 op/s 2024-04-12T11:20:38.737 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:38 smithi029 ceph-mon[97787]: Reconfiguring osd.2 (monmap changed)... 2024-04-12T11:20:38.737 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:38 smithi029 ceph-mon[97787]: Reconfiguring daemon osd.2 on smithi029 2024-04-12T11:20:38.737 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:38 smithi029 ceph-mon[93650]: pgmap v286: 89 pgs: 89 active+clean; 329 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.0 MiB/s wr, 16 op/s 2024-04-12T11:20:38.737 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:38 smithi029 ceph-mon[93650]: Reconfiguring osd.2 (monmap changed)... 2024-04-12T11:20:38.738 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:38 smithi029 ceph-mon[93650]: Reconfiguring daemon osd.2 on smithi029 2024-04-12T11:20:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:38 smithi087 ceph-mon[89014]: pgmap v286: 89 pgs: 89 active+clean; 329 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.0 MiB/s wr, 16 op/s 2024-04-12T11:20:38.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:38 smithi087 ceph-mon[89014]: Reconfiguring osd.2 (monmap changed)... 2024-04-12T11:20:38.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:38 smithi087 ceph-mon[89014]: Reconfiguring daemon osd.2 on smithi029 2024-04-12T11:20:39.571 INFO:tasks.rados.rados.0.smithi029.stdout:1492: done (0 left) 2024-04-12T11:20:39.571 INFO:tasks.rados.rados.0.smithi029.stdout:1493: snap_create 2024-04-12T11:20:39.826 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:39 smithi029 ceph-mon[97787]: osdmap e435: 8 total, 8 up, 8 in 2024-04-12T11:20:39.826 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:39.826 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:39.826 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T11:20:39.827 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:39.827 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:39 smithi029 ceph-mon[93650]: osdmap e435: 8 total, 8 up, 8 in 2024-04-12T11:20:39.827 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:39.827 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:39.827 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T11:20:39.827 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:39 smithi087 ceph-mon[89014]: osdmap e435: 8 total, 8 up, 8 in 2024-04-12T11:20:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T11:20:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:40.573 INFO:tasks.rados.rados.0.smithi029.stdout:1493: done (0 left) 2024-04-12T11:20:40.573 INFO:tasks.rados.rados.0.smithi029.stdout:1494: snap_remove snap 186 2024-04-12T11:20:40.863 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:40 smithi029 ceph-mon[97787]: pgmap v289: 89 pgs: 89 active+clean; 329 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.0 MiB/s wr, 16 op/s 2024-04-12T11:20:40.863 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:40 smithi029 ceph-mon[97787]: Reconfiguring osd.3 (monmap changed)... 2024-04-12T11:20:40.863 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:40 smithi029 ceph-mon[97787]: Reconfiguring daemon osd.3 on smithi029 2024-04-12T11:20:40.863 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:40 smithi029 ceph-mon[97787]: osdmap e436: 8 total, 8 up, 8 in 2024-04-12T11:20:40.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:40 smithi029 ceph-mon[93650]: pgmap v289: 89 pgs: 89 active+clean; 329 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.0 MiB/s wr, 16 op/s 2024-04-12T11:20:40.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:40 smithi029 ceph-mon[93650]: Reconfiguring osd.3 (monmap changed)... 2024-04-12T11:20:40.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:40 smithi029 ceph-mon[93650]: Reconfiguring daemon osd.3 on smithi029 2024-04-12T11:20:40.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:40 smithi029 ceph-mon[93650]: osdmap e436: 8 total, 8 up, 8 in 2024-04-12T11:20:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:40 smithi087 ceph-mon[89014]: pgmap v289: 89 pgs: 89 active+clean; 329 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 3.0 MiB/s wr, 16 op/s 2024-04-12T11:20:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:40 smithi087 ceph-mon[89014]: Reconfiguring osd.3 (monmap changed)... 2024-04-12T11:20:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:40 smithi087 ceph-mon[89014]: Reconfiguring daemon osd.3 on smithi029 2024-04-12T11:20:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:40 smithi087 ceph-mon[89014]: osdmap e436: 8 total, 8 up, 8 in 2024-04-12T11:20:41.573 INFO:tasks.rados.rados.0.smithi029.stdout:1494: done (0 left) 2024-04-12T11:20:41.573 INFO:tasks.rados.rados.0.smithi029.stdout:1495: snap_create 2024-04-12T11:20:41.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[97787]: osdmap e437: 8 total, 8 up, 8 in 2024-04-12T11:20:41.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:41.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:41.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.scbuux", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:20:41.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:41.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[97787]: osdmap e438: 8 total, 8 up, 8 in 2024-04-12T11:20:41.856 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[93650]: osdmap e437: 8 total, 8 up, 8 in 2024-04-12T11:20:41.856 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:41.856 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:41.856 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.scbuux", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:20:41.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:41.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:41 smithi029 ceph-mon[93650]: osdmap e438: 8 total, 8 up, 8 in 2024-04-12T11:20:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:41 smithi087 ceph-mon[89014]: osdmap e437: 8 total, 8 up, 8 in 2024-04-12T11:20:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.scbuux", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:20:41.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:41.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:41 smithi087 ceph-mon[89014]: osdmap e438: 8 total, 8 up, 8 in 2024-04-12T11:20:42.540 INFO:tasks.rados.rados.0.smithi029.stdout:1495: done (0 left) 2024-04-12T11:20:42.541 INFO:tasks.rados.rados.0.smithi029.stdout:1496: snap_remove snap 187 2024-04-12T11:20:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[93650]: Reconfiguring mds.foo.smithi029.scbuux (monmap changed)... 2024-04-12T11:20:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[93650]: Reconfiguring daemon mds.foo.smithi029.scbuux on smithi029 2024-04-12T11:20:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[93650]: pgmap v292: 89 pgs: 89 active+clean; 329 MiB data, 904 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[93650]: osdmap e439: 8 total, 8 up, 8 in 2024-04-12T11:20:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[97787]: Reconfiguring mds.foo.smithi029.scbuux (monmap changed)... 2024-04-12T11:20:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[97787]: Reconfiguring daemon mds.foo.smithi029.scbuux on smithi029 2024-04-12T11:20:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[97787]: pgmap v292: 89 pgs: 89 active+clean; 329 MiB data, 904 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[97787]: osdmap e439: 8 total, 8 up, 8 in 2024-04-12T11:20:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:43.310 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:42 smithi087 ceph-mon[89014]: Reconfiguring mds.foo.smithi029.scbuux (monmap changed)... 2024-04-12T11:20:43.311 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:42 smithi087 ceph-mon[89014]: Reconfiguring daemon mds.foo.smithi029.scbuux on smithi029 2024-04-12T11:20:43.311 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:42 smithi087 ceph-mon[89014]: pgmap v292: 89 pgs: 89 active+clean; 329 MiB data, 904 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:43.311 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:42 smithi087 ceph-mon[89014]: osdmap e439: 8 total, 8 up, 8 in 2024-04-12T11:20:43.311 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:43.311 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:43.311 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T11:20:43.311 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T11:20:43.311 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:43.542 INFO:tasks.rados.rados.0.smithi029.stdout:1496: done (0 left) 2024-04-12T11:20:43.542 INFO:tasks.rados.rados.0.smithi029.stdout:1497: rollback oid 50 current snap is 189 2024-04-12T11:20:43.542 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 50 to 163 2024-04-12T11:20:43.542 INFO:tasks.rados.rados.0.smithi029.stdout:1498: write oid 24 current snap is 189 2024-04-12T11:20:43.543 INFO:tasks.rados.rados.0.smithi029.stdout:1498: seq_num 584 ranges {615288=788244,1957837=562523,3121263=559679} 2024-04-12T11:20:43.550 INFO:tasks.rados.rados.0.smithi029.stdout:1498: writing smithi02979277-24 from 615288 to 1403532 tid 1 2024-04-12T11:20:43.556 INFO:tasks.rados.rados.0.smithi029.stdout:1498: writing smithi02979277-24 from 1957837 to 2520360 tid 2 2024-04-12T11:20:43.562 INFO:tasks.rados.rados.0.smithi029.stdout:1498: writing smithi02979277-24 from 3121263 to 3680942 tid 3 2024-04-12T11:20:43.563 INFO:tasks.rados.rados.0.smithi029.stdout:1499: write oid 49 current snap is 189 2024-04-12T11:20:43.563 INFO:tasks.rados.rados.0.smithi029.stdout:1499: seq_num 585 ranges {632095=463493,1616319=422534,2748686=487304} 2024-04-12T11:20:43.568 INFO:tasks.rados.rados.0.smithi029.stdout:1499: writing smithi02979277-49 from 632095 to 1095588 tid 1 2024-04-12T11:20:43.574 INFO:tasks.rados.rados.0.smithi029.stdout:1499: writing smithi02979277-49 from 1616319 to 2038853 tid 2 2024-04-12T11:20:43.580 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:43 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:43.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:43.581 INFO:tasks.rados.rados.0.smithi029.stdout:1499: writing smithi02979277-49 from 2748686 to 3235990 tid 3 2024-04-12T11:20:43.581 INFO:tasks.rados.rados.0.smithi029.stdout:1500: snap_remove snap 129 2024-04-12T11:20:44.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:43 smithi029 ceph-mon[97787]: Reconfiguring mon.b (monmap changed)... 2024-04-12T11:20:44.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:43 smithi029 ceph-mon[97787]: Reconfiguring daemon mon.b on smithi087 2024-04-12T11:20:44.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:43 smithi029 ceph-mon[97787]: osdmap e440: 8 total, 8 up, 8 in 2024-04-12T11:20:44.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:43 smithi029 ceph-mon[93650]: Reconfiguring mon.b (monmap changed)... 2024-04-12T11:20:44.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:43 smithi029 ceph-mon[93650]: Reconfiguring daemon mon.b on smithi087 2024-04-12T11:20:44.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:43 smithi029 ceph-mon[93650]: osdmap e440: 8 total, 8 up, 8 in 2024-04-12T11:20:44.381 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:43 smithi087 ceph-mon[89014]: Reconfiguring mon.b (monmap changed)... 2024-04-12T11:20:44.381 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:43 smithi087 ceph-mon[89014]: Reconfiguring daemon mon.b on smithi087 2024-04-12T11:20:44.381 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:43 smithi087 ceph-mon[89014]: osdmap e440: 8 total, 8 up, 8 in 2024-04-12T11:20:44.543 INFO:tasks.rados.rados.0.smithi029.stdout:1500: done (3 left) 2024-04-12T11:20:44.543 INFO:tasks.rados.rados.0.smithi029.stdout:1501: delete oid 4 current snap is 189 2024-04-12T11:20:44.543 INFO:tasks.rados.rados.0.smithi029.stdout:1497: finishing rollback tid 0 to smithi02979277-50 2024-04-12T11:20:44.543 INFO:tasks.rados.rados.0.smithi029.stdout:1497: finishing rollback tid 1 to smithi02979277-50 2024-04-12T11:20:44.543 INFO:tasks.rados.rados.0.smithi029.stdout:1497: finishing rollback tid 2 to smithi02979277-50 2024-04-12T11:20:44.543 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 499 (ObjNum 474 snap 156 seq_num 474) dirty exists 2024-04-12T11:20:44.543 INFO:tasks.rados.rados.0.smithi029.stdout:1498: finishing write tid 1 to smithi02979277-24 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1498: finishing write tid 2 to smithi02979277-24 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1498: finishing write tid 3 to smithi02979277-24 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1498: finishing write tid 4 to smithi02979277-24 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1498: finishing write tid 5 to smithi02979277-24 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1498: oid 24 updating version 0 to 954 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1498: oid 24 version 954 is already newer than 952 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1498: oid 24 version 954 is already newer than 953 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1498: oid 24 version 954 is already newer than 951 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 954 (ObjNum 584 snap 189 seq_num 584) dirty exists 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1498: left oid 24 (ObjNum 584 snap 189 seq_num 584) 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1499: finishing write tid 1 to smithi02979277-49 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1499: finishing write tid 2 to smithi02979277-49 2024-04-12T11:20:44.544 INFO:tasks.rados.rados.0.smithi029.stdout:1499: finishing write tid 3 to smithi02979277-49 2024-04-12T11:20:44.545 INFO:tasks.rados.rados.0.smithi029.stdout:1499: finishing write tid 4 to smithi02979277-49 2024-04-12T11:20:44.545 INFO:tasks.rados.rados.0.smithi029.stdout:1499: finishing write tid 5 to smithi02979277-49 2024-04-12T11:20:44.545 INFO:tasks.rados.rados.0.smithi029.stdout:1499: oid 49 updating version 0 to 559 2024-04-12T11:20:44.545 INFO:tasks.rados.rados.0.smithi029.stdout:1499: oid 49 version 559 is already newer than 558 2024-04-12T11:20:44.545 INFO:tasks.rados.rados.0.smithi029.stdout:1499: oid 49 version 559 is already newer than 557 2024-04-12T11:20:44.545 INFO:tasks.rados.rados.0.smithi029.stdout:1499: oid 49 version 559 is already newer than 556 2024-04-12T11:20:44.545 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 559 (ObjNum 585 snap 189 seq_num 585) dirty exists 2024-04-12T11:20:44.545 INFO:tasks.rados.rados.0.smithi029.stdout:1499: left oid 49 (ObjNum 585 snap 189 seq_num 585) 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1497: done (3 left) 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1498: done (2 left) 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1499: done (1 left) 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1501: done (0 left) 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1502: read oid 1 snap -1 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1502: expect (ObjNum 509 snap 164 seq_num 509) 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1503: read oid 47 snap -1 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1503: expect (ObjNum 415 snap 130 seq_num 415) 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1504: read oid 16 snap -1 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1504: expect (ObjNum 444 snap 144 seq_num 444) 2024-04-12T11:20:44.554 INFO:tasks.rados.rados.0.smithi029.stdout:1505: snap_remove snap 180 2024-04-12T11:20:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:20:44] "GET /metrics HTTP/1.1" 200 36251 "" "Prometheus/2.43.0" 2024-04-12T11:20:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[97787]: pgmap v295: 89 pgs: 89 active+clean; 304 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 449 KiB/s wr, 3 op/s 2024-04-12T11:20:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[97787]: osdmap e441: 8 total, 8 up, 8 in 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[93650]: pgmap v295: 89 pgs: 89 active+clean; 304 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 449 KiB/s wr, 3 op/s 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[93650]: osdmap e441: 8 total, 8 up, 8 in 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:20:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:45.312 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:44 smithi087 ceph-mon[89014]: pgmap v295: 89 pgs: 89 active+clean; 304 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 449 KiB/s wr, 3 op/s 2024-04-12T11:20:45.312 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:45.313 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:45.313 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:44 smithi087 ceph-mon[89014]: osdmap e441: 8 total, 8 up, 8 in 2024-04-12T11:20:45.313 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T11:20:45.313 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T11:20:45.313 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:45.546 INFO:tasks.rados.rados.0.smithi029.stdout:1505: done (3 left) 2024-04-12T11:20:45.546 INFO:tasks.rados.rados.0.smithi029.stdout:1506: write oid 50 current snap is 189 2024-04-12T11:20:45.546 INFO:tasks.rados.rados.0.smithi029.stdout:1506: seq_num 587 ranges {665709=730344,2133283=426203,3203532=509848} 2024-04-12T11:20:45.554 INFO:tasks.rados.rados.0.smithi029.stdout:1506: writing smithi02979277-50 from 665709 to 1396053 tid 1 2024-04-12T11:20:45.560 INFO:tasks.rados.rados.0.smithi029.stdout:1506: writing smithi02979277-50 from 2133283 to 2559486 tid 2 2024-04-12T11:20:45.567 INFO:tasks.rados.rados.0.smithi029.stdout:1506: writing smithi02979277-50 from 3203532 to 3713380 tid 3 2024-04-12T11:20:45.567 INFO:tasks.rados.rados.0.smithi029.stdout:1507: delete oid 42 current snap is 189 2024-04-12T11:20:45.567 INFO:tasks.rados.rados.0.smithi029.stdout:1503: expect (ObjNum 415 snap 130 seq_num 415) 2024-04-12T11:20:45.783 INFO:tasks.rados.rados.0.smithi029.stdout:1502: expect (ObjNum 509 snap 164 seq_num 509) 2024-04-12T11:20:45.895 INFO:tasks.rados.rados.0.smithi029.stdout:1504: expect (ObjNum 444 snap 144 seq_num 444) 2024-04-12T11:20:46.132 INFO:tasks.rados.rados.0.smithi029.stdout:1506: finishing write tid 1 to smithi02979277-50 2024-04-12T11:20:46.132 INFO:tasks.rados.rados.0.smithi029.stdout:1506: finishing write tid 2 to smithi02979277-50 2024-04-12T11:20:46.132 INFO:tasks.rados.rados.0.smithi029.stdout:1506: finishing write tid 3 to smithi02979277-50 2024-04-12T11:20:46.132 INFO:tasks.rados.rados.0.smithi029.stdout:1506: finishing write tid 4 to smithi02979277-50 2024-04-12T11:20:46.132 INFO:tasks.rados.rados.0.smithi029.stdout:1506: finishing write tid 5 to smithi02979277-50 2024-04-12T11:20:46.132 INFO:tasks.rados.rados.0.smithi029.stdout:1506: oid 50 updating version 0 to 507 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1506: oid 50 version 507 is already newer than 505 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1506: oid 50 version 507 is already newer than 502 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1506: oid 50 version 507 is already newer than 506 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 507 (ObjNum 587 snap 189 seq_num 587) dirty exists 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1506: left oid 50 (ObjNum 587 snap 189 seq_num 587) 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1502: done (4 left) 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1503: done (3 left) 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1504: done (2 left) 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1506: done (1 left) 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1507: done (0 left) 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1508: read oid 34 snap -1 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1508: expect (ObjNum 561 snap 177 seq_num 561) 2024-04-12T11:20:46.133 INFO:tasks.rados.rados.0.smithi029.stdout:1509: snap_create 2024-04-12T11:20:46.137 INFO:tasks.rados.rados.0.smithi029.stdout:1508: expect (ObjNum 561 snap 177 seq_num 561) 2024-04-12T11:20:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:45 smithi087 ceph-mon[89014]: Reconfiguring mgr.x (monmap changed)... 2024-04-12T11:20:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:45 smithi087 ceph-mon[89014]: Reconfiguring daemon mgr.x on smithi087 2024-04-12T11:20:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:45 smithi087 ceph-mon[89014]: osdmap e442: 8 total, 8 up, 8 in 2024-04-12T11:20:46.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:45 smithi029 ceph-mon[93650]: Reconfiguring mgr.x (monmap changed)... 2024-04-12T11:20:46.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:45 smithi029 ceph-mon[93650]: Reconfiguring daemon mgr.x on smithi087 2024-04-12T11:20:46.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:45 smithi029 ceph-mon[93650]: osdmap e442: 8 total, 8 up, 8 in 2024-04-12T11:20:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:45 smithi029 ceph-mon[97787]: Reconfiguring mgr.x (monmap changed)... 2024-04-12T11:20:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:45 smithi029 ceph-mon[97787]: Reconfiguring daemon mgr.x on smithi087 2024-04-12T11:20:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:45 smithi029 ceph-mon[97787]: osdmap e442: 8 total, 8 up, 8 in 2024-04-12T11:20:47.029 INFO:tasks.rados.rados.0.smithi029.stdout:1508: done (1 left) 2024-04-12T11:20:47.029 INFO:tasks.rados.rados.0.smithi029.stdout:1509: done (0 left) 2024-04-12T11:20:47.029 INFO:tasks.rados.rados.0.smithi029.stdout:1510: delete oid 15 current snap is 190 2024-04-12T11:20:47.034 INFO:tasks.rados.rados.0.smithi029.stdout:1510: done (0 left) 2024-04-12T11:20:47.034 INFO:tasks.rados.rados.0.smithi029.stdout:1511: read oid 24 snap -1 2024-04-12T11:20:47.034 INFO:tasks.rados.rados.0.smithi029.stdout:1511: expect (ObjNum 584 snap 189 seq_num 584) 2024-04-12T11:20:47.034 INFO:tasks.rados.rados.0.smithi029.stdout:1512: read oid 34 snap 157 2024-04-12T11:20:47.034 INFO:tasks.rados.rados.0.smithi029.stdout:1512: expect (ObjNum 310 snap 94 seq_num 310) 2024-04-12T11:20:47.034 INFO:tasks.rados.rados.0.smithi029.stdout:1513: snap_remove snap 182 2024-04-12T11:20:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:46 smithi087 ceph-mon[89014]: pgmap v298: 89 pgs: 89 active+clean; 304 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 449 KiB/s wr, 3 op/s 2024-04-12T11:20:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:46 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:46 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:46 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T11:20:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:46 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:46 smithi029 ceph-mon[93650]: pgmap v298: 89 pgs: 89 active+clean; 304 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 449 KiB/s wr, 3 op/s 2024-04-12T11:20:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:46 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:46 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:46 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T11:20:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:46 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:46 smithi029 ceph-mon[97787]: pgmap v298: 89 pgs: 89 active+clean; 304 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 449 KiB/s wr, 3 op/s 2024-04-12T11:20:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:46 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:46 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:47.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:46 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T11:20:47.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:46 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:48.033 INFO:tasks.rados.rados.0.smithi029.stdout:1513: done (2 left) 2024-04-12T11:20:48.033 INFO:tasks.rados.rados.0.smithi029.stdout:1514: write oid 29 current snap is 190 2024-04-12T11:20:48.033 INFO:tasks.rados.rados.0.smithi029.stdout:1514: seq_num 590 ranges {716130=556091,1792376=573530,2769448=460017,3821956=94174} 2024-04-12T11:20:48.040 INFO:tasks.rados.rados.0.smithi029.stdout:1514: writing smithi02979277-29 from 716130 to 1272221 tid 1 2024-04-12T11:20:48.047 INFO:tasks.rados.rados.0.smithi029.stdout:1514: writing smithi02979277-29 from 1792376 to 2365906 tid 2 2024-04-12T11:20:48.052 INFO:tasks.rados.rados.0.smithi029.stdout:1514: writing smithi02979277-29 from 2769448 to 3229465 tid 3 2024-04-12T11:20:48.057 INFO:tasks.rados.rados.0.smithi029.stdout:1514: writing smithi02979277-29 from 3821956 to 3916130 tid 4 2024-04-12T11:20:48.057 INFO:tasks.rados.rados.0.smithi029.stdout:1515: read oid 31 snap -1 2024-04-12T11:20:48.057 INFO:tasks.rados.rados.0.smithi029.stdout:1515: expect deleted 2024-04-12T11:20:48.057 INFO:tasks.rados.rados.0.smithi029.stdout:1516: snap_remove snap 184 2024-04-12T11:20:48.057 INFO:tasks.rados.rados.0.smithi029.stdout:1512: expect (ObjNum 310 snap 94 seq_num 310) 2024-04-12T11:20:48.234 INFO:tasks.rados.rados.0.smithi029.stdout:1511: expect (ObjNum 584 snap 189 seq_num 584) 2024-04-12T11:20:48.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[93650]: Reconfiguring osd.4 (monmap changed)... 2024-04-12T11:20:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[93650]: Reconfiguring daemon osd.4 on smithi087 2024-04-12T11:20:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[93650]: osdmap e443: 8 total, 8 up, 8 in 2024-04-12T11:20:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T11:20:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[97787]: Reconfiguring osd.4 (monmap changed)... 2024-04-12T11:20:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[97787]: Reconfiguring daemon osd.4 on smithi087 2024-04-12T11:20:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[97787]: osdmap e443: 8 total, 8 up, 8 in 2024-04-12T11:20:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:48.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:48.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T11:20:48.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:48.280 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:47 smithi087 ceph-mon[89014]: Reconfiguring osd.4 (monmap changed)... 2024-04-12T11:20:48.280 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:47 smithi087 ceph-mon[89014]: Reconfiguring daemon osd.4 on smithi087 2024-04-12T11:20:48.280 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:47 smithi087 ceph-mon[89014]: osdmap e443: 8 total, 8 up, 8 in 2024-04-12T11:20:48.280 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:48.280 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:48.280 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T11:20:48.281 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:48.444 INFO:tasks.rados.rados.0.smithi029.stdout:1514: finishing write tid 1 to smithi02979277-29 2024-04-12T11:20:48.445 INFO:tasks.rados.rados.0.smithi029.stdout:1514: finishing write tid 2 to smithi02979277-29 2024-04-12T11:20:48.445 INFO:tasks.rados.rados.0.smithi029.stdout:1514: finishing write tid 3 to smithi02979277-29 2024-04-12T11:20:48.445 INFO:tasks.rados.rados.0.smithi029.stdout:1514: finishing write tid 4 to smithi02979277-29 2024-04-12T11:20:49.036 INFO:tasks.rados.rados.0.smithi029.stdout:1511: done (4 left) 2024-04-12T11:20:49.036 INFO:tasks.rados.rados.0.smithi029.stdout:1512: done (3 left) 2024-04-12T11:20:49.037 INFO:tasks.rados.rados.0.smithi029.stdout:1515: done (2 left) 2024-04-12T11:20:49.037 INFO:tasks.rados.rados.0.smithi029.stdout:1516: done (1 left) 2024-04-12T11:20:49.037 INFO:tasks.rados.rados.0.smithi029.stdout:1517: write oid 38 current snap is 190 2024-04-12T11:20:49.037 INFO:tasks.rados.rados.0.smithi029.stdout:1517: seq_num 591 ranges {732937=631340,1850858=433541,2796871=671289,3888195=44742} 2024-04-12T11:20:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:1517: writing smithi02979277-38 from 732937 to 1364277 tid 1 2024-04-12T11:20:49.050 INFO:tasks.rados.rados.0.smithi029.stdout:1517: writing smithi02979277-38 from 1850858 to 2284399 tid 2 2024-04-12T11:20:49.056 INFO:tasks.rados.rados.0.smithi029.stdout:1517: writing smithi02979277-38 from 2796871 to 3468160 tid 3 2024-04-12T11:20:49.059 INFO:tasks.rados.rados.0.smithi029.stdout:1517: writing smithi02979277-38 from 3888195 to 3932937 tid 4 2024-04-12T11:20:49.059 INFO:tasks.rados.rados.0.smithi029.stdout:1518: read oid 3 snap -1 2024-04-12T11:20:49.059 INFO:tasks.rados.rados.0.smithi029.stdout:1518: expect (ObjNum 562 snap 177 seq_num 562) 2024-04-12T11:20:49.059 INFO:tasks.rados.rados.0.smithi029.stdout:1519: snap_remove snap 157 2024-04-12T11:20:49.224 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:48 smithi087 ceph-mon[89014]: pgmap v301: 89 pgs: 89 active+clean; 257 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 18 op/s 2024-04-12T11:20:49.224 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:48 smithi087 ceph-mon[89014]: Reconfiguring osd.5 (monmap changed)... 2024-04-12T11:20:49.224 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:48 smithi087 ceph-mon[89014]: Reconfiguring daemon osd.5 on smithi087 2024-04-12T11:20:49.224 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:48 smithi087 ceph-mon[89014]: osdmap e444: 8 total, 8 up, 8 in 2024-04-12T11:20:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:48 smithi029 ceph-mon[93650]: pgmap v301: 89 pgs: 89 active+clean; 257 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 18 op/s 2024-04-12T11:20:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:48 smithi029 ceph-mon[93650]: Reconfiguring osd.5 (monmap changed)... 2024-04-12T11:20:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:48 smithi029 ceph-mon[93650]: Reconfiguring daemon osd.5 on smithi087 2024-04-12T11:20:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:48 smithi029 ceph-mon[93650]: osdmap e444: 8 total, 8 up, 8 in 2024-04-12T11:20:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:48 smithi029 ceph-mon[97787]: pgmap v301: 89 pgs: 89 active+clean; 257 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 18 op/s 2024-04-12T11:20:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:48 smithi029 ceph-mon[97787]: Reconfiguring osd.5 (monmap changed)... 2024-04-12T11:20:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:48 smithi029 ceph-mon[97787]: Reconfiguring daemon osd.5 on smithi087 2024-04-12T11:20:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:48 smithi029 ceph-mon[97787]: osdmap e444: 8 total, 8 up, 8 in 2024-04-12T11:20:50.043 INFO:tasks.rados.rados.0.smithi029.stdout:1519: done (3 left) 2024-04-12T11:20:50.043 INFO:tasks.rados.rados.0.smithi029.stdout:1520: rollback oid 27 current snap is 190 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 171 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:1514: finishing write tid 5 to smithi02979277-29 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:1514: finishing write tid 6 to smithi02979277-29 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:1514: oid 29 updating version 0 to 588 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:1514: oid 29 updating version 588 to 589 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:1514: oid 29 version 589 is already newer than 587 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:1514: oid 29 updating version 589 to 590 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:1514: oid 29 version 590 is already newer than 586 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 590 (ObjNum 590 snap 190 seq_num 590) dirty exists 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:1514: left oid 29 (ObjNum 590 snap 190 seq_num 590) 2024-04-12T11:20:50.044 INFO:tasks.rados.rados.0.smithi029.stdout:1517: finishing write tid 1 to smithi02979277-38 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: finishing write tid 2 to smithi02979277-38 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: finishing write tid 3 to smithi02979277-38 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: finishing write tid 4 to smithi02979277-38 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: finishing write tid 5 to smithi02979277-38 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: finishing write tid 6 to smithi02979277-38 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: oid 38 updating version 0 to 410 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: oid 38 updating version 410 to 412 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: oid 38 updating version 412 to 413 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: oid 38 updating version 413 to 414 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: oid 38 version 414 is already newer than 411 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 414 (ObjNum 591 snap 190 seq_num 591) dirty exists 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1517: left oid 38 (ObjNum 591 snap 190 seq_num 591) 2024-04-12T11:20:50.045 INFO:tasks.rados.rados.0.smithi029.stdout:1518: expect (ObjNum 562 snap 177 seq_num 562) 2024-04-12T11:20:50.162 INFO:tasks.rados.rados.0.smithi029.stdout:1520: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:20:50.162 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 581 (ObjNum 177 snap 0 seq_num 1801073856) dirty dne 2024-04-12T11:20:50.162 INFO:tasks.rados.rados.0.smithi029.stdout:1514: done (3 left) 2024-04-12T11:20:50.162 INFO:tasks.rados.rados.0.smithi029.stdout:1517: done (2 left) 2024-04-12T11:20:50.162 INFO:tasks.rados.rados.0.smithi029.stdout:1518: done (1 left) 2024-04-12T11:20:50.162 INFO:tasks.rados.rados.0.smithi029.stdout:1520: done (0 left) 2024-04-12T11:20:50.162 INFO:tasks.rados.rados.0.smithi029.stdout:1521: write oid 9 current snap is 190 2024-04-12T11:20:50.163 INFO:tasks.rados.rados.0.smithi029.stdout:1521: seq_num 592 ranges {749744=706589,1909340=577199,3224294=482561} 2024-04-12T11:20:50.169 INFO:tasks.rados.rados.0.smithi029.stdout:1521: writing smithi02979277-9 from 749744 to 1456333 tid 1 2024-04-12T11:20:50.175 INFO:tasks.rados.rados.0.smithi029.stdout:1521: writing smithi02979277-9 from 1909340 to 2486539 tid 2 2024-04-12T11:20:50.182 INFO:tasks.rados.rados.0.smithi029.stdout:1521: writing smithi02979277-9 from 3224294 to 3706855 tid 3 2024-04-12T11:20:50.182 INFO:tasks.rados.rados.0.smithi029.stdout:1522: write oid 37 current snap is 190 2024-04-12T11:20:50.183 INFO:tasks.rados.rados.0.smithi029.stdout:1522: seq_num 593 ranges {766551=498191,1684175=720857,2851717=693833} 2024-04-12T11:20:50.189 INFO:tasks.rados.rados.0.smithi029.stdout:1522: writing smithi02979277-37 from 766551 to 1264742 tid 1 2024-04-12T11:20:50.195 INFO:tasks.rados.rados.0.smithi029.stdout:1522: writing smithi02979277-37 from 1684175 to 2405032 tid 2 2024-04-12T11:20:50.202 INFO:tasks.rados.rados.0.smithi029.stdout:1522: writing smithi02979277-37 from 2851717 to 3545550 tid 3 2024-04-12T11:20:50.202 INFO:tasks.rados.rados.0.smithi029.stdout:1523: write oid 30 current snap is 190 2024-04-12T11:20:50.202 INFO:tasks.rados.rados.0.smithi029.stdout:1523: seq_num 594 ranges {783358=573440,2026304=580868,3279140=621458} 2024-04-12T11:20:50.209 INFO:tasks.rados.rados.0.smithi029.stdout:1523: writing smithi02979277-30 from 783358 to 1356798 tid 1 2024-04-12T11:20:50.216 INFO:tasks.rados.rados.0.smithi029.stdout:1523: writing smithi02979277-30 from 2026304 to 2607172 tid 2 2024-04-12T11:20:50.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:50 smithi087 ceph-mon[89014]: osdmap e445: 8 total, 8 up, 8 in 2024-04-12T11:20:50.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:50 smithi087 ceph-mon[89014]: pgmap v304: 89 pgs: 89 active+clean; 257 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 18 op/s 2024-04-12T11:20:50.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:50 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:50.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:50 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:50.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:50 smithi087 ceph-mon[89014]: Reconfiguring osd.6 (monmap changed)... 2024-04-12T11:20:50.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:50 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T11:20:50.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:50 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:50.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:50 smithi087 ceph-mon[89014]: Reconfiguring daemon osd.6 on smithi087 2024-04-12T11:20:50.223 INFO:tasks.rados.rados.0.smithi029.stdout:1523: writing smithi02979277-30 from 3279140 to 3900598 tid 3 2024-04-12T11:20:50.224 INFO:tasks.rados.rados.0.smithi029.stdout:1524: snap_remove snap 189 2024-04-12T11:20:50.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[97787]: osdmap e445: 8 total, 8 up, 8 in 2024-04-12T11:20:50.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[97787]: pgmap v304: 89 pgs: 89 active+clean; 257 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 18 op/s 2024-04-12T11:20:50.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:50.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:50.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[97787]: Reconfiguring osd.6 (monmap changed)... 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[97787]: Reconfiguring daemon osd.6 on smithi087 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[93650]: osdmap e445: 8 total, 8 up, 8 in 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[93650]: pgmap v304: 89 pgs: 89 active+clean; 257 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 18 op/s 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[93650]: Reconfiguring osd.6 (monmap changed)... 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:50 smithi029 ceph-mon[93650]: Reconfiguring daemon osd.6 on smithi087 2024-04-12T11:20:51.048 INFO:tasks.rados.rados.0.smithi029.stdout:1521: finishing write tid 1 to smithi02979277-9 2024-04-12T11:20:51.048 INFO:tasks.rados.rados.0.smithi029.stdout:1521: finishing write tid 2 to smithi02979277-9 2024-04-12T11:20:51.048 INFO:tasks.rados.rados.0.smithi029.stdout:1521: finishing write tid 3 to smithi02979277-9 2024-04-12T11:20:51.048 INFO:tasks.rados.rados.0.smithi029.stdout:1521: finishing write tid 4 to smithi02979277-9 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1521: finishing write tid 5 to smithi02979277-9 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1521: oid 9 updating version 0 to 574 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1521: oid 9 version 574 is already newer than 571 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1521: oid 9 version 574 is already newer than 573 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1521: oid 9 version 574 is already newer than 572 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 574 (ObjNum 592 snap 190 seq_num 592) dirty exists 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1521: left oid 9 (ObjNum 592 snap 190 seq_num 592) 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1522: finishing write tid 1 to smithi02979277-37 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1522: finishing write tid 2 to smithi02979277-37 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1522: finishing write tid 3 to smithi02979277-37 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1522: finishing write tid 4 to smithi02979277-37 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1522: finishing write tid 5 to smithi02979277-37 2024-04-12T11:20:51.049 INFO:tasks.rados.rados.0.smithi029.stdout:1522: oid 37 updating version 0 to 576 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1522: oid 37 version 576 is already newer than 575 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1522: oid 37 updating version 576 to 577 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1522: oid 37 updating version 577 to 578 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 578 (ObjNum 593 snap 190 seq_num 593) dirty exists 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1522: left oid 37 (ObjNum 593 snap 190 seq_num 593) 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1523: finishing write tid 1 to smithi02979277-30 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1523: finishing write tid 2 to smithi02979277-30 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1523: finishing write tid 3 to smithi02979277-30 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1523: finishing write tid 4 to smithi02979277-30 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1523: finishing write tid 5 to smithi02979277-30 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1523: oid 30 updating version 0 to 617 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1523: oid 30 version 617 is already newer than 614 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1523: oid 30 version 617 is already newer than 616 2024-04-12T11:20:51.050 INFO:tasks.rados.rados.0.smithi029.stdout:1523: oid 30 version 617 is already newer than 615 2024-04-12T11:20:51.051 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 617 (ObjNum 594 snap 190 seq_num 594) dirty exists 2024-04-12T11:20:51.051 INFO:tasks.rados.rados.0.smithi029.stdout:1523: left oid 30 (ObjNum 594 snap 190 seq_num 594) 2024-04-12T11:20:51.051 INFO:tasks.rados.rados.0.smithi029.stdout:1521: done (3 left) 2024-04-12T11:20:51.051 INFO:tasks.rados.rados.0.smithi029.stdout:1522: done (2 left) 2024-04-12T11:20:51.051 INFO:tasks.rados.rados.0.smithi029.stdout:1523: done (1 left) 2024-04-12T11:20:51.051 INFO:tasks.rados.rados.0.smithi029.stdout:1524: done (0 left) 2024-04-12T11:20:51.051 INFO:tasks.rados.rados.0.smithi029.stdout:1525: delete oid 7 current snap is 190 2024-04-12T11:20:51.054 INFO:tasks.rados.rados.0.smithi029.stdout:1525: done (0 left) 2024-04-12T11:20:51.054 INFO:tasks.rados.rados.0.smithi029.stdout:1526: read oid 40 snap -1 2024-04-12T11:20:51.054 INFO:tasks.rados.rados.0.smithi029.stdout:1526: expect (ObjNum 458 snap 150 seq_num 458) 2024-04-12T11:20:51.054 INFO:tasks.rados.rados.0.smithi029.stdout:1527: write oid 14 current snap is 190 2024-04-12T11:20:51.054 INFO:tasks.rados.rados.0.smithi029.stdout:1527: seq_num 596 ranges {416972=723938,1743268=273704} 2024-04-12T11:20:51.060 INFO:tasks.rados.rados.0.smithi029.stdout:1527: writing smithi02979277-14 from 416972 to 1140910 tid 1 2024-04-12T11:20:51.065 INFO:tasks.rados.rados.0.smithi029.stdout:1527: writing smithi02979277-14 from 1743268 to 2016972 tid 2 2024-04-12T11:20:51.066 INFO:tasks.rados.rados.0.smithi029.stdout:1528: snap_create 2024-04-12T11:20:51.066 INFO:tasks.rados.rados.0.smithi029.stdout:1526: expect (ObjNum 458 snap 150 seq_num 458) 2024-04-12T11:20:51.189 INFO:tasks.rados.rados.0.smithi029.stdout:1527: finishing write tid 1 to smithi02979277-14 2024-04-12T11:20:51.189 INFO:tasks.rados.rados.0.smithi029.stdout:1527: finishing write tid 2 to smithi02979277-14 2024-04-12T11:20:51.189 INFO:tasks.rados.rados.0.smithi029.stdout:1527: finishing write tid 3 to smithi02979277-14 2024-04-12T11:20:51.190 INFO:tasks.rados.rados.0.smithi029.stdout:1527: finishing write tid 4 to smithi02979277-14 2024-04-12T11:20:51.190 INFO:tasks.rados.rados.0.smithi029.stdout:1527: oid 14 updating version 0 to 911 2024-04-12T11:20:51.190 INFO:tasks.rados.rados.0.smithi029.stdout:1527: oid 14 version 911 is already newer than 910 2024-04-12T11:20:51.190 INFO:tasks.rados.rados.0.smithi029.stdout:1527: oid 14 updating version 911 to 912 2024-04-12T11:20:51.190 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 912 (ObjNum 596 snap 190 seq_num 596) dirty exists 2024-04-12T11:20:51.190 INFO:tasks.rados.rados.0.smithi029.stdout:1527: left oid 14 (ObjNum 596 snap 190 seq_num 596) 2024-04-12T11:20:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:51 smithi087 ceph-mon[89014]: osdmap e446: 8 total, 8 up, 8 in 2024-04-12T11:20:51.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:51 smithi029 ceph-mon[93650]: osdmap e446: 8 total, 8 up, 8 in 2024-04-12T11:20:51.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:51 smithi029 ceph-mon[97787]: osdmap e446: 8 total, 8 up, 8 in 2024-04-12T11:20:52.055 INFO:tasks.rados.rados.0.smithi029.stdout:1526: done (2 left) 2024-04-12T11:20:52.055 INFO:tasks.rados.rados.0.smithi029.stdout:1527: done (1 left) 2024-04-12T11:20:52.055 INFO:tasks.rados.rados.0.smithi029.stdout:1528: done (0 left) 2024-04-12T11:20:52.056 INFO:tasks.rados.rados.0.smithi029.stdout:1529: write oid 26 current snap is 191 2024-04-12T11:20:52.056 INFO:tasks.rados.rados.0.smithi029.stdout:1529: seq_num 597 ranges {433779=799187,1801750=232029} 2024-04-12T11:20:52.061 INFO:tasks.rados.rados.0.smithi029.stdout:1529: writing smithi02979277-26 from 433779 to 1232966 tid 1 2024-04-12T11:20:52.066 INFO:tasks.rados.rados.0.smithi029.stdout:1529: writing smithi02979277-26 from 1801750 to 2033779 tid 2 2024-04-12T11:20:52.066 INFO:tasks.rados.rados.0.smithi029.stdout:1530: snap_remove snap 185 2024-04-12T11:20:52.314 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:52 smithi087 ceph-mon[89014]: osdmap e447: 8 total, 8 up, 8 in 2024-04-12T11:20:52.314 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:52.314 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:52.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:52 smithi087 ceph-mon[89014]: Reconfiguring osd.7 (monmap changed)... 2024-04-12T11:20:52.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T11:20:52.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:52.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:52 smithi087 ceph-mon[89014]: Reconfiguring daemon osd.7 on smithi087 2024-04-12T11:20:52.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:52 smithi087 ceph-mon[89014]: pgmap v307: 89 pgs: 89 active+clean; 257 MiB data, 799 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:52.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[93650]: osdmap e447: 8 total, 8 up, 8 in 2024-04-12T11:20:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[93650]: Reconfiguring osd.7 (monmap changed)... 2024-04-12T11:20:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T11:20:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[93650]: Reconfiguring daemon osd.7 on smithi087 2024-04-12T11:20:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[93650]: pgmap v307: 89 pgs: 89 active+clean; 257 MiB data, 799 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[97787]: osdmap e447: 8 total, 8 up, 8 in 2024-04-12T11:20:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[97787]: Reconfiguring osd.7 (monmap changed)... 2024-04-12T11:20:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T11:20:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[97787]: Reconfiguring daemon osd.7 on smithi087 2024-04-12T11:20:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[97787]: pgmap v307: 89 pgs: 89 active+clean; 257 MiB data, 799 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:20:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:1530: done (1 left) 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:1531: delete oid 22 current snap is 191 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:1529: finishing write tid 1 to smithi02979277-26 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:1529: finishing write tid 2 to smithi02979277-26 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:1529: finishing write tid 3 to smithi02979277-26 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:1529: finishing write tid 4 to smithi02979277-26 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:1529: oid 26 updating version 0 to 982 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:1529: oid 26 updating version 982 to 983 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:1529: oid 26 updating version 983 to 984 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 984 (ObjNum 597 snap 191 seq_num 597) dirty exists 2024-04-12T11:20:52.544 INFO:tasks.rados.rados.0.smithi029.stdout:1529: left oid 26 (ObjNum 597 snap 191 seq_num 597) 2024-04-12T11:20:52.551 INFO:tasks.rados.rados.0.smithi029.stdout:1529: done (1 left) 2024-04-12T11:20:52.551 INFO:tasks.rados.rados.0.smithi029.stdout:1531: done (0 left) 2024-04-12T11:20:52.552 INFO:tasks.rados.rados.0.smithi029.stdout:1532: snap_create 2024-04-12T11:20:52.653 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:53.317 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:53 smithi087 ceph-mon[89014]: osdmap e448: 8 total, 8 up, 8 in 2024-04-12T11:20:53.318 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:53 smithi087 ceph-mon[89014]: osdmap e449: 8 total, 8 up, 8 in 2024-04-12T11:20:53.318 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:53.318 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:53.318 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:53 smithi087 ceph-mon[89014]: Reconfiguring mds.foo.smithi087.mbflzu (monmap changed)... 2024-04-12T11:20:53.318 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi087.mbflzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:20:53.318 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:53.318 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:53 smithi087 ceph-mon[89014]: Reconfiguring daemon mds.foo.smithi087.mbflzu on smithi087 2024-04-12T11:20:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[97787]: osdmap e448: 8 total, 8 up, 8 in 2024-04-12T11:20:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[97787]: osdmap e449: 8 total, 8 up, 8 in 2024-04-12T11:20:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[97787]: Reconfiguring mds.foo.smithi087.mbflzu (monmap changed)... 2024-04-12T11:20:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi087.mbflzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:20:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[97787]: Reconfiguring daemon mds.foo.smithi087.mbflzu on smithi087 2024-04-12T11:20:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[93650]: osdmap e448: 8 total, 8 up, 8 in 2024-04-12T11:20:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[93650]: osdmap e449: 8 total, 8 up, 8 in 2024-04-12T11:20:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[93650]: Reconfiguring mds.foo.smithi087.mbflzu (monmap changed)... 2024-04-12T11:20:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi087.mbflzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T11:20:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:53 smithi029 ceph-mon[93650]: Reconfiguring daemon mds.foo.smithi087.mbflzu on smithi087 2024-04-12T11:20:53.545 INFO:tasks.rados.rados.0.smithi029.stdout:1532: done (0 left) 2024-04-12T11:20:53.545 INFO:tasks.rados.rados.0.smithi029.stdout:1533: snap_remove snap 181 2024-04-12T11:20:53.598 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:20:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:20:53.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:20:54.548 INFO:tasks.rados.rados.0.smithi029.stdout:1533: done (0 left) 2024-04-12T11:20:54.548 INFO:tasks.rados.rados.0.smithi029.stdout:1534: read oid 45 snap -1 2024-04-12T11:20:54.548 INFO:tasks.rados.rados.0.smithi029.stdout:1534: expect (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:20:54.548 INFO:tasks.rados.rados.0.smithi029.stdout:1535: snap_remove snap 176 2024-04-12T11:20:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[97787]: pgmap v310: 89 pgs: 89 active+clean; 238 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[97787]: osdmap e450: 8 total, 8 up, 8 in 2024-04-12T11:20:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:20:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[93650]: pgmap v310: 89 pgs: 89 active+clean; 238 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[93650]: osdmap e450: 8 total, 8 up, 8 in 2024-04-12T11:20:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:20:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:54 smithi087 ceph-mon[89014]: pgmap v310: 89 pgs: 89 active+clean; 238 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:54 smithi087 ceph-mon[89014]: osdmap e450: 8 total, 8 up, 8 in 2024-04-12T11:20:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:20:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:20:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:20:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:20:55.213 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:20:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:20:54] "GET /metrics HTTP/1.1" 200 36250 "" "Prometheus/2.43.0" 2024-04-12T11:20:55.548 INFO:tasks.rados.rados.0.smithi029.stdout:1535: done (1 left) 2024-04-12T11:20:55.548 INFO:tasks.rados.rados.0.smithi029.stdout:1536: read oid 38 snap -1 2024-04-12T11:20:55.548 INFO:tasks.rados.rados.0.smithi029.stdout:1536: expect (ObjNum 591 snap 190 seq_num 591) 2024-04-12T11:20:55.548 INFO:tasks.rados.rados.0.smithi029.stdout:1537: snap_remove snap 188 2024-04-12T11:20:55.548 INFO:tasks.rados.rados.0.smithi029.stdout:1534: expect (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:20:55.655 INFO:tasks.rados.rados.0.smithi029.stdout:1536: expect (ObjNum 591 snap 190 seq_num 591) 2024-04-12T11:20:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:55 smithi087 ceph-mon[89014]: osdmap e451: 8 total, 8 up, 8 in 2024-04-12T11:20:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:55 smithi087 ceph-mon[89014]: osdmap e452: 8 total, 8 up, 8 in 2024-04-12T11:20:56.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:55 smithi029 ceph-mon[93650]: osdmap e451: 8 total, 8 up, 8 in 2024-04-12T11:20:56.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:55 smithi029 ceph-mon[93650]: osdmap e452: 8 total, 8 up, 8 in 2024-04-12T11:20:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:55 smithi029 ceph-mon[97787]: osdmap e451: 8 total, 8 up, 8 in 2024-04-12T11:20:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:55 smithi029 ceph-mon[97787]: osdmap e452: 8 total, 8 up, 8 in 2024-04-12T11:20:56.568 INFO:tasks.rados.rados.0.smithi029.stdout:1534: done (2 left) 2024-04-12T11:20:56.568 INFO:tasks.rados.rados.0.smithi029.stdout:1536: done (1 left) 2024-04-12T11:20:56.568 INFO:tasks.rados.rados.0.smithi029.stdout:1537: done (0 left) 2024-04-12T11:20:56.568 INFO:tasks.rados.rados.0.smithi029.stdout:1538: read oid 26 snap -1 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1538: expect (ObjNum 597 snap 191 seq_num 597) 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1539: read oid 27 snap 191 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1539: expect deleted 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1540: read oid 36 snap 183 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1540: expect deleted 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1541: read oid 18 snap 190 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1541: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1542: read oid 21 snap -1 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1542: expect (ObjNum 579 snap 185 seq_num 579) 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1543: read oid 14 snap 177 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1543: expect (ObjNum 559 snap 177 seq_num 559) 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1544: read oid 22 snap -1 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1544: expect deleted 2024-04-12T11:20:56.569 INFO:tasks.rados.rados.0.smithi029.stdout:1545: delete oid 30 current snap is 192 2024-04-12T11:20:56.576 INFO:tasks.rados.rados.0.smithi029.stdout:1538: expect (ObjNum 597 snap 191 seq_num 597) 2024-04-12T11:20:56.692 INFO:tasks.rados.rados.0.smithi029.stdout:1541: expect (ObjNum 489 snap 158 seq_num 489) 2024-04-12T11:20:56.828 INFO:tasks.rados.rados.0.smithi029.stdout:1542: expect (ObjNum 579 snap 185 seq_num 579) 2024-04-12T11:20:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:56 smithi087 ceph-mon[89014]: pgmap v313: 89 pgs: 89 active+clean; 238 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:57.010 INFO:tasks.rados.rados.0.smithi029.stdout:1543: expect (ObjNum 559 snap 177 seq_num 559) 2024-04-12T11:20:57.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:56 smithi029 ceph-mon[93650]: pgmap v313: 89 pgs: 89 active+clean; 238 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:56 smithi029 ceph-mon[97787]: pgmap v313: 89 pgs: 89 active+clean; 238 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-04-12T11:20:57.157 INFO:tasks.rados.rados.0.smithi029.stdout:1538: done (7 left) 2024-04-12T11:20:57.157 INFO:tasks.rados.rados.0.smithi029.stdout:1539: done (6 left) 2024-04-12T11:20:57.157 INFO:tasks.rados.rados.0.smithi029.stdout:1540: done (5 left) 2024-04-12T11:20:57.157 INFO:tasks.rados.rados.0.smithi029.stdout:1541: done (4 left) 2024-04-12T11:20:57.157 INFO:tasks.rados.rados.0.smithi029.stdout:1542: done (3 left) 2024-04-12T11:20:57.157 INFO:tasks.rados.rados.0.smithi029.stdout:1543: done (2 left) 2024-04-12T11:20:57.157 INFO:tasks.rados.rados.0.smithi029.stdout:1544: done (1 left) 2024-04-12T11:20:57.157 INFO:tasks.rados.rados.0.smithi029.stdout:1545: done (0 left) 2024-04-12T11:20:57.157 INFO:tasks.rados.rados.0.smithi029.stdout:1546: read oid 10 snap -1 2024-04-12T11:20:57.158 INFO:tasks.rados.rados.0.smithi029.stdout:1546: expect (ObjNum 576 snap 184 seq_num 576) 2024-04-12T11:20:57.158 INFO:tasks.rados.rados.0.smithi029.stdout:1547: write oid 27 current snap is 192 2024-04-12T11:20:57.158 INFO:tasks.rados.rados.0.smithi029.stdout:1547: seq_num 600 ranges {484200=624934,1860843=223357} 2024-04-12T11:20:57.163 INFO:tasks.rados.rados.0.smithi029.stdout:1547: writing smithi02979277-27 from 484200 to 1109134 tid 1 2024-04-12T11:20:57.169 INFO:tasks.rados.rados.0.smithi029.stdout:1547: writing smithi02979277-27 from 1860843 to 2084200 tid 2 2024-04-12T11:20:57.169 INFO:tasks.rados.rados.0.smithi029.stdout:1548: read oid 7 snap -1 2024-04-12T11:20:57.169 INFO:tasks.rados.rados.0.smithi029.stdout:1548: expect deleted 2024-04-12T11:20:57.169 INFO:tasks.rados.rados.0.smithi029.stdout:1549: write oid 8 current snap is 192 2024-04-12T11:20:57.169 INFO:tasks.rados.rados.0.smithi029.stdout:1549: seq_num 601 ranges {501007=416536,1635678=465329} 2024-04-12T11:20:57.173 INFO:tasks.rados.rados.0.smithi029.stdout:1549: writing smithi02979277-8 from 501007 to 917543 tid 1 2024-04-12T11:20:57.180 INFO:tasks.rados.rados.0.smithi029.stdout:1549: writing smithi02979277-8 from 1635678 to 2101007 tid 2 2024-04-12T11:20:57.180 INFO:tasks.rados.rados.0.smithi029.stdout:1550: delete oid 30 current snap is 192 2024-04-12T11:20:57.180 INFO:tasks.rados.rados.0.smithi029.stdout:1546: expect (ObjNum 576 snap 184 seq_num 576) 2024-04-12T11:20:57.342 INFO:tasks.rados.rados.0.smithi029.stdout:1547: finishing write tid 1 to smithi02979277-27 2024-04-12T11:20:57.342 INFO:tasks.rados.rados.0.smithi029.stdout:1547: finishing write tid 2 to smithi02979277-27 2024-04-12T11:20:57.343 INFO:tasks.rados.rados.0.smithi029.stdout:1547: finishing write tid 3 to smithi02979277-27 2024-04-12T11:20:57.343 INFO:tasks.rados.rados.0.smithi029.stdout:1547: finishing write tid 4 to smithi02979277-27 2024-04-12T11:20:57.343 INFO:tasks.rados.rados.0.smithi029.stdout:1547: oid 27 updating version 0 to 640 2024-04-12T11:20:57.343 INFO:tasks.rados.rados.0.smithi029.stdout:1547: oid 27 updating version 640 to 642 2024-04-12T11:20:57.343 INFO:tasks.rados.rados.0.smithi029.stdout:1547: oid 27 version 642 is already newer than 641 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 642 (ObjNum 600 snap 192 seq_num 600) dirty exists 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1547: left oid 27 (ObjNum 600 snap 192 seq_num 600) 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1549: finishing write tid 1 to smithi02979277-8 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1546: done (4 left) 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1547: done (3 left) 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1548: done (2 left) 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1550: done (1 left) 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1551: read oid 11 snap -1 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1551: expect (ObjNum 566 snap 177 seq_num 566) 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1552: read oid 24 snap -1 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1552: expect (ObjNum 584 snap 189 seq_num 584) 2024-04-12T11:20:57.344 INFO:tasks.rados.rados.0.smithi029.stdout:1553: snap_remove snap 177 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:1553: done (3 left) 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:1554: delete oid 28 current snap is 192 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:1549: finishing write tid 2 to smithi02979277-8 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:1549: finishing write tid 3 to smithi02979277-8 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:1549: finishing write tid 4 to smithi02979277-8 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:1549: oid 8 updating version 0 to 145 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:1549: oid 8 updating version 145 to 146 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:1549: oid 8 version 146 is already newer than 144 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 146 (ObjNum 601 snap 192 seq_num 601) dirty exists 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:1549: left oid 8 (ObjNum 601 snap 192 seq_num 601) 2024-04-12T11:20:57.573 INFO:tasks.rados.rados.0.smithi029.stdout:1551: expect (ObjNum 566 snap 177 seq_num 566) 2024-04-12T11:20:57.692 INFO:tasks.rados.rados.0.smithi029.stdout:1552: expect (ObjNum 584 snap 189 seq_num 584) 2024-04-12T11:20:57.904 INFO:tasks.rados.rados.0.smithi029.stdout:1549: done (3 left) 2024-04-12T11:20:57.904 INFO:tasks.rados.rados.0.smithi029.stdout:1551: done (2 left) 2024-04-12T11:20:57.905 INFO:tasks.rados.rados.0.smithi029.stdout:1552: done (1 left) 2024-04-12T11:20:57.905 INFO:tasks.rados.rados.0.smithi029.stdout:1554: done (0 left) 2024-04-12T11:20:57.905 INFO:tasks.rados.rados.0.smithi029.stdout:1555: write oid 29 current snap is 192 2024-04-12T11:20:57.905 INFO:tasks.rados.rados.0.smithi029.stdout:1555: seq_num 604 ranges {551428=642283,1694771=456657} 2024-04-12T11:20:57.910 INFO:tasks.rados.rados.0.smithi029.stdout:1555: writing smithi02979277-29 from 551428 to 1193711 tid 1 2024-04-12T11:20:57.916 INFO:tasks.rados.rados.0.smithi029.stdout:1555: writing smithi02979277-29 from 1694771 to 2151428 tid 2 2024-04-12T11:20:57.916 INFO:tasks.rados.rados.0.smithi029.stdout:1556: snap_remove snap 171 2024-04-12T11:20:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:57 smithi087 ceph-mon[89014]: osdmap e453: 8 total, 8 up, 8 in 2024-04-12T11:20:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:57 smithi029 ceph-mon[93650]: osdmap e453: 8 total, 8 up, 8 in 2024-04-12T11:20:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:57 smithi029 ceph-mon[97787]: osdmap e453: 8 total, 8 up, 8 in 2024-04-12T11:20:58.580 INFO:tasks.rados.rados.0.smithi029.stdout:1556: done (1 left) 2024-04-12T11:20:58.580 INFO:tasks.rados.rados.0.smithi029.stdout:1557: read oid 46 snap -1 2024-04-12T11:20:58.580 INFO:tasks.rados.rados.0.smithi029.stdout:1557: expect (ObjNum 487 snap 158 seq_num 487) 2024-04-12T11:20:58.580 INFO:tasks.rados.rados.0.smithi029.stdout:1558: rollback oid 22 current snap is 192 2024-04-12T11:20:58.580 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 22 to 166 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1555: finishing write tid 1 to smithi02979277-29 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1555: finishing write tid 2 to smithi02979277-29 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1555: finishing write tid 3 to smithi02979277-29 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1555: finishing write tid 4 to smithi02979277-29 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1555: oid 29 updating version 0 to 605 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1555: oid 29 updating version 605 to 606 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1555: oid 29 version 606 is already newer than 604 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 606 (ObjNum 604 snap 192 seq_num 604) dirty exists 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1555: left oid 29 (ObjNum 604 snap 192 seq_num 604) 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1555: done (2 left) 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1559: read oid 29 snap -1 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1559: expect (ObjNum 604 snap 192 seq_num 604) 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1560: read oid 12 snap -1 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1560: expect (ObjNum 575 snap 183 seq_num 575) 2024-04-12T11:20:58.581 INFO:tasks.rados.rados.0.smithi029.stdout:1561: snap_create 2024-04-12T11:20:58.590 INFO:tasks.rados.rados.0.smithi029.stdout:1557: expect (ObjNum 487 snap 158 seq_num 487) 2024-04-12T11:20:58.723 INFO:tasks.rados.rados.0.smithi029.stdout:1558: finishing rollback tid 1 to smithi02979277-22 2024-04-12T11:20:58.724 INFO:tasks.rados.rados.0.smithi029.stdout:1558: finishing rollback tid 2 to smithi02979277-22 2024-04-12T11:20:58.724 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 526 (ObjNum 495 snap 159 seq_num 495) dirty exists 2024-04-12T11:20:58.724 INFO:tasks.rados.rados.0.smithi029.stdout:1559: expect (ObjNum 604 snap 192 seq_num 604) 2024-04-12T11:20:58.813 INFO:tasks.rados.rados.0.smithi029.stdout:1560: expect (ObjNum 575 snap 183 seq_num 575) 2024-04-12T11:20:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:58 smithi087 ceph-mon[89014]: pgmap v316: 89 pgs: 89 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 423 KiB/s wr, 17 op/s 2024-04-12T11:20:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:58 smithi087 ceph-mon[89014]: osdmap e454: 8 total, 8 up, 8 in 2024-04-12T11:20:59.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:58 smithi029 ceph-mon[97787]: pgmap v316: 89 pgs: 89 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 423 KiB/s wr, 17 op/s 2024-04-12T11:20:59.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:58 smithi029 ceph-mon[97787]: osdmap e454: 8 total, 8 up, 8 in 2024-04-12T11:20:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:58 smithi029 ceph-mon[93650]: pgmap v316: 89 pgs: 89 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 423 KiB/s wr, 17 op/s 2024-04-12T11:20:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:58 smithi029 ceph-mon[93650]: osdmap e454: 8 total, 8 up, 8 in 2024-04-12T11:20:59.586 INFO:tasks.rados.rados.0.smithi029.stdout:1557: done (4 left) 2024-04-12T11:20:59.586 INFO:tasks.rados.rados.0.smithi029.stdout:1558: done (3 left) 2024-04-12T11:20:59.586 INFO:tasks.rados.rados.0.smithi029.stdout:1559: done (2 left) 2024-04-12T11:20:59.586 INFO:tasks.rados.rados.0.smithi029.stdout:1560: done (1 left) 2024-04-12T11:20:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1561: done (0 left) 2024-04-12T11:20:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1562: snap_create 2024-04-12T11:20:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:20:59 smithi087 ceph-mon[89014]: osdmap e455: 8 total, 8 up, 8 in 2024-04-12T11:21:00.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:20:59 smithi029 ceph-mon[93650]: osdmap e455: 8 total, 8 up, 8 in 2024-04-12T11:21:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:20:59 smithi029 ceph-mon[97787]: osdmap e455: 8 total, 8 up, 8 in 2024-04-12T11:21:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:1562: done (0 left) 2024-04-12T11:21:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:1563: rollback oid 6 current snap is 194 2024-04-12T11:21:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 6 to 163 2024-04-12T11:21:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:1564: write oid 32 current snap is 194 2024-04-12T11:21:00.588 INFO:tasks.rados.rados.0.smithi029.stdout:1564: seq_num 605 ranges {568235=717532,1753253=414982} 2024-04-12T11:21:00.595 INFO:tasks.rados.rados.0.smithi029.stdout:1564: writing smithi02979277-32 from 568235 to 1285767 tid 1 2024-04-12T11:21:00.600 INFO:tasks.rados.rados.0.smithi029.stdout:1564: writing smithi02979277-32 from 1753253 to 2168235 tid 2 2024-04-12T11:21:00.601 INFO:tasks.rados.rados.0.smithi029.stdout:1565: rollback oid 46 current snap is 194 2024-04-12T11:21:00.601 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 46 to 192 2024-04-12T11:21:00.601 INFO:tasks.rados.rados.0.smithi029.stdout:1563: finishing rollback tid 0 to smithi02979277-6 2024-04-12T11:21:00.601 INFO:tasks.rados.rados.0.smithi029.stdout:1563: finishing rollback tid 1 to smithi02979277-6 2024-04-12T11:21:00.601 INFO:tasks.rados.rados.0.smithi029.stdout:1563: finishing rollback tid 2 to smithi02979277-6 2024-04-12T11:21:00.601 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 376 (ObjNum 424 snap 135 seq_num 424) dirty exists 2024-04-12T11:21:00.601 INFO:tasks.rados.rados.0.smithi029.stdout:1563: done (2 left) 2024-04-12T11:21:00.601 INFO:tasks.rados.rados.0.smithi029.stdout:1566: read oid 10 snap 193 2024-04-12T11:21:00.601 INFO:tasks.rados.rados.0.smithi029.stdout:1566: expect (ObjNum 576 snap 184 seq_num 576) 2024-04-12T11:21:00.601 INFO:tasks.rados.rados.0.smithi029.stdout:1567: snap_create 2024-04-12T11:21:00.604 INFO:tasks.rados.rados.0.smithi029.stdout:1564: finishing write tid 1 to smithi02979277-32 2024-04-12T11:21:00.604 INFO:tasks.rados.rados.0.smithi029.stdout:1565: finishing rollback tid 0 to smithi02979277-46 2024-04-12T11:21:00.605 INFO:tasks.rados.rados.0.smithi029.stdout:1564: finishing write tid 2 to smithi02979277-32 2024-04-12T11:21:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1564: finishing write tid 3 to smithi02979277-32 2024-04-12T11:21:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1565: finishing rollback tid 1 to smithi02979277-46 2024-04-12T11:21:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1564: finishing write tid 4 to smithi02979277-32 2024-04-12T11:21:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1564: oid 32 updating version 0 to 612 2024-04-12T11:21:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1564: oid 32 updating version 612 to 613 2024-04-12T11:21:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1564: oid 32 updating version 613 to 614 2024-04-12T11:21:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 614 (ObjNum 605 snap 194 seq_num 605) dirty exists 2024-04-12T11:21:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1564: left oid 32 (ObjNum 605 snap 194 seq_num 605) 2024-04-12T11:21:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1565: finishing rollback tid 2 to smithi02979277-46 2024-04-12T11:21:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 440 (ObjNum 487 snap 158 seq_num 487) dirty exists 2024-04-12T11:21:00.607 INFO:tasks.rados.rados.0.smithi029.stdout:1566: expect (ObjNum 576 snap 184 seq_num 576) 2024-04-12T11:21:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:00 smithi087 ceph-mon[89014]: pgmap v319: 89 pgs: 89 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 423 KiB/s wr, 17 op/s 2024-04-12T11:21:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:00 smithi087 ceph-mon[89014]: osdmap e456: 8 total, 8 up, 8 in 2024-04-12T11:21:01.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:00 smithi029 ceph-mon[97787]: pgmap v319: 89 pgs: 89 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 423 KiB/s wr, 17 op/s 2024-04-12T11:21:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:00 smithi029 ceph-mon[97787]: osdmap e456: 8 total, 8 up, 8 in 2024-04-12T11:21:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:00 smithi029 ceph-mon[93650]: pgmap v319: 89 pgs: 89 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 423 KiB/s wr, 17 op/s 2024-04-12T11:21:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:00 smithi029 ceph-mon[93650]: osdmap e456: 8 total, 8 up, 8 in 2024-04-12T11:21:01.598 INFO:tasks.rados.rados.0.smithi029.stdout:1564: done (3 left) 2024-04-12T11:21:01.598 INFO:tasks.rados.rados.0.smithi029.stdout:1565: done (2 left) 2024-04-12T11:21:01.598 INFO:tasks.rados.rados.0.smithi029.stdout:1566: done (1 left) 2024-04-12T11:21:01.598 INFO:tasks.rados.rados.0.smithi029.stdout:1567: done (0 left) 2024-04-12T11:21:01.598 INFO:tasks.rados.rados.0.smithi029.stdout:1568: read oid 17 snap 193 2024-04-12T11:21:01.598 INFO:tasks.rados.rados.0.smithi029.stdout:1568: expect (ObjNum 390 snap 121 seq_num 390) 2024-04-12T11:21:01.598 INFO:tasks.rados.rados.0.smithi029.stdout:1569: read oid 36 snap -1 2024-04-12T11:21:01.598 INFO:tasks.rados.rados.0.smithi029.stdout:1569: expect deleted 2024-04-12T11:21:01.599 INFO:tasks.rados.rados.0.smithi029.stdout:1570: snap_remove snap 183 2024-04-12T11:21:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:01 smithi087 ceph-mon[89014]: osdmap e457: 8 total, 8 up, 8 in 2024-04-12T11:21:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:01 smithi029 ceph-mon[93650]: osdmap e457: 8 total, 8 up, 8 in 2024-04-12T11:21:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:01 smithi029 ceph-mon[97787]: osdmap e457: 8 total, 8 up, 8 in 2024-04-12T11:21:02.546 INFO:tasks.rados.rados.0.smithi029.stdout:1570: done (2 left) 2024-04-12T11:21:02.547 INFO:tasks.rados.rados.0.smithi029.stdout:1571: snap_remove snap 194 2024-04-12T11:21:02.595 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:21:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:21:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:21:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:02 smithi087 ceph-mon[89014]: pgmap v322: 89 pgs: 89 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:21:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:02 smithi087 ceph-mon[89014]: osdmap e458: 8 total, 8 up, 8 in 2024-04-12T11:21:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:02 smithi087 ceph-mon[89014]: osdmap e459: 8 total, 8 up, 8 in 2024-04-12T11:21:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:02 smithi029 ceph-mon[97787]: pgmap v322: 89 pgs: 89 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:21:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:02 smithi029 ceph-mon[97787]: osdmap e458: 8 total, 8 up, 8 in 2024-04-12T11:21:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:02 smithi029 ceph-mon[97787]: osdmap e459: 8 total, 8 up, 8 in 2024-04-12T11:21:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:02 smithi029 ceph-mon[93650]: pgmap v322: 89 pgs: 89 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:21:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:02 smithi029 ceph-mon[93650]: osdmap e458: 8 total, 8 up, 8 in 2024-04-12T11:21:03.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:02 smithi029 ceph-mon[93650]: osdmap e459: 8 total, 8 up, 8 in 2024-04-12T11:21:03.547 INFO:tasks.rados.rados.0.smithi029.stdout:1571: done (2 left) 2024-04-12T11:21:03.547 INFO:tasks.rados.rados.0.smithi029.stdout:1572: write oid 29 current snap is 195 2024-04-12T11:21:03.548 INFO:tasks.rados.rados.0.smithi029.stdout:1572: seq_num 606 ranges {585042=792781,2095382=89660} 2024-04-12T11:21:03.555 INFO:tasks.rados.rados.0.smithi029.stdout:1572: writing smithi02979277-29 from 585042 to 1377823 tid 1 2024-04-12T11:21:03.560 INFO:tasks.rados.rados.0.smithi029.stdout:1572: writing smithi02979277-29 from 2095382 to 2185042 tid 2 2024-04-12T11:21:03.560 INFO:tasks.rados.rados.0.smithi029.stdout:1573: read oid 4 snap -1 2024-04-12T11:21:03.560 INFO:tasks.rados.rados.0.smithi029.stdout:1573: expect deleted 2024-04-12T11:21:03.560 INFO:tasks.rados.rados.0.smithi029.stdout:1574: read oid 44 snap -1 2024-04-12T11:21:03.560 INFO:tasks.rados.rados.0.smithi029.stdout:1574: expect (ObjNum 580 snap 185 seq_num 580) 2024-04-12T11:21:03.560 INFO:tasks.rados.rados.0.smithi029.stdout:1575: snap_remove snap 163 2024-04-12T11:21:03.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:21:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:21:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:21:04.552 INFO:tasks.rados.rados.0.smithi029.stdout:1575: done (5 left) 2024-04-12T11:21:04.552 INFO:tasks.rados.rados.0.smithi029.stdout:1576: snap_remove snap 166 2024-04-12T11:21:04.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:04 smithi029 ceph-mon[93650]: pgmap v325: 89 pgs: 89 active+clean; 225 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 545 KiB/s wr, 19 op/s 2024-04-12T11:21:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:04 smithi029 ceph-mon[93650]: osdmap e460: 8 total, 8 up, 8 in 2024-04-12T11:21:04.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:04 smithi029 ceph-mon[97787]: pgmap v325: 89 pgs: 89 active+clean; 225 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 545 KiB/s wr, 19 op/s 2024-04-12T11:21:04.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:04 smithi029 ceph-mon[97787]: osdmap e460: 8 total, 8 up, 8 in 2024-04-12T11:21:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:04 smithi087 ceph-mon[89014]: pgmap v325: 89 pgs: 89 active+clean; 225 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 545 KiB/s wr, 19 op/s 2024-04-12T11:21:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:04 smithi087 ceph-mon[89014]: osdmap e460: 8 total, 8 up, 8 in 2024-04-12T11:21:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:21:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:21:04] "GET /metrics HTTP/1.1" 200 36250 "" "Prometheus/2.43.0" 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:1576: done (5 left) 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:1577: read oid 34 snap -1 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:1577: expect (ObjNum 561 snap 177 seq_num 561) 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:1578: read oid 30 snap -1 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:1578: expect deleted 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:1579: rollback oid 24 current snap is 195 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 24 to 192 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:1580: read oid 38 snap 191 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:1580: expect (ObjNum 591 snap 190 seq_num 591) 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:1581: write oid 23 current snap is 195 2024-04-12T11:21:05.559 INFO:tasks.rados.rados.0.smithi029.stdout:1581: seq_num 607 ranges {601849=468030,1753864=447985} 2024-04-12T11:21:05.565 INFO:tasks.rados.rados.0.smithi029.stdout:1581: writing smithi02979277-23 from 601849 to 1069879 tid 1 2024-04-12T11:21:05.573 INFO:tasks.rados.rados.0.smithi029.stdout:1581: writing smithi02979277-23 from 1753864 to 2201849 tid 2 2024-04-12T11:21:05.573 INFO:tasks.rados.rados.0.smithi029.stdout:1582: write oid 15 current snap is 195 2024-04-12T11:21:05.573 INFO:tasks.rados.rados.0.smithi029.stdout:1582: seq_num 608 ranges {618656=543279,1812346=406310} 2024-04-12T11:21:05.580 INFO:tasks.rados.rados.0.smithi029.stdout:1582: writing smithi02979277-15 from 618656 to 1161935 tid 1 2024-04-12T11:21:05.586 INFO:tasks.rados.rados.0.smithi029.stdout:1582: writing smithi02979277-15 from 1812346 to 2218656 tid 2 2024-04-12T11:21:05.586 INFO:tasks.rados.rados.0.smithi029.stdout:1583: snap_remove snap 190 2024-04-12T11:21:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:05 smithi087 ceph-mon[89014]: osdmap e461: 8 total, 8 up, 8 in 2024-04-12T11:21:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:05 smithi029 ceph-mon[97787]: osdmap e461: 8 total, 8 up, 8 in 2024-04-12T11:21:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:05 smithi029 ceph-mon[93650]: osdmap e461: 8 total, 8 up, 8 in 2024-04-12T11:21:06.564 INFO:tasks.rados.rados.0.smithi029.stdout:1583: done (11 left) 2024-04-12T11:21:06.564 INFO:tasks.rados.rados.0.smithi029.stdout:1584: write oid 9 current snap is 195 2024-04-12T11:21:06.564 INFO:tasks.rados.rados.0.smithi029.stdout:1568: expect (ObjNum 390 snap 121 seq_num 390) 2024-04-12T11:21:06.684 INFO:tasks.rados.rados.0.smithi029.stdout:1572: finishing write tid 1 to smithi02979277-29 2024-04-12T11:21:06.684 INFO:tasks.rados.rados.0.smithi029.stdout:1572: finishing write tid 2 to smithi02979277-29 2024-04-12T11:21:06.684 INFO:tasks.rados.rados.0.smithi029.stdout:1572: finishing write tid 3 to smithi02979277-29 2024-04-12T11:21:06.684 INFO:tasks.rados.rados.0.smithi029.stdout:1584: seq_num 609 ranges {635463=734881,1987181=248282} 2024-04-12T11:21:06.691 INFO:tasks.rados.rados.0.smithi029.stdout:1584: writing smithi02979277-9 from 635463 to 1370344 tid 1 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1584: writing smithi02979277-9 from 1987181 to 2235463 tid 2 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1568: done (11 left) 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1569: done (10 left) 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1573: done (9 left) 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1585: read oid 46 snap -1 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1585: expect (ObjNum 487 snap 158 seq_num 487) 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1586: write oid 2 current snap is 195 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1572: finishing write tid 4 to smithi02979277-29 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1572: oid 29 updating version 0 to 618 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1572: oid 29 updating version 618 to 620 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1572: oid 29 version 620 is already newer than 619 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 620 (ObjNum 606 snap 195 seq_num 606) dirty exists 2024-04-12T11:21:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:1572: left oid 29 (ObjNum 606 snap 195 seq_num 606) 2024-04-12T11:21:06.698 INFO:tasks.rados.rados.0.smithi029.stdout:1574: expect (ObjNum 580 snap 185 seq_num 580) 2024-04-12T11:21:06.821 INFO:tasks.rados.rados.0.smithi029.stdout:1577: expect (ObjNum 561 snap 177 seq_num 561) 2024-04-12T11:21:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:06 smithi087 ceph-mon[89014]: pgmap v328: 89 pgs: 89 active+clean; 225 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 545 KiB/s wr, 19 op/s 2024-04-12T11:21:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:06 smithi087 ceph-mon[89014]: osdmap e462: 8 total, 8 up, 8 in 2024-04-12T11:21:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:07.006 INFO:tasks.rados.rados.0.smithi029.stdout:1581: finishing write tid 1 to smithi02979277-23 2024-04-12T11:21:07.006 INFO:tasks.rados.rados.0.smithi029.stdout:1580: expect (ObjNum 591 snap 190 seq_num 591) 2024-04-12T11:21:07.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:06 smithi029 ceph-mon[93650]: pgmap v328: 89 pgs: 89 active+clean; 225 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 545 KiB/s wr, 19 op/s 2024-04-12T11:21:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:06 smithi029 ceph-mon[93650]: osdmap e462: 8 total, 8 up, 8 in 2024-04-12T11:21:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:06 smithi029 ceph-mon[97787]: pgmap v328: 89 pgs: 89 active+clean; 225 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 545 KiB/s wr, 19 op/s 2024-04-12T11:21:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:06 smithi029 ceph-mon[97787]: osdmap e462: 8 total, 8 up, 8 in 2024-04-12T11:21:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:07.263 INFO:tasks.rados.rados.0.smithi029.stdout:1579: finishing rollback tid 0 to smithi02979277-24 2024-04-12T11:21:07.263 INFO:tasks.rados.rados.0.smithi029.stdout:1579: finishing rollback tid 1 to smithi02979277-24 2024-04-12T11:21:07.263 INFO:tasks.rados.rados.0.smithi029.stdout:1579: finishing rollback tid 2 to smithi02979277-24 2024-04-12T11:21:07.263 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1038 (ObjNum 584 snap 189 seq_num 584) dirty exists 2024-04-12T11:21:07.263 INFO:tasks.rados.rados.0.smithi029.stdout:1581: finishing write tid 2 to smithi02979277-23 2024-04-12T11:21:07.263 INFO:tasks.rados.rados.0.smithi029.stdout:1581: finishing write tid 3 to smithi02979277-23 2024-04-12T11:21:07.263 INFO:tasks.rados.rados.0.smithi029.stdout:1581: finishing write tid 4 to smithi02979277-23 2024-04-12T11:21:07.263 INFO:tasks.rados.rados.0.smithi029.stdout:1581: oid 23 updating version 0 to 550 2024-04-12T11:21:07.263 INFO:tasks.rados.rados.0.smithi029.stdout:1581: oid 23 updating version 550 to 552 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1581: oid 23 version 552 is already newer than 551 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 552 (ObjNum 607 snap 195 seq_num 607) dirty exists 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1581: left oid 23 (ObjNum 607 snap 195 seq_num 607) 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1582: finishing write tid 1 to smithi02979277-15 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1582: finishing write tid 2 to smithi02979277-15 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1582: finishing write tid 3 to smithi02979277-15 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1582: finishing write tid 4 to smithi02979277-15 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1582: oid 15 updating version 0 to 666 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1582: oid 15 updating version 666 to 667 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1582: oid 15 updating version 667 to 668 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 668 (ObjNum 608 snap 195 seq_num 608) dirty exists 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1582: left oid 15 (ObjNum 608 snap 195 seq_num 608) 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1584: finishing write tid 1 to smithi02979277-9 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1584: finishing write tid 2 to smithi02979277-9 2024-04-12T11:21:07.264 INFO:tasks.rados.rados.0.smithi029.stdout:1586: seq_num 610 ranges {652270=410130,1529310=493867} 2024-04-12T11:21:07.268 INFO:tasks.rados.rados.0.smithi029.stdout:1586: writing smithi02979277-2 from 652270 to 1062400 tid 1 2024-04-12T11:21:07.274 INFO:tasks.rados.rados.0.smithi029.stdout:1586: writing smithi02979277-2 from 1529310 to 2023177 tid 2 2024-04-12T11:21:07.274 INFO:tasks.rados.rados.0.smithi029.stdout:1572: done (10 left) 2024-04-12T11:21:07.274 INFO:tasks.rados.rados.0.smithi029.stdout:1574: done (9 left) 2024-04-12T11:21:07.274 INFO:tasks.rados.rados.0.smithi029.stdout:1577: done (8 left) 2024-04-12T11:21:07.274 INFO:tasks.rados.rados.0.smithi029.stdout:1578: done (7 left) 2024-04-12T11:21:07.274 INFO:tasks.rados.rados.0.smithi029.stdout:1579: done (6 left) 2024-04-12T11:21:07.275 INFO:tasks.rados.rados.0.smithi029.stdout:1580: done (5 left) 2024-04-12T11:21:07.275 INFO:tasks.rados.rados.0.smithi029.stdout:1581: done (4 left) 2024-04-12T11:21:07.275 INFO:tasks.rados.rados.0.smithi029.stdout:1582: done (3 left) 2024-04-12T11:21:07.275 INFO:tasks.rados.rados.0.smithi029.stdout:1587: snap_remove snap 192 2024-04-12T11:21:07.568 INFO:tasks.rados.rados.0.smithi029.stdout:1587: done (3 left) 2024-04-12T11:21:07.568 INFO:tasks.rados.rados.0.smithi029.stdout:1588: snap_remove snap 191 2024-04-12T11:21:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:07 smithi087 ceph-mon[89014]: osdmap e463: 8 total, 8 up, 8 in 2024-04-12T11:21:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:07 smithi029 ceph-mon[93650]: osdmap e463: 8 total, 8 up, 8 in 2024-04-12T11:21:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:07 smithi029 ceph-mon[97787]: osdmap e463: 8 total, 8 up, 8 in 2024-04-12T11:21:08.573 INFO:tasks.rados.rados.0.smithi029.stdout:1588: done (3 left) 2024-04-12T11:21:08.573 INFO:tasks.rados.rados.0.smithi029.stdout:1589: delete oid 35 current snap is 195 2024-04-12T11:21:08.573 INFO:tasks.rados.rados.0.smithi029.stdout:1584: finishing write tid 3 to smithi02979277-9 2024-04-12T11:21:08.573 INFO:tasks.rados.rados.0.smithi029.stdout:1584: finishing write tid 4 to smithi02979277-9 2024-04-12T11:21:08.573 INFO:tasks.rados.rados.0.smithi029.stdout:1584: oid 9 updating version 0 to 619 2024-04-12T11:21:08.573 INFO:tasks.rados.rados.0.smithi029.stdout:1584: oid 9 updating version 619 to 620 2024-04-12T11:21:08.573 INFO:tasks.rados.rados.0.smithi029.stdout:1584: oid 9 version 620 is already newer than 618 2024-04-12T11:21:08.574 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 620 (ObjNum 609 snap 195 seq_num 609) dirty exists 2024-04-12T11:21:08.574 INFO:tasks.rados.rados.0.smithi029.stdout:1584: left oid 9 (ObjNum 609 snap 195 seq_num 609) 2024-04-12T11:21:08.574 INFO:tasks.rados.rados.0.smithi029.stdout:1585: expect (ObjNum 487 snap 158 seq_num 487) 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:1586: finishing write tid 1 to smithi02979277-2 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:1586: finishing write tid 2 to smithi02979277-2 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:1586: finishing write tid 3 to smithi02979277-2 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:1586: finishing write tid 4 to smithi02979277-2 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:1586: oid 2 updating version 0 to 171 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:1586: oid 2 version 171 is already newer than 170 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:1586: oid 2 updating version 171 to 172 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 172 (ObjNum 610 snap 195 seq_num 610) dirty exists 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:1586: left oid 2 (ObjNum 610 snap 195 seq_num 610) 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:1584: done (3 left) 2024-04-12T11:21:08.705 INFO:tasks.rados.rados.0.smithi029.stdout:1585: done (2 left) 2024-04-12T11:21:08.706 INFO:tasks.rados.rados.0.smithi029.stdout:1586: done (1 left) 2024-04-12T11:21:08.706 INFO:tasks.rados.rados.0.smithi029.stdout:1589: done (0 left) 2024-04-12T11:21:08.706 INFO:tasks.rados.rados.0.smithi029.stdout:1590: snap_remove snap 193 2024-04-12T11:21:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:08 smithi087 ceph-mon[89014]: pgmap v331: 89 pgs: 89 active+clean; 136 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 912 KiB/s wr, 13 op/s 2024-04-12T11:21:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:08 smithi087 ceph-mon[89014]: osdmap e464: 8 total, 8 up, 8 in 2024-04-12T11:21:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:08 smithi029 ceph-mon[93650]: pgmap v331: 89 pgs: 89 active+clean; 136 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 912 KiB/s wr, 13 op/s 2024-04-12T11:21:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:08 smithi029 ceph-mon[93650]: osdmap e464: 8 total, 8 up, 8 in 2024-04-12T11:21:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:08 smithi029 ceph-mon[97787]: pgmap v331: 89 pgs: 89 active+clean; 136 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 912 KiB/s wr, 13 op/s 2024-04-12T11:21:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:08 smithi029 ceph-mon[97787]: osdmap e464: 8 total, 8 up, 8 in 2024-04-12T11:21:09.583 INFO:tasks.rados.rados.0.smithi029.stdout:1590: done (0 left) 2024-04-12T11:21:09.584 INFO:tasks.rados.rados.0.smithi029.stdout:1591: snap_create 2024-04-12T11:21:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:09 smithi087 ceph-mon[89014]: osdmap e465: 8 total, 8 up, 8 in 2024-04-12T11:21:10.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:09 smithi029 ceph-mon[93650]: osdmap e465: 8 total, 8 up, 8 in 2024-04-12T11:21:10.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:09 smithi029 ceph-mon[97787]: osdmap e465: 8 total, 8 up, 8 in 2024-04-12T11:21:10.588 INFO:tasks.rados.rados.0.smithi029.stdout:1591: done (0 left) 2024-04-12T11:21:10.588 INFO:tasks.rados.rados.0.smithi029.stdout:1592: rollback oid 25 current snap is 196 2024-04-12T11:21:10.588 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 25 to 195 2024-04-12T11:21:10.588 INFO:tasks.rados.rados.0.smithi029.stdout:1593: rollback oid 38 current snap is 196 2024-04-12T11:21:10.588 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 38 to 195 2024-04-12T11:21:10.588 INFO:tasks.rados.rados.0.smithi029.stdout:1594: rollback oid 43 current snap is 196 2024-04-12T11:21:10.588 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 195 2024-04-12T11:21:10.588 INFO:tasks.rados.rados.0.smithi029.stdout:1595: rollback oid 6 current snap is 196 2024-04-12T11:21:10.588 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 6 to 195 2024-04-12T11:21:10.589 INFO:tasks.rados.rados.0.smithi029.stdout:1596: read oid 2 snap -1 2024-04-12T11:21:10.589 INFO:tasks.rados.rados.0.smithi029.stdout:1596: expect (ObjNum 610 snap 195 seq_num 610) 2024-04-12T11:21:10.589 INFO:tasks.rados.rados.0.smithi029.stdout:1597: delete oid 9 current snap is 196 2024-04-12T11:21:10.594 INFO:tasks.rados.rados.0.smithi029.stdout:1597: done (5 left) 2024-04-12T11:21:10.594 INFO:tasks.rados.rados.0.smithi029.stdout:1598: write oid 31 current snap is 196 2024-04-12T11:21:10.595 INFO:tasks.rados.rados.0.smithi029.stdout:1598: seq_num 613 ranges {702691=635877,2104756=197935} 2024-04-12T11:21:10.601 INFO:tasks.rados.rados.0.smithi029.stdout:1598: writing smithi02979277-31 from 702691 to 1338568 tid 1 2024-04-12T11:21:10.607 INFO:tasks.rados.rados.0.smithi029.stdout:1598: writing smithi02979277-31 from 2104756 to 2302691 tid 2 2024-04-12T11:21:10.608 INFO:tasks.rados.rados.0.smithi029.stdout:1599: rollback oid 18 current snap is 196 2024-04-12T11:21:10.608 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 18 to 195 2024-04-12T11:21:10.608 INFO:tasks.rados.rados.0.smithi029.stdout:1595: finishing rollback tid 0 to smithi02979277-6 2024-04-12T11:21:10.608 INFO:tasks.rados.rados.0.smithi029.stdout:1595: finishing rollback tid 1 to smithi02979277-6 2024-04-12T11:21:10.608 INFO:tasks.rados.rados.0.smithi029.stdout:1596: expect (ObjNum 610 snap 195 seq_num 610) 2024-04-12T11:21:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:1595: finishing rollback tid 2 to smithi02979277-6 2024-04-12T11:21:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 396 (ObjNum 424 snap 135 seq_num 424) dirty exists 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1593: finishing rollback tid 0 to smithi02979277-38 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1592: finishing rollback tid 0 to smithi02979277-25 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1593: finishing rollback tid 1 to smithi02979277-38 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1594: finishing rollback tid 0 to smithi02979277-43 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1595: done (6 left) 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1596: done (5 left) 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1600: snap_create 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1592: finishing rollback tid 1 to smithi02979277-25 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1592: finishing rollback tid 2 to smithi02979277-25 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 826 (ObjNum 567 snap 177 seq_num 567) dirty exists 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1594: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1594: finishing rollback tid 2 to smithi02979277-43 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 644 (ObjNum 427 snap 135 seq_num 427) dirty exists 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:1593: finishing rollback tid 2 to smithi02979277-38 2024-04-12T11:21:10.716 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 468 (ObjNum 591 snap 190 seq_num 591) dirty exists 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1598: finishing write tid 1 to smithi02979277-31 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1598: finishing write tid 2 to smithi02979277-31 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1598: finishing write tid 3 to smithi02979277-31 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1599: finishing rollback tid 0 to smithi02979277-18 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1598: finishing write tid 4 to smithi02979277-31 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1598: oid 31 updating version 0 to 742 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1598: oid 31 updating version 742 to 744 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1598: oid 31 version 744 is already newer than 743 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 744 (ObjNum 613 snap 196 seq_num 613) dirty exists 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1598: left oid 31 (ObjNum 613 snap 196 seq_num 613) 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1599: finishing rollback tid 1 to smithi02979277-18 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:1599: finishing rollback tid 2 to smithi02979277-18 2024-04-12T11:21:10.717 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 830 (ObjNum 489 snap 158 seq_num 489) dirty exists 2024-04-12T11:21:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:10 smithi087 ceph-mon[89014]: pgmap v334: 89 pgs: 89 active+clean; 136 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 912 KiB/s wr, 13 op/s 2024-04-12T11:21:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:10 smithi087 ceph-mon[89014]: osdmap e466: 8 total, 8 up, 8 in 2024-04-12T11:21:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:10 smithi029 ceph-mon[97787]: pgmap v334: 89 pgs: 89 active+clean; 136 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 912 KiB/s wr, 13 op/s 2024-04-12T11:21:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:10 smithi029 ceph-mon[97787]: osdmap e466: 8 total, 8 up, 8 in 2024-04-12T11:21:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:10 smithi029 ceph-mon[93650]: pgmap v334: 89 pgs: 89 active+clean; 136 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 912 KiB/s wr, 13 op/s 2024-04-12T11:21:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:10 smithi029 ceph-mon[93650]: osdmap e466: 8 total, 8 up, 8 in 2024-04-12T11:21:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:1592: done (5 left) 2024-04-12T11:21:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:1593: done (4 left) 2024-04-12T11:21:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1594: done (3 left) 2024-04-12T11:21:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1598: done (2 left) 2024-04-12T11:21:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1599: done (1 left) 2024-04-12T11:21:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1600: done (0 left) 2024-04-12T11:21:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1601: read oid 25 snap -1 2024-04-12T11:21:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1601: expect (ObjNum 567 snap 177 seq_num 567) 2024-04-12T11:21:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1602: snap_create 2024-04-12T11:21:11.603 INFO:tasks.rados.rados.0.smithi029.stdout:1601: expect (ObjNum 567 snap 177 seq_num 567) 2024-04-12T11:21:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:11 smithi087 ceph-mon[89014]: osdmap e467: 8 total, 8 up, 8 in 2024-04-12T11:21:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:11 smithi029 ceph-mon[93650]: osdmap e467: 8 total, 8 up, 8 in 2024-04-12T11:21:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:11 smithi029 ceph-mon[97787]: osdmap e467: 8 total, 8 up, 8 in 2024-04-12T11:21:12.548 INFO:tasks.rados.rados.0.smithi029.stdout:1601: done (1 left) 2024-04-12T11:21:12.548 INFO:tasks.rados.rados.0.smithi029.stdout:1602: done (0 left) 2024-04-12T11:21:12.548 INFO:tasks.rados.rados.0.smithi029.stdout:1603: delete oid 41 current snap is 198 2024-04-12T11:21:12.553 INFO:tasks.rados.rados.0.smithi029.stdout:1603: done (0 left) 2024-04-12T11:21:12.553 INFO:tasks.rados.rados.0.smithi029.stdout:1604: write oid 41 current snap is 198 2024-04-12T11:21:12.553 INFO:tasks.rados.rados.0.smithi029.stdout:1604: seq_num 615 ranges {736305=786375,2105367=230938} 2024-04-12T11:21:12.561 INFO:tasks.rados.rados.0.smithi029.stdout:1604: writing smithi02979277-41 from 736305 to 1522680 tid 1 2024-04-12T11:21:12.566 INFO:tasks.rados.rados.0.smithi029.stdout:1604: writing smithi02979277-41 from 2105367 to 2336305 tid 2 2024-04-12T11:21:12.566 INFO:tasks.rados.rados.0.smithi029.stdout:1604: finishing write tid 1 to smithi02979277-41 2024-04-12T11:21:12.566 INFO:tasks.rados.rados.0.smithi029.stdout:1605: rollback oid 43 current snap is 198 2024-04-12T11:21:12.566 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 197 2024-04-12T11:21:12.566 INFO:tasks.rados.rados.0.smithi029.stdout:1606: snap_create 2024-04-12T11:21:12.568 INFO:tasks.rados.rados.0.smithi029.stdout:1604: finishing write tid 2 to smithi02979277-41 2024-04-12T11:21:12.568 INFO:tasks.rados.rados.0.smithi029.stdout:1604: finishing write tid 3 to smithi02979277-41 2024-04-12T11:21:12.569 INFO:tasks.rados.rados.0.smithi029.stdout:1604: finishing write tid 4 to smithi02979277-41 2024-04-12T11:21:12.569 INFO:tasks.rados.rados.0.smithi029.stdout:1604: oid 41 updating version 0 to 802 2024-04-12T11:21:12.569 INFO:tasks.rados.rados.0.smithi029.stdout:1604: oid 41 version 802 is already newer than 801 2024-04-12T11:21:12.569 INFO:tasks.rados.rados.0.smithi029.stdout:1604: oid 41 updating version 802 to 803 2024-04-12T11:21:12.569 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 803 (ObjNum 615 snap 198 seq_num 615) dirty exists 2024-04-12T11:21:12.569 INFO:tasks.rados.rados.0.smithi029.stdout:1604: left oid 41 (ObjNum 615 snap 198 seq_num 615) 2024-04-12T11:21:12.571 INFO:tasks.rados.rados.0.smithi029.stdout:1605: finishing rollback tid 0 to smithi02979277-43 2024-04-12T11:21:12.571 INFO:tasks.rados.rados.0.smithi029.stdout:1605: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:21:12.571 INFO:tasks.rados.rados.0.smithi029.stdout:1605: finishing rollback tid 2 to smithi02979277-43 2024-04-12T11:21:12.571 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 648 (ObjNum 427 snap 135 seq_num 427) dirty exists 2024-04-12T11:21:12.590 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:21:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:21:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:21:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:12 smithi087 ceph-mon[89014]: pgmap v337: 89 pgs: 89 active+clean; 136 MiB data, 604 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:12 smithi087 ceph-mon[89014]: osdmap e468: 8 total, 8 up, 8 in 2024-04-12T11:21:12.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:12 smithi087 ceph-mon[89014]: osdmap e469: 8 total, 8 up, 8 in 2024-04-12T11:21:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:12 smithi029 ceph-mon[97787]: pgmap v337: 89 pgs: 89 active+clean; 136 MiB data, 604 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:12 smithi029 ceph-mon[97787]: osdmap e468: 8 total, 8 up, 8 in 2024-04-12T11:21:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:12 smithi029 ceph-mon[97787]: osdmap e469: 8 total, 8 up, 8 in 2024-04-12T11:21:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:12 smithi029 ceph-mon[93650]: pgmap v337: 89 pgs: 89 active+clean; 136 MiB data, 604 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:12 smithi029 ceph-mon[93650]: osdmap e468: 8 total, 8 up, 8 in 2024-04-12T11:21:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:12 smithi029 ceph-mon[93650]: osdmap e469: 8 total, 8 up, 8 in 2024-04-12T11:21:13.550 INFO:tasks.rados.rados.0.smithi029.stdout:1604: done (2 left) 2024-04-12T11:21:13.550 INFO:tasks.rados.rados.0.smithi029.stdout:1605: done (1 left) 2024-04-12T11:21:13.550 INFO:tasks.rados.rados.0.smithi029.stdout:1606: done (0 left) 2024-04-12T11:21:13.550 INFO:tasks.rados.rados.0.smithi029.stdout:1607: write oid 37 current snap is 199 2024-04-12T11:21:13.550 INFO:tasks.rados.rados.0.smithi029.stdout:1607: seq_num 616 ranges {753112=577977,1880202=472910} 2024-04-12T11:21:13.557 INFO:tasks.rados.rados.0.smithi029.stdout:1607: writing smithi02979277-37 from 753112 to 1331089 tid 1 2024-04-12T11:21:13.564 INFO:tasks.rados.rados.0.smithi029.stdout:1607: writing smithi02979277-37 from 1880202 to 2353112 tid 2 2024-04-12T11:21:13.564 INFO:tasks.rados.rados.0.smithi029.stdout:1608: snap_remove snap 196 2024-04-12T11:21:13.966 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:21:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:21:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi029\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", domainname=\"(none)\", instance=\"smithi029\", job=\"node\", machine=\"x86_64\", nodename=\"smithi029\", release=\"5.14.0-435.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:21:14.555 INFO:tasks.rados.rados.0.smithi029.stdout:1608: done (1 left) 2024-04-12T11:21:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:1609: read oid 48 snap -1 2024-04-12T11:21:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:1609: expect deleted 2024-04-12T11:21:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:1610: read oid 19 snap -1 2024-04-12T11:21:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:1610: expect deleted 2024-04-12T11:21:14.556 INFO:tasks.rados.rados.0.smithi029.stdout:1611: snap_remove snap 195 2024-04-12T11:21:14.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:14 smithi029 ceph-mon[97787]: pgmap v340: 89 pgs: 89 active+clean; 112 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 425 KiB/s wr, 9 op/s 2024-04-12T11:21:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:14 smithi029 ceph-mon[97787]: osdmap e470: 8 total, 8 up, 8 in 2024-04-12T11:21:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:14 smithi029 ceph-mon[93650]: pgmap v340: 89 pgs: 89 active+clean; 112 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 425 KiB/s wr, 9 op/s 2024-04-12T11:21:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:14 smithi029 ceph-mon[93650]: osdmap e470: 8 total, 8 up, 8 in 2024-04-12T11:21:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:14 smithi087 ceph-mon[89014]: pgmap v340: 89 pgs: 89 active+clean; 112 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 425 KiB/s wr, 9 op/s 2024-04-12T11:21:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:14 smithi087 ceph-mon[89014]: osdmap e470: 8 total, 8 up, 8 in 2024-04-12T11:21:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:21:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:21:14] "GET /metrics HTTP/1.1" 200 36248 "" "Prometheus/2.43.0" 2024-04-12T11:21:15.561 INFO:tasks.rados.rados.0.smithi029.stdout:1611: done (3 left) 2024-04-12T11:21:15.610 INFO:tasks.rados.rados.0.smithi029.stdout:1612: write oid 41 current snap is 199 2024-04-12T11:21:15.610 INFO:tasks.rados.rados.0.smithi029.stdout:1607: finishing write tid 1 to smithi02979277-37 2024-04-12T11:21:15.610 INFO:tasks.rados.rados.0.smithi029.stdout:1612: seq_num 617 ranges {769919=653226,1938684=431235} 2024-04-12T11:21:15.611 INFO:tasks.rados.rados.0.smithi029.stdout:1612: writing smithi02979277-41 from 769919 to 1423145 tid 1 2024-04-12T11:21:15.611 INFO:tasks.rados.rados.0.smithi029.stdout:1612: writing smithi02979277-41 from 1938684 to 2369919 tid 2 2024-04-12T11:21:15.611 INFO:tasks.rados.rados.0.smithi029.stdout:1613: snap_remove snap 198 2024-04-12T11:21:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:15 smithi087 ceph-mon[89014]: osdmap e471: 8 total, 8 up, 8 in 2024-04-12T11:21:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:15 smithi029 ceph-mon[93650]: osdmap e471: 8 total, 8 up, 8 in 2024-04-12T11:21:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:15 smithi029 ceph-mon[97787]: osdmap e471: 8 total, 8 up, 8 in 2024-04-12T11:21:16.566 INFO:tasks.rados.rados.0.smithi029.stdout:1613: done (4 left) 2024-04-12T11:21:16.566 INFO:tasks.rados.rados.0.smithi029.stdout:1614: read oid 1 snap -1 2024-04-12T11:21:16.566 INFO:tasks.rados.rados.0.smithi029.stdout:1614: expect (ObjNum 509 snap 164 seq_num 509) 2024-04-12T11:21:16.566 INFO:tasks.rados.rados.0.smithi029.stdout:1615: snap_create 2024-04-12T11:21:16.566 INFO:tasks.rados.rados.0.smithi029.stdout:1607: finishing write tid 2 to smithi02979277-37 2024-04-12T11:21:16.566 INFO:tasks.rados.rados.0.smithi029.stdout:1607: finishing write tid 3 to smithi02979277-37 2024-04-12T11:21:16.566 INFO:tasks.rados.rados.0.smithi029.stdout:1607: finishing write tid 4 to smithi02979277-37 2024-04-12T11:21:16.566 INFO:tasks.rados.rados.0.smithi029.stdout:1607: oid 37 updating version 0 to 631 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1607: oid 37 updating version 631 to 632 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1607: oid 37 version 632 is already newer than 630 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 632 (ObjNum 616 snap 199 seq_num 616) dirty exists 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1607: left oid 37 (ObjNum 616 snap 199 seq_num 616) 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1612: finishing write tid 1 to smithi02979277-41 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1612: finishing write tid 2 to smithi02979277-41 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1612: finishing write tid 3 to smithi02979277-41 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1612: finishing write tid 4 to smithi02979277-41 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1612: oid 41 updating version 0 to 807 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1612: oid 41 updating version 807 to 810 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1612: oid 41 updating version 810 to 811 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 811 (ObjNum 617 snap 199 seq_num 617) dirty exists 2024-04-12T11:21:16.567 INFO:tasks.rados.rados.0.smithi029.stdout:1612: left oid 41 (ObjNum 617 snap 199 seq_num 617) 2024-04-12T11:21:16.578 INFO:tasks.rados.rados.0.smithi029.stdout:1614: expect (ObjNum 509 snap 164 seq_num 509) 2024-04-12T11:21:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:16 smithi087 ceph-mon[89014]: pgmap v343: 89 pgs: 89 active+clean; 112 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 425 KiB/s wr, 9 op/s 2024-04-12T11:21:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:16 smithi087 ceph-mon[89014]: osdmap e472: 8 total, 8 up, 8 in 2024-04-12T11:21:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:16 smithi029 ceph-mon[93650]: pgmap v343: 89 pgs: 89 active+clean; 112 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 425 KiB/s wr, 9 op/s 2024-04-12T11:21:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:16 smithi029 ceph-mon[93650]: osdmap e472: 8 total, 8 up, 8 in 2024-04-12T11:21:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:16 smithi029 ceph-mon[97787]: pgmap v343: 89 pgs: 89 active+clean; 112 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 425 KiB/s wr, 9 op/s 2024-04-12T11:21:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:16 smithi029 ceph-mon[97787]: osdmap e472: 8 total, 8 up, 8 in 2024-04-12T11:21:17.571 INFO:tasks.rados.rados.0.smithi029.stdout:1607: done (5 left) 2024-04-12T11:21:17.571 INFO:tasks.rados.rados.0.smithi029.stdout:1609: done (4 left) 2024-04-12T11:21:17.571 INFO:tasks.rados.rados.0.smithi029.stdout:1610: done (3 left) 2024-04-12T11:21:17.571 INFO:tasks.rados.rados.0.smithi029.stdout:1612: done (2 left) 2024-04-12T11:21:17.571 INFO:tasks.rados.rados.0.smithi029.stdout:1614: done (1 left) 2024-04-12T11:21:17.571 INFO:tasks.rados.rados.0.smithi029.stdout:1615: done (0 left) 2024-04-12T11:21:17.571 INFO:tasks.rados.rados.0.smithi029.stdout:1616: snap_remove snap 197 2024-04-12T11:21:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:17 smithi087 ceph-mon[89014]: osdmap e473: 8 total, 8 up, 8 in 2024-04-12T11:21:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:17 smithi029 ceph-mon[93650]: osdmap e473: 8 total, 8 up, 8 in 2024-04-12T11:21:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:17 smithi029 ceph-mon[97787]: osdmap e473: 8 total, 8 up, 8 in 2024-04-12T11:21:18.573 INFO:tasks.rados.rados.0.smithi029.stdout:1616: done (0 left) 2024-04-12T11:21:18.573 INFO:tasks.rados.rados.0.smithi029.stdout:1617: read oid 34 snap -1 2024-04-12T11:21:18.573 INFO:tasks.rados.rados.0.smithi029.stdout:1617: expect (ObjNum 561 snap 177 seq_num 561) 2024-04-12T11:21:18.573 INFO:tasks.rados.rados.0.smithi029.stdout:1618: read oid 35 snap -1 2024-04-12T11:21:18.573 INFO:tasks.rados.rados.0.smithi029.stdout:1618: expect deleted 2024-04-12T11:21:18.574 INFO:tasks.rados.rados.0.smithi029.stdout:1619: read oid 30 snap -1 2024-04-12T11:21:18.574 INFO:tasks.rados.rados.0.smithi029.stdout:1619: expect deleted 2024-04-12T11:21:18.574 INFO:tasks.rados.rados.0.smithi029.stdout:1620: delete oid 14 current snap is 200 2024-04-12T11:21:18.582 INFO:tasks.rados.rados.0.smithi029.stdout:1618: done (3 left) 2024-04-12T11:21:18.582 INFO:tasks.rados.rados.0.smithi029.stdout:1619: done (2 left) 2024-04-12T11:21:18.582 INFO:tasks.rados.rados.0.smithi029.stdout:1620: done (1 left) 2024-04-12T11:21:18.582 INFO:tasks.rados.rados.0.smithi029.stdout:1621: delete oid 26 current snap is 200 2024-04-12T11:21:18.583 INFO:tasks.rados.rados.0.smithi029.stdout:1617: expect (ObjNum 561 snap 177 seq_num 561) 2024-04-12T11:21:18.771 INFO:tasks.rados.rados.0.smithi029.stdout:1617: done (1 left) 2024-04-12T11:21:18.771 INFO:tasks.rados.rados.0.smithi029.stdout:1621: done (0 left) 2024-04-12T11:21:18.771 INFO:tasks.rados.rados.0.smithi029.stdout:1622: read oid 13 snap -1 2024-04-12T11:21:18.771 INFO:tasks.rados.rados.0.smithi029.stdout:1622: expect deleted 2024-04-12T11:21:18.771 INFO:tasks.rados.rados.0.smithi029.stdout:1623: read oid 46 snap -1 2024-04-12T11:21:18.772 INFO:tasks.rados.rados.0.smithi029.stdout:1623: expect (ObjNum 487 snap 158 seq_num 487) 2024-04-12T11:21:18.772 INFO:tasks.rados.rados.0.smithi029.stdout:1624: write oid 39 current snap is 200 2024-04-12T11:21:18.772 INFO:tasks.rados.rados.0.smithi029.stdout:1624: seq_num 620 ranges {420340=478973,1597777=512212} 2024-04-12T11:21:18.775 INFO:tasks.rados.rados.0.smithi029.stdout:1624: writing smithi02979277-39 from 420340 to 899313 tid 1 2024-04-12T11:21:18.782 INFO:tasks.rados.rados.0.smithi029.stdout:1624: writing smithi02979277-39 from 1597777 to 2109989 tid 2 2024-04-12T11:21:18.782 INFO:tasks.rados.rados.0.smithi029.stdout:1625: delete oid 22 current snap is 200 2024-04-12T11:21:18.782 INFO:tasks.rados.rados.0.smithi029.stdout:1623: expect (ObjNum 487 snap 158 seq_num 487) 2024-04-12T11:21:18.914 INFO:tasks.rados.rados.0.smithi029.stdout:1624: finishing write tid 1 to smithi02979277-39 2024-04-12T11:21:18.914 INFO:tasks.rados.rados.0.smithi029.stdout:1624: finishing write tid 2 to smithi02979277-39 2024-04-12T11:21:18.914 INFO:tasks.rados.rados.0.smithi029.stdout:1624: finishing write tid 3 to smithi02979277-39 2024-04-12T11:21:18.914 INFO:tasks.rados.rados.0.smithi029.stdout:1624: finishing write tid 4 to smithi02979277-39 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1624: oid 39 updating version 0 to 191 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1624: oid 39 updating version 191 to 193 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1624: oid 39 version 193 is already newer than 192 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 193 (ObjNum 620 snap 200 seq_num 620) dirty exists 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1624: left oid 39 (ObjNum 620 snap 200 seq_num 620) 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1622: done (3 left) 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1623: done (2 left) 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1624: done (1 left) 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1625: done (0 left) 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1626: read oid 36 snap 199 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1626: expect deleted 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1627: read oid 50 snap -1 2024-04-12T11:21:18.915 INFO:tasks.rados.rados.0.smithi029.stdout:1627: expect (ObjNum 587 snap 189 seq_num 587) 2024-04-12T11:21:18.916 INFO:tasks.rados.rados.0.smithi029.stdout:1628: snap_create 2024-04-12T11:21:18.928 INFO:tasks.rados.rados.0.smithi029.stdout:1627: expect (ObjNum 587 snap 189 seq_num 587) 2024-04-12T11:21:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:18 smithi087 ceph-mon[89014]: pgmap v346: 89 pgs: 89 active+clean; 113 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 770 KiB/s wr, 6 op/s 2024-04-12T11:21:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:18 smithi087 ceph-mon[89014]: osdmap e474: 8 total, 8 up, 8 in 2024-04-12T11:21:19.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:18 smithi029 ceph-mon[93650]: pgmap v346: 89 pgs: 89 active+clean; 113 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 770 KiB/s wr, 6 op/s 2024-04-12T11:21:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:18 smithi029 ceph-mon[93650]: osdmap e474: 8 total, 8 up, 8 in 2024-04-12T11:21:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:18 smithi029 ceph-mon[97787]: pgmap v346: 89 pgs: 89 active+clean; 113 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 770 KiB/s wr, 6 op/s 2024-04-12T11:21:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:18 smithi029 ceph-mon[97787]: osdmap e474: 8 total, 8 up, 8 in 2024-04-12T11:21:19.585 INFO:tasks.rados.rados.0.smithi029.stdout:1626: done (2 left) 2024-04-12T11:21:19.585 INFO:tasks.rados.rados.0.smithi029.stdout:1627: done (1 left) 2024-04-12T11:21:19.585 INFO:tasks.rados.rados.0.smithi029.stdout:1628: done (0 left) 2024-04-12T11:21:19.585 INFO:tasks.rados.rados.0.smithi029.stdout:1629: read oid 48 snap -1 2024-04-12T11:21:19.585 INFO:tasks.rados.rados.0.smithi029.stdout:1629: expect deleted 2024-04-12T11:21:19.585 INFO:tasks.rados.rados.0.smithi029.stdout:1630: write oid 22 current snap is 201 2024-04-12T11:21:19.585 INFO:tasks.rados.rados.0.smithi029.stdout:1630: seq_num 622 ranges {453954=629471,1598388=515881} 2024-04-12T11:21:19.590 INFO:tasks.rados.rados.0.smithi029.stdout:1630: writing smithi02979277-22 from 453954 to 1083425 tid 1 2024-04-12T11:21:19.596 INFO:tasks.rados.rados.0.smithi029.stdout:1630: writing smithi02979277-22 from 1598388 to 2114269 tid 2 2024-04-12T11:21:19.596 INFO:tasks.rados.rados.0.smithi029.stdout:1631: snap_remove snap 199 2024-04-12T11:21:19.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:19 smithi087 ceph-mon[89014]: osdmap e475: 8 total, 8 up, 8 in 2024-04-12T11:21:20.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:19 smithi029 ceph-mon[93650]: osdmap e475: 8 total, 8 up, 8 in 2024-04-12T11:21:20.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:19 smithi029 ceph-mon[97787]: osdmap e475: 8 total, 8 up, 8 in 2024-04-12T11:21:20.592 INFO:tasks.rados.rados.0.smithi029.stdout:1631: done (2 left) 2024-04-12T11:21:20.592 INFO:tasks.rados.rados.0.smithi029.stdout:1632: read oid 28 snap -1 2024-04-12T11:21:20.592 INFO:tasks.rados.rados.0.smithi029.stdout:1632: expect deleted 2024-04-12T11:21:20.592 INFO:tasks.rados.rados.0.smithi029.stdout:1633: snap_remove snap 200 2024-04-12T11:21:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:20 smithi087 ceph-mon[89014]: pgmap v349: 89 pgs: 89 active+clean; 113 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 770 KiB/s wr, 6 op/s 2024-04-12T11:21:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:20 smithi087 ceph-mon[89014]: osdmap e476: 8 total, 8 up, 8 in 2024-04-12T11:21:21.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:20 smithi029 ceph-mon[93650]: pgmap v349: 89 pgs: 89 active+clean; 113 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 770 KiB/s wr, 6 op/s 2024-04-12T11:21:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:20 smithi029 ceph-mon[93650]: osdmap e476: 8 total, 8 up, 8 in 2024-04-12T11:21:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:20 smithi029 ceph-mon[97787]: pgmap v349: 89 pgs: 89 active+clean; 113 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 770 KiB/s wr, 6 op/s 2024-04-12T11:21:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:20 smithi029 ceph-mon[97787]: osdmap e476: 8 total, 8 up, 8 in 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:1633: done (3 left) 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:1634: delete oid 41 current snap is 201 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:1630: finishing write tid 1 to smithi02979277-22 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:1630: finishing write tid 2 to smithi02979277-22 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:1630: finishing write tid 3 to smithi02979277-22 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:1630: finishing write tid 4 to smithi02979277-22 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:1630: oid 22 updating version 0 to 564 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:1630: oid 22 updating version 564 to 565 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:1630: oid 22 version 565 is already newer than 563 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 565 (ObjNum 622 snap 201 seq_num 622) dirty exists 2024-04-12T11:21:21.597 INFO:tasks.rados.rados.0.smithi029.stdout:1630: left oid 22 (ObjNum 622 snap 201 seq_num 622) 2024-04-12T11:21:21.602 INFO:tasks.rados.rados.0.smithi029.stdout:1629: done (3 left) 2024-04-12T11:21:21.602 INFO:tasks.rados.rados.0.smithi029.stdout:1630: done (2 left) 2024-04-12T11:21:21.603 INFO:tasks.rados.rados.0.smithi029.stdout:1632: done (1 left) 2024-04-12T11:21:21.603 INFO:tasks.rados.rados.0.smithi029.stdout:1634: done (0 left) 2024-04-12T11:21:21.603 INFO:tasks.rados.rados.0.smithi029.stdout:1635: read oid 37 snap -1 2024-04-12T11:21:21.603 INFO:tasks.rados.rados.0.smithi029.stdout:1635: expect (ObjNum 616 snap 199 seq_num 616) 2024-04-12T11:21:21.603 INFO:tasks.rados.rados.0.smithi029.stdout:1636: snap_create 2024-04-12T11:21:21.606 INFO:tasks.rados.rados.0.smithi029.stdout:1635: expect (ObjNum 616 snap 199 seq_num 616) 2024-04-12T11:21:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:21 smithi087 ceph-mon[89014]: osdmap e477: 8 total, 8 up, 8 in 2024-04-12T11:21:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:21 smithi029 ceph-mon[93650]: osdmap e477: 8 total, 8 up, 8 in 2024-04-12T11:21:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:21 smithi029 ceph-mon[97787]: osdmap e477: 8 total, 8 up, 8 in 2024-04-12T11:21:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:22.551 INFO:tasks.rados.rados.0.smithi029.stdout:1635: done (1 left) 2024-04-12T11:21:22.551 INFO:tasks.rados.rados.0.smithi029.stdout:1636: done (0 left) 2024-04-12T11:21:22.551 INFO:tasks.rados.rados.0.smithi029.stdout:1637: rollback oid 21 current snap is 202 2024-04-12T11:21:22.551 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 21 to 201 2024-04-12T11:21:22.551 INFO:tasks.rados.rados.0.smithi029.stdout:1638: rollback oid 48 current snap is 202 2024-04-12T11:21:22.551 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 201 2024-04-12T11:21:22.551 INFO:tasks.rados.rados.0.smithi029.stdout:1639: snap_create 2024-04-12T11:21:22.553 INFO:tasks.rados.rados.0.smithi029.stdout:1638: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:21:22.553 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 0 (ObjNum 289 snap 0 seq_num 862916346) dirty dne 2024-04-12T11:21:22.556 INFO:tasks.rados.rados.0.smithi029.stdout:1637: finishing rollback tid 0 to smithi02979277-21 2024-04-12T11:21:22.556 INFO:tasks.rados.rados.0.smithi029.stdout:1637: finishing rollback tid 1 to smithi02979277-21 2024-04-12T11:21:22.557 INFO:tasks.rados.rados.0.smithi029.stdout:1637: finishing rollback tid 2 to smithi02979277-21 2024-04-12T11:21:22.557 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 820 (ObjNum 579 snap 185 seq_num 579) dirty exists 2024-04-12T11:21:22.595 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:21:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:21:22.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:21:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:22 smithi087 ceph-mon[89014]: pgmap v352: 89 pgs: 89 active+clean; 113 MiB data, 570 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:22.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:22 smithi087 ceph-mon[89014]: osdmap e478: 8 total, 8 up, 8 in 2024-04-12T11:21:22.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:22 smithi087 ceph-mon[89014]: osdmap e479: 8 total, 8 up, 8 in 2024-04-12T11:21:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:22 smithi029 ceph-mon[93650]: pgmap v352: 89 pgs: 89 active+clean; 113 MiB data, 570 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:22 smithi029 ceph-mon[93650]: osdmap e478: 8 total, 8 up, 8 in 2024-04-12T11:21:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:22 smithi029 ceph-mon[93650]: osdmap e479: 8 total, 8 up, 8 in 2024-04-12T11:21:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:22 smithi029 ceph-mon[97787]: pgmap v352: 89 pgs: 89 active+clean; 113 MiB data, 570 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:22 smithi029 ceph-mon[97787]: osdmap e478: 8 total, 8 up, 8 in 2024-04-12T11:21:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:22 smithi029 ceph-mon[97787]: osdmap e479: 8 total, 8 up, 8 in 2024-04-12T11:21:23.552 INFO:tasks.rados.rados.0.smithi029.stdout:1637: done (2 left) 2024-04-12T11:21:23.552 INFO:tasks.rados.rados.0.smithi029.stdout:1638: done (1 left) 2024-04-12T11:21:23.553 INFO:tasks.rados.rados.0.smithi029.stdout:1639: done (0 left) 2024-04-12T11:21:23.553 INFO:tasks.rados.rados.0.smithi029.stdout:1640: write oid 40 current snap is 203 2024-04-12T11:21:23.553 INFO:tasks.rados.rados.0.smithi029.stdout:1640: seq_num 624 ranges {487568=496322,1431705=519550} 2024-04-12T11:21:23.557 INFO:tasks.rados.rados.0.smithi029.stdout:1640: writing smithi02979277-40 from 487568 to 983890 tid 1 2024-04-12T11:21:23.563 INFO:tasks.rados.rados.0.smithi029.stdout:1640: writing smithi02979277-40 from 1431705 to 1951255 tid 2 2024-04-12T11:21:23.563 INFO:tasks.rados.rados.0.smithi029.stdout:1641: rollback oid 8 current snap is 203 2024-04-12T11:21:23.563 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 201 2024-04-12T11:21:23.563 INFO:tasks.rados.rados.0.smithi029.stdout:1642: write oid 47 current snap is 203 2024-04-12T11:21:23.564 INFO:tasks.rados.rados.0.smithi029.stdout:1642: seq_num 625 ranges {504375=571571,1490187=663208} 2024-04-12T11:21:23.569 INFO:tasks.rados.rados.0.smithi029.stdout:1642: writing smithi02979277-47 from 504375 to 1075946 tid 1 2024-04-12T11:21:23.576 INFO:tasks.rados.rados.0.smithi029.stdout:1642: writing smithi02979277-47 from 1490187 to 2153395 tid 2 2024-04-12T11:21:23.576 INFO:tasks.rados.rados.0.smithi029.stdout:1643: read oid 7 snap -1 2024-04-12T11:21:23.576 INFO:tasks.rados.rados.0.smithi029.stdout:1643: expect deleted 2024-04-12T11:21:23.576 INFO:tasks.rados.rados.0.smithi029.stdout:1644: write oid 13 current snap is 203 2024-04-12T11:21:23.576 INFO:tasks.rados.rados.0.smithi029.stdout:1640: finishing write tid 1 to smithi02979277-40 2024-04-12T11:21:23.576 INFO:tasks.rados.rados.0.smithi029.stdout:1641: finishing rollback tid 0 to smithi02979277-8 2024-04-12T11:21:23.576 INFO:tasks.rados.rados.0.smithi029.stdout:1641: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:21:23.576 INFO:tasks.rados.rados.0.smithi029.stdout:1641: finishing rollback tid 2 to smithi02979277-8 2024-04-12T11:21:23.576 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 164 (ObjNum 601 snap 192 seq_num 601) dirty exists 2024-04-12T11:21:23.577 INFO:tasks.rados.rados.0.smithi029.stdout:1640: finishing write tid 2 to smithi02979277-40 2024-04-12T11:21:23.577 INFO:tasks.rados.rados.0.smithi029.stdout:1644: seq_num 626 ranges {521182=646820,1948669=523219} 2024-04-12T11:21:23.582 INFO:tasks.rados.rados.0.smithi029.stdout:1644: writing smithi02979277-13 from 521182 to 1168002 tid 1 2024-04-12T11:21:23.589 INFO:tasks.rados.rados.0.smithi029.stdout:1644: writing smithi02979277-13 from 1948669 to 2471888 tid 2 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:1641: done (4 left) 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:1645: write oid 23 current snap is 203 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:1640: finishing write tid 3 to smithi02979277-40 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:1640: finishing write tid 4 to smithi02979277-40 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:1640: oid 40 updating version 0 to 1067 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:1640: oid 40 version 1067 is already newer than 1066 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:1640: oid 40 updating version 1067 to 1068 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 1068 (ObjNum 624 snap 203 seq_num 624) dirty exists 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:1640: left oid 40 (ObjNum 624 snap 203 seq_num 624) 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:1642: finishing write tid 1 to smithi02979277-47 2024-04-12T11:21:23.590 INFO:tasks.rados.rados.0.smithi029.stdout:1645: seq_num 627 ranges {537989=722069,1890798=647191} 2024-04-12T11:21:23.597 INFO:tasks.rados.rados.0.smithi029.stdout:1645: writing smithi02979277-23 from 537989 to 1260058 tid 1 2024-04-12T11:21:23.604 INFO:tasks.rados.rados.0.smithi029.stdout:1645: writing smithi02979277-23 from 1890798 to 2537989 tid 2 2024-04-12T11:21:23.604 INFO:tasks.rados.rados.0.smithi029.stdout:1640: done (4 left) 2024-04-12T11:21:23.604 INFO:tasks.rados.rados.0.smithi029.stdout:1646: read oid 36 snap -1 2024-04-12T11:21:23.604 INFO:tasks.rados.rados.0.smithi029.stdout:1646: expect deleted 2024-04-12T11:21:23.604 INFO:tasks.rados.rados.0.smithi029.stdout:1647: snap_remove snap 202 2024-04-12T11:21:24.601 INFO:tasks.rados.rados.0.smithi029.stdout:1647: done (5 left) 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:1648: rollback oid 9 current snap is 203 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 9 to 201 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:1642: finishing write tid 2 to smithi02979277-47 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:1642: finishing write tid 3 to smithi02979277-47 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:1642: finishing write tid 4 to smithi02979277-47 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:1642: oid 47 updating version 0 to 781 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:1642: oid 47 updating version 781 to 782 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:1642: oid 47 updating version 782 to 783 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 783 (ObjNum 625 snap 203 seq_num 625) dirty exists 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:1642: left oid 47 (ObjNum 625 snap 203 seq_num 625) 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:1644: finishing write tid 1 to smithi02979277-13 2024-04-12T11:21:24.602 INFO:tasks.rados.rados.0.smithi029.stdout:1644: finishing write tid 2 to smithi02979277-13 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1644: finishing write tid 3 to smithi02979277-13 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1644: finishing write tid 4 to smithi02979277-13 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1644: oid 13 updating version 0 to 835 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1644: oid 13 updating version 835 to 836 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1644: oid 13 updating version 836 to 837 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 837 (ObjNum 626 snap 203 seq_num 626) dirty exists 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1644: left oid 13 (ObjNum 626 snap 203 seq_num 626) 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1645: finishing write tid 1 to smithi02979277-23 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1645: finishing write tid 2 to smithi02979277-23 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1645: finishing write tid 3 to smithi02979277-23 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1645: finishing write tid 4 to smithi02979277-23 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1645: oid 23 updating version 0 to 571 2024-04-12T11:21:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:1645: oid 23 version 571 is already newer than 570 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:1645: oid 23 version 571 is already newer than 569 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 571 (ObjNum 627 snap 203 seq_num 627) dirty exists 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:1645: left oid 23 (ObjNum 627 snap 203 seq_num 627) 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:1642: done (5 left) 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:1643: done (4 left) 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:1644: done (3 left) 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:1645: done (2 left) 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:1646: done (1 left) 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:1649: read oid 21 snap -1 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:1649: expect (ObjNum 579 snap 185 seq_num 579) 2024-04-12T11:21:24.604 INFO:tasks.rados.rados.0.smithi029.stdout:1650: snap_create 2024-04-12T11:21:24.605 INFO:tasks.rados.rados.0.smithi029.stdout:1648: finishing rollback tid 1 to smithi02979277-9 2024-04-12T11:21:24.605 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:21:24.613 INFO:tasks.rados.rados.0.smithi029.stdout:1649: expect (ObjNum 579 snap 185 seq_num 579) 2024-04-12T11:21:24.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:24 smithi029 ceph-mon[97787]: pgmap v355: 89 pgs: 89 active+clean; 100 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 522 KiB/s wr, 12 op/s 2024-04-12T11:21:24.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:24 smithi029 ceph-mon[97787]: osdmap e480: 8 total, 8 up, 8 in 2024-04-12T11:21:24.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:24 smithi029 ceph-mon[93650]: pgmap v355: 89 pgs: 89 active+clean; 100 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 522 KiB/s wr, 12 op/s 2024-04-12T11:21:24.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:24 smithi029 ceph-mon[93650]: osdmap e480: 8 total, 8 up, 8 in 2024-04-12T11:21:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:24 smithi087 ceph-mon[89014]: pgmap v355: 89 pgs: 89 active+clean; 100 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 522 KiB/s wr, 12 op/s 2024-04-12T11:21:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:24 smithi087 ceph-mon[89014]: osdmap e480: 8 total, 8 up, 8 in 2024-04-12T11:21:25.085 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:21:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:21:24] "GET /metrics HTTP/1.1" 200 36243 "" "Prometheus/2.43.0" 2024-04-12T11:21:25.604 INFO:tasks.rados.rados.0.smithi029.stdout:1648: done (2 left) 2024-04-12T11:21:25.604 INFO:tasks.rados.rados.0.smithi029.stdout:1649: done (1 left) 2024-04-12T11:21:25.604 INFO:tasks.rados.rados.0.smithi029.stdout:1650: done (0 left) 2024-04-12T11:21:25.604 INFO:tasks.rados.rados.0.smithi029.stdout:1651: delete oid 27 current snap is 204 2024-04-12T11:21:25.607 INFO:tasks.rados.rados.0.smithi029.stdout:1651: done (0 left) 2024-04-12T11:21:25.608 INFO:tasks.rados.rados.0.smithi029.stdout:1652: delete oid 15 current snap is 204 2024-04-12T11:21:25.609 INFO:tasks.rados.rados.0.smithi029.stdout:1652: done (0 left) 2024-04-12T11:21:25.609 INFO:tasks.rados.rados.0.smithi029.stdout:1653: snap_remove snap 203 2024-04-12T11:21:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:25 smithi087 ceph-mon[89014]: osdmap e481: 8 total, 8 up, 8 in 2024-04-12T11:21:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:25 smithi029 ceph-mon[97787]: osdmap e481: 8 total, 8 up, 8 in 2024-04-12T11:21:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:25 smithi029 ceph-mon[93650]: osdmap e481: 8 total, 8 up, 8 in 2024-04-12T11:21:26.609 INFO:tasks.rados.rados.0.smithi029.stdout:1653: done (0 left) 2024-04-12T11:21:26.609 INFO:tasks.rados.rados.0.smithi029.stdout:1654: read oid 37 snap 201 2024-04-12T11:21:26.609 INFO:tasks.rados.rados.0.smithi029.stdout:1654: expect (ObjNum 616 snap 199 seq_num 616) 2024-04-12T11:21:26.609 INFO:tasks.rados.rados.0.smithi029.stdout:1655: rollback oid 14 current snap is 204 2024-04-12T11:21:26.609 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 201 2024-04-12T11:21:26.609 INFO:tasks.rados.rados.0.smithi029.stdout:1656: write oid 30 current snap is 204 2024-04-12T11:21:26.610 INFO:tasks.rados.rados.0.smithi029.stdout:1656: seq_num 630 ranges {588410=547816,1666244=414204,2484603=103807} 2024-04-12T11:21:26.615 INFO:tasks.rados.rados.0.smithi029.stdout:1656: writing smithi02979277-30 from 588410 to 1136226 tid 1 2024-04-12T11:21:26.620 INFO:tasks.rados.rados.0.smithi029.stdout:1656: writing smithi02979277-30 from 1666244 to 2080448 tid 2 2024-04-12T11:21:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:1656: writing smithi02979277-30 from 2484603 to 2588410 tid 3 2024-04-12T11:21:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:1657: delete oid 9 current snap is 204 2024-04-12T11:21:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:1655: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:21:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 0 (ObjNum 33 snap 0 seq_num 1800936016) dirty dne 2024-04-12T11:21:26.624 INFO:tasks.rados.rados.0.smithi029.stdout:1654: expect (ObjNum 616 snap 199 seq_num 616) 2024-04-12T11:21:26.731 INFO:tasks.rados.rados.0.smithi029.stdout:1656: finishing write tid 1 to smithi02979277-30 2024-04-12T11:21:26.731 INFO:tasks.rados.rados.0.smithi029.stdout:1656: finishing write tid 2 to smithi02979277-30 2024-04-12T11:21:26.732 INFO:tasks.rados.rados.0.smithi029.stdout:1656: finishing write tid 3 to smithi02979277-30 2024-04-12T11:21:26.732 INFO:tasks.rados.rados.0.smithi029.stdout:1656: finishing write tid 4 to smithi02979277-30 2024-04-12T11:21:26.732 INFO:tasks.rados.rados.0.smithi029.stdout:1656: finishing write tid 5 to smithi02979277-30 2024-04-12T11:21:26.732 INFO:tasks.rados.rados.0.smithi029.stdout:1656: oid 30 updating version 0 to 684 2024-04-12T11:21:26.732 INFO:tasks.rados.rados.0.smithi029.stdout:1656: oid 30 version 684 is already newer than 677 2024-04-12T11:21:26.732 INFO:tasks.rados.rados.0.smithi029.stdout:1656: oid 30 version 684 is already newer than 682 2024-04-12T11:21:26.733 INFO:tasks.rados.rados.0.smithi029.stdout:1656: oid 30 version 684 is already newer than 683 2024-04-12T11:21:26.733 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 684 (ObjNum 630 snap 204 seq_num 630) dirty exists 2024-04-12T11:21:26.733 INFO:tasks.rados.rados.0.smithi029.stdout:1656: left oid 30 (ObjNum 630 snap 204 seq_num 630) 2024-04-12T11:21:26.733 INFO:tasks.rados.rados.0.smithi029.stdout:1654: done (3 left) 2024-04-12T11:21:26.733 INFO:tasks.rados.rados.0.smithi029.stdout:1655: done (2 left) 2024-04-12T11:21:26.733 INFO:tasks.rados.rados.0.smithi029.stdout:1656: done (1 left) 2024-04-12T11:21:26.733 INFO:tasks.rados.rados.0.smithi029.stdout:1657: done (0 left) 2024-04-12T11:21:26.734 INFO:tasks.rados.rados.0.smithi029.stdout:1658: snap_create 2024-04-12T11:21:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:26 smithi087 ceph-mon[89014]: pgmap v358: 89 pgs: 89 active+clean; 100 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 522 KiB/s wr, 12 op/s 2024-04-12T11:21:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:26 smithi087 ceph-mon[89014]: osdmap e482: 8 total, 8 up, 8 in 2024-04-12T11:21:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:26 smithi029 ceph-mon[93650]: pgmap v358: 89 pgs: 89 active+clean; 100 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 522 KiB/s wr, 12 op/s 2024-04-12T11:21:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:26 smithi029 ceph-mon[93650]: osdmap e482: 8 total, 8 up, 8 in 2024-04-12T11:21:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:26 smithi029 ceph-mon[97787]: pgmap v358: 89 pgs: 89 active+clean; 100 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 522 KiB/s wr, 12 op/s 2024-04-12T11:21:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:26 smithi029 ceph-mon[97787]: osdmap e482: 8 total, 8 up, 8 in 2024-04-12T11:21:27.615 INFO:tasks.rados.rados.0.smithi029.stdout:1658: done (0 left) 2024-04-12T11:21:27.615 INFO:tasks.rados.rados.0.smithi029.stdout:1659: write oid 33 current snap is 205 2024-04-12T11:21:27.615 INFO:tasks.rados.rados.0.smithi029.stdout:1659: seq_num 632 ranges {622024=414667,1783208=417873} 2024-04-12T11:21:27.620 INFO:tasks.rados.rados.0.smithi029.stdout:1659: writing smithi02979277-33 from 622024 to 1036691 tid 1 2024-04-12T11:21:27.627 INFO:tasks.rados.rados.0.smithi029.stdout:1659: writing smithi02979277-33 from 1783208 to 2201081 tid 2 2024-04-12T11:21:27.627 INFO:tasks.rados.rados.0.smithi029.stdout:1660: delete oid 15 current snap is 205 2024-04-12T11:21:27.627 INFO:tasks.rados.rados.0.smithi029.stdout:1659: finishing write tid 1 to smithi02979277-33 2024-04-12T11:21:27.630 INFO:tasks.rados.rados.0.smithi029.stdout:1659: finishing write tid 2 to smithi02979277-33 2024-04-12T11:21:27.630 INFO:tasks.rados.rados.0.smithi029.stdout:1659: finishing write tid 3 to smithi02979277-33 2024-04-12T11:21:27.630 INFO:tasks.rados.rados.0.smithi029.stdout:1659: finishing write tid 4 to smithi02979277-33 2024-04-12T11:21:27.630 INFO:tasks.rados.rados.0.smithi029.stdout:1659: oid 33 updating version 0 to 575 2024-04-12T11:21:27.630 INFO:tasks.rados.rados.0.smithi029.stdout:1659: oid 33 updating version 575 to 577 2024-04-12T11:21:27.630 INFO:tasks.rados.rados.0.smithi029.stdout:1659: oid 33 version 577 is already newer than 576 2024-04-12T11:21:27.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 577 (ObjNum 632 snap 205 seq_num 632) dirty exists 2024-04-12T11:21:27.631 INFO:tasks.rados.rados.0.smithi029.stdout:1659: left oid 33 (ObjNum 632 snap 205 seq_num 632) 2024-04-12T11:21:27.631 INFO:tasks.rados.rados.0.smithi029.stdout:1659: done (1 left) 2024-04-12T11:21:27.631 INFO:tasks.rados.rados.0.smithi029.stdout:1660: done (0 left) 2024-04-12T11:21:27.631 INFO:tasks.rados.rados.0.smithi029.stdout:1661: write oid 41 current snap is 205 2024-04-12T11:21:27.631 INFO:tasks.rados.rados.0.smithi029.stdout:1661: seq_num 634 ranges {655638=565165,1900172=421542} 2024-04-12T11:21:27.638 INFO:tasks.rados.rados.0.smithi029.stdout:1661: writing smithi02979277-41 from 655638 to 1220803 tid 1 2024-04-12T11:21:27.644 INFO:tasks.rados.rados.0.smithi029.stdout:1661: writing smithi02979277-41 from 1900172 to 2321714 tid 2 2024-04-12T11:21:27.644 INFO:tasks.rados.rados.0.smithi029.stdout:1662: read oid 15 snap -1 2024-04-12T11:21:27.644 INFO:tasks.rados.rados.0.smithi029.stdout:1662: expect deleted 2024-04-12T11:21:27.645 INFO:tasks.rados.rados.0.smithi029.stdout:1663: write oid 10 current snap is 205 2024-04-12T11:21:27.645 INFO:tasks.rados.rados.0.smithi029.stdout:1661: finishing write tid 1 to smithi02979277-41 2024-04-12T11:21:27.645 INFO:tasks.rados.rados.0.smithi029.stdout:1663: seq_num 635 ranges {672445=640414,1842301=681553} 2024-04-12T11:21:27.651 INFO:tasks.rados.rados.0.smithi029.stdout:1663: writing smithi02979277-10 from 672445 to 1312859 tid 1 2024-04-12T11:21:27.658 INFO:tasks.rados.rados.0.smithi029.stdout:1663: writing smithi02979277-10 from 1842301 to 2523854 tid 2 2024-04-12T11:21:27.658 INFO:tasks.rados.rados.0.smithi029.stdout:1664: delete oid 7 current snap is 205 2024-04-12T11:21:27.659 INFO:tasks.rados.rados.0.smithi029.stdout:1661: finishing write tid 2 to smithi02979277-41 2024-04-12T11:21:27.659 INFO:tasks.rados.rados.0.smithi029.stdout:1661: finishing write tid 3 to smithi02979277-41 2024-04-12T11:21:27.659 INFO:tasks.rados.rados.0.smithi029.stdout:1661: finishing write tid 4 to smithi02979277-41 2024-04-12T11:21:27.659 INFO:tasks.rados.rados.0.smithi029.stdout:1661: oid 41 updating version 0 to 823 2024-04-12T11:21:27.659 INFO:tasks.rados.rados.0.smithi029.stdout:1661: oid 41 version 823 is already newer than 822 2024-04-12T11:21:27.659 INFO:tasks.rados.rados.0.smithi029.stdout:1661: oid 41 version 823 is already newer than 821 2024-04-12T11:21:27.659 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 823 (ObjNum 634 snap 205 seq_num 634) dirty exists 2024-04-12T11:21:27.659 INFO:tasks.rados.rados.0.smithi029.stdout:1661: left oid 41 (ObjNum 634 snap 205 seq_num 634) 2024-04-12T11:21:27.659 INFO:tasks.rados.rados.0.smithi029.stdout:1663: finishing write tid 1 to smithi02979277-10 2024-04-12T11:21:27.660 INFO:tasks.rados.rados.0.smithi029.stdout:1661: done (3 left) 2024-04-12T11:21:27.660 INFO:tasks.rados.rados.0.smithi029.stdout:1662: done (2 left) 2024-04-12T11:21:27.660 INFO:tasks.rados.rados.0.smithi029.stdout:1664: done (1 left) 2024-04-12T11:21:27.660 INFO:tasks.rados.rados.0.smithi029.stdout:1665: snap_remove snap 204 2024-04-12T11:21:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:27 smithi087 ceph-mon[89014]: osdmap e483: 8 total, 8 up, 8 in 2024-04-12T11:21:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:27 smithi029 ceph-mon[93650]: osdmap e483: 8 total, 8 up, 8 in 2024-04-12T11:21:28.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:27 smithi029 ceph-mon[97787]: osdmap e483: 8 total, 8 up, 8 in 2024-04-12T11:21:28.623 INFO:tasks.rados.rados.0.smithi029.stdout:1665: done (1 left) 2024-04-12T11:21:28.623 INFO:tasks.rados.rados.0.smithi029.stdout:1666: rollback oid 2 current snap is 205 2024-04-12T11:21:28.623 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 2 to 201 2024-04-12T11:21:28.623 INFO:tasks.rados.rados.0.smithi029.stdout:1663: finishing write tid 2 to smithi02979277-10 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:1663: finishing write tid 3 to smithi02979277-10 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:1663: finishing write tid 4 to smithi02979277-10 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:1663: oid 10 updating version 0 to 746 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:1663: oid 10 updating version 746 to 747 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:1663: oid 10 updating version 747 to 748 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 748 (ObjNum 635 snap 205 seq_num 635) dirty exists 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:1663: left oid 10 (ObjNum 635 snap 205 seq_num 635) 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:1663: done (1 left) 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:1667: read oid 11 snap -1 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:1667: expect (ObjNum 566 snap 177 seq_num 566) 2024-04-12T11:21:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:1668: delete oid 44 current snap is 205 2024-04-12T11:21:28.629 INFO:tasks.rados.rados.0.smithi029.stdout:1666: finishing rollback tid 0 to smithi02979277-2 2024-04-12T11:21:28.629 INFO:tasks.rados.rados.0.smithi029.stdout:1666: finishing rollback tid 1 to smithi02979277-2 2024-04-12T11:21:28.630 INFO:tasks.rados.rados.0.smithi029.stdout:1668: done (2 left) 2024-04-12T11:21:28.630 INFO:tasks.rados.rados.0.smithi029.stdout:1669: snap_create 2024-04-12T11:21:28.633 INFO:tasks.rados.rados.0.smithi029.stdout:1666: finishing rollback tid 2 to smithi02979277-2 2024-04-12T11:21:28.633 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 182 (ObjNum 610 snap 195 seq_num 610) dirty exists 2024-04-12T11:21:28.634 INFO:tasks.rados.rados.0.smithi029.stdout:1667: expect (ObjNum 566 snap 177 seq_num 566) 2024-04-12T11:21:28.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:28 smithi087 ceph-mon[89014]: pgmap v361: 89 pgs: 89 active+clean; 109 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-04-12T11:21:28.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:28 smithi087 ceph-mon[89014]: osdmap e484: 8 total, 8 up, 8 in 2024-04-12T11:21:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:28 smithi029 ceph-mon[93650]: pgmap v361: 89 pgs: 89 active+clean; 109 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-04-12T11:21:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:28 smithi029 ceph-mon[93650]: osdmap e484: 8 total, 8 up, 8 in 2024-04-12T11:21:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:28 smithi029 ceph-mon[97787]: pgmap v361: 89 pgs: 89 active+clean; 109 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-04-12T11:21:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:28 smithi029 ceph-mon[97787]: osdmap e484: 8 total, 8 up, 8 in 2024-04-12T11:21:29.627 INFO:tasks.rados.rados.0.smithi029.stdout:1666: done (2 left) 2024-04-12T11:21:29.628 INFO:tasks.rados.rados.0.smithi029.stdout:1667: done (1 left) 2024-04-12T11:21:29.628 INFO:tasks.rados.rados.0.smithi029.stdout:1669: done (0 left) 2024-04-12T11:21:29.628 INFO:tasks.rados.rados.0.smithi029.stdout:1670: snap_create 2024-04-12T11:21:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:29 smithi087 ceph-mon[89014]: osdmap e485: 8 total, 8 up, 8 in 2024-04-12T11:21:30.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:29 smithi029 ceph-mon[93650]: osdmap e485: 8 total, 8 up, 8 in 2024-04-12T11:21:30.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:29 smithi029 ceph-mon[97787]: osdmap e485: 8 total, 8 up, 8 in 2024-04-12T11:21:30.633 INFO:tasks.rados.rados.0.smithi029.stdout:1670: done (0 left) 2024-04-12T11:21:30.634 INFO:tasks.rados.rados.0.smithi029.stdout:1671: snap_remove snap 201 2024-04-12T11:21:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:30 smithi087 ceph-mon[89014]: pgmap v364: 89 pgs: 89 active+clean; 109 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-04-12T11:21:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:30 smithi087 ceph-mon[89014]: osdmap e486: 8 total, 8 up, 8 in 2024-04-12T11:21:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:30 smithi029 ceph-mon[93650]: pgmap v364: 89 pgs: 89 active+clean; 109 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-04-12T11:21:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:30 smithi029 ceph-mon[93650]: osdmap e486: 8 total, 8 up, 8 in 2024-04-12T11:21:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:30 smithi029 ceph-mon[97787]: pgmap v364: 89 pgs: 89 active+clean; 109 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-04-12T11:21:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:30 smithi029 ceph-mon[97787]: osdmap e486: 8 total, 8 up, 8 in 2024-04-12T11:21:31.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch upgrade resume' 2024-04-12T11:21:31.638 INFO:tasks.rados.rados.0.smithi029.stdout:1671: done (0 left) 2024-04-12T11:21:31.638 INFO:tasks.rados.rados.0.smithi029.stdout:1672: snap_remove snap 205 2024-04-12T11:21:31.668 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:21:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:31 smithi029 ceph-mon[93650]: osdmap e487: 8 total, 8 up, 8 in 2024-04-12T11:21:31.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:31 smithi029 ceph-mon[97787]: osdmap e487: 8 total, 8 up, 8 in 2024-04-12T11:21:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:31 smithi087 ceph-mon[89014]: osdmap e487: 8 total, 8 up, 8 in 2024-04-12T11:21:32.552 INFO:tasks.rados.rados.0.smithi029.stdout:1672: done (0 left) 2024-04-12T11:21:32.552 INFO:tasks.rados.rados.0.smithi029.stdout:1673: write oid 25 current snap is 207 2024-04-12T11:21:32.552 INFO:tasks.rados.rados.0.smithi029.stdout:1673: seq_num 638 ranges {722866=466161,1617747=428880,2587634=135232} 2024-04-12T11:21:32.558 INFO:tasks.rados.rados.0.smithi029.stdout:1673: writing smithi02979277-25 from 722866 to 1189027 tid 1 2024-04-12T11:21:32.563 INFO:tasks.rados.rados.0.smithi029.stdout:1673: writing smithi02979277-25 from 1617747 to 2046627 tid 2 2024-04-12T11:21:32.567 INFO:tasks.rados.rados.0.smithi029.stdout:1673: writing smithi02979277-25 from 2587634 to 2722866 tid 3 2024-04-12T11:21:32.567 INFO:tasks.rados.rados.0.smithi029.stdout:1674: write oid 31 current snap is 207 2024-04-12T11:21:32.568 INFO:tasks.rados.rados.0.smithi029.stdout:1673: finishing write tid 1 to smithi02979277-25 2024-04-12T11:21:32.568 INFO:tasks.rados.rados.0.smithi029.stdout:1674: seq_num 639 ranges {739673=657763,2076229=663444} 2024-04-12T11:21:32.575 INFO:tasks.rados.rados.0.smithi029.stdout:1674: writing smithi02979277-31 from 739673 to 1397436 tid 1 2024-04-12T11:21:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:1674: writing smithi02979277-31 from 2076229 to 2739673 tid 2 2024-04-12T11:21:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:1675: snap_create 2024-04-12T11:21:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:1673: finishing write tid 2 to smithi02979277-25 2024-04-12T11:21:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:1673: finishing write tid 3 to smithi02979277-25 2024-04-12T11:21:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:1673: finishing write tid 4 to smithi02979277-25 2024-04-12T11:21:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:1673: finishing write tid 5 to smithi02979277-25 2024-04-12T11:21:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:1673: oid 25 updating version 0 to 840 2024-04-12T11:21:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:1673: oid 25 updating version 840 to 841 2024-04-12T11:21:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:1673: oid 25 version 841 is already newer than 839 2024-04-12T11:21:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:1673: oid 25 updating version 841 to 842 2024-04-12T11:21:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 842 (ObjNum 638 snap 207 seq_num 638) dirty exists 2024-04-12T11:21:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:1673: left oid 25 (ObjNum 638 snap 207 seq_num 638) 2024-04-12T11:21:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:1674: finishing write tid 1 to smithi02979277-31 2024-04-12T11:21:32.586 INFO:tasks.rados.rados.0.smithi029.stdout:1674: finishing write tid 2 to smithi02979277-31 2024-04-12T11:21:32.586 INFO:tasks.rados.rados.0.smithi029.stdout:1674: finishing write tid 3 to smithi02979277-31 2024-04-12T11:21:32.586 INFO:tasks.rados.rados.0.smithi029.stdout:1674: finishing write tid 4 to smithi02979277-31 2024-04-12T11:21:32.586 INFO:tasks.rados.rados.0.smithi029.stdout:1674: oid 31 updating version 0 to 756 2024-04-12T11:21:32.586 INFO:tasks.rados.rados.0.smithi029.stdout:1674: oid 31 version 756 is already newer than 754 2024-04-12T11:21:32.586 INFO:tasks.rados.rados.0.smithi029.stdout:1674: oid 31 version 756 is already newer than 755 2024-04-12T11:21:32.586 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 756 (ObjNum 639 snap 207 seq_num 639) dirty exists 2024-04-12T11:21:32.587 INFO:tasks.rados.rados.0.smithi029.stdout:1674: left oid 31 (ObjNum 639 snap 207 seq_num 639) 2024-04-12T11:21:32.636 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:21:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:21:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:21:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:32 smithi029 ceph-mon[93650]: pgmap v367: 89 pgs: 89 active+clean; 109 MiB data, 551 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:32 smithi029 ceph-mon[93650]: osdmap e488: 8 total, 8 up, 8 in 2024-04-12T11:21:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:32 smithi029 ceph-mon[93650]: osdmap e489: 8 total, 8 up, 8 in 2024-04-12T11:21:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:32 smithi029 ceph-mon[97787]: pgmap v367: 89 pgs: 89 active+clean; 109 MiB data, 551 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:32 smithi029 ceph-mon[97787]: osdmap e488: 8 total, 8 up, 8 in 2024-04-12T11:21:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:32 smithi029 ceph-mon[97787]: osdmap e489: 8 total, 8 up, 8 in 2024-04-12T11:21:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:32 smithi087 ceph-mon[89014]: pgmap v367: 89 pgs: 89 active+clean; 109 MiB data, 551 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:32 smithi087 ceph-mon[89014]: osdmap e488: 8 total, 8 up, 8 in 2024-04-12T11:21:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:32 smithi087 ceph-mon[89014]: osdmap e489: 8 total, 8 up, 8 in 2024-04-12T11:21:33.554 INFO:tasks.rados.rados.0.smithi029.stdout:1673: done (2 left) 2024-04-12T11:21:33.554 INFO:tasks.rados.rados.0.smithi029.stdout:1674: done (1 left) 2024-04-12T11:21:33.554 INFO:tasks.rados.rados.0.smithi029.stdout:1675: done (0 left) 2024-04-12T11:21:33.554 INFO:tasks.rados.rados.0.smithi029.stdout:1676: snap_remove snap 206 2024-04-12T11:21:33.563 INFO:teuthology.orchestra.run.smithi029.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:21:34.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'echo wait for half of osds to upgrade' 2024-04-12T11:21:34.555 INFO:tasks.rados.rados.0.smithi029.stdout:1676: done (0 left) 2024-04-12T11:21:34.555 INFO:tasks.rados.rados.0.smithi029.stdout:1677: delete oid 40 current snap is 208 2024-04-12T11:21:34.563 INFO:tasks.rados.rados.0.smithi029.stdout:1677: done (0 left) 2024-04-12T11:21:34.563 INFO:tasks.rados.rados.0.smithi029.stdout:1678: read oid 22 snap -1 2024-04-12T11:21:34.563 INFO:tasks.rados.rados.0.smithi029.stdout:1678: expect (ObjNum 622 snap 201 seq_num 622) 2024-04-12T11:21:34.563 INFO:tasks.rados.rados.0.smithi029.stdout:1679: write oid 35 current snap is 208 2024-04-12T11:21:34.563 INFO:tasks.rados.rados.0.smithi029.stdout:1679: seq_num 641 ranges {773287=408261,1793193=576207} 2024-04-12T11:21:34.569 INFO:tasks.rados.rados.0.smithi029.stdout:1679: writing smithi02979277-35 from 773287 to 1181548 tid 1 2024-04-12T11:21:34.576 INFO:tasks.rados.rados.0.smithi029.stdout:1679: writing smithi02979277-35 from 1793193 to 2369400 tid 2 2024-04-12T11:21:34.576 INFO:tasks.rados.rados.0.smithi029.stdout:1680: delete oid 20 current snap is 208 2024-04-12T11:21:34.576 INFO:tasks.rados.rados.0.smithi029.stdout:1678: expect (ObjNum 622 snap 201 seq_num 622) 2024-04-12T11:21:34.658 INFO:tasks.rados.rados.0.smithi029.stdout:1679: finishing write tid 1 to smithi02979277-35 2024-04-12T11:21:34.658 INFO:tasks.rados.rados.0.smithi029.stdout:1679: finishing write tid 2 to smithi02979277-35 2024-04-12T11:21:34.658 INFO:tasks.rados.rados.0.smithi029.stdout:1679: finishing write tid 3 to smithi02979277-35 2024-04-12T11:21:34.658 INFO:tasks.rados.rados.0.smithi029.stdout:1679: finishing write tid 4 to smithi02979277-35 2024-04-12T11:21:34.658 INFO:tasks.rados.rados.0.smithi029.stdout:1679: oid 35 updating version 0 to 1075 2024-04-12T11:21:34.658 INFO:tasks.rados.rados.0.smithi029.stdout:1679: oid 35 updating version 1075 to 1077 2024-04-12T11:21:34.658 INFO:tasks.rados.rados.0.smithi029.stdout:1679: oid 35 version 1077 is already newer than 1076 2024-04-12T11:21:34.658 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 1077 (ObjNum 641 snap 208 seq_num 641) dirty exists 2024-04-12T11:21:34.658 INFO:tasks.rados.rados.0.smithi029.stdout:1679: left oid 35 (ObjNum 641 snap 208 seq_num 641) 2024-04-12T11:21:34.658 INFO:tasks.rados.rados.0.smithi029.stdout:1678: done (2 left) 2024-04-12T11:21:34.659 INFO:tasks.rados.rados.0.smithi029.stdout:1679: done (1 left) 2024-04-12T11:21:34.659 INFO:tasks.rados.rados.0.smithi029.stdout:1680: done (0 left) 2024-04-12T11:21:34.659 INFO:tasks.rados.rados.0.smithi029.stdout:1681: write oid 12 current snap is 208 2024-04-12T11:21:34.659 INFO:tasks.rados.rados.0.smithi029.stdout:1681: seq_num 643 ranges {406901=558759,1393804=579876,2608396=198505} 2024-04-12T11:21:34.663 INFO:tasks.rados.rados.0.smithi029.stdout:1681: writing smithi02979277-12 from 406901 to 965660 tid 1 2024-04-12T11:21:34.668 INFO:tasks.rados.rados.0.smithi029.stdout:1681: writing smithi02979277-12 from 1393804 to 1973680 tid 2 2024-04-12T11:21:34.673 INFO:tasks.rados.rados.0.smithi029.stdout:1681: writing smithi02979277-12 from 2608396 to 2806901 tid 3 2024-04-12T11:21:34.674 INFO:tasks.rados.rados.0.smithi029.stdout:1682: read oid 9 snap -1 2024-04-12T11:21:34.674 INFO:tasks.rados.rados.0.smithi029.stdout:1682: expect deleted 2024-04-12T11:21:34.674 INFO:tasks.rados.rados.0.smithi029.stdout:1683: write oid 23 current snap is 208 2024-04-12T11:21:34.674 INFO:tasks.rados.rados.0.smithi029.stdout:1681: finishing write tid 1 to smithi02979277-12 2024-04-12T11:21:34.674 INFO:tasks.rados.rados.0.smithi029.stdout:1681: finishing write tid 2 to smithi02979277-12 2024-04-12T11:21:34.674 INFO:tasks.rados.rados.0.smithi029.stdout:1683: seq_num 644 ranges {423708=634008,1852286=439887,2752172=71536} 2024-04-12T11:21:34.679 INFO:tasks.rados.rados.0.smithi029.stdout:1683: writing smithi02979277-23 from 423708 to 1057716 tid 1 2024-04-12T11:21:34.686 INFO:tasks.rados.rados.0.smithi029.stdout:1683: writing smithi02979277-23 from 1852286 to 2292173 tid 2 2024-04-12T11:21:34.689 INFO:tasks.rados.rados.0.smithi029.stdout:1683: writing smithi02979277-23 from 2752172 to 2823708 tid 3 2024-04-12T11:21:34.689 INFO:tasks.rados.rados.0.smithi029.stdout:1684: read oid 19 snap -1 2024-04-12T11:21:34.689 INFO:tasks.rados.rados.0.smithi029.stdout:1684: expect deleted 2024-04-12T11:21:34.689 INFO:tasks.rados.rados.0.smithi029.stdout:1685: delete oid 21 current snap is 208 2024-04-12T11:21:34.689 INFO:tasks.rados.rados.0.smithi029.stdout:1681: finishing write tid 3 to smithi02979277-12 2024-04-12T11:21:34.689 INFO:tasks.rados.rados.0.smithi029.stdout:1681: finishing write tid 4 to smithi02979277-12 2024-04-12T11:21:34.689 INFO:tasks.rados.rados.0.smithi029.stdout:1681: finishing write tid 5 to smithi02979277-12 2024-04-12T11:21:34.689 INFO:tasks.rados.rados.0.smithi029.stdout:1681: oid 12 updating version 0 to 793 2024-04-12T11:21:34.689 INFO:tasks.rados.rados.0.smithi029.stdout:1681: oid 12 version 793 is already newer than 792 2024-04-12T11:21:34.690 INFO:tasks.rados.rados.0.smithi029.stdout:1681: oid 12 version 793 is already newer than 791 2024-04-12T11:21:34.690 INFO:tasks.rados.rados.0.smithi029.stdout:1681: oid 12 version 793 is already newer than 790 2024-04-12T11:21:34.690 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 793 (ObjNum 643 snap 208 seq_num 643) dirty exists 2024-04-12T11:21:34.690 INFO:tasks.rados.rados.0.smithi029.stdout:1681: left oid 12 (ObjNum 643 snap 208 seq_num 643) 2024-04-12T11:21:34.690 INFO:tasks.rados.rados.0.smithi029.stdout:1683: finishing write tid 1 to smithi02979277-23 2024-04-12T11:21:34.690 INFO:tasks.rados.rados.0.smithi029.stdout:1683: finishing write tid 2 to smithi02979277-23 2024-04-12T11:21:34.691 INFO:tasks.rados.rados.0.smithi029.stdout:1683: finishing write tid 3 to smithi02979277-23 2024-04-12T11:21:34.691 INFO:tasks.rados.rados.0.smithi029.stdout:1681: done (4 left) 2024-04-12T11:21:34.691 INFO:tasks.rados.rados.0.smithi029.stdout:1682: done (3 left) 2024-04-12T11:21:34.691 INFO:tasks.rados.rados.0.smithi029.stdout:1684: done (2 left) 2024-04-12T11:21:34.692 INFO:tasks.rados.rados.0.smithi029.stdout:1685: done (1 left) 2024-04-12T11:21:34.692 INFO:tasks.rados.rados.0.smithi029.stdout:1686: write oid 1 current snap is 208 2024-04-12T11:21:34.692 INFO:tasks.rados.rados.0.smithi029.stdout:1686: seq_num 646 ranges {457322=784506,1969250=443556} 2024-04-12T11:21:34.698 INFO:tasks.rados.rados.0.smithi029.stdout:1686: writing smithi02979277-1 from 457322 to 1241828 tid 1 2024-04-12T11:21:34.704 INFO:tasks.rados.rados.0.smithi029.stdout:1686: writing smithi02979277-1 from 1969250 to 2412806 tid 2 2024-04-12T11:21:34.705 INFO:tasks.rados.rados.0.smithi029.stdout:1687: delete oid 37 current snap is 208 2024-04-12T11:21:34.705 INFO:tasks.rados.rados.0.smithi029.stdout:1683: finishing write tid 4 to smithi02979277-23 2024-04-12T11:21:34.705 INFO:tasks.rados.rados.0.smithi029.stdout:1683: finishing write tid 5 to smithi02979277-23 2024-04-12T11:21:34.705 INFO:tasks.rados.rados.0.smithi029.stdout:1683: oid 23 updating version 0 to 588 2024-04-12T11:21:34.705 INFO:tasks.rados.rados.0.smithi029.stdout:1683: oid 23 version 588 is already newer than 587 2024-04-12T11:21:34.705 INFO:tasks.rados.rados.0.smithi029.stdout:1683: oid 23 version 588 is already newer than 585 2024-04-12T11:21:34.705 INFO:tasks.rados.rados.0.smithi029.stdout:1683: oid 23 version 588 is already newer than 586 2024-04-12T11:21:34.705 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 588 (ObjNum 644 snap 208 seq_num 644) dirty exists 2024-04-12T11:21:34.705 INFO:tasks.rados.rados.0.smithi029.stdout:1683: left oid 23 (ObjNum 644 snap 208 seq_num 644) 2024-04-12T11:21:34.705 INFO:tasks.rados.rados.0.smithi029.stdout:1686: finishing write tid 1 to smithi02979277-1 2024-04-12T11:21:34.706 INFO:tasks.rados.rados.0.smithi029.stdout:1683: done (2 left) 2024-04-12T11:21:34.706 INFO:tasks.rados.rados.0.smithi029.stdout:1687: done (1 left) 2024-04-12T11:21:34.706 INFO:tasks.rados.rados.0.smithi029.stdout:1688: read oid 19 snap -1 2024-04-12T11:21:34.706 INFO:tasks.rados.rados.0.smithi029.stdout:1688: expect deleted 2024-04-12T11:21:34.707 INFO:tasks.rados.rados.0.smithi029.stdout:1689: snap_create 2024-04-12T11:21:34.707 INFO:tasks.rados.rados.0.smithi029.stdout:1686: finishing write tid 2 to smithi02979277-1 2024-04-12T11:21:34.707 INFO:tasks.rados.rados.0.smithi029.stdout:1686: finishing write tid 3 to smithi02979277-1 2024-04-12T11:21:34.707 INFO:tasks.rados.rados.0.smithi029.stdout:1686: finishing write tid 4 to smithi02979277-1 2024-04-12T11:21:34.707 INFO:tasks.rados.rados.0.smithi029.stdout:1686: oid 1 updating version 0 to 834 2024-04-12T11:21:34.708 INFO:tasks.rados.rados.0.smithi029.stdout:1686: oid 1 version 834 is already newer than 833 2024-04-12T11:21:34.708 INFO:tasks.rados.rados.0.smithi029.stdout:1686: oid 1 updating version 834 to 835 2024-04-12T11:21:34.708 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 835 (ObjNum 646 snap 208 seq_num 646) dirty exists 2024-04-12T11:21:34.708 INFO:tasks.rados.rados.0.smithi029.stdout:1686: left oid 1 (ObjNum 646 snap 208 seq_num 646) 2024-04-12T11:21:34.750 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[97787]: pgmap v370: 89 pgs: 89 active+clean; 97 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 767 KiB/s wr, 6 op/s 2024-04-12T11:21:34.750 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[97787]: osdmap e490: 8 total, 8 up, 8 in 2024-04-12T11:21:34.750 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[97787]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:21:34.751 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[97787]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:21:34.751 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:34.751 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:21:34.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[93650]: pgmap v370: 89 pgs: 89 active+clean; 97 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 767 KiB/s wr, 6 op/s 2024-04-12T11:21:34.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[93650]: osdmap e490: 8 total, 8 up, 8 in 2024-04-12T11:21:34.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[93650]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:21:34.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[93650]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:21:34.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:34.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:21:34.840 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:34 smithi087 ceph-mon[89014]: pgmap v370: 89 pgs: 89 active+clean; 97 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 767 KiB/s wr, 6 op/s 2024-04-12T11:21:34.840 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:34 smithi087 ceph-mon[89014]: osdmap e490: 8 total, 8 up, 8 in 2024-04-12T11:21:34.840 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:34 smithi087 ceph-mon[89014]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:21:34.840 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:34 smithi087 ceph-mon[89014]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:21:34.840 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:34 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:34.840 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:34 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:21:35.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:21:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:21:34] "GET /metrics HTTP/1.1" 200 36243 "" "Prometheus/2.43.0" 2024-04-12T11:21:35.285 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:21:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:1686: done (2 left) 2024-04-12T11:21:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:1688: done (1 left) 2024-04-12T11:21:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:1689: done (0 left) 2024-04-12T11:21:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:1690: read oid 48 snap 208 2024-04-12T11:21:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:1690: expect deleted 2024-04-12T11:21:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:1691: rollback oid 3 current snap is 209 2024-04-12T11:21:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 208 2024-04-12T11:21:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:1692: read oid 15 snap -1 2024-04-12T11:21:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:1692: expect deleted 2024-04-12T11:21:35.564 INFO:tasks.rados.rados.0.smithi029.stdout:1693: delete oid 42 current snap is 209 2024-04-12T11:21:35.568 INFO:tasks.rados.rados.0.smithi029.stdout:1690: done (3 left) 2024-04-12T11:21:35.568 INFO:tasks.rados.rados.0.smithi029.stdout:1692: done (2 left) 2024-04-12T11:21:35.568 INFO:tasks.rados.rados.0.smithi029.stdout:1693: done (1 left) 2024-04-12T11:21:35.568 INFO:tasks.rados.rados.0.smithi029.stdout:1694: snap_remove snap 207 2024-04-12T11:21:35.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:35 smithi029 ceph-mon[97787]: osdmap e491: 8 total, 8 up, 8 in 2024-04-12T11:21:35.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:35 smithi029 ceph-mon[93650]: osdmap e491: 8 total, 8 up, 8 in 2024-04-12T11:21:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:35 smithi087 ceph-mon[89014]: osdmap e491: 8 total, 8 up, 8 in 2024-04-12T11:21:36.571 INFO:tasks.rados.rados.0.smithi029.stdout:1694: done (1 left) 2024-04-12T11:21:36.571 INFO:tasks.rados.rados.0.smithi029.stdout:1695: read oid 21 snap 208 2024-04-12T11:21:36.571 INFO:tasks.rados.rados.0.smithi029.stdout:1695: expect deleted 2024-04-12T11:21:36.571 INFO:tasks.rados.rados.0.smithi029.stdout:1696: read oid 8 snap -1 2024-04-12T11:21:36.571 INFO:tasks.rados.rados.0.smithi029.stdout:1696: expect (ObjNum 601 snap 192 seq_num 601) 2024-04-12T11:21:36.571 INFO:tasks.rados.rados.0.smithi029.stdout:1697: rollback oid 27 current snap is 209 2024-04-12T11:21:36.571 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 208 2024-04-12T11:21:36.571 INFO:tasks.rados.rados.0.smithi029.stdout:1691: finishing rollback tid 0 to smithi02979277-3 2024-04-12T11:21:36.572 INFO:tasks.rados.rados.0.smithi029.stdout:1691: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:21:36.572 INFO:tasks.rados.rados.0.smithi029.stdout:1691: finishing rollback tid 2 to smithi02979277-3 2024-04-12T11:21:36.572 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1081 (ObjNum 562 snap 177 seq_num 562) dirty exists 2024-04-12T11:21:36.572 INFO:tasks.rados.rados.0.smithi029.stdout:1691: done (3 left) 2024-04-12T11:21:36.572 INFO:tasks.rados.rados.0.smithi029.stdout:1698: snap_create 2024-04-12T11:21:36.572 INFO:tasks.rados.rados.0.smithi029.stdout:1697: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:21:36.572 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 0 (ObjNum 33 snap 0 seq_num 1800919216) dirty dne 2024-04-12T11:21:36.580 INFO:tasks.rados.rados.0.smithi029.stdout:1696: expect (ObjNum 601 snap 192 seq_num 601) 2024-04-12T11:21:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:36 smithi029 ceph-mon[93650]: pgmap v373: 89 pgs: 89 active+clean; 97 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 767 KiB/s wr, 6 op/s 2024-04-12T11:21:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:36 smithi029 ceph-mon[93650]: osdmap e492: 8 total, 8 up, 8 in 2024-04-12T11:21:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:36.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:36 smithi029 ceph-mon[97787]: pgmap v373: 89 pgs: 89 active+clean; 97 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 767 KiB/s wr, 6 op/s 2024-04-12T11:21:36.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:36 smithi029 ceph-mon[97787]: osdmap e492: 8 total, 8 up, 8 in 2024-04-12T11:21:36.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:36.900 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:36 smithi087 ceph-mon[89014]: pgmap v373: 89 pgs: 89 active+clean; 97 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 767 KiB/s wr, 6 op/s 2024-04-12T11:21:36.900 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:36 smithi087 ceph-mon[89014]: osdmap e492: 8 total, 8 up, 8 in 2024-04-12T11:21:36.900 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:37.231 INFO:teuthology.orchestra.run.smithi029.stdout:wait for half of osds to upgrade 2024-04-12T11:21:37.575 INFO:tasks.rados.rados.0.smithi029.stdout:1695: done (3 left) 2024-04-12T11:21:37.575 INFO:tasks.rados.rados.0.smithi029.stdout:1696: done (2 left) 2024-04-12T11:21:37.575 INFO:tasks.rados.rados.0.smithi029.stdout:1697: done (1 left) 2024-04-12T11:21:37.575 INFO:tasks.rados.rados.0.smithi029.stdout:1698: done (0 left) 2024-04-12T11:21:37.575 INFO:tasks.rados.rados.0.smithi029.stdout:1699: write oid 47 current snap is 210 2024-04-12T11:21:37.575 INFO:tasks.rados.rados.0.smithi029.stdout:1699: seq_num 649 ranges {507743=726606,1744696=590883,2772934=134809} 2024-04-12T11:21:37.581 INFO:tasks.rados.rados.0.smithi029.stdout:1699: writing smithi02979277-47 from 507743 to 1234349 tid 1 2024-04-12T11:21:37.588 INFO:tasks.rados.rados.0.smithi029.stdout:1699: writing smithi02979277-47 from 1744696 to 2335579 tid 2 2024-04-12T11:21:37.591 INFO:tasks.rados.rados.0.smithi029.stdout:1699: writing smithi02979277-47 from 2772934 to 2907743 tid 3 2024-04-12T11:21:37.592 INFO:tasks.rados.rados.0.smithi029.stdout:1700: snap_create 2024-04-12T11:21:37.592 INFO:tasks.rados.rados.0.smithi029.stdout:1699: finishing write tid 1 to smithi02979277-47 2024-04-12T11:21:37.592 INFO:tasks.rados.rados.0.smithi029.stdout:1699: finishing write tid 2 to smithi02979277-47 2024-04-12T11:21:37.593 INFO:tasks.rados.rados.0.smithi029.stdout:1699: finishing write tid 3 to smithi02979277-47 2024-04-12T11:21:37.593 INFO:tasks.rados.rados.0.smithi029.stdout:1699: finishing write tid 4 to smithi02979277-47 2024-04-12T11:21:37.593 INFO:tasks.rados.rados.0.smithi029.stdout:1699: finishing write tid 5 to smithi02979277-47 2024-04-12T11:21:37.593 INFO:tasks.rados.rados.0.smithi029.stdout:1699: oid 47 updating version 0 to 797 2024-04-12T11:21:37.594 INFO:tasks.rados.rados.0.smithi029.stdout:1699: oid 47 updating version 797 to 799 2024-04-12T11:21:37.594 INFO:tasks.rados.rados.0.smithi029.stdout:1699: oid 47 version 799 is already newer than 798 2024-04-12T11:21:37.594 INFO:tasks.rados.rados.0.smithi029.stdout:1699: oid 47 updating version 799 to 800 2024-04-12T11:21:37.594 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 800 (ObjNum 649 snap 210 seq_num 649) dirty exists 2024-04-12T11:21:37.594 INFO:tasks.rados.rados.0.smithi029.stdout:1699: left oid 47 (ObjNum 649 snap 210 seq_num 649) 2024-04-12T11:21:37.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:37 smithi087 ceph-mon[89014]: osdmap e493: 8 total, 8 up, 8 in 2024-04-12T11:21:37.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:37 smithi029 ceph-mon[93650]: osdmap e493: 8 total, 8 up, 8 in 2024-04-12T11:21:37.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:37 smithi029 ceph-mon[97787]: osdmap e493: 8 total, 8 up, 8 in 2024-04-12T11:21:37.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-12T11:21:38.579 INFO:tasks.rados.rados.0.smithi029.stdout:1699: done (1 left) 2024-04-12T11:21:38.580 INFO:tasks.rados.rados.0.smithi029.stdout:1700: done (0 left) 2024-04-12T11:21:38.580 INFO:tasks.rados.rados.0.smithi029.stdout:1701: write oid 37 current snap is 211 2024-04-12T11:21:38.580 INFO:tasks.rados.rados.0.smithi029.stdout:1701: seq_num 650 ranges {524550=401855,1403178=450894,2516710=407840} 2024-04-12T11:21:38.584 INFO:tasks.rados.rados.0.smithi029.stdout:1701: writing smithi02979277-37 from 524550 to 926405 tid 1 2024-04-12T11:21:38.590 INFO:tasks.rados.rados.0.smithi029.stdout:1701: writing smithi02979277-37 from 1403178 to 1854072 tid 2 2024-04-12T11:21:38.596 INFO:tasks.rados.rados.0.smithi029.stdout:1701: writing smithi02979277-37 from 2516710 to 2924550 tid 3 2024-04-12T11:21:38.596 INFO:tasks.rados.rados.0.smithi029.stdout:1702: snap_remove snap 208 2024-04-12T11:21:38.617 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:21:38.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:38 smithi029 ceph-mon[97787]: pgmap v376: 89 pgs: 89 active+clean; 88 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-04-12T11:21:38.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:38 smithi029 ceph-mon[97787]: osdmap e494: 8 total, 8 up, 8 in 2024-04-12T11:21:38.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:38 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:38.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:38 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:38.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:38 smithi029 ceph-mon[93650]: pgmap v376: 89 pgs: 89 active+clean; 88 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-04-12T11:21:38.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:38 smithi029 ceph-mon[93650]: osdmap e494: 8 total, 8 up, 8 in 2024-04-12T11:21:38.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:38 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:38.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:38 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:38 smithi087 ceph-mon[89014]: pgmap v376: 89 pgs: 89 active+clean; 88 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-04-12T11:21:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:38 smithi087 ceph-mon[89014]: osdmap e494: 8 total, 8 up, 8 in 2024-04-12T11:21:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:38 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:38 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:39.589 INFO:tasks.rados.rados.0.smithi029.stdout:1702: done (1 left) 2024-04-12T11:21:39.589 INFO:tasks.rados.rados.0.smithi029.stdout:1703: write oid 38 current snap is 211 2024-04-12T11:21:39.589 INFO:tasks.rados.rados.0.smithi029.stdout:1701: finishing write tid 1 to smithi02979277-37 2024-04-12T11:21:39.589 INFO:tasks.rados.rados.0.smithi029.stdout:1703: seq_num 651 ranges {541357=477104,1745307=594552} 2024-04-12T11:21:39.594 INFO:tasks.rados.rados.0.smithi029.stdout:1703: writing smithi02979277-38 from 541357 to 1018461 tid 1 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1703: writing smithi02979277-38 from 1745307 to 2339859 tid 2 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1704: read oid 9 snap -1 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1704: expect deleted 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1705: read oid 4 snap -1 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1705: expect deleted 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1706: delete oid 47 current snap is 211 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1701: finishing write tid 2 to smithi02979277-37 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1701: finishing write tid 3 to smithi02979277-37 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1701: finishing write tid 4 to smithi02979277-37 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1701: finishing write tid 5 to smithi02979277-37 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1701: oid 37 updating version 0 to 649 2024-04-12T11:21:39.603 INFO:tasks.rados.rados.0.smithi029.stdout:1701: oid 37 version 649 is already newer than 648 2024-04-12T11:21:39.604 INFO:tasks.rados.rados.0.smithi029.stdout:1701: oid 37 updating version 649 to 651 2024-04-12T11:21:39.604 INFO:tasks.rados.rados.0.smithi029.stdout:1701: oid 37 version 651 is already newer than 650 2024-04-12T11:21:39.604 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 651 (ObjNum 650 snap 211 seq_num 650) dirty exists 2024-04-12T11:21:39.604 INFO:tasks.rados.rados.0.smithi029.stdout:1701: left oid 37 (ObjNum 650 snap 211 seq_num 650) 2024-04-12T11:21:39.604 INFO:tasks.rados.rados.0.smithi029.stdout:1703: finishing write tid 1 to smithi02979277-38 2024-04-12T11:21:39.606 INFO:tasks.rados.rados.0.smithi029.stdout:1703: finishing write tid 2 to smithi02979277-38 2024-04-12T11:21:39.606 INFO:tasks.rados.rados.0.smithi029.stdout:1703: finishing write tid 3 to smithi02979277-38 2024-04-12T11:21:39.606 INFO:tasks.rados.rados.0.smithi029.stdout:1703: finishing write tid 4 to smithi02979277-38 2024-04-12T11:21:39.606 INFO:tasks.rados.rados.0.smithi029.stdout:1703: oid 38 updating version 0 to 477 2024-04-12T11:21:39.606 INFO:tasks.rados.rados.0.smithi029.stdout:1703: oid 38 version 477 is already newer than 476 2024-04-12T11:21:39.607 INFO:tasks.rados.rados.0.smithi029.stdout:1703: oid 38 updating version 477 to 478 2024-04-12T11:21:39.607 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 478 (ObjNum 651 snap 211 seq_num 651) dirty exists 2024-04-12T11:21:39.607 INFO:tasks.rados.rados.0.smithi029.stdout:1703: left oid 38 (ObjNum 651 snap 211 seq_num 651) 2024-04-12T11:21:39.609 INFO:tasks.rados.rados.0.smithi029.stdout:1701: done (4 left) 2024-04-12T11:21:39.609 INFO:tasks.rados.rados.0.smithi029.stdout:1703: done (3 left) 2024-04-12T11:21:39.609 INFO:tasks.rados.rados.0.smithi029.stdout:1704: done (2 left) 2024-04-12T11:21:39.609 INFO:tasks.rados.rados.0.smithi029.stdout:1705: done (1 left) 2024-04-12T11:21:39.609 INFO:tasks.rados.rados.0.smithi029.stdout:1706: done (0 left) 2024-04-12T11:21:39.610 INFO:tasks.rados.rados.0.smithi029.stdout:1707: write oid 47 current snap is 211 2024-04-12T11:21:39.610 INFO:tasks.rados.rados.0.smithi029.stdout:1707: seq_num 653 ranges {574971=627602,1862271=598221} 2024-04-12T11:21:39.616 INFO:tasks.rados.rados.0.smithi029.stdout:1707: writing smithi02979277-47 from 574971 to 1202573 tid 1 2024-04-12T11:21:39.624 INFO:tasks.rados.rados.0.smithi029.stdout:1707: writing smithi02979277-47 from 1862271 to 2460492 tid 2 2024-04-12T11:21:39.624 INFO:tasks.rados.rados.0.smithi029.stdout:1708: delete oid 40 current snap is 211 2024-04-12T11:21:39.629 INFO:tasks.rados.rados.0.smithi029.stdout:1708: done (1 left) 2024-04-12T11:21:39.629 INFO:tasks.rados.rados.0.smithi029.stdout:1709: snap_create 2024-04-12T11:21:39.629 INFO:tasks.rados.rados.0.smithi029.stdout:1707: finishing write tid 1 to smithi02979277-47 2024-04-12T11:21:39.629 INFO:tasks.rados.rados.0.smithi029.stdout:1707: finishing write tid 2 to smithi02979277-47 2024-04-12T11:21:39.629 INFO:tasks.rados.rados.0.smithi029.stdout:1707: finishing write tid 3 to smithi02979277-47 2024-04-12T11:21:39.630 INFO:tasks.rados.rados.0.smithi029.stdout:1707: finishing write tid 4 to smithi02979277-47 2024-04-12T11:21:39.630 INFO:tasks.rados.rados.0.smithi029.stdout:1707: oid 47 updating version 0 to 806 2024-04-12T11:21:39.630 INFO:tasks.rados.rados.0.smithi029.stdout:1707: oid 47 updating version 806 to 807 2024-04-12T11:21:39.630 INFO:tasks.rados.rados.0.smithi029.stdout:1707: oid 47 version 807 is already newer than 805 2024-04-12T11:21:39.630 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 807 (ObjNum 653 snap 211 seq_num 653) dirty exists 2024-04-12T11:21:39.630 INFO:tasks.rados.rados.0.smithi029.stdout:1707: left oid 47 (ObjNum 653 snap 211 seq_num 653) 2024-04-12T11:21:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:39 smithi029 ceph-mon[93650]: osdmap e495: 8 total, 8 up, 8 in 2024-04-12T11:21:39.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:39 smithi029 ceph-mon[97787]: osdmap e495: 8 total, 8 up, 8 in 2024-04-12T11:21:39.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:39 smithi087 ceph-mon[89014]: osdmap e495: 8 total, 8 up, 8 in 2024-04-12T11:21:40.590 INFO:tasks.rados.rados.0.smithi029.stdout:1707: done (1 left) 2024-04-12T11:21:40.590 INFO:tasks.rados.rados.0.smithi029.stdout:1709: done (0 left) 2024-04-12T11:21:40.590 INFO:tasks.rados.rados.0.smithi029.stdout:1710: snap_remove snap 210 2024-04-12T11:21:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:40 smithi087 ceph-mon[89014]: pgmap v379: 89 pgs: 89 active+clean; 88 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-04-12T11:21:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:40 smithi087 ceph-mon[89014]: osdmap e496: 8 total, 8 up, 8 in 2024-04-12T11:21:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:40 smithi029 ceph-mon[93650]: pgmap v379: 89 pgs: 89 active+clean; 88 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-04-12T11:21:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:40 smithi029 ceph-mon[93650]: osdmap e496: 8 total, 8 up, 8 in 2024-04-12T11:21:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:40 smithi029 ceph-mon[97787]: pgmap v379: 89 pgs: 89 active+clean; 88 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-04-12T11:21:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:40 smithi029 ceph-mon[97787]: osdmap e496: 8 total, 8 up, 8 in 2024-04-12T11:21:41.595 INFO:tasks.rados.rados.0.smithi029.stdout:1710: done (0 left) 2024-04-12T11:21:41.595 INFO:tasks.rados.rados.0.smithi029.stdout:1711: delete oid 29 current snap is 212 2024-04-12T11:21:41.599 INFO:tasks.rados.rados.0.smithi029.stdout:1711: done (0 left) 2024-04-12T11:21:41.599 INFO:tasks.rados.rados.0.smithi029.stdout:1712: rollback oid 42 current snap is 212 2024-04-12T11:21:41.599 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 209 2024-04-12T11:21:41.599 INFO:tasks.rados.rados.0.smithi029.stdout:1713: write oid 17 current snap is 212 2024-04-12T11:21:41.599 INFO:tasks.rados.rados.0.smithi029.stdout:1713: seq_num 656 ranges {625392=569702,1637717=745548,2848542=176850} 2024-04-12T11:21:41.605 INFO:tasks.rados.rados.0.smithi029.stdout:1713: writing smithi02979277-17 from 625392 to 1195094 tid 1 2024-04-12T11:21:41.612 INFO:tasks.rados.rados.0.smithi029.stdout:1713: writing smithi02979277-17 from 1637717 to 2383265 tid 2 2024-04-12T11:21:41.616 INFO:tasks.rados.rados.0.smithi029.stdout:1713: writing smithi02979277-17 from 2848542 to 3025392 tid 3 2024-04-12T11:21:41.616 INFO:tasks.rados.rados.0.smithi029.stdout:1714: write oid 23 current snap is 212 2024-04-12T11:21:41.616 INFO:tasks.rados.rados.0.smithi029.stdout:1712: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:21:41.616 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-04-12T11:21:41.617 INFO:tasks.rados.rados.0.smithi029.stdout:1714: seq_num 657 ranges {642199=644951,1696199=605559,2992318=49881} 2024-04-12T11:21:41.623 INFO:tasks.rados.rados.0.smithi029.stdout:1714: writing smithi02979277-23 from 642199 to 1287150 tid 1 2024-04-12T11:21:41.628 INFO:tasks.rados.rados.0.smithi029.stdout:1714: writing smithi02979277-23 from 1696199 to 2301758 tid 2 2024-04-12T11:21:41.633 INFO:tasks.rados.rados.0.smithi029.stdout:1714: writing smithi02979277-23 from 2992318 to 3042199 tid 3 2024-04-12T11:21:41.633 INFO:tasks.rados.rados.0.smithi029.stdout:1712: done (2 left) 2024-04-12T11:21:41.633 INFO:tasks.rados.rados.0.smithi029.stdout:1715: write oid 47 current snap is 212 2024-04-12T11:21:41.633 INFO:tasks.rados.rados.0.smithi029.stdout:1715: seq_num 658 ranges {659006=720200,2154681=749217} 2024-04-12T11:21:41.640 INFO:tasks.rados.rados.0.smithi029.stdout:1715: writing smithi02979277-47 from 659006 to 1379206 tid 1 2024-04-12T11:21:41.649 INFO:tasks.rados.rados.0.smithi029.stdout:1715: writing smithi02979277-47 from 2154681 to 2903898 tid 2 2024-04-12T11:21:41.649 INFO:tasks.rados.rados.0.smithi029.stdout:1716: read oid 43 snap -1 2024-04-12T11:21:41.649 INFO:tasks.rados.rados.0.smithi029.stdout:1716: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:21:41.649 INFO:tasks.rados.rados.0.smithi029.stdout:1717: read oid 12 snap -1 2024-04-12T11:21:41.650 INFO:tasks.rados.rados.0.smithi029.stdout:1717: expect (ObjNum 643 snap 208 seq_num 643) 2024-04-12T11:21:41.650 INFO:tasks.rados.rados.0.smithi029.stdout:1718: write oid 3 current snap is 212 2024-04-12T11:21:41.650 INFO:tasks.rados.rados.0.smithi029.stdout:1713: finishing write tid 1 to smithi02979277-17 2024-04-12T11:21:41.650 INFO:tasks.rados.rados.0.smithi029.stdout:1713: finishing write tid 2 to smithi02979277-17 2024-04-12T11:21:41.650 INFO:tasks.rados.rados.0.smithi029.stdout:1718: seq_num 659 ranges {675813=795449,2096810=609228} 2024-04-12T11:21:41.657 INFO:tasks.rados.rados.0.smithi029.stdout:1718: writing smithi02979277-3 from 675813 to 1471262 tid 1 2024-04-12T11:21:41.664 INFO:tasks.rados.rados.0.smithi029.stdout:1718: writing smithi02979277-3 from 2096810 to 2706038 tid 2 2024-04-12T11:21:41.664 INFO:tasks.rados.rados.0.smithi029.stdout:1719: snap_remove snap 211 2024-04-12T11:21:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:41 smithi087 ceph-mon[89014]: osdmap e497: 8 total, 8 up, 8 in 2024-04-12T11:21:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:41 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3646726143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:41 smithi029 ceph-mon[97787]: osdmap e497: 8 total, 8 up, 8 in 2024-04-12T11:21:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:41 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3646726143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:41 smithi029 ceph-mon[93650]: osdmap e497: 8 total, 8 up, 8 in 2024-04-12T11:21:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:41 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3646726143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:42.553 INFO:tasks.rados.rados.0.smithi029.stdout:1719: done (6 left) 2024-04-12T11:21:42.554 INFO:tasks.rados.rados.0.smithi029.stdout:1720: snap_remove snap 209 2024-04-12T11:21:42.592 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:21:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: pgmap v382: 89 pgs: 89 active+clean; 88 MiB data, 536 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: osdmap e498: 8 total, 8 up, 8 in 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-12T11:21:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-12T11:21:42.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-12T11:21:42.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-12T11:21:42.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-12T11:21:42.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:21:42.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:42.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T11:21:42.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:42 smithi087 ceph-mon[89014]: osdmap e499: 8 total, 8 up, 8 in 2024-04-12T11:21:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: pgmap v382: 89 pgs: 89 active+clean; 88 MiB data, 536 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: osdmap e498: 8 total, 8 up, 8 in 2024-04-12T11:21:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:21:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[93650]: osdmap e499: 8 total, 8 up, 8 in 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: pgmap v382: 89 pgs: 89 active+clean; 88 MiB data, 536 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: osdmap e498: 8 total, 8 up, 8 in 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:43.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-12T11:21:43.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-12T11:21:43.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-12T11:21:43.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-12T11:21:43.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-12T11:21:43.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-12T11:21:43.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:21:43.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:43.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T11:21:43.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:42 smithi029 ceph-mon[97787]: osdmap e499: 8 total, 8 up, 8 in 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1720: done (6 left) 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1721: rollback oid 19 current snap is 212 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1721: done (6 left) 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1722: read oid 19 snap -1 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1722: expect deleted 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1723: delete oid 30 current snap is 212 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1713: finishing write tid 3 to smithi02979277-17 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1713: finishing write tid 4 to smithi02979277-17 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1713: finishing write tid 5 to smithi02979277-17 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1713: oid 17 updating version 0 to 846 2024-04-12T11:21:43.557 INFO:tasks.rados.rados.0.smithi029.stdout:1713: oid 17 updating version 846 to 849 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1713: oid 17 version 849 is already newer than 847 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1713: oid 17 version 849 is already newer than 848 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 849 (ObjNum 656 snap 212 seq_num 656) dirty exists 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1713: left oid 17 (ObjNum 656 snap 212 seq_num 656) 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1714: finishing write tid 1 to smithi02979277-23 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1714: finishing write tid 2 to smithi02979277-23 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1714: finishing write tid 3 to smithi02979277-23 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1714: finishing write tid 4 to smithi02979277-23 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1714: finishing write tid 5 to smithi02979277-23 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1714: oid 23 updating version 0 to 593 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1714: oid 23 updating version 593 to 595 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1714: oid 23 version 595 is already newer than 592 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1714: oid 23 version 595 is already newer than 594 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 595 (ObjNum 657 snap 212 seq_num 657) dirty exists 2024-04-12T11:21:43.558 INFO:tasks.rados.rados.0.smithi029.stdout:1714: left oid 23 (ObjNum 657 snap 212 seq_num 657) 2024-04-12T11:21:43.559 INFO:tasks.rados.rados.0.smithi029.stdout:1715: finishing write tid 1 to smithi02979277-47 2024-04-12T11:21:43.559 INFO:tasks.rados.rados.0.smithi029.stdout:1716: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:21:43.667 INFO:tasks.rados.rados.0.smithi029.stdout:1715: finishing write tid 2 to smithi02979277-47 2024-04-12T11:21:43.667 INFO:tasks.rados.rados.0.smithi029.stdout:1715: finishing write tid 3 to smithi02979277-47 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1713: done (7 left) 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1714: done (6 left) 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1716: done (5 left) 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1723: done (4 left) 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1724: read oid 6 snap -1 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1724: expect (ObjNum 424 snap 135 seq_num 424) 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1725: read oid 33 snap -1 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1725: expect (ObjNum 632 snap 205 seq_num 632) 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1726: read oid 50 snap -1 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1726: expect (ObjNum 587 snap 189 seq_num 587) 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1727: write oid 25 current snap is 212 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1715: finishing write tid 4 to smithi02979277-47 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1715: oid 47 updating version 0 to 813 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1715: oid 47 version 813 is already newer than 811 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:1715: oid 47 version 813 is already newer than 812 2024-04-12T11:21:43.668 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 813 (ObjNum 658 snap 212 seq_num 658) dirty exists 2024-04-12T11:21:43.669 INFO:tasks.rados.rados.0.smithi029.stdout:1715: left oid 47 (ObjNum 658 snap 212 seq_num 658) 2024-04-12T11:21:43.669 INFO:tasks.rados.rados.0.smithi029.stdout:1717: expect (ObjNum 643 snap 208 seq_num 643) 2024-04-12T11:21:43.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[93650]: Upgrade: Setting container_image for all mon 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[93650]: Upgrade: Setting container_image for all crash 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[93650]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[93650]: Upgrade: osd.0 is safe to restart 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/382239759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[93650]: osdmap e500: 8 total, 8 up, 8 in 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[97787]: Upgrade: Setting container_image for all mon 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[97787]: Upgrade: Setting container_image for all crash 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[97787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[97787]: Upgrade: osd.0 is safe to restart 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/382239759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:43 smithi029 ceph-mon[97787]: osdmap e500: 8 total, 8 up, 8 in 2024-04-12T11:21:43.828 INFO:tasks.rados.rados.0.smithi029.stdout:1718: finishing write tid 1 to smithi02979277-3 2024-04-12T11:21:43.829 INFO:tasks.rados.rados.0.smithi029.stdout:1718: finishing write tid 2 to smithi02979277-3 2024-04-12T11:21:43.829 INFO:tasks.rados.rados.0.smithi029.stdout:1718: finishing write tid 3 to smithi02979277-3 2024-04-12T11:21:43.829 INFO:tasks.rados.rados.0.smithi029.stdout:1718: finishing write tid 4 to smithi02979277-3 2024-04-12T11:21:43.829 INFO:tasks.rados.rados.0.smithi029.stdout:1718: oid 3 updating version 0 to 1090 2024-04-12T11:21:43.829 INFO:tasks.rados.rados.0.smithi029.stdout:1718: oid 3 updating version 1090 to 1092 2024-04-12T11:21:43.829 INFO:tasks.rados.rados.0.smithi029.stdout:1718: oid 3 version 1092 is already newer than 1091 2024-04-12T11:21:43.829 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1092 (ObjNum 659 snap 212 seq_num 659) dirty exists 2024-04-12T11:21:43.829 INFO:tasks.rados.rados.0.smithi029.stdout:1718: left oid 3 (ObjNum 659 snap 212 seq_num 659) 2024-04-12T11:21:43.829 INFO:tasks.rados.rados.0.smithi029.stdout:1727: seq_num 661 ranges {709427=545947,1813774=612897,2985657=123770} 2024-04-12T11:21:43.835 INFO:tasks.rados.rados.0.smithi029.stdout:1727: writing smithi02979277-25 from 709427 to 1255374 tid 1 2024-04-12T11:21:43.842 INFO:tasks.rados.rados.0.smithi029.stdout:1727: writing smithi02979277-25 from 1813774 to 2426671 tid 2 2024-04-12T11:21:43.846 INFO:tasks.rados.rados.0.smithi029.stdout:1727: writing smithi02979277-25 from 2985657 to 3109427 tid 3 2024-04-12T11:21:43.846 INFO:tasks.rados.rados.0.smithi029.stdout:1715: done (7 left) 2024-04-12T11:21:43.846 INFO:tasks.rados.rados.0.smithi029.stdout:1717: done (6 left) 2024-04-12T11:21:43.846 INFO:tasks.rados.rados.0.smithi029.stdout:1718: done (5 left) 2024-04-12T11:21:43.846 INFO:tasks.rados.rados.0.smithi029.stdout:1728: read oid 17 snap -1 2024-04-12T11:21:43.846 INFO:tasks.rados.rados.0.smithi029.stdout:1728: expect (ObjNum 656 snap 212 seq_num 656) 2024-04-12T11:21:43.846 INFO:tasks.rados.rados.0.smithi029.stdout:1729: delete oid 49 current snap is 212 2024-04-12T11:21:43.846 INFO:tasks.rados.rados.0.smithi029.stdout:1724: expect (ObjNum 424 snap 135 seq_num 424) 2024-04-12T11:21:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:43 smithi087 ceph-mon[89014]: Upgrade: Setting container_image for all mon 2024-04-12T11:21:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:43 smithi087 ceph-mon[89014]: Upgrade: Setting container_image for all crash 2024-04-12T11:21:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:43 smithi087 ceph-mon[89014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T11:21:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:43 smithi087 ceph-mon[89014]: Upgrade: osd.0 is safe to restart 2024-04-12T11:21:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:43 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/382239759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:43 smithi087 ceph-mon[89014]: osdmap e500: 8 total, 8 up, 8 in 2024-04-12T11:21:44.013 INFO:tasks.rados.rados.0.smithi029.stdout:1725: expect (ObjNum 632 snap 205 seq_num 632) 2024-04-12T11:21:44.173 INFO:tasks.rados.rados.0.smithi029.stdout:1726: expect (ObjNum 587 snap 189 seq_num 587) 2024-04-12T11:21:44.348 INFO:tasks.rados.rados.0.smithi029.stdout:1727: finishing write tid 1 to smithi02979277-25 2024-04-12T11:21:44.348 INFO:tasks.rados.rados.0.smithi029.stdout:1727: finishing write tid 2 to smithi02979277-25 2024-04-12T11:21:44.349 INFO:tasks.rados.rados.0.smithi029.stdout:1727: finishing write tid 3 to smithi02979277-25 2024-04-12T11:21:44.349 INFO:tasks.rados.rados.0.smithi029.stdout:1727: finishing write tid 4 to smithi02979277-25 2024-04-12T11:21:44.349 INFO:tasks.rados.rados.0.smithi029.stdout:1727: finishing write tid 5 to smithi02979277-25 2024-04-12T11:21:44.349 INFO:tasks.rados.rados.0.smithi029.stdout:1727: oid 25 updating version 0 to 856 2024-04-12T11:21:44.349 INFO:tasks.rados.rados.0.smithi029.stdout:1727: oid 25 version 856 is already newer than 855 2024-04-12T11:21:44.349 INFO:tasks.rados.rados.0.smithi029.stdout:1727: oid 25 updating version 856 to 857 2024-04-12T11:21:44.349 INFO:tasks.rados.rados.0.smithi029.stdout:1727: oid 25 version 857 is already newer than 854 2024-04-12T11:21:44.349 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 857 (ObjNum 661 snap 212 seq_num 661) dirty exists 2024-04-12T11:21:44.349 INFO:tasks.rados.rados.0.smithi029.stdout:1727: left oid 25 (ObjNum 661 snap 212 seq_num 661) 2024-04-12T11:21:44.349 INFO:tasks.rados.rados.0.smithi029.stdout:1728: expect (ObjNum 656 snap 212 seq_num 656) 2024-04-12T11:21:44.532 INFO:tasks.rados.rados.0.smithi029.stdout:1722: done (6 left) 2024-04-12T11:21:44.532 INFO:tasks.rados.rados.0.smithi029.stdout:1724: done (5 left) 2024-04-12T11:21:44.532 INFO:tasks.rados.rados.0.smithi029.stdout:1725: done (4 left) 2024-04-12T11:21:44.532 INFO:tasks.rados.rados.0.smithi029.stdout:1726: done (3 left) 2024-04-12T11:21:44.532 INFO:tasks.rados.rados.0.smithi029.stdout:1727: done (2 left) 2024-04-12T11:21:44.532 INFO:tasks.rados.rados.0.smithi029.stdout:1728: done (1 left) 2024-04-12T11:21:44.532 INFO:tasks.rados.rados.0.smithi029.stdout:1729: done (0 left) 2024-04-12T11:21:44.532 INFO:tasks.rados.rados.0.smithi029.stdout:1730: delete oid 36 current snap is 212 2024-04-12T11:21:44.533 INFO:tasks.rados.rados.0.smithi029.stdout:1730: done (0 left) 2024-04-12T11:21:44.533 INFO:tasks.rados.rados.0.smithi029.stdout:1731: delete oid 27 current snap is 212 2024-04-12T11:21:44.535 INFO:tasks.rados.rados.0.smithi029.stdout:1731: done (0 left) 2024-04-12T11:21:44.535 INFO:tasks.rados.rados.0.smithi029.stdout:1732: read oid 13 snap -1 2024-04-12T11:21:44.535 INFO:tasks.rados.rados.0.smithi029.stdout:1732: expect (ObjNum 626 snap 203 seq_num 626) 2024-04-12T11:21:44.535 INFO:tasks.rados.rados.0.smithi029.stdout:1733: snap_create 2024-04-12T11:21:44.540 INFO:tasks.rados.rados.0.smithi029.stdout:1732: expect (ObjNum 626 snap 203 seq_num 626) 2024-04-12T11:21:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:44 smithi029 ceph-mon[93650]: pgmap v385: 89 pgs: 89 active+clean; 100 MiB data, 542 MiB used, 715 GiB / 715 GiB avail; 5.4 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-04-12T11:21:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T11:21:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:21:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:44 smithi029 ceph-mon[97787]: pgmap v385: 89 pgs: 89 active+clean; 100 MiB data, 542 MiB used, 715 GiB / 715 GiB avail; 5.4 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-04-12T11:21:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T11:21:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:21:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:44 smithi087 ceph-mon[89014]: pgmap v385: 89 pgs: 89 active+clean; 100 MiB data, 542 MiB used, 715 GiB / 715 GiB avail; 5.4 MiB/s rd, 2.5 MiB/s wr, 15 op/s 2024-04-12T11:21:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T11:21:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:21:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:21:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:21:44] "GET /metrics HTTP/1.1" 200 36241 "" "Prometheus/2.43.0" 2024-04-12T11:21:45.404 INFO:tasks.rados.rados.0.smithi029.stdout:1732: done (1 left) 2024-04-12T11:21:45.405 INFO:tasks.rados.rados.0.smithi029.stdout:1733: done (0 left) 2024-04-12T11:21:45.405 INFO:tasks.rados.rados.0.smithi029.stdout:1734: snap_create 2024-04-12T11:21:45.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:45 smithi087 ceph-mon[89014]: Upgrade: Updating osd.0 2024-04-12T11:21:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:45 smithi087 ceph-mon[89014]: Deploying daemon osd.0 on smithi029 2024-04-12T11:21:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:45 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/873993367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:45 smithi087 ceph-mon[89014]: osdmap e501: 8 total, 8 up, 8 in 2024-04-12T11:21:46.003 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:45 smithi029 ceph-mon[97787]: Upgrade: Updating osd.0 2024-04-12T11:21:46.003 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:45 smithi029 ceph-mon[97787]: Deploying daemon osd.0 on smithi029 2024-04-12T11:21:46.003 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:45 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/873993367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:46.003 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:45 smithi029 ceph-mon[97787]: osdmap e501: 8 total, 8 up, 8 in 2024-04-12T11:21:46.003 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:45 smithi029 ceph-mon[93650]: Upgrade: Updating osd.0 2024-04-12T11:21:46.003 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:45 smithi029 ceph-mon[93650]: Deploying daemon osd.0 on smithi029 2024-04-12T11:21:46.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:45 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/873993367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:46.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:45 smithi029 ceph-mon[93650]: osdmap e501: 8 total, 8 up, 8 in 2024-04-12T11:21:46.407 INFO:tasks.rados.rados.0.smithi029.stdout:1734: done (0 left) 2024-04-12T11:21:46.407 INFO:tasks.rados.rados.0.smithi029.stdout:1735: write oid 50 current snap is 214 2024-04-12T11:21:46.407 INFO:tasks.rados.rados.0.smithi029.stdout:1735: seq_num 665 ranges {776655=563296,2047702=620235,3095349=81306} 2024-04-12T11:21:46.414 INFO:tasks.rados.rados.0.smithi029.stdout:1735: writing smithi02979277-50 from 776655 to 1339951 tid 1 2024-04-12T11:21:46.421 INFO:tasks.rados.rados.0.smithi029.stdout:1735: writing smithi02979277-50 from 2047702 to 2667937 tid 2 2024-04-12T11:21:46.425 INFO:tasks.rados.rados.0.smithi029.stdout:1735: writing smithi02979277-50 from 3095349 to 3176655 tid 3 2024-04-12T11:21:46.425 INFO:tasks.rados.rados.0.smithi029.stdout:1736: read oid 40 snap -1 2024-04-12T11:21:46.425 INFO:tasks.rados.rados.0.smithi029.stdout:1736: expect deleted 2024-04-12T11:21:46.425 INFO:tasks.rados.rados.0.smithi029.stdout:1737: snap_create 2024-04-12T11:21:46.425 INFO:tasks.rados.rados.0.smithi029.stdout:1735: finishing write tid 1 to smithi02979277-50 2024-04-12T11:21:46.426 INFO:tasks.rados.rados.0.smithi029.stdout:1735: finishing write tid 2 to smithi02979277-50 2024-04-12T11:21:46.426 INFO:tasks.rados.rados.0.smithi029.stdout:1735: finishing write tid 3 to smithi02979277-50 2024-04-12T11:21:46.427 INFO:tasks.rados.rados.0.smithi029.stdout:1735: finishing write tid 4 to smithi02979277-50 2024-04-12T11:21:46.427 INFO:tasks.rados.rados.0.smithi029.stdout:1735: finishing write tid 5 to smithi02979277-50 2024-04-12T11:21:46.427 INFO:tasks.rados.rados.0.smithi029.stdout:1735: oid 50 updating version 0 to 601 2024-04-12T11:21:46.427 INFO:tasks.rados.rados.0.smithi029.stdout:1735: oid 50 updating version 601 to 604 2024-04-12T11:21:46.427 INFO:tasks.rados.rados.0.smithi029.stdout:1735: oid 50 version 604 is already newer than 602 2024-04-12T11:21:46.427 INFO:tasks.rados.rados.0.smithi029.stdout:1735: oid 50 version 604 is already newer than 603 2024-04-12T11:21:46.427 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 604 (ObjNum 665 snap 214 seq_num 665) dirty exists 2024-04-12T11:21:46.427 INFO:tasks.rados.rados.0.smithi029.stdout:1735: left oid 50 (ObjNum 665 snap 214 seq_num 665) 2024-04-12T11:21:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:46 smithi087 ceph-mon[89014]: pgmap v387: 89 pgs: 89 active+clean; 100 MiB data, 542 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-04-12T11:21:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:46 smithi087 ceph-mon[89014]: osdmap e502: 8 total, 8 up, 8 in 2024-04-12T11:21:47.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:46 smithi029 ceph-mon[93650]: pgmap v387: 89 pgs: 89 active+clean; 100 MiB data, 542 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-04-12T11:21:47.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:46 smithi029 ceph-mon[93650]: osdmap e502: 8 total, 8 up, 8 in 2024-04-12T11:21:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:46 smithi029 ceph-mon[97787]: pgmap v387: 89 pgs: 89 active+clean; 100 MiB data, 542 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-04-12T11:21:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:46 smithi029 ceph-mon[97787]: osdmap e502: 8 total, 8 up, 8 in 2024-04-12T11:21:47.408 INFO:tasks.rados.rados.0.smithi029.stdout:1735: done (2 left) 2024-04-12T11:21:47.409 INFO:tasks.rados.rados.0.smithi029.stdout:1736: done (1 left) 2024-04-12T11:21:47.409 INFO:tasks.rados.rados.0.smithi029.stdout:1737: done (0 left) 2024-04-12T11:21:47.409 INFO:tasks.rados.rados.0.smithi029.stdout:1738: read oid 25 snap -1 2024-04-12T11:21:47.409 INFO:tasks.rados.rados.0.smithi029.stdout:1738: expect (ObjNum 661 snap 212 seq_num 661) 2024-04-12T11:21:47.409 INFO:tasks.rados.rados.0.smithi029.stdout:1739: read oid 1 snap -1 2024-04-12T11:21:47.409 INFO:tasks.rados.rados.0.smithi029.stdout:1739: expect (ObjNum 646 snap 208 seq_num 646) 2024-04-12T11:21:47.409 INFO:tasks.rados.rados.0.smithi029.stdout:1740: write oid 29 current snap is 215 2024-04-12T11:21:47.409 INFO:tasks.rados.rados.0.smithi029.stdout:1740: seq_num 666 ranges {793462=638545,2106184=763893} 2024-04-12T11:21:47.416 INFO:tasks.rados.rados.0.smithi029.stdout:1740: writing smithi02979277-29 from 793462 to 1432007 tid 1 2024-04-12T11:21:47.425 INFO:tasks.rados.rados.0.smithi029.stdout:1740: writing smithi02979277-29 from 2106184 to 2870077 tid 2 2024-04-12T11:21:47.425 INFO:tasks.rados.rados.0.smithi029.stdout:1741: delete oid 3 current snap is 215 2024-04-12T11:21:47.425 INFO:tasks.rados.rados.0.smithi029.stdout:1740: finishing write tid 1 to smithi02979277-29 2024-04-12T11:21:47.425 INFO:tasks.rados.rados.0.smithi029.stdout:1739: expect (ObjNum 646 snap 208 seq_num 646) 2024-04-12T11:21:47.602 INFO:tasks.rados.rados.0.smithi029.stdout:1738: expect (ObjNum 661 snap 212 seq_num 661) 2024-04-12T11:21:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:1740: finishing write tid 2 to smithi02979277-29 2024-04-12T11:21:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:1740: finishing write tid 3 to smithi02979277-29 2024-04-12T11:21:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:1740: finishing write tid 4 to smithi02979277-29 2024-04-12T11:21:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:1740: oid 29 updating version 0 to 661 2024-04-12T11:21:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:1740: oid 29 updating version 661 to 663 2024-04-12T11:21:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:1740: oid 29 version 663 is already newer than 662 2024-04-12T11:21:47.837 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 663 (ObjNum 666 snap 215 seq_num 666) dirty exists 2024-04-12T11:21:47.838 INFO:tasks.rados.rados.0.smithi029.stdout:1740: left oid 29 (ObjNum 666 snap 215 seq_num 666) 2024-04-12T11:21:47.838 INFO:tasks.rados.rados.0.smithi029.stdout:1738: done (3 left) 2024-04-12T11:21:47.838 INFO:tasks.rados.rados.0.smithi029.stdout:1739: done (2 left) 2024-04-12T11:21:47.838 INFO:tasks.rados.rados.0.smithi029.stdout:1740: done (1 left) 2024-04-12T11:21:47.838 INFO:tasks.rados.rados.0.smithi029.stdout:1741: done (0 left) 2024-04-12T11:21:47.838 INFO:tasks.rados.rados.0.smithi029.stdout:1742: delete oid 44 current snap is 215 2024-04-12T11:21:47.839 INFO:tasks.rados.rados.0.smithi029.stdout:1742: done (0 left) 2024-04-12T11:21:47.839 INFO:tasks.rados.rados.0.smithi029.stdout:1743: write oid 25 current snap is 215 2024-04-12T11:21:47.839 INFO:tasks.rados.rados.0.smithi029.stdout:1743: seq_num 669 ranges {443883=464292,1365277=511220,2572335=671548} 2024-04-12T11:21:47.845 INFO:tasks.rados.rados.0.smithi029.stdout:1743: writing smithi02979277-25 from 443883 to 908175 tid 1 2024-04-12T11:21:47.850 INFO:tasks.rados.rados.0.smithi029.stdout:1743: writing smithi02979277-25 from 1365277 to 1876497 tid 2 2024-04-12T11:21:47.858 INFO:tasks.rados.rados.0.smithi029.stdout:1743: writing smithi02979277-25 from 2572335 to 3243883 tid 3 2024-04-12T11:21:47.858 INFO:tasks.rados.rados.0.smithi029.stdout:1744: snap_remove snap 214 2024-04-12T11:21:47.977 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:47 smithi029 systemd[1]: Stopping Ceph osd.0 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:21:48.268 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:47 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[43889]: 2024-04-12T11:21:47.976+0000 7f3163a0b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:21:48.268 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:47 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[43889]: 2024-04-12T11:21:47.976+0000 7f3163a0b700 -1 osd.0 503 *** Got signal Terminated *** 2024-04-12T11:21:48.268 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:47 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[43889]: 2024-04-12T11:21:47.976+0000 7f3163a0b700 -1 osd.0 503 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T11:21:48.613 INFO:tasks.rados.rados.0.smithi029.stdout:1744: done (1 left) 2024-04-12T11:21:48.613 INFO:tasks.rados.rados.0.smithi029.stdout:1745: snap_remove snap 213 2024-04-12T11:21:48.613 INFO:tasks.rados.rados.0.smithi029.stdout:1743: finishing write tid 1 to smithi02979277-25 2024-04-12T11:21:48.613 INFO:tasks.rados.rados.0.smithi029.stdout:1743: finishing write tid 2 to smithi02979277-25 2024-04-12T11:21:48.613 INFO:tasks.rados.rados.0.smithi029.stdout:1743: finishing write tid 3 to smithi02979277-25 2024-04-12T11:21:48.613 INFO:tasks.rados.rados.0.smithi029.stdout:1743: finishing write tid 4 to smithi02979277-25 2024-04-12T11:21:48.613 INFO:tasks.rados.rados.0.smithi029.stdout:1743: finishing write tid 5 to smithi02979277-25 2024-04-12T11:21:48.613 INFO:tasks.rados.rados.0.smithi029.stdout:1743: oid 25 updating version 0 to 862 2024-04-12T11:21:48.613 INFO:tasks.rados.rados.0.smithi029.stdout:1743: oid 25 version 862 is already newer than 859 2024-04-12T11:21:48.614 INFO:tasks.rados.rados.0.smithi029.stdout:1743: oid 25 version 862 is already newer than 860 2024-04-12T11:21:48.614 INFO:tasks.rados.rados.0.smithi029.stdout:1743: oid 25 version 862 is already newer than 861 2024-04-12T11:21:48.614 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 862 (ObjNum 669 snap 215 seq_num 669) dirty exists 2024-04-12T11:21:48.614 INFO:tasks.rados.rados.0.smithi029.stdout:1743: left oid 25 (ObjNum 669 snap 215 seq_num 669) 2024-04-12T11:21:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:48 smithi087 ceph-mon[89014]: pgmap v390: 89 pgs: 89 active+clean; 84 MiB data, 515 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 532 KiB/s wr, 11 op/s 2024-04-12T11:21:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:48 smithi087 ceph-mon[89014]: osdmap e503: 8 total, 8 up, 8 in 2024-04-12T11:21:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:48 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1556595888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:48 smithi087 ceph-mon[89014]: osd.0 marked itself down and dead 2024-04-12T11:21:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:48 smithi029 ceph-mon[93650]: pgmap v390: 89 pgs: 89 active+clean; 84 MiB data, 515 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 532 KiB/s wr, 11 op/s 2024-04-12T11:21:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:48 smithi029 ceph-mon[93650]: osdmap e503: 8 total, 8 up, 8 in 2024-04-12T11:21:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:48 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1556595888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:48 smithi029 ceph-mon[93650]: osd.0 marked itself down and dead 2024-04-12T11:21:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:48 smithi029 ceph-mon[97787]: pgmap v390: 89 pgs: 89 active+clean; 84 MiB data, 515 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 532 KiB/s wr, 11 op/s 2024-04-12T11:21:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:48 smithi029 ceph-mon[97787]: osdmap e503: 8 total, 8 up, 8 in 2024-04-12T11:21:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:48 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1556595888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:48 smithi029 ceph-mon[97787]: osd.0 marked itself down and dead 2024-04-12T11:21:49.408 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:49 smithi029 podman[105776]: 2024-04-12 11:21:49.143663184 +0000 UTC m=+1.268843593 container died f65c324b008450ceca2ddbb3f7a74c2b0e0038e631f2021a765f3a7254a42489 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=reef-d540eba, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:21:49.619 INFO:tasks.rados.rados.0.smithi029.stdout:1743: done (1 left) 2024-04-12T11:21:49.619 INFO:tasks.rados.rados.0.smithi029.stdout:1745: done (0 left) 2024-04-12T11:21:49.619 INFO:tasks.rados.rados.0.smithi029.stdout:1746: snap_remove snap 212 2024-04-12T11:21:49.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:49 smithi087 ceph-mon[89014]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:21:49.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:49 smithi087 ceph-mon[89014]: osdmap e504: 8 total, 7 up, 8 in 2024-04-12T11:21:49.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:49 smithi029 ceph-mon[97787]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:21:49.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:49 smithi029 ceph-mon[97787]: osdmap e504: 8 total, 7 up, 8 in 2024-04-12T11:21:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:49 smithi029 ceph-mon[93650]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:21:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:49 smithi029 ceph-mon[93650]: osdmap e504: 8 total, 7 up, 8 in 2024-04-12T11:21:49.768 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:49 smithi029 podman[105776]: 2024-04-12 11:21:49.555845821 +0000 UTC m=+1.681026231 container cleanup f65c324b008450ceca2ddbb3f7a74c2b0e0038e631f2021a765f3a7254a42489 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0, org.label-schema.build-date=20240403, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=reef-d540eba, ceph=True) 2024-04-12T11:21:49.768 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:49 smithi029 bash[105776]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0 2024-04-12T11:21:50.102 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:49 smithi029 podman[105794]: 2024-04-12 11:21:49.7837285 +0000 UTC m=+0.631977212 container remove f65c324b008450ceca2ddbb3f7a74c2b0e0038e631f2021a765f3a7254a42489 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=reef-d540eba) 2024-04-12T11:21:50.375 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:50 smithi029 podman[105903]: 2024-04-12 11:21:50.102346787 +0000 UTC m=+0.109385153 container create 332f17c40b00a0324d44208e5d2c1c2172d22b90f0740f56b6c585d0d0059d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409) 2024-04-12T11:21:50.375 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:50 smithi029 podman[105903]: 2024-04-12 11:21:50.011592424 +0000 UTC m=+0.018630804 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:21:50.375 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:50 smithi029 podman[105903]: 2024-04-12 11:21:50.375249872 +0000 UTC m=+0.382288251 container init 332f17c40b00a0324d44208e5d2c1c2172d22b90f0740f56b6c585d0d0059d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409) 2024-04-12T11:21:50.618 INFO:tasks.rados.rados.0.smithi029.stdout:1746: done (0 left) 2024-04-12T11:21:50.618 INFO:tasks.rados.rados.0.smithi029.stdout:1747: write oid 12 current snap is 215 2024-04-12T11:21:50.618 INFO:tasks.rados.rados.0.smithi029.stdout:1747: seq_num 670 ranges {460690=655894,1540112=771231,2716111=539778} 2024-04-12T11:21:50.625 INFO:tasks.rados.rados.0.smithi029.stdout:1747: writing smithi02979277-12 from 460690 to 1116584 tid 1 2024-04-12T11:21:50.632 INFO:tasks.rados.rados.0.smithi029.stdout:1747: writing smithi02979277-12 from 1540112 to 2311343 tid 2 2024-04-12T11:21:50.637 INFO:tasks.rados.rados.0.smithi029.stdout:1747: writing smithi02979277-12 from 2716111 to 3255889 tid 3 2024-04-12T11:21:50.637 INFO:tasks.rados.rados.0.smithi029.stdout:1748: rollback oid 41 current snap is 215 2024-04-12T11:21:50.637 INFO:tasks.rados.rados.0.smithi029.stdout:1748: done (1 left) 2024-04-12T11:21:50.637 INFO:tasks.rados.rados.0.smithi029.stdout:1749: write oid 20 current snap is 215 2024-04-12T11:21:50.637 INFO:tasks.rados.rados.0.smithi029.stdout:1747: finishing write tid 1 to smithi02979277-12 2024-04-12T11:21:50.637 INFO:tasks.rados.rados.0.smithi029.stdout:1747: finishing write tid 2 to smithi02979277-12 2024-04-12T11:21:50.637 INFO:tasks.rados.rados.0.smithi029.stdout:1749: seq_num 671 ranges {477497=731143,1998594=514889,3143534=133963} 2024-04-12T11:21:50.643 INFO:tasks.rados.rados.0.smithi029.stdout:1749: writing smithi02979277-20 from 477497 to 1208640 tid 1 2024-04-12T11:21:50.651 INFO:tasks.rados.rados.0.smithi029.stdout:1749: writing smithi02979277-20 from 1998594 to 2513483 tid 2 2024-04-12T11:21:50.655 INFO:tasks.rados.rados.0.smithi029.stdout:1749: writing smithi02979277-20 from 3143534 to 3277497 tid 3 2024-04-12T11:21:50.655 INFO:tasks.rados.rados.0.smithi029.stdout:1750: read oid 19 snap -1 2024-04-12T11:21:50.655 INFO:tasks.rados.rados.0.smithi029.stdout:1750: expect deleted 2024-04-12T11:21:50.656 INFO:tasks.rados.rados.0.smithi029.stdout:1751: write oid 21 current snap is 215 2024-04-12T11:21:50.656 INFO:tasks.rados.rados.0.smithi029.stdout:1747: finishing write tid 3 to smithi02979277-12 2024-04-12T11:21:50.656 INFO:tasks.rados.rados.0.smithi029.stdout:1751: seq_num 672 ranges {494304=406392,1540723=774900,2770957=523347} 2024-04-12T11:21:50.660 INFO:tasks.rados.rados.0.smithi029.stdout:1751: writing smithi02979277-21 from 494304 to 900696 tid 1 2024-04-12T11:21:50.668 INFO:tasks.rados.rados.0.smithi029.stdout:1751: writing smithi02979277-21 from 1540723 to 2315623 tid 2 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1751: writing smithi02979277-21 from 2770957 to 3294304 tid 3 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1752: write oid 18 current snap is 215 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1747: finishing write tid 4 to smithi02979277-12 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1747: finishing write tid 5 to smithi02979277-12 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1747: oid 12 updating version 0 to 820 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1747: oid 12 version 820 is already newer than 819 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1747: oid 12 updating version 820 to 821 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1747: oid 12 version 821 is already newer than 818 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 821 (ObjNum 670 snap 215 seq_num 670) dirty exists 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1747: left oid 12 (ObjNum 670 snap 215 seq_num 670) 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1749: finishing write tid 1 to smithi02979277-20 2024-04-12T11:21:50.674 INFO:tasks.rados.rados.0.smithi029.stdout:1752: seq_num 673 ranges {511111=481641,1599205=518558,2682027=489947} 2024-04-12T11:21:50.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:50 smithi029 ceph-mon[97787]: pgmap v393: 89 pgs: 13 stale+active+clean, 76 active+clean; 84 MiB data, 515 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 623 KiB/s wr, 13 op/s 2024-04-12T11:21:50.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:50 smithi029 ceph-mon[97787]: osdmap e505: 8 total, 7 up, 8 in 2024-04-12T11:21:50.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:50 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3304676132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:50.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:50 smithi029 ceph-mon[93650]: pgmap v393: 89 pgs: 13 stale+active+clean, 76 active+clean; 84 MiB data, 515 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 623 KiB/s wr, 13 op/s 2024-04-12T11:21:50.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:50 smithi029 ceph-mon[93650]: osdmap e505: 8 total, 7 up, 8 in 2024-04-12T11:21:50.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:50 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3304676132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:50.677 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:50 smithi029 podman[105903]: 2024-04-12 11:21:50.378572592 +0000 UTC m=+0.385610960 container start 332f17c40b00a0324d44208e5d2c1c2172d22b90f0740f56b6c585d0d0059d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185) 2024-04-12T11:21:50.677 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:50 smithi029 podman[105903]: 2024-04-12 11:21:50.475480929 +0000 UTC m=+0.482519302 container attach 332f17c40b00a0324d44208e5d2c1c2172d22b90f0740f56b6c585d0d0059d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T11:21:50.679 INFO:tasks.rados.rados.0.smithi029.stdout:1752: writing smithi02979277-18 from 511111 to 992752 tid 1 2024-04-12T11:21:50.685 INFO:tasks.rados.rados.0.smithi029.stdout:1752: writing smithi02979277-18 from 1599205 to 2117763 tid 2 2024-04-12T11:21:50.691 INFO:tasks.rados.rados.0.smithi029.stdout:1752: writing smithi02979277-18 from 2682027 to 3171974 tid 3 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1747: done (4 left) 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1753: snap_create 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1749: finishing write tid 2 to smithi02979277-20 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1749: finishing write tid 3 to smithi02979277-20 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1749: finishing write tid 4 to smithi02979277-20 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1749: finishing write tid 5 to smithi02979277-20 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1749: oid 20 updating version 0 to 485 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1749: oid 20 updating version 485 to 486 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1749: oid 20 version 486 is already newer than 483 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1749: oid 20 version 486 is already newer than 484 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 486 (ObjNum 671 snap 215 seq_num 671) dirty exists 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1749: left oid 20 (ObjNum 671 snap 215 seq_num 671) 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1751: finishing write tid 1 to smithi02979277-21 2024-04-12T11:21:50.692 INFO:tasks.rados.rados.0.smithi029.stdout:1751: finishing write tid 2 to smithi02979277-21 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:1751: finishing write tid 3 to smithi02979277-21 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:1751: finishing write tid 4 to smithi02979277-21 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:1751: finishing write tid 5 to smithi02979277-21 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:1751: oid 21 updating version 0 to 841 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:1751: oid 21 updating version 841 to 842 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:1751: oid 21 updating version 842 to 843 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:1751: oid 21 updating version 843 to 844 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 844 (ObjNum 672 snap 215 seq_num 672) dirty exists 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:1751: left oid 21 (ObjNum 672 snap 215 seq_num 672) 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:1752: finishing write tid 1 to smithi02979277-18 2024-04-12T11:21:50.693 INFO:tasks.rados.rados.0.smithi029.stdout:1752: finishing write tid 2 to smithi02979277-18 2024-04-12T11:21:50.694 INFO:tasks.rados.rados.0.smithi029.stdout:1752: finishing write tid 3 to smithi02979277-18 2024-04-12T11:21:50.694 INFO:tasks.rados.rados.0.smithi029.stdout:1752: finishing write tid 4 to smithi02979277-18 2024-04-12T11:21:50.694 INFO:tasks.rados.rados.0.smithi029.stdout:1752: finishing write tid 5 to smithi02979277-18 2024-04-12T11:21:50.694 INFO:tasks.rados.rados.0.smithi029.stdout:1752: oid 18 updating version 0 to 872 2024-04-12T11:21:50.695 INFO:tasks.rados.rados.0.smithi029.stdout:1752: oid 18 version 872 is already newer than 869 2024-04-12T11:21:50.695 INFO:tasks.rados.rados.0.smithi029.stdout:1752: oid 18 version 872 is already newer than 871 2024-04-12T11:21:50.695 INFO:tasks.rados.rados.0.smithi029.stdout:1752: oid 18 version 872 is already newer than 870 2024-04-12T11:21:50.695 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 872 (ObjNum 673 snap 215 seq_num 673) dirty exists 2024-04-12T11:21:50.695 INFO:tasks.rados.rados.0.smithi029.stdout:1752: left oid 18 (ObjNum 673 snap 215 seq_num 673) 2024-04-12T11:21:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:50 smithi087 ceph-mon[89014]: pgmap v393: 89 pgs: 13 stale+active+clean, 76 active+clean; 84 MiB data, 515 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 623 KiB/s wr, 13 op/s 2024-04-12T11:21:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:50 smithi087 ceph-mon[89014]: osdmap e505: 8 total, 7 up, 8 in 2024-04-12T11:21:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:50 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3304676132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:51.018 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:50 smithi029 podman[105903]: 2024-04-12 11:21:50.677115522 +0000 UTC m=+0.684153881 container died 332f17c40b00a0324d44208e5d2c1c2172d22b90f0740f56b6c585d0d0059d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=) 2024-04-12T11:21:51.304 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:51 smithi029 podman[105932]: 2024-04-12 11:21:51.284430501 +0000 UTC m=+0.599497677 container remove 332f17c40b00a0324d44208e5d2c1c2172d22b90f0740f56b6c585d0d0059d33 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-rf-comb1-2cee185, CEPH_POINT_RELEASE=) 2024-04-12T11:21:51.626 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:51 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-12T11:21:51.626 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:51 smithi029 systemd[1]: Stopped Ceph osd.0 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:21:51.626 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:51 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.0.service: Consumed 14.234s CPU time. 2024-04-12T11:21:51.626 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:51 smithi029 systemd[1]: Starting Ceph osd.0 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:21:51.629 INFO:tasks.rados.rados.0.smithi029.stdout:1749: done (4 left) 2024-04-12T11:21:51.629 INFO:tasks.rados.rados.0.smithi029.stdout:1750: done (3 left) 2024-04-12T11:21:51.629 INFO:tasks.rados.rados.0.smithi029.stdout:1751: done (2 left) 2024-04-12T11:21:51.629 INFO:tasks.rados.rados.0.smithi029.stdout:1752: done (1 left) 2024-04-12T11:21:51.629 INFO:tasks.rados.rados.0.smithi029.stdout:1753: done (0 left) 2024-04-12T11:21:51.629 INFO:tasks.rados.rados.0.smithi029.stdout:1754: snap_remove snap 215 2024-04-12T11:21:51.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:51 smithi029 ceph-mon[97787]: osdmap e506: 8 total, 7 up, 8 in 2024-04-12T11:21:51.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:51.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:51.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:51 smithi029 ceph-mon[93650]: osdmap e506: 8 total, 7 up, 8 in 2024-04-12T11:21:51.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:51.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:51.909 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:51 smithi029 podman[106036]: 2024-04-12 11:21:51.629401375 +0000 UTC m=+0.020074077 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:21:51.909 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:51 smithi029 podman[106036]: 2024-04-12 11:21:51.75412932 +0000 UTC m=+0.144802043 container create 5f994880b35bd599edcfa0f9bce5739f4ed9eceafebf70c08d0e6c535e3717c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-12T11:21:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:51 smithi087 ceph-mon[89014]: osdmap e506: 8 total, 7 up, 8 in 2024-04-12T11:21:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:21:52.268 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 podman[106036]: 2024-04-12 11:21:52.027765898 +0000 UTC m=+0.418438583 container init 5f994880b35bd599edcfa0f9bce5739f4ed9eceafebf70c08d0e6c535e3717c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-12T11:21:52.268 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 podman[106036]: 2024-04-12 11:21:52.033010169 +0000 UTC m=+0.423682865 container start 5f994880b35bd599edcfa0f9bce5739f4ed9eceafebf70c08d0e6c535e3717c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-04-12T11:21:52.268 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 podman[106036]: 2024-04-12 11:21:52.136044973 +0000 UTC m=+0.526717676 container attach 5f994880b35bd599edcfa0f9bce5739f4ed9eceafebf70c08d0e6c535e3717c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-12T11:21:52.556 INFO:tasks.rados.rados.0.smithi029.stdout:1754: done (0 left) 2024-04-12T11:21:52.556 INFO:tasks.rados.rados.0.smithi029.stdout:1755: snap_create 2024-04-12T11:21:52.626 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:21:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:21:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:21:52.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-mon[93650]: pgmap v396: 89 pgs: 13 stale+active+clean, 76 active+clean; 84 MiB data, 515 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:52.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-mon[93650]: osdmap e507: 8 total, 7 up, 8 in 2024-04-12T11:21:52.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1425356043' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:52.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-mon[93650]: osdmap e508: 8 total, 7 up, 8 in 2024-04-12T11:21:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-mon[97787]: pgmap v396: 89 pgs: 13 stale+active+clean, 76 active+clean; 84 MiB data, 515 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-mon[97787]: osdmap e507: 8 total, 7 up, 8 in 2024-04-12T11:21:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1425356043' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-mon[97787]: osdmap e508: 8 total, 7 up, 8 in 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate[106051]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate[106051]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 bash[106036]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 bash[106036]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 bash[106036]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 bash[106036]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 bash[106036]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 bash[106036]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 bash[106036]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate[106051]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate[106051]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate[106051]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate[106051]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T11:21:52.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate[106051]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-12T11:21:52.770 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:52 smithi029 podman[106036]: 2024-04-12 11:21:52.507430357 +0000 UTC m=+0.898103044 container died 5f994880b35bd599edcfa0f9bce5739f4ed9eceafebf70c08d0e6c535e3717c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-12T11:21:52.898 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:52 smithi087 ceph-mon[89014]: pgmap v396: 89 pgs: 13 stale+active+clean, 76 active+clean; 84 MiB data, 515 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:21:52.898 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:52 smithi087 ceph-mon[89014]: osdmap e507: 8 total, 7 up, 8 in 2024-04-12T11:21:52.898 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:52 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1425356043' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:52.898 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:52 smithi087 ceph-mon[89014]: osdmap e508: 8 total, 7 up, 8 in 2024-04-12T11:21:53.518 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:53 smithi029 podman[106194]: 2024-04-12 11:21:53.162139299 +0000 UTC m=+0.649931524 container remove 5f994880b35bd599edcfa0f9bce5739f4ed9eceafebf70c08d0e6c535e3717c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True) 2024-04-12T11:21:53.518 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:53 smithi029 podman[106256]: 2024-04-12 11:21:53.283184707 +0000 UTC m=+0.018393195 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:21:53.518 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:53 smithi029 podman[106256]: 2024-04-12 11:21:53.405575632 +0000 UTC m=+0.140784108 container create 78ce9733fe38346de4f4bc0a3f3a08409a968bf9c477438680ad11dbb22f8102 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, ceph=True, io.buildah.version=1.33.5) 2024-04-12T11:21:53.557 INFO:tasks.rados.rados.0.smithi029.stdout:1755: done (0 left) 2024-04-12T11:21:53.557 INFO:tasks.rados.rados.0.smithi029.stdout:1756: read oid 19 snap -1 2024-04-12T11:21:53.557 INFO:tasks.rados.rados.0.smithi029.stdout:1756: expect deleted 2024-04-12T11:21:53.557 INFO:tasks.rados.rados.0.smithi029.stdout:1757: snap_create 2024-04-12T11:21:54.018 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:53 smithi029 podman[106256]: 2024-04-12 11:21:53.636657285 +0000 UTC m=+0.371865771 container init 78ce9733fe38346de4f4bc0a3f3a08409a968bf9c477438680ad11dbb22f8102 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-12T11:21:54.018 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:53 smithi029 podman[106256]: 2024-04-12 11:21:53.639889499 +0000 UTC m=+0.375097978 container start 78ce9733fe38346de4f4bc0a3f3a08409a968bf9c477438680ad11dbb22f8102 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-12T11:21:54.018 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:53 smithi029 ceph-osd[106275]: -- 172.21.15.29:0/3162048498 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a7edec5680 con 0x55a7ede8ec00 2024-04-12T11:21:54.018 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:53 smithi029 bash[106256]: 78ce9733fe38346de4f4bc0a3f3a08409a968bf9c477438680ad11dbb22f8102 2024-04-12T11:21:54.018 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:53 smithi029 systemd[1]: Started Ceph osd.0 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:21:54.518 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[106271]: 2024-04-12T11:21:54.176+0000 7f4b345097c0 -1 Falling back to public interface 2024-04-12T11:21:54.559 INFO:tasks.rados.rados.0.smithi029.stdout:1756: done (1 left) 2024-04-12T11:21:54.559 INFO:tasks.rados.rados.0.smithi029.stdout:1757: done (0 left) 2024-04-12T11:21:54.559 INFO:tasks.rados.rados.0.smithi029.stdout:1758: delete oid 42 current snap is 218 2024-04-12T11:21:54.565 INFO:tasks.rados.rados.0.smithi029.stdout:1758: done (0 left) 2024-04-12T11:21:54.565 INFO:tasks.rados.rados.0.smithi029.stdout:1759: rollback oid 48 current snap is 218 2024-04-12T11:21:54.565 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 217 2024-04-12T11:21:54.565 INFO:tasks.rados.rados.0.smithi029.stdout:1760: write oid 24 current snap is 218 2024-04-12T11:21:54.565 INFO:tasks.rados.rados.0.smithi029.stdout:1760: seq_num 675 ranges {544725=632139,1716169=522227,2736873=512491} 2024-04-12T11:21:54.571 INFO:tasks.rados.rados.0.smithi029.stdout:1760: writing smithi02979277-24 from 544725 to 1176864 tid 1 2024-04-12T11:21:54.577 INFO:tasks.rados.rados.0.smithi029.stdout:1760: writing smithi02979277-24 from 1716169 to 2238396 tid 2 2024-04-12T11:21:54.583 INFO:tasks.rados.rados.0.smithi029.stdout:1760: writing smithi02979277-24 from 2736873 to 3249364 tid 3 2024-04-12T11:21:54.583 INFO:tasks.rados.rados.0.smithi029.stdout:1761: delete oid 11 current snap is 218 2024-04-12T11:21:54.583 INFO:tasks.rados.rados.0.smithi029.stdout:1759: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:21:54.583 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 0 (ObjNum 177 snap 0 seq_num 1798758080) dirty dne 2024-04-12T11:21:54.583 INFO:tasks.rados.rados.0.smithi029.stdout:1760: finishing write tid 1 to smithi02979277-24 2024-04-12T11:21:54.583 INFO:tasks.rados.rados.0.smithi029.stdout:1760: finishing write tid 2 to smithi02979277-24 2024-04-12T11:21:54.587 INFO:tasks.rados.rados.0.smithi029.stdout:1760: finishing write tid 3 to smithi02979277-24 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:1760: finishing write tid 4 to smithi02979277-24 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:1760: finishing write tid 5 to smithi02979277-24 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:1760: oid 24 updating version 0 to 1106 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:1760: oid 24 updating version 1106 to 1108 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:1760: oid 24 updating version 1108 to 1109 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:1760: oid 24 version 1109 is already newer than 1107 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1109 (ObjNum 675 snap 218 seq_num 675) dirty exists 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:1760: left oid 24 (ObjNum 675 snap 218 seq_num 675) 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:1759: done (2 left) 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:1760: done (1 left) 2024-04-12T11:21:54.588 INFO:tasks.rados.rados.0.smithi029.stdout:1761: done (0 left) 2024-04-12T11:21:54.589 INFO:tasks.rados.rados.0.smithi029.stdout:1762: read oid 9 snap 217 2024-04-12T11:21:54.589 INFO:tasks.rados.rados.0.smithi029.stdout:1762: expect deleted 2024-04-12T11:21:54.589 INFO:tasks.rados.rados.0.smithi029.stdout:1763: write oid 20 current snap is 218 2024-04-12T11:21:54.589 INFO:tasks.rados.rados.0.smithi029.stdout:1763: seq_num 677 ranges {578339=498990,1833133=525896,2791719=586620} 2024-04-12T11:21:54.593 INFO:tasks.rados.rados.0.smithi029.stdout:1763: writing smithi02979277-20 from 578339 to 1077329 tid 1 2024-04-12T11:21:54.600 INFO:tasks.rados.rados.0.smithi029.stdout:1763: writing smithi02979277-20 from 1833133 to 2359029 tid 2 2024-04-12T11:21:54.606 INFO:tasks.rados.rados.0.smithi029.stdout:1763: writing smithi02979277-20 from 2791719 to 3378339 tid 3 2024-04-12T11:21:54.606 INFO:tasks.rados.rados.0.smithi029.stdout:1764: snap_create 2024-04-12T11:21:54.606 INFO:tasks.rados.rados.0.smithi029.stdout:1763: finishing write tid 1 to smithi02979277-20 2024-04-12T11:21:54.606 INFO:tasks.rados.rados.0.smithi029.stdout:1763: finishing write tid 2 to smithi02979277-20 2024-04-12T11:21:54.609 INFO:tasks.rados.rados.0.smithi029.stdout:1763: finishing write tid 3 to smithi02979277-20 2024-04-12T11:21:54.609 INFO:tasks.rados.rados.0.smithi029.stdout:1763: finishing write tid 4 to smithi02979277-20 2024-04-12T11:21:54.609 INFO:tasks.rados.rados.0.smithi029.stdout:1763: finishing write tid 5 to smithi02979277-20 2024-04-12T11:21:54.609 INFO:tasks.rados.rados.0.smithi029.stdout:1763: oid 20 updating version 0 to 489 2024-04-12T11:21:54.609 INFO:tasks.rados.rados.0.smithi029.stdout:1763: oid 20 updating version 489 to 491 2024-04-12T11:21:54.609 INFO:tasks.rados.rados.0.smithi029.stdout:1763: oid 20 version 491 is already newer than 490 2024-04-12T11:21:54.609 INFO:tasks.rados.rados.0.smithi029.stdout:1763: oid 20 version 491 is already newer than 488 2024-04-12T11:21:54.609 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 491 (ObjNum 677 snap 218 seq_num 677) dirty exists 2024-04-12T11:21:54.609 INFO:tasks.rados.rados.0.smithi029.stdout:1763: left oid 20 (ObjNum 677 snap 218 seq_num 677) 2024-04-12T11:21:54.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[97787]: pgmap v399: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 90 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.3 MiB/s wr, 12 op/s; 26/168 objects degraded (15.476%) 2024-04-12T11:21:54.885 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[97787]: osdmap e509: 8 total, 7 up, 8 in 2024-04-12T11:21:54.885 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 26/168 objects degraded (15.476%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T11:21:54.885 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:54.885 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:54.885 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:21:54.885 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[93650]: pgmap v399: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 90 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.3 MiB/s wr, 12 op/s; 26/168 objects degraded (15.476%) 2024-04-12T11:21:54.885 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[93650]: osdmap e509: 8 total, 7 up, 8 in 2024-04-12T11:21:54.885 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 26/168 objects degraded (15.476%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T11:21:54.886 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:54.886 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:54.886 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:21:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:54 smithi087 ceph-mon[89014]: pgmap v399: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 90 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.3 MiB/s wr, 12 op/s; 26/168 objects degraded (15.476%) 2024-04-12T11:21:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:54 smithi087 ceph-mon[89014]: osdmap e509: 8 total, 7 up, 8 in 2024-04-12T11:21:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:54 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 26/168 objects degraded (15.476%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T11:21:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:21:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:21:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:21:54] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.43.0" 2024-04-12T11:21:55.571 INFO:tasks.rados.rados.0.smithi029.stdout:1762: done (2 left) 2024-04-12T11:21:55.571 INFO:tasks.rados.rados.0.smithi029.stdout:1763: done (1 left) 2024-04-12T11:21:55.571 INFO:tasks.rados.rados.0.smithi029.stdout:1764: done (0 left) 2024-04-12T11:21:55.572 INFO:tasks.rados.rados.0.smithi029.stdout:1765: rollback oid 26 current snap is 219 2024-04-12T11:21:55.572 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 216 2024-04-12T11:21:55.572 INFO:tasks.rados.rados.0.smithi029.stdout:1766: read oid 24 snap -1 2024-04-12T11:21:55.572 INFO:tasks.rados.rados.0.smithi029.stdout:1766: expect (ObjNum 675 snap 218 seq_num 675) 2024-04-12T11:21:55.572 INFO:tasks.rados.rados.0.smithi029.stdout:1767: delete oid 31 current snap is 219 2024-04-12T11:21:55.574 INFO:tasks.rados.rados.0.smithi029.stdout:1765: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:21:55.574 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 0 (ObjNum 129 snap 0 seq_num 1800866304) dirty dne 2024-04-12T11:21:55.575 INFO:tasks.rados.rados.0.smithi029.stdout:1765: done (2 left) 2024-04-12T11:21:55.576 INFO:tasks.rados.rados.0.smithi029.stdout:1767: done (1 left) 2024-04-12T11:21:55.576 INFO:tasks.rados.rados.0.smithi029.stdout:1768: rollback oid 39 current snap is 219 2024-04-12T11:21:55.576 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 216 2024-04-12T11:21:55.576 INFO:tasks.rados.rados.0.smithi029.stdout:1769: read oid 27 snap -1 2024-04-12T11:21:55.576 INFO:tasks.rados.rados.0.smithi029.stdout:1769: expect deleted 2024-04-12T11:21:55.576 INFO:tasks.rados.rados.0.smithi029.stdout:1770: snap_create 2024-04-12T11:21:55.578 INFO:tasks.rados.rados.0.smithi029.stdout:1766: expect (ObjNum 675 snap 218 seq_num 675) 2024-04-12T11:21:55.704 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:55 smithi029 ceph-mon[93650]: osdmap e510: 8 total, 7 up, 8 in 2024-04-12T11:21:55.704 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:55 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/679447813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:55.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:55 smithi029 ceph-mon[97787]: osdmap e510: 8 total, 7 up, 8 in 2024-04-12T11:21:55.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:55 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/679447813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:55.817 INFO:tasks.rados.rados.0.smithi029.stdout:1768: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:21:55.818 INFO:tasks.rados.rados.0.smithi029.stdout:1768: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:21:55.818 INFO:tasks.rados.rados.0.smithi029.stdout:1768: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:21:55.818 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 199 (ObjNum 620 snap 200 seq_num 620) dirty exists 2024-04-12T11:21:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:55 smithi087 ceph-mon[89014]: osdmap e510: 8 total, 7 up, 8 in 2024-04-12T11:21:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:55 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/679447813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:56.576 INFO:tasks.rados.rados.0.smithi029.stdout:1766: done (3 left) 2024-04-12T11:21:56.576 INFO:tasks.rados.rados.0.smithi029.stdout:1768: done (2 left) 2024-04-12T11:21:56.576 INFO:tasks.rados.rados.0.smithi029.stdout:1769: done (1 left) 2024-04-12T11:21:56.576 INFO:tasks.rados.rados.0.smithi029.stdout:1770: done (0 left) 2024-04-12T11:21:56.576 INFO:tasks.rados.rados.0.smithi029.stdout:1771: write oid 35 current snap is 220 2024-04-12T11:21:56.576 INFO:tasks.rados.rados.0.smithi029.stdout:1771: seq_num 679 ranges {611953=649488,1950097=529565,3246565=165388} 2024-04-12T11:21:56.583 INFO:tasks.rados.rados.0.smithi029.stdout:1771: writing smithi02979277-35 from 611953 to 1261441 tid 1 2024-04-12T11:21:56.590 INFO:tasks.rados.rados.0.smithi029.stdout:1771: writing smithi02979277-35 from 1950097 to 2479662 tid 2 2024-04-12T11:21:56.595 INFO:tasks.rados.rados.0.smithi029.stdout:1771: writing smithi02979277-35 from 3246565 to 3411953 tid 3 2024-04-12T11:21:56.596 INFO:tasks.rados.rados.0.smithi029.stdout:1772: write oid 48 current snap is 220 2024-04-12T11:21:56.596 INFO:tasks.rados.rados.0.smithi029.stdout:1771: finishing write tid 1 to smithi02979277-35 2024-04-12T11:21:56.596 INFO:tasks.rados.rados.0.smithi029.stdout:1772: seq_num 680 ranges {628760=724737,1892226=673223,3157635=271125} 2024-04-12T11:21:56.603 INFO:tasks.rados.rados.0.smithi029.stdout:1772: writing smithi02979277-48 from 628760 to 1353497 tid 1 2024-04-12T11:21:56.610 INFO:tasks.rados.rados.0.smithi029.stdout:1772: writing smithi02979277-48 from 1892226 to 2565449 tid 2 2024-04-12T11:21:56.616 INFO:tasks.rados.rados.0.smithi029.stdout:1772: writing smithi02979277-48 from 3157635 to 3428760 tid 3 2024-04-12T11:21:56.616 INFO:tasks.rados.rados.0.smithi029.stdout:1773: rollback oid 20 current snap is 220 2024-04-12T11:21:56.616 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 20 to 216 2024-04-12T11:21:56.616 INFO:tasks.rados.rados.0.smithi029.stdout:1771: finishing write tid 2 to smithi02979277-35 2024-04-12T11:21:56.616 INFO:tasks.rados.rados.0.smithi029.stdout:1771: finishing write tid 3 to smithi02979277-35 2024-04-12T11:21:56.616 INFO:tasks.rados.rados.0.smithi029.stdout:1771: finishing write tid 4 to smithi02979277-35 2024-04-12T11:21:56.616 INFO:tasks.rados.rados.0.smithi029.stdout:1771: finishing write tid 5 to smithi02979277-35 2024-04-12T11:21:56.617 INFO:tasks.rados.rados.0.smithi029.stdout:1771: oid 35 updating version 0 to 1113 2024-04-12T11:21:56.617 INFO:tasks.rados.rados.0.smithi029.stdout:1771: oid 35 updating version 1113 to 1114 2024-04-12T11:21:56.617 INFO:tasks.rados.rados.0.smithi029.stdout:1771: oid 35 updating version 1114 to 1116 2024-04-12T11:21:56.617 INFO:tasks.rados.rados.0.smithi029.stdout:1771: oid 35 version 1116 is already newer than 1115 2024-04-12T11:21:56.617 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 1116 (ObjNum 679 snap 220 seq_num 679) dirty exists 2024-04-12T11:21:56.618 INFO:tasks.rados.rados.0.smithi029.stdout:1771: left oid 35 (ObjNum 679 snap 220 seq_num 679) 2024-04-12T11:21:56.618 INFO:tasks.rados.rados.0.smithi029.stdout:1772: finishing write tid 1 to smithi02979277-48 2024-04-12T11:21:56.618 INFO:tasks.rados.rados.0.smithi029.stdout:1772: finishing write tid 2 to smithi02979277-48 2024-04-12T11:21:56.618 INFO:tasks.rados.rados.0.smithi029.stdout:1771: done (2 left) 2024-04-12T11:21:56.618 INFO:tasks.rados.rados.0.smithi029.stdout:1774: read oid 35 snap -1 2024-04-12T11:21:56.618 INFO:tasks.rados.rados.0.smithi029.stdout:1774: expect (ObjNum 679 snap 220 seq_num 679) 2024-04-12T11:21:56.618 INFO:tasks.rados.rados.0.smithi029.stdout:1775: snap_create 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:1772: finishing write tid 3 to smithi02979277-48 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:1772: finishing write tid 4 to smithi02979277-48 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:1772: finishing write tid 5 to smithi02979277-48 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:1772: oid 48 updating version 0 to 764 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:1772: oid 48 version 764 is already newer than 763 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:1772: oid 48 version 764 is already newer than 762 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:1772: oid 48 updating version 764 to 765 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 765 (ObjNum 680 snap 220 seq_num 680) dirty exists 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:1772: left oid 48 (ObjNum 680 snap 220 seq_num 680) 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:1773: finishing rollback tid 0 to smithi02979277-20 2024-04-12T11:21:56.620 INFO:tasks.rados.rados.0.smithi029.stdout:1773: finishing rollback tid 1 to smithi02979277-20 2024-04-12T11:21:56.621 INFO:tasks.rados.rados.0.smithi029.stdout:1774: expect (ObjNum 679 snap 220 seq_num 679) 2024-04-12T11:21:56.781 INFO:tasks.rados.rados.0.smithi029.stdout:1773: finishing rollback tid 2 to smithi02979277-20 2024-04-12T11:21:56.781 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 495 (ObjNum 671 snap 215 seq_num 671) dirty exists 2024-04-12T11:21:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:56 smithi087 ceph-mon[89014]: pgmap v402: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 90 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.3 MiB/s wr, 12 op/s; 26/168 objects degraded (15.476%) 2024-04-12T11:21:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:56 smithi087 ceph-mon[89014]: osdmap e511: 8 total, 7 up, 8 in 2024-04-12T11:21:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:56 smithi029 ceph-mon[97787]: pgmap v402: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 90 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.3 MiB/s wr, 12 op/s; 26/168 objects degraded (15.476%) 2024-04-12T11:21:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:56 smithi029 ceph-mon[97787]: osdmap e511: 8 total, 7 up, 8 in 2024-04-12T11:21:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:56 smithi029 ceph-mon[93650]: pgmap v402: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 90 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.3 MiB/s wr, 12 op/s; 26/168 objects degraded (15.476%) 2024-04-12T11:21:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:56 smithi029 ceph-mon[93650]: osdmap e511: 8 total, 7 up, 8 in 2024-04-12T11:21:57.303 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:57 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[106271]: 2024-04-12T11:21:57.039+0000 7f4b345097c0 -1 osd.0 503 log_to_monitors true 2024-04-12T11:21:57.580 INFO:tasks.rados.rados.0.smithi029.stdout:1772: done (3 left) 2024-04-12T11:21:57.580 INFO:tasks.rados.rados.0.smithi029.stdout:1773: done (2 left) 2024-04-12T11:21:57.581 INFO:tasks.rados.rados.0.smithi029.stdout:1774: done (1 left) 2024-04-12T11:21:57.581 INFO:tasks.rados.rados.0.smithi029.stdout:1775: done (0 left) 2024-04-12T11:21:57.581 INFO:tasks.rados.rados.0.smithi029.stdout:1776: delete oid 48 current snap is 221 2024-04-12T11:21:57.586 INFO:tasks.rados.rados.0.smithi029.stdout:1776: done (0 left) 2024-04-12T11:21:57.586 INFO:tasks.rados.rados.0.smithi029.stdout:1777: write oid 37 current snap is 221 2024-04-12T11:21:57.586 INFO:tasks.rados.rados.0.smithi029.stdout:1777: seq_num 682 ranges {662374=475235,1609190=676892,2812481=624101} 2024-04-12T11:21:57.593 INFO:tasks.rados.rados.0.smithi029.stdout:1777: writing smithi02979277-37 from 662374 to 1137609 tid 1 2024-04-12T11:21:57.599 INFO:tasks.rados.rados.0.smithi029.stdout:1777: writing smithi02979277-37 from 1609190 to 2286082 tid 2 2024-04-12T11:21:57.606 INFO:tasks.rados.rados.0.smithi029.stdout:1777: writing smithi02979277-37 from 2812481 to 3436582 tid 3 2024-04-12T11:21:57.606 INFO:tasks.rados.rados.0.smithi029.stdout:1778: snap_create 2024-04-12T11:21:57.606 INFO:tasks.rados.rados.0.smithi029.stdout:1777: finishing write tid 1 to smithi02979277-37 2024-04-12T11:21:57.606 INFO:tasks.rados.rados.0.smithi029.stdout:1777: finishing write tid 2 to smithi02979277-37 2024-04-12T11:21:57.608 INFO:tasks.rados.rados.0.smithi029.stdout:1777: finishing write tid 3 to smithi02979277-37 2024-04-12T11:21:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:1777: finishing write tid 4 to smithi02979277-37 2024-04-12T11:21:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:1777: finishing write tid 5 to smithi02979277-37 2024-04-12T11:21:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:1777: oid 37 updating version 0 to 654 2024-04-12T11:21:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:1777: oid 37 updating version 654 to 656 2024-04-12T11:21:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:1777: oid 37 updating version 656 to 657 2024-04-12T11:21:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:1777: oid 37 version 657 is already newer than 655 2024-04-12T11:21:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 657 (ObjNum 682 snap 221 seq_num 682) dirty exists 2024-04-12T11:21:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:1777: left oid 37 (ObjNum 682 snap 221 seq_num 682) 2024-04-12T11:21:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:57 smithi087 ceph-mon[89014]: osdmap e512: 8 total, 7 up, 8 in 2024-04-12T11:21:57.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:57 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4275334557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:57.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:57 smithi087 ceph-mon[89014]: from='osd.0 [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T11:21:57.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:57 smithi087 ceph-mon[89014]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T11:21:58.018 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 11:21:57 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[106271]: 2024-04-12T11:21:57.601+0000 7f4b21959700 -1 osd.0 503 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:21:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:57 smithi029 ceph-mon[93650]: osdmap e512: 8 total, 7 up, 8 in 2024-04-12T11:21:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:57 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4275334557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:57 smithi029 ceph-mon[93650]: from='osd.0 [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T11:21:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:57 smithi029 ceph-mon[93650]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T11:21:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:57 smithi029 ceph-mon[97787]: osdmap e512: 8 total, 7 up, 8 in 2024-04-12T11:21:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:57 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4275334557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:58.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:57 smithi029 ceph-mon[97787]: from='osd.0 [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T11:21:58.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:57 smithi029 ceph-mon[97787]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T11:21:58.583 INFO:tasks.rados.rados.0.smithi029.stdout:1777: done (1 left) 2024-04-12T11:21:58.583 INFO:tasks.rados.rados.0.smithi029.stdout:1778: done (0 left) 2024-04-12T11:21:58.584 INFO:tasks.rados.rados.0.smithi029.stdout:1779: delete oid 20 current snap is 222 2024-04-12T11:21:58.600 INFO:tasks.rados.rados.0.smithi029.stdout:1779: done (0 left) 2024-04-12T11:21:58.600 INFO:tasks.rados.rados.0.smithi029.stdout:1780: write oid 16 current snap is 222 2024-04-12T11:21:58.600 INFO:tasks.rados.rados.0.smithi029.stdout:1780: seq_num 684 ranges {695988=625733,2009801=680561,3150974=345014} 2024-04-12T11:21:58.603 INFO:tasks.rados.rados.0.smithi029.stdout:1780: writing smithi02979277-16 from 695988 to 1321721 tid 1 2024-04-12T11:21:58.611 INFO:tasks.rados.rados.0.smithi029.stdout:1780: writing smithi02979277-16 from 2009801 to 2690362 tid 2 2024-04-12T11:21:58.616 INFO:tasks.rados.rados.0.smithi029.stdout:1780: writing smithi02979277-16 from 3150974 to 3495988 tid 3 2024-04-12T11:21:58.616 INFO:tasks.rados.rados.0.smithi029.stdout:1781: read oid 11 snap -1 2024-04-12T11:21:58.616 INFO:tasks.rados.rados.0.smithi029.stdout:1781: expect deleted 2024-04-12T11:21:58.616 INFO:tasks.rados.rados.0.smithi029.stdout:1782: read oid 3 snap -1 2024-04-12T11:21:58.616 INFO:tasks.rados.rados.0.smithi029.stdout:1782: expect deleted 2024-04-12T11:21:58.617 INFO:tasks.rados.rados.0.smithi029.stdout:1783: rollback oid 28 current snap is 222 2024-04-12T11:21:58.617 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 28 to 220 2024-04-12T11:21:58.617 INFO:tasks.rados.rados.0.smithi029.stdout:1784: write oid 26 current snap is 222 2024-04-12T11:21:58.617 INFO:tasks.rados.rados.0.smithi029.stdout:1784: seq_num 685 ranges {712795=417335,1784636=540572,2894750=457917} 2024-04-12T11:21:58.622 INFO:tasks.rados.rados.0.smithi029.stdout:1784: writing smithi02979277-26 from 712795 to 1130130 tid 1 2024-04-12T11:21:58.629 INFO:tasks.rados.rados.0.smithi029.stdout:1784: writing smithi02979277-26 from 1784636 to 2325208 tid 2 2024-04-12T11:21:58.635 INFO:tasks.rados.rados.0.smithi029.stdout:1784: writing smithi02979277-26 from 2894750 to 3352667 tid 3 2024-04-12T11:21:58.635 INFO:tasks.rados.rados.0.smithi029.stdout:1785: rollback oid 45 current snap is 222 2024-04-12T11:21:58.635 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 45 to 218 2024-04-12T11:21:58.635 INFO:tasks.rados.rados.0.smithi029.stdout:1783: finishing rollback tid 1 to smithi02979277-28 2024-04-12T11:21:58.635 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 0 (ObjNum 49 snap 0 seq_num 3) dirty dne 2024-04-12T11:21:58.636 INFO:tasks.rados.rados.0.smithi029.stdout:1784: finishing write tid 1 to smithi02979277-26 2024-04-12T11:21:58.636 INFO:tasks.rados.rados.0.smithi029.stdout:1784: finishing write tid 2 to smithi02979277-26 2024-04-12T11:21:58.636 INFO:tasks.rados.rados.0.smithi029.stdout:1781: done (5 left) 2024-04-12T11:21:58.636 INFO:tasks.rados.rados.0.smithi029.stdout:1782: done (4 left) 2024-04-12T11:21:58.636 INFO:tasks.rados.rados.0.smithi029.stdout:1783: done (3 left) 2024-04-12T11:21:58.636 INFO:tasks.rados.rados.0.smithi029.stdout:1786: write oid 24 current snap is 222 2024-04-12T11:21:58.636 INFO:tasks.rados.rados.0.smithi029.stdout:1786: seq_num 686 ranges {729602=492584,1843118=684230,3322173=207429} 2024-04-12T11:21:58.641 INFO:tasks.rados.rados.0.smithi029.stdout:1786: writing smithi02979277-24 from 729602 to 1222186 tid 1 2024-04-12T11:21:58.648 INFO:tasks.rados.rados.0.smithi029.stdout:1786: writing smithi02979277-24 from 1843118 to 2527348 tid 2 2024-04-12T11:21:58.655 INFO:tasks.rados.rados.0.smithi029.stdout:1786: writing smithi02979277-24 from 3322173 to 3529602 tid 3 2024-04-12T11:21:58.655 INFO:tasks.rados.rados.0.smithi029.stdout:1787: read oid 27 snap -1 2024-04-12T11:21:58.655 INFO:tasks.rados.rados.0.smithi029.stdout:1787: expect deleted 2024-04-12T11:21:58.655 INFO:tasks.rados.rados.0.smithi029.stdout:1788: rollback oid 17 current snap is 222 2024-04-12T11:21:58.655 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 218 2024-04-12T11:21:58.655 INFO:tasks.rados.rados.0.smithi029.stdout:1784: finishing write tid 3 to smithi02979277-26 2024-04-12T11:21:58.655 INFO:tasks.rados.rados.0.smithi029.stdout:1789: read oid 43 snap -1 2024-04-12T11:21:58.655 INFO:tasks.rados.rados.0.smithi029.stdout:1789: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:21:58.655 INFO:tasks.rados.rados.0.smithi029.stdout:1790: snap_remove snap 216 2024-04-12T11:21:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:58 smithi087 ceph-mon[89014]: pgmap v405: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 100 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.5 MiB/s wr, 11 op/s; 27/192 objects degraded (14.062%) 2024-04-12T11:21:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:58 smithi087 ceph-mon[89014]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T11:21:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:58 smithi087 ceph-mon[89014]: osdmap e513: 8 total, 7 up, 8 in 2024-04-12T11:21:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:58 smithi087 ceph-mon[89014]: from='osd.0 [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:21:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:58 smithi087 ceph-mon[89014]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:21:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:58 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:58 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[97787]: pgmap v405: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 100 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.5 MiB/s wr, 11 op/s; 27/192 objects degraded (14.062%) 2024-04-12T11:21:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[97787]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T11:21:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[97787]: osdmap e513: 8 total, 7 up, 8 in 2024-04-12T11:21:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[97787]: from='osd.0 [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:21:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[97787]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:21:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[93650]: pgmap v405: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 100 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.5 MiB/s wr, 11 op/s; 27/192 objects degraded (14.062%) 2024-04-12T11:21:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[93650]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T11:21:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[93650]: osdmap e513: 8 total, 7 up, 8 in 2024-04-12T11:21:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[93650]: from='osd.0 [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:21:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[93650]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:21:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:58 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.586 INFO:tasks.rados.rados.0.smithi029.stdout:1790: done (7 left) 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1791: snap_create 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1784: finishing write tid 4 to smithi02979277-26 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1784: finishing write tid 5 to smithi02979277-26 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1784: oid 26 updating version 0 to 1117 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1784: oid 26 updating version 1117 to 1119 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1784: oid 26 updating version 1119 to 1120 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1784: oid 26 version 1120 is already newer than 1118 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 1120 (ObjNum 685 snap 222 seq_num 685) dirty exists 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1784: left oid 26 (ObjNum 685 snap 222 seq_num 685) 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1786: finishing write tid 1 to smithi02979277-24 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1786: finishing write tid 2 to smithi02979277-24 2024-04-12T11:21:59.587 INFO:tasks.rados.rados.0.smithi029.stdout:1786: finishing write tid 3 to smithi02979277-24 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:1786: finishing write tid 4 to smithi02979277-24 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:1786: finishing write tid 5 to smithi02979277-24 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:1786: oid 24 updating version 0 to 1122 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:1786: oid 24 updating version 1122 to 1123 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:1786: oid 24 updating version 1123 to 1124 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:1786: oid 24 updating version 1124 to 1125 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1125 (ObjNum 686 snap 222 seq_num 686) dirty exists 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:1786: left oid 24 (ObjNum 686 snap 222 seq_num 686) 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:1788: finishing rollback tid 0 to smithi02979277-17 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:1788: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:1788: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:21:59.588 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 876 (ObjNum 656 snap 212 seq_num 656) dirty exists 2024-04-12T11:21:59.601 INFO:tasks.rados.rados.0.smithi029.stdout:1789: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:21:59.715 INFO:tasks.rados.rados.0.smithi029.stdout:1785: finishing rollback tid 0 to smithi02979277-45 2024-04-12T11:21:59.715 INFO:tasks.rados.rados.0.smithi029.stdout:1785: finishing rollback tid 1 to smithi02979277-45 2024-04-12T11:21:59.715 INFO:tasks.rados.rados.0.smithi029.stdout:1785: finishing rollback tid 2 to smithi02979277-45 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 825 (ObjNum 537 snap 173 seq_num 537) dirty exists 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:1780: finishing write tid 1 to smithi02979277-16 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:1780: finishing write tid 2 to smithi02979277-16 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:1780: finishing write tid 3 to smithi02979277-16 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:1780: finishing write tid 4 to smithi02979277-16 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:1780: finishing write tid 5 to smithi02979277-16 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:1780: oid 16 updating version 0 to 952 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:1780: oid 16 updating version 952 to 953 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:1780: oid 16 version 953 is already newer than 950 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:1780: oid 16 version 953 is already newer than 951 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 953 (ObjNum 684 snap 222 seq_num 684) dirty exists 2024-04-12T11:21:59.716 INFO:tasks.rados.rados.0.smithi029.stdout:1780: left oid 16 (ObjNum 684 snap 222 seq_num 684) 2024-04-12T11:21:59.866 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[97787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:21:59.866 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[97787]: osd.0 [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946] boot 2024-04-12T11:21:59.866 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[97787]: osdmap e514: 8 total, 8 up, 8 in 2024-04-12T11:21:59.866 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:21:59.866 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.866 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.866 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3219194885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:59.867 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[97787]: osdmap e515: 8 total, 8 up, 8 in 2024-04-12T11:21:59.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[93650]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:21:59.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[93650]: osd.0 [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946] boot 2024-04-12T11:21:59.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[93650]: osdmap e514: 8 total, 8 up, 8 in 2024-04-12T11:21:59.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:21:59.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3219194885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:59.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:21:59 smithi029 ceph-mon[93650]: osdmap e515: 8 total, 8 up, 8 in 2024-04-12T11:21:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:59 smithi087 ceph-mon[89014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:21:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:59 smithi087 ceph-mon[89014]: osd.0 [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946] boot 2024-04-12T11:21:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:59 smithi087 ceph-mon[89014]: osdmap e514: 8 total, 8 up, 8 in 2024-04-12T11:21:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T11:21:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:21:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:59 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3219194885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:21:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:21:59 smithi087 ceph-mon[89014]: osdmap e515: 8 total, 8 up, 8 in 2024-04-12T11:22:00.585 INFO:tasks.rados.rados.0.smithi029.stdout:1780: done (7 left) 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1784: done (6 left) 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1785: done (5 left) 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1786: done (4 left) 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1787: done (3 left) 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1788: done (2 left) 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1789: done (1 left) 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1791: done (0 left) 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1792: read oid 43 snap 222 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1792: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1793: write oid 48 current snap is 223 2024-04-12T11:22:00.586 INFO:tasks.rados.rados.0.smithi029.stdout:1793: seq_num 687 ranges {746409=567833,1901600=544241,3065949=480460} 2024-04-12T11:22:00.592 INFO:tasks.rados.rados.0.smithi029.stdout:1793: writing smithi02979277-48 from 746409 to 1314242 tid 1 2024-04-12T11:22:00.599 INFO:tasks.rados.rados.0.smithi029.stdout:1793: writing smithi02979277-48 from 1901600 to 2445841 tid 2 2024-04-12T11:22:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1793: writing smithi02979277-48 from 3065949 to 3546409 tid 3 2024-04-12T11:22:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1794: write oid 7 current snap is 223 2024-04-12T11:22:00.606 INFO:tasks.rados.rados.0.smithi029.stdout:1794: seq_num 688 ranges {763216=643082,1843729=687899,3260666=302550} 2024-04-12T11:22:00.613 INFO:tasks.rados.rados.0.smithi029.stdout:1794: writing smithi02979277-7 from 763216 to 1406298 tid 1 2024-04-12T11:22:00.619 INFO:tasks.rados.rados.0.smithi029.stdout:1794: writing smithi02979277-7 from 1843729 to 2531628 tid 2 2024-04-12T11:22:00.625 INFO:tasks.rados.rados.0.smithi029.stdout:1794: writing smithi02979277-7 from 3260666 to 3563216 tid 3 2024-04-12T11:22:00.625 INFO:tasks.rados.rados.0.smithi029.stdout:1795: snap_create 2024-04-12T11:22:00.625 INFO:tasks.rados.rados.0.smithi029.stdout:1792: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:22:00.734 INFO:tasks.rados.rados.0.smithi029.stdout:1793: finishing write tid 1 to smithi02979277-48 2024-04-12T11:22:00.734 INFO:tasks.rados.rados.0.smithi029.stdout:1793: finishing write tid 2 to smithi02979277-48 2024-04-12T11:22:00.734 INFO:tasks.rados.rados.0.smithi029.stdout:1793: finishing write tid 3 to smithi02979277-48 2024-04-12T11:22:00.734 INFO:tasks.rados.rados.0.smithi029.stdout:1793: finishing write tid 4 to smithi02979277-48 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1793: finishing write tid 5 to smithi02979277-48 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1793: oid 48 updating version 0 to 770 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1793: oid 48 updating version 770 to 772 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1793: oid 48 version 772 is already newer than 771 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1793: oid 48 updating version 772 to 773 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 773 (ObjNum 687 snap 223 seq_num 687) dirty exists 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1793: left oid 48 (ObjNum 687 snap 223 seq_num 687) 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1794: finishing write tid 1 to smithi02979277-7 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1794: finishing write tid 2 to smithi02979277-7 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1794: finishing write tid 3 to smithi02979277-7 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1794: finishing write tid 4 to smithi02979277-7 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1794: finishing write tid 5 to smithi02979277-7 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1794: oid 7 updating version 0 to 826 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1794: oid 7 updating version 826 to 829 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1794: oid 7 version 829 is already newer than 827 2024-04-12T11:22:00.735 INFO:tasks.rados.rados.0.smithi029.stdout:1794: oid 7 version 829 is already newer than 828 2024-04-12T11:22:00.736 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 829 (ObjNum 688 snap 223 seq_num 688) dirty exists 2024-04-12T11:22:00.736 INFO:tasks.rados.rados.0.smithi029.stdout:1794: left oid 7 (ObjNum 688 snap 223 seq_num 688) 2024-04-12T11:22:00.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:00 smithi029 ceph-mon[93650]: pgmap v408: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 100 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.5 MiB/s wr, 11 op/s; 27/192 objects degraded (14.062%) 2024-04-12T11:22:00.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:00 smithi029 ceph-mon[93650]: osdmap e516: 8 total, 8 up, 8 in 2024-04-12T11:22:00.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:00 smithi029 ceph-mon[97787]: pgmap v408: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 100 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.5 MiB/s wr, 11 op/s; 27/192 objects degraded (14.062%) 2024-04-12T11:22:00.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:00 smithi029 ceph-mon[97787]: osdmap e516: 8 total, 8 up, 8 in 2024-04-12T11:22:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:00 smithi087 ceph-mon[89014]: pgmap v408: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 100 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.5 MiB/s wr, 11 op/s; 27/192 objects degraded (14.062%) 2024-04-12T11:22:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:00 smithi087 ceph-mon[89014]: osdmap e516: 8 total, 8 up, 8 in 2024-04-12T11:22:01.604 INFO:tasks.rados.rados.0.smithi029.stdout:1792: done (3 left) 2024-04-12T11:22:01.605 INFO:tasks.rados.rados.0.smithi029.stdout:1793: done (2 left) 2024-04-12T11:22:01.605 INFO:tasks.rados.rados.0.smithi029.stdout:1794: done (1 left) 2024-04-12T11:22:01.605 INFO:tasks.rados.rados.0.smithi029.stdout:1795: done (0 left) 2024-04-12T11:22:01.605 INFO:tasks.rados.rados.0.smithi029.stdout:1796: rollback oid 29 current snap is 224 2024-04-12T11:22:01.605 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 29 to 223 2024-04-12T11:22:01.605 INFO:tasks.rados.rados.0.smithi029.stdout:1797: write oid 18 current snap is 224 2024-04-12T11:22:01.605 INFO:tasks.rados.rados.0.smithi029.stdout:1797: seq_num 689 ranges {780023=718331,1902211=547910,3004442=575581} 2024-04-12T11:22:01.612 INFO:tasks.rados.rados.0.smithi029.stdout:1797: writing smithi02979277-18 from 780023 to 1498354 tid 1 2024-04-12T11:22:01.618 INFO:tasks.rados.rados.0.smithi029.stdout:1797: writing smithi02979277-18 from 1902211 to 2450121 tid 2 2024-04-12T11:22:01.624 INFO:tasks.rados.rados.0.smithi029.stdout:1797: writing smithi02979277-18 from 3004442 to 3580023 tid 3 2024-04-12T11:22:01.624 INFO:tasks.rados.rados.0.smithi029.stdout:1798: write oid 34 current snap is 224 2024-04-12T11:22:01.625 INFO:tasks.rados.rados.0.smithi029.stdout:1796: finishing rollback tid 0 to smithi02979277-29 2024-04-12T11:22:01.625 INFO:tasks.rados.rados.0.smithi029.stdout:1798: seq_num 690 ranges {796830=793580,2360693=691568} 2024-04-12T11:22:01.633 INFO:tasks.rados.rados.0.smithi029.stdout:1798: writing smithi02979277-34 from 796830 to 1590410 tid 1 2024-04-12T11:22:01.642 INFO:tasks.rados.rados.0.smithi029.stdout:1798: writing smithi02979277-34 from 2360693 to 3052261 tid 2 2024-04-12T11:22:01.642 INFO:tasks.rados.rados.0.smithi029.stdout:1799: write oid 38 current snap is 224 2024-04-12T11:22:01.642 INFO:tasks.rados.rados.0.smithi029.stdout:1796: finishing rollback tid 1 to smithi02979277-29 2024-04-12T11:22:01.642 INFO:tasks.rados.rados.0.smithi029.stdout:1796: finishing rollback tid 2 to smithi02979277-29 2024-04-12T11:22:01.642 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 667 (ObjNum 666 snap 215 seq_num 666) dirty exists 2024-04-12T11:22:01.642 INFO:tasks.rados.rados.0.smithi029.stdout:1797: finishing write tid 1 to smithi02979277-18 2024-04-12T11:22:01.643 INFO:tasks.rados.rados.0.smithi029.stdout:1799: seq_num 691 ranges {413637=468829,1619175=435226,2542935=758255} 2024-04-12T11:22:01.647 INFO:tasks.rados.rados.0.smithi029.stdout:1799: writing smithi02979277-38 from 413637 to 882466 tid 1 2024-04-12T11:22:01.654 INFO:tasks.rados.rados.0.smithi029.stdout:1799: writing smithi02979277-38 from 1619175 to 2054401 tid 2 2024-04-12T11:22:01.661 INFO:tasks.rados.rados.0.smithi029.stdout:1799: writing smithi02979277-38 from 2542935 to 3301190 tid 3 2024-04-12T11:22:01.661 INFO:tasks.rados.rados.0.smithi029.stdout:1796: done (3 left) 2024-04-12T11:22:01.661 INFO:tasks.rados.rados.0.smithi029.stdout:1800: write oid 37 current snap is 224 2024-04-12T11:22:01.661 INFO:tasks.rados.rados.0.smithi029.stdout:1797: finishing write tid 2 to smithi02979277-18 2024-04-12T11:22:01.661 INFO:tasks.rados.rados.0.smithi029.stdout:1797: finishing write tid 3 to smithi02979277-18 2024-04-12T11:22:01.661 INFO:tasks.rados.rados.0.smithi029.stdout:1800: seq_num 692 ranges {430444=660431,1677657=695237,2970358=569527} 2024-04-12T11:22:01.667 INFO:tasks.rados.rados.0.smithi029.stdout:1800: writing smithi02979277-37 from 430444 to 1090875 tid 1 2024-04-12T11:22:01.675 INFO:tasks.rados.rados.0.smithi029.stdout:1800: writing smithi02979277-37 from 1677657 to 2372894 tid 2 2024-04-12T11:22:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:1800: writing smithi02979277-37 from 2970358 to 3539885 tid 3 2024-04-12T11:22:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:1801: snap_create 2024-04-12T11:22:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:1797: finishing write tid 4 to smithi02979277-18 2024-04-12T11:22:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:1797: finishing write tid 5 to smithi02979277-18 2024-04-12T11:22:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:1797: oid 18 updating version 0 to 882 2024-04-12T11:22:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:1797: oid 18 version 882 is already newer than 881 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1797: oid 18 version 882 is already newer than 880 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1797: oid 18 updating version 882 to 883 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 883 (ObjNum 689 snap 224 seq_num 689) dirty exists 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1797: left oid 18 (ObjNum 689 snap 224 seq_num 689) 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1798: finishing write tid 1 to smithi02979277-34 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1798: finishing write tid 2 to smithi02979277-34 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1798: finishing write tid 3 to smithi02979277-34 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1798: finishing write tid 4 to smithi02979277-34 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1798: oid 34 updating version 0 to 669 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1798: oid 34 updating version 669 to 670 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1798: oid 34 updating version 670 to 671 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 671 (ObjNum 690 snap 224 seq_num 690) dirty exists 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1798: left oid 34 (ObjNum 690 snap 224 seq_num 690) 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1799: finishing write tid 1 to smithi02979277-38 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1799: finishing write tid 2 to smithi02979277-38 2024-04-12T11:22:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:1799: finishing write tid 3 to smithi02979277-38 2024-04-12T11:22:01.684 INFO:tasks.rados.rados.0.smithi029.stdout:1799: finishing write tid 4 to smithi02979277-38 2024-04-12T11:22:01.684 INFO:tasks.rados.rados.0.smithi029.stdout:1799: finishing write tid 5 to smithi02979277-38 2024-04-12T11:22:01.684 INFO:tasks.rados.rados.0.smithi029.stdout:1799: oid 38 updating version 0 to 503 2024-04-12T11:22:01.684 INFO:tasks.rados.rados.0.smithi029.stdout:1799: oid 38 version 503 is already newer than 501 2024-04-12T11:22:01.684 INFO:tasks.rados.rados.0.smithi029.stdout:1799: oid 38 updating version 503 to 504 2024-04-12T11:22:01.684 INFO:tasks.rados.rados.0.smithi029.stdout:1799: oid 38 version 504 is already newer than 502 2024-04-12T11:22:01.684 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 504 (ObjNum 691 snap 224 seq_num 691) dirty exists 2024-04-12T11:22:01.684 INFO:tasks.rados.rados.0.smithi029.stdout:1799: left oid 38 (ObjNum 691 snap 224 seq_num 691) 2024-04-12T11:22:01.684 INFO:tasks.rados.rados.0.smithi029.stdout:1800: finishing write tid 1 to smithi02979277-37 2024-04-12T11:22:01.684 INFO:tasks.rados.rados.0.smithi029.stdout:1800: finishing write tid 2 to smithi02979277-37 2024-04-12T11:22:01.685 INFO:tasks.rados.rados.0.smithi029.stdout:1800: finishing write tid 3 to smithi02979277-37 2024-04-12T11:22:01.685 INFO:tasks.rados.rados.0.smithi029.stdout:1800: finishing write tid 4 to smithi02979277-37 2024-04-12T11:22:01.685 INFO:tasks.rados.rados.0.smithi029.stdout:1800: finishing write tid 5 to smithi02979277-37 2024-04-12T11:22:01.685 INFO:tasks.rados.rados.0.smithi029.stdout:1800: oid 37 updating version 0 to 661 2024-04-12T11:22:01.685 INFO:tasks.rados.rados.0.smithi029.stdout:1800: oid 37 updating version 661 to 662 2024-04-12T11:22:01.685 INFO:tasks.rados.rados.0.smithi029.stdout:1800: oid 37 updating version 662 to 664 2024-04-12T11:22:01.685 INFO:tasks.rados.rados.0.smithi029.stdout:1800: oid 37 version 664 is already newer than 663 2024-04-12T11:22:01.685 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 664 (ObjNum 692 snap 224 seq_num 692) dirty exists 2024-04-12T11:22:01.685 INFO:tasks.rados.rados.0.smithi029.stdout:1800: left oid 37 (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:22:02.603 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:22:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:22:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:22:02.610 INFO:tasks.rados.rados.0.smithi029.stdout:1797: done (4 left) 2024-04-12T11:22:02.610 INFO:tasks.rados.rados.0.smithi029.stdout:1798: done (3 left) 2024-04-12T11:22:02.610 INFO:tasks.rados.rados.0.smithi029.stdout:1799: done (2 left) 2024-04-12T11:22:02.610 INFO:tasks.rados.rados.0.smithi029.stdout:1800: done (1 left) 2024-04-12T11:22:02.610 INFO:tasks.rados.rados.0.smithi029.stdout:1801: done (0 left) 2024-04-12T11:22:02.610 INFO:tasks.rados.rados.0.smithi029.stdout:1802: snap_create 2024-04-12T11:22:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:02 smithi087 ceph-mon[89014]: pgmap v411: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 100 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 27/192 objects degraded (14.062%) 2024-04-12T11:22:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:02 smithi087 ceph-mon[89014]: osdmap e517: 8 total, 8 up, 8 in 2024-04-12T11:22:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:02 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3731066439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:02 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 27/192 objects degraded (14.062%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:02 smithi029 ceph-mon[93650]: pgmap v411: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 100 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 27/192 objects degraded (14.062%) 2024-04-12T11:22:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:02 smithi029 ceph-mon[93650]: osdmap e517: 8 total, 8 up, 8 in 2024-04-12T11:22:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:02 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3731066439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:02 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 27/192 objects degraded (14.062%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:02 smithi029 ceph-mon[97787]: pgmap v411: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 100 MiB data, 501 MiB used, 715 GiB / 715 GiB avail; 27/192 objects degraded (14.062%) 2024-04-12T11:22:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:02 smithi029 ceph-mon[97787]: osdmap e517: 8 total, 8 up, 8 in 2024-04-12T11:22:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:02 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3731066439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:02 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 27/192 objects degraded (14.062%), 14 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:03.612 INFO:tasks.rados.rados.0.smithi029.stdout:1802: done (0 left) 2024-04-12T11:22:03.612 INFO:tasks.rados.rados.0.smithi029.stdout:1803: read oid 48 snap 224 2024-04-12T11:22:03.612 INFO:tasks.rados.rados.0.smithi029.stdout:1803: expect (ObjNum 687 snap 223 seq_num 687) 2024-04-12T11:22:03.612 INFO:tasks.rados.rados.0.smithi029.stdout:1804: read oid 5 snap 225 2024-04-12T11:22:03.612 INFO:tasks.rados.rados.0.smithi029.stdout:1804: expect deleted 2024-04-12T11:22:03.612 INFO:tasks.rados.rados.0.smithi029.stdout:1805: snap_create 2024-04-12T11:22:03.619 INFO:tasks.rados.rados.0.smithi029.stdout:1803: expect (ObjNum 687 snap 223 seq_num 687) 2024-04-12T11:22:03.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: osdmap e518: 8 total, 8 up, 8 in 2024-04-12T11:22:03.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:03.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:03.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:03.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:22:03.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:03.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[97787]: Upgrade: osd.1 is safe to restart 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: osdmap e518: 8 total, 8 up, 8 in 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:03.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:03.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:03.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:03.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T11:22:03.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T11:22:03.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:03 smithi029 ceph-mon[93650]: Upgrade: osd.1 is safe to restart 2024-04-12T11:22:03.895 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: osdmap e518: 8 total, 8 up, 8 in 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T11:22:03.896 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:03 smithi087 ceph-mon[89014]: Upgrade: osd.1 is safe to restart 2024-04-12T11:22:04.612 INFO:tasks.rados.rados.0.smithi029.stdout:1803: done (2 left) 2024-04-12T11:22:04.612 INFO:tasks.rados.rados.0.smithi029.stdout:1804: done (1 left) 2024-04-12T11:22:04.612 INFO:tasks.rados.rados.0.smithi029.stdout:1805: done (0 left) 2024-04-12T11:22:04.612 INFO:tasks.rados.rados.0.smithi029.stdout:1806: snap_create 2024-04-12T11:22:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[93650]: pgmap v414: 89 pgs: 89 active+clean; 121 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.9 MiB/s wr, 20 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[93650]: osdmap e519: 8 total, 8 up, 8 in 2024-04-12T11:22:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/192 objects degraded (14.062%), 14 pgs degraded) 2024-04-12T11:22:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:22:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1199764782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T11:22:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[93650]: osdmap e520: 8 total, 8 up, 8 in 2024-04-12T11:22:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[97787]: pgmap v414: 89 pgs: 89 active+clean; 121 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.9 MiB/s wr, 20 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[97787]: osdmap e519: 8 total, 8 up, 8 in 2024-04-12T11:22:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/192 objects degraded (14.062%), 14 pgs degraded) 2024-04-12T11:22:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:22:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1199764782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T11:22:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-mon[97787]: osdmap e520: 8 total, 8 up, 8 in 2024-04-12T11:22:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:04 smithi087 ceph-mon[89014]: pgmap v414: 89 pgs: 89 active+clean; 121 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.9 MiB/s wr, 20 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:04 smithi087 ceph-mon[89014]: osdmap e519: 8 total, 8 up, 8 in 2024-04-12T11:22:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:04 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/192 objects degraded (14.062%), 14 pgs degraded) 2024-04-12T11:22:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:04 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:22:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:04 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1199764782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:04 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:04 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T11:22:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:04 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:04 smithi087 ceph-mon[89014]: osdmap e520: 8 total, 8 up, 8 in 2024-04-12T11:22:05.202 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:22:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:22:04] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.43.0" 2024-04-12T11:22:05.615 INFO:tasks.rados.rados.0.smithi029.stdout:1806: done (0 left) 2024-04-12T11:22:05.615 INFO:tasks.rados.rados.0.smithi029.stdout:1807: read oid 31 snap 222 2024-04-12T11:22:05.615 INFO:tasks.rados.rados.0.smithi029.stdout:1807: expect deleted 2024-04-12T11:22:05.615 INFO:tasks.rados.rados.0.smithi029.stdout:1808: rollback oid 3 current snap is 228 2024-04-12T11:22:05.615 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 218 2024-04-12T11:22:05.615 INFO:tasks.rados.rados.0.smithi029.stdout:1809: rollback oid 37 current snap is 228 2024-04-12T11:22:05.615 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 226 2024-04-12T11:22:05.615 INFO:tasks.rados.rados.0.smithi029.stdout:1810: write oid 34 current snap is 228 2024-04-12T11:22:05.615 INFO:tasks.rados.rados.0.smithi029.stdout:1810: seq_num 693 ranges {447251=735680,1736139=438895,2597781=780799} 2024-04-12T11:22:05.621 INFO:tasks.rados.rados.0.smithi029.stdout:1810: writing smithi02979277-34 from 447251 to 1182931 tid 1 2024-04-12T11:22:05.627 INFO:tasks.rados.rados.0.smithi029.stdout:1810: writing smithi02979277-34 from 1736139 to 2175034 tid 2 2024-04-12T11:22:05.634 INFO:tasks.rados.rados.0.smithi029.stdout:1810: writing smithi02979277-34 from 2597781 to 3378580 tid 3 2024-04-12T11:22:05.634 INFO:tasks.rados.rados.0.smithi029.stdout:1811: snap_create 2024-04-12T11:22:05.635 INFO:tasks.rados.rados.0.smithi029.stdout:1808: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:22:05.635 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-04-12T11:22:05.635 INFO:tasks.rados.rados.0.smithi029.stdout:1809: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:22:05.635 INFO:tasks.rados.rados.0.smithi029.stdout:1809: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:22:05.635 INFO:tasks.rados.rados.0.smithi029.stdout:1809: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:22:05.635 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 668 (ObjNum 692 snap 224 seq_num 692) dirty exists 2024-04-12T11:22:05.635 INFO:tasks.rados.rados.0.smithi029.stdout:1810: finishing write tid 1 to smithi02979277-34 2024-04-12T11:22:05.635 INFO:tasks.rados.rados.0.smithi029.stdout:1810: finishing write tid 2 to smithi02979277-34 2024-04-12T11:22:05.638 INFO:tasks.rados.rados.0.smithi029.stdout:1810: finishing write tid 3 to smithi02979277-34 2024-04-12T11:22:05.638 INFO:tasks.rados.rados.0.smithi029.stdout:1810: finishing write tid 4 to smithi02979277-34 2024-04-12T11:22:05.638 INFO:tasks.rados.rados.0.smithi029.stdout:1810: finishing write tid 5 to smithi02979277-34 2024-04-12T11:22:05.638 INFO:tasks.rados.rados.0.smithi029.stdout:1810: oid 34 updating version 0 to 673 2024-04-12T11:22:05.638 INFO:tasks.rados.rados.0.smithi029.stdout:1810: oid 34 updating version 673 to 674 2024-04-12T11:22:05.638 INFO:tasks.rados.rados.0.smithi029.stdout:1810: oid 34 updating version 674 to 675 2024-04-12T11:22:05.638 INFO:tasks.rados.rados.0.smithi029.stdout:1810: oid 34 updating version 675 to 676 2024-04-12T11:22:05.638 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 676 (ObjNum 693 snap 228 seq_num 693) dirty exists 2024-04-12T11:22:05.638 INFO:tasks.rados.rados.0.smithi029.stdout:1810: left oid 34 (ObjNum 693 snap 228 seq_num 693) 2024-04-12T11:22:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:05 smithi087 ceph-mon[89014]: Upgrade: Updating osd.1 2024-04-12T11:22:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:05 smithi087 ceph-mon[89014]: Deploying daemon osd.1 on smithi029 2024-04-12T11:22:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:05 smithi087 ceph-mon[89014]: osdmap e521: 8 total, 8 up, 8 in 2024-04-12T11:22:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:05 smithi029 ceph-mon[93650]: Upgrade: Updating osd.1 2024-04-12T11:22:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:05 smithi029 ceph-mon[93650]: Deploying daemon osd.1 on smithi029 2024-04-12T11:22:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:05 smithi029 ceph-mon[93650]: osdmap e521: 8 total, 8 up, 8 in 2024-04-12T11:22:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:05 smithi029 ceph-mon[97787]: Upgrade: Updating osd.1 2024-04-12T11:22:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:05 smithi029 ceph-mon[97787]: Deploying daemon osd.1 on smithi029 2024-04-12T11:22:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:05 smithi029 ceph-mon[97787]: osdmap e521: 8 total, 8 up, 8 in 2024-04-12T11:22:06.630 INFO:tasks.rados.rados.0.smithi029.stdout:1807: done (4 left) 2024-04-12T11:22:06.630 INFO:tasks.rados.rados.0.smithi029.stdout:1808: done (3 left) 2024-04-12T11:22:06.631 INFO:tasks.rados.rados.0.smithi029.stdout:1809: done (2 left) 2024-04-12T11:22:06.631 INFO:tasks.rados.rados.0.smithi029.stdout:1810: done (1 left) 2024-04-12T11:22:06.631 INFO:tasks.rados.rados.0.smithi029.stdout:1811: done (0 left) 2024-04-12T11:22:06.631 INFO:tasks.rados.rados.0.smithi029.stdout:1812: rollback oid 49 current snap is 229 2024-04-12T11:22:06.631 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 49 to 222 2024-04-12T11:22:06.631 INFO:tasks.rados.rados.0.smithi029.stdout:1813: snap_create 2024-04-12T11:22:06.633 INFO:tasks.rados.rados.0.smithi029.stdout:1812: finishing rollback tid 1 to smithi02979277-49 2024-04-12T11:22:06.633 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 0 (ObjNum 33 snap 0 seq_num 1800776864) dirty dne 2024-04-12T11:22:06.764 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:06 smithi029 ceph-mon[97787]: pgmap v417: 89 pgs: 89 active+clean; 121 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.9 MiB/s wr, 20 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:06.764 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:06.764 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:06.764 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1810273350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:06.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:06 smithi029 ceph-mon[93650]: pgmap v417: 89 pgs: 89 active+clean; 121 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.9 MiB/s wr, 20 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:06.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:06.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:06.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1810273350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:06 smithi087 ceph-mon[89014]: pgmap v417: 89 pgs: 89 active+clean; 121 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.9 MiB/s wr, 20 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1810273350' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:07.561 INFO:tasks.rados.rados.0.smithi029.stdout:1812: done (1 left) 2024-04-12T11:22:07.561 INFO:tasks.rados.rados.0.smithi029.stdout:1813: done (0 left) 2024-04-12T11:22:07.561 INFO:tasks.rados.rados.0.smithi029.stdout:1814: delete oid 18 current snap is 230 2024-04-12T11:22:07.570 INFO:tasks.rados.rados.0.smithi029.stdout:1814: done (0 left) 2024-04-12T11:22:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:1815: write oid 38 current snap is 230 2024-04-12T11:22:07.571 INFO:tasks.rados.rados.0.smithi029.stdout:1815: seq_num 695 ranges {480865=486178,1453103=442564,2652627=519696} 2024-04-12T11:22:07.576 INFO:tasks.rados.rados.0.smithi029.stdout:1815: writing smithi02979277-38 from 480865 to 967043 tid 1 2024-04-12T11:22:07.581 INFO:tasks.rados.rados.0.smithi029.stdout:1815: writing smithi02979277-38 from 1453103 to 1895667 tid 2 2024-04-12T11:22:07.588 INFO:tasks.rados.rados.0.smithi029.stdout:1815: writing smithi02979277-38 from 2652627 to 3172323 tid 3 2024-04-12T11:22:07.588 INFO:tasks.rados.rados.0.smithi029.stdout:1816: snap_create 2024-04-12T11:22:07.588 INFO:tasks.rados.rados.0.smithi029.stdout:1815: finishing write tid 1 to smithi02979277-38 2024-04-12T11:22:07.588 INFO:tasks.rados.rados.0.smithi029.stdout:1815: finishing write tid 2 to smithi02979277-38 2024-04-12T11:22:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:1815: finishing write tid 3 to smithi02979277-38 2024-04-12T11:22:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:1815: finishing write tid 4 to smithi02979277-38 2024-04-12T11:22:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:1815: finishing write tid 5 to smithi02979277-38 2024-04-12T11:22:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:1815: oid 38 updating version 0 to 509 2024-04-12T11:22:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:1815: oid 38 version 509 is already newer than 506 2024-04-12T11:22:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:1815: oid 38 version 509 is already newer than 507 2024-04-12T11:22:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:1815: oid 38 version 509 is already newer than 508 2024-04-12T11:22:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 509 (ObjNum 695 snap 230 seq_num 695) dirty exists 2024-04-12T11:22:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:1815: left oid 38 (ObjNum 695 snap 230 seq_num 695) 2024-04-12T11:22:07.758 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:07 smithi029 ceph-mon[97787]: osdmap e522: 8 total, 8 up, 8 in 2024-04-12T11:22:07.758 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:07 smithi029 ceph-mon[97787]: osdmap e523: 8 total, 8 up, 8 in 2024-04-12T11:22:07.758 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:07 smithi029 ceph-mon[93650]: osdmap e522: 8 total, 8 up, 8 in 2024-04-12T11:22:07.758 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:07 smithi029 ceph-mon[93650]: osdmap e523: 8 total, 8 up, 8 in 2024-04-12T11:22:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:07 smithi087 ceph-mon[89014]: osdmap e522: 8 total, 8 up, 8 in 2024-04-12T11:22:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:07 smithi087 ceph-mon[89014]: osdmap e523: 8 total, 8 up, 8 in 2024-04-12T11:22:08.018 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:07 smithi029 systemd[1]: Stopping Ceph osd.1 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:22:08.518 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[50016]: 2024-04-12T11:22:08.205+0000 7f743171c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:22:08.518 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[50016]: 2024-04-12T11:22:08.205+0000 7f743171c700 -1 osd.1 523 *** Got signal Terminated *** 2024-04-12T11:22:08.518 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[50016]: 2024-04-12T11:22:08.205+0000 7f743171c700 -1 osd.1 523 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T11:22:08.568 INFO:tasks.rados.rados.0.smithi029.stdout:1815: done (1 left) 2024-04-12T11:22:08.568 INFO:tasks.rados.rados.0.smithi029.stdout:1816: done (0 left) 2024-04-12T11:22:08.568 INFO:tasks.rados.rados.0.smithi029.stdout:1817: snap_remove snap 225 2024-04-12T11:22:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:08 smithi087 ceph-mon[89014]: pgmap v420: 89 pgs: 89 active+clean; 123 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 478 KiB/s wr, 4 op/s 2024-04-12T11:22:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:08 smithi087 ceph-mon[89014]: osd.1 marked itself down and dead 2024-04-12T11:22:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:08 smithi087 ceph-mon[89014]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:22:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:08 smithi087 ceph-mon[89014]: osdmap e524: 8 total, 7 up, 8 in 2024-04-12T11:22:09.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-mon[93650]: pgmap v420: 89 pgs: 89 active+clean; 123 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 478 KiB/s wr, 4 op/s 2024-04-12T11:22:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-mon[93650]: osd.1 marked itself down and dead 2024-04-12T11:22:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-mon[93650]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:22:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-mon[93650]: osdmap e524: 8 total, 7 up, 8 in 2024-04-12T11:22:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-mon[97787]: pgmap v420: 89 pgs: 89 active+clean; 123 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 478 KiB/s wr, 4 op/s 2024-04-12T11:22:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-mon[97787]: osd.1 marked itself down and dead 2024-04-12T11:22:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-mon[97787]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:22:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:08 smithi029 ceph-mon[97787]: osdmap e524: 8 total, 7 up, 8 in 2024-04-12T11:22:09.435 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:09 smithi029 podman[110761]: 2024-04-12 11:22:09.120818865 +0000 UTC m=+1.019008033 container died bb552e04f5782204cc59db35d8e331b65ecda6319a8452465f6faca2dc7ca603 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=reef-d540eba, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_CLEAN=True, ceph=True) 2024-04-12T11:22:09.569 INFO:tasks.rados.rados.0.smithi029.stdout:1817: done (0 left) 2024-04-12T11:22:09.569 INFO:tasks.rados.rados.0.smithi029.stdout:1818: read oid 4 snap -1 2024-04-12T11:22:09.569 INFO:tasks.rados.rados.0.smithi029.stdout:1818: expect deleted 2024-04-12T11:22:09.569 INFO:tasks.rados.rados.0.smithi029.stdout:1819: snap_remove snap 221 2024-04-12T11:22:09.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3687910621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:09 smithi029 ceph-mon[93650]: osdmap e525: 8 total, 7 up, 8 in 2024-04-12T11:22:09.768 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:09 smithi029 podman[110761]: 2024-04-12 11:22:09.652697912 +0000 UTC m=+1.550887133 container cleanup bb552e04f5782204cc59db35d8e331b65ecda6319a8452465f6faca2dc7ca603 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, RELEASE=reef-d540eba, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-12T11:22:09.768 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:09 smithi029 bash[110761]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1 2024-04-12T11:22:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3687910621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:09 smithi029 ceph-mon[97787]: osdmap e525: 8 total, 7 up, 8 in 2024-04-12T11:22:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3687910621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:09 smithi087 ceph-mon[89014]: osdmap e525: 8 total, 7 up, 8 in 2024-04-12T11:22:10.268 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:09 smithi029 podman[110815]: 2024-04-12 11:22:09.974446597 +0000 UTC m=+0.855870693 container remove bb552e04f5782204cc59db35d8e331b65ecda6319a8452465f6faca2dc7ca603 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1, org.label-schema.license=GPLv2, RELEASE=reef-d540eba, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True) 2024-04-12T11:22:10.569 INFO:tasks.rados.rados.0.smithi029.stdout:1819: done (1 left) 2024-04-12T11:22:10.570 INFO:tasks.rados.rados.0.smithi029.stdout:1820: delete oid 17 current snap is 231 2024-04-12T11:22:10.587 INFO:tasks.rados.rados.0.smithi029.stdout:1818: done (1 left) 2024-04-12T11:22:10.587 INFO:tasks.rados.rados.0.smithi029.stdout:1820: done (0 left) 2024-04-12T11:22:10.587 INFO:tasks.rados.rados.0.smithi029.stdout:1821: read oid 11 snap -1 2024-04-12T11:22:10.587 INFO:tasks.rados.rados.0.smithi029.stdout:1821: expect deleted 2024-04-12T11:22:10.587 INFO:tasks.rados.rados.0.smithi029.stdout:1822: rollback oid 39 current snap is 231 2024-04-12T11:22:10.587 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 217 2024-04-12T11:22:10.587 INFO:tasks.rados.rados.0.smithi029.stdout:1823: write oid 36 current snap is 231 2024-04-12T11:22:10.587 INFO:tasks.rados.rados.0.smithi029.stdout:1823: seq_num 697 ranges {514479=636676,1853714=446233,2991120=542240} 2024-04-12T11:22:10.593 INFO:tasks.rados.rados.0.smithi029.stdout:1823: writing smithi02979277-36 from 514479 to 1151155 tid 1 2024-04-12T11:22:10.599 INFO:tasks.rados.rados.0.smithi029.stdout:1823: writing smithi02979277-36 from 1853714 to 2299947 tid 2 2024-04-12T11:22:10.606 INFO:tasks.rados.rados.0.smithi029.stdout:1823: writing smithi02979277-36 from 2991120 to 3533360 tid 3 2024-04-12T11:22:10.606 INFO:tasks.rados.rados.0.smithi029.stdout:1824: write oid 9 current snap is 231 2024-04-12T11:22:10.606 INFO:tasks.rados.rados.0.smithi029.stdout:1824: seq_num 698 ranges {531286=711925,1912196=706244,3018543=712743} 2024-04-12T11:22:10.613 INFO:tasks.rados.rados.0.smithi029.stdout:1824: writing smithi02979277-9 from 531286 to 1243211 tid 1 2024-04-12T11:22:10.620 INFO:tasks.rados.rados.0.smithi029.stdout:1824: writing smithi02979277-9 from 1912196 to 2618440 tid 2 2024-04-12T11:22:10.627 INFO:tasks.rados.rados.0.smithi029.stdout:1824: writing smithi02979277-9 from 3018543 to 3731286 tid 3 2024-04-12T11:22:10.627 INFO:tasks.rados.rados.0.smithi029.stdout:1825: read oid 20 snap -1 2024-04-12T11:22:10.627 INFO:tasks.rados.rados.0.smithi029.stdout:1825: expect deleted 2024-04-12T11:22:10.627 INFO:tasks.rados.rados.0.smithi029.stdout:1826: snap_remove snap 227 2024-04-12T11:22:10.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:10 smithi029 ceph-mon[93650]: pgmap v423: 89 pgs: 11 stale+active+clean, 78 active+clean; 123 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 478 KiB/s wr, 4 op/s 2024-04-12T11:22:10.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:10 smithi029 ceph-mon[93650]: osdmap e526: 8 total, 7 up, 8 in 2024-04-12T11:22:10.723 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:10 smithi029 ceph-mon[97787]: pgmap v423: 89 pgs: 11 stale+active+clean, 78 active+clean; 123 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 478 KiB/s wr, 4 op/s 2024-04-12T11:22:10.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:10 smithi029 ceph-mon[97787]: osdmap e526: 8 total, 7 up, 8 in 2024-04-12T11:22:10.724 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:10 smithi029 podman[110908]: 2024-04-12 11:22:10.319234205 +0000 UTC m=+0.017959425 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:22:10.724 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:10 smithi029 podman[110908]: 2024-04-12 11:22:10.426314965 +0000 UTC m=+0.125040173 container create 0550d9c7610cfa7e96c6092c3ba25323819c5dc3af1cb3bd8314ebafb75057da (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:22:10.724 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:10 smithi029 podman[110908]: 2024-04-12 11:22:10.640462191 +0000 UTC m=+0.339187414 container init 0550d9c7610cfa7e96c6092c3ba25323819c5dc3af1cb3bd8314ebafb75057da (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-12T11:22:10.724 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:10 smithi029 podman[110908]: 2024-04-12 11:22:10.644576905 +0000 UTC m=+0.343302121 container start 0550d9c7610cfa7e96c6092c3ba25323819c5dc3af1cb3bd8314ebafb75057da (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-rf-comb1-2cee185, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_BRANCH=HEAD) 2024-04-12T11:22:10.724 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:10 smithi029 podman[110908]: 2024-04-12 11:22:10.723409335 +0000 UTC m=+0.422134555 container attach 0550d9c7610cfa7e96c6092c3ba25323819c5dc3af1cb3bd8314ebafb75057da (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409) 2024-04-12T11:22:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:10 smithi087 ceph-mon[89014]: pgmap v423: 89 pgs: 11 stale+active+clean, 78 active+clean; 123 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 478 KiB/s wr, 4 op/s 2024-04-12T11:22:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:10 smithi087 ceph-mon[89014]: osdmap e526: 8 total, 7 up, 8 in 2024-04-12T11:22:11.018 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:10 smithi029 podman[110908]: 2024-04-12 11:22:10.957478375 +0000 UTC m=+0.656203589 container died 0550d9c7610cfa7e96c6092c3ba25323819c5dc3af1cb3bd8314ebafb75057da (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-12T11:22:11.571 INFO:tasks.rados.rados.0.smithi029.stdout:1826: done (5 left) 2024-04-12T11:22:11.571 INFO:tasks.rados.rados.0.smithi029.stdout:1827: write oid 44 current snap is 231 2024-04-12T11:22:11.572 INFO:tasks.rados.rados.0.smithi029.stdout:1827: seq_num 699 ranges {548093=787174,1970678=449902,3045966=681137} 2024-04-12T11:22:11.578 INFO:tasks.rados.rados.0.smithi029.stdout:1827: writing smithi02979277-44 from 548093 to 1335267 tid 1 2024-04-12T11:22:11.584 INFO:tasks.rados.rados.0.smithi029.stdout:1827: writing smithi02979277-44 from 1970678 to 2420580 tid 2 2024-04-12T11:22:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:1827: writing smithi02979277-44 from 3045966 to 3727103 tid 3 2024-04-12T11:22:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:1828: rollback oid 33 current snap is 231 2024-04-12T11:22:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 218 2024-04-12T11:22:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:1822: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:22:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:1822: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:22:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:1822: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:22:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 205 (ObjNum 620 snap 200 seq_num 620) dirty exists 2024-04-12T11:22:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:1821: done (6 left) 2024-04-12T11:22:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:1822: done (5 left) 2024-04-12T11:22:11.593 INFO:tasks.rados.rados.0.smithi029.stdout:1829: delete oid 8 current snap is 231 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1823: finishing write tid 1 to smithi02979277-36 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1823: finishing write tid 2 to smithi02979277-36 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1823: finishing write tid 3 to smithi02979277-36 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1823: finishing write tid 4 to smithi02979277-36 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1823: finishing write tid 5 to smithi02979277-36 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1823: oid 36 updating version 0 to 400 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1823: oid 36 updating version 400 to 403 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1823: oid 36 version 403 is already newer than 402 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1823: oid 36 version 403 is already newer than 401 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 403 (ObjNum 697 snap 231 seq_num 697) dirty exists 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1823: left oid 36 (ObjNum 697 snap 231 seq_num 697) 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1824: finishing write tid 1 to smithi02979277-9 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1824: finishing write tid 2 to smithi02979277-9 2024-04-12T11:22:11.594 INFO:tasks.rados.rados.0.smithi029.stdout:1824: finishing write tid 3 to smithi02979277-9 2024-04-12T11:22:11.595 INFO:tasks.rados.rados.0.smithi029.stdout:1824: finishing write tid 4 to smithi02979277-9 2024-04-12T11:22:11.595 INFO:tasks.rados.rados.0.smithi029.stdout:1824: finishing write tid 5 to smithi02979277-9 2024-04-12T11:22:11.595 INFO:tasks.rados.rados.0.smithi029.stdout:1824: oid 9 updating version 0 to 671 2024-04-12T11:22:11.595 INFO:tasks.rados.rados.0.smithi029.stdout:1824: oid 9 updating version 671 to 673 2024-04-12T11:22:11.595 INFO:tasks.rados.rados.0.smithi029.stdout:1824: oid 9 version 673 is already newer than 672 2024-04-12T11:22:11.595 INFO:tasks.rados.rados.0.smithi029.stdout:1824: oid 9 version 673 is already newer than 670 2024-04-12T11:22:11.595 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 673 (ObjNum 698 snap 231 seq_num 698) dirty exists 2024-04-12T11:22:11.595 INFO:tasks.rados.rados.0.smithi029.stdout:1824: left oid 9 (ObjNum 698 snap 231 seq_num 698) 2024-04-12T11:22:11.595 INFO:tasks.rados.rados.0.smithi029.stdout:1827: finishing write tid 1 to smithi02979277-44 2024-04-12T11:22:11.595 INFO:tasks.rados.rados.0.smithi029.stdout:1827: finishing write tid 2 to smithi02979277-44 2024-04-12T11:22:11.596 INFO:tasks.rados.rados.0.smithi029.stdout:1823: done (5 left) 2024-04-12T11:22:11.596 INFO:tasks.rados.rados.0.smithi029.stdout:1824: done (4 left) 2024-04-12T11:22:11.596 INFO:tasks.rados.rados.0.smithi029.stdout:1825: done (3 left) 2024-04-12T11:22:11.596 INFO:tasks.rados.rados.0.smithi029.stdout:1829: done (2 left) 2024-04-12T11:22:11.596 INFO:tasks.rados.rados.0.smithi029.stdout:1830: write oid 40 current snap is 231 2024-04-12T11:22:11.596 INFO:tasks.rados.rados.0.smithi029.stdout:1830: seq_num 701 ranges {581707=654025,1687642=453571,2700812=703681} 2024-04-12T11:22:11.601 INFO:tasks.rados.rados.0.smithi029.stdout:1830: writing smithi02979277-40 from 581707 to 1235732 tid 1 2024-04-12T11:22:11.606 INFO:tasks.rados.rados.0.smithi029.stdout:1830: writing smithi02979277-40 from 1687642 to 2141213 tid 2 2024-04-12T11:22:11.614 INFO:tasks.rados.rados.0.smithi029.stdout:1830: writing smithi02979277-40 from 2700812 to 3404493 tid 3 2024-04-12T11:22:11.614 INFO:tasks.rados.rados.0.smithi029.stdout:1831: write oid 16 current snap is 231 2024-04-12T11:22:11.614 INFO:tasks.rados.rados.0.smithi029.stdout:1831: seq_num 702 ranges {598514=729274,1746124=597229,3128235=514953} 2024-04-12T11:22:11.621 INFO:tasks.rados.rados.0.smithi029.stdout:1831: writing smithi02979277-16 from 598514 to 1327788 tid 1 2024-04-12T11:22:11.626 INFO:tasks.rados.rados.0.smithi029.stdout:1831: writing smithi02979277-16 from 1746124 to 2343353 tid 2 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:1831: writing smithi02979277-16 from 3128235 to 3643188 tid 3 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:1832: rollback oid 21 current snap is 231 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 21 to 220 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:1827: finishing write tid 3 to smithi02979277-44 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:1827: finishing write tid 4 to smithi02979277-44 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:1833: rollback oid 20 current snap is 231 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 20 to 223 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:1827: finishing write tid 5 to smithi02979277-44 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:1827: oid 44 updating version 0 to 845 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:1827: oid 44 updating version 845 to 846 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:1827: oid 44 updating version 846 to 848 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:1827: oid 44 version 848 is already newer than 847 2024-04-12T11:22:11.635 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 848 (ObjNum 699 snap 231 seq_num 699) dirty exists 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1827: left oid 44 (ObjNum 699 snap 231 seq_num 699) 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1828: finishing rollback tid 0 to smithi02979277-33 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1828: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1827: done (5 left) 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1834: snap_create 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1828: finishing rollback tid 2 to smithi02979277-33 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 612 (ObjNum 632 snap 205 seq_num 632) dirty exists 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1830: finishing write tid 1 to smithi02979277-40 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1830: finishing write tid 2 to smithi02979277-40 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1830: finishing write tid 3 to smithi02979277-40 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1830: finishing write tid 4 to smithi02979277-40 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1830: finishing write tid 5 to smithi02979277-40 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1830: oid 40 updating version 0 to 1137 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1830: oid 40 version 1137 is already newer than 1136 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1830: oid 40 version 1137 is already newer than 1135 2024-04-12T11:22:11.636 INFO:tasks.rados.rados.0.smithi029.stdout:1830: oid 40 version 1137 is already newer than 1134 2024-04-12T11:22:11.637 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 1137 (ObjNum 701 snap 231 seq_num 701) dirty exists 2024-04-12T11:22:11.637 INFO:tasks.rados.rados.0.smithi029.stdout:1830: left oid 40 (ObjNum 701 snap 231 seq_num 701) 2024-04-12T11:22:11.637 INFO:tasks.rados.rados.0.smithi029.stdout:1831: finishing write tid 1 to smithi02979277-16 2024-04-12T11:22:11.637 INFO:tasks.rados.rados.0.smithi029.stdout:1831: finishing write tid 2 to smithi02979277-16 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:1833: finishing rollback tid 1 to smithi02979277-20 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 497 (ObjNum 33 snap 0 seq_num 1777349880) dirty dne 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:1831: finishing write tid 3 to smithi02979277-16 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:1831: finishing write tid 4 to smithi02979277-16 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:1831: finishing write tid 5 to smithi02979277-16 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:1831: oid 16 updating version 0 to 957 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:1831: oid 16 version 957 is already newer than 955 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:1831: oid 16 updating version 957 to 958 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:1831: oid 16 version 958 is already newer than 956 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 958 (ObjNum 702 snap 231 seq_num 702) dirty exists 2024-04-12T11:22:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:1831: left oid 16 (ObjNum 702 snap 231 seq_num 702) 2024-04-12T11:22:11.639 INFO:tasks.rados.rados.0.smithi029.stdout:1832: finishing rollback tid 0 to smithi02979277-21 2024-04-12T11:22:11.639 INFO:tasks.rados.rados.0.smithi029.stdout:1832: finishing rollback tid 1 to smithi02979277-21 2024-04-12T11:22:11.639 INFO:tasks.rados.rados.0.smithi029.stdout:1832: finishing rollback tid 2 to smithi02979277-21 2024-04-12T11:22:11.639 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 852 (ObjNum 672 snap 215 seq_num 672) dirty exists 2024-04-12T11:22:11.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:11 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3285949564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:11.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:11 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:11.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:11 smithi029 ceph-mon[97787]: osdmap e527: 8 total, 7 up, 8 in 2024-04-12T11:22:11.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:11 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3285949564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:11.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:11 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:11.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:11 smithi029 ceph-mon[93650]: osdmap e527: 8 total, 7 up, 8 in 2024-04-12T11:22:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:11 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3285949564' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:11 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:11 smithi087 ceph-mon[89014]: osdmap e527: 8 total, 7 up, 8 in 2024-04-12T11:22:12.268 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:11 smithi029 podman[110957]: 2024-04-12 11:22:11.894749057 +0000 UTC m=+0.930905799 container remove 0550d9c7610cfa7e96c6092c3ba25323819c5dc3af1cb3bd8314ebafb75057da (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, RELEASE=wip-rf-comb1-2cee185, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240409) 2024-04-12T11:22:12.268 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:11 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-12T11:22:12.268 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:11 smithi029 systemd[1]: Stopped Ceph osd.1 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:22:12.268 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:11 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.1.service: Consumed 12.079s CPU time. 2024-04-12T11:22:12.268 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:12 smithi029 systemd[1]: Starting Ceph osd.1 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:22:12.572 INFO:tasks.rados.rados.0.smithi029.stdout:1828: done (5 left) 2024-04-12T11:22:12.572 INFO:tasks.rados.rados.0.smithi029.stdout:1830: done (4 left) 2024-04-12T11:22:12.572 INFO:tasks.rados.rados.0.smithi029.stdout:1831: done (3 left) 2024-04-12T11:22:12.572 INFO:tasks.rados.rados.0.smithi029.stdout:1832: done (2 left) 2024-04-12T11:22:12.572 INFO:tasks.rados.rados.0.smithi029.stdout:1833: done (1 left) 2024-04-12T11:22:12.572 INFO:tasks.rados.rados.0.smithi029.stdout:1834: done (0 left) 2024-04-12T11:22:12.572 INFO:tasks.rados.rados.0.smithi029.stdout:1835: snap_create 2024-04-12T11:22:12.620 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:12 smithi029 podman[111064]: 2024-04-12 11:22:12.226529171 +0000 UTC m=+0.017942879 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:22:12.620 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:12 smithi029 podman[111064]: 2024-04-12 11:22:12.389612418 +0000 UTC m=+0.181026153 container create 4cff3370fb50982632ea0e8284d6b99f20abb7a0d0c075ef56b9acd9f2090ee8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=wip-rf-comb1-2cee185, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-12T11:22:12.620 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:12 smithi029 podman[111064]: 2024-04-12 11:22:12.620530734 +0000 UTC m=+0.411944428 container init 4cff3370fb50982632ea0e8284d6b99f20abb7a0d0c075ef56b9acd9f2090ee8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-rf-comb1-2cee185, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-12T11:22:12.645 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:22:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:22:12.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:22:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:12 smithi087 ceph-mon[89014]: pgmap v426: 89 pgs: 11 stale+active+clean, 78 active+clean; 123 MiB data, 565 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:22:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:12 smithi087 ceph-mon[89014]: osdmap e528: 8 total, 7 up, 8 in 2024-04-12T11:22:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:12 smithi029 ceph-mon[93650]: pgmap v426: 89 pgs: 11 stale+active+clean, 78 active+clean; 123 MiB data, 565 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:22:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:12 smithi029 ceph-mon[93650]: osdmap e528: 8 total, 7 up, 8 in 2024-04-12T11:22:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:12 smithi029 ceph-mon[97787]: pgmap v426: 89 pgs: 11 stale+active+clean, 78 active+clean; 123 MiB data, 565 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:22:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:12 smithi029 ceph-mon[97787]: osdmap e528: 8 total, 7 up, 8 in 2024-04-12T11:22:13.018 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:12 smithi029 podman[111064]: 2024-04-12 11:22:12.624243099 +0000 UTC m=+0.415656798 container start 4cff3370fb50982632ea0e8284d6b99f20abb7a0d0c075ef56b9acd9f2090ee8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate, ceph=True, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:22:13.018 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:12 smithi029 podman[111064]: 2024-04-12 11:22:12.729481824 +0000 UTC m=+0.520895529 container attach 4cff3370fb50982632ea0e8284d6b99f20abb7a0d0c075ef56b9acd9f2090ee8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-12T11:22:13.435 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate[111078]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T11:22:13.435 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate[111078]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T11:22:13.435 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 bash[111064]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T11:22:13.435 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 bash[111064]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 bash[111064]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 bash[111064]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 bash[111064]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 bash[111064]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 bash[111064]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate[111078]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate[111078]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate[111078]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate[111078]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate[111078]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-12T11:22:13.436 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 podman[111064]: 2024-04-12 11:22:13.098519663 +0000 UTC m=+0.889933350 container died 4cff3370fb50982632ea0e8284d6b99f20abb7a0d0c075ef56b9acd9f2090ee8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-12T11:22:13.574 INFO:tasks.rados.rados.0.smithi029.stdout:1835: done (0 left) 2024-04-12T11:22:13.574 INFO:tasks.rados.rados.0.smithi029.stdout:1836: snap_create 2024-04-12T11:22:13.756 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4015182490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:13.756 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-mon[97787]: osdmap e529: 8 total, 7 up, 8 in 2024-04-12T11:22:13.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4015182490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:13.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:13 smithi029 ceph-mon[93650]: osdmap e529: 8 total, 7 up, 8 in 2024-04-12T11:22:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:13 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4015182490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:13 smithi087 ceph-mon[89014]: osdmap e529: 8 total, 7 up, 8 in 2024-04-12T11:22:14.018 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 podman[111211]: 2024-04-12 11:22:13.756289326 +0000 UTC m=+0.649341530 container remove 4cff3370fb50982632ea0e8284d6b99f20abb7a0d0c075ef56b9acd9f2090ee8 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, org.label-schema.schema-version=1.0) 2024-04-12T11:22:14.018 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 podman[111303]: 2024-04-12 11:22:13.868283946 +0000 UTC m=+0.017751805 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:22:14.018 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:13 smithi029 podman[111303]: 2024-04-12 11:22:13.96640015 +0000 UTC m=+0.115867989 container create 260ced18c216f31b0394d7c1f6bdf8540c18349534482ec31848288db75daffc (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-12T11:22:14.518 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:14 smithi029 podman[111303]: 2024-04-12 11:22:14.162314242 +0000 UTC m=+0.311782084 container init 260ced18c216f31b0394d7c1f6bdf8540c18349534482ec31848288db75daffc (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-12T11:22:14.518 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:14 smithi029 podman[111303]: 2024-04-12 11:22:14.165338424 +0000 UTC m=+0.314806264 container start 260ced18c216f31b0394d7c1f6bdf8540c18349534482ec31848288db75daffc (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:22:14.518 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-osd[111322]: -- 172.21.15.29:0/40584370 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5607fc28d680 con 0x5607fc256c00 2024-04-12T11:22:14.518 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:14 smithi029 bash[111303]: 260ced18c216f31b0394d7c1f6bdf8540c18349534482ec31848288db75daffc 2024-04-12T11:22:14.518 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:14 smithi029 systemd[1]: Started Ceph osd.1 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:22:14.576 INFO:tasks.rados.rados.0.smithi029.stdout:1836: done (0 left) 2024-04-12T11:22:14.576 INFO:tasks.rados.rados.0.smithi029.stdout:1837: read oid 26 snap 232 2024-04-12T11:22:14.576 INFO:tasks.rados.rados.0.smithi029.stdout:1837: expect (ObjNum 685 snap 222 seq_num 685) 2024-04-12T11:22:14.577 INFO:tasks.rados.rados.0.smithi029.stdout:1838: write oid 30 current snap is 234 2024-04-12T11:22:14.577 INFO:tasks.rados.rados.0.smithi029.stdout:1838: seq_num 703 ranges {615321=404523,1804606=457240,2755658=726225} 2024-04-12T11:22:14.582 INFO:tasks.rados.rados.0.smithi029.stdout:1838: writing smithi02979277-30 from 615321 to 1019844 tid 1 2024-04-12T11:22:14.589 INFO:tasks.rados.rados.0.smithi029.stdout:1838: writing smithi02979277-30 from 1804606 to 2261846 tid 2 2024-04-12T11:22:14.596 INFO:tasks.rados.rados.0.smithi029.stdout:1838: writing smithi02979277-30 from 2755658 to 3481883 tid 3 2024-04-12T11:22:14.596 INFO:tasks.rados.rados.0.smithi029.stdout:1839: snap_create 2024-04-12T11:22:14.596 INFO:tasks.rados.rados.0.smithi029.stdout:1837: expect (ObjNum 685 snap 222 seq_num 685) 2024-04-12T11:22:14.717 INFO:tasks.rados.rados.0.smithi029.stdout:1838: finishing write tid 1 to smithi02979277-30 2024-04-12T11:22:14.718 INFO:tasks.rados.rados.0.smithi029.stdout:1838: finishing write tid 2 to smithi02979277-30 2024-04-12T11:22:14.718 INFO:tasks.rados.rados.0.smithi029.stdout:1838: finishing write tid 3 to smithi02979277-30 2024-04-12T11:22:14.718 INFO:tasks.rados.rados.0.smithi029.stdout:1838: finishing write tid 4 to smithi02979277-30 2024-04-12T11:22:14.718 INFO:tasks.rados.rados.0.smithi029.stdout:1838: finishing write tid 5 to smithi02979277-30 2024-04-12T11:22:14.718 INFO:tasks.rados.rados.0.smithi029.stdout:1838: oid 30 updating version 0 to 694 2024-04-12T11:22:14.718 INFO:tasks.rados.rados.0.smithi029.stdout:1838: oid 30 version 694 is already newer than 693 2024-04-12T11:22:14.718 INFO:tasks.rados.rados.0.smithi029.stdout:1838: oid 30 updating version 694 to 695 2024-04-12T11:22:14.718 INFO:tasks.rados.rados.0.smithi029.stdout:1838: oid 30 version 695 is already newer than 692 2024-04-12T11:22:14.718 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 695 (ObjNum 703 snap 234 seq_num 703) dirty exists 2024-04-12T11:22:14.718 INFO:tasks.rados.rados.0.smithi029.stdout:1838: left oid 30 (ObjNum 703 snap 234 seq_num 703) 2024-04-12T11:22:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[97787]: pgmap v429: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 141 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.6 MiB/s wr, 11 op/s; 19/276 objects degraded (6.884%) 2024-04-12T11:22:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 19/276 objects degraded (6.884%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:14.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[97787]: osdmap e530: 8 total, 7 up, 8 in 2024-04-12T11:22:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[93650]: pgmap v429: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 141 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.6 MiB/s wr, 11 op/s; 19/276 objects degraded (6.884%) 2024-04-12T11:22:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 19/276 objects degraded (6.884%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-mon[93650]: osdmap e530: 8 total, 7 up, 8 in 2024-04-12T11:22:14.927 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[111318]: 2024-04-12T11:22:14.703+0000 7fd5be7fb7c0 -1 Falling back to public interface 2024-04-12T11:22:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:14 smithi087 ceph-mon[89014]: pgmap v429: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 141 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.6 MiB/s wr, 11 op/s; 19/276 objects degraded (6.884%) 2024-04-12T11:22:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:14 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 19/276 objects degraded (6.884%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:14 smithi087 ceph-mon[89014]: osdmap e530: 8 total, 7 up, 8 in 2024-04-12T11:22:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:22:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:22:14] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.43.0" 2024-04-12T11:22:15.578 INFO:tasks.rados.rados.0.smithi029.stdout:1837: done (2 left) 2024-04-12T11:22:15.578 INFO:tasks.rados.rados.0.smithi029.stdout:1838: done (1 left) 2024-04-12T11:22:15.578 INFO:tasks.rados.rados.0.smithi029.stdout:1839: done (0 left) 2024-04-12T11:22:15.578 INFO:tasks.rados.rados.0.smithi029.stdout:1840: delete oid 30 current snap is 235 2024-04-12T11:22:15.580 INFO:tasks.rados.rados.0.smithi029.stdout:1840: done (0 left) 2024-04-12T11:22:15.580 INFO:tasks.rados.rados.0.smithi029.stdout:1841: read oid 24 snap -1 2024-04-12T11:22:15.581 INFO:tasks.rados.rados.0.smithi029.stdout:1841: expect (ObjNum 686 snap 222 seq_num 686) 2024-04-12T11:22:15.581 INFO:tasks.rados.rados.0.smithi029.stdout:1842: read oid 19 snap 228 2024-04-12T11:22:15.581 INFO:tasks.rados.rados.0.smithi029.stdout:1842: expect deleted 2024-04-12T11:22:15.581 INFO:tasks.rados.rados.0.smithi029.stdout:1843: snap_create 2024-04-12T11:22:15.586 INFO:tasks.rados.rados.0.smithi029.stdout:1841: expect (ObjNum 686 snap 222 seq_num 686) 2024-04-12T11:22:16.583 INFO:tasks.rados.rados.0.smithi029.stdout:1841: done (2 left) 2024-04-12T11:22:16.583 INFO:tasks.rados.rados.0.smithi029.stdout:1842: done (1 left) 2024-04-12T11:22:16.583 INFO:tasks.rados.rados.0.smithi029.stdout:1843: done (0 left) 2024-04-12T11:22:16.583 INFO:tasks.rados.rados.0.smithi029.stdout:1844: write oid 21 current snap is 236 2024-04-12T11:22:16.583 INFO:tasks.rados.rados.0.smithi029.stdout:1844: seq_num 705 ranges {648935=555021,1805217=460909,2694151=465122,3817442=31493} 2024-04-12T11:22:16.589 INFO:tasks.rados.rados.0.smithi029.stdout:1844: writing smithi02979277-21 from 648935 to 1203956 tid 1 2024-04-12T11:22:16.595 INFO:tasks.rados.rados.0.smithi029.stdout:1844: writing smithi02979277-21 from 1805217 to 2266126 tid 2 2024-04-12T11:22:16.600 INFO:tasks.rados.rados.0.smithi029.stdout:1844: writing smithi02979277-21 from 2694151 to 3159273 tid 3 2024-04-12T11:22:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:1844: writing smithi02979277-21 from 3817442 to 3848935 tid 4 2024-04-12T11:22:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:1845: write oid 30 current snap is 236 2024-04-12T11:22:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:1844: finishing write tid 1 to smithi02979277-21 2024-04-12T11:22:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:1844: finishing write tid 2 to smithi02979277-21 2024-04-12T11:22:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:1844: finishing write tid 3 to smithi02979277-21 2024-04-12T11:22:16.604 INFO:tasks.rados.rados.0.smithi029.stdout:1845: seq_num 706 ranges {665742=630270,1863699=604567,3121574=676394} 2024-04-12T11:22:16.612 INFO:tasks.rados.rados.0.smithi029.stdout:1845: writing smithi02979277-30 from 665742 to 1296012 tid 1 2024-04-12T11:22:16.619 INFO:tasks.rados.rados.0.smithi029.stdout:1845: writing smithi02979277-30 from 1863699 to 2468266 tid 2 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1845: writing smithi02979277-30 from 3121574 to 3797968 tid 3 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1846: delete oid 4 current snap is 236 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1844: finishing write tid 4 to smithi02979277-21 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1844: finishing write tid 5 to smithi02979277-21 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1844: finishing write tid 6 to smithi02979277-21 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1844: oid 21 updating version 0 to 854 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1844: oid 21 updating version 854 to 856 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1844: oid 21 updating version 856 to 858 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1844: oid 21 version 858 is already newer than 855 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1844: oid 21 version 858 is already newer than 857 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 858 (ObjNum 705 snap 236 seq_num 705) dirty exists 2024-04-12T11:22:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:1844: left oid 21 (ObjNum 705 snap 236 seq_num 705) 2024-04-12T11:22:16.628 INFO:tasks.rados.rados.0.smithi029.stdout:1845: finishing write tid 1 to smithi02979277-30 2024-04-12T11:22:16.628 INFO:tasks.rados.rados.0.smithi029.stdout:1845: finishing write tid 2 to smithi02979277-30 2024-04-12T11:22:16.629 INFO:tasks.rados.rados.0.smithi029.stdout:1844: done (2 left) 2024-04-12T11:22:16.629 INFO:tasks.rados.rados.0.smithi029.stdout:1846: done (1 left) 2024-04-12T11:22:16.629 INFO:tasks.rados.rados.0.smithi029.stdout:1847: snap_create 2024-04-12T11:22:16.630 INFO:tasks.rados.rados.0.smithi029.stdout:1845: finishing write tid 3 to smithi02979277-30 2024-04-12T11:22:16.630 INFO:tasks.rados.rados.0.smithi029.stdout:1845: finishing write tid 4 to smithi02979277-30 2024-04-12T11:22:16.630 INFO:tasks.rados.rados.0.smithi029.stdout:1845: finishing write tid 5 to smithi02979277-30 2024-04-12T11:22:16.630 INFO:tasks.rados.rados.0.smithi029.stdout:1845: oid 30 updating version 0 to 699 2024-04-12T11:22:16.630 INFO:tasks.rados.rados.0.smithi029.stdout:1845: oid 30 updating version 699 to 700 2024-04-12T11:22:16.630 INFO:tasks.rados.rados.0.smithi029.stdout:1845: oid 30 version 700 is already newer than 698 2024-04-12T11:22:16.630 INFO:tasks.rados.rados.0.smithi029.stdout:1845: oid 30 updating version 700 to 701 2024-04-12T11:22:16.630 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 701 (ObjNum 706 snap 236 seq_num 706) dirty exists 2024-04-12T11:22:16.630 INFO:tasks.rados.rados.0.smithi029.stdout:1845: left oid 30 (ObjNum 706 snap 236 seq_num 706) 2024-04-12T11:22:16.959 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:16 smithi029 ceph-mon[97787]: pgmap v432: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 141 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.6 MiB/s wr, 11 op/s; 19/276 objects degraded (6.884%) 2024-04-12T11:22:16.959 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:16 smithi029 ceph-mon[97787]: osdmap e531: 8 total, 7 up, 8 in 2024-04-12T11:22:16.960 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:16 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3342980287' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:16.960 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:16 smithi029 ceph-mon[93650]: pgmap v432: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 141 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.6 MiB/s wr, 11 op/s; 19/276 objects degraded (6.884%) 2024-04-12T11:22:16.960 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:16 smithi029 ceph-mon[93650]: osdmap e531: 8 total, 7 up, 8 in 2024-04-12T11:22:16.960 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:16 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3342980287' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:16 smithi087 ceph-mon[89014]: pgmap v432: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 141 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 2.6 MiB/s wr, 11 op/s; 19/276 objects degraded (6.884%) 2024-04-12T11:22:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:16 smithi087 ceph-mon[89014]: osdmap e531: 8 total, 7 up, 8 in 2024-04-12T11:22:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:16 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3342980287' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:17.588 INFO:tasks.rados.rados.0.smithi029.stdout:1845: done (1 left) 2024-04-12T11:22:17.588 INFO:tasks.rados.rados.0.smithi029.stdout:1847: done (0 left) 2024-04-12T11:22:17.588 INFO:tasks.rados.rados.0.smithi029.stdout:1848: read oid 7 snap -1 2024-04-12T11:22:17.588 INFO:tasks.rados.rados.0.smithi029.stdout:1848: expect (ObjNum 688 snap 223 seq_num 688) 2024-04-12T11:22:17.588 INFO:tasks.rados.rados.0.smithi029.stdout:1849: snap_remove snap 219 2024-04-12T11:22:17.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:17 smithi029 ceph-mon[97787]: osdmap e532: 8 total, 7 up, 8 in 2024-04-12T11:22:17.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:17 smithi029 ceph-mon[97787]: from='osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T11:22:17.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:17 smithi029 ceph-mon[93650]: osdmap e532: 8 total, 7 up, 8 in 2024-04-12T11:22:17.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:17 smithi029 ceph-mon[93650]: from='osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T11:22:17.768 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:17 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[111318]: 2024-04-12T11:22:17.516+0000 7fd5be7fb7c0 -1 osd.1 523 log_to_monitors true 2024-04-12T11:22:17.769 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 11:22:17 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[111318]: 2024-04-12T11:22:17.602+0000 7fd5afc53700 -1 osd.1 523 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:22:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:17 smithi087 ceph-mon[89014]: osdmap e532: 8 total, 7 up, 8 in 2024-04-12T11:22:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:17 smithi087 ceph-mon[89014]: from='osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T11:22:18.595 INFO:tasks.rados.rados.0.smithi029.stdout:1849: done (1 left) 2024-04-12T11:22:18.595 INFO:tasks.rados.rados.0.smithi029.stdout:1850: snap_create 2024-04-12T11:22:18.595 INFO:tasks.rados.rados.0.smithi029.stdout:1848: expect (ObjNum 688 snap 223 seq_num 688) 2024-04-12T11:22:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:18 smithi087 ceph-mon[89014]: pgmap v435: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 143 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 370 KiB/s wr, 6 op/s; 19/279 objects degraded (6.810%) 2024-04-12T11:22:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:18 smithi087 ceph-mon[89014]: from='osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T11:22:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:18 smithi087 ceph-mon[89014]: osdmap e533: 8 total, 7 up, 8 in 2024-04-12T11:22:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:18 smithi087 ceph-mon[89014]: from='osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:22:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:18 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1540093324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:18 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:18 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[93650]: pgmap v435: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 143 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 370 KiB/s wr, 6 op/s; 19/279 objects degraded (6.810%) 2024-04-12T11:22:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[93650]: from='osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T11:22:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[93650]: osdmap e533: 8 total, 7 up, 8 in 2024-04-12T11:22:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[93650]: from='osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:22:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1540093324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:19.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[97787]: pgmap v435: 89 pgs: 7 active+undersized+degraded, 30 active+undersized, 52 active+clean; 143 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 370 KiB/s wr, 6 op/s; 19/279 objects degraded (6.810%) 2024-04-12T11:22:19.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[97787]: from='osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T11:22:19.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[97787]: osdmap e533: 8 total, 7 up, 8 in 2024-04-12T11:22:19.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[97787]: from='osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:22:19.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1540093324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:19.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:19.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:18 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:19.597 INFO:tasks.rados.rados.0.smithi029.stdout:1848: done (1 left) 2024-04-12T11:22:19.598 INFO:tasks.rados.rados.0.smithi029.stdout:1850: done (0 left) 2024-04-12T11:22:19.598 INFO:tasks.rados.rados.0.smithi029.stdout:1851: read oid 38 snap -1 2024-04-12T11:22:19.598 INFO:tasks.rados.rados.0.smithi029.stdout:1851: expect (ObjNum 695 snap 230 seq_num 695) 2024-04-12T11:22:19.598 INFO:tasks.rados.rados.0.smithi029.stdout:1852: snap_create 2024-04-12T11:22:19.606 INFO:tasks.rados.rados.0.smithi029.stdout:1851: expect (ObjNum 695 snap 230 seq_num 695) 2024-04-12T11:22:19.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:19 smithi087 ceph-mon[89014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:22:19.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:19 smithi087 ceph-mon[89014]: osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505] boot 2024-04-12T11:22:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:19 smithi087 ceph-mon[89014]: osdmap e534: 8 total, 8 up, 8 in 2024-04-12T11:22:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:22:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:20.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[93650]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:22:20.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[93650]: osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505] boot 2024-04-12T11:22:20.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[93650]: osdmap e534: 8 total, 8 up, 8 in 2024-04-12T11:22:20.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:22:20.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:20.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:20.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[97787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:22:20.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[97787]: osd.1 [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505] boot 2024-04-12T11:22:20.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[97787]: osdmap e534: 8 total, 8 up, 8 in 2024-04-12T11:22:20.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T11:22:20.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:20.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:20.596 INFO:tasks.rados.rados.0.smithi029.stdout:1851: done (1 left) 2024-04-12T11:22:20.596 INFO:tasks.rados.rados.0.smithi029.stdout:1852: done (0 left) 2024-04-12T11:22:20.596 INFO:tasks.rados.rados.0.smithi029.stdout:1853: snap_create 2024-04-12T11:22:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:20 smithi087 ceph-mon[89014]: pgmap v438: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 150 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 11 op/s; 21/285 objects degraded (7.368%) 2024-04-12T11:22:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:20 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 21/285 objects degraded (7.368%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:20 smithi087 ceph-mon[89014]: osdmap e535: 8 total, 8 up, 8 in 2024-04-12T11:22:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:20 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/564843898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:20 smithi029 ceph-mon[93650]: pgmap v438: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 150 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 11 op/s; 21/285 objects degraded (7.368%) 2024-04-12T11:22:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:20 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 21/285 objects degraded (7.368%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:20 smithi029 ceph-mon[93650]: osdmap e535: 8 total, 8 up, 8 in 2024-04-12T11:22:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:20 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/564843898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:20 smithi029 ceph-mon[97787]: pgmap v438: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 150 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 11 op/s; 21/285 objects degraded (7.368%) 2024-04-12T11:22:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:20 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 21/285 objects degraded (7.368%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:20 smithi029 ceph-mon[97787]: osdmap e535: 8 total, 8 up, 8 in 2024-04-12T11:22:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:20 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/564843898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:21.602 INFO:tasks.rados.rados.0.smithi029.stdout:1853: done (0 left) 2024-04-12T11:22:21.602 INFO:tasks.rados.rados.0.smithi029.stdout:1854: write oid 5 current snap is 240 2024-04-12T11:22:21.602 INFO:tasks.rados.rados.0.smithi029.stdout:1854: seq_num 708 ranges {699356=497121,1980663=608236,3176420=415291} 2024-04-12T11:22:21.608 INFO:tasks.rados.rados.0.smithi029.stdout:1854: writing smithi02979277-5 from 699356 to 1196477 tid 1 2024-04-12T11:22:21.617 INFO:tasks.rados.rados.0.smithi029.stdout:1854: writing smithi02979277-5 from 1980663 to 2588899 tid 2 2024-04-12T11:22:21.623 INFO:tasks.rados.rados.0.smithi029.stdout:1854: writing smithi02979277-5 from 3176420 to 3591711 tid 3 2024-04-12T11:22:21.623 INFO:tasks.rados.rados.0.smithi029.stdout:1855: rollback oid 4 current snap is 240 2024-04-12T11:22:21.623 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 4 to 218 2024-04-12T11:22:21.623 INFO:tasks.rados.rados.0.smithi029.stdout:1854: finishing write tid 1 to smithi02979277-5 2024-04-12T11:22:21.623 INFO:tasks.rados.rados.0.smithi029.stdout:1854: finishing write tid 2 to smithi02979277-5 2024-04-12T11:22:21.623 INFO:tasks.rados.rados.0.smithi029.stdout:1856: snap_create 2024-04-12T11:22:21.626 INFO:tasks.rados.rados.0.smithi029.stdout:1854: finishing write tid 3 to smithi02979277-5 2024-04-12T11:22:21.626 INFO:tasks.rados.rados.0.smithi029.stdout:1854: finishing write tid 4 to smithi02979277-5 2024-04-12T11:22:21.626 INFO:tasks.rados.rados.0.smithi029.stdout:1854: finishing write tid 5 to smithi02979277-5 2024-04-12T11:22:21.626 INFO:tasks.rados.rados.0.smithi029.stdout:1854: oid 5 updating version 0 to 965 2024-04-12T11:22:21.627 INFO:tasks.rados.rados.0.smithi029.stdout:1854: oid 5 version 965 is already newer than 963 2024-04-12T11:22:21.627 INFO:tasks.rados.rados.0.smithi029.stdout:1854: oid 5 version 965 is already newer than 964 2024-04-12T11:22:21.627 INFO:tasks.rados.rados.0.smithi029.stdout:1854: oid 5 version 965 is already newer than 962 2024-04-12T11:22:21.627 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 965 (ObjNum 708 snap 240 seq_num 708) dirty exists 2024-04-12T11:22:21.627 INFO:tasks.rados.rados.0.smithi029.stdout:1854: left oid 5 (ObjNum 708 snap 240 seq_num 708) 2024-04-12T11:22:21.627 INFO:tasks.rados.rados.0.smithi029.stdout:1855: finishing rollback tid 1 to smithi02979277-4 2024-04-12T11:22:21.627 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 0 (ObjNum 193 snap 0 seq_num 1800886688) dirty dne 2024-04-12T11:22:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:21 smithi029 ceph-mon[93650]: osdmap e536: 8 total, 8 up, 8 in 2024-04-12T11:22:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:21 smithi029 ceph-mon[97787]: osdmap e536: 8 total, 8 up, 8 in 2024-04-12T11:22:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:21 smithi087 ceph-mon[89014]: osdmap e536: 8 total, 8 up, 8 in 2024-04-12T11:22:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:22.599 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:22:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:22:22.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:22:22.607 INFO:tasks.rados.rados.0.smithi029.stdout:1854: done (2 left) 2024-04-12T11:22:22.607 INFO:tasks.rados.rados.0.smithi029.stdout:1855: done (1 left) 2024-04-12T11:22:22.607 INFO:tasks.rados.rados.0.smithi029.stdout:1856: done (0 left) 2024-04-12T11:22:22.607 INFO:tasks.rados.rados.0.smithi029.stdout:1857: read oid 32 snap -1 2024-04-12T11:22:22.607 INFO:tasks.rados.rados.0.smithi029.stdout:1857: expect (ObjNum 605 snap 194 seq_num 605) 2024-04-12T11:22:22.607 INFO:tasks.rados.rados.0.smithi029.stdout:1858: snap_create 2024-04-12T11:22:22.611 INFO:tasks.rados.rados.0.smithi029.stdout:1857: expect (ObjNum 605 snap 194 seq_num 605) 2024-04-12T11:22:22.866 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:22 smithi087 ceph-mon[89014]: pgmap v441: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 150 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 855 KiB/s wr, 5 op/s; 21/285 objects degraded (7.368%) 2024-04-12T11:22:22.866 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:22 smithi087 ceph-mon[89014]: osdmap e537: 8 total, 8 up, 8 in 2024-04-12T11:22:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:22 smithi029 ceph-mon[93650]: pgmap v441: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 150 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 855 KiB/s wr, 5 op/s; 21/285 objects degraded (7.368%) 2024-04-12T11:22:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:22 smithi029 ceph-mon[93650]: osdmap e537: 8 total, 8 up, 8 in 2024-04-12T11:22:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:22 smithi029 ceph-mon[97787]: pgmap v441: 89 pgs: 8 active+undersized+degraded, 29 active+undersized, 52 active+clean; 150 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 855 KiB/s wr, 5 op/s; 21/285 objects degraded (7.368%) 2024-04-12T11:22:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:22 smithi029 ceph-mon[97787]: osdmap e537: 8 total, 8 up, 8 in 2024-04-12T11:22:23.613 INFO:tasks.rados.rados.0.smithi029.stdout:1857: done (1 left) 2024-04-12T11:22:23.614 INFO:tasks.rados.rados.0.smithi029.stdout:1858: done (0 left) 2024-04-12T11:22:23.614 INFO:tasks.rados.rados.0.smithi029.stdout:1859: snap_create 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: osdmap e538: 8 total, 8 up, 8 in 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3862116746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:23.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:23.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T11:22:23.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: osdmap e538: 8 total, 8 up, 8 in 2024-04-12T11:22:23.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3862116746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:23.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:23.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:23.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:23.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:22:23.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:23.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:23.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:23.904 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:23.904 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:23.904 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T11:22:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: osdmap e538: 8 total, 8 up, 8 in 2024-04-12T11:22:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3862116746' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:22:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:23.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:23.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:23.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:23.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:23.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T11:22:24.615 INFO:tasks.rados.rados.0.smithi029.stdout:1859: done (0 left) 2024-04-12T11:22:24.615 INFO:tasks.rados.rados.0.smithi029.stdout:1860: write oid 39 current snap is 243 2024-04-12T11:22:24.615 INFO:tasks.rados.rados.0.smithi029.stdout:1860: seq_num 709 ranges {716163=572370,2039145=468247,3203843=626563} 2024-04-12T11:22:24.621 INFO:tasks.rados.rados.0.smithi029.stdout:1860: writing smithi02979277-39 from 716163 to 1288533 tid 1 2024-04-12T11:22:24.628 INFO:tasks.rados.rados.0.smithi029.stdout:1860: writing smithi02979277-39 from 2039145 to 2507392 tid 2 2024-04-12T11:22:24.636 INFO:tasks.rados.rados.0.smithi029.stdout:1860: writing smithi02979277-39 from 3203843 to 3830406 tid 3 2024-04-12T11:22:24.636 INFO:tasks.rados.rados.0.smithi029.stdout:1861: read oid 32 snap -1 2024-04-12T11:22:24.636 INFO:tasks.rados.rados.0.smithi029.stdout:1861: expect (ObjNum 605 snap 194 seq_num 605) 2024-04-12T11:22:24.636 INFO:tasks.rados.rados.0.smithi029.stdout:1862: snap_create 2024-04-12T11:22:24.636 INFO:tasks.rados.rados.0.smithi029.stdout:1860: finishing write tid 1 to smithi02979277-39 2024-04-12T11:22:24.636 INFO:tasks.rados.rados.0.smithi029.stdout:1860: finishing write tid 2 to smithi02979277-39 2024-04-12T11:22:24.639 INFO:tasks.rados.rados.0.smithi029.stdout:1861: expect (ObjNum 605 snap 194 seq_num 605) 2024-04-12T11:22:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:1860: finishing write tid 3 to smithi02979277-39 2024-04-12T11:22:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:1860: finishing write tid 4 to smithi02979277-39 2024-04-12T11:22:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:1860: finishing write tid 5 to smithi02979277-39 2024-04-12T11:22:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:1860: oid 39 updating version 0 to 211 2024-04-12T11:22:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:1860: oid 39 version 211 is already newer than 209 2024-04-12T11:22:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:1860: oid 39 version 211 is already newer than 210 2024-04-12T11:22:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:1860: oid 39 updating version 211 to 212 2024-04-12T11:22:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 212 (ObjNum 709 snap 243 seq_num 709) dirty exists 2024-04-12T11:22:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:1860: left oid 39 (ObjNum 709 snap 243 seq_num 709) 2024-04-12T11:22:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[97787]: pgmap v444: 89 pgs: 89 active+clean; 154 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 371 KiB/s wr, 2 op/s; 2.2 MiB/s, 1 objects/s recovering 2024-04-12T11:22:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[97787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T11:22:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[97787]: Upgrade: osd.2 is safe to restart 2024-04-12T11:22:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[97787]: osdmap e539: 8 total, 8 up, 8 in 2024-04-12T11:22:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/285 objects degraded (7.368%), 8 pgs degraded) 2024-04-12T11:22:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:22:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[93650]: pgmap v444: 89 pgs: 89 active+clean; 154 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 371 KiB/s wr, 2 op/s; 2.2 MiB/s, 1 objects/s recovering 2024-04-12T11:22:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[93650]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T11:22:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[93650]: Upgrade: osd.2 is safe to restart 2024-04-12T11:22:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[93650]: osdmap e539: 8 total, 8 up, 8 in 2024-04-12T11:22:24.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/285 objects degraded (7.368%), 8 pgs degraded) 2024-04-12T11:22:24.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:22:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:24 smithi087 ceph-mon[89014]: pgmap v444: 89 pgs: 89 active+clean; 154 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 371 KiB/s wr, 2 op/s; 2.2 MiB/s, 1 objects/s recovering 2024-04-12T11:22:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:24 smithi087 ceph-mon[89014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T11:22:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:24 smithi087 ceph-mon[89014]: Upgrade: osd.2 is safe to restart 2024-04-12T11:22:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:24 smithi087 ceph-mon[89014]: osdmap e539: 8 total, 8 up, 8 in 2024-04-12T11:22:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:24 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/285 objects degraded (7.368%), 8 pgs degraded) 2024-04-12T11:22:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:24 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:22:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:22:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:22:24] "GET /metrics HTTP/1.1" 200 36232 "" "Prometheus/2.43.0" 2024-04-12T11:22:25.624 INFO:tasks.rados.rados.0.smithi029.stdout:1860: done (2 left) 2024-04-12T11:22:25.624 INFO:tasks.rados.rados.0.smithi029.stdout:1861: done (1 left) 2024-04-12T11:22:25.624 INFO:tasks.rados.rados.0.smithi029.stdout:1862: done (0 left) 2024-04-12T11:22:25.625 INFO:tasks.rados.rados.0.smithi029.stdout:1863: snap_create 2024-04-12T11:22:25.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:25 smithi029 ceph-mon[97787]: osdmap e540: 8 total, 8 up, 8 in 2024-04-12T11:22:25.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:25.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T11:22:25.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:25.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:25 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1200299982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:25.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:25 smithi029 ceph-mon[93650]: osdmap e540: 8 total, 8 up, 8 in 2024-04-12T11:22:25.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:25.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T11:22:25.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:25.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:25 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1200299982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:25 smithi087 ceph-mon[89014]: osdmap e540: 8 total, 8 up, 8 in 2024-04-12T11:22:25.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:25.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T11:22:25.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:25.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:25 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1200299982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:26.627 INFO:tasks.rados.rados.0.smithi029.stdout:1863: done (0 left) 2024-04-12T11:22:26.627 INFO:tasks.rados.rados.0.smithi029.stdout:1864: read oid 35 snap -1 2024-04-12T11:22:26.627 INFO:tasks.rados.rados.0.smithi029.stdout:1864: expect (ObjNum 679 snap 220 seq_num 679) 2024-04-12T11:22:26.627 INFO:tasks.rados.rados.0.smithi029.stdout:1865: snap_remove snap 244 2024-04-12T11:22:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:26 smithi029 ceph-mon[93650]: Upgrade: Updating osd.2 2024-04-12T11:22:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:26 smithi029 ceph-mon[93650]: Deploying daemon osd.2 on smithi029 2024-04-12T11:22:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:26 smithi029 ceph-mon[93650]: pgmap v447: 89 pgs: 89 active+clean; 154 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 371 KiB/s wr, 2 op/s; 2.2 MiB/s, 1 objects/s recovering 2024-04-12T11:22:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:26 smithi029 ceph-mon[93650]: osdmap e541: 8 total, 8 up, 8 in 2024-04-12T11:22:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:26 smithi029 ceph-mon[97787]: Upgrade: Updating osd.2 2024-04-12T11:22:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:26 smithi029 ceph-mon[97787]: Deploying daemon osd.2 on smithi029 2024-04-12T11:22:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:26 smithi029 ceph-mon[97787]: pgmap v447: 89 pgs: 89 active+clean; 154 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 371 KiB/s wr, 2 op/s; 2.2 MiB/s, 1 objects/s recovering 2024-04-12T11:22:26.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:26 smithi029 ceph-mon[97787]: osdmap e541: 8 total, 8 up, 8 in 2024-04-12T11:22:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:26 smithi087 ceph-mon[89014]: Upgrade: Updating osd.2 2024-04-12T11:22:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:26 smithi087 ceph-mon[89014]: Deploying daemon osd.2 on smithi029 2024-04-12T11:22:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:26 smithi087 ceph-mon[89014]: pgmap v447: 89 pgs: 89 active+clean; 154 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 371 KiB/s wr, 2 op/s; 2.2 MiB/s, 1 objects/s recovering 2024-04-12T11:22:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:26 smithi087 ceph-mon[89014]: osdmap e541: 8 total, 8 up, 8 in 2024-04-12T11:22:27.567 INFO:tasks.rados.rados.0.smithi029.stdout:1865: done (1 left) 2024-04-12T11:22:27.567 INFO:tasks.rados.rados.0.smithi029.stdout:1866: write oid 25 current snap is 245 2024-04-12T11:22:27.567 INFO:tasks.rados.rados.0.smithi029.stdout:1866: seq_num 710 ranges {732970=647619,2097627=611905,3231266=437835} 2024-04-12T11:22:27.574 INFO:tasks.rados.rados.0.smithi029.stdout:1866: writing smithi02979277-25 from 732970 to 1380589 tid 1 2024-04-12T11:22:27.582 INFO:tasks.rados.rados.0.smithi029.stdout:1866: writing smithi02979277-25 from 2097627 to 2709532 tid 2 2024-04-12T11:22:27.587 INFO:tasks.rados.rados.0.smithi029.stdout:1866: writing smithi02979277-25 from 3231266 to 3669101 tid 3 2024-04-12T11:22:27.587 INFO:tasks.rados.rados.0.smithi029.stdout:1867: snap_remove snap 217 2024-04-12T11:22:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:27 smithi029 ceph-mon[93650]: osdmap e542: 8 total, 8 up, 8 in 2024-04-12T11:22:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:27 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1292252518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:27 smithi029 ceph-mon[93650]: osdmap e543: 8 total, 8 up, 8 in 2024-04-12T11:22:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:27 smithi029 ceph-mon[97787]: osdmap e542: 8 total, 8 up, 8 in 2024-04-12T11:22:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:27 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1292252518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:27 smithi029 ceph-mon[97787]: osdmap e543: 8 total, 8 up, 8 in 2024-04-12T11:22:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:27 smithi087 ceph-mon[89014]: osdmap e542: 8 total, 8 up, 8 in 2024-04-12T11:22:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:27 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1292252518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:27 smithi087 ceph-mon[89014]: osdmap e543: 8 total, 8 up, 8 in 2024-04-12T11:22:28.568 INFO:tasks.rados.rados.0.smithi029.stdout:1867: done (2 left) 2024-04-12T11:22:28.568 INFO:tasks.rados.rados.0.smithi029.stdout:1868: write oid 2 current snap is 245 2024-04-12T11:22:28.569 INFO:tasks.rados.rados.0.smithi029.stdout:1868: seq_num 711 ranges {749777=722868,2156109=471916,3258689=649107} 2024-04-12T11:22:28.576 INFO:tasks.rados.rados.0.smithi029.stdout:1868: writing smithi02979277-2 from 749777 to 1472645 tid 1 2024-04-12T11:22:28.583 INFO:tasks.rados.rados.0.smithi029.stdout:1868: writing smithi02979277-2 from 2156109 to 2628025 tid 2 2024-04-12T11:22:28.590 INFO:tasks.rados.rados.0.smithi029.stdout:1868: writing smithi02979277-2 from 3258689 to 3907796 tid 3 2024-04-12T11:22:28.591 INFO:tasks.rados.rados.0.smithi029.stdout:1869: read oid 6 snap -1 2024-04-12T11:22:28.591 INFO:tasks.rados.rados.0.smithi029.stdout:1869: expect (ObjNum 424 snap 135 seq_num 424) 2024-04-12T11:22:28.591 INFO:tasks.rados.rados.0.smithi029.stdout:1870: snap_remove snap 226 2024-04-12T11:22:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:28 smithi029 ceph-mon[93650]: pgmap v450: 89 pgs: 89 active+clean; 156 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 407 KiB/s wr, 7 op/s 2024-04-12T11:22:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:28 smithi029 ceph-mon[93650]: osdmap e544: 8 total, 8 up, 8 in 2024-04-12T11:22:28.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:28 smithi029 ceph-mon[97787]: pgmap v450: 89 pgs: 89 active+clean; 156 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 407 KiB/s wr, 7 op/s 2024-04-12T11:22:28.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:28 smithi029 ceph-mon[97787]: osdmap e544: 8 total, 8 up, 8 in 2024-04-12T11:22:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:28 smithi087 ceph-mon[89014]: pgmap v450: 89 pgs: 89 active+clean; 156 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 407 KiB/s wr, 7 op/s 2024-04-12T11:22:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:28 smithi087 ceph-mon[89014]: osdmap e544: 8 total, 8 up, 8 in 2024-04-12T11:22:29.064 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:28 smithi029 systemd[1]: Stopping Ceph osd.2 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:22:29.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[55643]: 2024-04-12T11:22:29.063+0000 7feca36e4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:22:29.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[55643]: 2024-04-12T11:22:29.063+0000 7feca36e4700 -1 osd.2 544 *** Got signal Terminated *** 2024-04-12T11:22:29.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[55643]: 2024-04-12T11:22:29.063+0000 7feca36e4700 -1 osd.2 544 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T11:22:29.571 INFO:tasks.rados.rados.0.smithi029.stdout:1870: done (4 left) 2024-04-12T11:22:29.571 INFO:tasks.rados.rados.0.smithi029.stdout:1871: snap_create 2024-04-12T11:22:29.575 INFO:tasks.rados.rados.0.smithi029.stdout:1864: expect (ObjNum 679 snap 220 seq_num 679) 2024-04-12T11:22:29.746 INFO:tasks.rados.rados.0.smithi029.stdout:1866: finishing write tid 1 to smithi02979277-25 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1866: finishing write tid 2 to smithi02979277-25 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1866: finishing write tid 3 to smithi02979277-25 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1866: finishing write tid 4 to smithi02979277-25 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1866: finishing write tid 5 to smithi02979277-25 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1866: oid 25 updating version 0 to 905 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1866: oid 25 version 905 is already newer than 903 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1866: oid 25 updating version 905 to 906 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1866: oid 25 version 906 is already newer than 904 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 906 (ObjNum 710 snap 245 seq_num 710) dirty exists 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1866: left oid 25 (ObjNum 710 snap 245 seq_num 710) 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1868: finishing write tid 1 to smithi02979277-2 2024-04-12T11:22:29.747 INFO:tasks.rados.rados.0.smithi029.stdout:1868: finishing write tid 2 to smithi02979277-2 2024-04-12T11:22:29.748 INFO:tasks.rados.rados.0.smithi029.stdout:1868: finishing write tid 3 to smithi02979277-2 2024-04-12T11:22:29.748 INFO:tasks.rados.rados.0.smithi029.stdout:1868: finishing write tid 4 to smithi02979277-2 2024-04-12T11:22:29.748 INFO:tasks.rados.rados.0.smithi029.stdout:1868: finishing write tid 5 to smithi02979277-2 2024-04-12T11:22:29.748 INFO:tasks.rados.rados.0.smithi029.stdout:1868: oid 2 updating version 0 to 187 2024-04-12T11:22:29.748 INFO:tasks.rados.rados.0.smithi029.stdout:1868: oid 2 updating version 187 to 188 2024-04-12T11:22:29.748 INFO:tasks.rados.rados.0.smithi029.stdout:1868: oid 2 updating version 188 to 189 2024-04-12T11:22:29.748 INFO:tasks.rados.rados.0.smithi029.stdout:1868: oid 2 version 189 is already newer than 186 2024-04-12T11:22:29.748 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 189 (ObjNum 711 snap 245 seq_num 711) dirty exists 2024-04-12T11:22:29.748 INFO:tasks.rados.rados.0.smithi029.stdout:1868: left oid 2 (ObjNum 711 snap 245 seq_num 711) 2024-04-12T11:22:29.748 INFO:tasks.rados.rados.0.smithi029.stdout:1869: expect (ObjNum 424 snap 135 seq_num 424) 2024-04-12T11:22:29.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:29 smithi087 ceph-mon[89014]: osd.2 marked itself down and dead 2024-04-12T11:22:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:29 smithi087 ceph-mon[89014]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:22:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:29 smithi087 ceph-mon[89014]: osdmap e545: 8 total, 7 up, 8 in 2024-04-12T11:22:30.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:29 smithi029 ceph-mon[93650]: osd.2 marked itself down and dead 2024-04-12T11:22:30.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:29 smithi029 ceph-mon[93650]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:22:30.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:29 smithi029 ceph-mon[93650]: osdmap e545: 8 total, 7 up, 8 in 2024-04-12T11:22:30.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:29 smithi029 ceph-mon[97787]: osd.2 marked itself down and dead 2024-04-12T11:22:30.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:29 smithi029 ceph-mon[97787]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:22:30.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:29 smithi029 ceph-mon[97787]: osdmap e545: 8 total, 7 up, 8 in 2024-04-12T11:22:30.474 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:30 smithi029 podman[115507]: 2024-04-12 11:22:30.102407711 +0000 UTC m=+1.148870525 container died dbb7abb8911b0568db6d0a5d6d9a6a1c974bcec75381ba7173dd80fd0e32f7d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, RELEASE=reef-d540eba, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:22:30.574 INFO:tasks.rados.rados.0.smithi029.stdout:1864: done (4 left) 2024-04-12T11:22:30.574 INFO:tasks.rados.rados.0.smithi029.stdout:1866: done (3 left) 2024-04-12T11:22:30.574 INFO:tasks.rados.rados.0.smithi029.stdout:1868: done (2 left) 2024-04-12T11:22:30.574 INFO:tasks.rados.rados.0.smithi029.stdout:1869: done (1 left) 2024-04-12T11:22:30.574 INFO:tasks.rados.rados.0.smithi029.stdout:1871: done (0 left) 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:1872: rollback oid 36 current snap is 246 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 36 to 238 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:1873: read oid 34 snap -1 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:1873: expect (ObjNum 693 snap 228 seq_num 693) 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:1874: rollback oid 46 current snap is 246 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 46 to 220 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:1875: rollback oid 8 current snap is 246 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 218 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:1876: read oid 14 snap -1 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:1876: expect deleted 2024-04-12T11:22:30.575 INFO:tasks.rados.rados.0.smithi029.stdout:1877: snap_remove snap 228 2024-04-12T11:22:30.768 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:30 smithi029 podman[115507]: 2024-04-12 11:22:30.574501506 +0000 UTC m=+1.620964321 container cleanup dbb7abb8911b0568db6d0a5d6d9a6a1c974bcec75381ba7173dd80fd0e32f7d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=reef-d540eba) 2024-04-12T11:22:30.768 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:30 smithi029 bash[115507]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2 2024-04-12T11:22:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:30 smithi029 ceph-mon[93650]: pgmap v453: 89 pgs: 89 active+clean; 156 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 407 KiB/s wr, 7 op/s 2024-04-12T11:22:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:30 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3452330855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:30 smithi029 ceph-mon[93650]: osdmap e546: 8 total, 7 up, 8 in 2024-04-12T11:22:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:30 smithi029 ceph-mon[97787]: pgmap v453: 89 pgs: 89 active+clean; 156 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 407 KiB/s wr, 7 op/s 2024-04-12T11:22:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:30 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3452330855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:30 smithi029 ceph-mon[97787]: osdmap e546: 8 total, 7 up, 8 in 2024-04-12T11:22:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:30 smithi087 ceph-mon[89014]: pgmap v453: 89 pgs: 89 active+clean; 156 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 407 KiB/s wr, 7 op/s 2024-04-12T11:22:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:30 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3452330855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:30 smithi087 ceph-mon[89014]: osdmap e546: 8 total, 7 up, 8 in 2024-04-12T11:22:31.193 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:30 smithi029 podman[115551]: 2024-04-12 11:22:30.879178031 +0000 UTC m=+0.777623070 container remove dbb7abb8911b0568db6d0a5d6d9a6a1c974bcec75381ba7173dd80fd0e32f7d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=reef-d540eba, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403) 2024-04-12T11:22:31.504 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:31 smithi029 podman[115646]: 2024-04-12 11:22:31.096471587 +0000 UTC m=+0.017590464 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:22:31.504 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:31 smithi029 podman[115646]: 2024-04-12 11:22:31.197644203 +0000 UTC m=+0.118763061 container create ac2108a7636b789be457951f4e221bb53afd91bb66ffdaadcccbc60dd6fb5e0a (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-12T11:22:31.504 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:31 smithi029 podman[115646]: 2024-04-12 11:22:31.428134669 +0000 UTC m=+0.349253527 container init ac2108a7636b789be457951f4e221bb53afd91bb66ffdaadcccbc60dd6fb5e0a (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-rf-comb1-2cee185, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-12T11:22:31.504 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:31 smithi029 podman[115646]: 2024-04-12 11:22:31.431412162 +0000 UTC m=+0.352531032 container start ac2108a7636b789be457951f4e221bb53afd91bb66ffdaadcccbc60dd6fb5e0a (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185) 2024-04-12T11:22:31.505 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:31 smithi029 podman[115646]: 2024-04-12 11:22:31.50413133 +0000 UTC m=+0.425250197 container attach ac2108a7636b789be457951f4e221bb53afd91bb66ffdaadcccbc60dd6fb5e0a (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, RELEASE=wip-rf-comb1-2cee185, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-12T11:22:31.572 INFO:tasks.rados.rados.0.smithi029.stdout:1877: done (5 left) 2024-04-12T11:22:31.572 INFO:tasks.rados.rados.0.smithi029.stdout:1878: write oid 6 current snap is 246 2024-04-12T11:22:31.572 INFO:tasks.rados.rados.0.smithi029.stdout:1878: seq_num 712 ranges {766584=798117,2098238=615574,3169759=460379} 2024-04-12T11:22:31.580 INFO:tasks.rados.rados.0.smithi029.stdout:1878: writing smithi02979277-6 from 766584 to 1564701 tid 1 2024-04-12T11:22:31.587 INFO:tasks.rados.rados.0.smithi029.stdout:1878: writing smithi02979277-6 from 2098238 to 2713812 tid 2 2024-04-12T11:22:31.593 INFO:tasks.rados.rados.0.smithi029.stdout:1878: writing smithi02979277-6 from 3169759 to 3630138 tid 3 2024-04-12T11:22:31.593 INFO:tasks.rados.rados.0.smithi029.stdout:1879: write oid 21 current snap is 246 2024-04-12T11:22:31.593 INFO:tasks.rados.rados.0.smithi029.stdout:1879: seq_num 713 ranges {783391=473366,1756720=475585,2797182=788004} 2024-04-12T11:22:31.600 INFO:tasks.rados.rados.0.smithi029.stdout:1879: writing smithi02979277-21 from 783391 to 1256757 tid 1 2024-04-12T11:22:31.605 INFO:tasks.rados.rados.0.smithi029.stdout:1879: writing smithi02979277-21 from 1756720 to 2232305 tid 2 2024-04-12T11:22:31.613 INFO:tasks.rados.rados.0.smithi029.stdout:1879: writing smithi02979277-21 from 2797182 to 3585186 tid 3 2024-04-12T11:22:31.613 INFO:tasks.rados.rados.0.smithi029.stdout:1880: read oid 27 snap -1 2024-04-12T11:22:31.613 INFO:tasks.rados.rados.0.smithi029.stdout:1880: expect deleted 2024-04-12T11:22:31.613 INFO:tasks.rados.rados.0.smithi029.stdout:1881: write oid 49 current snap is 246 2024-04-12T11:22:31.613 INFO:tasks.rados.rados.0.smithi029.stdout:1881: seq_num 714 ranges {400198=548615,1415202=584996} 2024-04-12T11:22:31.618 INFO:tasks.rados.rados.0.smithi029.stdout:1881: writing smithi02979277-49 from 400198 to 948813 tid 1 2024-04-12T11:22:31.624 INFO:tasks.rados.rados.0.smithi029.stdout:1881: writing smithi02979277-49 from 1415202 to 2000198 tid 2 2024-04-12T11:22:31.624 INFO:tasks.rados.rados.0.smithi029.stdout:1882: read oid 2 snap -1 2024-04-12T11:22:31.624 INFO:tasks.rados.rados.0.smithi029.stdout:1882: expect (ObjNum 711 snap 245 seq_num 711) 2024-04-12T11:22:31.624 INFO:tasks.rados.rados.0.smithi029.stdout:1883: snap_remove snap 224 2024-04-12T11:22:31.768 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:31 smithi029 podman[115646]: 2024-04-12 11:22:31.742414593 +0000 UTC m=+0.663533454 container died ac2108a7636b789be457951f4e221bb53afd91bb66ffdaadcccbc60dd6fb5e0a (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-12T11:22:32.362 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:32 smithi029 ceph-mon[97787]: pgmap v456: 89 pgs: 7 stale+active+clean, 82 active+clean; 156 MiB data, 606 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:22:32.385 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:32 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:32.385 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:32 smithi029 ceph-mon[97787]: osdmap e547: 8 total, 7 up, 8 in 2024-04-12T11:22:32.385 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:32 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1740086690' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:32.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:32 smithi029 ceph-mon[93650]: pgmap v456: 89 pgs: 7 stale+active+clean, 82 active+clean; 156 MiB data, 606 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:22:32.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:32 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:32.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:32 smithi029 ceph-mon[93650]: osdmap e547: 8 total, 7 up, 8 in 2024-04-12T11:22:32.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:32 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1740086690' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:32.574 INFO:tasks.rados.rados.0.smithi029.stdout:1883: done (10 left) 2024-04-12T11:22:32.574 INFO:tasks.rados.rados.0.smithi029.stdout:1884: snap_remove snap 243 2024-04-12T11:22:32.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:32 smithi087 ceph-mon[89014]: pgmap v456: 89 pgs: 7 stale+active+clean, 82 active+clean; 156 MiB data, 606 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:22:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:32 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:32 smithi087 ceph-mon[89014]: osdmap e547: 8 total, 7 up, 8 in 2024-04-12T11:22:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:32 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1740086690' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:22:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:22:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:22:32.768 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:32 smithi029 podman[115672]: 2024-04-12 11:22:32.362329724 +0000 UTC m=+0.610519661 container remove ac2108a7636b789be457951f4e221bb53afd91bb66ffdaadcccbc60dd6fb5e0a (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T11:22:32.768 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:32 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-12T11:22:32.768 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:32 smithi029 systemd[1]: Stopped Ceph osd.2 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:22:32.768 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:32 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.2.service: Consumed 13.200s CPU time. 2024-04-12T11:22:32.768 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:32 smithi029 systemd[1]: Starting Ceph osd.2 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:22:33.105 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:32 smithi029 podman[115799]: 2024-04-12 11:22:32.697171147 +0000 UTC m=+0.017686215 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:22:33.106 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:32 smithi029 podman[115799]: 2024-04-12 11:22:32.858461081 +0000 UTC m=+0.178976195 container create 5a8e1f63ca47373d5c24a42cb11be0023cdf097ca02fbfa8e36040ac3bb5bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185) 2024-04-12T11:22:33.106 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 podman[115799]: 2024-04-12 11:22:33.1045457 +0000 UTC m=+0.425060769 container init 5a8e1f63ca47373d5c24a42cb11be0023cdf097ca02fbfa8e36040ac3bb5bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate, io.buildah.version=1.33.5, RELEASE=wip-rf-comb1-2cee185, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-12T11:22:33.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 podman[115799]: 2024-04-12 11:22:33.10775451 +0000 UTC m=+0.428269579 container start 5a8e1f63ca47373d5c24a42cb11be0023cdf097ca02fbfa8e36040ac3bb5bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-12T11:22:33.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 podman[115799]: 2024-04-12 11:22:33.196657755 +0000 UTC m=+0.517172832 container attach 5a8e1f63ca47373d5c24a42cb11be0023cdf097ca02fbfa8e36040ac3bb5bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185) 2024-04-12T11:22:33.578 INFO:tasks.rados.rados.0.smithi029.stdout:1884: done (10 left) 2024-04-12T11:22:33.579 INFO:tasks.rados.rados.0.smithi029.stdout:1885: snap_remove snap 230 2024-04-12T11:22:33.856 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:33 smithi087 ceph-mon[89014]: osdmap e548: 8 total, 7 up, 8 in 2024-04-12T11:22:33.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:33 smithi029 ceph-mon[97787]: osdmap e548: 8 total, 7 up, 8 in 2024-04-12T11:22:33.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:33 smithi029 ceph-mon[93650]: osdmap e548: 8 total, 7 up, 8 in 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate[115813]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate[115813]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 bash[115799]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 bash[115799]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 bash[115799]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 bash[115799]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 bash[115799]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 bash[115799]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 bash[115799]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate[115813]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate[115813]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T11:22:33.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate[115813]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-12T11:22:33.948 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate[115813]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T11:22:33.948 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate[115813]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-12T11:22:33.948 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:33 smithi029 podman[115799]: 2024-04-12 11:22:33.583701753 +0000 UTC m=+0.904216826 container died 5a8e1f63ca47373d5c24a42cb11be0023cdf097ca02fbfa8e36040ac3bb5bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-12T11:22:34.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:34 smithi029 podman[115941]: 2024-04-12 11:22:34.256472265 +0000 UTC m=+0.668071866 container remove 5a8e1f63ca47373d5c24a42cb11be0023cdf097ca02fbfa8e36040ac3bb5bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-12T11:22:34.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:34 smithi029 podman[116029]: 2024-04-12 11:22:34.377667467 +0000 UTC m=+0.018513988 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:22:34.518 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:34 smithi029 podman[116029]: 2024-04-12 11:22:34.484948855 +0000 UTC m=+0.125795376 container create db6a92e3e974206b07f65766d558d949ca670fdcfc52de985ba9c8a6072b7ef7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-rf-comb1-2cee185) 2024-04-12T11:22:34.580 INFO:tasks.rados.rados.0.smithi029.stdout:1885: done (10 left) 2024-04-12T11:22:34.580 INFO:tasks.rados.rados.0.smithi029.stdout:1886: rollback oid 41 current snap is 246 2024-04-12T11:22:34.580 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 234 2024-04-12T11:22:34.580 INFO:tasks.rados.rados.0.smithi029.stdout:1875: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:22:34.580 INFO:tasks.rados.rados.0.smithi029.stdout:1875: finishing rollback tid 2 to smithi02979277-8 2024-04-12T11:22:34.580 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 178 (ObjNum 601 snap 192 seq_num 601) dirty exists 2024-04-12T11:22:34.580 INFO:tasks.rados.rados.0.smithi029.stdout:1874: finishing rollback tid 0 to smithi02979277-46 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:1872: finishing rollback tid 0 to smithi02979277-36 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:1874: finishing rollback tid 1 to smithi02979277-46 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:1874: finishing rollback tid 2 to smithi02979277-46 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 530 (ObjNum 487 snap 158 seq_num 487) dirty exists 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:1872: finishing rollback tid 1 to smithi02979277-36 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:1874: done (10 left) 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:1875: done (9 left) 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:1876: done (8 left) 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:1887: rollback oid 31 current snap is 246 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 31 to 239 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:1872: finishing rollback tid 2 to smithi02979277-36 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 407 (ObjNum 697 snap 231 seq_num 697) dirty exists 2024-04-12T11:22:34.581 INFO:tasks.rados.rados.0.smithi029.stdout:1873: expect (ObjNum 693 snap 228 seq_num 693) 2024-04-12T11:22:34.809 INFO:tasks.rados.rados.0.smithi029.stdout:1878: finishing write tid 1 to smithi02979277-6 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1878: finishing write tid 2 to smithi02979277-6 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1878: finishing write tid 3 to smithi02979277-6 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1878: finishing write tid 4 to smithi02979277-6 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1878: finishing write tid 5 to smithi02979277-6 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1878: oid 6 updating version 0 to 411 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1878: oid 6 updating version 411 to 412 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1878: oid 6 version 412 is already newer than 409 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1878: oid 6 version 412 is already newer than 410 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 412 (ObjNum 712 snap 246 seq_num 712) dirty exists 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1878: left oid 6 (ObjNum 712 snap 246 seq_num 712) 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1879: finishing write tid 1 to smithi02979277-21 2024-04-12T11:22:34.810 INFO:tasks.rados.rados.0.smithi029.stdout:1879: finishing write tid 2 to smithi02979277-21 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1879: finishing write tid 3 to smithi02979277-21 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1872: done (8 left) 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1873: done (7 left) 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1878: done (6 left) 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1880: done (5 left) 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1888: rollback oid 27 current snap is 246 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 220 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1879: finishing write tid 4 to smithi02979277-21 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1879: finishing write tid 5 to smithi02979277-21 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1879: oid 21 updating version 0 to 868 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1879: oid 21 updating version 868 to 870 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1879: oid 21 version 870 is already newer than 869 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:1879: oid 21 updating version 870 to 871 2024-04-12T11:22:34.811 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 871 (ObjNum 713 snap 246 seq_num 713) dirty exists 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:1879: left oid 21 (ObjNum 713 snap 246 seq_num 713) 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:1881: finishing write tid 1 to smithi02979277-49 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:1881: finishing write tid 2 to smithi02979277-49 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:1881: finishing write tid 3 to smithi02979277-49 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:1881: finishing write tid 4 to smithi02979277-49 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:1881: oid 49 updating version 0 to 684 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:1881: oid 49 updating version 684 to 685 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:1881: oid 49 updating version 685 to 686 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 686 (ObjNum 714 snap 246 seq_num 714) dirty exists 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:1881: left oid 49 (ObjNum 714 snap 246 seq_num 714) 2024-04-12T11:22:34.812 INFO:tasks.rados.rados.0.smithi029.stdout:1882: expect (ObjNum 711 snap 245 seq_num 711) 2024-04-12T11:22:34.876 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:34 smithi029 ceph-mon[97787]: pgmap v459: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 162 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 40/297 objects degraded (13.468%) 2024-04-12T11:22:34.876 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:34 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 40/297 objects degraded (13.468%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:34.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:34 smithi029 ceph-mon[97787]: osdmap e549: 8 total, 7 up, 8 in 2024-04-12T11:22:34.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:34 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/537282789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:34.877 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:34 smithi029 ceph-mon[93650]: pgmap v459: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 162 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 40/297 objects degraded (13.468%) 2024-04-12T11:22:34.877 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:34 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 40/297 objects degraded (13.468%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:34.877 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:34 smithi029 ceph-mon[93650]: osdmap e549: 8 total, 7 up, 8 in 2024-04-12T11:22:34.877 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:34 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/537282789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:34.877 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:34 smithi029 podman[116029]: 2024-04-12 11:22:34.723570739 +0000 UTC m=+0.364417262 container init db6a92e3e974206b07f65766d558d949ca670fdcfc52de985ba9c8a6072b7ef7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, ceph=True, RELEASE=wip-rf-comb1-2cee185, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-12T11:22:34.877 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:34 smithi029 podman[116029]: 2024-04-12 11:22:34.726538707 +0000 UTC m=+0.367385237 container start db6a92e3e974206b07f65766d558d949ca670fdcfc52de985ba9c8a6072b7ef7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240409) 2024-04-12T11:22:34.877 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:34 smithi029 ceph-osd[116051]: -- 172.21.15.29:0/2611744498 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55eb306c1680 con 0x55eb3068ac00 2024-04-12T11:22:34.878 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:34 smithi029 bash[116029]: db6a92e3e974206b07f65766d558d949ca670fdcfc52de985ba9c8a6072b7ef7 2024-04-12T11:22:34.878 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:34 smithi029 systemd[1]: Started Ceph osd.2 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:22:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:34 smithi087 ceph-mon[89014]: pgmap v459: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 162 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 40/297 objects degraded (13.468%) 2024-04-12T11:22:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:34 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 40/297 objects degraded (13.468%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:34 smithi087 ceph-mon[89014]: osdmap e549: 8 total, 7 up, 8 in 2024-04-12T11:22:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:34 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/537282789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:1886: finishing rollback tid 0 to smithi02979277-41 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:1886: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:1886: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 881 (ObjNum 634 snap 205 seq_num 634) dirty exists 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:1887: finishing rollback tid 1 to smithi02979277-31 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 761 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:1879: done (5 left) 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:1881: done (4 left) 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:1882: done (3 left) 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:1886: done (2 left) 2024-04-12T11:22:35.032 INFO:tasks.rados.rados.0.smithi029.stdout:1887: done (1 left) 2024-04-12T11:22:35.033 INFO:tasks.rados.rados.0.smithi029.stdout:1889: read oid 16 snap -1 2024-04-12T11:22:35.033 INFO:tasks.rados.rados.0.smithi029.stdout:1889: expect (ObjNum 702 snap 231 seq_num 702) 2024-04-12T11:22:35.033 INFO:tasks.rados.rados.0.smithi029.stdout:1890: read oid 33 snap -1 2024-04-12T11:22:35.033 INFO:tasks.rados.rados.0.smithi029.stdout:1890: expect (ObjNum 632 snap 205 seq_num 632) 2024-04-12T11:22:35.033 INFO:tasks.rados.rados.0.smithi029.stdout:1891: read oid 38 snap -1 2024-04-12T11:22:35.033 INFO:tasks.rados.rados.0.smithi029.stdout:1891: expect (ObjNum 695 snap 230 seq_num 695) 2024-04-12T11:22:35.033 INFO:tasks.rados.rados.0.smithi029.stdout:1892: snap_create 2024-04-12T11:22:35.033 INFO:tasks.rados.rados.0.smithi029.stdout:1888: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:22:35.033 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 0 (ObjNum 129 snap 0 seq_num 1801115184) dirty dne 2024-04-12T11:22:35.048 INFO:tasks.rados.rados.0.smithi029.stdout:1890: expect (ObjNum 632 snap 205 seq_num 632) 2024-04-12T11:22:35.209 INFO:tasks.rados.rados.0.smithi029.stdout:1889: expect (ObjNum 702 snap 231 seq_num 702) 2024-04-12T11:22:35.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:22:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:22:34] "GET /metrics HTTP/1.1" 200 36232 "" "Prometheus/2.43.0" 2024-04-12T11:22:35.517 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:35 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[116047]: 2024-04-12T11:22:35.266+0000 7fbe789f97c0 -1 Falling back to public interface 2024-04-12T11:22:35.531 INFO:tasks.rados.rados.0.smithi029.stdout:1891: expect (ObjNum 695 snap 230 seq_num 695) 2024-04-12T11:22:35.901 INFO:tasks.rados.rados.0.smithi029.stdout:1888: done (4 left) 2024-04-12T11:22:35.901 INFO:tasks.rados.rados.0.smithi029.stdout:1889: done (3 left) 2024-04-12T11:22:35.901 INFO:tasks.rados.rados.0.smithi029.stdout:1890: done (2 left) 2024-04-12T11:22:35.901 INFO:tasks.rados.rados.0.smithi029.stdout:1891: done (1 left) 2024-04-12T11:22:35.901 INFO:tasks.rados.rados.0.smithi029.stdout:1892: done (0 left) 2024-04-12T11:22:35.901 INFO:tasks.rados.rados.0.smithi029.stdout:1893: rollback oid 11 current snap is 247 2024-04-12T11:22:35.901 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 11 to 246 2024-04-12T11:22:35.901 INFO:tasks.rados.rados.0.smithi029.stdout:1894: snap_remove snap 236 2024-04-12T11:22:35.913 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:35 smithi029 ceph-mon[97787]: osdmap e550: 8 total, 7 up, 8 in 2024-04-12T11:22:35.913 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:35 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:35.913 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:35 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:35.913 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:35 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:35.914 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:35 smithi029 ceph-mon[93650]: osdmap e550: 8 total, 7 up, 8 in 2024-04-12T11:22:35.914 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:35 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:35.914 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:35 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:35.914 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:35 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:35 smithi087 ceph-mon[89014]: osdmap e550: 8 total, 7 up, 8 in 2024-04-12T11:22:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:35 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:35 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:35 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:36.896 INFO:tasks.rados.rados.0.smithi029.stdout:1894: done (1 left) 2024-04-12T11:22:36.896 INFO:tasks.rados.rados.0.smithi029.stdout:1895: write oid 41 current snap is 247 2024-04-12T11:22:36.896 INFO:tasks.rados.rados.0.smithi029.stdout:1893: finishing rollback tid 1 to smithi02979277-11 2024-04-12T11:22:36.896 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 0 (ObjNum 305 snap 0 seq_num 862798720) dirty dne 2024-04-12T11:22:36.896 INFO:tasks.rados.rados.0.smithi029.stdout:1895: seq_num 715 ranges {417005=740217,1590037=426968} 2024-04-12T11:22:36.902 INFO:tasks.rados.rados.0.smithi029.stdout:1895: writing smithi02979277-41 from 417005 to 1157222 tid 1 2024-04-12T11:22:36.908 INFO:tasks.rados.rados.0.smithi029.stdout:1895: writing smithi02979277-41 from 1590037 to 2017005 tid 2 2024-04-12T11:22:36.908 INFO:tasks.rados.rados.0.smithi029.stdout:1893: done (1 left) 2024-04-12T11:22:36.908 INFO:tasks.rados.rados.0.smithi029.stdout:1896: snap_remove snap 246 2024-04-12T11:22:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:36 smithi029 ceph-mon[93650]: pgmap v462: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 162 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 40/297 objects degraded (13.468%) 2024-04-12T11:22:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:36 smithi029 ceph-mon[93650]: osdmap e551: 8 total, 7 up, 8 in 2024-04-12T11:22:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:36 smithi029 ceph-mon[97787]: pgmap v462: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 162 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 40/297 objects degraded (13.468%) 2024-04-12T11:22:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:36 smithi029 ceph-mon[97787]: osdmap e551: 8 total, 7 up, 8 in 2024-04-12T11:22:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:36 smithi087 ceph-mon[89014]: pgmap v462: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 162 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 40/297 objects degraded (13.468%) 2024-04-12T11:22:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:36 smithi087 ceph-mon[89014]: osdmap e551: 8 total, 7 up, 8 in 2024-04-12T11:22:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:37.569 INFO:tasks.rados.rados.0.smithi029.stdout:1896: done (1 left) 2024-04-12T11:22:37.569 INFO:tasks.rados.rados.0.smithi029.stdout:1897: delete oid 40 current snap is 247 2024-04-12T11:22:37.570 INFO:tasks.rados.rados.0.smithi029.stdout:1895: finishing write tid 1 to smithi02979277-41 2024-04-12T11:22:37.570 INFO:tasks.rados.rados.0.smithi029.stdout:1895: finishing write tid 2 to smithi02979277-41 2024-04-12T11:22:37.570 INFO:tasks.rados.rados.0.smithi029.stdout:1895: finishing write tid 3 to smithi02979277-41 2024-04-12T11:22:37.570 INFO:tasks.rados.rados.0.smithi029.stdout:1895: finishing write tid 4 to smithi02979277-41 2024-04-12T11:22:37.570 INFO:tasks.rados.rados.0.smithi029.stdout:1895: oid 41 updating version 0 to 888 2024-04-12T11:22:37.570 INFO:tasks.rados.rados.0.smithi029.stdout:1895: oid 41 updating version 888 to 889 2024-04-12T11:22:37.570 INFO:tasks.rados.rados.0.smithi029.stdout:1895: oid 41 version 889 is already newer than 883 2024-04-12T11:22:37.570 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 889 (ObjNum 715 snap 247 seq_num 715) dirty exists 2024-04-12T11:22:37.570 INFO:tasks.rados.rados.0.smithi029.stdout:1895: left oid 41 (ObjNum 715 snap 247 seq_num 715) 2024-04-12T11:22:37.575 INFO:tasks.rados.rados.0.smithi029.stdout:1895: done (1 left) 2024-04-12T11:22:37.575 INFO:tasks.rados.rados.0.smithi029.stdout:1897: done (0 left) 2024-04-12T11:22:37.575 INFO:tasks.rados.rados.0.smithi029.stdout:1898: snap_create 2024-04-12T11:22:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:37 smithi087 ceph-mon[89014]: osdmap e552: 8 total, 7 up, 8 in 2024-04-12T11:22:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:37 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3517262198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:37 smithi087 ceph-mon[89014]: osdmap e553: 8 total, 7 up, 8 in 2024-04-12T11:22:38.268 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[116047]: 2024-04-12T11:22:38.098+0000 7fbe789f97c0 -1 osd.2 544 log_to_monitors true 2024-04-12T11:22:38.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:37 smithi029 ceph-mon[93650]: osdmap e552: 8 total, 7 up, 8 in 2024-04-12T11:22:38.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:37 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3517262198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:38.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:37 smithi029 ceph-mon[93650]: osdmap e553: 8 total, 7 up, 8 in 2024-04-12T11:22:38.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:37 smithi029 ceph-mon[97787]: osdmap e552: 8 total, 7 up, 8 in 2024-04-12T11:22:38.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:37 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3517262198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:38.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:37 smithi029 ceph-mon[97787]: osdmap e553: 8 total, 7 up, 8 in 2024-04-12T11:22:38.570 INFO:tasks.rados.rados.0.smithi029.stdout:1898: done (0 left) 2024-04-12T11:22:38.570 INFO:tasks.rados.rados.0.smithi029.stdout:1899: write oid 7 current snap is 248 2024-04-12T11:22:38.570 INFO:tasks.rados.rados.0.smithi029.stdout:1899: seq_num 717 ranges {450619=490715,1590648=459971} 2024-04-12T11:22:38.575 INFO:tasks.rados.rados.0.smithi029.stdout:1899: writing smithi02979277-7 from 450619 to 941334 tid 1 2024-04-12T11:22:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:1899: writing smithi02979277-7 from 1590648 to 2050619 tid 2 2024-04-12T11:22:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:1900: rollback oid 45 current snap is 248 2024-04-12T11:22:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 45 to 223 2024-04-12T11:22:38.582 INFO:tasks.rados.rados.0.smithi029.stdout:1899: finishing write tid 1 to smithi02979277-7 2024-04-12T11:22:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:1901: read oid 44 snap -1 2024-04-12T11:22:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:1901: expect (ObjNum 699 snap 231 seq_num 699) 2024-04-12T11:22:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:1902: write oid 9 current snap is 248 2024-04-12T11:22:38.583 INFO:tasks.rados.rados.0.smithi029.stdout:1902: seq_num 718 ranges {467426=565964,1649130=418296} 2024-04-12T11:22:38.588 INFO:tasks.rados.rados.0.smithi029.stdout:1902: writing smithi02979277-9 from 467426 to 1033390 tid 1 2024-04-12T11:22:38.594 INFO:tasks.rados.rados.0.smithi029.stdout:1902: writing smithi02979277-9 from 1649130 to 2067426 tid 2 2024-04-12T11:22:38.594 INFO:tasks.rados.rados.0.smithi029.stdout:1903: write oid 3 current snap is 248 2024-04-12T11:22:38.594 INFO:tasks.rados.rados.0.smithi029.stdout:1903: seq_num 719 ranges {484233=641213,1707612=376621} 2024-04-12T11:22:38.600 INFO:tasks.rados.rados.0.smithi029.stdout:1903: writing smithi02979277-3 from 484233 to 1125446 tid 1 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1903: writing smithi02979277-3 from 1707612 to 2084233 tid 2 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1904: read oid 15 snap -1 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1904: expect deleted 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1905: rollback oid 47 current snap is 248 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 242 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1899: finishing write tid 2 to smithi02979277-7 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1899: finishing write tid 3 to smithi02979277-7 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1900: finishing rollback tid 0 to smithi02979277-45 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1899: finishing write tid 4 to smithi02979277-7 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1899: oid 7 updating version 0 to 839 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1899: oid 7 version 839 is already newer than 837 2024-04-12T11:22:38.607 INFO:tasks.rados.rados.0.smithi029.stdout:1899: oid 7 version 839 is already newer than 838 2024-04-12T11:22:38.608 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 839 (ObjNum 717 snap 248 seq_num 717) dirty exists 2024-04-12T11:22:38.608 INFO:tasks.rados.rados.0.smithi029.stdout:1899: left oid 7 (ObjNum 717 snap 248 seq_num 717) 2024-04-12T11:22:38.608 INFO:tasks.rados.rados.0.smithi029.stdout:1900: finishing rollback tid 1 to smithi02979277-45 2024-04-12T11:22:38.608 INFO:tasks.rados.rados.0.smithi029.stdout:1901: expect (ObjNum 699 snap 231 seq_num 699) 2024-04-12T11:22:38.753 INFO:tasks.rados.rados.0.smithi029.stdout:1900: finishing rollback tid 2 to smithi02979277-45 2024-04-12T11:22:38.753 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 843 (ObjNum 537 snap 173 seq_num 537) dirty exists 2024-04-12T11:22:38.754 INFO:tasks.rados.rados.0.smithi029.stdout:1903: finishing write tid 1 to smithi02979277-3 2024-04-12T11:22:38.754 INFO:tasks.rados.rados.0.smithi029.stdout:1899: done (6 left) 2024-04-12T11:22:38.754 INFO:tasks.rados.rados.0.smithi029.stdout:1900: done (5 left) 2024-04-12T11:22:38.754 INFO:tasks.rados.rados.0.smithi029.stdout:1901: done (4 left) 2024-04-12T11:22:38.754 INFO:tasks.rados.rados.0.smithi029.stdout:1906: write oid 22 current snap is 248 2024-04-12T11:22:38.754 INFO:tasks.rados.rados.0.smithi029.stdout:1902: finishing write tid 1 to smithi02979277-9 2024-04-12T11:22:38.754 INFO:tasks.rados.rados.0.smithi029.stdout:1906: seq_num 720 ranges {501040=716462,1766094=334946} 2024-04-12T11:22:38.759 INFO:tasks.rados.rados.0.smithi029.stdout:1906: writing smithi02979277-22 from 501040 to 1217502 tid 1 2024-04-12T11:22:38.765 INFO:tasks.rados.rados.0.smithi029.stdout:1906: writing smithi02979277-22 from 1766094 to 2101040 tid 2 2024-04-12T11:22:38.765 INFO:tasks.rados.rados.0.smithi029.stdout:1907: write oid 2 current snap is 248 2024-04-12T11:22:38.765 INFO:tasks.rados.rados.0.smithi029.stdout:1907: seq_num 721 ranges {517847=791711,2108223=9624} 2024-04-12T11:22:38.772 INFO:tasks.rados.rados.0.smithi029.stdout:1907: writing smithi02979277-2 from 517847 to 1309558 tid 1 2024-04-12T11:22:38.777 INFO:tasks.rados.rados.0.smithi029.stdout:1907: writing smithi02979277-2 from 2108223 to 2117847 tid 2 2024-04-12T11:22:38.777 INFO:tasks.rados.rados.0.smithi029.stdout:1908: delete oid 17 current snap is 248 2024-04-12T11:22:38.777 INFO:tasks.rados.rados.0.smithi029.stdout:1903: finishing write tid 2 to smithi02979277-3 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1903: finishing write tid 3 to smithi02979277-3 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1902: finishing write tid 2 to smithi02979277-9 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1903: finishing write tid 4 to smithi02979277-3 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1903: oid 3 updating version 0 to 1151 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1903: oid 3 updating version 1151 to 1152 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1903: oid 3 updating version 1152 to 1153 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1153 (ObjNum 719 snap 248 seq_num 719) dirty exists 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1903: left oid 3 (ObjNum 719 snap 248 seq_num 719) 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1902: finishing write tid 3 to smithi02979277-9 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1902: finishing write tid 4 to smithi02979277-9 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1902: oid 9 updating version 0 to 692 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1902: oid 9 version 692 is already newer than 691 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:1902: oid 9 version 692 is already newer than 690 2024-04-12T11:22:38.778 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 692 (ObjNum 718 snap 248 seq_num 718) dirty exists 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1902: left oid 9 (ObjNum 718 snap 248 seq_num 718) 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1905: finishing rollback tid 0 to smithi02979277-47 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1905: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1905: finishing rollback tid 2 to smithi02979277-47 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 847 (ObjNum 658 snap 212 seq_num 658) dirty exists 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1906: finishing write tid 1 to smithi02979277-22 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1906: finishing write tid 2 to smithi02979277-22 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1906: finishing write tid 3 to smithi02979277-22 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1906: finishing write tid 4 to smithi02979277-22 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1906: oid 22 updating version 0 to 628 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1906: oid 22 version 628 is already newer than 626 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1906: oid 22 version 628 is already newer than 627 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 628 (ObjNum 720 snap 248 seq_num 720) dirty exists 2024-04-12T11:22:38.779 INFO:tasks.rados.rados.0.smithi029.stdout:1906: left oid 22 (ObjNum 720 snap 248 seq_num 720) 2024-04-12T11:22:38.780 INFO:tasks.rados.rados.0.smithi029.stdout:1907: finishing write tid 1 to smithi02979277-2 2024-04-12T11:22:38.780 INFO:tasks.rados.rados.0.smithi029.stdout:1907: finishing write tid 2 to smithi02979277-2 2024-04-12T11:22:38.780 INFO:tasks.rados.rados.0.smithi029.stdout:1902: done (6 left) 2024-04-12T11:22:38.780 INFO:tasks.rados.rados.0.smithi029.stdout:1903: done (5 left) 2024-04-12T11:22:38.781 INFO:tasks.rados.rados.0.smithi029.stdout:1904: done (4 left) 2024-04-12T11:22:38.781 INFO:tasks.rados.rados.0.smithi029.stdout:1905: done (3 left) 2024-04-12T11:22:38.781 INFO:tasks.rados.rados.0.smithi029.stdout:1906: done (2 left) 2024-04-12T11:22:38.781 INFO:tasks.rados.rados.0.smithi029.stdout:1908: done (1 left) 2024-04-12T11:22:38.781 INFO:tasks.rados.rados.0.smithi029.stdout:1909: rollback oid 6 current snap is 248 2024-04-12T11:22:38.781 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 6 to 232 2024-04-12T11:22:38.781 INFO:tasks.rados.rados.0.smithi029.stdout:1907: finishing write tid 3 to smithi02979277-2 2024-04-12T11:22:38.781 INFO:tasks.rados.rados.0.smithi029.stdout:1910: snap_remove snap 245 2024-04-12T11:22:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:38 smithi087 ceph-mon[89014]: pgmap v465: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 163 MiB data, 620 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 285 KiB/s wr, 8 op/s; 40/303 objects degraded (13.201%) 2024-04-12T11:22:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:38 smithi087 ceph-mon[89014]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T11:22:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:38 smithi087 ceph-mon[89014]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T11:22:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:38 smithi087 ceph-mon[89014]: osdmap e554: 8 total, 7 up, 8 in 2024-04-12T11:22:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:38 smithi087 ceph-mon[89014]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:22:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:38 smithi087 ceph-mon[89014]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' 2024-04-12T11:22:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:38 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:38 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:39.268 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[116047]: 2024-04-12T11:22:38.861+0000 7fbe65e49700 -1 osd.2 544 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:22:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[93650]: pgmap v465: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 163 MiB data, 620 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 285 KiB/s wr, 8 op/s; 40/303 objects degraded (13.201%) 2024-04-12T11:22:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[93650]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T11:22:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[93650]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[93650]: osdmap e554: 8 total, 7 up, 8 in 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[93650]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[93650]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[97787]: pgmap v465: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 163 MiB data, 620 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 285 KiB/s wr, 8 op/s; 40/303 objects degraded (13.201%) 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[97787]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[97787]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[97787]: osdmap e554: 8 total, 7 up, 8 in 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[97787]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[97787]: from='osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687]' entity='osd.2' 2024-04-12T11:22:39.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:39.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:38 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:39.576 INFO:tasks.rados.rados.0.smithi029.stdout:1910: done (2 left) 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:1911: delete oid 29 current snap is 248 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:1907: finishing write tid 4 to smithi02979277-2 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:1907: oid 2 updating version 0 to 193 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:1907: oid 2 version 193 is already newer than 191 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:1907: oid 2 version 193 is already newer than 192 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 193 (ObjNum 721 snap 248 seq_num 721) dirty exists 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:1907: left oid 2 (ObjNum 721 snap 248 seq_num 721) 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:1909: finishing rollback tid 0 to smithi02979277-6 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:1909: finishing rollback tid 1 to smithi02979277-6 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:1909: finishing rollback tid 2 to smithi02979277-6 2024-04-12T11:22:39.577 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 428 (ObjNum 424 snap 135 seq_num 424) dirty exists 2024-04-12T11:22:40.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:39 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3563819870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:40.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:39 smithi087 ceph-mon[89014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:22:40.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:39 smithi087 ceph-mon[89014]: osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687] boot 2024-04-12T11:22:40.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:39 smithi087 ceph-mon[89014]: osdmap e555: 8 total, 8 up, 8 in 2024-04-12T11:22:40.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:22:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:39 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3563819870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:39 smithi029 ceph-mon[97787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:22:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:39 smithi029 ceph-mon[97787]: osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687] boot 2024-04-12T11:22:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:39 smithi029 ceph-mon[97787]: osdmap e555: 8 total, 8 up, 8 in 2024-04-12T11:22:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:22:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:39 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3563819870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:39 smithi029 ceph-mon[93650]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:22:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:39 smithi029 ceph-mon[93650]: osd.2 [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687] boot 2024-04-12T11:22:40.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:39 smithi029 ceph-mon[93650]: osdmap e555: 8 total, 8 up, 8 in 2024-04-12T11:22:40.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T11:22:40.589 INFO:tasks.rados.rados.0.smithi029.stdout:1907: done (2 left) 2024-04-12T11:22:40.589 INFO:tasks.rados.rados.0.smithi029.stdout:1909: done (1 left) 2024-04-12T11:22:40.589 INFO:tasks.rados.rados.0.smithi029.stdout:1911: done (0 left) 2024-04-12T11:22:40.589 INFO:tasks.rados.rados.0.smithi029.stdout:1912: snap_remove snap 231 2024-04-12T11:22:40.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:40 smithi029 ceph-mon[93650]: pgmap v468: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 163 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 285 KiB/s wr, 8 op/s; 40/303 objects degraded (13.201%) 2024-04-12T11:22:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:40 smithi087 ceph-mon[89014]: pgmap v468: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 163 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 285 KiB/s wr, 8 op/s; 40/303 objects degraded (13.201%) 2024-04-12T11:22:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:40 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:40 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:41.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:40 smithi087 ceph-mon[89014]: osdmap e556: 8 total, 8 up, 8 in 2024-04-12T11:22:41.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:41.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:40 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:41.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:40 smithi029 ceph-mon[93650]: osdmap e556: 8 total, 8 up, 8 in 2024-04-12T11:22:41.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:40 smithi029 ceph-mon[97787]: pgmap v468: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 163 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 285 KiB/s wr, 8 op/s; 40/303 objects degraded (13.201%) 2024-04-12T11:22:41.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:40 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:41.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:40 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:41.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:40 smithi029 ceph-mon[97787]: osdmap e556: 8 total, 8 up, 8 in 2024-04-12T11:22:41.574 INFO:tasks.rados.rados.0.smithi029.stdout:1912: done (0 left) 2024-04-12T11:22:41.574 INFO:tasks.rados.rados.0.smithi029.stdout:1913: rollback oid 38 current snap is 248 2024-04-12T11:22:41.574 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 38 to 241 2024-04-12T11:22:41.574 INFO:tasks.rados.rados.0.smithi029.stdout:1914: read oid 36 snap -1 2024-04-12T11:22:41.574 INFO:tasks.rados.rados.0.smithi029.stdout:1914: expect (ObjNum 697 snap 231 seq_num 697) 2024-04-12T11:22:41.574 INFO:tasks.rados.rados.0.smithi029.stdout:1915: delete oid 20 current snap is 248 2024-04-12T11:22:41.582 INFO:tasks.rados.rados.0.smithi029.stdout:1913: finishing rollback tid 0 to smithi02979277-38 2024-04-12T11:22:41.583 INFO:tasks.rados.rados.0.smithi029.stdout:1913: finishing rollback tid 1 to smithi02979277-38 2024-04-12T11:22:41.583 INFO:tasks.rados.rados.0.smithi029.stdout:1913: finishing rollback tid 2 to smithi02979277-38 2024-04-12T11:22:41.584 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 538 (ObjNum 695 snap 230 seq_num 695) dirty exists 2024-04-12T11:22:41.584 INFO:tasks.rados.rados.0.smithi029.stdout:1913: done (2 left) 2024-04-12T11:22:41.585 INFO:tasks.rados.rados.0.smithi029.stdout:1915: done (1 left) 2024-04-12T11:22:41.585 INFO:tasks.rados.rados.0.smithi029.stdout:1916: read oid 5 snap -1 2024-04-12T11:22:41.585 INFO:tasks.rados.rados.0.smithi029.stdout:1916: expect (ObjNum 708 snap 240 seq_num 708) 2024-04-12T11:22:41.585 INFO:tasks.rados.rados.0.smithi029.stdout:1917: snap_remove snap 232 2024-04-12T11:22:42.572 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:22:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:22:42.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:22:42.579 INFO:tasks.rados.rados.0.smithi029.stdout:1917: done (2 left) 2024-04-12T11:22:42.579 INFO:tasks.rados.rados.0.smithi029.stdout:1918: delete oid 9 current snap is 248 2024-04-12T11:22:42.579 INFO:tasks.rados.rados.0.smithi029.stdout:1914: expect (ObjNum 697 snap 231 seq_num 697) 2024-04-12T11:22:42.718 INFO:tasks.rados.rados.0.smithi029.stdout:1916: expect (ObjNum 708 snap 240 seq_num 708) 2024-04-12T11:22:42.799 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:42 smithi029 ceph-mon[97787]: pgmap v471: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 163 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 40/303 objects degraded (13.201%) 2024-04-12T11:22:42.799 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:42 smithi029 ceph-mon[97787]: osdmap e557: 8 total, 8 up, 8 in 2024-04-12T11:22:42.799 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:42 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3179595865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:42.799 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:42 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 40/303 objects degraded (13.201%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:42.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:42 smithi029 ceph-mon[93650]: pgmap v471: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 163 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 40/303 objects degraded (13.201%) 2024-04-12T11:22:42.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:42 smithi029 ceph-mon[93650]: osdmap e557: 8 total, 8 up, 8 in 2024-04-12T11:22:42.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:42 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3179595865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:42.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:42 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 40/303 objects degraded (13.201%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:42.852 INFO:tasks.rados.rados.0.smithi029.stdout:1914: done (2 left) 2024-04-12T11:22:42.853 INFO:tasks.rados.rados.0.smithi029.stdout:1916: done (1 left) 2024-04-12T11:22:42.853 INFO:tasks.rados.rados.0.smithi029.stdout:1918: done (0 left) 2024-04-12T11:22:42.853 INFO:tasks.rados.rados.0.smithi029.stdout:1919: rollback oid 47 current snap is 248 2024-04-12T11:22:42.853 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 235 2024-04-12T11:22:42.853 INFO:tasks.rados.rados.0.smithi029.stdout:1920: write oid 48 current snap is 248 2024-04-12T11:22:42.853 INFO:tasks.rados.rados.0.smithi029.stdout:1920: seq_num 726 ranges {601882=484309,1600633=524904} 2024-04-12T11:22:42.858 INFO:tasks.rados.rados.0.smithi029.stdout:1920: writing smithi02979277-48 from 601882 to 1086191 tid 1 2024-04-12T11:22:42.864 INFO:tasks.rados.rados.0.smithi029.stdout:1920: writing smithi02979277-48 from 1600633 to 2125537 tid 2 2024-04-12T11:22:42.864 INFO:tasks.rados.rados.0.smithi029.stdout:1921: delete oid 35 current snap is 248 2024-04-12T11:22:42.865 INFO:tasks.rados.rados.0.smithi029.stdout:1919: finishing rollback tid 0 to smithi02979277-47 2024-04-12T11:22:42.865 INFO:tasks.rados.rados.0.smithi029.stdout:1919: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:22:42.865 INFO:tasks.rados.rados.0.smithi029.stdout:1919: finishing rollback tid 2 to smithi02979277-47 2024-04-12T11:22:42.865 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 868 (ObjNum 658 snap 212 seq_num 658) dirty exists 2024-04-12T11:22:42.865 INFO:tasks.rados.rados.0.smithi029.stdout:1920: finishing write tid 1 to smithi02979277-48 2024-04-12T11:22:42.868 INFO:tasks.rados.rados.0.smithi029.stdout:1920: finishing write tid 2 to smithi02979277-48 2024-04-12T11:22:42.868 INFO:tasks.rados.rados.0.smithi029.stdout:1919: done (2 left) 2024-04-12T11:22:42.868 INFO:tasks.rados.rados.0.smithi029.stdout:1921: done (1 left) 2024-04-12T11:22:42.868 INFO:tasks.rados.rados.0.smithi029.stdout:1922: delete oid 1 current snap is 248 2024-04-12T11:22:42.868 INFO:tasks.rados.rados.0.smithi029.stdout:1920: finishing write tid 3 to smithi02979277-48 2024-04-12T11:22:42.868 INFO:tasks.rados.rados.0.smithi029.stdout:1920: finishing write tid 4 to smithi02979277-48 2024-04-12T11:22:42.868 INFO:tasks.rados.rados.0.smithi029.stdout:1920: oid 48 updating version 0 to 779 2024-04-12T11:22:42.868 INFO:tasks.rados.rados.0.smithi029.stdout:1920: oid 48 updating version 779 to 780 2024-04-12T11:22:42.868 INFO:tasks.rados.rados.0.smithi029.stdout:1920: oid 48 version 780 is already newer than 778 2024-04-12T11:22:42.868 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 780 (ObjNum 726 snap 248 seq_num 726) dirty exists 2024-04-12T11:22:42.869 INFO:tasks.rados.rados.0.smithi029.stdout:1920: left oid 48 (ObjNum 726 snap 248 seq_num 726) 2024-04-12T11:22:42.870 INFO:tasks.rados.rados.0.smithi029.stdout:1920: done (1 left) 2024-04-12T11:22:42.870 INFO:tasks.rados.rados.0.smithi029.stdout:1922: done (0 left) 2024-04-12T11:22:42.870 INFO:tasks.rados.rados.0.smithi029.stdout:1923: snap_create 2024-04-12T11:22:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:42 smithi087 ceph-mon[89014]: pgmap v471: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 163 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 40/303 objects degraded (13.201%) 2024-04-12T11:22:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:42 smithi087 ceph-mon[89014]: osdmap e557: 8 total, 8 up, 8 in 2024-04-12T11:22:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:42 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3179595865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:42 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 40/303 objects degraded (13.201%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:22:43.595 INFO:tasks.rados.rados.0.smithi029.stdout:1923: done (0 left) 2024-04-12T11:22:43.595 INFO:tasks.rados.rados.0.smithi029.stdout:1924: read oid 34 snap -1 2024-04-12T11:22:43.595 INFO:tasks.rados.rados.0.smithi029.stdout:1924: expect (ObjNum 693 snap 228 seq_num 693) 2024-04-12T11:22:43.595 INFO:tasks.rados.rados.0.smithi029.stdout:1925: read oid 33 snap -1 2024-04-12T11:22:43.595 INFO:tasks.rados.rados.0.smithi029.stdout:1925: expect (ObjNum 632 snap 205 seq_num 632) 2024-04-12T11:22:43.596 INFO:tasks.rados.rados.0.smithi029.stdout:1926: rollback oid 12 current snap is 249 2024-04-12T11:22:43.596 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 223 2024-04-12T11:22:43.596 INFO:tasks.rados.rados.0.smithi029.stdout:1927: write oid 26 current snap is 249 2024-04-12T11:22:43.596 INFO:tasks.rados.rados.0.smithi029.stdout:1927: seq_num 729 ranges {652303=710056,2059726=192577} 2024-04-12T11:22:43.603 INFO:tasks.rados.rados.0.smithi029.stdout:1927: writing smithi02979277-26 from 652303 to 1362359 tid 1 2024-04-12T11:22:43.609 INFO:tasks.rados.rados.0.smithi029.stdout:1927: writing smithi02979277-26 from 2059726 to 2252303 tid 2 2024-04-12T11:22:43.609 INFO:tasks.rados.rados.0.smithi029.stdout:1928: read oid 48 snap -1 2024-04-12T11:22:43.610 INFO:tasks.rados.rados.0.smithi029.stdout:1928: expect (ObjNum 726 snap 248 seq_num 726) 2024-04-12T11:22:43.610 INFO:tasks.rados.rados.0.smithi029.stdout:1929: read oid 9 snap 240 2024-04-12T11:22:43.610 INFO:tasks.rados.rados.0.smithi029.stdout:1929: expect (ObjNum 698 snap 231 seq_num 698) 2024-04-12T11:22:43.610 INFO:tasks.rados.rados.0.smithi029.stdout:1930: read oid 20 snap 239 2024-04-12T11:22:43.610 INFO:tasks.rados.rados.0.smithi029.stdout:1930: expect deleted 2024-04-12T11:22:43.610 INFO:tasks.rados.rados.0.smithi029.stdout:1931: snap_remove snap 229 2024-04-12T11:22:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:43 smithi029 ceph-mon[97787]: osdmap e558: 8 total, 8 up, 8 in 2024-04-12T11:22:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:43 smithi029 ceph-mon[93650]: osdmap e558: 8 total, 8 up, 8 in 2024-04-12T11:22:43.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:43 smithi087 ceph-mon[89014]: osdmap e558: 8 total, 8 up, 8 in 2024-04-12T11:22:44.518 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:22:44.081+0000 7f4c1f740700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T11:22:44.591 INFO:tasks.rados.rados.0.smithi029.stdout:1931: done (7 left) 2024-04-12T11:22:44.591 INFO:tasks.rados.rados.0.smithi029.stdout:1932: delete oid 44 current snap is 249 2024-04-12T11:22:44.591 INFO:tasks.rados.rados.0.smithi029.stdout:1926: finishing rollback tid 0 to smithi02979277-12 2024-04-12T11:22:44.591 INFO:tasks.rados.rados.0.smithi029.stdout:1926: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:22:44.591 INFO:tasks.rados.rados.0.smithi029.stdout:1926: finishing rollback tid 2 to smithi02979277-12 2024-04-12T11:22:44.591 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 872 (ObjNum 670 snap 215 seq_num 670) dirty exists 2024-04-12T11:22:44.591 INFO:tasks.rados.rados.0.smithi029.stdout:1924: expect (ObjNum 693 snap 228 seq_num 693) 2024-04-12T11:22:44.815 INFO:tasks.rados.rados.0.smithi029.stdout:1925: expect (ObjNum 632 snap 205 seq_num 632) 2024-04-12T11:22:44.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: pgmap v474: 89 pgs: 5 active+undersized+degraded, 2 active+undersized, 82 active+clean; 166 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 730 KiB/s wr, 11 op/s; 27/318 objects degraded (8.491%) 2024-04-12T11:22:44.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: osdmap e559: 8 total, 8 up, 8 in 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/511821266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:22:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: pgmap v474: 89 pgs: 5 active+undersized+degraded, 2 active+undersized, 82 active+clean; 166 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 730 KiB/s wr, 11 op/s; 27/318 objects degraded (8.491%) 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: osdmap e559: 8 total, 8 up, 8 in 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/511821266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:22:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: pgmap v474: 89 pgs: 5 active+undersized+degraded, 2 active+undersized, 82 active+clean; 166 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 730 KiB/s wr, 11 op/s; 27/318 objects degraded (8.491%) 2024-04-12T11:22:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: osdmap e559: 8 total, 8 up, 8 in 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/511821266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:22:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:22:44.974 INFO:tasks.rados.rados.0.smithi029.stdout:1927: finishing write tid 1 to smithi02979277-26 2024-04-12T11:22:44.975 INFO:tasks.rados.rados.0.smithi029.stdout:1927: finishing write tid 2 to smithi02979277-26 2024-04-12T11:22:44.975 INFO:tasks.rados.rados.0.smithi029.stdout:1928: expect (ObjNum 726 snap 248 seq_num 726) 2024-04-12T11:22:45.080 INFO:tasks.rados.rados.0.smithi029.stdout:1927: finishing write tid 3 to smithi02979277-26 2024-04-12T11:22:45.080 INFO:tasks.rados.rados.0.smithi029.stdout:1927: finishing write tid 4 to smithi02979277-26 2024-04-12T11:22:45.080 INFO:tasks.rados.rados.0.smithi029.stdout:1927: oid 26 updating version 0 to 1165 2024-04-12T11:22:45.080 INFO:tasks.rados.rados.0.smithi029.stdout:1927: oid 26 version 1165 is already newer than 1164 2024-04-12T11:22:45.080 INFO:tasks.rados.rados.0.smithi029.stdout:1927: oid 26 version 1165 is already newer than 1163 2024-04-12T11:22:45.080 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 1165 (ObjNum 729 snap 249 seq_num 729) dirty exists 2024-04-12T11:22:45.080 INFO:tasks.rados.rados.0.smithi029.stdout:1927: left oid 26 (ObjNum 729 snap 249 seq_num 729) 2024-04-12T11:22:45.080 INFO:tasks.rados.rados.0.smithi029.stdout:1924: done (7 left) 2024-04-12T11:22:45.080 INFO:tasks.rados.rados.0.smithi029.stdout:1925: done (6 left) 2024-04-12T11:22:45.080 INFO:tasks.rados.rados.0.smithi029.stdout:1926: done (5 left) 2024-04-12T11:22:45.081 INFO:tasks.rados.rados.0.smithi029.stdout:1927: done (4 left) 2024-04-12T11:22:45.081 INFO:tasks.rados.rados.0.smithi029.stdout:1928: done (3 left) 2024-04-12T11:22:45.081 INFO:tasks.rados.rados.0.smithi029.stdout:1930: done (2 left) 2024-04-12T11:22:45.081 INFO:tasks.rados.rados.0.smithi029.stdout:1932: done (1 left) 2024-04-12T11:22:45.081 INFO:tasks.rados.rados.0.smithi029.stdout:1933: write oid 7 current snap is 249 2024-04-12T11:22:45.081 INFO:tasks.rados.rados.0.smithi029.stdout:1929: expect (ObjNum 698 snap 231 seq_num 698) 2024-04-12T11:22:45.198 INFO:tasks.rados.rados.0.smithi029.stdout:1933: seq_num 731 ranges {685917=576907,1893043=392874} 2024-04-12T11:22:45.204 INFO:tasks.rados.rados.0.smithi029.stdout:1933: writing smithi02979277-7 from 685917 to 1262824 tid 1 2024-04-12T11:22:45.210 INFO:tasks.rados.rados.0.smithi029.stdout:1933: writing smithi02979277-7 from 1893043 to 2285917 tid 2 2024-04-12T11:22:45.210 INFO:tasks.rados.rados.0.smithi029.stdout:1929: done (1 left) 2024-04-12T11:22:45.211 INFO:tasks.rados.rados.0.smithi029.stdout:1934: read oid 39 snap -1 2024-04-12T11:22:45.211 INFO:tasks.rados.rados.0.smithi029.stdout:1934: expect (ObjNum 709 snap 243 seq_num 709) 2024-04-12T11:22:45.211 INFO:tasks.rados.rados.0.smithi029.stdout:1935: snap_create 2024-04-12T11:22:45.211 INFO:tasks.rados.rados.0.smithi029.stdout:1933: finishing write tid 1 to smithi02979277-7 2024-04-12T11:22:45.213 INFO:tasks.rados.rados.0.smithi029.stdout:1933: finishing write tid 2 to smithi02979277-7 2024-04-12T11:22:45.214 INFO:tasks.rados.rados.0.smithi029.stdout:1933: finishing write tid 3 to smithi02979277-7 2024-04-12T11:22:45.214 INFO:tasks.rados.rados.0.smithi029.stdout:1933: finishing write tid 4 to smithi02979277-7 2024-04-12T11:22:45.214 INFO:tasks.rados.rados.0.smithi029.stdout:1933: oid 7 updating version 0 to 882 2024-04-12T11:22:45.214 INFO:tasks.rados.rados.0.smithi029.stdout:1933: oid 7 updating version 882 to 883 2024-04-12T11:22:45.214 INFO:tasks.rados.rados.0.smithi029.stdout:1933: oid 7 updating version 883 to 884 2024-04-12T11:22:45.214 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 884 (ObjNum 731 snap 249 seq_num 731) dirty exists 2024-04-12T11:22:45.214 INFO:tasks.rados.rados.0.smithi029.stdout:1933: left oid 7 (ObjNum 731 snap 249 seq_num 731) 2024-04-12T11:22:45.216 INFO:tasks.rados.rados.0.smithi029.stdout:1934: expect (ObjNum 709 snap 243 seq_num 709) 2024-04-12T11:22:45.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:22:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:22:44] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.43.0" 2024-04-12T11:22:45.594 INFO:tasks.rados.rados.0.smithi029.stdout:1933: done (2 left) 2024-04-12T11:22:45.595 INFO:tasks.rados.rados.0.smithi029.stdout:1934: done (1 left) 2024-04-12T11:22:45.595 INFO:tasks.rados.rados.0.smithi029.stdout:1935: done (0 left) 2024-04-12T11:22:45.595 INFO:tasks.rados.rados.0.smithi029.stdout:1936: read oid 49 snap -1 2024-04-12T11:22:45.595 INFO:tasks.rados.rados.0.smithi029.stdout:1936: expect (ObjNum 714 snap 246 seq_num 714) 2024-04-12T11:22:45.595 INFO:tasks.rados.rados.0.smithi029.stdout:1937: snap_create 2024-04-12T11:22:45.599 INFO:tasks.rados.rados.0.smithi029.stdout:1936: expect (ObjNum 714 snap 246 seq_num 714) 2024-04-12T11:22:45.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:45 smithi087 ceph-mon[89014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:22:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:45 smithi087 ceph-mon[89014]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T11:22:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:45 smithi087 ceph-mon[89014]: osdmap e560: 8 total, 8 up, 8 in 2024-04-12T11:22:46.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:45 smithi029 ceph-mon[93650]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:22:46.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:45 smithi029 ceph-mon[93650]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T11:22:46.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:45 smithi029 ceph-mon[93650]: osdmap e560: 8 total, 8 up, 8 in 2024-04-12T11:22:46.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:45 smithi029 ceph-mon[97787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:22:46.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:45 smithi029 ceph-mon[97787]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T11:22:46.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:45 smithi029 ceph-mon[97787]: osdmap e560: 8 total, 8 up, 8 in 2024-04-12T11:22:46.595 INFO:tasks.rados.rados.0.smithi029.stdout:1936: done (1 left) 2024-04-12T11:22:46.595 INFO:tasks.rados.rados.0.smithi029.stdout:1937: done (0 left) 2024-04-12T11:22:46.595 INFO:tasks.rados.rados.0.smithi029.stdout:1938: rollback oid 34 current snap is 251 2024-04-12T11:22:46.595 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 223 2024-04-12T11:22:46.595 INFO:tasks.rados.rados.0.smithi029.stdout:1939: snap_create 2024-04-12T11:22:46.600 INFO:tasks.rados.rados.0.smithi029.stdout:1938: finishing rollback tid 0 to smithi02979277-34 2024-04-12T11:22:46.600 INFO:tasks.rados.rados.0.smithi029.stdout:1938: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:22:46.600 INFO:tasks.rados.rados.0.smithi029.stdout:1938: finishing rollback tid 2 to smithi02979277-34 2024-04-12T11:22:46.600 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 706 (ObjNum 561 snap 177 seq_num 561) dirty exists 2024-04-12T11:22:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:46 smithi087 ceph-mon[89014]: pgmap v477: 89 pgs: 89 active+clean; 168 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.1 MiB/s wr, 17 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:46 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/318 objects degraded (8.491%), 5 pgs degraded) 2024-04-12T11:22:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:46 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:22:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:46 smithi087 ceph-mon[89014]: osdmap e561: 8 total, 8 up, 8 in 2024-04-12T11:22:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:46 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1231187715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:46 smithi029 ceph-mon[93650]: pgmap v477: 89 pgs: 89 active+clean; 168 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.1 MiB/s wr, 17 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:46 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/318 objects degraded (8.491%), 5 pgs degraded) 2024-04-12T11:22:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:46 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:22:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:46 smithi029 ceph-mon[93650]: osdmap e561: 8 total, 8 up, 8 in 2024-04-12T11:22:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:46 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1231187715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:46 smithi029 ceph-mon[97787]: pgmap v477: 89 pgs: 89 active+clean; 168 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.1 MiB/s wr, 17 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:46 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/318 objects degraded (8.491%), 5 pgs degraded) 2024-04-12T11:22:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:46 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:22:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:46 smithi029 ceph-mon[97787]: osdmap e561: 8 total, 8 up, 8 in 2024-04-12T11:22:47.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:46 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1231187715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:47.571 INFO:tasks.rados.rados.0.smithi029.stdout:1938: done (1 left) 2024-04-12T11:22:47.571 INFO:tasks.rados.rados.0.smithi029.stdout:1939: done (0 left) 2024-04-12T11:22:47.572 INFO:tasks.rados.rados.0.smithi029.stdout:1940: read oid 37 snap -1 2024-04-12T11:22:47.572 INFO:tasks.rados.rados.0.smithi029.stdout:1940: expect (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:22:47.572 INFO:tasks.rados.rados.0.smithi029.stdout:1941: write oid 18 current snap is 252 2024-04-12T11:22:47.572 INFO:tasks.rados.rados.0.smithi029.stdout:1941: seq_num 732 ranges {702724=652156,1951525=351199} 2024-04-12T11:22:47.579 INFO:tasks.rados.rados.0.smithi029.stdout:1941: writing smithi02979277-18 from 702724 to 1354880 tid 1 2024-04-12T11:22:47.584 INFO:tasks.rados.rados.0.smithi029.stdout:1941: writing smithi02979277-18 from 1951525 to 2302724 tid 2 2024-04-12T11:22:47.585 INFO:tasks.rados.rados.0.smithi029.stdout:1942: delete oid 29 current snap is 252 2024-04-12T11:22:47.585 INFO:tasks.rados.rados.0.smithi029.stdout:1940: expect (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1941: finishing write tid 1 to smithi02979277-18 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1941: finishing write tid 2 to smithi02979277-18 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1941: finishing write tid 3 to smithi02979277-18 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1941: finishing write tid 4 to smithi02979277-18 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1941: oid 18 updating version 0 to 931 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1941: oid 18 version 931 is already newer than 930 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1941: oid 18 updating version 931 to 932 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 932 (ObjNum 732 snap 252 seq_num 732) dirty exists 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1941: left oid 18 (ObjNum 732 snap 252 seq_num 732) 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1940: done (2 left) 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1941: done (1 left) 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1942: done (0 left) 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1943: read oid 12 snap -1 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1943: expect (ObjNum 670 snap 215 seq_num 670) 2024-04-12T11:22:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:1944: read oid 17 snap 240 2024-04-12T11:22:47.800 INFO:tasks.rados.rados.0.smithi029.stdout:1944: expect deleted 2024-04-12T11:22:47.800 INFO:tasks.rados.rados.0.smithi029.stdout:1945: read oid 23 snap -1 2024-04-12T11:22:47.800 INFO:tasks.rados.rados.0.smithi029.stdout:1945: expect (ObjNum 657 snap 212 seq_num 657) 2024-04-12T11:22:47.800 INFO:tasks.rados.rados.0.smithi029.stdout:1946: snap_create 2024-04-12T11:22:47.802 INFO:tasks.rados.rados.0.smithi029.stdout:1943: expect (ObjNum 670 snap 215 seq_num 670) 2024-04-12T11:22:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:47 smithi087 ceph-mon[89014]: osdmap e562: 8 total, 8 up, 8 in 2024-04-12T11:22:47.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:47 smithi087 ceph-mon[89014]: osdmap e563: 8 total, 8 up, 8 in 2024-04-12T11:22:47.967 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:22:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=sqlstore.transactions t=2024-04-12T11:22:47.545480659Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-12T11:22:47.967 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:22:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=cleanup t=2024-04-12T11:22:47.546842701Z level=info msg="Completed cleanup jobs" duration=12.629009ms 2024-04-12T11:22:48.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:47 smithi029 ceph-mon[93650]: osdmap e562: 8 total, 8 up, 8 in 2024-04-12T11:22:48.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:47 smithi029 ceph-mon[93650]: osdmap e563: 8 total, 8 up, 8 in 2024-04-12T11:22:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:47 smithi029 ceph-mon[97787]: osdmap e562: 8 total, 8 up, 8 in 2024-04-12T11:22:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:47 smithi029 ceph-mon[97787]: osdmap e563: 8 total, 8 up, 8 in 2024-04-12T11:22:48.028 INFO:tasks.rados.rados.0.smithi029.stdout:1945: expect (ObjNum 657 snap 212 seq_num 657) 2024-04-12T11:22:48.608 INFO:tasks.rados.rados.0.smithi029.stdout:1943: done (3 left) 2024-04-12T11:22:48.608 INFO:tasks.rados.rados.0.smithi029.stdout:1944: done (2 left) 2024-04-12T11:22:48.608 INFO:tasks.rados.rados.0.smithi029.stdout:1945: done (1 left) 2024-04-12T11:22:48.608 INFO:tasks.rados.rados.0.smithi029.stdout:1946: done (0 left) 2024-04-12T11:22:48.609 INFO:tasks.rados.rados.0.smithi029.stdout:1947: snap_remove snap 233 2024-04-12T11:22:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:48 smithi087 ceph-mon[89014]: pgmap v480: 89 pgs: 89 active+clean; 167 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 691 KiB/s wr, 11 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:48 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1015422674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:49.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:48 smithi029 ceph-mon[93650]: pgmap v480: 89 pgs: 89 active+clean; 167 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 691 KiB/s wr, 11 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:48 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1015422674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:48 smithi029 ceph-mon[97787]: pgmap v480: 89 pgs: 89 active+clean; 167 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 691 KiB/s wr, 11 op/s; 2.1 MiB/s, 1 objects/s recovering 2024-04-12T11:22:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:48 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1015422674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:49.616 INFO:tasks.rados.rados.0.smithi029.stdout:1947: done (0 left) 2024-04-12T11:22:49.616 INFO:tasks.rados.rados.0.smithi029.stdout:1948: read oid 43 snap 248 2024-04-12T11:22:49.616 INFO:tasks.rados.rados.0.smithi029.stdout:1948: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:22:49.616 INFO:tasks.rados.rados.0.smithi029.stdout:1949: delete oid 11 current snap is 253 2024-04-12T11:22:49.623 INFO:tasks.rados.rados.0.smithi029.stdout:1949: done (1 left) 2024-04-12T11:22:49.623 INFO:tasks.rados.rados.0.smithi029.stdout:1950: read oid 21 snap 238 2024-04-12T11:22:49.623 INFO:tasks.rados.rados.0.smithi029.stdout:1950: expect (ObjNum 705 snap 236 seq_num 705) 2024-04-12T11:22:49.623 INFO:tasks.rados.rados.0.smithi029.stdout:1951: snap_remove snap 250 2024-04-12T11:22:49.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:49 smithi087 ceph-mon[89014]: osdmap e564: 8 total, 8 up, 8 in 2024-04-12T11:22:50.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:49 smithi029 ceph-mon[97787]: osdmap e564: 8 total, 8 up, 8 in 2024-04-12T11:22:50.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:49 smithi029 ceph-mon[93650]: osdmap e564: 8 total, 8 up, 8 in 2024-04-12T11:22:50.621 INFO:tasks.rados.rados.0.smithi029.stdout:1951: done (2 left) 2024-04-12T11:22:50.621 INFO:tasks.rados.rados.0.smithi029.stdout:1952: snap_remove snap 241 2024-04-12T11:22:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:50 smithi087 ceph-mon[89014]: pgmap v483: 89 pgs: 89 active+clean; 170 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 704 KiB/s wr, 17 op/s 2024-04-12T11:22:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:50 smithi087 ceph-mon[89014]: osdmap e565: 8 total, 8 up, 8 in 2024-04-12T11:22:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:50 smithi029 ceph-mon[93650]: pgmap v483: 89 pgs: 89 active+clean; 170 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 704 KiB/s wr, 17 op/s 2024-04-12T11:22:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:50 smithi029 ceph-mon[93650]: osdmap e565: 8 total, 8 up, 8 in 2024-04-12T11:22:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:50 smithi029 ceph-mon[97787]: pgmap v483: 89 pgs: 89 active+clean; 170 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 704 KiB/s wr, 17 op/s 2024-04-12T11:22:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:50 smithi029 ceph-mon[97787]: osdmap e565: 8 total, 8 up, 8 in 2024-04-12T11:22:51.629 INFO:tasks.rados.rados.0.smithi029.stdout:1952: done (2 left) 2024-04-12T11:22:51.629 INFO:tasks.rados.rados.0.smithi029.stdout:1953: write oid 24 current snap is 253 2024-04-12T11:22:51.629 INFO:tasks.rados.rados.0.smithi029.stdout:1953: seq_num 735 ranges {753145=477903,2010618=342527} 2024-04-12T11:22:51.635 INFO:tasks.rados.rados.0.smithi029.stdout:1953: writing smithi02979277-24 from 753145 to 1231048 tid 1 2024-04-12T11:22:51.642 INFO:tasks.rados.rados.0.smithi029.stdout:1953: writing smithi02979277-24 from 2010618 to 2353145 tid 2 2024-04-12T11:22:51.642 INFO:tasks.rados.rados.0.smithi029.stdout:1954: write oid 29 current snap is 253 2024-04-12T11:22:51.642 INFO:tasks.rados.rados.0.smithi029.stdout:1954: seq_num 736 ranges {769952=553152,2069100=300852} 2024-04-12T11:22:51.649 INFO:tasks.rados.rados.0.smithi029.stdout:1954: writing smithi02979277-29 from 769952 to 1323104 tid 1 2024-04-12T11:22:51.655 INFO:tasks.rados.rados.0.smithi029.stdout:1954: writing smithi02979277-29 from 2069100 to 2369952 tid 2 2024-04-12T11:22:51.656 INFO:tasks.rados.rados.0.smithi029.stdout:1955: snap_remove snap 234 2024-04-12T11:22:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:51 smithi087 ceph-mon[89014]: osdmap e566: 8 total, 8 up, 8 in 2024-04-12T11:22:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:51 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2791297736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:51 smithi029 ceph-mon[93650]: osdmap e566: 8 total, 8 up, 8 in 2024-04-12T11:22:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:51 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2791297736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:51 smithi029 ceph-mon[97787]: osdmap e566: 8 total, 8 up, 8 in 2024-04-12T11:22:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:51 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2791297736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:22:52.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:22:52.573 INFO:tasks.rados.rados.0.smithi029.stdout:1955: done (4 left) 2024-04-12T11:22:52.573 INFO:tasks.rados.rados.0.smithi029.stdout:1956: read oid 40 snap 222 2024-04-12T11:22:52.573 INFO:tasks.rados.rados.0.smithi029.stdout:1956: expect deleted 2024-04-12T11:22:52.573 INFO:tasks.rados.rados.0.smithi029.stdout:1948: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:22:52.623 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:22:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:22:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:22:52.686 INFO:tasks.rados.rados.0.smithi029.stdout:1950: expect (ObjNum 705 snap 236 seq_num 705) 2024-04-12T11:22:52.864 INFO:tasks.rados.rados.0.smithi029.stdout:1953: finishing write tid 1 to smithi02979277-24 2024-04-12T11:22:52.908 INFO:tasks.rados.rados.0.smithi029.stdout:1953: finishing write tid 2 to smithi02979277-24 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1953: finishing write tid 3 to smithi02979277-24 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1953: finishing write tid 4 to smithi02979277-24 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1953: oid 24 updating version 0 to 1178 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1953: oid 24 updating version 1178 to 1184 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1953: oid 24 version 1184 is already newer than 1183 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1184 (ObjNum 735 snap 253 seq_num 735) dirty exists 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1953: left oid 24 (ObjNum 735 snap 253 seq_num 735) 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1948: done (4 left) 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1950: done (3 left) 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1953: done (2 left) 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1957: write oid 9 current snap is 253 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1954: finishing write tid 1 to smithi02979277-29 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1954: finishing write tid 2 to smithi02979277-29 2024-04-12T11:22:52.909 INFO:tasks.rados.rados.0.smithi029.stdout:1954: finishing write tid 3 to smithi02979277-29 2024-04-12T11:22:52.910 INFO:tasks.rados.rados.0.smithi029.stdout:1954: finishing write tid 4 to smithi02979277-29 2024-04-12T11:22:52.910 INFO:tasks.rados.rados.0.smithi029.stdout:1954: oid 29 updating version 0 to 718 2024-04-12T11:22:52.910 INFO:tasks.rados.rados.0.smithi029.stdout:1954: oid 29 updating version 718 to 719 2024-04-12T11:22:52.910 INFO:tasks.rados.rados.0.smithi029.stdout:1954: oid 29 updating version 719 to 720 2024-04-12T11:22:52.910 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 720 (ObjNum 736 snap 253 seq_num 736) dirty exists 2024-04-12T11:22:52.910 INFO:tasks.rados.rados.0.smithi029.stdout:1954: left oid 29 (ObjNum 736 snap 253 seq_num 736) 2024-04-12T11:22:52.910 INFO:tasks.rados.rados.0.smithi029.stdout:1957: seq_num 737 ranges {786759=628401,2011229=375530} 2024-04-12T11:22:52.912 INFO:tasks.rados.rados.0.smithi029.stdout:1957: writing smithi02979277-9 from 786759 to 1415160 tid 1 2024-04-12T11:22:52.912 INFO:tasks.rados.rados.0.smithi029.stdout:1957: writing smithi02979277-9 from 2011229 to 2386759 tid 2 2024-04-12T11:22:52.912 INFO:tasks.rados.rados.0.smithi029.stdout:1954: done (2 left) 2024-04-12T11:22:52.912 INFO:tasks.rados.rados.0.smithi029.stdout:1956: done (1 left) 2024-04-12T11:22:52.912 INFO:tasks.rados.rados.0.smithi029.stdout:1958: write oid 31 current snap is 253 2024-04-12T11:22:52.912 INFO:tasks.rados.rados.0.smithi029.stdout:1957: finishing write tid 1 to smithi02979277-9 2024-04-12T11:22:52.912 INFO:tasks.rados.rados.0.smithi029.stdout:1958: seq_num 738 ranges {403566=420003,1386064=546918} 2024-04-12T11:22:52.913 INFO:tasks.rados.rados.0.smithi029.stdout:1958: writing smithi02979277-31 from 403566 to 823569 tid 1 2024-04-12T11:22:52.913 INFO:tasks.rados.rados.0.smithi029.stdout:1958: writing smithi02979277-31 from 1386064 to 1932982 tid 2 2024-04-12T11:22:52.913 INFO:tasks.rados.rados.0.smithi029.stdout:1959: snap_create 2024-04-12T11:22:52.913 INFO:tasks.rados.rados.0.smithi029.stdout:1957: finishing write tid 2 to smithi02979277-9 2024-04-12T11:22:52.913 INFO:tasks.rados.rados.0.smithi029.stdout:1957: finishing write tid 3 to smithi02979277-9 2024-04-12T11:22:52.913 INFO:tasks.rados.rados.0.smithi029.stdout:1957: finishing write tid 4 to smithi02979277-9 2024-04-12T11:22:52.913 INFO:tasks.rados.rados.0.smithi029.stdout:1957: oid 9 updating version 0 to 707 2024-04-12T11:22:52.913 INFO:tasks.rados.rados.0.smithi029.stdout:1957: oid 9 updating version 707 to 708 2024-04-12T11:22:52.913 INFO:tasks.rados.rados.0.smithi029.stdout:1957: oid 9 version 708 is already newer than 706 2024-04-12T11:22:52.913 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 708 (ObjNum 737 snap 253 seq_num 737) dirty exists 2024-04-12T11:22:52.914 INFO:tasks.rados.rados.0.smithi029.stdout:1957: left oid 9 (ObjNum 737 snap 253 seq_num 737) 2024-04-12T11:22:52.914 INFO:tasks.rados.rados.0.smithi029.stdout:1958: finishing write tid 1 to smithi02979277-31 2024-04-12T11:22:52.914 INFO:tasks.rados.rados.0.smithi029.stdout:1958: finishing write tid 2 to smithi02979277-31 2024-04-12T11:22:52.914 INFO:tasks.rados.rados.0.smithi029.stdout:1958: finishing write tid 3 to smithi02979277-31 2024-04-12T11:22:52.914 INFO:tasks.rados.rados.0.smithi029.stdout:1958: finishing write tid 4 to smithi02979277-31 2024-04-12T11:22:52.914 INFO:tasks.rados.rados.0.smithi029.stdout:1958: oid 31 updating version 0 to 790 2024-04-12T11:22:52.914 INFO:tasks.rados.rados.0.smithi029.stdout:1958: oid 31 updating version 790 to 791 2024-04-12T11:22:52.914 INFO:tasks.rados.rados.0.smithi029.stdout:1958: oid 31 version 791 is already newer than 789 2024-04-12T11:22:52.914 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 791 (ObjNum 738 snap 253 seq_num 738) dirty exists 2024-04-12T11:22:52.914 INFO:tasks.rados.rados.0.smithi029.stdout:1958: left oid 31 (ObjNum 738 snap 253 seq_num 738) 2024-04-12T11:22:52.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:52 smithi087 ceph-mon[89014]: pgmap v486: 89 pgs: 89 active+clean; 170 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 458 KiB/s wr, 12 op/s 2024-04-12T11:22:52.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:52 smithi087 ceph-mon[89014]: osdmap e567: 8 total, 8 up, 8 in 2024-04-12T11:22:52.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:52 smithi087 ceph-mon[89014]: osdmap e568: 8 total, 8 up, 8 in 2024-04-12T11:22:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:52 smithi029 ceph-mon[93650]: pgmap v486: 89 pgs: 89 active+clean; 170 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 458 KiB/s wr, 12 op/s 2024-04-12T11:22:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:52 smithi029 ceph-mon[93650]: osdmap e567: 8 total, 8 up, 8 in 2024-04-12T11:22:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:52 smithi029 ceph-mon[93650]: osdmap e568: 8 total, 8 up, 8 in 2024-04-12T11:22:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:52 smithi029 ceph-mon[97787]: pgmap v486: 89 pgs: 89 active+clean; 170 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 458 KiB/s wr, 12 op/s 2024-04-12T11:22:53.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:52 smithi029 ceph-mon[97787]: osdmap e567: 8 total, 8 up, 8 in 2024-04-12T11:22:53.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:52 smithi029 ceph-mon[97787]: osdmap e568: 8 total, 8 up, 8 in 2024-04-12T11:22:53.631 INFO:tasks.rados.rados.0.smithi029.stdout:1957: done (2 left) 2024-04-12T11:22:53.632 INFO:tasks.rados.rados.0.smithi029.stdout:1958: done (1 left) 2024-04-12T11:22:53.632 INFO:tasks.rados.rados.0.smithi029.stdout:1959: done (0 left) 2024-04-12T11:22:53.632 INFO:tasks.rados.rados.0.smithi029.stdout:1960: snap_remove snap 223 2024-04-12T11:22:53.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:53 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/361420603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:54.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:53 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/361420603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:54.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:53 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/361420603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:54.634 INFO:tasks.rados.rados.0.smithi029.stdout:1960: done (0 left) 2024-04-12T11:22:54.634 INFO:tasks.rados.rados.0.smithi029.stdout:1961: snap_create 2024-04-12T11:22:54.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:54 smithi029 ceph-mon[97787]: pgmap v489: 89 pgs: 89 active+clean; 169 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 245 KiB/s wr, 7 op/s 2024-04-12T11:22:54.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:54 smithi029 ceph-mon[97787]: osdmap e569: 8 total, 8 up, 8 in 2024-04-12T11:22:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:54 smithi029 ceph-mon[93650]: pgmap v489: 89 pgs: 89 active+clean; 169 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 245 KiB/s wr, 7 op/s 2024-04-12T11:22:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:54 smithi029 ceph-mon[93650]: osdmap e569: 8 total, 8 up, 8 in 2024-04-12T11:22:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:54 smithi087 ceph-mon[89014]: pgmap v489: 89 pgs: 89 active+clean; 169 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 245 KiB/s wr, 7 op/s 2024-04-12T11:22:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:54 smithi087 ceph-mon[89014]: osdmap e569: 8 total, 8 up, 8 in 2024-04-12T11:22:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:22:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:22:54] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.43.0" 2024-04-12T11:22:55.635 INFO:tasks.rados.rados.0.smithi029.stdout:1961: done (0 left) 2024-04-12T11:22:55.635 INFO:tasks.rados.rados.0.smithi029.stdout:1962: read oid 49 snap -1 2024-04-12T11:22:55.635 INFO:tasks.rados.rados.0.smithi029.stdout:1962: expect (ObjNum 714 snap 246 seq_num 714) 2024-04-12T11:22:55.635 INFO:tasks.rados.rados.0.smithi029.stdout:1963: read oid 35 snap 237 2024-04-12T11:22:55.635 INFO:tasks.rados.rados.0.smithi029.stdout:1963: expect (ObjNum 679 snap 220 seq_num 679) 2024-04-12T11:22:55.635 INFO:tasks.rados.rados.0.smithi029.stdout:1964: snap_remove snap 253 2024-04-12T11:22:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:55 smithi087 ceph-mon[89014]: osdmap e570: 8 total, 8 up, 8 in 2024-04-12T11:22:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:55 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2564318866' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:55.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:55 smithi087 ceph-mon[89014]: osdmap e571: 8 total, 8 up, 8 in 2024-04-12T11:22:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:55 smithi029 ceph-mon[93650]: osdmap e570: 8 total, 8 up, 8 in 2024-04-12T11:22:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:55 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2564318866' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:55 smithi029 ceph-mon[93650]: osdmap e571: 8 total, 8 up, 8 in 2024-04-12T11:22:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:55 smithi029 ceph-mon[97787]: osdmap e570: 8 total, 8 up, 8 in 2024-04-12T11:22:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:55 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2564318866' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:55 smithi029 ceph-mon[97787]: osdmap e571: 8 total, 8 up, 8 in 2024-04-12T11:22:56.638 INFO:tasks.rados.rados.0.smithi029.stdout:1964: done (2 left) 2024-04-12T11:22:56.638 INFO:tasks.rados.rados.0.smithi029.stdout:1965: read oid 28 snap -1 2024-04-12T11:22:56.638 INFO:tasks.rados.rados.0.smithi029.stdout:1965: expect deleted 2024-04-12T11:22:56.638 INFO:tasks.rados.rados.0.smithi029.stdout:1966: write oid 45 current snap is 255 2024-04-12T11:22:56.638 INFO:tasks.rados.rados.0.smithi029.stdout:1966: seq_num 739 ranges {420373=495252,1444546=690576} 2024-04-12T11:22:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:1966: writing smithi02979277-45 from 420373 to 915625 tid 1 2024-04-12T11:22:56.650 INFO:tasks.rados.rados.0.smithi029.stdout:1966: writing smithi02979277-45 from 1444546 to 2135122 tid 2 2024-04-12T11:22:56.650 INFO:tasks.rados.rados.0.smithi029.stdout:1967: snap_remove snap 239 2024-04-12T11:22:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:56 smithi087 ceph-mon[89014]: pgmap v492: 89 pgs: 89 active+clean; 172 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 654 KiB/s wr, 12 op/s 2024-04-12T11:22:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:56 smithi087 ceph-mon[89014]: osdmap e572: 8 total, 8 up, 8 in 2024-04-12T11:22:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:56 smithi029 ceph-mon[93650]: pgmap v492: 89 pgs: 89 active+clean; 172 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 654 KiB/s wr, 12 op/s 2024-04-12T11:22:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:56 smithi029 ceph-mon[93650]: osdmap e572: 8 total, 8 up, 8 in 2024-04-12T11:22:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:56 smithi029 ceph-mon[97787]: pgmap v492: 89 pgs: 89 active+clean; 172 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 654 KiB/s wr, 12 op/s 2024-04-12T11:22:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:56 smithi029 ceph-mon[97787]: osdmap e572: 8 total, 8 up, 8 in 2024-04-12T11:22:57.574 INFO:tasks.rados.rados.0.smithi029.stdout:1967: done (4 left) 2024-04-12T11:22:57.575 INFO:tasks.rados.rados.0.smithi029.stdout:1968: write oid 40 current snap is 255 2024-04-12T11:22:57.575 INFO:tasks.rados.rados.0.smithi029.stdout:1962: expect (ObjNum 714 snap 246 seq_num 714) 2024-04-12T11:22:57.644 INFO:tasks.rados.rados.0.smithi029.stdout:1963: expect (ObjNum 679 snap 220 seq_num 679) 2024-04-12T11:22:57.804 INFO:tasks.rados.rados.0.smithi029.stdout:1966: finishing write tid 1 to smithi02979277-45 2024-04-12T11:22:57.805 INFO:tasks.rados.rados.0.smithi029.stdout:1966: finishing write tid 2 to smithi02979277-45 2024-04-12T11:22:57.805 INFO:tasks.rados.rados.0.smithi029.stdout:1966: finishing write tid 3 to smithi02979277-45 2024-04-12T11:22:57.805 INFO:tasks.rados.rados.0.smithi029.stdout:1966: finishing write tid 4 to smithi02979277-45 2024-04-12T11:22:57.805 INFO:tasks.rados.rados.0.smithi029.stdout:1966: oid 45 updating version 0 to 920 2024-04-12T11:22:57.805 INFO:tasks.rados.rados.0.smithi029.stdout:1966: oid 45 version 920 is already newer than 919 2024-04-12T11:22:57.805 INFO:tasks.rados.rados.0.smithi029.stdout:1966: oid 45 version 920 is already newer than 918 2024-04-12T11:22:57.805 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 920 (ObjNum 739 snap 255 seq_num 739) dirty exists 2024-04-12T11:22:57.805 INFO:tasks.rados.rados.0.smithi029.stdout:1966: left oid 45 (ObjNum 739 snap 255 seq_num 739) 2024-04-12T11:22:57.805 INFO:tasks.rados.rados.0.smithi029.stdout:1968: seq_num 740 ranges {437180=570501,1503028=550587} 2024-04-12T11:22:57.811 INFO:tasks.rados.rados.0.smithi029.stdout:1968: writing smithi02979277-40 from 437180 to 1007681 tid 1 2024-04-12T11:22:57.818 INFO:tasks.rados.rados.0.smithi029.stdout:1968: writing smithi02979277-40 from 1503028 to 2053615 tid 2 2024-04-12T11:22:57.819 INFO:tasks.rados.rados.0.smithi029.stdout:1962: done (4 left) 2024-04-12T11:22:57.819 INFO:tasks.rados.rados.0.smithi029.stdout:1963: done (3 left) 2024-04-12T11:22:57.819 INFO:tasks.rados.rados.0.smithi029.stdout:1965: done (2 left) 2024-04-12T11:22:57.819 INFO:tasks.rados.rados.0.smithi029.stdout:1966: done (1 left) 2024-04-12T11:22:57.819 INFO:tasks.rados.rados.0.smithi029.stdout:1969: rollback oid 50 current snap is 255 2024-04-12T11:22:57.819 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 50 to 238 2024-04-12T11:22:57.819 INFO:tasks.rados.rados.0.smithi029.stdout:1968: finishing write tid 1 to smithi02979277-40 2024-04-12T11:22:57.819 INFO:tasks.rados.rados.0.smithi029.stdout:1970: snap_create 2024-04-12T11:22:57.822 INFO:tasks.rados.rados.0.smithi029.stdout:1968: finishing write tid 2 to smithi02979277-40 2024-04-12T11:22:57.822 INFO:tasks.rados.rados.0.smithi029.stdout:1968: finishing write tid 3 to smithi02979277-40 2024-04-12T11:22:57.823 INFO:tasks.rados.rados.0.smithi029.stdout:1969: finishing rollback tid 0 to smithi02979277-50 2024-04-12T11:22:57.823 INFO:tasks.rados.rados.0.smithi029.stdout:1968: finishing write tid 4 to smithi02979277-40 2024-04-12T11:22:57.823 INFO:tasks.rados.rados.0.smithi029.stdout:1968: oid 40 updating version 0 to 1210 2024-04-12T11:22:57.823 INFO:tasks.rados.rados.0.smithi029.stdout:1968: oid 40 updating version 1210 to 1211 2024-04-12T11:22:57.823 INFO:tasks.rados.rados.0.smithi029.stdout:1968: oid 40 version 1211 is already newer than 1209 2024-04-12T11:22:57.823 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 1211 (ObjNum 740 snap 255 seq_num 740) dirty exists 2024-04-12T11:22:57.824 INFO:tasks.rados.rados.0.smithi029.stdout:1968: left oid 40 (ObjNum 740 snap 255 seq_num 740) 2024-04-12T11:22:57.824 INFO:tasks.rados.rados.0.smithi029.stdout:1969: finishing rollback tid 1 to smithi02979277-50 2024-04-12T11:22:57.824 INFO:tasks.rados.rados.0.smithi029.stdout:1969: finishing rollback tid 2 to smithi02979277-50 2024-04-12T11:22:57.824 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 654 (ObjNum 665 snap 214 seq_num 665) dirty exists 2024-04-12T11:22:58.648 INFO:tasks.rados.rados.0.smithi029.stdout:1968: done (2 left) 2024-04-12T11:22:58.648 INFO:tasks.rados.rados.0.smithi029.stdout:1969: done (1 left) 2024-04-12T11:22:58.648 INFO:tasks.rados.rados.0.smithi029.stdout:1970: done (0 left) 2024-04-12T11:22:58.649 INFO:tasks.rados.rados.0.smithi029.stdout:1971: read oid 22 snap -1 2024-04-12T11:22:58.649 INFO:tasks.rados.rados.0.smithi029.stdout:1971: expect (ObjNum 720 snap 248 seq_num 720) 2024-04-12T11:22:58.649 INFO:tasks.rados.rados.0.smithi029.stdout:1972: delete oid 21 current snap is 256 2024-04-12T11:22:58.653 INFO:tasks.rados.rados.0.smithi029.stdout:1972: done (1 left) 2024-04-12T11:22:58.653 INFO:tasks.rados.rados.0.smithi029.stdout:1973: delete oid 28 current snap is 256 2024-04-12T11:22:58.656 INFO:tasks.rados.rados.0.smithi029.stdout:1973: done (1 left) 2024-04-12T11:22:58.656 INFO:tasks.rados.rados.0.smithi029.stdout:1974: snap_remove snap 218 2024-04-12T11:22:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:58 smithi087 ceph-mon[89014]: pgmap v495: 89 pgs: 89 active+clean; 174 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 646 KiB/s wr, 6 op/s 2024-04-12T11:22:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:58 smithi087 ceph-mon[89014]: osdmap e573: 8 total, 8 up, 8 in 2024-04-12T11:22:58.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:58 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/467527836' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:58 smithi029 ceph-mon[93650]: pgmap v495: 89 pgs: 89 active+clean; 174 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 646 KiB/s wr, 6 op/s 2024-04-12T11:22:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:58 smithi029 ceph-mon[93650]: osdmap e573: 8 total, 8 up, 8 in 2024-04-12T11:22:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:58 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/467527836' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:58 smithi029 ceph-mon[97787]: pgmap v495: 89 pgs: 89 active+clean; 174 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 646 KiB/s wr, 6 op/s 2024-04-12T11:22:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:58 smithi029 ceph-mon[97787]: osdmap e573: 8 total, 8 up, 8 in 2024-04-12T11:22:59.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:58 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/467527836' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:22:59.653 INFO:tasks.rados.rados.0.smithi029.stdout:1974: done (1 left) 2024-04-12T11:22:59.653 INFO:tasks.rados.rados.0.smithi029.stdout:1975: write oid 6 current snap is 256 2024-04-12T11:22:59.653 INFO:tasks.rados.rados.0.smithi029.stdout:1975: seq_num 743 ranges {487601=796248,1962121=525480} 2024-04-12T11:22:59.662 INFO:tasks.rados.rados.0.smithi029.stdout:1975: writing smithi02979277-6 from 487601 to 1283849 tid 1 2024-04-12T11:22:59.669 INFO:tasks.rados.rados.0.smithi029.stdout:1975: writing smithi02979277-6 from 1962121 to 2487601 tid 2 2024-04-12T11:22:59.669 INFO:tasks.rados.rados.0.smithi029.stdout:1976: read oid 40 snap -1 2024-04-12T11:22:59.669 INFO:tasks.rados.rados.0.smithi029.stdout:1976: expect (ObjNum 740 snap 255 seq_num 740) 2024-04-12T11:22:59.669 INFO:tasks.rados.rados.0.smithi029.stdout:1971: expect (ObjNum 720 snap 248 seq_num 720) 2024-04-12T11:22:59.765 INFO:tasks.rados.rados.0.smithi029.stdout:1975: finishing write tid 1 to smithi02979277-6 2024-04-12T11:22:59.765 INFO:tasks.rados.rados.0.smithi029.stdout:1975: finishing write tid 2 to smithi02979277-6 2024-04-12T11:22:59.765 INFO:tasks.rados.rados.0.smithi029.stdout:1975: finishing write tid 3 to smithi02979277-6 2024-04-12T11:22:59.765 INFO:tasks.rados.rados.0.smithi029.stdout:1977: read oid 36 snap 248 2024-04-12T11:22:59.765 INFO:tasks.rados.rados.0.smithi029.stdout:1977: expect (ObjNum 697 snap 231 seq_num 697) 2024-04-12T11:22:59.765 INFO:tasks.rados.rados.0.smithi029.stdout:1971: done (3 left) 2024-04-12T11:22:59.765 INFO:tasks.rados.rados.0.smithi029.stdout:1978: read oid 8 snap -1 2024-04-12T11:22:59.765 INFO:tasks.rados.rados.0.smithi029.stdout:1978: expect (ObjNum 601 snap 192 seq_num 601) 2024-04-12T11:22:59.766 INFO:tasks.rados.rados.0.smithi029.stdout:1979: delete oid 15 current snap is 256 2024-04-12T11:22:59.766 INFO:tasks.rados.rados.0.smithi029.stdout:1975: finishing write tid 4 to smithi02979277-6 2024-04-12T11:22:59.766 INFO:tasks.rados.rados.0.smithi029.stdout:1975: oid 6 updating version 0 to 465 2024-04-12T11:22:59.766 INFO:tasks.rados.rados.0.smithi029.stdout:1975: oid 6 version 465 is already newer than 464 2024-04-12T11:22:59.766 INFO:tasks.rados.rados.0.smithi029.stdout:1975: oid 6 updating version 465 to 466 2024-04-12T11:22:59.766 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 466 (ObjNum 743 snap 256 seq_num 743) dirty exists 2024-04-12T11:22:59.766 INFO:tasks.rados.rados.0.smithi029.stdout:1975: left oid 6 (ObjNum 743 snap 256 seq_num 743) 2024-04-12T11:22:59.766 INFO:tasks.rados.rados.0.smithi029.stdout:1976: expect (ObjNum 740 snap 255 seq_num 740) 2024-04-12T11:22:59.849 INFO:tasks.rados.rados.0.smithi029.stdout:1978: expect (ObjNum 601 snap 192 seq_num 601) 2024-04-12T11:22:59.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:59 smithi029 ceph-mon[97787]: osdmap e574: 8 total, 8 up, 8 in 2024-04-12T11:22:59.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:22:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:22:59.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:59 smithi029 ceph-mon[93650]: osdmap e574: 8 total, 8 up, 8 in 2024-04-12T11:22:59.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:22:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:22:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:59 smithi087 ceph-mon[89014]: osdmap e574: 8 total, 8 up, 8 in 2024-04-12T11:22:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:22:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:22:59.992 INFO:tasks.rados.rados.0.smithi029.stdout:1977: expect (ObjNum 697 snap 231 seq_num 697) 2024-04-12T11:23:00.121 INFO:tasks.rados.rados.0.smithi029.stdout:1975: done (4 left) 2024-04-12T11:23:00.121 INFO:tasks.rados.rados.0.smithi029.stdout:1976: done (3 left) 2024-04-12T11:23:00.121 INFO:tasks.rados.rados.0.smithi029.stdout:1977: done (2 left) 2024-04-12T11:23:00.122 INFO:tasks.rados.rados.0.smithi029.stdout:1978: done (1 left) 2024-04-12T11:23:00.122 INFO:tasks.rados.rados.0.smithi029.stdout:1979: done (0 left) 2024-04-12T11:23:00.122 INFO:tasks.rados.rados.0.smithi029.stdout:1980: read oid 12 snap -1 2024-04-12T11:23:00.122 INFO:tasks.rados.rados.0.smithi029.stdout:1980: expect (ObjNum 670 snap 215 seq_num 670) 2024-04-12T11:23:00.122 INFO:tasks.rados.rados.0.smithi029.stdout:1981: read oid 27 snap 248 2024-04-12T11:23:00.122 INFO:tasks.rados.rados.0.smithi029.stdout:1981: expect deleted 2024-04-12T11:23:00.122 INFO:tasks.rados.rados.0.smithi029.stdout:1982: snap_remove snap 252 2024-04-12T11:23:00.660 INFO:tasks.rados.rados.0.smithi029.stdout:1980: expect (ObjNum 670 snap 215 seq_num 670) 2024-04-12T11:23:00.887 INFO:tasks.rados.rados.0.smithi029.stdout:1980: done (2 left) 2024-04-12T11:23:00.887 INFO:tasks.rados.rados.0.smithi029.stdout:1981: done (1 left) 2024-04-12T11:23:00.888 INFO:tasks.rados.rados.0.smithi029.stdout:1982: done (0 left) 2024-04-12T11:23:00.888 INFO:tasks.rados.rados.0.smithi029.stdout:1983: rollback oid 11 current snap is 256 2024-04-12T11:23:00.888 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 11 to 248 2024-04-12T11:23:00.888 INFO:tasks.rados.rados.0.smithi029.stdout:1984: read oid 25 snap -1 2024-04-12T11:23:00.888 INFO:tasks.rados.rados.0.smithi029.stdout:1984: expect (ObjNum 710 snap 245 seq_num 710) 2024-04-12T11:23:00.888 INFO:tasks.rados.rados.0.smithi029.stdout:1985: write oid 2 current snap is 256 2024-04-12T11:23:00.888 INFO:tasks.rados.rados.0.smithi029.stdout:1985: seq_num 745 ranges {521215=546746,1562732=701583} 2024-04-12T11:23:00.893 INFO:tasks.rados.rados.0.smithi029.stdout:1985: writing smithi02979277-2 from 521215 to 1067961 tid 1 2024-04-12T11:23:00.900 INFO:tasks.rados.rados.0.smithi029.stdout:1985: writing smithi02979277-2 from 1562732 to 2264315 tid 2 2024-04-12T11:23:00.900 INFO:tasks.rados.rados.0.smithi029.stdout:1986: snap_create 2024-04-12T11:23:00.900 INFO:tasks.rados.rados.0.smithi029.stdout:1983: finishing rollback tid 1 to smithi02979277-11 2024-04-12T11:23:00.900 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-04-12T11:23:00.900 INFO:tasks.rados.rados.0.smithi029.stdout:1985: finishing write tid 1 to smithi02979277-2 2024-04-12T11:23:00.903 INFO:tasks.rados.rados.0.smithi029.stdout:1984: expect (ObjNum 710 snap 245 seq_num 710) 2024-04-12T11:23:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:00 smithi087 ceph-mon[89014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:23:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:00 smithi087 ceph-mon[89014]: Upgrade: osd.3 is safe to restart 2024-04-12T11:23:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:00 smithi087 ceph-mon[89014]: pgmap v498: 89 pgs: 89 active+clean; 180 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.0 MiB/s wr, 8 op/s 2024-04-12T11:23:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:00 smithi087 ceph-mon[89014]: osdmap e575: 8 total, 8 up, 8 in 2024-04-12T11:23:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:00 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3956345986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:01.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:00 smithi029 ceph-mon[97787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:23:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:00 smithi029 ceph-mon[97787]: Upgrade: osd.3 is safe to restart 2024-04-12T11:23:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:00 smithi029 ceph-mon[97787]: pgmap v498: 89 pgs: 89 active+clean; 180 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.0 MiB/s wr, 8 op/s 2024-04-12T11:23:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:00 smithi029 ceph-mon[97787]: osdmap e575: 8 total, 8 up, 8 in 2024-04-12T11:23:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:00 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3956345986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:00 smithi029 ceph-mon[93650]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T11:23:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:00 smithi029 ceph-mon[93650]: Upgrade: osd.3 is safe to restart 2024-04-12T11:23:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:00 smithi029 ceph-mon[93650]: pgmap v498: 89 pgs: 89 active+clean; 180 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.0 MiB/s wr, 8 op/s 2024-04-12T11:23:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:00 smithi029 ceph-mon[93650]: osdmap e575: 8 total, 8 up, 8 in 2024-04-12T11:23:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:00 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3956345986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:01.112 INFO:tasks.rados.rados.0.smithi029.stdout:1985: finishing write tid 2 to smithi02979277-2 2024-04-12T11:23:01.112 INFO:tasks.rados.rados.0.smithi029.stdout:1985: finishing write tid 3 to smithi02979277-2 2024-04-12T11:23:01.112 INFO:tasks.rados.rados.0.smithi029.stdout:1985: finishing write tid 4 to smithi02979277-2 2024-04-12T11:23:01.112 INFO:tasks.rados.rados.0.smithi029.stdout:1985: oid 2 updating version 0 to 218 2024-04-12T11:23:01.112 INFO:tasks.rados.rados.0.smithi029.stdout:1985: oid 2 version 218 is already newer than 217 2024-04-12T11:23:01.112 INFO:tasks.rados.rados.0.smithi029.stdout:1985: oid 2 updating version 218 to 219 2024-04-12T11:23:01.112 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 219 (ObjNum 745 snap 256 seq_num 745) dirty exists 2024-04-12T11:23:01.112 INFO:tasks.rados.rados.0.smithi029.stdout:1985: left oid 2 (ObjNum 745 snap 256 seq_num 745) 2024-04-12T11:23:01.725 INFO:tasks.rados.rados.0.smithi029.stdout:1983: done (3 left) 2024-04-12T11:23:01.725 INFO:tasks.rados.rados.0.smithi029.stdout:1984: done (2 left) 2024-04-12T11:23:01.725 INFO:tasks.rados.rados.0.smithi029.stdout:1985: done (1 left) 2024-04-12T11:23:01.725 INFO:tasks.rados.rados.0.smithi029.stdout:1986: done (0 left) 2024-04-12T11:23:01.726 INFO:tasks.rados.rados.0.smithi029.stdout:1987: snap_remove snap 220 2024-04-12T11:23:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:01 smithi087 ceph-mon[89014]: osdmap e576: 8 total, 8 up, 8 in 2024-04-12T11:23:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:01 smithi087 ceph-mon[89014]: Upgrade: Updating osd.3 2024-04-12T11:23:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:01 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:01 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T11:23:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:01 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:01 smithi087 ceph-mon[89014]: Deploying daemon osd.3 on smithi029 2024-04-12T11:23:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[93650]: osdmap e576: 8 total, 8 up, 8 in 2024-04-12T11:23:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[93650]: Upgrade: Updating osd.3 2024-04-12T11:23:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T11:23:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[93650]: Deploying daemon osd.3 on smithi029 2024-04-12T11:23:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[97787]: osdmap e576: 8 total, 8 up, 8 in 2024-04-12T11:23:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[97787]: Upgrade: Updating osd.3 2024-04-12T11:23:02.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:02.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T11:23:02.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:02.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:01 smithi029 ceph-mon[97787]: Deploying daemon osd.3 on smithi029 2024-04-12T11:23:02.575 INFO:tasks.rados.rados.0.smithi029.stdout:1987: done (0 left) 2024-04-12T11:23:02.575 INFO:tasks.rados.rados.0.smithi029.stdout:1988: read oid 40 snap 222 2024-04-12T11:23:02.575 INFO:tasks.rados.rados.0.smithi029.stdout:1988: expect deleted 2024-04-12T11:23:02.575 INFO:tasks.rados.rados.0.smithi029.stdout:1989: delete oid 42 current snap is 257 2024-04-12T11:23:02.583 INFO:tasks.rados.rados.0.smithi029.stdout:1988: done (1 left) 2024-04-12T11:23:02.583 INFO:tasks.rados.rados.0.smithi029.stdout:1989: done (0 left) 2024-04-12T11:23:02.583 INFO:tasks.rados.rados.0.smithi029.stdout:1990: snap_create 2024-04-12T11:23:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:23:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:23:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:23:02.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:02 smithi029 ceph-mon[97787]: pgmap v501: 89 pgs: 89 active+clean; 180 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 809 KiB/s wr, 7 op/s 2024-04-12T11:23:02.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:02 smithi029 ceph-mon[97787]: osdmap e577: 8 total, 8 up, 8 in 2024-04-12T11:23:02.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:02 smithi029 ceph-mon[97787]: osdmap e578: 8 total, 8 up, 8 in 2024-04-12T11:23:02.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:02 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3521321895' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:02.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:02 smithi029 ceph-mon[93650]: pgmap v501: 89 pgs: 89 active+clean; 180 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 809 KiB/s wr, 7 op/s 2024-04-12T11:23:02.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:02 smithi029 ceph-mon[93650]: osdmap e577: 8 total, 8 up, 8 in 2024-04-12T11:23:02.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:02 smithi029 ceph-mon[93650]: osdmap e578: 8 total, 8 up, 8 in 2024-04-12T11:23:02.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:02 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3521321895' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:02 smithi087 ceph-mon[89014]: pgmap v501: 89 pgs: 89 active+clean; 180 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 809 KiB/s wr, 7 op/s 2024-04-12T11:23:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:02 smithi087 ceph-mon[89014]: osdmap e577: 8 total, 8 up, 8 in 2024-04-12T11:23:03.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:02 smithi087 ceph-mon[89014]: osdmap e578: 8 total, 8 up, 8 in 2024-04-12T11:23:03.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:02 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3521321895' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:03.576 INFO:tasks.rados.rados.0.smithi029.stdout:1990: done (0 left) 2024-04-12T11:23:03.576 INFO:tasks.rados.rados.0.smithi029.stdout:1991: rollback oid 31 current snap is 258 2024-04-12T11:23:03.576 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 31 to 240 2024-04-12T11:23:03.576 INFO:tasks.rados.rados.0.smithi029.stdout:1992: delete oid 14 current snap is 258 2024-04-12T11:23:03.580 INFO:tasks.rados.rados.0.smithi029.stdout:1991: finishing rollback tid 0 to smithi02979277-31 2024-04-12T11:23:03.580 INFO:tasks.rados.rados.0.smithi029.stdout:1991: finishing rollback tid 1 to smithi02979277-31 2024-04-12T11:23:03.580 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 803 (ObjNum 529 snap 0 seq_num 1800734208) dirty dne 2024-04-12T11:23:03.581 INFO:tasks.rados.rados.0.smithi029.stdout:1991: done (1 left) 2024-04-12T11:23:03.581 INFO:tasks.rados.rados.0.smithi029.stdout:1992: done (0 left) 2024-04-12T11:23:03.581 INFO:tasks.rados.rados.0.smithi029.stdout:1993: write oid 26 current snap is 258 2024-04-12T11:23:03.581 INFO:tasks.rados.rados.0.smithi029.stdout:1993: seq_num 748 ranges {571636=488846,1854531=565263} 2024-04-12T11:23:03.586 INFO:tasks.rados.rados.0.smithi029.stdout:1993: writing smithi02979277-26 from 571636 to 1060482 tid 1 2024-04-12T11:23:03.594 INFO:tasks.rados.rados.0.smithi029.stdout:1993: writing smithi02979277-26 from 1854531 to 2419794 tid 2 2024-04-12T11:23:03.594 INFO:tasks.rados.rados.0.smithi029.stdout:1994: write oid 48 current snap is 258 2024-04-12T11:23:03.594 INFO:tasks.rados.rados.0.smithi029.stdout:1993: finishing write tid 1 to smithi02979277-26 2024-04-12T11:23:03.595 INFO:tasks.rados.rados.0.smithi029.stdout:1994: seq_num 749 ranges {588443=564095,1796660=708921} 2024-04-12T11:23:03.601 INFO:tasks.rados.rados.0.smithi029.stdout:1994: writing smithi02979277-48 from 588443 to 1152538 tid 1 2024-04-12T11:23:03.608 INFO:tasks.rados.rados.0.smithi029.stdout:1994: writing smithi02979277-48 from 1796660 to 2505581 tid 2 2024-04-12T11:23:03.609 INFO:tasks.rados.rados.0.smithi029.stdout:1995: snap_remove snap 256 2024-04-12T11:23:03.609 INFO:tasks.rados.rados.0.smithi029.stdout:1993: finishing write tid 2 to smithi02979277-26 2024-04-12T11:23:04.518 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:04 smithi029 systemd[1]: Stopping Ceph osd.3 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:23:04.518 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[61361]: 2024-04-12T11:23:04.385+0000 7fbf0107b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:23:04.518 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[61361]: 2024-04-12T11:23:04.385+0000 7fbf0107b700 -1 osd.3 579 *** Got signal Terminated *** 2024-04-12T11:23:04.518 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[61361]: 2024-04-12T11:23:04.385+0000 7fbf0107b700 -1 osd.3 579 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T11:23:04.585 INFO:tasks.rados.rados.0.smithi029.stdout:1995: done (2 left) 2024-04-12T11:23:04.585 INFO:tasks.rados.rados.0.smithi029.stdout:1996: delete oid 49 current snap is 258 2024-04-12T11:23:04.585 INFO:tasks.rados.rados.0.smithi029.stdout:1993: finishing write tid 3 to smithi02979277-26 2024-04-12T11:23:04.585 INFO:tasks.rados.rados.0.smithi029.stdout:1993: finishing write tid 4 to smithi02979277-26 2024-04-12T11:23:04.585 INFO:tasks.rados.rados.0.smithi029.stdout:1993: oid 26 updating version 0 to 1226 2024-04-12T11:23:04.585 INFO:tasks.rados.rados.0.smithi029.stdout:1993: oid 26 version 1226 is already newer than 1225 2024-04-12T11:23:04.585 INFO:tasks.rados.rados.0.smithi029.stdout:1993: oid 26 version 1226 is already newer than 1224 2024-04-12T11:23:04.585 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 1226 (ObjNum 748 snap 258 seq_num 748) dirty exists 2024-04-12T11:23:04.585 INFO:tasks.rados.rados.0.smithi029.stdout:1993: left oid 26 (ObjNum 748 snap 258 seq_num 748) 2024-04-12T11:23:04.586 INFO:tasks.rados.rados.0.smithi029.stdout:1994: finishing write tid 1 to smithi02979277-48 2024-04-12T11:23:04.586 INFO:tasks.rados.rados.0.smithi029.stdout:1994: finishing write tid 2 to smithi02979277-48 2024-04-12T11:23:04.586 INFO:tasks.rados.rados.0.smithi029.stdout:1994: finishing write tid 3 to smithi02979277-48 2024-04-12T11:23:04.586 INFO:tasks.rados.rados.0.smithi029.stdout:1994: finishing write tid 4 to smithi02979277-48 2024-04-12T11:23:04.586 INFO:tasks.rados.rados.0.smithi029.stdout:1994: oid 48 updating version 0 to 805 2024-04-12T11:23:04.586 INFO:tasks.rados.rados.0.smithi029.stdout:1994: oid 48 updating version 805 to 807 2024-04-12T11:23:04.586 INFO:tasks.rados.rados.0.smithi029.stdout:1994: oid 48 version 807 is already newer than 806 2024-04-12T11:23:04.586 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 807 (ObjNum 749 snap 258 seq_num 749) dirty exists 2024-04-12T11:23:04.586 INFO:tasks.rados.rados.0.smithi029.stdout:1994: left oid 48 (ObjNum 749 snap 258 seq_num 749) 2024-04-12T11:23:04.592 INFO:tasks.rados.rados.0.smithi029.stdout:1993: done (2 left) 2024-04-12T11:23:04.593 INFO:tasks.rados.rados.0.smithi029.stdout:1994: done (1 left) 2024-04-12T11:23:04.593 INFO:tasks.rados.rados.0.smithi029.stdout:1996: done (0 left) 2024-04-12T11:23:04.593 INFO:tasks.rados.rados.0.smithi029.stdout:1997: rollback oid 25 current snap is 258 2024-04-12T11:23:04.593 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 25 to 248 2024-04-12T11:23:04.593 INFO:tasks.rados.rados.0.smithi029.stdout:1998: snap_create 2024-04-12T11:23:04.596 INFO:tasks.rados.rados.0.smithi029.stdout:1997: finishing rollback tid 0 to smithi02979277-25 2024-04-12T11:23:04.596 INFO:tasks.rados.rados.0.smithi029.stdout:1997: finishing rollback tid 1 to smithi02979277-25 2024-04-12T11:23:04.596 INFO:tasks.rados.rados.0.smithi029.stdout:1997: finishing rollback tid 2 to smithi02979277-25 2024-04-12T11:23:04.596 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 962 (ObjNum 710 snap 245 seq_num 710) dirty exists 2024-04-12T11:23:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:04 smithi029 ceph-mon[97787]: pgmap v504: 89 pgs: 89 active+clean; 177 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 323 KiB/s wr, 13 op/s 2024-04-12T11:23:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:04 smithi029 ceph-mon[97787]: osdmap e579: 8 total, 8 up, 8 in 2024-04-12T11:23:04.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:04 smithi029 ceph-mon[97787]: osd.3 marked itself down and dead 2024-04-12T11:23:04.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:04 smithi029 ceph-mon[93650]: pgmap v504: 89 pgs: 89 active+clean; 177 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 323 KiB/s wr, 13 op/s 2024-04-12T11:23:04.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:04 smithi029 ceph-mon[93650]: osdmap e579: 8 total, 8 up, 8 in 2024-04-12T11:23:04.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:04 smithi029 ceph-mon[93650]: osd.3 marked itself down and dead 2024-04-12T11:23:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:04 smithi087 ceph-mon[89014]: pgmap v504: 89 pgs: 89 active+clean; 177 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 323 KiB/s wr, 13 op/s 2024-04-12T11:23:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:04 smithi087 ceph-mon[89014]: osdmap e579: 8 total, 8 up, 8 in 2024-04-12T11:23:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:04 smithi087 ceph-mon[89014]: osd.3 marked itself down and dead 2024-04-12T11:23:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:23:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:23:04] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.43.0" 2024-04-12T11:23:05.268 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:05 smithi029 podman[120402]: 2024-04-12 11:23:05.099557291 +0000 UTC m=+0.808470518 container died a9b5c72d15dace8afd879991e3436b9b66e9f8df686109358b7f470be4317f69 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, RELEASE=reef-d540eba, org.label-schema.build-date=20240403) 2024-04-12T11:23:05.586 INFO:tasks.rados.rados.0.smithi029.stdout:1997: done (1 left) 2024-04-12T11:23:05.586 INFO:tasks.rados.rados.0.smithi029.stdout:1998: done (0 left) 2024-04-12T11:23:05.586 INFO:tasks.rados.rados.0.smithi029.stdout:1999: write oid 42 current snap is 259 2024-04-12T11:23:05.586 INFO:tasks.rados.rados.0.smithi029.stdout:1999: seq_num 751 ranges {622057=714593,1913624=708433} 2024-04-12T11:23:05.593 INFO:tasks.rados.rados.0.smithi029.stdout:1999: writing smithi02979277-42 from 622057 to 1336650 tid 1 2024-04-12T11:23:05.601 INFO:tasks.rados.rados.0.smithi029.stdout:1999: writing smithi02979277-42 from 1913624 to 2622057 tid 2 2024-04-12T11:23:05.601 INFO:tasks.rados.rados.0.smithi029.stdout:2000: delete oid 14 current snap is 259 2024-04-12T11:23:05.601 INFO:tasks.rados.rados.0.smithi029.stdout:1999: finishing write tid 1 to smithi02979277-42 2024-04-12T11:23:05.603 INFO:tasks.rados.rados.0.smithi029.stdout:2000: done (1 left) 2024-04-12T11:23:05.603 INFO:tasks.rados.rados.0.smithi029.stdout:2001: read oid 3 snap -1 2024-04-12T11:23:05.603 INFO:tasks.rados.rados.0.smithi029.stdout:2001: expect (ObjNum 719 snap 248 seq_num 719) 2024-04-12T11:23:05.603 INFO:tasks.rados.rados.0.smithi029.stdout:2002: write oid 23 current snap is 259 2024-04-12T11:23:05.603 INFO:tasks.rados.rados.0.smithi029.stdout:2002: seq_num 753 ranges {655671=581444,2030588=625083} 2024-04-12T11:23:05.609 INFO:tasks.rados.rados.0.smithi029.stdout:2002: writing smithi02979277-23 from 655671 to 1237115 tid 1 2024-04-12T11:23:05.618 INFO:tasks.rados.rados.0.smithi029.stdout:2002: writing smithi02979277-23 from 2030588 to 2655671 tid 2 2024-04-12T11:23:05.618 INFO:tasks.rados.rados.0.smithi029.stdout:2003: write oid 27 current snap is 259 2024-04-12T11:23:05.618 INFO:tasks.rados.rados.0.smithi029.stdout:1999: finishing write tid 2 to smithi02979277-42 2024-04-12T11:23:05.618 INFO:tasks.rados.rados.0.smithi029.stdout:1999: finishing write tid 3 to smithi02979277-42 2024-04-12T11:23:05.618 INFO:tasks.rados.rados.0.smithi029.stdout:2003: seq_num 754 ranges {672478=656693,2089070=459917} 2024-04-12T11:23:05.624 INFO:tasks.rados.rados.0.smithi029.stdout:2003: writing smithi02979277-27 from 672478 to 1329171 tid 1 2024-04-12T11:23:05.632 INFO:tasks.rados.rados.0.smithi029.stdout:2003: writing smithi02979277-27 from 2089070 to 2548987 tid 2 2024-04-12T11:23:05.632 INFO:tasks.rados.rados.0.smithi029.stdout:2004: write oid 50 current snap is 259 2024-04-12T11:23:05.632 INFO:tasks.rados.rados.0.smithi029.stdout:1999: finishing write tid 4 to smithi02979277-42 2024-04-12T11:23:05.632 INFO:tasks.rados.rados.0.smithi029.stdout:1999: oid 42 updating version 0 to 289 2024-04-12T11:23:05.632 INFO:tasks.rados.rados.0.smithi029.stdout:1999: oid 42 updating version 289 to 291 2024-04-12T11:23:05.632 INFO:tasks.rados.rados.0.smithi029.stdout:1999: oid 42 version 291 is already newer than 290 2024-04-12T11:23:05.632 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 291 (ObjNum 751 snap 259 seq_num 751) dirty exists 2024-04-12T11:23:05.632 INFO:tasks.rados.rados.0.smithi029.stdout:1999: left oid 42 (ObjNum 751 snap 259 seq_num 751) 2024-04-12T11:23:05.632 INFO:tasks.rados.rados.0.smithi029.stdout:2001: expect (ObjNum 719 snap 248 seq_num 719) 2024-04-12T11:23:05.702 INFO:tasks.rados.rados.0.smithi029.stdout:2002: finishing write tid 1 to smithi02979277-23 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:2002: finishing write tid 2 to smithi02979277-23 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:2002: finishing write tid 3 to smithi02979277-23 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:2002: finishing write tid 4 to smithi02979277-23 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:2002: oid 23 updating version 0 to 670 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:2002: oid 23 updating version 670 to 672 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:2002: oid 23 version 672 is already newer than 671 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 672 (ObjNum 753 snap 259 seq_num 753) dirty exists 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:2002: left oid 23 (ObjNum 753 snap 259 seq_num 753) 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:2003: finishing write tid 1 to smithi02979277-27 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:2003: finishing write tid 2 to smithi02979277-27 2024-04-12T11:23:05.703 INFO:tasks.rados.rados.0.smithi029.stdout:2004: seq_num 755 ranges {689285=731942,2147552=541733} 2024-04-12T11:23:05.709 INFO:tasks.rados.rados.0.smithi029.stdout:2004: writing smithi02979277-50 from 689285 to 1421227 tid 1 2024-04-12T11:23:05.717 INFO:tasks.rados.rados.0.smithi029.stdout:2004: writing smithi02979277-50 from 2147552 to 2689285 tid 2 2024-04-12T11:23:05.717 INFO:tasks.rados.rados.0.smithi029.stdout:1999: done (4 left) 2024-04-12T11:23:05.717 INFO:tasks.rados.rados.0.smithi029.stdout:2001: done (3 left) 2024-04-12T11:23:05.717 INFO:tasks.rados.rados.0.smithi029.stdout:2002: done (2 left) 2024-04-12T11:23:05.717 INFO:tasks.rados.rados.0.smithi029.stdout:2005: read oid 41 snap -1 2024-04-12T11:23:05.717 INFO:tasks.rados.rados.0.smithi029.stdout:2005: expect (ObjNum 715 snap 247 seq_num 715) 2024-04-12T11:23:05.718 INFO:tasks.rados.rados.0.smithi029.stdout:2006: snap_create 2024-04-12T11:23:05.718 INFO:tasks.rados.rados.0.smithi029.stdout:2003: finishing write tid 3 to smithi02979277-27 2024-04-12T11:23:05.718 INFO:tasks.rados.rados.0.smithi029.stdout:2003: finishing write tid 4 to smithi02979277-27 2024-04-12T11:23:05.718 INFO:tasks.rados.rados.0.smithi029.stdout:2003: oid 27 updating version 0 to 708 2024-04-12T11:23:05.718 INFO:tasks.rados.rados.0.smithi029.stdout:2003: oid 27 version 708 is already newer than 706 2024-04-12T11:23:05.718 INFO:tasks.rados.rados.0.smithi029.stdout:2003: oid 27 version 708 is already newer than 707 2024-04-12T11:23:05.718 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 708 (ObjNum 754 snap 259 seq_num 754) dirty exists 2024-04-12T11:23:05.718 INFO:tasks.rados.rados.0.smithi029.stdout:2003: left oid 27 (ObjNum 754 snap 259 seq_num 754) 2024-04-12T11:23:05.718 INFO:tasks.rados.rados.0.smithi029.stdout:2004: finishing write tid 1 to smithi02979277-50 2024-04-12T11:23:05.720 INFO:tasks.rados.rados.0.smithi029.stdout:2004: finishing write tid 2 to smithi02979277-50 2024-04-12T11:23:05.721 INFO:tasks.rados.rados.0.smithi029.stdout:2004: finishing write tid 3 to smithi02979277-50 2024-04-12T11:23:05.721 INFO:tasks.rados.rados.0.smithi029.stdout:2004: finishing write tid 4 to smithi02979277-50 2024-04-12T11:23:05.721 INFO:tasks.rados.rados.0.smithi029.stdout:2004: oid 50 updating version 0 to 674 2024-04-12T11:23:05.721 INFO:tasks.rados.rados.0.smithi029.stdout:2004: oid 50 updating version 674 to 675 2024-04-12T11:23:05.721 INFO:tasks.rados.rados.0.smithi029.stdout:2004: oid 50 updating version 675 to 676 2024-04-12T11:23:05.721 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 676 (ObjNum 755 snap 259 seq_num 755) dirty exists 2024-04-12T11:23:05.721 INFO:tasks.rados.rados.0.smithi029.stdout:2004: left oid 50 (ObjNum 755 snap 259 seq_num 755) 2024-04-12T11:23:05.724 INFO:tasks.rados.rados.0.smithi029.stdout:2005: expect (ObjNum 715 snap 247 seq_num 715) 2024-04-12T11:23:05.742 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:05 smithi029 ceph-mon[97787]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:23:05.742 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:05 smithi029 ceph-mon[97787]: osdmap e580: 8 total, 7 up, 8 in 2024-04-12T11:23:05.742 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:05 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4239517603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:05.743 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:05 smithi029 podman[120402]: 2024-04-12 11:23:05.47310295 +0000 UTC m=+1.182016210 container cleanup a9b5c72d15dace8afd879991e3436b9b66e9f8df686109358b7f470be4317f69 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, GIT_BRANCH=HEAD, RELEASE=reef-d540eba, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-04-12T11:23:05.743 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:05 smithi029 bash[120402]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3 2024-04-12T11:23:05.743 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:05 smithi029 ceph-mon[93650]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:23:05.743 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:05 smithi029 ceph-mon[93650]: osdmap e580: 8 total, 7 up, 8 in 2024-04-12T11:23:05.743 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:05 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4239517603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:05 smithi087 ceph-mon[89014]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:23:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:05 smithi087 ceph-mon[89014]: osdmap e580: 8 total, 7 up, 8 in 2024-04-12T11:23:05.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:05 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4239517603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:06.018 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:05 smithi029 podman[120445]: 2024-04-12 11:23:05.742732108 +0000 UTC m=+0.639997354 container remove a9b5c72d15dace8afd879991e3436b9b66e9f8df686109358b7f470be4317f69 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=reef-d540eba, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, ceph=True) 2024-04-12T11:23:06.304 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:06 smithi029 podman[120535]: 2024-04-12 11:23:05.956851674 +0000 UTC m=+0.018057590 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:23:06.304 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:06 smithi029 podman[120535]: 2024-04-12 11:23:06.080027048 +0000 UTC m=+0.141232935 container create 8f1affcb5a01be2b2adef189c3c7938c1bd9bbcc672966bab00156bcde76f394 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD) 2024-04-12T11:23:06.304 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:06 smithi029 podman[120535]: 2024-04-12 11:23:06.300946664 +0000 UTC m=+0.362152561 container init 8f1affcb5a01be2b2adef189c3c7938c1bd9bbcc672966bab00156bcde76f394 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-rf-comb1-2cee185, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409) 2024-04-12T11:23:06.304 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:06 smithi029 podman[120535]: 2024-04-12 11:23:06.304101164 +0000 UTC m=+0.365307057 container start 8f1affcb5a01be2b2adef189c3c7938c1bd9bbcc672966bab00156bcde76f394 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-12T11:23:06.583 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:06 smithi029 podman[120535]: 2024-04-12 11:23:06.384657012 +0000 UTC m=+0.445862917 container attach 8f1affcb5a01be2b2adef189c3c7938c1bd9bbcc672966bab00156bcde76f394 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240409, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-12T11:23:06.591 INFO:tasks.rados.rados.0.smithi029.stdout:2003: done (3 left) 2024-04-12T11:23:06.592 INFO:tasks.rados.rados.0.smithi029.stdout:2004: done (2 left) 2024-04-12T11:23:06.592 INFO:tasks.rados.rados.0.smithi029.stdout:2005: done (1 left) 2024-04-12T11:23:06.592 INFO:tasks.rados.rados.0.smithi029.stdout:2006: done (0 left) 2024-04-12T11:23:06.592 INFO:tasks.rados.rados.0.smithi029.stdout:2007: snap_remove snap 247 2024-04-12T11:23:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:06 smithi087 ceph-mon[89014]: pgmap v507: 89 pgs: 14 stale+active+clean, 75 active+clean; 173 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 886 KiB/s wr, 17 op/s 2024-04-12T11:23:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:06 smithi087 ceph-mon[89014]: osdmap e581: 8 total, 7 up, 8 in 2024-04-12T11:23:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:06.974 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:06 smithi029 ceph-mon[97787]: pgmap v507: 89 pgs: 14 stale+active+clean, 75 active+clean; 173 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 886 KiB/s wr, 17 op/s 2024-04-12T11:23:06.974 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:06 smithi029 ceph-mon[97787]: osdmap e581: 8 total, 7 up, 8 in 2024-04-12T11:23:06.974 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:06.974 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:06.975 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:06 smithi029 podman[120535]: 2024-04-12 11:23:06.609195371 +0000 UTC m=+0.670401266 container died 8f1affcb5a01be2b2adef189c3c7938c1bd9bbcc672966bab00156bcde76f394 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-12T11:23:06.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:06 smithi029 ceph-mon[93650]: pgmap v507: 89 pgs: 14 stale+active+clean, 75 active+clean; 173 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 886 KiB/s wr, 17 op/s 2024-04-12T11:23:06.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:06 smithi029 ceph-mon[93650]: osdmap e581: 8 total, 7 up, 8 in 2024-04-12T11:23:06.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:06.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:07.577 INFO:tasks.rados.rados.0.smithi029.stdout:2007: done (0 left) 2024-04-12T11:23:07.578 INFO:tasks.rados.rados.0.smithi029.stdout:2008: delete oid 10 current snap is 260 2024-04-12T11:23:07.581 INFO:tasks.rados.rados.0.smithi029.stdout:2008: done (0 left) 2024-04-12T11:23:07.582 INFO:tasks.rados.rados.0.smithi029.stdout:2009: read oid 18 snap -1 2024-04-12T11:23:07.582 INFO:tasks.rados.rados.0.smithi029.stdout:2009: expect (ObjNum 732 snap 252 seq_num 732) 2024-04-12T11:23:07.582 INFO:tasks.rados.rados.0.smithi029.stdout:2010: read oid 28 snap -1 2024-04-12T11:23:07.582 INFO:tasks.rados.rados.0.smithi029.stdout:2010: expect deleted 2024-04-12T11:23:07.582 INFO:tasks.rados.rados.0.smithi029.stdout:2011: snap_remove snap 235 2024-04-12T11:23:07.595 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:07 smithi029 podman[120561]: 2024-04-12 11:23:07.285705406 +0000 UTC m=+0.667915059 container remove 8f1affcb5a01be2b2adef189c3c7938c1bd9bbcc672966bab00156bcde76f394 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, RELEASE=wip-rf-comb1-2cee185, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T11:23:07.595 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:07 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-12T11:23:07.596 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:07 smithi029 systemd[1]: Stopped Ceph osd.3 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:23:07.596 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:07 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.3.service: Consumed 10.429s CPU time. 2024-04-12T11:23:07.596 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:07 smithi029 systemd[1]: Starting Ceph osd.3 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:23:07.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:07 smithi029 ceph-mon[97787]: osdmap e582: 8 total, 7 up, 8 in 2024-04-12T11:23:07.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:07 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3492369153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:07.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:07 smithi029 ceph-mon[97787]: osdmap e583: 8 total, 7 up, 8 in 2024-04-12T11:23:07.853 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:07 smithi029 podman[120688]: 2024-04-12 11:23:07.72138138 +0000 UTC m=+0.112850525 container create 7244c9eae71283b0a19f6e266dd07a3922424f833f52c47592b35ea32c07e673 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-12T11:23:07.853 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:07 smithi029 podman[120688]: 2024-04-12 11:23:07.627507433 +0000 UTC m=+0.018976598 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:23:07.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:07 smithi029 ceph-mon[93650]: osdmap e582: 8 total, 7 up, 8 in 2024-04-12T11:23:07.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:07 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3492369153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:07.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:07 smithi029 ceph-mon[93650]: osdmap e583: 8 total, 7 up, 8 in 2024-04-12T11:23:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:07 smithi087 ceph-mon[89014]: osdmap e582: 8 total, 7 up, 8 in 2024-04-12T11:23:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:07 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3492369153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:07.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:07 smithi087 ceph-mon[89014]: osdmap e583: 8 total, 7 up, 8 in 2024-04-12T11:23:08.144 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:07 smithi029 podman[120688]: 2024-04-12 11:23:07.952357536 +0000 UTC m=+0.343826691 container init 7244c9eae71283b0a19f6e266dd07a3922424f833f52c47592b35ea32c07e673 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=) 2024-04-12T11:23:08.144 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:07 smithi029 podman[120688]: 2024-04-12 11:23:07.955370982 +0000 UTC m=+0.346840128 container start 7244c9eae71283b0a19f6e266dd07a3922424f833f52c47592b35ea32c07e673 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409, ceph=True, org.label-schema.schema-version=1.0) 2024-04-12T11:23:08.406 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 podman[120688]: 2024-04-12 11:23:08.144427857 +0000 UTC m=+0.535897018 container attach 7244c9eae71283b0a19f6e266dd07a3922424f833f52c47592b35ea32c07e673 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-12T11:23:08.578 INFO:tasks.rados.rados.0.smithi029.stdout:2011: done (2 left) 2024-04-12T11:23:08.578 INFO:tasks.rados.rados.0.smithi029.stdout:2012: read oid 10 snap -1 2024-04-12T11:23:08.579 INFO:tasks.rados.rados.0.smithi029.stdout:2012: expect deleted 2024-04-12T11:23:08.579 INFO:tasks.rados.rados.0.smithi029.stdout:2013: write oid 6 current snap is 260 2024-04-12T11:23:08.579 INFO:tasks.rados.rados.0.smithi029.stdout:2013: seq_num 757 ranges {722899=482440,1748163=723597} 2024-04-12T11:23:08.585 INFO:tasks.rados.rados.0.smithi029.stdout:2013: writing smithi02979277-6 from 722899 to 1205339 tid 1 2024-04-12T11:23:08.594 INFO:tasks.rados.rados.0.smithi029.stdout:2013: writing smithi02979277-6 from 1748163 to 2471760 tid 2 2024-04-12T11:23:08.594 INFO:tasks.rados.rados.0.smithi029.stdout:2009: expect (ObjNum 732 snap 252 seq_num 732) 2024-04-12T11:23:08.681 INFO:tasks.rados.rados.0.smithi029.stdout:2013: finishing write tid 1 to smithi02979277-6 2024-04-12T11:23:08.681 INFO:tasks.rados.rados.0.smithi029.stdout:2013: finishing write tid 2 to smithi02979277-6 2024-04-12T11:23:08.681 INFO:tasks.rados.rados.0.smithi029.stdout:2013: finishing write tid 3 to smithi02979277-6 2024-04-12T11:23:08.681 INFO:tasks.rados.rados.0.smithi029.stdout:2013: finishing write tid 4 to smithi02979277-6 2024-04-12T11:23:08.681 INFO:tasks.rados.rados.0.smithi029.stdout:2013: oid 6 updating version 0 to 480 2024-04-12T11:23:08.681 INFO:tasks.rados.rados.0.smithi029.stdout:2013: oid 6 version 480 is already newer than 478 2024-04-12T11:23:08.681 INFO:tasks.rados.rados.0.smithi029.stdout:2013: oid 6 version 480 is already newer than 479 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 480 (ObjNum 757 snap 260 seq_num 757) dirty exists 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:2013: left oid 6 (ObjNum 757 snap 260 seq_num 757) 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:2009: done (3 left) 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:2010: done (2 left) 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:2012: done (1 left) 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:2013: done (0 left) 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:2014: read oid 48 snap -1 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:2014: expect (ObjNum 749 snap 258 seq_num 749) 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:2015: read oid 32 snap -1 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:2015: expect (ObjNum 605 snap 194 seq_num 605) 2024-04-12T11:23:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:2016: snap_create 2024-04-12T11:23:08.686 INFO:tasks.rados.rados.0.smithi029.stdout:2014: expect (ObjNum 749 snap 258 seq_num 749) 2024-04-12T11:23:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-mon[93650]: pgmap v510: 89 pgs: 3 active+undersized+degraded, 8 stale+active+clean, 12 active+undersized, 66 active+clean; 176 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 4/345 objects degraded (1.159%) 2024-04-12T11:23:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 4/345 objects degraded (1.159%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-mon[93650]: osdmap e584: 8 total, 7 up, 8 in 2024-04-12T11:23:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-mon[97787]: pgmap v510: 89 pgs: 3 active+undersized+degraded, 8 stale+active+clean, 12 active+undersized, 66 active+clean; 176 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 4/345 objects degraded (1.159%) 2024-04-12T11:23:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 4/345 objects degraded (1.159%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-mon[97787]: osdmap e584: 8 total, 7 up, 8 in 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[120703]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[120703]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 bash[120688]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 bash[120688]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 bash[120688]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 bash[120688]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 bash[120688]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 bash[120688]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 bash[120688]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[120703]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[120703]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T11:23:08.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[120703]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-12T11:23:08.770 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[120703]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T11:23:08.770 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[120703]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-12T11:23:08.770 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:08 smithi029 podman[120688]: 2024-04-12 11:23:08.430349798 +0000 UTC m=+0.821818935 container died 7244c9eae71283b0a19f6e266dd07a3922424f833f52c47592b35ea32c07e673 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-12T11:23:08.832 INFO:tasks.rados.rados.0.smithi029.stdout:2015: expect (ObjNum 605 snap 194 seq_num 605) 2024-04-12T11:23:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:08 smithi087 ceph-mon[89014]: pgmap v510: 89 pgs: 3 active+undersized+degraded, 8 stale+active+clean, 12 active+undersized, 66 active+clean; 176 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 4/345 objects degraded (1.159%) 2024-04-12T11:23:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:08 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 4/345 objects degraded (1.159%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:08 smithi087 ceph-mon[89014]: osdmap e584: 8 total, 7 up, 8 in 2024-04-12T11:23:09.238 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:09 smithi029 podman[120825]: 2024-04-12 11:23:09.029305012 +0000 UTC m=+0.593805560 container remove 7244c9eae71283b0a19f6e266dd07a3922424f833f52c47592b35ea32c07e673 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-12T11:23:09.518 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:09 smithi029 podman[120890]: 2024-04-12 11:23:09.141944564 +0000 UTC m=+0.017842448 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:23:09.518 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:09 smithi029 podman[120890]: 2024-04-12 11:23:09.23925801 +0000 UTC m=+0.115155878 container create 1f975ff65c6b1962f6e35ea3d80c12daf0b429ae704ddda7c5fb5af1fc27376d (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , RELEASE=wip-rf-comb1-2cee185, CEPH_POINT_RELEASE=) 2024-04-12T11:23:09.518 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:09 smithi029 podman[120890]: 2024-04-12 11:23:09.461869848 +0000 UTC m=+0.337767714 container init 1f975ff65c6b1962f6e35ea3d80c12daf0b429ae704ddda7c5fb5af1fc27376d (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-rf-comb1-2cee185, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:23:09.518 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:09 smithi029 podman[120890]: 2024-04-12 11:23:09.464958491 +0000 UTC m=+0.340856364 container start 1f975ff65c6b1962f6e35ea3d80c12daf0b429ae704ddda7c5fb5af1fc27376d (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , ceph=True) 2024-04-12T11:23:09.518 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:09 smithi029 ceph-osd[120931]: -- 172.21.15.29:0/3593739727 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5626aa159680 con 0x5626aa122c00 2024-04-12T11:23:09.604 INFO:tasks.rados.rados.0.smithi029.stdout:2014: done (2 left) 2024-04-12T11:23:09.604 INFO:tasks.rados.rados.0.smithi029.stdout:2015: done (1 left) 2024-04-12T11:23:09.604 INFO:tasks.rados.rados.0.smithi029.stdout:2016: done (0 left) 2024-04-12T11:23:09.604 INFO:tasks.rados.rados.0.smithi029.stdout:2017: rollback oid 21 current snap is 261 2024-04-12T11:23:09.604 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 21 to 240 2024-04-12T11:23:09.604 INFO:tasks.rados.rados.0.smithi029.stdout:2018: snap_create 2024-04-12T11:23:09.609 INFO:tasks.rados.rados.0.smithi029.stdout:2017: finishing rollback tid 1 to smithi02979277-21 2024-04-12T11:23:09.609 INFO:tasks.rados.rados.0.smithi029.stdout:2017: finishing rollback tid 2 to smithi02979277-21 2024-04-12T11:23:09.609 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 983 (ObjNum 705 snap 236 seq_num 705) dirty exists 2024-04-12T11:23:10.004 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:09 smithi029 bash[120890]: 1f975ff65c6b1962f6e35ea3d80c12daf0b429ae704ddda7c5fb5af1fc27376d 2024-04-12T11:23:10.004 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:09 smithi029 systemd[1]: Started Ceph osd.3 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:23:10.268 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[120907]: 2024-04-12T11:23:10.003+0000 7fad14a187c0 -1 Falling back to public interface 2024-04-12T11:23:10.605 INFO:tasks.rados.rados.0.smithi029.stdout:2017: done (1 left) 2024-04-12T11:23:10.605 INFO:tasks.rados.rados.0.smithi029.stdout:2018: done (0 left) 2024-04-12T11:23:10.605 INFO:tasks.rados.rados.0.smithi029.stdout:2019: read oid 43 snap -1 2024-04-12T11:23:10.605 INFO:tasks.rados.rados.0.smithi029.stdout:2019: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:23:10.605 INFO:tasks.rados.rados.0.smithi029.stdout:2020: write oid 8 current snap is 262 2024-04-12T11:23:10.605 INFO:tasks.rados.rados.0.smithi029.stdout:2020: seq_num 758 ranges {739706=557689,1806645=467255} 2024-04-12T11:23:10.612 INFO:tasks.rados.rados.0.smithi029.stdout:2020: writing smithi02979277-8 from 739706 to 1297395 tid 1 2024-04-12T11:23:10.618 INFO:tasks.rados.rados.0.smithi029.stdout:2020: writing smithi02979277-8 from 1806645 to 2273900 tid 2 2024-04-12T11:23:10.618 INFO:tasks.rados.rados.0.smithi029.stdout:2021: snap_remove snap 240 2024-04-12T11:23:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[93650]: pgmap v513: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 177 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 12 op/s; 16/357 objects degraded (4.482%) 2024-04-12T11:23:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[93650]: osdmap e585: 8 total, 7 up, 8 in 2024-04-12T11:23:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/478135293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[97787]: pgmap v513: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 177 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 12 op/s; 16/357 objects degraded (4.482%) 2024-04-12T11:23:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[97787]: osdmap e585: 8 total, 7 up, 8 in 2024-04-12T11:23:10.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:10.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/478135293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:10.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:10.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:10 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:10 smithi087 ceph-mon[89014]: pgmap v513: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 177 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 12 op/s; 16/357 objects degraded (4.482%) 2024-04-12T11:23:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:10 smithi087 ceph-mon[89014]: osdmap e585: 8 total, 7 up, 8 in 2024-04-12T11:23:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:10 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/478135293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:10 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:10 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:11.616 INFO:tasks.rados.rados.0.smithi029.stdout:2021: done (2 left) 2024-04-12T11:23:11.617 INFO:tasks.rados.rados.0.smithi029.stdout:2022: write oid 25 current snap is 262 2024-04-12T11:23:11.617 INFO:tasks.rados.rados.0.smithi029.stdout:2022: seq_num 759 ranges {756513=632938,1865127=727266} 2024-04-12T11:23:11.618 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:11 smithi087 ceph-mon[89014]: osdmap e586: 8 total, 7 up, 8 in 2024-04-12T11:23:11.623 INFO:tasks.rados.rados.0.smithi029.stdout:2022: writing smithi02979277-25 from 756513 to 1389451 tid 1 2024-04-12T11:23:11.632 INFO:tasks.rados.rados.0.smithi029.stdout:2022: writing smithi02979277-25 from 1865127 to 2592393 tid 2 2024-04-12T11:23:11.632 INFO:tasks.rados.rados.0.smithi029.stdout:2023: rollback oid 14 current snap is 262 2024-04-12T11:23:11.632 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 255 2024-04-12T11:23:11.632 INFO:tasks.rados.rados.0.smithi029.stdout:2019: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2020: finishing write tid 1 to smithi02979277-8 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2020: finishing write tid 2 to smithi02979277-8 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2019: done (3 left) 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2024: delete oid 7 current snap is 262 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2020: finishing write tid 3 to smithi02979277-8 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2020: finishing write tid 4 to smithi02979277-8 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2020: oid 8 updating version 0 to 194 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2020: oid 8 updating version 194 to 196 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2020: oid 8 version 196 is already newer than 195 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 196 (ObjNum 758 snap 262 seq_num 758) dirty exists 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2020: left oid 8 (ObjNum 758 snap 262 seq_num 758) 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2022: finishing write tid 1 to smithi02979277-25 2024-04-12T11:23:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:2023: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:23:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 0 (ObjNum 129 snap 0 seq_num 1800810880) dirty dne 2024-04-12T11:23:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:2022: finishing write tid 2 to smithi02979277-25 2024-04-12T11:23:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:2022: finishing write tid 3 to smithi02979277-25 2024-04-12T11:23:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:2022: finishing write tid 4 to smithi02979277-25 2024-04-12T11:23:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:2022: oid 25 updating version 0 to 968 2024-04-12T11:23:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:2022: oid 25 updating version 968 to 971 2024-04-12T11:23:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:2022: oid 25 updating version 971 to 972 2024-04-12T11:23:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 972 (ObjNum 759 snap 262 seq_num 759) dirty exists 2024-04-12T11:23:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:2022: left oid 25 (ObjNum 759 snap 262 seq_num 759) 2024-04-12T11:23:11.745 INFO:tasks.rados.rados.0.smithi029.stdout:2020: done (3 left) 2024-04-12T11:23:11.745 INFO:tasks.rados.rados.0.smithi029.stdout:2022: done (2 left) 2024-04-12T11:23:11.745 INFO:tasks.rados.rados.0.smithi029.stdout:2023: done (1 left) 2024-04-12T11:23:11.745 INFO:tasks.rados.rados.0.smithi029.stdout:2024: done (0 left) 2024-04-12T11:23:11.745 INFO:tasks.rados.rados.0.smithi029.stdout:2025: snap_remove snap 260 2024-04-12T11:23:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:11 smithi029 ceph-mon[97787]: osdmap e586: 8 total, 7 up, 8 in 2024-04-12T11:23:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:11 smithi029 ceph-mon[93650]: osdmap e586: 8 total, 7 up, 8 in 2024-04-12T11:23:12.577 INFO:tasks.rados.rados.0.smithi029.stdout:2025: done (0 left) 2024-04-12T11:23:12.577 INFO:tasks.rados.rados.0.smithi029.stdout:2026: write oid 7 current snap is 262 2024-04-12T11:23:12.577 INFO:tasks.rados.rados.0.smithi029.stdout:2026: seq_num 761 ranges {790127=499789,1698444=730935} 2024-04-12T11:23:12.584 INFO:tasks.rados.rados.0.smithi029.stdout:2026: writing smithi02979277-7 from 790127 to 1289916 tid 1 2024-04-12T11:23:12.591 INFO:tasks.rados.rados.0.smithi029.stdout:2026: writing smithi02979277-7 from 1698444 to 2429379 tid 2 2024-04-12T11:23:12.591 INFO:tasks.rados.rados.0.smithi029.stdout:2027: rollback oid 45 current snap is 262 2024-04-12T11:23:12.591 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 45 to 242 2024-04-12T11:23:12.591 INFO:tasks.rados.rados.0.smithi029.stdout:2026: finishing write tid 1 to smithi02979277-7 2024-04-12T11:23:12.591 INFO:tasks.rados.rados.0.smithi029.stdout:2028: read oid 17 snap -1 2024-04-12T11:23:12.591 INFO:tasks.rados.rados.0.smithi029.stdout:2028: expect deleted 2024-04-12T11:23:12.592 INFO:tasks.rados.rados.0.smithi029.stdout:2029: read oid 32 snap -1 2024-04-12T11:23:12.592 INFO:tasks.rados.rados.0.smithi029.stdout:2029: expect (ObjNum 605 snap 194 seq_num 605) 2024-04-12T11:23:12.592 INFO:tasks.rados.rados.0.smithi029.stdout:2030: snap_create 2024-04-12T11:23:12.596 INFO:tasks.rados.rados.0.smithi029.stdout:2029: expect (ObjNum 605 snap 194 seq_num 605) 2024-04-12T11:23:12.613 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:23:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:23:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:23:12.702 INFO:tasks.rados.rados.0.smithi029.stdout:2026: finishing write tid 2 to smithi02979277-7 2024-04-12T11:23:12.702 INFO:tasks.rados.rados.0.smithi029.stdout:2026: finishing write tid 3 to smithi02979277-7 2024-04-12T11:23:12.702 INFO:tasks.rados.rados.0.smithi029.stdout:2026: finishing write tid 4 to smithi02979277-7 2024-04-12T11:23:12.703 INFO:tasks.rados.rados.0.smithi029.stdout:2026: oid 7 updating version 0 to 972 2024-04-12T11:23:12.703 INFO:tasks.rados.rados.0.smithi029.stdout:2026: oid 7 updating version 972 to 973 2024-04-12T11:23:12.703 INFO:tasks.rados.rados.0.smithi029.stdout:2026: oid 7 version 973 is already newer than 971 2024-04-12T11:23:12.703 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 973 (ObjNum 761 snap 262 seq_num 761) dirty exists 2024-04-12T11:23:12.703 INFO:tasks.rados.rados.0.smithi029.stdout:2026: left oid 7 (ObjNum 761 snap 262 seq_num 761) 2024-04-12T11:23:12.703 INFO:tasks.rados.rados.0.smithi029.stdout:2027: finishing rollback tid 0 to smithi02979277-45 2024-04-12T11:23:12.703 INFO:tasks.rados.rados.0.smithi029.stdout:2027: finishing rollback tid 1 to smithi02979277-45 2024-04-12T11:23:12.703 INFO:tasks.rados.rados.0.smithi029.stdout:2027: finishing rollback tid 2 to smithi02979277-45 2024-04-12T11:23:12.703 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 977 (ObjNum 537 snap 173 seq_num 537) dirty exists 2024-04-12T11:23:12.866 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:12 smithi087 ceph-mon[89014]: pgmap v516: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 177 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 584 KiB/s wr, 6 op/s; 16/357 objects degraded (4.482%) 2024-04-12T11:23:12.866 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:12 smithi087 ceph-mon[89014]: osdmap e587: 8 total, 7 up, 8 in 2024-04-12T11:23:12.866 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:12 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2976235574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:12.866 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:12 smithi087 ceph-mon[89014]: osdmap e588: 8 total, 7 up, 8 in 2024-04-12T11:23:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:12 smithi029 ceph-mon[93650]: pgmap v516: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 177 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 584 KiB/s wr, 6 op/s; 16/357 objects degraded (4.482%) 2024-04-12T11:23:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:12 smithi029 ceph-mon[93650]: osdmap e587: 8 total, 7 up, 8 in 2024-04-12T11:23:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:12 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2976235574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:12 smithi029 ceph-mon[93650]: osdmap e588: 8 total, 7 up, 8 in 2024-04-12T11:23:13.018 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:12 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[120907]: 2024-04-12T11:23:12.787+0000 7fad14a187c0 -1 osd.3 579 log_to_monitors true 2024-04-12T11:23:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:12 smithi029 ceph-mon[97787]: pgmap v516: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 177 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 584 KiB/s wr, 6 op/s; 16/357 objects degraded (4.482%) 2024-04-12T11:23:13.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:12 smithi029 ceph-mon[97787]: osdmap e587: 8 total, 7 up, 8 in 2024-04-12T11:23:13.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:12 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2976235574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:13.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:12 smithi029 ceph-mon[97787]: osdmap e588: 8 total, 7 up, 8 in 2024-04-12T11:23:13.580 INFO:tasks.rados.rados.0.smithi029.stdout:2026: done (4 left) 2024-04-12T11:23:13.580 INFO:tasks.rados.rados.0.smithi029.stdout:2027: done (3 left) 2024-04-12T11:23:13.580 INFO:tasks.rados.rados.0.smithi029.stdout:2028: done (2 left) 2024-04-12T11:23:13.580 INFO:tasks.rados.rados.0.smithi029.stdout:2029: done (1 left) 2024-04-12T11:23:13.580 INFO:tasks.rados.rados.0.smithi029.stdout:2030: done (0 left) 2024-04-12T11:23:13.580 INFO:tasks.rados.rados.0.smithi029.stdout:2031: write oid 20 current snap is 263 2024-04-12T11:23:13.580 INFO:tasks.rados.rados.0.smithi029.stdout:2031: seq_num 762 ranges {406934=575038,1640573=474593,2642791=164143} 2024-04-12T11:23:13.585 INFO:tasks.rados.rados.0.smithi029.stdout:2031: writing smithi02979277-20 from 406934 to 981972 tid 1 2024-04-12T11:23:13.593 INFO:tasks.rados.rados.0.smithi029.stdout:2031: writing smithi02979277-20 from 1640573 to 2115166 tid 2 2024-04-12T11:23:13.598 INFO:tasks.rados.rados.0.smithi029.stdout:2031: writing smithi02979277-20 from 2642791 to 2806934 tid 3 2024-04-12T11:23:13.598 INFO:tasks.rados.rados.0.smithi029.stdout:2032: write oid 36 current snap is 263 2024-04-12T11:23:13.598 INFO:tasks.rados.rados.0.smithi029.stdout:2031: finishing write tid 1 to smithi02979277-20 2024-04-12T11:23:13.598 INFO:tasks.rados.rados.0.smithi029.stdout:2031: finishing write tid 2 to smithi02979277-20 2024-04-12T11:23:13.598 INFO:tasks.rados.rados.0.smithi029.stdout:2032: seq_num 763 ranges {423741=650287,1699055=618251} 2024-04-12T11:23:13.603 INFO:tasks.rados.rados.0.smithi029.stdout:2032: writing smithi02979277-36 from 423741 to 1074028 tid 1 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:2032: writing smithi02979277-36 from 1699055 to 2317306 tid 2 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:2033: delete oid 2 current snap is 263 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:2031: finishing write tid 3 to smithi02979277-20 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:2031: finishing write tid 4 to smithi02979277-20 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:2031: finishing write tid 5 to smithi02979277-20 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:2031: oid 20 updating version 0 to 593 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:2031: oid 20 updating version 593 to 594 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:2031: oid 20 updating version 594 to 595 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:2031: oid 20 version 595 is already newer than 592 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 595 (ObjNum 762 snap 263 seq_num 762) dirty exists 2024-04-12T11:23:13.611 INFO:tasks.rados.rados.0.smithi029.stdout:2031: left oid 20 (ObjNum 762 snap 263 seq_num 762) 2024-04-12T11:23:13.612 INFO:tasks.rados.rados.0.smithi029.stdout:2032: finishing write tid 1 to smithi02979277-36 2024-04-12T11:23:13.612 INFO:tasks.rados.rados.0.smithi029.stdout:2031: done (2 left) 2024-04-12T11:23:13.612 INFO:tasks.rados.rados.0.smithi029.stdout:2033: done (1 left) 2024-04-12T11:23:13.612 INFO:tasks.rados.rados.0.smithi029.stdout:2034: read oid 20 snap -1 2024-04-12T11:23:13.612 INFO:tasks.rados.rados.0.smithi029.stdout:2034: expect (ObjNum 762 snap 263 seq_num 762) 2024-04-12T11:23:13.612 INFO:tasks.rados.rados.0.smithi029.stdout:2035: snap_create 2024-04-12T11:23:13.614 INFO:tasks.rados.rados.0.smithi029.stdout:2032: finishing write tid 2 to smithi02979277-36 2024-04-12T11:23:13.614 INFO:tasks.rados.rados.0.smithi029.stdout:2032: finishing write tid 3 to smithi02979277-36 2024-04-12T11:23:13.614 INFO:tasks.rados.rados.0.smithi029.stdout:2032: finishing write tid 4 to smithi02979277-36 2024-04-12T11:23:13.614 INFO:tasks.rados.rados.0.smithi029.stdout:2032: oid 36 updating version 0 to 486 2024-04-12T11:23:13.614 INFO:tasks.rados.rados.0.smithi029.stdout:2032: oid 36 updating version 486 to 487 2024-04-12T11:23:13.614 INFO:tasks.rados.rados.0.smithi029.stdout:2032: oid 36 updating version 487 to 488 2024-04-12T11:23:13.614 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 488 (ObjNum 763 snap 263 seq_num 763) dirty exists 2024-04-12T11:23:13.615 INFO:tasks.rados.rados.0.smithi029.stdout:2032: left oid 36 (ObjNum 763 snap 263 seq_num 763) 2024-04-12T11:23:13.617 INFO:tasks.rados.rados.0.smithi029.stdout:2034: expect (ObjNum 762 snap 263 seq_num 762) 2024-04-12T11:23:13.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:13 smithi087 ceph-mon[89014]: from='osd.3 [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:23:13.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:13 smithi087 ceph-mon[89014]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:23:13.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:13 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:13.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:13 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:13 smithi087 ceph-mon[89014]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T11:23:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:13 smithi087 ceph-mon[89014]: osdmap e589: 8 total, 7 up, 8 in 2024-04-12T11:23:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:13 smithi087 ceph-mon[89014]: from='osd.3 [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:23:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:13 smithi087 ceph-mon[89014]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:23:14.018 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[120907]: 2024-04-12T11:23:13.603+0000 7fad01e68700 -1 osd.3 579 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:23:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[93650]: from='osd.3 [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:23:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[93650]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:23:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[93650]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[93650]: osdmap e589: 8 total, 7 up, 8 in 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[93650]: from='osd.3 [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[93650]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[97787]: from='osd.3 [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[97787]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[97787]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[97787]: osdmap e589: 8 total, 7 up, 8 in 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[97787]: from='osd.3 [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:23:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:13 smithi029 ceph-mon[97787]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T11:23:14.587 INFO:tasks.rados.rados.0.smithi029.stdout:2032: done (2 left) 2024-04-12T11:23:14.587 INFO:tasks.rados.rados.0.smithi029.stdout:2034: done (1 left) 2024-04-12T11:23:14.587 INFO:tasks.rados.rados.0.smithi029.stdout:2035: done (0 left) 2024-04-12T11:23:14.587 INFO:tasks.rados.rados.0.smithi029.stdout:2036: read oid 22 snap -1 2024-04-12T11:23:14.587 INFO:tasks.rados.rados.0.smithi029.stdout:2036: expect (ObjNum 720 snap 248 seq_num 720) 2024-04-12T11:23:14.587 INFO:tasks.rados.rados.0.smithi029.stdout:2037: delete oid 41 current snap is 264 2024-04-12T11:23:14.590 INFO:tasks.rados.rados.0.smithi029.stdout:2037: done (1 left) 2024-04-12T11:23:14.590 INFO:tasks.rados.rados.0.smithi029.stdout:2038: snap_create 2024-04-12T11:23:14.592 INFO:tasks.rados.rados.0.smithi029.stdout:2036: expect (ObjNum 720 snap 248 seq_num 720) 2024-04-12T11:23:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[97787]: pgmap v519: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 187 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.2 MiB/s wr, 10 op/s; 16/366 objects degraded (4.372%) 2024-04-12T11:23:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16/366 objects degraded (4.372%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1799512571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[97787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:23:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[97787]: osd.3 [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308] boot 2024-04-12T11:23:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[97787]: osdmap e590: 8 total, 8 up, 8 in 2024-04-12T11:23:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:23:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[93650]: pgmap v519: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 187 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.2 MiB/s wr, 10 op/s; 16/366 objects degraded (4.372%) 2024-04-12T11:23:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16/366 objects degraded (4.372%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1799512571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[93650]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:23:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[93650]: osd.3 [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308] boot 2024-04-12T11:23:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[93650]: osdmap e590: 8 total, 8 up, 8 in 2024-04-12T11:23:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:23:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:14 smithi087 ceph-mon[89014]: pgmap v519: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 187 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.2 MiB/s wr, 10 op/s; 16/366 objects degraded (4.372%) 2024-04-12T11:23:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:14 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16/366 objects degraded (4.372%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:14 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1799512571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:14 smithi087 ceph-mon[89014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:23:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:14 smithi087 ceph-mon[89014]: osd.3 [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308] boot 2024-04-12T11:23:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:14 smithi087 ceph-mon[89014]: osdmap e590: 8 total, 8 up, 8 in 2024-04-12T11:23:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T11:23:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:23:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:23:14] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.43.0" 2024-04-12T11:23:15.584 INFO:tasks.rados.rados.0.smithi029.stdout:2036: done (1 left) 2024-04-12T11:23:15.584 INFO:tasks.rados.rados.0.smithi029.stdout:2038: done (0 left) 2024-04-12T11:23:15.584 INFO:tasks.rados.rados.0.smithi029.stdout:2039: read oid 25 snap -1 2024-04-12T11:23:15.585 INFO:tasks.rados.rados.0.smithi029.stdout:2039: expect (ObjNum 759 snap 262 seq_num 759) 2024-04-12T11:23:15.585 INFO:tasks.rados.rados.0.smithi029.stdout:2040: read oid 11 snap -1 2024-04-12T11:23:15.585 INFO:tasks.rados.rados.0.smithi029.stdout:2040: expect deleted 2024-04-12T11:23:15.585 INFO:tasks.rados.rados.0.smithi029.stdout:2041: snap_remove snap 258 2024-04-12T11:23:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:15 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:15 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:15.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:15 smithi087 ceph-mon[89014]: osdmap e591: 8 total, 8 up, 8 in 2024-04-12T11:23:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:15 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:15 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:15 smithi029 ceph-mon[93650]: osdmap e591: 8 total, 8 up, 8 in 2024-04-12T11:23:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:15 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:15 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:15 smithi029 ceph-mon[97787]: osdmap e591: 8 total, 8 up, 8 in 2024-04-12T11:23:16.584 INFO:tasks.rados.rados.0.smithi029.stdout:2041: done (2 left) 2024-04-12T11:23:16.584 INFO:tasks.rados.rados.0.smithi029.stdout:2042: read oid 31 snap -1 2024-04-12T11:23:16.584 INFO:tasks.rados.rados.0.smithi029.stdout:2042: expect deleted 2024-04-12T11:23:16.584 INFO:tasks.rados.rados.0.smithi029.stdout:2043: read oid 21 snap -1 2024-04-12T11:23:16.584 INFO:tasks.rados.rados.0.smithi029.stdout:2043: expect (ObjNum 705 snap 236 seq_num 705) 2024-04-12T11:23:16.584 INFO:tasks.rados.rados.0.smithi029.stdout:2044: read oid 47 snap -1 2024-04-12T11:23:16.585 INFO:tasks.rados.rados.0.smithi029.stdout:2044: expect (ObjNum 658 snap 212 seq_num 658) 2024-04-12T11:23:16.585 INFO:tasks.rados.rados.0.smithi029.stdout:2045: delete oid 42 current snap is 265 2024-04-12T11:23:16.585 INFO:tasks.rados.rados.0.smithi029.stdout:2039: expect (ObjNum 759 snap 262 seq_num 759) 2024-04-12T11:23:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev)": 4 2024-04-12T11:23:16.759 INFO:tasks.rados.rados.0.smithi029.stdout:2044: expect (ObjNum 658 snap 212 seq_num 658) 2024-04-12T11:23:16.905 INFO:tasks.rados.rados.0.smithi029.stdout:2043: expect (ObjNum 705 snap 236 seq_num 705) 2024-04-12T11:23:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:16 smithi087 ceph-mon[89014]: pgmap v522: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 189 MiB data, 674 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 17 op/s; 16/369 objects degraded (4.336%) 2024-04-12T11:23:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:16 smithi087 ceph-mon[89014]: osdmap e592: 8 total, 8 up, 8 in 2024-04-12T11:23:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:16 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/647594610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:17.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:16 smithi029 ceph-mon[97787]: pgmap v522: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 189 MiB data, 674 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 17 op/s; 16/369 objects degraded (4.336%) 2024-04-12T11:23:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:16 smithi029 ceph-mon[97787]: osdmap e592: 8 total, 8 up, 8 in 2024-04-12T11:23:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:16 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/647594610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:16 smithi029 ceph-mon[93650]: pgmap v522: 89 pgs: 6 active+undersized+degraded, 28 active+undersized, 55 active+clean; 189 MiB data, 674 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 17 op/s; 16/369 objects degraded (4.336%) 2024-04-12T11:23:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:16 smithi029 ceph-mon[93650]: osdmap e592: 8 total, 8 up, 8 in 2024-04-12T11:23:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:16 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/647594610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-12T11:23:17.082 INFO:tasks.rados.rados.0.smithi029.stdout:2039: done (5 left) 2024-04-12T11:23:17.082 INFO:tasks.rados.rados.0.smithi029.stdout:2040: done (4 left) 2024-04-12T11:23:17.082 INFO:tasks.rados.rados.0.smithi029.stdout:2042: done (3 left) 2024-04-12T11:23:17.082 INFO:tasks.rados.rados.0.smithi029.stdout:2043: done (2 left) 2024-04-12T11:23:17.083 INFO:tasks.rados.rados.0.smithi029.stdout:2044: done (1 left) 2024-04-12T11:23:17.083 INFO:tasks.rados.rados.0.smithi029.stdout:2045: done (0 left) 2024-04-12T11:23:17.083 INFO:tasks.rados.rados.0.smithi029.stdout:2046: write oid 46 current snap is 265 2024-04-12T11:23:17.083 INFO:tasks.rados.rados.0.smithi029.stdout:2046: seq_num 767 ranges {490969=551283,1816630=625589} 2024-04-12T11:23:17.088 INFO:tasks.rados.rados.0.smithi029.stdout:2046: writing smithi02979277-46 from 490969 to 1042252 tid 1 2024-04-12T11:23:17.096 INFO:tasks.rados.rados.0.smithi029.stdout:2046: writing smithi02979277-46 from 1816630 to 2442219 tid 2 2024-04-12T11:23:17.096 INFO:tasks.rados.rados.0.smithi029.stdout:2047: read oid 2 snap 249 2024-04-12T11:23:17.096 INFO:tasks.rados.rados.0.smithi029.stdout:2047: expect (ObjNum 721 snap 248 seq_num 721) 2024-04-12T11:23:17.096 INFO:tasks.rados.rados.0.smithi029.stdout:2048: snap_create 2024-04-12T11:23:17.096 INFO:tasks.rados.rados.0.smithi029.stdout:2046: finishing write tid 1 to smithi02979277-46 2024-04-12T11:23:17.099 INFO:tasks.rados.rados.0.smithi029.stdout:2046: finishing write tid 2 to smithi02979277-46 2024-04-12T11:23:17.099 INFO:tasks.rados.rados.0.smithi029.stdout:2047: expect (ObjNum 721 snap 248 seq_num 721) 2024-04-12T11:23:17.253 INFO:tasks.rados.rados.0.smithi029.stdout:2046: finishing write tid 3 to smithi02979277-46 2024-04-12T11:23:17.253 INFO:tasks.rados.rados.0.smithi029.stdout:2046: finishing write tid 4 to smithi02979277-46 2024-04-12T11:23:17.253 INFO:tasks.rados.rados.0.smithi029.stdout:2046: oid 46 updating version 0 to 599 2024-04-12T11:23:17.253 INFO:tasks.rados.rados.0.smithi029.stdout:2046: oid 46 version 599 is already newer than 598 2024-04-12T11:23:17.253 INFO:tasks.rados.rados.0.smithi029.stdout:2046: oid 46 version 599 is already newer than 597 2024-04-12T11:23:17.253 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 599 (ObjNum 767 snap 265 seq_num 767) dirty exists 2024-04-12T11:23:17.254 INFO:tasks.rados.rados.0.smithi029.stdout:2046: left oid 46 (ObjNum 767 snap 265 seq_num 767) 2024-04-12T11:23:17.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch upgrade pause' 2024-04-12T11:23:17.697 INFO:tasks.rados.rados.0.smithi029.stdout:2046: done (2 left) 2024-04-12T11:23:17.697 INFO:tasks.rados.rados.0.smithi029.stdout:2047: done (1 left) 2024-04-12T11:23:17.698 INFO:tasks.rados.rados.0.smithi029.stdout:2048: done (0 left) 2024-04-12T11:23:17.698 INFO:tasks.rados.rados.0.smithi029.stdout:2049: snap_remove snap 262 2024-04-12T11:23:17.799 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:23:18.702 INFO:tasks.rados.rados.0.smithi029.stdout:2049: done (0 left) 2024-04-12T11:23:18.703 INFO:tasks.rados.rados.0.smithi029.stdout:2050: rollback oid 10 current snap is 266 2024-04-12T11:23:18.703 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 10 to 259 2024-04-12T11:23:18.703 INFO:tasks.rados.rados.0.smithi029.stdout:2051: read oid 44 snap 222 2024-04-12T11:23:18.703 INFO:tasks.rados.rados.0.smithi029.stdout:2051: expect deleted 2024-04-12T11:23:18.703 INFO:tasks.rados.rados.0.smithi029.stdout:2052: rollback oid 43 current snap is 266 2024-04-12T11:23:18.703 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 254 2024-04-12T11:23:18.703 INFO:tasks.rados.rados.0.smithi029.stdout:2053: snap_create 2024-04-12T11:23:18.709 INFO:tasks.rados.rados.0.smithi029.stdout:2050: finishing rollback tid 1 to smithi02979277-10 2024-04-12T11:23:18.709 INFO:tasks.rados.rados.0.smithi029.stdout:2050: finishing rollback tid 2 to smithi02979277-10 2024-04-12T11:23:18.709 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 821 (ObjNum 635 snap 205 seq_num 635) dirty exists 2024-04-12T11:23:18.710 INFO:tasks.rados.rados.0.smithi029.stdout:2052: finishing rollback tid 0 to smithi02979277-43 2024-04-12T11:23:18.710 INFO:tasks.rados.rados.0.smithi029.stdout:2052: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:23:18.710 INFO:tasks.rados.rados.0.smithi029.stdout:2052: finishing rollback tid 2 to smithi02979277-43 2024-04-12T11:23:18.710 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 756 (ObjNum 427 snap 135 seq_num 427) dirty exists 2024-04-12T11:23:18.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:18 smithi029 ceph-mon[97787]: pgmap v525: 89 pgs: 4 active+undersized+degraded, 21 active+undersized, 64 active+clean; 191 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 607 KiB/s wr, 15 op/s; 13/378 objects degraded (3.439%); 347 KiB/s, 0 objects/s recovering 2024-04-12T11:23:18.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:18 smithi029 ceph-mon[97787]: osdmap e593: 8 total, 8 up, 8 in 2024-04-12T11:23:18.850 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:18 smithi029 ceph-mon[93650]: pgmap v525: 89 pgs: 4 active+undersized+degraded, 21 active+undersized, 64 active+clean; 191 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 607 KiB/s wr, 15 op/s; 13/378 objects degraded (3.439%); 347 KiB/s, 0 objects/s recovering 2024-04-12T11:23:18.850 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:18 smithi029 ceph-mon[93650]: osdmap e593: 8 total, 8 up, 8 in 2024-04-12T11:23:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:18 smithi087 ceph-mon[89014]: pgmap v525: 89 pgs: 4 active+undersized+degraded, 21 active+undersized, 64 active+clean; 191 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 607 KiB/s wr, 15 op/s; 13/378 objects degraded (3.439%); 347 KiB/s, 0 objects/s recovering 2024-04-12T11:23:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:18 smithi087 ceph-mon[89014]: osdmap e593: 8 total, 8 up, 8 in 2024-04-12T11:23:19.707 INFO:tasks.rados.rados.0.smithi029.stdout:2050: done (3 left) 2024-04-12T11:23:19.707 INFO:tasks.rados.rados.0.smithi029.stdout:2051: done (2 left) 2024-04-12T11:23:19.707 INFO:tasks.rados.rados.0.smithi029.stdout:2052: done (1 left) 2024-04-12T11:23:19.707 INFO:tasks.rados.rados.0.smithi029.stdout:2053: done (0 left) 2024-04-12T11:23:19.707 INFO:tasks.rados.rados.0.smithi029.stdout:2054: snap_remove snap 265 2024-04-12T11:23:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:19 smithi087 ceph-mon[89014]: osdmap e594: 8 total, 8 up, 8 in 2024-04-12T11:23:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:23:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:20.006 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[93650]: osdmap e594: 8 total, 8 up, 8 in 2024-04-12T11:23:20.006 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:20.007 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:20.007 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:20.007 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:23:20.007 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:20.007 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[97787]: osdmap e594: 8 total, 8 up, 8 in 2024-04-12T11:23:20.007 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:20.007 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:20.007 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:20.007 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:23:20.007 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:20.350 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:23:20.712 INFO:tasks.rados.rados.0.smithi029.stdout:2054: done (0 left) 2024-04-12T11:23:20.712 INFO:tasks.rados.rados.0.smithi029.stdout:2055: write oid 3 current snap is 267 2024-04-12T11:23:20.712 INFO:tasks.rados.rados.0.smithi029.stdout:2055: seq_num 768 ranges {507776=742885,1991465=485600} 2024-04-12T11:23:20.719 INFO:tasks.rados.rados.0.smithi029.stdout:2055: writing smithi02979277-3 from 507776 to 1250661 tid 1 2024-04-12T11:23:20.725 INFO:tasks.rados.rados.0.smithi029.stdout:2055: writing smithi02979277-3 from 1991465 to 2477065 tid 2 2024-04-12T11:23:20.726 INFO:tasks.rados.rados.0.smithi029.stdout:2055: finishing write tid 1 to smithi02979277-3 2024-04-12T11:23:20.726 INFO:tasks.rados.rados.0.smithi029.stdout:2056: snap_create 2024-04-12T11:23:20.729 INFO:tasks.rados.rados.0.smithi029.stdout:2055: finishing write tid 2 to smithi02979277-3 2024-04-12T11:23:20.729 INFO:tasks.rados.rados.0.smithi029.stdout:2055: finishing write tid 3 to smithi02979277-3 2024-04-12T11:23:20.729 INFO:tasks.rados.rados.0.smithi029.stdout:2055: finishing write tid 4 to smithi02979277-3 2024-04-12T11:23:20.729 INFO:tasks.rados.rados.0.smithi029.stdout:2055: oid 3 updating version 0 to 1253 2024-04-12T11:23:20.729 INFO:tasks.rados.rados.0.smithi029.stdout:2055: oid 3 updating version 1253 to 1254 2024-04-12T11:23:20.729 INFO:tasks.rados.rados.0.smithi029.stdout:2055: oid 3 version 1254 is already newer than 1252 2024-04-12T11:23:20.729 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1254 (ObjNum 768 snap 267 seq_num 768) dirty exists 2024-04-12T11:23:20.729 INFO:tasks.rados.rados.0.smithi029.stdout:2055: left oid 3 (ObjNum 768 snap 267 seq_num 768) 2024-04-12T11:23:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: pgmap v528: 89 pgs: 89 active+clean; 190 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 597 KiB/s wr, 15 op/s; 1.1 MiB/s, 1 objects/s recovering 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/378 objects degraded (3.439%), 4 pgs degraded) 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: osdmap e595: 8 total, 8 up, 8 in 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: Upgrade: osd.4 is safe to restart 2024-04-12T11:23:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: pgmap v528: 89 pgs: 89 active+clean; 190 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 597 KiB/s wr, 15 op/s; 1.1 MiB/s, 1 objects/s recovering 2024-04-12T11:23:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/378 objects degraded (3.439%), 4 pgs degraded) 2024-04-12T11:23:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:23:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: osdmap e595: 8 total, 8 up, 8 in 2024-04-12T11:23:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T11:23:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T11:23:21.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: Upgrade: osd.4 is safe to restart 2024-04-12T11:23:21.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: pgmap v528: 89 pgs: 89 active+clean; 190 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 597 KiB/s wr, 15 op/s; 1.1 MiB/s, 1 objects/s recovering 2024-04-12T11:23:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/378 objects degraded (3.439%), 4 pgs degraded) 2024-04-12T11:23:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:23:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: osdmap e595: 8 total, 8 up, 8 in 2024-04-12T11:23:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:21.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T11:23:21.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T11:23:21.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: Upgrade: osd.4 is safe to restart 2024-04-12T11:23:21.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:21.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch ps' 2024-04-12T11:23:21.403 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:23:21.717 INFO:tasks.rados.rados.0.smithi029.stdout:2055: done (1 left) 2024-04-12T11:23:21.717 INFO:tasks.rados.rados.0.smithi029.stdout:2056: done (0 left) 2024-04-12T11:23:21.717 INFO:tasks.rados.rados.0.smithi029.stdout:2057: delete oid 20 current snap is 268 2024-04-12T11:23:21.719 INFO:tasks.rados.rados.0.smithi029.stdout:2057: done (0 left) 2024-04-12T11:23:21.719 INFO:tasks.rados.rados.0.smithi029.stdout:2058: rollback oid 21 current snap is 268 2024-04-12T11:23:21.720 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 21 to 222 2024-04-12T11:23:21.720 INFO:tasks.rados.rados.0.smithi029.stdout:2059: snap_remove snap 248 2024-04-12T11:23:21.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[97787]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:23:21.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[97787]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:23:21.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[97787]: osdmap e596: 8 total, 8 up, 8 in 2024-04-12T11:23:21.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:21.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:21.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:21.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T11:23:21.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:21.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[93650]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:23:21.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[93650]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:23:21.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[93650]: osdmap e596: 8 total, 8 up, 8 in 2024-04-12T11:23:21.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:21.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:21.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:21.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T11:23:21.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:21 smithi087 ceph-mon[89014]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:23:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:21 smithi087 ceph-mon[89014]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:23:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:21 smithi087 ceph-mon[89014]: osdmap e596: 8 total, 8 up, 8 in 2024-04-12T11:23:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T11:23:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:22.579 INFO:tasks.rados.rados.0.smithi029.stdout:2059: done (1 left) 2024-04-12T11:23:22.579 INFO:tasks.rados.rados.0.smithi029.stdout:2060: snap_create 2024-04-12T11:23:22.580 INFO:tasks.rados.rados.0.smithi029.stdout:2058: finishing rollback tid 0 to smithi02979277-21 2024-04-12T11:23:22.580 INFO:tasks.rados.rados.0.smithi029.stdout:2058: finishing rollback tid 1 to smithi02979277-21 2024-04-12T11:23:22.580 INFO:tasks.rados.rados.0.smithi029.stdout:2058: finishing rollback tid 2 to smithi02979277-21 2024-04-12T11:23:22.580 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1001 (ObjNum 672 snap 215 seq_num 672) dirty exists 2024-04-12T11:23:22.708 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:23:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:23:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:23:22.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:22 smithi029 ceph-mon[97787]: pgmap v531: 89 pgs: 89 active+clean; 190 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 287 KiB/s wr, 7 op/s; 758 KiB/s, 0 objects/s recovering 2024-04-12T11:23:22.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:22 smithi029 ceph-mon[97787]: Upgrade: Updating osd.4 2024-04-12T11:23:22.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:22 smithi029 ceph-mon[97787]: Deploying daemon osd.4 on smithi087 2024-04-12T11:23:22.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:22 smithi029 ceph-mon[97787]: osdmap e597: 8 total, 8 up, 8 in 2024-04-12T11:23:22.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:22 smithi029 ceph-mon[97787]: osdmap e598: 8 total, 8 up, 8 in 2024-04-12T11:23:22.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:22 smithi029 ceph-mon[93650]: pgmap v531: 89 pgs: 89 active+clean; 190 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 287 KiB/s wr, 7 op/s; 758 KiB/s, 0 objects/s recovering 2024-04-12T11:23:22.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:22 smithi029 ceph-mon[93650]: Upgrade: Updating osd.4 2024-04-12T11:23:22.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:22 smithi029 ceph-mon[93650]: Deploying daemon osd.4 on smithi087 2024-04-12T11:23:22.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:22 smithi029 ceph-mon[93650]: osdmap e597: 8 total, 8 up, 8 in 2024-04-12T11:23:22.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:22 smithi029 ceph-mon[93650]: osdmap e598: 8 total, 8 up, 8 in 2024-04-12T11:23:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:22 smithi087 ceph-mon[89014]: pgmap v531: 89 pgs: 89 active+clean; 190 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 287 KiB/s wr, 7 op/s; 758 KiB/s, 0 objects/s recovering 2024-04-12T11:23:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:22 smithi087 ceph-mon[89014]: Upgrade: Updating osd.4 2024-04-12T11:23:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:22 smithi087 ceph-mon[89014]: Deploying daemon osd.4 on smithi087 2024-04-12T11:23:22.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:22 smithi087 ceph-mon[89014]: osdmap e597: 8 total, 8 up, 8 in 2024-04-12T11:23:22.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:22 smithi087 ceph-mon[89014]: osdmap e598: 8 total, 8 up, 8 in 2024-04-12T11:23:23.458 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T11:23:23.458 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (9m) 9s ago 10m 25.2M - 0.25.0 c8568f914cd2 5f36b9c37cec 2024-04-12T11:23:23.458 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi087 *:3000 running (10m) 9s ago 10m 86.1M - 9.4.7 954c08fa6188 d4ba4d587fdb 2024-04-12T11:23:23.458 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.scbuux smithi029 running (10m) 9s ago 10m 19.8M - 18.2.2-1187-gd540ebac bb2c5eac26a4 a659ce0bccaf 2024-04-12T11:23:23.458 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi087.mbflzu smithi087 running (10m) 9s ago 10m 18.4M - 18.2.2-1187-gd540ebac bb2c5eac26a4 64bafd1e6a4c 2024-04-12T11:23:23.458 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi087 *:8443,9283,8765 running (6m) 9s ago 15m 452M - 19.0.0-2874-g2cee1857 4a0e59462345 5a46135dfdbc 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (6m) 9s ago 17m 509M - 19.0.0-2874-g2cee1857 4a0e59462345 05c187489262 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (6m) 9s ago 17m 65.1M 2048M 19.0.0-2874-g2cee1857 4a0e59462345 d1506def0998 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi087 running (3m) 9s ago 15m 44.1M 2048M 19.0.0-2874-g2cee1857 4a0e59462345 ae22114389c9 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (4m) 9s ago 15m 53.1M 2048M 19.0.0-2874-g2cee1857 4a0e59462345 24b3f58e1d66 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (10m) 9s ago 10m 15.6M - 1.5.0 0da6a335fe13 32124106e912 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi087 *:9100 running (10m) 9s ago 10m 16.0M - 1.5.0 0da6a335fe13 78910b2d30c6 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (90s) 9s ago 14m 66.2M 2481M 19.0.0-2874-g2cee1857 4a0e59462345 78ce9733fe38 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (69s) 9s ago 14m 54.9M 2481M 19.0.0-2874-g2cee1857 4a0e59462345 260ced18c216 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (49s) 9s ago 14m 67.2M 2481M 19.0.0-2874-g2cee1857 4a0e59462345 db6a92e3e974 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (14s) 9s ago 13m 12.7M 2481M 19.0.0-2874-g2cee1857 4a0e59462345 1f975ff65c6b 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi087 running (13m) 9s ago 13m 104M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 ab3a8e864afd 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi087 running (12m) 9s ago 12m 119M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 573e2f90e4da 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi087 running (12m) 9s ago 12m 84.4M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 2b67d3f477dd 2024-04-12T11:23:23.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi087 running (12m) 9s ago 12m 106M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 9c484b9f0790 2024-04-12T11:23:23.460 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi087 *:9095 running (6m) 9s ago 11m 60.7M - 2.43.0 a07b618ecd1d 569140fc5a53 2024-04-12T11:23:23.580 INFO:tasks.rados.rados.0.smithi029.stdout:2058: done (1 left) 2024-04-12T11:23:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:2060: done (0 left) 2024-04-12T11:23:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:2061: write oid 8 current snap is 269 2024-04-12T11:23:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:2061: seq_num 770 ranges {541390=493383,1592076=489269,2862175=79215} 2024-04-12T11:23:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:2061: writing smithi02979277-8 from 541390 to 1034773 tid 1 2024-04-12T11:23:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:2061: writing smithi02979277-8 from 1592076 to 2081345 tid 2 2024-04-12T11:23:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:2061: writing smithi02979277-8 from 2862175 to 2941390 tid 3 2024-04-12T11:23:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:2062: snap_create 2024-04-12T11:23:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:2061: finishing write tid 1 to smithi02979277-8 2024-04-12T11:23:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:2061: finishing write tid 2 to smithi02979277-8 2024-04-12T11:23:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:2061: finishing write tid 3 to smithi02979277-8 2024-04-12T11:23:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:2061: finishing write tid 4 to smithi02979277-8 2024-04-12T11:23:23.600 INFO:tasks.rados.rados.0.smithi029.stdout:2061: finishing write tid 5 to smithi02979277-8 2024-04-12T11:23:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:2061: oid 8 updating version 0 to 207 2024-04-12T11:23:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:2061: oid 8 version 207 is already newer than 206 2024-04-12T11:23:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:2061: oid 8 version 207 is already newer than 205 2024-04-12T11:23:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:2061: oid 8 version 207 is already newer than 204 2024-04-12T11:23:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 207 (ObjNum 770 snap 269 seq_num 770) dirty exists 2024-04-12T11:23:23.601 INFO:tasks.rados.rados.0.smithi029.stdout:2061: left oid 8 (ObjNum 770 snap 269 seq_num 770) 2024-04-12T11:23:24.185 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph orch ps' 2024-04-12T11:23:24.395 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:23:24.586 INFO:tasks.rados.rados.0.smithi029.stdout:2061: done (1 left) 2024-04-12T11:23:24.587 INFO:tasks.rados.rados.0.smithi029.stdout:2062: done (0 left) 2024-04-12T11:23:24.587 INFO:tasks.rados.rados.0.smithi029.stdout:2063: write oid 10 current snap is 270 2024-04-12T11:23:24.587 INFO:tasks.rados.rados.0.smithi029.stdout:2063: seq_num 771 ranges {558197=568632,1650558=632927,2773245=184952} 2024-04-12T11:23:24.592 INFO:tasks.rados.rados.0.smithi029.stdout:2063: writing smithi02979277-10 from 558197 to 1126829 tid 1 2024-04-12T11:23:24.599 INFO:tasks.rados.rados.0.smithi029.stdout:2063: writing smithi02979277-10 from 1650558 to 2283485 tid 2 2024-04-12T11:23:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:2063: writing smithi02979277-10 from 2773245 to 2958197 tid 3 2024-04-12T11:23:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:2064: delete oid 47 current snap is 270 2024-04-12T11:23:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:2063: finishing write tid 1 to smithi02979277-10 2024-04-12T11:23:24.603 INFO:tasks.rados.rados.0.smithi029.stdout:2063: finishing write tid 2 to smithi02979277-10 2024-04-12T11:23:24.605 INFO:tasks.rados.rados.0.smithi029.stdout:2063: finishing write tid 3 to smithi02979277-10 2024-04-12T11:23:24.605 INFO:tasks.rados.rados.0.smithi029.stdout:2063: finishing write tid 4 to smithi02979277-10 2024-04-12T11:23:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2063: finishing write tid 5 to smithi02979277-10 2024-04-12T11:23:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2063: oid 10 updating version 0 to 827 2024-04-12T11:23:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2063: oid 10 updating version 827 to 830 2024-04-12T11:23:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2063: oid 10 version 830 is already newer than 829 2024-04-12T11:23:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2063: oid 10 version 830 is already newer than 828 2024-04-12T11:23:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 830 (ObjNum 771 snap 270 seq_num 771) dirty exists 2024-04-12T11:23:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2063: left oid 10 (ObjNum 771 snap 270 seq_num 771) 2024-04-12T11:23:24.607 INFO:tasks.rados.rados.0.smithi029.stdout:2063: done (1 left) 2024-04-12T11:23:24.607 INFO:tasks.rados.rados.0.smithi029.stdout:2064: done (0 left) 2024-04-12T11:23:24.607 INFO:tasks.rados.rados.0.smithi029.stdout:2065: snap_create 2024-04-12T11:23:24.716 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:24 smithi087 systemd[1]: Stopping Ceph osd.4 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:23:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:24 smithi087 ceph-mon[89014]: pgmap v534: 89 pgs: 89 active+clean; 193 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:23:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:24 smithi087 ceph-mon[89014]: from='client.44438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:23:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:24 smithi087 ceph-mon[89014]: osdmap e599: 8 total, 8 up, 8 in 2024-04-12T11:23:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:24 smithi029 ceph-mon[93650]: pgmap v534: 89 pgs: 89 active+clean; 193 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:23:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:24 smithi029 ceph-mon[93650]: from='client.44438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:23:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:24 smithi029 ceph-mon[93650]: osdmap e599: 8 total, 8 up, 8 in 2024-04-12T11:23:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:24 smithi029 ceph-mon[97787]: pgmap v534: 89 pgs: 89 active+clean; 193 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:23:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:24 smithi029 ceph-mon[97787]: from='client.44438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:23:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:24 smithi029 ceph-mon[97787]: osdmap e599: 8 total, 8 up, 8 in 2024-04-12T11:23:25.216 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:24 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[37843]: 2024-04-12T11:23:24.912+0000 7fd60d178700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:23:25.216 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:24 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[37843]: 2024-04-12T11:23:24.912+0000 7fd60d178700 -1 osd.4 600 *** Got signal Terminated *** 2024-04-12T11:23:25.217 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:24 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[37843]: 2024-04-12T11:23:24.912+0000 7fd60d178700 -1 osd.4 600 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T11:23:25.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:23:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:23:24] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.43.0" 2024-04-12T11:23:25.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:25 smithi029 ceph-mon[93650]: osdmap e600: 8 total, 8 up, 8 in 2024-04-12T11:23:25.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:25 smithi029 ceph-mon[93650]: osd.4 marked itself down and dead 2024-04-12T11:23:25.594 INFO:tasks.rados.rados.0.smithi029.stdout:2065: done (0 left) 2024-04-12T11:23:25.594 INFO:tasks.rados.rados.0.smithi029.stdout:2066: rollback oid 33 current snap is 271 2024-04-12T11:23:25.594 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 270 2024-04-12T11:23:25.594 INFO:tasks.rados.rados.0.smithi029.stdout:2067: read oid 44 snap -1 2024-04-12T11:23:25.594 INFO:tasks.rados.rados.0.smithi029.stdout:2067: expect deleted 2024-04-12T11:23:25.595 INFO:tasks.rados.rados.0.smithi029.stdout:2068: write oid 23 current snap is 271 2024-04-12T11:23:25.595 INFO:tasks.rados.rados.0.smithi029.stdout:2068: seq_num 773 ranges {591811=719130,1767522=636596,2828091=163720} 2024-04-12T11:23:25.603 INFO:tasks.rados.rados.0.smithi029.stdout:2068: writing smithi02979277-23 from 591811 to 1310941 tid 1 2024-04-12T11:23:25.609 INFO:tasks.rados.rados.0.smithi029.stdout:2068: writing smithi02979277-23 from 1767522 to 2404118 tid 2 2024-04-12T11:23:25.612 INFO:tasks.rados.rados.0.smithi029.stdout:2068: writing smithi02979277-23 from 2828091 to 2991811 tid 3 2024-04-12T11:23:25.612 INFO:tasks.rados.rados.0.smithi029.stdout:2069: snap_remove snap 237 2024-04-12T11:23:25.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:25 smithi029 ceph-mon[97787]: osdmap e600: 8 total, 8 up, 8 in 2024-04-12T11:23:25.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:25 smithi029 ceph-mon[97787]: osd.4 marked itself down and dead 2024-04-12T11:23:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:25 smithi087 ceph-mon[89014]: osdmap e600: 8 total, 8 up, 8 in 2024-04-12T11:23:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:25 smithi087 ceph-mon[89014]: osd.4 marked itself down and dead 2024-04-12T11:23:26.382 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (9m) 12s ago 10m 25.2M - 0.25.0 c8568f914cd2 5f36b9c37cec 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi087 *:3000 running (10m) 12s ago 10m 86.1M - 9.4.7 954c08fa6188 d4ba4d587fdb 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.scbuux smithi029 running (10m) 12s ago 10m 19.8M - 18.2.2-1187-gd540ebac bb2c5eac26a4 a659ce0bccaf 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi087.mbflzu smithi087 running (10m) 12s ago 10m 18.4M - 18.2.2-1187-gd540ebac bb2c5eac26a4 64bafd1e6a4c 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi087 *:8443,9283,8765 running (6m) 12s ago 15m 452M - 19.0.0-2874-g2cee1857 4a0e59462345 5a46135dfdbc 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (6m) 12s ago 17m 509M - 19.0.0-2874-g2cee1857 4a0e59462345 05c187489262 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (6m) 12s ago 17m 65.1M 2048M 19.0.0-2874-g2cee1857 4a0e59462345 d1506def0998 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi087 running (3m) 12s ago 15m 44.1M 2048M 19.0.0-2874-g2cee1857 4a0e59462345 ae22114389c9 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (4m) 12s ago 15m 53.1M 2048M 19.0.0-2874-g2cee1857 4a0e59462345 24b3f58e1d66 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (10m) 12s ago 10m 15.6M - 1.5.0 0da6a335fe13 32124106e912 2024-04-12T11:23:26.383 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi087 *:9100 running (10m) 12s ago 10m 16.0M - 1.5.0 0da6a335fe13 78910b2d30c6 2024-04-12T11:23:26.384 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (93s) 12s ago 15m 66.2M 2481M 19.0.0-2874-g2cee1857 4a0e59462345 78ce9733fe38 2024-04-12T11:23:26.384 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (72s) 12s ago 14m 54.9M 2481M 19.0.0-2874-g2cee1857 4a0e59462345 260ced18c216 2024-04-12T11:23:26.384 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (51s) 12s ago 14m 67.2M 2481M 19.0.0-2874-g2cee1857 4a0e59462345 db6a92e3e974 2024-04-12T11:23:26.384 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (17s) 12s ago 13m 12.7M 2481M 19.0.0-2874-g2cee1857 4a0e59462345 1f975ff65c6b 2024-04-12T11:23:26.384 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi087 running (13m) 12s ago 13m 104M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 ab3a8e864afd 2024-04-12T11:23:26.384 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi087 running (13m) 12s ago 13m 119M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 573e2f90e4da 2024-04-12T11:23:26.384 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi087 running (12m) 12s ago 12m 84.4M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 2b67d3f477dd 2024-04-12T11:23:26.384 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi087 running (12m) 12s ago 12m 106M 2480M 18.2.2-1187-gd540ebac bb2c5eac26a4 9c484b9f0790 2024-04-12T11:23:26.384 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi087 *:9095 running (6m) 12s ago 11m 60.7M - 2.43.0 a07b618ecd1d 569140fc5a53 2024-04-12T11:23:26.439 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:26 smithi087 podman[95276]: 2024-04-12 11:23:26.12790887 +0000 UTC m=+1.412289584 container died ab3a8e864afdb08fdebc7cdb69728a45f566a378161e9ef8401fe9338d1cb5da (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=reef-d540eba, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-12T11:23:26.598 INFO:tasks.rados.rados.0.smithi029.stdout:2067: done (3 left) 2024-04-12T11:23:26.598 INFO:tasks.rados.rados.0.smithi029.stdout:2069: done (2 left) 2024-04-12T11:23:26.598 INFO:tasks.rados.rados.0.smithi029.stdout:2070: read oid 26 snap 254 2024-04-12T11:23:26.598 INFO:tasks.rados.rados.0.smithi029.stdout:2070: expect (ObjNum 729 snap 249 seq_num 729) 2024-04-12T11:23:26.598 INFO:tasks.rados.rados.0.smithi029.stdout:2071: read oid 44 snap -1 2024-04-12T11:23:26.598 INFO:tasks.rados.rados.0.smithi029.stdout:2071: expect deleted 2024-04-12T11:23:26.598 INFO:tasks.rados.rados.0.smithi029.stdout:2072: delete oid 31 current snap is 271 2024-04-12T11:23:26.604 INFO:tasks.rados.rados.0.smithi029.stdout:2070: expect (ObjNum 729 snap 249 seq_num 729) 2024-04-12T11:23:26.715 INFO:tasks.rados.rados.0.smithi029.stdout:2066: finishing rollback tid 0 to smithi02979277-33 2024-04-12T11:23:26.715 INFO:tasks.rados.rados.0.smithi029.stdout:2066: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:23:26.715 INFO:tasks.rados.rados.0.smithi029.stdout:2066: finishing rollback tid 2 to smithi02979277-33 2024-04-12T11:23:26.715 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 706 (ObjNum 632 snap 205 seq_num 632) dirty exists 2024-04-12T11:23:26.715 INFO:tasks.rados.rados.0.smithi029.stdout:2068: finishing write tid 1 to smithi02979277-23 2024-04-12T11:23:26.715 INFO:tasks.rados.rados.0.smithi029.stdout:2068: finishing write tid 2 to smithi02979277-23 2024-04-12T11:23:26.715 INFO:tasks.rados.rados.0.smithi029.stdout:2068: finishing write tid 3 to smithi02979277-23 2024-04-12T11:23:26.715 INFO:tasks.rados.rados.0.smithi029.stdout:2066: done (4 left) 2024-04-12T11:23:26.715 INFO:tasks.rados.rados.0.smithi029.stdout:2070: done (3 left) 2024-04-12T11:23:26.716 INFO:tasks.rados.rados.0.smithi029.stdout:2071: done (2 left) 2024-04-12T11:23:26.716 INFO:tasks.rados.rados.0.smithi029.stdout:2072: done (1 left) 2024-04-12T11:23:26.716 INFO:tasks.rados.rados.0.smithi029.stdout:2073: snap_remove snap 266 2024-04-12T11:23:26.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:26 smithi087 ceph-mon[89014]: pgmap v537: 89 pgs: 89 active+clean; 197 MiB data, 694 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 300 KiB/s wr, 2 op/s 2024-04-12T11:23:26.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:26 smithi087 ceph-mon[89014]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:23:26.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:26 smithi087 ceph-mon[89014]: osdmap e601: 8 total, 7 up, 8 in 2024-04-12T11:23:26.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:26 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:26.717 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:26 smithi087 podman[95276]: 2024-04-12 11:23:26.512730425 +0000 UTC m=+1.797111148 container cleanup ab3a8e864afdb08fdebc7cdb69728a45f566a378161e9ef8401fe9338d1cb5da (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=reef-d540eba, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5) 2024-04-12T11:23:26.717 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:26 smithi087 bash[95276]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4 2024-04-12T11:23:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:26 smithi029 ceph-mon[93650]: pgmap v537: 89 pgs: 89 active+clean; 197 MiB data, 694 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 300 KiB/s wr, 2 op/s 2024-04-12T11:23:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:26 smithi029 ceph-mon[93650]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:23:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:26 smithi029 ceph-mon[93650]: osdmap e601: 8 total, 7 up, 8 in 2024-04-12T11:23:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:26 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:26 smithi029 ceph-mon[97787]: pgmap v537: 89 pgs: 89 active+clean; 197 MiB data, 694 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 300 KiB/s wr, 2 op/s 2024-04-12T11:23:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:26 smithi029 ceph-mon[97787]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:23:26.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:26 smithi029 ceph-mon[97787]: osdmap e601: 8 total, 7 up, 8 in 2024-04-12T11:23:26.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:26 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:27.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -e sha1=2cee1857f8068f7ca29db3a37937b8e9f919ba95 -- bash -c 'ceph versions' 2024-04-12T11:23:27.090 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:26 smithi087 podman[95304]: 2024-04-12 11:23:26.772797911 +0000 UTC m=+0.644758266 container remove ab3a8e864afdb08fdebc7cdb69728a45f566a378161e9ef8401fe9338d1cb5da (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, RELEASE=reef-d540eba, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T11:23:27.216 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:23:27.409 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:27 smithi087 podman[95397]: 2024-04-12 11:23:26.992296748 +0000 UTC m=+0.018188538 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:23:27.409 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:27 smithi087 podman[95397]: 2024-04-12 11:23:27.107029769 +0000 UTC m=+0.132921546 container create ed95b0eaf78eca04660d1fdf4572323272b58b449e7077b2b0b31534721f47b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-12T11:23:27.409 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:27 smithi087 podman[95397]: 2024-04-12 11:23:27.326472827 +0000 UTC m=+0.352364616 container init ed95b0eaf78eca04660d1fdf4572323272b58b449e7077b2b0b31534721f47b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, ceph=True, org.label-schema.license=GPLv2) 2024-04-12T11:23:27.409 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:27 smithi087 podman[95397]: 2024-04-12 11:23:27.329997163 +0000 UTC m=+0.355888941 container start ed95b0eaf78eca04660d1fdf4572323272b58b449e7077b2b0b31534721f47b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409) 2024-04-12T11:23:27.580 INFO:tasks.rados.rados.0.smithi029.stdout:2073: done (1 left) 2024-04-12T11:23:27.580 INFO:tasks.rados.rados.0.smithi029.stdout:2074: snap_remove snap 263 2024-04-12T11:23:27.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:27 smithi087 ceph-mon[89014]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:23:27.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:27 smithi087 ceph-mon[89014]: osdmap e602: 8 total, 7 up, 8 in 2024-04-12T11:23:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:27 smithi087 ceph-mon[89014]: osdmap e603: 8 total, 7 up, 8 in 2024-04-12T11:23:27.717 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:27 smithi087 podman[95397]: 2024-04-12 11:23:27.409421613 +0000 UTC m=+0.435313409 container attach ed95b0eaf78eca04660d1fdf4572323272b58b449e7077b2b0b31534721f47b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-12T11:23:27.717 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:27 smithi087 podman[95397]: 2024-04-12 11:23:27.620184384 +0000 UTC m=+0.646076162 container died ed95b0eaf78eca04660d1fdf4572323272b58b449e7077b2b0b31534721f47b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True) 2024-04-12T11:23:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:27 smithi029 ceph-mon[97787]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:23:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:27 smithi029 ceph-mon[97787]: osdmap e602: 8 total, 7 up, 8 in 2024-04-12T11:23:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:27 smithi029 ceph-mon[97787]: osdmap e603: 8 total, 7 up, 8 in 2024-04-12T11:23:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:27 smithi029 ceph-mon[93650]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:23:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:27 smithi029 ceph-mon[93650]: osdmap e602: 8 total, 7 up, 8 in 2024-04-12T11:23:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:27 smithi029 ceph-mon[93650]: osdmap e603: 8 total, 7 up, 8 in 2024-04-12T11:23:28.174 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:28 smithi087 podman[95423]: 2024-04-12 11:23:28.171775893 +0000 UTC m=+0.546191879 container remove ed95b0eaf78eca04660d1fdf4572323272b58b449e7077b2b0b31534721f47b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409) 2024-04-12T11:23:28.466 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:28 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-12T11:23:28.467 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:28 smithi087 systemd[1]: Stopped Ceph osd.4 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:23:28.467 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:28 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.4.service: Consumed 14.275s CPU time. 2024-04-12T11:23:28.467 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:28 smithi087 systemd[1]: Starting Ceph osd.4 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:23:28.582 INFO:tasks.rados.rados.0.smithi029.stdout:2074: done (1 left) 2024-04-12T11:23:28.582 INFO:tasks.rados.rados.0.smithi029.stdout:2075: write oid 34 current snap is 271 2024-04-12T11:23:28.582 INFO:tasks.rados.rados.0.smithi029.stdout:2075: seq_num 775 ranges {625425=469628,1768133=640265} 2024-04-12T11:23:28.588 INFO:tasks.rados.rados.0.smithi029.stdout:2075: writing smithi02979277-34 from 625425 to 1095053 tid 1 2024-04-12T11:23:28.596 INFO:tasks.rados.rados.0.smithi029.stdout:2075: writing smithi02979277-34 from 1768133 to 2408398 tid 2 2024-04-12T11:23:28.596 INFO:tasks.rados.rados.0.smithi029.stdout:2076: snap_create 2024-04-12T11:23:28.596 INFO:tasks.rados.rados.0.smithi029.stdout:2068: finishing write tid 4 to smithi02979277-23 2024-04-12T11:23:28.596 INFO:tasks.rados.rados.0.smithi029.stdout:2068: finishing write tid 5 to smithi02979277-23 2024-04-12T11:23:28.596 INFO:tasks.rados.rados.0.smithi029.stdout:2068: oid 23 updating version 0 to 710 2024-04-12T11:23:28.596 INFO:tasks.rados.rados.0.smithi029.stdout:2068: oid 23 version 710 is already newer than 709 2024-04-12T11:23:28.597 INFO:tasks.rados.rados.0.smithi029.stdout:2068: oid 23 updating version 710 to 711 2024-04-12T11:23:28.597 INFO:tasks.rados.rados.0.smithi029.stdout:2068: oid 23 version 711 is already newer than 708 2024-04-12T11:23:28.597 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 711 (ObjNum 773 snap 271 seq_num 773) dirty exists 2024-04-12T11:23:28.597 INFO:tasks.rados.rados.0.smithi029.stdout:2068: left oid 23 (ObjNum 773 snap 271 seq_num 773) 2024-04-12T11:23:28.597 INFO:tasks.rados.rados.0.smithi029.stdout:2075: finishing write tid 1 to smithi02979277-34 2024-04-12T11:23:28.599 INFO:tasks.rados.rados.0.smithi029.stdout:2075: finishing write tid 2 to smithi02979277-34 2024-04-12T11:23:28.599 INFO:tasks.rados.rados.0.smithi029.stdout:2075: finishing write tid 3 to smithi02979277-34 2024-04-12T11:23:28.599 INFO:tasks.rados.rados.0.smithi029.stdout:2075: finishing write tid 4 to smithi02979277-34 2024-04-12T11:23:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:2075: oid 34 updating version 0 to 770 2024-04-12T11:23:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:2075: oid 34 updating version 770 to 773 2024-04-12T11:23:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:2075: oid 34 updating version 773 to 774 2024-04-12T11:23:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 774 (ObjNum 775 snap 271 seq_num 775) dirty exists 2024-04-12T11:23:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:2075: left oid 34 (ObjNum 775 snap 271 seq_num 775) 2024-04-12T11:23:28.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:28 smithi029 ceph-mon[97787]: pgmap v540: 89 pgs: 10 active+undersized+degraded, 8 stale+active+clean, 9 active+undersized, 62 active+clean; 200 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 931 KiB/s wr, 5 op/s; 34/399 objects degraded (8.521%) 2024-04-12T11:23:28.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:28 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 34/399 objects degraded (8.521%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:28.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:28 smithi029 ceph-mon[97787]: osdmap e604: 8 total, 7 up, 8 in 2024-04-12T11:23:28.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:28 smithi029 ceph-mon[93650]: pgmap v540: 89 pgs: 10 active+undersized+degraded, 8 stale+active+clean, 9 active+undersized, 62 active+clean; 200 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 931 KiB/s wr, 5 op/s; 34/399 objects degraded (8.521%) 2024-04-12T11:23:28.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:28 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 34/399 objects degraded (8.521%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:28.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:28 smithi029 ceph-mon[93650]: osdmap e604: 8 total, 7 up, 8 in 2024-04-12T11:23:28.876 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:28 smithi087 ceph-mon[89014]: pgmap v540: 89 pgs: 10 active+undersized+degraded, 8 stale+active+clean, 9 active+undersized, 62 active+clean; 200 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 931 KiB/s wr, 5 op/s; 34/399 objects degraded (8.521%) 2024-04-12T11:23:28.876 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:28 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 34/399 objects degraded (8.521%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:28.876 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:28 smithi087 ceph-mon[89014]: osdmap e604: 8 total, 7 up, 8 in 2024-04-12T11:23:28.876 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:28 smithi087 podman[95523]: 2024-04-12 11:23:28.507568162 +0000 UTC m=+0.017640751 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:23:28.876 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:28 smithi087 podman[95523]: 2024-04-12 11:23:28.639487296 +0000 UTC m=+0.149559885 container create 8de1294ad7858bf37a77e85c8c608efbea0a43a09a66082750395b841b51f100 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, ceph=True, org.label-schema.vendor=CentOS) 2024-04-12T11:23:29.203 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: "mon": { 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev)": 3 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr": { 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev)": 2 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: "osd": { 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.2-1187-gd540ebac (d540ebaca6b131a1dd560e7f69e024b133bbaa42) reef (stable)": 3, 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev)": 4 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: "mds": { 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.2-1187-gd540ebac (d540ebaca6b131a1dd560e7f69e024b133bbaa42) reef (stable)": 2 2024-04-12T11:23:29.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:23:29.205 INFO:teuthology.orchestra.run.smithi029.stdout: "overall": { 2024-04-12T11:23:29.205 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.2-1187-gd540ebac (d540ebaca6b131a1dd560e7f69e024b133bbaa42) reef (stable)": 5, 2024-04-12T11:23:29.205 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-2874-g2cee1857 (2cee1857f8068f7ca29db3a37937b8e9f919ba95) squid (dev)": 9 2024-04-12T11:23:29.205 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:23:29.205 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:23:29.216 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:28 smithi087 podman[95523]: 2024-04-12 11:23:28.875871462 +0000 UTC m=+0.385944061 container init 8de1294ad7858bf37a77e85c8c608efbea0a43a09a66082750395b841b51f100 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-12T11:23:29.216 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:28 smithi087 podman[95523]: 2024-04-12 11:23:28.879456727 +0000 UTC m=+0.389529317 container start 8de1294ad7858bf37a77e85c8c608efbea0a43a09a66082750395b841b51f100 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-12T11:23:29.216 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:28 smithi087 podman[95523]: 2024-04-12 11:23:28.983675097 +0000 UTC m=+0.493747689 container attach 8de1294ad7858bf37a77e85c8c608efbea0a43a09a66082750395b841b51f100 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-12T11:23:29.583 INFO:tasks.rados.rados.0.smithi029.stdout:2068: done (2 left) 2024-04-12T11:23:29.583 INFO:tasks.rados.rados.0.smithi029.stdout:2075: done (1 left) 2024-04-12T11:23:29.583 INFO:tasks.rados.rados.0.smithi029.stdout:2076: done (0 left) 2024-04-12T11:23:29.583 INFO:tasks.rados.rados.0.smithi029.stdout:2077: snap_remove snap 222 2024-04-12T11:23:29.601 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate[95538]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate[95538]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 bash[95523]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 bash[95523]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 bash[95523]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 bash[95523]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 bash[95523]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 bash[95523]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 bash[95523]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate[95538]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate[95538]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate[95538]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate[95538]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate[95538]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-12T11:23:29.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 podman[95523]: 2024-04-12 11:23:29.344171374 +0000 UTC m=+0.854243957 container died 8de1294ad7858bf37a77e85c8c608efbea0a43a09a66082750395b841b51f100 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T11:23:29.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:29 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/612716932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:29.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:29 smithi029 ceph-mon[97787]: osdmap e605: 8 total, 7 up, 8 in 2024-04-12T11:23:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:29 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/612716932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:29 smithi029 ceph-mon[93650]: osdmap e605: 8 total, 7 up, 8 in 2024-04-12T11:23:29.949 DEBUG:teuthology.parallel:result is None 2024-04-12T11:23:29.962 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:29 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/612716932' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T11:23:29.962 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:29 smithi087 ceph-mon[89014]: osdmap e605: 8 total, 7 up, 8 in 2024-04-12T11:23:30.216 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:29 smithi087 podman[95669]: 2024-04-12 11:23:29.962662314 +0000 UTC m=+0.609542811 container remove 8de1294ad7858bf37a77e85c8c608efbea0a43a09a66082750395b841b51f100 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-12T11:23:30.216 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:30 smithi087 podman[95735]: 2024-04-12 11:23:30.081012755 +0000 UTC m=+0.018158102 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T11:23:30.217 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:30 smithi087 podman[95735]: 2024-04-12 11:23:30.196784306 +0000 UTC m=+0.133929643 container create fb553cefdd39034be0f8dfca2e163cde9d03ab9d40572f7279cfe8007f4fccb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T11:23:30.585 INFO:tasks.rados.rados.0.smithi029.stdout:2077: done (0 left) 2024-04-12T11:23:30.585 INFO:tasks.rados.rados.0.smithi029.stdout:2078: write oid 8 current snap is 272 2024-04-12T11:23:30.585 INFO:tasks.rados.rados.0.smithi029.stdout:2078: seq_num 776 ranges {642232=661230,1942968=783923} 2024-04-12T11:23:30.592 INFO:tasks.rados.rados.0.smithi029.stdout:2078: writing smithi02979277-8 from 642232 to 1303462 tid 1 2024-04-12T11:23:30.600 INFO:tasks.rados.rados.0.smithi029.stdout:2078: writing smithi02979277-8 from 1942968 to 2726891 tid 2 2024-04-12T11:23:30.600 INFO:tasks.rados.rados.0.smithi029.stdout:2079: delete oid 15 current snap is 272 2024-04-12T11:23:30.600 INFO:tasks.rados.rados.0.smithi029.stdout:2078: finishing write tid 1 to smithi02979277-8 2024-04-12T11:23:30.602 INFO:tasks.rados.rados.0.smithi029.stdout:2079: done (1 left) 2024-04-12T11:23:30.602 INFO:tasks.rados.rados.0.smithi029.stdout:2080: delete oid 34 current snap is 272 2024-04-12T11:23:30.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:30 smithi087 podman[95735]: 2024-04-12 11:23:30.399385987 +0000 UTC m=+0.336531322 container init fb553cefdd39034be0f8dfca2e163cde9d03ab9d40572f7279cfe8007f4fccb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-12T11:23:30.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:30 smithi087 podman[95735]: 2024-04-12 11:23:30.402354706 +0000 UTC m=+0.339500040 container start fb553cefdd39034be0f8dfca2e163cde9d03ab9d40572f7279cfe8007f4fccb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4, RELEASE=wip-rf-comb1-2cee185, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-12T11:23:30.602 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:30 smithi087 ceph-osd[95755]: -- 172.21.15.87:0/3328087873 <== mon.1 v2:172.21.15.87:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55dd65409680 con 0x55dd653d3000 2024-04-12T11:23:30.603 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:30 smithi087 bash[95735]: fb553cefdd39034be0f8dfca2e163cde9d03ab9d40572f7279cfe8007f4fccb2 2024-04-12T11:23:30.603 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:30 smithi087 systemd[1]: Started Ceph osd.4 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:23:30.604 INFO:tasks.rados.rados.0.smithi029.stdout:2078: finishing write tid 2 to smithi02979277-8 2024-04-12T11:23:30.604 INFO:tasks.rados.rados.0.smithi029.stdout:2078: finishing write tid 3 to smithi02979277-8 2024-04-12T11:23:30.604 INFO:tasks.rados.rados.0.smithi029.stdout:2078: finishing write tid 4 to smithi02979277-8 2024-04-12T11:23:30.604 INFO:tasks.rados.rados.0.smithi029.stdout:2078: oid 8 updating version 0 to 211 2024-04-12T11:23:30.604 INFO:tasks.rados.rados.0.smithi029.stdout:2078: oid 8 version 211 is already newer than 210 2024-04-12T11:23:30.604 INFO:tasks.rados.rados.0.smithi029.stdout:2078: oid 8 version 211 is already newer than 209 2024-04-12T11:23:30.604 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 211 (ObjNum 776 snap 272 seq_num 776) dirty exists 2024-04-12T11:23:30.604 INFO:tasks.rados.rados.0.smithi029.stdout:2078: left oid 8 (ObjNum 776 snap 272 seq_num 776) 2024-04-12T11:23:30.611 INFO:tasks.rados.rados.0.smithi029.stdout:2078: done (1 left) 2024-04-12T11:23:30.611 INFO:tasks.rados.rados.0.smithi029.stdout:2080: done (0 left) 2024-04-12T11:23:30.612 INFO:tasks.rados.rados.0.smithi029.stdout:2081: write oid 3 current snap is 272 2024-04-12T11:23:30.612 INFO:tasks.rados.rados.0.smithi029.stdout:2081: seq_num 779 ranges {692653=486977,1602061=647603,2992629=100024} 2024-04-12T11:23:30.618 INFO:tasks.rados.rados.0.smithi029.stdout:2081: writing smithi02979277-3 from 692653 to 1179630 tid 1 2024-04-12T11:23:30.624 INFO:tasks.rados.rados.0.smithi029.stdout:2081: writing smithi02979277-3 from 1602061 to 2249664 tid 2 2024-04-12T11:23:30.629 INFO:tasks.rados.rados.0.smithi029.stdout:2081: writing smithi02979277-3 from 2992629 to 3092653 tid 3 2024-04-12T11:23:30.629 INFO:tasks.rados.rados.0.smithi029.stdout:2082: snap_create 2024-04-12T11:23:30.629 INFO:tasks.rados.rados.0.smithi029.stdout:2081: finishing write tid 1 to smithi02979277-3 2024-04-12T11:23:30.629 INFO:tasks.rados.rados.0.smithi029.stdout:2081: finishing write tid 2 to smithi02979277-3 2024-04-12T11:23:30.631 INFO:tasks.rados.rados.0.smithi029.stdout:2081: finishing write tid 3 to smithi02979277-3 2024-04-12T11:23:30.631 INFO:tasks.rados.rados.0.smithi029.stdout:2081: finishing write tid 4 to smithi02979277-3 2024-04-12T11:23:30.631 INFO:tasks.rados.rados.0.smithi029.stdout:2081: finishing write tid 5 to smithi02979277-3 2024-04-12T11:23:30.631 INFO:tasks.rados.rados.0.smithi029.stdout:2081: oid 3 updating version 0 to 1280 2024-04-12T11:23:30.631 INFO:tasks.rados.rados.0.smithi029.stdout:2081: oid 3 updating version 1280 to 1281 2024-04-12T11:23:30.631 INFO:tasks.rados.rados.0.smithi029.stdout:2081: oid 3 updating version 1281 to 1283 2024-04-12T11:23:30.631 INFO:tasks.rados.rados.0.smithi029.stdout:2081: oid 3 version 1283 is already newer than 1282 2024-04-12T11:23:30.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1283 (ObjNum 779 snap 272 seq_num 779) dirty exists 2024-04-12T11:23:30.632 INFO:tasks.rados.rados.0.smithi029.stdout:2081: left oid 3 (ObjNum 779 snap 272 seq_num 779) 2024-04-12T11:23:30.947 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:30 smithi087 ceph-mon[89014]: pgmap v543: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 202 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 865 KiB/s rd, 1.2 MiB/s wr, 7 op/s; 58/408 objects degraded (14.216%) 2024-04-12T11:23:30.947 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:30 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:30.947 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:30 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:30.947 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:30 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:30.947 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:30 smithi087 ceph-mon[89014]: osdmap e606: 8 total, 7 up, 8 in 2024-04-12T11:23:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:30 smithi029 ceph-mon[97787]: pgmap v543: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 202 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 865 KiB/s rd, 1.2 MiB/s wr, 7 op/s; 58/408 objects degraded (14.216%) 2024-04-12T11:23:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:30 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:30 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:30 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:30 smithi029 ceph-mon[97787]: osdmap e606: 8 total, 7 up, 8 in 2024-04-12T11:23:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:30 smithi029 ceph-mon[93650]: pgmap v543: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 202 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 865 KiB/s rd, 1.2 MiB/s wr, 7 op/s; 58/408 objects degraded (14.216%) 2024-04-12T11:23:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:31.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:30 smithi029 ceph-mon[93650]: osdmap e606: 8 total, 7 up, 8 in 2024-04-12T11:23:31.216 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:30 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T11:23:30.945+0000 7f7eb47347c0 -1 Falling back to public interface 2024-04-12T11:23:31.610 INFO:tasks.rados.rados.0.smithi029.stdout:2081: done (1 left) 2024-04-12T11:23:31.610 INFO:tasks.rados.rados.0.smithi029.stdout:2082: done (0 left) 2024-04-12T11:23:31.610 INFO:tasks.rados.rados.0.smithi029.stdout:2083: write oid 18 current snap is 273 2024-04-12T11:23:31.610 INFO:tasks.rados.rados.0.smithi029.stdout:2083: seq_num 780 ranges {709460=562226,2060543=791261} 2024-04-12T11:23:31.616 INFO:tasks.rados.rados.0.smithi029.stdout:2083: writing smithi02979277-18 from 709460 to 1271686 tid 1 2024-04-12T11:23:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:2083: writing smithi02979277-18 from 2060543 to 2851804 tid 2 2024-04-12T11:23:31.626 INFO:tasks.rados.rados.0.smithi029.stdout:2084: snap_remove snap 267 2024-04-12T11:23:32.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:23:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:23:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:23:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:2084: done (1 left) 2024-04-12T11:23:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:2085: delete oid 38 current snap is 273 2024-04-12T11:23:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:2083: finishing write tid 1 to smithi02979277-18 2024-04-12T11:23:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:2083: finishing write tid 2 to smithi02979277-18 2024-04-12T11:23:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:2083: finishing write tid 3 to smithi02979277-18 2024-04-12T11:23:32.583 INFO:tasks.rados.rados.0.smithi029.stdout:2083: finishing write tid 4 to smithi02979277-18 2024-04-12T11:23:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:2083: oid 18 updating version 0 to 990 2024-04-12T11:23:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:2083: oid 18 version 990 is already newer than 989 2024-04-12T11:23:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:2083: oid 18 version 990 is already newer than 988 2024-04-12T11:23:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 990 (ObjNum 780 snap 273 seq_num 780) dirty exists 2024-04-12T11:23:32.584 INFO:tasks.rados.rados.0.smithi029.stdout:2083: left oid 18 (ObjNum 780 snap 273 seq_num 780) 2024-04-12T11:23:32.590 INFO:tasks.rados.rados.0.smithi029.stdout:2083: done (1 left) 2024-04-12T11:23:32.590 INFO:tasks.rados.rados.0.smithi029.stdout:2085: done (0 left) 2024-04-12T11:23:32.590 INFO:tasks.rados.rados.0.smithi029.stdout:2086: read oid 43 snap -1 2024-04-12T11:23:32.590 INFO:tasks.rados.rados.0.smithi029.stdout:2086: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:23:32.590 INFO:tasks.rados.rados.0.smithi029.stdout:2087: snap_remove snap 251 2024-04-12T11:23:32.891 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:32 smithi087 ceph-mon[89014]: pgmap v546: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 202 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 864 KiB/s rd, 610 KiB/s wr, 4 op/s; 58/408 objects degraded (14.216%) 2024-04-12T11:23:32.891 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:32 smithi087 ceph-mon[89014]: osdmap e607: 8 total, 7 up, 8 in 2024-04-12T11:23:32.891 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:32 smithi087 ceph-mon[89014]: osdmap e608: 8 total, 7 up, 8 in 2024-04-12T11:23:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:32 smithi029 ceph-mon[97787]: pgmap v546: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 202 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 864 KiB/s rd, 610 KiB/s wr, 4 op/s; 58/408 objects degraded (14.216%) 2024-04-12T11:23:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:32 smithi029 ceph-mon[97787]: osdmap e607: 8 total, 7 up, 8 in 2024-04-12T11:23:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:32 smithi029 ceph-mon[97787]: osdmap e608: 8 total, 7 up, 8 in 2024-04-12T11:23:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:32 smithi029 ceph-mon[93650]: pgmap v546: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 202 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 864 KiB/s rd, 610 KiB/s wr, 4 op/s; 58/408 objects degraded (14.216%) 2024-04-12T11:23:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:32 smithi029 ceph-mon[93650]: osdmap e607: 8 total, 7 up, 8 in 2024-04-12T11:23:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:32 smithi029 ceph-mon[93650]: osdmap e608: 8 total, 7 up, 8 in 2024-04-12T11:23:33.584 INFO:tasks.rados.rados.0.smithi029.stdout:2086: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:23:33.695 INFO:tasks.rados.rados.0.smithi029.stdout:2086: done (1 left) 2024-04-12T11:23:33.696 INFO:tasks.rados.rados.0.smithi029.stdout:2087: done (0 left) 2024-04-12T11:23:33.696 INFO:tasks.rados.rados.0.smithi029.stdout:2088: rollback oid 48 current snap is 273 2024-04-12T11:23:33.696 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 261 2024-04-12T11:23:33.696 INFO:tasks.rados.rados.0.smithi029.stdout:2089: snap_create 2024-04-12T11:23:33.699 INFO:tasks.rados.rados.0.smithi029.stdout:2088: finishing rollback tid 0 to smithi02979277-48 2024-04-12T11:23:33.699 INFO:tasks.rados.rados.0.smithi029.stdout:2088: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:23:33.699 INFO:tasks.rados.rados.0.smithi029.stdout:2088: finishing rollback tid 2 to smithi02979277-48 2024-04-12T11:23:33.699 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 835 (ObjNum 749 snap 258 seq_num 749) dirty exists 2024-04-12T11:23:33.966 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T11:23:33.809+0000 7f7eb47347c0 -1 osd.4 600 log_to_monitors true 2024-04-12T11:23:34.615 INFO:tasks.rados.rados.0.smithi029.stdout:2088: done (1 left) 2024-04-12T11:23:34.615 INFO:tasks.rados.rados.0.smithi029.stdout:2089: done (0 left) 2024-04-12T11:23:34.615 INFO:tasks.rados.rados.0.smithi029.stdout:2090: read oid 24 snap -1 2024-04-12T11:23:34.615 INFO:tasks.rados.rados.0.smithi029.stdout:2090: expect (ObjNum 735 snap 253 seq_num 735) 2024-04-12T11:23:34.615 INFO:tasks.rados.rados.0.smithi029.stdout:2091: read oid 23 snap -1 2024-04-12T11:23:34.615 INFO:tasks.rados.rados.0.smithi029.stdout:2091: expect (ObjNum 773 snap 271 seq_num 773) 2024-04-12T11:23:34.615 INFO:tasks.rados.rados.0.smithi029.stdout:2092: delete oid 46 current snap is 274 2024-04-12T11:23:34.619 INFO:tasks.rados.rados.0.smithi029.stdout:2092: done (2 left) 2024-04-12T11:23:34.620 INFO:tasks.rados.rados.0.smithi029.stdout:2093: write oid 6 current snap is 274 2024-04-12T11:23:34.620 INFO:tasks.rados.rados.0.smithi029.stdout:2093: seq_num 783 ranges {759881=787973,2119636=654941} 2024-04-12T11:23:34.629 INFO:tasks.rados.rados.0.smithi029.stdout:2093: writing smithi02979277-6 from 759881 to 1547854 tid 1 2024-04-12T11:23:34.636 INFO:tasks.rados.rados.0.smithi029.stdout:2093: writing smithi02979277-6 from 2119636 to 2774577 tid 2 2024-04-12T11:23:34.637 INFO:tasks.rados.rados.0.smithi029.stdout:2094: snap_remove snap 268 2024-04-12T11:23:34.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-mon[97787]: pgmap v549: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 196 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 684 KiB/s wr, 2 op/s; 59/399 objects degraded (14.787%) 2024-04-12T11:23:34.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-mon[97787]: osdmap e609: 8 total, 7 up, 8 in 2024-04-12T11:23:34.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 59/399 objects degraded (14.787%), 17 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:34.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-mon[97787]: from='osd.4 [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:23:34.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-mon[97787]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:23:34.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-mon[93650]: pgmap v549: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 196 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 684 KiB/s wr, 2 op/s; 59/399 objects degraded (14.787%) 2024-04-12T11:23:34.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-mon[93650]: osdmap e609: 8 total, 7 up, 8 in 2024-04-12T11:23:34.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 59/399 objects degraded (14.787%), 17 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:34.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-mon[93650]: from='osd.4 [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:23:34.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-mon[93650]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:23:34.966 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 11:23:34 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T11:23:34.639+0000 7f7ea5b8c700 -1 osd.4 600 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:23:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:34 smithi087 ceph-mon[89014]: pgmap v549: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 196 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 684 KiB/s wr, 2 op/s; 59/399 objects degraded (14.787%) 2024-04-12T11:23:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:34 smithi087 ceph-mon[89014]: osdmap e609: 8 total, 7 up, 8 in 2024-04-12T11:23:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:34 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 59/399 objects degraded (14.787%), 17 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:34 smithi087 ceph-mon[89014]: from='osd.4 [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:23:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:34 smithi087 ceph-mon[89014]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T11:23:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:23:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:23:34] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.43.0" 2024-04-12T11:23:35.625 INFO:tasks.rados.rados.0.smithi029.stdout:2094: done (3 left) 2024-04-12T11:23:35.625 INFO:tasks.rados.rados.0.smithi029.stdout:2095: snap_remove snap 264 2024-04-12T11:23:35.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[97787]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T11:23:35.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[97787]: osdmap e610: 8 total, 7 up, 8 in 2024-04-12T11:23:35.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[97787]: from='osd.4 [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:23:35.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[97787]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:23:35.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:35.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:35.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[93650]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T11:23:35.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[93650]: osdmap e610: 8 total, 7 up, 8 in 2024-04-12T11:23:35.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[93650]: from='osd.4 [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:23:35.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[93650]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:23:35.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:35.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:35 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:35 smithi087 ceph-mon[89014]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T11:23:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:35 smithi087 ceph-mon[89014]: osdmap e610: 8 total, 7 up, 8 in 2024-04-12T11:23:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:35 smithi087 ceph-mon[89014]: from='osd.4 [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:23:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:35 smithi087 ceph-mon[89014]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:23:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:35 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:35 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:36.627 INFO:tasks.rados.rados.0.smithi029.stdout:2095: done (3 left) 2024-04-12T11:23:36.627 INFO:tasks.rados.rados.0.smithi029.stdout:2096: rollback oid 27 current snap is 274 2024-04-12T11:23:36.627 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 238 2024-04-12T11:23:36.628 INFO:tasks.rados.rados.0.smithi029.stdout:2097: write oid 22 current snap is 274 2024-04-12T11:23:36.628 INFO:tasks.rados.rados.0.smithi029.stdout:2097: seq_num 784 ranges {776688=579575,1894471=798599} 2024-04-12T11:23:36.636 INFO:tasks.rados.rados.0.smithi029.stdout:2097: writing smithi02979277-22 from 776688 to 1356263 tid 1 2024-04-12T11:23:36.644 INFO:tasks.rados.rados.0.smithi029.stdout:2097: writing smithi02979277-22 from 1894471 to 2693070 tid 2 2024-04-12T11:23:36.645 INFO:tasks.rados.rados.0.smithi029.stdout:2090: expect (ObjNum 735 snap 253 seq_num 735) 2024-04-12T11:23:36.821 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:36 smithi087 ceph-mon[89014]: pgmap v552: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 193 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 3.3 MiB/s rd, 985 KiB/s wr, 6 op/s; 59/396 objects degraded (14.899%) 2024-04-12T11:23:36.821 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:36 smithi087 ceph-mon[89014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:23:36.821 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:36 smithi087 ceph-mon[89014]: osd.4 [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210] boot 2024-04-12T11:23:36.821 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:36 smithi087 ceph-mon[89014]: osdmap e611: 8 total, 8 up, 8 in 2024-04-12T11:23:36.821 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:23:36.821 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:36.821 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:36.822 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:36.822 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:36.831 INFO:tasks.rados.rados.0.smithi029.stdout:2091: expect (ObjNum 773 snap 271 seq_num 773) 2024-04-12T11:23:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[97787]: pgmap v552: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 193 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 3.3 MiB/s rd, 985 KiB/s wr, 6 op/s; 59/396 objects degraded (14.899%) 2024-04-12T11:23:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[97787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:23:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[97787]: osd.4 [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210] boot 2024-04-12T11:23:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[97787]: osdmap e611: 8 total, 8 up, 8 in 2024-04-12T11:23:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:23:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:37.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[93650]: pgmap v552: 89 pgs: 17 active+undersized+degraded, 24 active+undersized, 48 active+clean; 193 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 3.3 MiB/s rd, 985 KiB/s wr, 6 op/s; 59/396 objects degraded (14.899%) 2024-04-12T11:23:37.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[93650]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:23:37.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[93650]: osd.4 [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210] boot 2024-04-12T11:23:37.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[93650]: osdmap e611: 8 total, 8 up, 8 in 2024-04-12T11:23:37.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T11:23:37.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:37.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:37.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:37.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:37.026 INFO:tasks.rados.rados.0.smithi029.stdout:2093: finishing write tid 1 to smithi02979277-6 2024-04-12T11:23:37.027 INFO:tasks.rados.rados.0.smithi029.stdout:2093: finishing write tid 2 to smithi02979277-6 2024-04-12T11:23:37.027 INFO:tasks.rados.rados.0.smithi029.stdout:2090: done (4 left) 2024-04-12T11:23:37.027 INFO:tasks.rados.rados.0.smithi029.stdout:2091: done (3 left) 2024-04-12T11:23:37.027 INFO:tasks.rados.rados.0.smithi029.stdout:2098: snap_remove snap 270 2024-04-12T11:23:37.631 INFO:tasks.rados.rados.0.smithi029.stdout:2098: done (3 left) 2024-04-12T11:23:37.631 INFO:tasks.rados.rados.0.smithi029.stdout:2099: read oid 3 snap -1 2024-04-12T11:23:37.631 INFO:tasks.rados.rados.0.smithi029.stdout:2099: expect (ObjNum 779 snap 272 seq_num 779) 2024-04-12T11:23:37.631 INFO:tasks.rados.rados.0.smithi029.stdout:2100: snap_remove snap 272 2024-04-12T11:23:37.783 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:37 smithi087 ceph-mon[89014]: osdmap e612: 8 total, 8 up, 8 in 2024-04-12T11:23:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:37 smithi029 ceph-mon[93650]: osdmap e612: 8 total, 8 up, 8 in 2024-04-12T11:23:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:37 smithi029 ceph-mon[97787]: osdmap e612: 8 total, 8 up, 8 in 2024-04-12T11:23:38.632 INFO:tasks.rados.rados.0.smithi029.stdout:2100: done (4 left) 2024-04-12T11:23:38.632 INFO:tasks.rados.rados.0.smithi029.stdout:2101: read oid 17 snap -1 2024-04-12T11:23:38.632 INFO:tasks.rados.rados.0.smithi029.stdout:2101: expect deleted 2024-04-12T11:23:38.632 INFO:tasks.rados.rados.0.smithi029.stdout:2102: write oid 33 current snap is 274 2024-04-12T11:23:38.632 INFO:tasks.rados.rados.0.smithi029.stdout:2093: finishing write tid 3 to smithi02979277-6 2024-04-12T11:23:38.632 INFO:tasks.rados.rados.0.smithi029.stdout:2093: finishing write tid 4 to smithi02979277-6 2024-04-12T11:23:38.632 INFO:tasks.rados.rados.0.smithi029.stdout:2093: oid 6 updating version 0 to 502 2024-04-12T11:23:38.632 INFO:tasks.rados.rados.0.smithi029.stdout:2093: oid 6 version 502 is already newer than 501 2024-04-12T11:23:38.632 INFO:tasks.rados.rados.0.smithi029.stdout:2093: oid 6 version 502 is already newer than 500 2024-04-12T11:23:38.632 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 502 (ObjNum 783 snap 274 seq_num 783) dirty exists 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2093: left oid 6 (ObjNum 783 snap 274 seq_num 783) 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2096: finishing rollback tid 0 to smithi02979277-27 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2096: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 712 (ObjNum 193 snap 0 seq_num 1800965184) dirty dne 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2097: finishing write tid 1 to smithi02979277-22 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2097: finishing write tid 2 to smithi02979277-22 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2097: finishing write tid 3 to smithi02979277-22 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2097: finishing write tid 4 to smithi02979277-22 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2097: oid 22 updating version 0 to 714 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2097: oid 22 version 714 is already newer than 713 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2097: oid 22 updating version 714 to 715 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 715 (ObjNum 784 snap 274 seq_num 784) dirty exists 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2097: left oid 22 (ObjNum 784 snap 274 seq_num 784) 2024-04-12T11:23:38.633 INFO:tasks.rados.rados.0.smithi029.stdout:2099: expect (ObjNum 779 snap 272 seq_num 779) 2024-04-12T11:23:38.820 INFO:tasks.rados.rados.0.smithi029.stdout:2102: seq_num 785 ranges {793495=654824,1952953=658610,3157167=36328} 2024-04-12T11:23:38.828 INFO:tasks.rados.rados.0.smithi029.stdout:2102: writing smithi02979277-33 from 793495 to 1448319 tid 1 2024-04-12T11:23:38.834 INFO:tasks.rados.rados.0.smithi029.stdout:2102: writing smithi02979277-33 from 1952953 to 2611563 tid 2 2024-04-12T11:23:38.838 INFO:tasks.rados.rados.0.smithi029.stdout:2102: writing smithi02979277-33 from 3157167 to 3193495 tid 3 2024-04-12T11:23:38.838 INFO:tasks.rados.rados.0.smithi029.stdout:2093: done (5 left) 2024-04-12T11:23:38.838 INFO:tasks.rados.rados.0.smithi029.stdout:2096: done (4 left) 2024-04-12T11:23:38.838 INFO:tasks.rados.rados.0.smithi029.stdout:2097: done (3 left) 2024-04-12T11:23:38.838 INFO:tasks.rados.rados.0.smithi029.stdout:2099: done (2 left) 2024-04-12T11:23:38.838 INFO:tasks.rados.rados.0.smithi029.stdout:2101: done (1 left) 2024-04-12T11:23:38.838 INFO:tasks.rados.rados.0.smithi029.stdout:2103: read oid 25 snap -1 2024-04-12T11:23:38.839 INFO:tasks.rados.rados.0.smithi029.stdout:2103: expect (ObjNum 759 snap 262 seq_num 759) 2024-04-12T11:23:38.839 INFO:tasks.rados.rados.0.smithi029.stdout:2104: delete oid 32 current snap is 274 2024-04-12T11:23:38.839 INFO:tasks.rados.rados.0.smithi029.stdout:2102: finishing write tid 1 to smithi02979277-33 2024-04-12T11:23:38.839 INFO:tasks.rados.rados.0.smithi029.stdout:2102: finishing write tid 2 to smithi02979277-33 2024-04-12T11:23:38.840 INFO:tasks.rados.rados.0.smithi029.stdout:2104: done (2 left) 2024-04-12T11:23:38.840 INFO:tasks.rados.rados.0.smithi029.stdout:2105: write oid 40 current snap is 274 2024-04-12T11:23:38.840 INFO:tasks.rados.rados.0.smithi029.stdout:2105: seq_num 787 ranges {427109=405322,1553564=545926,2579307=647802} 2024-04-12T11:23:38.845 INFO:tasks.rados.rados.0.smithi029.stdout:2105: writing smithi02979277-40 from 427109 to 832431 tid 1 2024-04-12T11:23:38.852 INFO:tasks.rados.rados.0.smithi029.stdout:2105: writing smithi02979277-40 from 1553564 to 2099490 tid 2 2024-04-12T11:23:38.858 INFO:tasks.rados.rados.0.smithi029.stdout:2105: writing smithi02979277-40 from 2579307 to 3227109 tid 3 2024-04-12T11:23:38.858 INFO:tasks.rados.rados.0.smithi029.stdout:2106: snap_remove snap 271 2024-04-12T11:23:38.873 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:38 smithi087 ceph-mon[89014]: pgmap v555: 89 pgs: 9 active+undersized+degraded, 20 active+undersized, 60 active+clean; 193 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 3.3 MiB/s rd, 638 KiB/s wr, 5 op/s; 35/390 objects degraded (8.974%); 1.2 MiB/s, 1 objects/s recovering 2024-04-12T11:23:38.873 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:38 smithi087 ceph-mon[89014]: osdmap e613: 8 total, 8 up, 8 in 2024-04-12T11:23:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:38 smithi029 ceph-mon[97787]: pgmap v555: 89 pgs: 9 active+undersized+degraded, 20 active+undersized, 60 active+clean; 193 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 3.3 MiB/s rd, 638 KiB/s wr, 5 op/s; 35/390 objects degraded (8.974%); 1.2 MiB/s, 1 objects/s recovering 2024-04-12T11:23:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:38 smithi029 ceph-mon[97787]: osdmap e613: 8 total, 8 up, 8 in 2024-04-12T11:23:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:38 smithi029 ceph-mon[93650]: pgmap v555: 89 pgs: 9 active+undersized+degraded, 20 active+undersized, 60 active+clean; 193 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 3.3 MiB/s rd, 638 KiB/s wr, 5 op/s; 35/390 objects degraded (8.974%); 1.2 MiB/s, 1 objects/s recovering 2024-04-12T11:23:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:38 smithi029 ceph-mon[93650]: osdmap e613: 8 total, 8 up, 8 in 2024-04-12T11:23:39.644 INFO:tasks.rados.rados.0.smithi029.stdout:2106: done (3 left) 2024-04-12T11:23:39.644 INFO:tasks.rados.rados.0.smithi029.stdout:2107: delete oid 30 current snap is 274 2024-04-12T11:23:39.644 INFO:tasks.rados.rados.0.smithi029.stdout:2103: expect (ObjNum 759 snap 262 seq_num 759) 2024-04-12T11:23:39.813 INFO:tasks.rados.rados.0.smithi029.stdout:2102: finishing write tid 3 to smithi02979277-33 2024-04-12T11:23:39.813 INFO:tasks.rados.rados.0.smithi029.stdout:2102: finishing write tid 4 to smithi02979277-33 2024-04-12T11:23:39.813 INFO:tasks.rados.rados.0.smithi029.stdout:2102: finishing write tid 5 to smithi02979277-33 2024-04-12T11:23:39.813 INFO:tasks.rados.rados.0.smithi029.stdout:2102: oid 33 updating version 0 to 739 2024-04-12T11:23:39.813 INFO:tasks.rados.rados.0.smithi029.stdout:2102: oid 33 version 739 is already newer than 738 2024-04-12T11:23:39.813 INFO:tasks.rados.rados.0.smithi029.stdout:2102: oid 33 version 739 is already newer than 737 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2102: oid 33 updating version 739 to 740 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 740 (ObjNum 785 snap 274 seq_num 785) dirty exists 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2102: left oid 33 (ObjNum 785 snap 274 seq_num 785) 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2105: finishing write tid 1 to smithi02979277-40 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2105: finishing write tid 2 to smithi02979277-40 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2105: finishing write tid 3 to smithi02979277-40 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2105: finishing write tid 4 to smithi02979277-40 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2105: finishing write tid 5 to smithi02979277-40 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2105: oid 40 updating version 0 to 1302 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2105: oid 40 version 1302 is already newer than 1300 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2105: oid 40 version 1302 is already newer than 1299 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2105: oid 40 version 1302 is already newer than 1301 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 1302 (ObjNum 787 snap 274 seq_num 787) dirty exists 2024-04-12T11:23:39.814 INFO:tasks.rados.rados.0.smithi029.stdout:2105: left oid 40 (ObjNum 787 snap 274 seq_num 787) 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2102: done (3 left) 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2103: done (2 left) 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2105: done (1 left) 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2107: done (0 left) 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2108: read oid 20 snap -1 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2108: expect deleted 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2109: read oid 33 snap -1 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2109: expect (ObjNum 785 snap 274 seq_num 785) 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2110: read oid 42 snap -1 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2110: expect deleted 2024-04-12T11:23:39.815 INFO:tasks.rados.rados.0.smithi029.stdout:2111: snap_remove snap 261 2024-04-12T11:23:39.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:39 smithi087 ceph-mon[89014]: osdmap e614: 8 total, 8 up, 8 in 2024-04-12T11:23:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:23:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:23:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[97787]: osdmap e614: 8 total, 8 up, 8 in 2024-04-12T11:23:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:23:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:23:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[93650]: osdmap e614: 8 total, 8 up, 8 in 2024-04-12T11:23:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:23:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:23:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:23:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:23:40.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:40.650 INFO:tasks.rados.rados.0.smithi029.stdout:2111: done (3 left) 2024-04-12T11:23:40.650 INFO:tasks.rados.rados.0.smithi029.stdout:2112: delete oid 17 current snap is 274 2024-04-12T11:23:40.650 INFO:tasks.rados.rados.0.smithi029.stdout:2109: expect (ObjNum 785 snap 274 seq_num 785) 2024-04-12T11:23:40.864 INFO:tasks.rados.rados.0.smithi029.stdout:2108: done (3 left) 2024-04-12T11:23:40.865 INFO:tasks.rados.rados.0.smithi029.stdout:2109: done (2 left) 2024-04-12T11:23:40.865 INFO:tasks.rados.rados.0.smithi029.stdout:2110: done (1 left) 2024-04-12T11:23:40.865 INFO:tasks.rados.rados.0.smithi029.stdout:2112: done (0 left) 2024-04-12T11:23:40.865 INFO:tasks.rados.rados.0.smithi029.stdout:2113: snap_remove snap 257 2024-04-12T11:23:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:40 smithi087 ceph-mon[89014]: pgmap v558: 89 pgs: 3 active+undersized+degraded, 12 active+undersized, 74 active+clean; 188 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 6.1 MiB/s rd, 336 KiB/s wr, 7 op/s; 13/390 objects degraded (3.333%); 1.6 MiB/s, 1 objects/s recovering 2024-04-12T11:23:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:40 smithi087 ceph-mon[89014]: osdmap e615: 8 total, 8 up, 8 in 2024-04-12T11:23:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:40 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13/390 objects degraded (3.333%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:40 smithi029 ceph-mon[97787]: pgmap v558: 89 pgs: 3 active+undersized+degraded, 12 active+undersized, 74 active+clean; 188 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 6.1 MiB/s rd, 336 KiB/s wr, 7 op/s; 13/390 objects degraded (3.333%); 1.6 MiB/s, 1 objects/s recovering 2024-04-12T11:23:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:40 smithi029 ceph-mon[97787]: osdmap e615: 8 total, 8 up, 8 in 2024-04-12T11:23:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:40 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13/390 objects degraded (3.333%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:40 smithi029 ceph-mon[93650]: pgmap v558: 89 pgs: 3 active+undersized+degraded, 12 active+undersized, 74 active+clean; 188 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 6.1 MiB/s rd, 336 KiB/s wr, 7 op/s; 13/390 objects degraded (3.333%); 1.6 MiB/s, 1 objects/s recovering 2024-04-12T11:23:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:40 smithi029 ceph-mon[93650]: osdmap e615: 8 total, 8 up, 8 in 2024-04-12T11:23:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:40 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13/390 objects degraded (3.333%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:23:41.656 INFO:tasks.rados.rados.0.smithi029.stdout:2113: done (0 left) 2024-04-12T11:23:41.656 INFO:tasks.rados.rados.0.smithi029.stdout:2114: snap_create 2024-04-12T11:23:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:41 smithi087 ceph-mon[89014]: osdmap e616: 8 total, 8 up, 8 in 2024-04-12T11:23:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:41 smithi029 ceph-mon[93650]: osdmap e616: 8 total, 8 up, 8 in 2024-04-12T11:23:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:41 smithi029 ceph-mon[97787]: osdmap e616: 8 total, 8 up, 8 in 2024-04-12T11:23:42.585 INFO:tasks.rados.rados.0.smithi029.stdout:2114: done (0 left) 2024-04-12T11:23:42.585 INFO:tasks.rados.rados.0.smithi029.stdout:2115: write oid 17 current snap is 275 2024-04-12T11:23:42.585 INFO:tasks.rados.rados.0.smithi029.stdout:2115: seq_num 790 ranges {477530=631069,1612657=409606,2545223=517596} 2024-04-12T11:23:42.591 INFO:tasks.rados.rados.0.smithi029.stdout:2115: writing smithi02979277-17 from 477530 to 1108599 tid 1 2024-04-12T11:23:42.596 INFO:tasks.rados.rados.0.smithi029.stdout:2115: writing smithi02979277-17 from 1612657 to 2022263 tid 2 2024-04-12T11:23:42.602 INFO:tasks.rados.rados.0.smithi029.stdout:2115: writing smithi02979277-17 from 2545223 to 3062819 tid 3 2024-04-12T11:23:42.602 INFO:tasks.rados.rados.0.smithi029.stdout:2116: write oid 49 current snap is 275 2024-04-12T11:23:42.602 INFO:tasks.rados.rados.0.smithi029.stdout:2115: finishing write tid 1 to smithi02979277-17 2024-04-12T11:23:42.602 INFO:tasks.rados.rados.0.smithi029.stdout:2115: finishing write tid 2 to smithi02979277-17 2024-04-12T11:23:42.602 INFO:tasks.rados.rados.0.smithi029.stdout:2116: seq_num 791 ranges {494337=422671,1387492=553264,2688999=445221} 2024-04-12T11:23:42.607 INFO:tasks.rados.rados.0.smithi029.stdout:2116: writing smithi02979277-49 from 494337 to 917008 tid 1 2024-04-12T11:23:42.613 INFO:tasks.rados.rados.0.smithi029.stdout:2116: writing smithi02979277-49 from 1387492 to 1940756 tid 2 2024-04-12T11:23:42.620 INFO:tasks.rados.rados.0.smithi029.stdout:2116: writing smithi02979277-49 from 2688999 to 3134220 tid 3 2024-04-12T11:23:42.620 INFO:tasks.rados.rados.0.smithi029.stdout:2117: delete oid 4 current snap is 275 2024-04-12T11:23:42.620 INFO:tasks.rados.rados.0.smithi029.stdout:2115: finishing write tid 3 to smithi02979277-17 2024-04-12T11:23:42.620 INFO:tasks.rados.rados.0.smithi029.stdout:2115: finishing write tid 4 to smithi02979277-17 2024-04-12T11:23:42.620 INFO:tasks.rados.rados.0.smithi029.stdout:2115: finishing write tid 5 to smithi02979277-17 2024-04-12T11:23:42.620 INFO:tasks.rados.rados.0.smithi029.stdout:2115: oid 17 updating version 0 to 1014 2024-04-12T11:23:42.620 INFO:tasks.rados.rados.0.smithi029.stdout:2115: oid 17 updating version 1014 to 1016 2024-04-12T11:23:42.620 INFO:tasks.rados.rados.0.smithi029.stdout:2115: oid 17 version 1016 is already newer than 1015 2024-04-12T11:23:42.621 INFO:tasks.rados.rados.0.smithi029.stdout:2115: oid 17 updating version 1016 to 1017 2024-04-12T11:23:42.621 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1017 (ObjNum 790 snap 275 seq_num 790) dirty exists 2024-04-12T11:23:42.621 INFO:tasks.rados.rados.0.smithi029.stdout:2115: left oid 17 (ObjNum 790 snap 275 seq_num 790) 2024-04-12T11:23:42.621 INFO:tasks.rados.rados.0.smithi029.stdout:2116: finishing write tid 1 to smithi02979277-49 2024-04-12T11:23:42.621 INFO:tasks.rados.rados.0.smithi029.stdout:2116: finishing write tid 2 to smithi02979277-49 2024-04-12T11:23:42.622 INFO:tasks.rados.rados.0.smithi029.stdout:2115: done (2 left) 2024-04-12T11:23:42.622 INFO:tasks.rados.rados.0.smithi029.stdout:2117: done (1 left) 2024-04-12T11:23:42.622 INFO:tasks.rados.rados.0.smithi029.stdout:2118: snap_create 2024-04-12T11:23:42.623 INFO:tasks.rados.rados.0.smithi029.stdout:2116: finishing write tid 3 to smithi02979277-49 2024-04-12T11:23:42.623 INFO:tasks.rados.rados.0.smithi029.stdout:2116: finishing write tid 4 to smithi02979277-49 2024-04-12T11:23:42.623 INFO:tasks.rados.rados.0.smithi029.stdout:2116: finishing write tid 5 to smithi02979277-49 2024-04-12T11:23:42.623 INFO:tasks.rados.rados.0.smithi029.stdout:2116: oid 49 updating version 0 to 739 2024-04-12T11:23:42.623 INFO:tasks.rados.rados.0.smithi029.stdout:2116: oid 49 updating version 739 to 740 2024-04-12T11:23:42.623 INFO:tasks.rados.rados.0.smithi029.stdout:2116: oid 49 version 740 is already newer than 737 2024-04-12T11:23:42.623 INFO:tasks.rados.rados.0.smithi029.stdout:2116: oid 49 version 740 is already newer than 738 2024-04-12T11:23:42.623 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 740 (ObjNum 791 snap 275 seq_num 791) dirty exists 2024-04-12T11:23:42.623 INFO:tasks.rados.rados.0.smithi029.stdout:2116: left oid 49 (ObjNum 791 snap 275 seq_num 791) 2024-04-12T11:23:42.653 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:23:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:23:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:23:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:42 smithi087 ceph-mon[89014]: pgmap v561: 89 pgs: 89 active+clean; 190 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 6.1 MiB/s rd, 352 KiB/s wr, 6 op/s; 957 KiB/s, 0 objects/s recovering 2024-04-12T11:23:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:42 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/390 objects degraded (3.333%), 3 pgs degraded) 2024-04-12T11:23:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:42 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:23:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:42 smithi087 ceph-mon[89014]: osdmap e617: 8 total, 8 up, 8 in 2024-04-12T11:23:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:42 smithi087 ceph-mon[89014]: osdmap e618: 8 total, 8 up, 8 in 2024-04-12T11:23:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:42 smithi029 ceph-mon[93650]: pgmap v561: 89 pgs: 89 active+clean; 190 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 6.1 MiB/s rd, 352 KiB/s wr, 6 op/s; 957 KiB/s, 0 objects/s recovering 2024-04-12T11:23:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:42 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/390 objects degraded (3.333%), 3 pgs degraded) 2024-04-12T11:23:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:42 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:23:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:42 smithi029 ceph-mon[93650]: osdmap e617: 8 total, 8 up, 8 in 2024-04-12T11:23:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:42 smithi029 ceph-mon[93650]: osdmap e618: 8 total, 8 up, 8 in 2024-04-12T11:23:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:42 smithi029 ceph-mon[97787]: pgmap v561: 89 pgs: 89 active+clean; 190 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 6.1 MiB/s rd, 352 KiB/s wr, 6 op/s; 957 KiB/s, 0 objects/s recovering 2024-04-12T11:23:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:42 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/390 objects degraded (3.333%), 3 pgs degraded) 2024-04-12T11:23:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:42 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:23:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:42 smithi029 ceph-mon[97787]: osdmap e617: 8 total, 8 up, 8 in 2024-04-12T11:23:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:42 smithi029 ceph-mon[97787]: osdmap e618: 8 total, 8 up, 8 in 2024-04-12T11:23:43.587 INFO:tasks.rados.rados.0.smithi029.stdout:2116: done (1 left) 2024-04-12T11:23:43.587 INFO:tasks.rados.rados.0.smithi029.stdout:2118: done (0 left) 2024-04-12T11:23:43.587 INFO:tasks.rados.rados.0.smithi029.stdout:2119: read oid 4 snap 273 2024-04-12T11:23:43.587 INFO:tasks.rados.rados.0.smithi029.stdout:2119: expect deleted 2024-04-12T11:23:43.587 INFO:tasks.rados.rados.0.smithi029.stdout:2120: snap_create 2024-04-12T11:23:44.588 INFO:tasks.rados.rados.0.smithi029.stdout:2119: done (1 left) 2024-04-12T11:23:44.588 INFO:tasks.rados.rados.0.smithi029.stdout:2120: done (0 left) 2024-04-12T11:23:44.588 INFO:tasks.rados.rados.0.smithi029.stdout:2121: snap_remove snap 238 2024-04-12T11:23:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:44 smithi029 ceph-mon[93650]: pgmap v564: 89 pgs: 89 active+clean; 188 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 682 KiB/s wr, 6 op/s; 647 KiB/s, 0 objects/s recovering 2024-04-12T11:23:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:44 smithi029 ceph-mon[93650]: osdmap e619: 8 total, 8 up, 8 in 2024-04-12T11:23:44.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:44 smithi029 ceph-mon[97787]: pgmap v564: 89 pgs: 89 active+clean; 188 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 682 KiB/s wr, 6 op/s; 647 KiB/s, 0 objects/s recovering 2024-04-12T11:23:44.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:44 smithi029 ceph-mon[97787]: osdmap e619: 8 total, 8 up, 8 in 2024-04-12T11:23:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:44 smithi087 ceph-mon[89014]: pgmap v564: 89 pgs: 89 active+clean; 188 MiB data, 692 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 682 KiB/s wr, 6 op/s; 647 KiB/s, 0 objects/s recovering 2024-04-12T11:23:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:44 smithi087 ceph-mon[89014]: osdmap e619: 8 total, 8 up, 8 in 2024-04-12T11:23:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:23:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:23:44] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.43.0" 2024-04-12T11:23:45.600 INFO:tasks.rados.rados.0.smithi029.stdout:2121: done (0 left) 2024-04-12T11:23:45.600 INFO:tasks.rados.rados.0.smithi029.stdout:2122: snap_remove snap 273 2024-04-12T11:23:45.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:45 smithi087 ceph-mon[89014]: osdmap e620: 8 total, 8 up, 8 in 2024-04-12T11:23:46.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:45 smithi029 ceph-mon[93650]: osdmap e620: 8 total, 8 up, 8 in 2024-04-12T11:23:46.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:45 smithi029 ceph-mon[97787]: osdmap e620: 8 total, 8 up, 8 in 2024-04-12T11:23:46.600 INFO:tasks.rados.rados.0.smithi029.stdout:2122: done (0 left) 2024-04-12T11:23:46.601 INFO:tasks.rados.rados.0.smithi029.stdout:2123: delete oid 34 current snap is 277 2024-04-12T11:23:46.610 INFO:tasks.rados.rados.0.smithi029.stdout:2123: done (0 left) 2024-04-12T11:23:46.610 INFO:tasks.rados.rados.0.smithi029.stdout:2124: write oid 31 current snap is 277 2024-04-12T11:23:46.610 INFO:tasks.rados.rados.0.smithi029.stdout:2124: seq_num 794 ranges {544758=648418,1846585=416944,2771268=573490} 2024-04-12T11:23:46.616 INFO:tasks.rados.rados.0.smithi029.stdout:2124: writing smithi02979277-31 from 544758 to 1193176 tid 1 2024-04-12T11:23:46.622 INFO:tasks.rados.rados.0.smithi029.stdout:2124: writing smithi02979277-31 from 1846585 to 2263529 tid 2 2024-04-12T11:23:46.629 INFO:tasks.rados.rados.0.smithi029.stdout:2124: writing smithi02979277-31 from 2771268 to 3344758 tid 3 2024-04-12T11:23:46.629 INFO:tasks.rados.rados.0.smithi029.stdout:2125: read oid 23 snap 259 2024-04-12T11:23:46.629 INFO:tasks.rados.rados.0.smithi029.stdout:2125: expect (ObjNum 753 snap 259 seq_num 753) 2024-04-12T11:23:46.629 INFO:tasks.rados.rados.0.smithi029.stdout:2126: snap_create 2024-04-12T11:23:46.629 INFO:tasks.rados.rados.0.smithi029.stdout:2124: finishing write tid 1 to smithi02979277-31 2024-04-12T11:23:46.629 INFO:tasks.rados.rados.0.smithi029.stdout:2124: finishing write tid 2 to smithi02979277-31 2024-04-12T11:23:46.631 INFO:tasks.rados.rados.0.smithi029.stdout:2124: finishing write tid 3 to smithi02979277-31 2024-04-12T11:23:46.632 INFO:tasks.rados.rados.0.smithi029.stdout:2124: finishing write tid 4 to smithi02979277-31 2024-04-12T11:23:46.632 INFO:tasks.rados.rados.0.smithi029.stdout:2124: finishing write tid 5 to smithi02979277-31 2024-04-12T11:23:46.632 INFO:tasks.rados.rados.0.smithi029.stdout:2124: oid 31 updating version 0 to 864 2024-04-12T11:23:46.632 INFO:tasks.rados.rados.0.smithi029.stdout:2124: oid 31 updating version 864 to 866 2024-04-12T11:23:46.632 INFO:tasks.rados.rados.0.smithi029.stdout:2124: oid 31 updating version 866 to 867 2024-04-12T11:23:46.632 INFO:tasks.rados.rados.0.smithi029.stdout:2124: oid 31 version 867 is already newer than 865 2024-04-12T11:23:46.632 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 867 (ObjNum 794 snap 277 seq_num 794) dirty exists 2024-04-12T11:23:46.632 INFO:tasks.rados.rados.0.smithi029.stdout:2124: left oid 31 (ObjNum 794 snap 277 seq_num 794) 2024-04-12T11:23:46.638 INFO:tasks.rados.rados.0.smithi029.stdout:2125: expect (ObjNum 753 snap 259 seq_num 753) 2024-04-12T11:23:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:46 smithi087 ceph-mon[89014]: pgmap v567: 89 pgs: 89 active+clean; 189 MiB data, 696 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 721 KiB/s wr, 7 op/s 2024-04-12T11:23:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:46 smithi087 ceph-mon[89014]: osdmap e621: 8 total, 8 up, 8 in 2024-04-12T11:23:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:46 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:46 smithi029 ceph-mon[93650]: pgmap v567: 89 pgs: 89 active+clean; 189 MiB data, 696 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 721 KiB/s wr, 7 op/s 2024-04-12T11:23:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:46 smithi029 ceph-mon[93650]: osdmap e621: 8 total, 8 up, 8 in 2024-04-12T11:23:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:46 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:46 smithi029 ceph-mon[97787]: pgmap v567: 89 pgs: 89 active+clean; 189 MiB data, 696 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 721 KiB/s wr, 7 op/s 2024-04-12T11:23:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:46 smithi029 ceph-mon[97787]: osdmap e621: 8 total, 8 up, 8 in 2024-04-12T11:23:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:46 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:47.586 INFO:tasks.rados.rados.0.smithi029.stdout:2124: done (2 left) 2024-04-12T11:23:47.586 INFO:tasks.rados.rados.0.smithi029.stdout:2125: done (1 left) 2024-04-12T11:23:47.587 INFO:tasks.rados.rados.0.smithi029.stdout:2126: done (0 left) 2024-04-12T11:23:47.587 INFO:tasks.rados.rados.0.smithi029.stdout:2127: write oid 9 current snap is 278 2024-04-12T11:23:47.587 INFO:tasks.rados.rados.0.smithi029.stdout:2127: seq_num 795 ranges {561565=723667,1905067=560602,3082338=279227} 2024-04-12T11:23:47.593 INFO:tasks.rados.rados.0.smithi029.stdout:2127: writing smithi02979277-9 from 561565 to 1285232 tid 1 2024-04-12T11:23:47.600 INFO:tasks.rados.rados.0.smithi029.stdout:2127: writing smithi02979277-9 from 1905067 to 2465669 tid 2 2024-04-12T11:23:47.606 INFO:tasks.rados.rados.0.smithi029.stdout:2127: writing smithi02979277-9 from 3082338 to 3361565 tid 3 2024-04-12T11:23:47.606 INFO:tasks.rados.rados.0.smithi029.stdout:2128: read oid 10 snap 269 2024-04-12T11:23:47.606 INFO:tasks.rados.rados.0.smithi029.stdout:2128: expect (ObjNum 635 snap 205 seq_num 635) 2024-04-12T11:23:47.606 INFO:tasks.rados.rados.0.smithi029.stdout:2129: snap_create 2024-04-12T11:23:47.606 INFO:tasks.rados.rados.0.smithi029.stdout:2127: finishing write tid 1 to smithi02979277-9 2024-04-12T11:23:47.606 INFO:tasks.rados.rados.0.smithi029.stdout:2127: finishing write tid 2 to smithi02979277-9 2024-04-12T11:23:47.611 INFO:tasks.rados.rados.0.smithi029.stdout:2127: finishing write tid 3 to smithi02979277-9 2024-04-12T11:23:47.611 INFO:tasks.rados.rados.0.smithi029.stdout:2127: finishing write tid 4 to smithi02979277-9 2024-04-12T11:23:47.611 INFO:tasks.rados.rados.0.smithi029.stdout:2127: finishing write tid 5 to smithi02979277-9 2024-04-12T11:23:47.611 INFO:tasks.rados.rados.0.smithi029.stdout:2127: oid 9 updating version 0 to 746 2024-04-12T11:23:47.612 INFO:tasks.rados.rados.0.smithi029.stdout:2127: oid 9 updating version 746 to 747 2024-04-12T11:23:47.612 INFO:tasks.rados.rados.0.smithi029.stdout:2127: oid 9 version 747 is already newer than 745 2024-04-12T11:23:47.612 INFO:tasks.rados.rados.0.smithi029.stdout:2127: oid 9 version 747 is already newer than 744 2024-04-12T11:23:47.612 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 747 (ObjNum 795 snap 278 seq_num 795) dirty exists 2024-04-12T11:23:47.612 INFO:tasks.rados.rados.0.smithi029.stdout:2127: left oid 9 (ObjNum 795 snap 278 seq_num 795) 2024-04-12T11:23:47.615 INFO:tasks.rados.rados.0.smithi029.stdout:2128: expect (ObjNum 635 snap 205 seq_num 635) 2024-04-12T11:23:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:47 smithi087 ceph-mon[89014]: osdmap e622: 8 total, 8 up, 8 in 2024-04-12T11:23:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:47 smithi087 ceph-mon[89014]: osdmap e623: 8 total, 8 up, 8 in 2024-04-12T11:23:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:47 smithi029 ceph-mon[97787]: osdmap e622: 8 total, 8 up, 8 in 2024-04-12T11:23:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:47 smithi029 ceph-mon[97787]: osdmap e623: 8 total, 8 up, 8 in 2024-04-12T11:23:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:47 smithi029 ceph-mon[93650]: osdmap e622: 8 total, 8 up, 8 in 2024-04-12T11:23:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:47 smithi029 ceph-mon[93650]: osdmap e623: 8 total, 8 up, 8 in 2024-04-12T11:23:48.609 INFO:tasks.rados.rados.0.smithi029.stdout:2127: done (2 left) 2024-04-12T11:23:48.609 INFO:tasks.rados.rados.0.smithi029.stdout:2128: done (1 left) 2024-04-12T11:23:48.609 INFO:tasks.rados.rados.0.smithi029.stdout:2129: done (0 left) 2024-04-12T11:23:48.610 INFO:tasks.rados.rados.0.smithi029.stdout:2130: read oid 3 snap -1 2024-04-12T11:23:48.610 INFO:tasks.rados.rados.0.smithi029.stdout:2130: expect (ObjNum 779 snap 272 seq_num 779) 2024-04-12T11:23:48.610 INFO:tasks.rados.rados.0.smithi029.stdout:2131: write oid 39 current snap is 279 2024-04-12T11:23:48.610 INFO:tasks.rados.rados.0.smithi029.stdout:2131: seq_num 796 ranges {578372=798916,1963549=420613,2826114=417934} 2024-04-12T11:23:48.617 INFO:tasks.rados.rados.0.smithi029.stdout:2131: writing smithi02979277-39 from 578372 to 1377288 tid 1 2024-04-12T11:23:48.623 INFO:tasks.rados.rados.0.smithi029.stdout:2131: writing smithi02979277-39 from 1963549 to 2384162 tid 2 2024-04-12T11:23:48.628 INFO:tasks.rados.rados.0.smithi029.stdout:2131: writing smithi02979277-39 from 2826114 to 3244048 tid 3 2024-04-12T11:23:48.628 INFO:tasks.rados.rados.0.smithi029.stdout:2132: read oid 5 snap 275 2024-04-12T11:23:48.628 INFO:tasks.rados.rados.0.smithi029.stdout:2132: expect (ObjNum 708 snap 240 seq_num 708) 2024-04-12T11:23:48.629 INFO:tasks.rados.rados.0.smithi029.stdout:2133: snap_create 2024-04-12T11:23:48.629 INFO:tasks.rados.rados.0.smithi029.stdout:2130: expect (ObjNum 779 snap 272 seq_num 779) 2024-04-12T11:23:48.816 INFO:tasks.rados.rados.0.smithi029.stdout:2131: finishing write tid 1 to smithi02979277-39 2024-04-12T11:23:48.816 INFO:tasks.rados.rados.0.smithi029.stdout:2131: finishing write tid 2 to smithi02979277-39 2024-04-12T11:23:48.816 INFO:tasks.rados.rados.0.smithi029.stdout:2131: finishing write tid 3 to smithi02979277-39 2024-04-12T11:23:48.816 INFO:tasks.rados.rados.0.smithi029.stdout:2131: finishing write tid 4 to smithi02979277-39 2024-04-12T11:23:48.816 INFO:tasks.rados.rados.0.smithi029.stdout:2131: finishing write tid 5 to smithi02979277-39 2024-04-12T11:23:48.816 INFO:tasks.rados.rados.0.smithi029.stdout:2131: oid 39 updating version 0 to 250 2024-04-12T11:23:48.816 INFO:tasks.rados.rados.0.smithi029.stdout:2131: oid 39 updating version 250 to 251 2024-04-12T11:23:48.816 INFO:tasks.rados.rados.0.smithi029.stdout:2131: oid 39 updating version 251 to 252 2024-04-12T11:23:48.816 INFO:tasks.rados.rados.0.smithi029.stdout:2131: oid 39 updating version 252 to 253 2024-04-12T11:23:48.816 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 253 (ObjNum 796 snap 279 seq_num 796) dirty exists 2024-04-12T11:23:48.817 INFO:tasks.rados.rados.0.smithi029.stdout:2131: left oid 39 (ObjNum 796 snap 279 seq_num 796) 2024-04-12T11:23:48.817 INFO:tasks.rados.rados.0.smithi029.stdout:2132: expect (ObjNum 708 snap 240 seq_num 708) 2024-04-12T11:23:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:48 smithi087 ceph-mon[89014]: pgmap v570: 89 pgs: 89 active+clean; 192 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 772 KiB/s wr, 5 op/s 2024-04-12T11:23:49.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:48 smithi029 ceph-mon[93650]: pgmap v570: 89 pgs: 89 active+clean; 192 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 772 KiB/s wr, 5 op/s 2024-04-12T11:23:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:48 smithi029 ceph-mon[97787]: pgmap v570: 89 pgs: 89 active+clean; 192 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 772 KiB/s wr, 5 op/s 2024-04-12T11:23:49.619 INFO:tasks.rados.rados.0.smithi029.stdout:2130: done (3 left) 2024-04-12T11:23:49.619 INFO:tasks.rados.rados.0.smithi029.stdout:2131: done (2 left) 2024-04-12T11:23:49.619 INFO:tasks.rados.rados.0.smithi029.stdout:2132: done (1 left) 2024-04-12T11:23:49.619 INFO:tasks.rados.rados.0.smithi029.stdout:2133: done (0 left) 2024-04-12T11:23:49.619 INFO:tasks.rados.rados.0.smithi029.stdout:2134: delete oid 11 current snap is 280 2024-04-12T11:23:49.623 INFO:tasks.rados.rados.0.smithi029.stdout:2134: done (0 left) 2024-04-12T11:23:49.623 INFO:tasks.rados.rados.0.smithi029.stdout:2135: snap_create 2024-04-12T11:23:49.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:49 smithi087 ceph-mon[89014]: osdmap e624: 8 total, 8 up, 8 in 2024-04-12T11:23:50.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:49 smithi029 ceph-mon[93650]: osdmap e624: 8 total, 8 up, 8 in 2024-04-12T11:23:50.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:49 smithi029 ceph-mon[97787]: osdmap e624: 8 total, 8 up, 8 in 2024-04-12T11:23:50.620 INFO:tasks.rados.rados.0.smithi029.stdout:2135: done (0 left) 2024-04-12T11:23:50.620 INFO:tasks.rados.rados.0.smithi029.stdout:2136: read oid 43 snap -1 2024-04-12T11:23:50.621 INFO:tasks.rados.rados.0.smithi029.stdout:2136: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:23:50.621 INFO:tasks.rados.rados.0.smithi029.stdout:2137: read oid 1 snap -1 2024-04-12T11:23:50.621 INFO:tasks.rados.rados.0.smithi029.stdout:2137: expect deleted 2024-04-12T11:23:50.621 INFO:tasks.rados.rados.0.smithi029.stdout:2138: write oid 42 current snap is 281 2024-04-12T11:23:50.621 INFO:tasks.rados.rados.0.smithi029.stdout:2138: seq_num 798 ranges {611986=549414,1564160=424282,2764607=440478} 2024-04-12T11:23:50.626 INFO:tasks.rados.rados.0.smithi029.stdout:2138: writing smithi02979277-42 from 611986 to 1161400 tid 1 2024-04-12T11:23:50.631 INFO:tasks.rados.rados.0.smithi029.stdout:2138: writing smithi02979277-42 from 1564160 to 1988442 tid 2 2024-04-12T11:23:50.638 INFO:tasks.rados.rados.0.smithi029.stdout:2138: writing smithi02979277-42 from 2764607 to 3205085 tid 3 2024-04-12T11:23:50.639 INFO:tasks.rados.rados.0.smithi029.stdout:2139: delete oid 34 current snap is 281 2024-04-12T11:23:50.639 INFO:tasks.rados.rados.0.smithi029.stdout:2138: finishing write tid 1 to smithi02979277-42 2024-04-12T11:23:50.639 INFO:tasks.rados.rados.0.smithi029.stdout:2136: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2138: finishing write tid 2 to smithi02979277-42 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2138: finishing write tid 3 to smithi02979277-42 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2138: finishing write tid 4 to smithi02979277-42 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2138: finishing write tid 5 to smithi02979277-42 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2138: oid 42 updating version 0 to 303 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2138: oid 42 version 303 is already newer than 300 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2138: oid 42 version 303 is already newer than 302 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2138: oid 42 version 303 is already newer than 301 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 303 (ObjNum 798 snap 281 seq_num 798) dirty exists 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2138: left oid 42 (ObjNum 798 snap 281 seq_num 798) 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2136: done (3 left) 2024-04-12T11:23:50.748 INFO:tasks.rados.rados.0.smithi029.stdout:2137: done (2 left) 2024-04-12T11:23:50.749 INFO:tasks.rados.rados.0.smithi029.stdout:2138: done (1 left) 2024-04-12T11:23:50.749 INFO:tasks.rados.rados.0.smithi029.stdout:2139: done (0 left) 2024-04-12T11:23:50.749 INFO:tasks.rados.rados.0.smithi029.stdout:2140: rollback oid 37 current snap is 281 2024-04-12T11:23:50.749 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 242 2024-04-12T11:23:50.749 INFO:tasks.rados.rados.0.smithi029.stdout:2141: snap_remove snap 255 2024-04-12T11:23:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:50 smithi087 ceph-mon[89014]: pgmap v573: 89 pgs: 89 active+clean; 199 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-04-12T11:23:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:50 smithi087 ceph-mon[89014]: osdmap e625: 8 total, 8 up, 8 in 2024-04-12T11:23:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:50 smithi029 ceph-mon[97787]: pgmap v573: 89 pgs: 89 active+clean; 199 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-04-12T11:23:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:50 smithi029 ceph-mon[97787]: osdmap e625: 8 total, 8 up, 8 in 2024-04-12T11:23:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:50 smithi029 ceph-mon[93650]: pgmap v573: 89 pgs: 89 active+clean; 199 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-04-12T11:23:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:50 smithi029 ceph-mon[93650]: osdmap e625: 8 total, 8 up, 8 in 2024-04-12T11:23:51.630 INFO:tasks.rados.rados.0.smithi029.stdout:2141: done (1 left) 2024-04-12T11:23:51.631 INFO:tasks.rados.rados.0.smithi029.stdout:2142: read oid 2 snap -1 2024-04-12T11:23:51.631 INFO:tasks.rados.rados.0.smithi029.stdout:2142: expect deleted 2024-04-12T11:23:51.631 INFO:tasks.rados.rados.0.smithi029.stdout:2143: snap_create 2024-04-12T11:23:51.631 INFO:tasks.rados.rados.0.smithi029.stdout:2140: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:23:51.631 INFO:tasks.rados.rados.0.smithi029.stdout:2140: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:23:51.631 INFO:tasks.rados.rados.0.smithi029.stdout:2140: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:23:51.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 751 (ObjNum 692 snap 224 seq_num 692) dirty exists 2024-04-12T11:23:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:51 smithi087 ceph-mon[89014]: osdmap e626: 8 total, 8 up, 8 in 2024-04-12T11:23:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:51 smithi029 ceph-mon[93650]: osdmap e626: 8 total, 8 up, 8 in 2024-04-12T11:23:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:51 smithi029 ceph-mon[97787]: osdmap e626: 8 total, 8 up, 8 in 2024-04-12T11:23:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:23:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:23:52.588 INFO:tasks.rados.rados.0.smithi029.stdout:2140: done (2 left) 2024-04-12T11:23:52.588 INFO:tasks.rados.rados.0.smithi029.stdout:2142: done (1 left) 2024-04-12T11:23:52.589 INFO:tasks.rados.rados.0.smithi029.stdout:2143: done (0 left) 2024-04-12T11:23:52.589 INFO:tasks.rados.rados.0.smithi029.stdout:2144: delete oid 13 current snap is 282 2024-04-12T11:23:52.593 INFO:tasks.rados.rados.0.smithi029.stdout:2144: done (0 left) 2024-04-12T11:23:52.593 INFO:tasks.rados.rados.0.smithi029.stdout:2145: write oid 20 current snap is 282 2024-04-12T11:23:52.594 INFO:tasks.rados.rados.0.smithi029.stdout:2145: seq_num 801 ranges {662407=491514,1855959=571609,2846876=615531} 2024-04-12T11:23:52.600 INFO:tasks.rados.rados.0.smithi029.stdout:2145: writing smithi02979277-20 from 662407 to 1153921 tid 1 2024-04-12T11:23:52.607 INFO:tasks.rados.rados.0.smithi029.stdout:2145: writing smithi02979277-20 from 1855959 to 2427568 tid 2 2024-04-12T11:23:52.613 INFO:tasks.rados.rados.0.smithi029.stdout:2145: writing smithi02979277-20 from 2846876 to 3462407 tid 3 2024-04-12T11:23:52.613 INFO:tasks.rados.rados.0.smithi029.stdout:2146: rollback oid 43 current snap is 282 2024-04-12T11:23:52.613 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 280 2024-04-12T11:23:52.613 INFO:tasks.rados.rados.0.smithi029.stdout:2145: finishing write tid 1 to smithi02979277-20 2024-04-12T11:23:52.613 INFO:tasks.rados.rados.0.smithi029.stdout:2145: finishing write tid 2 to smithi02979277-20 2024-04-12T11:23:52.613 INFO:tasks.rados.rados.0.smithi029.stdout:2147: rollback oid 5 current snap is 282 2024-04-12T11:23:52.613 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 5 to 269 2024-04-12T11:23:52.614 INFO:tasks.rados.rados.0.smithi029.stdout:2148: write oid 16 current snap is 282 2024-04-12T11:23:52.614 INFO:tasks.rados.rados.0.smithi029.stdout:2148: seq_num 802 ranges {679214=566763,1798088=715267,3157946=321268} 2024-04-12T11:23:52.620 INFO:tasks.rados.rados.0.smithi029.stdout:2148: writing smithi02979277-16 from 679214 to 1245977 tid 1 2024-04-12T11:23:52.627 INFO:tasks.rados.rados.0.smithi029.stdout:2148: writing smithi02979277-16 from 1798088 to 2513355 tid 2 2024-04-12T11:23:52.628 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:23:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:23:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:23:52.633 INFO:tasks.rados.rados.0.smithi029.stdout:2148: writing smithi02979277-16 from 3157946 to 3479214 tid 3 2024-04-12T11:23:52.633 INFO:tasks.rados.rados.0.smithi029.stdout:2149: read oid 42 snap 269 2024-04-12T11:23:52.633 INFO:tasks.rados.rados.0.smithi029.stdout:2149: expect deleted 2024-04-12T11:23:52.634 INFO:tasks.rados.rados.0.smithi029.stdout:2150: write oid 17 current snap is 282 2024-04-12T11:23:52.634 INFO:tasks.rados.rados.0.smithi029.stdout:2145: finishing write tid 3 to smithi02979277-20 2024-04-12T11:23:52.634 INFO:tasks.rados.rados.0.smithi029.stdout:2150: seq_num 803 ranges {696021=642012,1856570=575278,3185369=310652} 2024-04-12T11:23:52.640 INFO:tasks.rados.rados.0.smithi029.stdout:2150: writing smithi02979277-17 from 696021 to 1338033 tid 1 2024-04-12T11:23:52.647 INFO:tasks.rados.rados.0.smithi029.stdout:2150: writing smithi02979277-17 from 1856570 to 2431848 tid 2 2024-04-12T11:23:52.653 INFO:tasks.rados.rados.0.smithi029.stdout:2150: writing smithi02979277-17 from 3185369 to 3496021 tid 3 2024-04-12T11:23:52.653 INFO:tasks.rados.rados.0.smithi029.stdout:2151: write oid 48 current snap is 282 2024-04-12T11:23:52.653 INFO:tasks.rados.rados.0.smithi029.stdout:2145: finishing write tid 4 to smithi02979277-20 2024-04-12T11:23:52.653 INFO:tasks.rados.rados.0.smithi029.stdout:2145: finishing write tid 5 to smithi02979277-20 2024-04-12T11:23:52.653 INFO:tasks.rados.rados.0.smithi029.stdout:2145: oid 20 updating version 0 to 673 2024-04-12T11:23:52.654 INFO:tasks.rados.rados.0.smithi029.stdout:2145: oid 20 version 673 is already newer than 670 2024-04-12T11:23:52.654 INFO:tasks.rados.rados.0.smithi029.stdout:2145: oid 20 version 673 is already newer than 672 2024-04-12T11:23:52.654 INFO:tasks.rados.rados.0.smithi029.stdout:2145: oid 20 version 673 is already newer than 671 2024-04-12T11:23:52.654 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 673 (ObjNum 801 snap 282 seq_num 801) dirty exists 2024-04-12T11:23:52.654 INFO:tasks.rados.rados.0.smithi029.stdout:2145: left oid 20 (ObjNum 801 snap 282 seq_num 801) 2024-04-12T11:23:52.654 INFO:tasks.rados.rados.0.smithi029.stdout:2146: finishing rollback tid 0 to smithi02979277-43 2024-04-12T11:23:52.654 INFO:tasks.rados.rados.0.smithi029.stdout:2151: seq_num 804 ranges {712828=717261,1915052=718936,3212792=300036} 2024-04-12T11:23:52.661 INFO:tasks.rados.rados.0.smithi029.stdout:2151: writing smithi02979277-48 from 712828 to 1430089 tid 1 2024-04-12T11:23:52.667 INFO:tasks.rados.rados.0.smithi029.stdout:2151: writing smithi02979277-48 from 1915052 to 2633988 tid 2 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2151: writing smithi02979277-48 from 3212792 to 3512828 tid 3 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2145: done (6 left) 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2152: write oid 1 current snap is 282 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2147: finishing rollback tid 0 to smithi02979277-5 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2146: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2147: finishing rollback tid 1 to smithi02979277-5 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2146: finishing rollback tid 2 to smithi02979277-43 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 836 (ObjNum 427 snap 135 seq_num 427) dirty exists 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2147: finishing rollback tid 2 to smithi02979277-5 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 994 (ObjNum 708 snap 240 seq_num 708) dirty exists 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2148: finishing write tid 1 to smithi02979277-16 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2148: finishing write tid 2 to smithi02979277-16 2024-04-12T11:23:52.673 INFO:tasks.rados.rados.0.smithi029.stdout:2152: seq_num 805 ranges {729635=792510,1973534=578947,3240215=289420} 2024-04-12T11:23:52.681 INFO:tasks.rados.rados.0.smithi029.stdout:2152: writing smithi02979277-1 from 729635 to 1522145 tid 1 2024-04-12T11:23:52.686 INFO:tasks.rados.rados.0.smithi029.stdout:2152: writing smithi02979277-1 from 1973534 to 2552481 tid 2 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2152: writing smithi02979277-1 from 3240215 to 3529635 tid 3 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2146: done (6 left) 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2147: done (5 left) 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2153: delete oid 5 current snap is 282 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2148: finishing write tid 3 to smithi02979277-16 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2148: finishing write tid 4 to smithi02979277-16 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2148: finishing write tid 5 to smithi02979277-16 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2148: oid 16 updating version 0 to 997 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2148: oid 16 updating version 997 to 998 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2148: oid 16 version 998 is already newer than 996 2024-04-12T11:23:52.693 INFO:tasks.rados.rados.0.smithi029.stdout:2148: oid 16 updating version 998 to 999 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 999 (ObjNum 802 snap 282 seq_num 802) dirty exists 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2148: left oid 16 (ObjNum 802 snap 282 seq_num 802) 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2150: finishing write tid 1 to smithi02979277-17 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2150: finishing write tid 2 to smithi02979277-17 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2150: finishing write tid 3 to smithi02979277-17 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2150: finishing write tid 4 to smithi02979277-17 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2150: finishing write tid 5 to smithi02979277-17 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2150: oid 17 updating version 0 to 1029 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2150: oid 17 version 1029 is already newer than 1028 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2150: oid 17 updating version 1029 to 1030 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2150: oid 17 version 1030 is already newer than 1027 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1030 (ObjNum 803 snap 282 seq_num 803) dirty exists 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2150: left oid 17 (ObjNum 803 snap 282 seq_num 803) 2024-04-12T11:23:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:2151: finishing write tid 1 to smithi02979277-48 2024-04-12T11:23:52.695 INFO:tasks.rados.rados.0.smithi029.stdout:2151: finishing write tid 2 to smithi02979277-48 2024-04-12T11:23:52.695 INFO:tasks.rados.rados.0.smithi029.stdout:2151: finishing write tid 3 to smithi02979277-48 2024-04-12T11:23:52.695 INFO:tasks.rados.rados.0.smithi029.stdout:2151: finishing write tid 4 to smithi02979277-48 2024-04-12T11:23:52.695 INFO:tasks.rados.rados.0.smithi029.stdout:2151: finishing write tid 5 to smithi02979277-48 2024-04-12T11:23:52.695 INFO:tasks.rados.rados.0.smithi029.stdout:2151: oid 48 updating version 0 to 876 2024-04-12T11:23:52.695 INFO:tasks.rados.rados.0.smithi029.stdout:2151: oid 48 version 876 is already newer than 875 2024-04-12T11:23:52.695 INFO:tasks.rados.rados.0.smithi029.stdout:2151: oid 48 updating version 876 to 878 2024-04-12T11:23:52.695 INFO:tasks.rados.rados.0.smithi029.stdout:2151: oid 48 version 878 is already newer than 877 2024-04-12T11:23:52.695 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 878 (ObjNum 804 snap 282 seq_num 804) dirty exists 2024-04-12T11:23:52.695 INFO:tasks.rados.rados.0.smithi029.stdout:2151: left oid 48 (ObjNum 804 snap 282 seq_num 804) 2024-04-12T11:23:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:2148: done (5 left) 2024-04-12T11:23:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:2149: done (4 left) 2024-04-12T11:23:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:2150: done (3 left) 2024-04-12T11:23:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:2151: done (2 left) 2024-04-12T11:23:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:2153: done (1 left) 2024-04-12T11:23:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:2154: delete oid 9 current snap is 282 2024-04-12T11:23:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:2152: finishing write tid 1 to smithi02979277-1 2024-04-12T11:23:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:2152: finishing write tid 2 to smithi02979277-1 2024-04-12T11:23:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:2152: finishing write tid 3 to smithi02979277-1 2024-04-12T11:23:52.697 INFO:tasks.rados.rados.0.smithi029.stdout:2152: finishing write tid 4 to smithi02979277-1 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2152: finishing write tid 5 to smithi02979277-1 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2152: oid 1 updating version 0 to 1057 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2152: oid 1 version 1057 is already newer than 1055 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2152: oid 1 version 1057 is already newer than 1056 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2152: oid 1 version 1057 is already newer than 1054 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1057 (ObjNum 805 snap 282 seq_num 805) dirty exists 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2152: left oid 1 (ObjNum 805 snap 282 seq_num 805) 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2152: done (1 left) 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2154: done (0 left) 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2155: read oid 43 snap -1 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2155: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2156: read oid 19 snap -1 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2156: expect deleted 2024-04-12T11:23:52.698 INFO:tasks.rados.rados.0.smithi029.stdout:2157: read oid 35 snap -1 2024-04-12T11:23:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:2157: expect deleted 2024-04-12T11:23:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:2158: rollback oid 22 current snap is 282 2024-04-12T11:23:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 22 to 269 2024-04-12T11:23:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:2159: read oid 31 snap -1 2024-04-12T11:23:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:2159: expect (ObjNum 794 snap 277 seq_num 794) 2024-04-12T11:23:52.699 INFO:tasks.rados.rados.0.smithi029.stdout:2160: snap_remove snap 278 2024-04-12T11:23:52.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:52 smithi087 ceph-mon[89014]: pgmap v576: 89 pgs: 89 active+clean; 199 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.0 MiB/s wr, 5 op/s 2024-04-12T11:23:52.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:52 smithi087 ceph-mon[89014]: osdmap e627: 8 total, 8 up, 8 in 2024-04-12T11:23:52.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:52 smithi087 ceph-mon[89014]: osdmap e628: 8 total, 8 up, 8 in 2024-04-12T11:23:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:52 smithi029 ceph-mon[97787]: pgmap v576: 89 pgs: 89 active+clean; 199 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.0 MiB/s wr, 5 op/s 2024-04-12T11:23:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:52 smithi029 ceph-mon[97787]: osdmap e627: 8 total, 8 up, 8 in 2024-04-12T11:23:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:52 smithi029 ceph-mon[97787]: osdmap e628: 8 total, 8 up, 8 in 2024-04-12T11:23:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:52 smithi029 ceph-mon[93650]: pgmap v576: 89 pgs: 89 active+clean; 199 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.0 MiB/s wr, 5 op/s 2024-04-12T11:23:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:52 smithi029 ceph-mon[93650]: osdmap e627: 8 total, 8 up, 8 in 2024-04-12T11:23:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:52 smithi029 ceph-mon[93650]: osdmap e628: 8 total, 8 up, 8 in 2024-04-12T11:23:53.637 INFO:tasks.rados.rados.0.smithi029.stdout:2158: finishing rollback tid 0 to smithi02979277-22 2024-04-12T11:23:53.637 INFO:tasks.rados.rados.0.smithi029.stdout:2158: finishing rollback tid 1 to smithi02979277-22 2024-04-12T11:23:53.637 INFO:tasks.rados.rados.0.smithi029.stdout:2158: finishing rollback tid 2 to smithi02979277-22 2024-04-12T11:23:53.637 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 782 (ObjNum 720 snap 248 seq_num 720) dirty exists 2024-04-12T11:23:53.637 INFO:tasks.rados.rados.0.smithi029.stdout:2159: expect (ObjNum 794 snap 277 seq_num 794) 2024-04-12T11:23:53.749 INFO:tasks.rados.rados.0.smithi029.stdout:2155: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:23:53.858 INFO:tasks.rados.rados.0.smithi029.stdout:2155: done (5 left) 2024-04-12T11:23:53.858 INFO:tasks.rados.rados.0.smithi029.stdout:2156: done (4 left) 2024-04-12T11:23:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:2157: done (3 left) 2024-04-12T11:23:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:2158: done (2 left) 2024-04-12T11:23:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:2159: done (1 left) 2024-04-12T11:23:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:2160: done (0 left) 2024-04-12T11:23:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:2161: read oid 12 snap -1 2024-04-12T11:23:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:2161: expect (ObjNum 670 snap 215 seq_num 670) 2024-04-12T11:23:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:2162: write oid 32 current snap is 282 2024-04-12T11:23:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:2162: seq_num 808 ranges {780056=734610,2148980=726274,3322484=257572} 2024-04-12T11:23:53.866 INFO:tasks.rados.rados.0.smithi029.stdout:2162: writing smithi02979277-32 from 780056 to 1514666 tid 1 2024-04-12T11:23:53.874 INFO:tasks.rados.rados.0.smithi029.stdout:2162: writing smithi02979277-32 from 2148980 to 2875254 tid 2 2024-04-12T11:23:53.878 INFO:tasks.rados.rados.0.smithi029.stdout:2162: writing smithi02979277-32 from 3322484 to 3580056 tid 3 2024-04-12T11:23:53.878 INFO:tasks.rados.rados.0.smithi029.stdout:2163: snap_remove snap 274 2024-04-12T11:23:54.642 INFO:tasks.rados.rados.0.smithi029.stdout:2163: done (2 left) 2024-04-12T11:23:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:2164: delete oid 17 current snap is 282 2024-04-12T11:23:54.649 INFO:tasks.rados.rados.0.smithi029.stdout:2161: expect (ObjNum 670 snap 215 seq_num 670) 2024-04-12T11:23:54.869 INFO:tasks.rados.rados.0.smithi029.stdout:2162: finishing write tid 1 to smithi02979277-32 2024-04-12T11:23:54.869 INFO:tasks.rados.rados.0.smithi029.stdout:2162: finishing write tid 2 to smithi02979277-32 2024-04-12T11:23:54.869 INFO:tasks.rados.rados.0.smithi029.stdout:2162: finishing write tid 3 to smithi02979277-32 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2162: finishing write tid 4 to smithi02979277-32 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2162: finishing write tid 5 to smithi02979277-32 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2162: oid 32 updating version 0 to 842 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2162: oid 32 version 842 is already newer than 839 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2162: oid 32 version 842 is already newer than 841 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2162: oid 32 version 842 is already newer than 840 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 842 (ObjNum 808 snap 282 seq_num 808) dirty exists 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2162: left oid 32 (ObjNum 808 snap 282 seq_num 808) 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2161: done (2 left) 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2162: done (1 left) 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2164: done (0 left) 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2165: rollback oid 13 current snap is 282 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 13 to 242 2024-04-12T11:23:54.870 INFO:tasks.rados.rados.0.smithi029.stdout:2166: delete oid 33 current snap is 282 2024-04-12T11:23:54.871 INFO:tasks.rados.rados.0.smithi029.stdout:2166: done (1 left) 2024-04-12T11:23:54.871 INFO:tasks.rados.rados.0.smithi029.stdout:2167: snap_create 2024-04-12T11:23:54.872 INFO:tasks.rados.rados.0.smithi029.stdout:2165: finishing rollback tid 1 to smithi02979277-13 2024-04-12T11:23:54.873 INFO:tasks.rados.rados.0.smithi029.stdout:2165: finishing rollback tid 2 to smithi02979277-13 2024-04-12T11:23:54.873 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1039 (ObjNum 626 snap 203 seq_num 626) dirty exists 2024-04-12T11:23:54.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:54 smithi029 ceph-mon[97787]: pgmap v579: 89 pgs: 89 active+clean; 200 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 68 KiB/s wr, 3 op/s 2024-04-12T11:23:54.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:54 smithi029 ceph-mon[97787]: osdmap e629: 8 total, 8 up, 8 in 2024-04-12T11:23:54.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:54 smithi029 ceph-mon[93650]: pgmap v579: 89 pgs: 89 active+clean; 200 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 68 KiB/s wr, 3 op/s 2024-04-12T11:23:54.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:54 smithi029 ceph-mon[93650]: osdmap e629: 8 total, 8 up, 8 in 2024-04-12T11:23:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:54 smithi087 ceph-mon[89014]: pgmap v579: 89 pgs: 89 active+clean; 200 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 68 KiB/s wr, 3 op/s 2024-04-12T11:23:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:54 smithi087 ceph-mon[89014]: osdmap e629: 8 total, 8 up, 8 in 2024-04-12T11:23:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:23:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:23:54] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:23:55.647 INFO:tasks.rados.rados.0.smithi029.stdout:2165: done (1 left) 2024-04-12T11:23:55.647 INFO:tasks.rados.rados.0.smithi029.stdout:2167: done (0 left) 2024-04-12T11:23:55.648 INFO:tasks.rados.rados.0.smithi029.stdout:2168: write oid 4 current snap is 283 2024-04-12T11:23:55.648 INFO:tasks.rados.rados.0.smithi029.stdout:2168: seq_num 811 ranges {430477=560357,1408073=473601,2488400=736073} 2024-04-12T11:23:55.652 INFO:tasks.rados.rados.0.smithi029.stdout:2168: writing smithi02979277-4 from 430477 to 990834 tid 1 2024-04-12T11:23:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2168: writing smithi02979277-4 from 1408073 to 1881674 tid 2 2024-04-12T11:23:55.665 INFO:tasks.rados.rados.0.smithi029.stdout:2168: writing smithi02979277-4 from 2488400 to 3224473 tid 3 2024-04-12T11:23:55.665 INFO:tasks.rados.rados.0.smithi029.stdout:2169: read oid 37 snap -1 2024-04-12T11:23:55.665 INFO:tasks.rados.rados.0.smithi029.stdout:2169: expect (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:23:55.665 INFO:tasks.rados.rados.0.smithi029.stdout:2170: snap_remove snap 269 2024-04-12T11:23:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:55 smithi087 ceph-mon[89014]: osdmap e630: 8 total, 8 up, 8 in 2024-04-12T11:23:56.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:55 smithi029 ceph-mon[93650]: osdmap e630: 8 total, 8 up, 8 in 2024-04-12T11:23:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:55 smithi029 ceph-mon[97787]: osdmap e630: 8 total, 8 up, 8 in 2024-04-12T11:23:56.648 INFO:tasks.rados.rados.0.smithi029.stdout:2170: done (2 left) 2024-04-12T11:23:56.648 INFO:tasks.rados.rados.0.smithi029.stdout:2171: write oid 33 current snap is 283 2024-04-12T11:23:56.648 INFO:tasks.rados.rados.0.smithi029.stdout:2168: finishing write tid 1 to smithi02979277-4 2024-04-12T11:23:56.649 INFO:tasks.rados.rados.0.smithi029.stdout:2168: finishing write tid 2 to smithi02979277-4 2024-04-12T11:23:56.649 INFO:tasks.rados.rados.0.smithi029.stdout:2169: expect (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:23:56.863 INFO:tasks.rados.rados.0.smithi029.stdout:2168: finishing write tid 3 to smithi02979277-4 2024-04-12T11:23:56.863 INFO:tasks.rados.rados.0.smithi029.stdout:2168: finishing write tid 4 to smithi02979277-4 2024-04-12T11:23:56.863 INFO:tasks.rados.rados.0.smithi029.stdout:2168: finishing write tid 5 to smithi02979277-4 2024-04-12T11:23:56.863 INFO:tasks.rados.rados.0.smithi029.stdout:2168: oid 4 updating version 0 to 1012 2024-04-12T11:23:56.863 INFO:tasks.rados.rados.0.smithi029.stdout:2168: oid 4 version 1012 is already newer than 1011 2024-04-12T11:23:56.863 INFO:tasks.rados.rados.0.smithi029.stdout:2168: oid 4 version 1012 is already newer than 1010 2024-04-12T11:23:56.864 INFO:tasks.rados.rados.0.smithi029.stdout:2168: oid 4 version 1012 is already newer than 1009 2024-04-12T11:23:56.864 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1012 (ObjNum 811 snap 283 seq_num 811) dirty exists 2024-04-12T11:23:56.864 INFO:tasks.rados.rados.0.smithi029.stdout:2168: left oid 4 (ObjNum 811 snap 283 seq_num 811) 2024-04-12T11:23:56.864 INFO:tasks.rados.rados.0.smithi029.stdout:2171: seq_num 812 ranges {447284=635606,1866555=733612,3315823=331461} 2024-04-12T11:23:56.869 INFO:tasks.rados.rados.0.smithi029.stdout:2171: writing smithi02979277-33 from 447284 to 1082890 tid 1 2024-04-12T11:23:56.877 INFO:tasks.rados.rados.0.smithi029.stdout:2171: writing smithi02979277-33 from 1866555 to 2600167 tid 2 2024-04-12T11:23:56.883 INFO:tasks.rados.rados.0.smithi029.stdout:2171: writing smithi02979277-33 from 3315823 to 3647284 tid 3 2024-04-12T11:23:56.883 INFO:tasks.rados.rados.0.smithi029.stdout:2168: done (2 left) 2024-04-12T11:23:56.883 INFO:tasks.rados.rados.0.smithi029.stdout:2169: done (1 left) 2024-04-12T11:23:56.883 INFO:tasks.rados.rados.0.smithi029.stdout:2172: rollback oid 49 current snap is 283 2024-04-12T11:23:56.883 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 49 to 282 2024-04-12T11:23:56.883 INFO:tasks.rados.rados.0.smithi029.stdout:2171: finishing write tid 1 to smithi02979277-33 2024-04-12T11:23:56.883 INFO:tasks.rados.rados.0.smithi029.stdout:2171: finishing write tid 2 to smithi02979277-33 2024-04-12T11:23:56.883 INFO:tasks.rados.rados.0.smithi029.stdout:2173: snap_create 2024-04-12T11:23:56.886 INFO:tasks.rados.rados.0.smithi029.stdout:2172: finishing rollback tid 0 to smithi02979277-49 2024-04-12T11:23:56.886 INFO:tasks.rados.rados.0.smithi029.stdout:2172: finishing rollback tid 1 to smithi02979277-49 2024-04-12T11:23:56.886 INFO:tasks.rados.rados.0.smithi029.stdout:2171: finishing write tid 3 to smithi02979277-33 2024-04-12T11:23:56.886 INFO:tasks.rados.rados.0.smithi029.stdout:2172: finishing rollback tid 2 to smithi02979277-49 2024-04-12T11:23:56.886 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 775 (ObjNum 791 snap 275 seq_num 791) dirty exists 2024-04-12T11:23:56.886 INFO:tasks.rados.rados.0.smithi029.stdout:2171: finishing write tid 4 to smithi02979277-33 2024-04-12T11:23:56.887 INFO:tasks.rados.rados.0.smithi029.stdout:2171: finishing write tid 5 to smithi02979277-33 2024-04-12T11:23:56.887 INFO:tasks.rados.rados.0.smithi029.stdout:2171: oid 33 updating version 0 to 798 2024-04-12T11:23:56.887 INFO:tasks.rados.rados.0.smithi029.stdout:2171: oid 33 version 798 is already newer than 797 2024-04-12T11:23:56.887 INFO:tasks.rados.rados.0.smithi029.stdout:2171: oid 33 version 798 is already newer than 796 2024-04-12T11:23:56.887 INFO:tasks.rados.rados.0.smithi029.stdout:2171: oid 33 version 798 is already newer than 795 2024-04-12T11:23:56.887 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 798 (ObjNum 812 snap 283 seq_num 812) dirty exists 2024-04-12T11:23:56.887 INFO:tasks.rados.rados.0.smithi029.stdout:2171: left oid 33 (ObjNum 812 snap 283 seq_num 812) 2024-04-12T11:23:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:56 smithi087 ceph-mon[89014]: pgmap v582: 89 pgs: 89 active+clean; 210 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-04-12T11:23:56.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:56 smithi087 ceph-mon[89014]: osdmap e631: 8 total, 8 up, 8 in 2024-04-12T11:23:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:56 smithi029 ceph-mon[97787]: pgmap v582: 89 pgs: 89 active+clean; 210 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-04-12T11:23:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:56 smithi029 ceph-mon[97787]: osdmap e631: 8 total, 8 up, 8 in 2024-04-12T11:23:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:56 smithi029 ceph-mon[93650]: pgmap v582: 89 pgs: 89 active+clean; 210 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-04-12T11:23:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:56 smithi029 ceph-mon[93650]: osdmap e631: 8 total, 8 up, 8 in 2024-04-12T11:23:57.589 INFO:tasks.rados.rados.0.smithi029.stdout:2171: done (2 left) 2024-04-12T11:23:57.589 INFO:tasks.rados.rados.0.smithi029.stdout:2172: done (1 left) 2024-04-12T11:23:57.589 INFO:tasks.rados.rados.0.smithi029.stdout:2173: done (0 left) 2024-04-12T11:23:57.589 INFO:tasks.rados.rados.0.smithi029.stdout:2174: write oid 29 current snap is 284 2024-04-12T11:23:57.589 INFO:tasks.rados.rados.0.smithi029.stdout:2174: seq_num 813 ranges {464091=710855,1925037=477270,2943246=474970} 2024-04-12T11:23:57.595 INFO:tasks.rados.rados.0.smithi029.stdout:2174: writing smithi02979277-29 from 464091 to 1174946 tid 1 2024-04-12T11:23:57.605 INFO:tasks.rados.rados.0.smithi029.stdout:2174: writing smithi02979277-29 from 1925037 to 2402307 tid 2 2024-04-12T11:23:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2174: writing smithi02979277-29 from 2943246 to 3418216 tid 3 2024-04-12T11:23:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2175: read oid 21 snap 249 2024-04-12T11:23:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2175: expect (ObjNum 713 snap 246 seq_num 713) 2024-04-12T11:23:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2176: rollback oid 46 current snap is 284 2024-04-12T11:23:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 46 to 275 2024-04-12T11:23:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2174: finishing write tid 1 to smithi02979277-29 2024-04-12T11:23:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2174: finishing write tid 2 to smithi02979277-29 2024-04-12T11:23:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2177: write oid 34 current snap is 284 2024-04-12T11:23:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2177: seq_num 814 ranges {480898=502457,1699872=737281,3087022=593876} 2024-04-12T11:23:57.614 INFO:tasks.rados.rados.0.smithi029.stdout:2177: writing smithi02979277-34 from 480898 to 983355 tid 1 2024-04-12T11:23:57.622 INFO:tasks.rados.rados.0.smithi029.stdout:2177: writing smithi02979277-34 from 1699872 to 2437153 tid 2 2024-04-12T11:23:57.630 INFO:tasks.rados.rados.0.smithi029.stdout:2177: writing smithi02979277-34 from 3087022 to 3680898 tid 3 2024-04-12T11:23:57.630 INFO:tasks.rados.rados.0.smithi029.stdout:2178: delete oid 28 current snap is 284 2024-04-12T11:23:57.630 INFO:tasks.rados.rados.0.smithi029.stdout:2176: finishing rollback tid 1 to smithi02979277-46 2024-04-12T11:23:57.630 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 629 (ObjNum 113 snap 0 seq_num 2925505856) dirty dne 2024-04-12T11:23:57.630 INFO:tasks.rados.rados.0.smithi029.stdout:2174: finishing write tid 3 to smithi02979277-29 2024-04-12T11:23:57.630 INFO:tasks.rados.rados.0.smithi029.stdout:2174: finishing write tid 4 to smithi02979277-29 2024-04-12T11:23:57.630 INFO:tasks.rados.rados.0.smithi029.stdout:2174: finishing write tid 5 to smithi02979277-29 2024-04-12T11:23:57.630 INFO:tasks.rados.rados.0.smithi029.stdout:2174: oid 29 updating version 0 to 855 2024-04-12T11:23:57.630 INFO:tasks.rados.rados.0.smithi029.stdout:2174: oid 29 version 855 is already newer than 854 2024-04-12T11:23:57.631 INFO:tasks.rados.rados.0.smithi029.stdout:2174: oid 29 version 855 is already newer than 852 2024-04-12T11:23:57.631 INFO:tasks.rados.rados.0.smithi029.stdout:2174: oid 29 version 855 is already newer than 853 2024-04-12T11:23:57.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 855 (ObjNum 813 snap 284 seq_num 813) dirty exists 2024-04-12T11:23:57.631 INFO:tasks.rados.rados.0.smithi029.stdout:2174: left oid 29 (ObjNum 813 snap 284 seq_num 813) 2024-04-12T11:23:57.631 INFO:tasks.rados.rados.0.smithi029.stdout:2177: finishing write tid 1 to smithi02979277-34 2024-04-12T11:23:57.631 INFO:tasks.rados.rados.0.smithi029.stdout:2177: finishing write tid 2 to smithi02979277-34 2024-04-12T11:23:57.631 INFO:tasks.rados.rados.0.smithi029.stdout:2175: expect (ObjNum 713 snap 246 seq_num 713) 2024-04-12T11:23:57.877 INFO:tasks.rados.rados.0.smithi029.stdout:2177: finishing write tid 3 to smithi02979277-34 2024-04-12T11:23:57.877 INFO:tasks.rados.rados.0.smithi029.stdout:2177: finishing write tid 4 to smithi02979277-34 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2177: finishing write tid 5 to smithi02979277-34 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2177: oid 34 updating version 0 to 858 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2177: oid 34 version 858 is already newer than 856 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2177: oid 34 version 858 is already newer than 857 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2177: oid 34 updating version 858 to 859 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 859 (ObjNum 814 snap 284 seq_num 814) dirty exists 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2177: left oid 34 (ObjNum 814 snap 284 seq_num 814) 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2174: done (4 left) 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2175: done (3 left) 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2176: done (2 left) 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2177: done (1 left) 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2178: done (0 left) 2024-04-12T11:23:57.878 INFO:tasks.rados.rados.0.smithi029.stdout:2179: snap_create 2024-04-12T11:23:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:57 smithi087 ceph-mon[89014]: osdmap e632: 8 total, 8 up, 8 in 2024-04-12T11:23:57.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:57 smithi087 ceph-mon[89014]: osdmap e633: 8 total, 8 up, 8 in 2024-04-12T11:23:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:57 smithi029 ceph-mon[97787]: osdmap e632: 8 total, 8 up, 8 in 2024-04-12T11:23:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:57 smithi029 ceph-mon[97787]: osdmap e633: 8 total, 8 up, 8 in 2024-04-12T11:23:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:57 smithi029 ceph-mon[93650]: osdmap e632: 8 total, 8 up, 8 in 2024-04-12T11:23:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:57 smithi029 ceph-mon[93650]: osdmap e633: 8 total, 8 up, 8 in 2024-04-12T11:23:58.664 INFO:tasks.rados.rados.0.smithi029.stdout:2179: done (0 left) 2024-04-12T11:23:58.664 INFO:tasks.rados.rados.0.smithi029.stdout:2180: read oid 40 snap -1 2024-04-12T11:23:58.665 INFO:tasks.rados.rados.0.smithi029.stdout:2180: expect (ObjNum 787 snap 274 seq_num 787) 2024-04-12T11:23:58.665 INFO:tasks.rados.rados.0.smithi029.stdout:2181: delete oid 33 current snap is 285 2024-04-12T11:23:58.669 INFO:tasks.rados.rados.0.smithi029.stdout:2181: done (1 left) 2024-04-12T11:23:58.669 INFO:tasks.rados.rados.0.smithi029.stdout:2182: write oid 32 current snap is 285 2024-04-12T11:23:58.669 INFO:tasks.rados.rados.0.smithi029.stdout:2182: seq_num 817 ranges {531319=728204,1758965=484608,2652938=520058} 2024-04-12T11:23:58.676 INFO:tasks.rados.rados.0.smithi029.stdout:2182: writing smithi02979277-32 from 531319 to 1259523 tid 1 2024-04-12T11:23:58.682 INFO:tasks.rados.rados.0.smithi029.stdout:2182: writing smithi02979277-32 from 1758965 to 2243573 tid 2 2024-04-12T11:23:58.687 INFO:tasks.rados.rados.0.smithi029.stdout:2182: writing smithi02979277-32 from 2652938 to 3172996 tid 3 2024-04-12T11:23:58.687 INFO:tasks.rados.rados.0.smithi029.stdout:2183: snap_remove snap 276 2024-04-12T11:23:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:58 smithi087 ceph-mon[89014]: pgmap v585: 89 pgs: 89 active+clean; 224 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.5 MiB/s wr, 25 op/s 2024-04-12T11:23:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:58 smithi029 ceph-mon[93650]: pgmap v585: 89 pgs: 89 active+clean; 224 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.5 MiB/s wr, 25 op/s 2024-04-12T11:23:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:58 smithi029 ceph-mon[97787]: pgmap v585: 89 pgs: 89 active+clean; 224 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.5 MiB/s wr, 25 op/s 2024-04-12T11:23:59.670 INFO:tasks.rados.rados.0.smithi029.stdout:2183: done (2 left) 2024-04-12T11:23:59.670 INFO:tasks.rados.rados.0.smithi029.stdout:2184: read oid 31 snap 282 2024-04-12T11:23:59.670 INFO:tasks.rados.rados.0.smithi029.stdout:2184: expect (ObjNum 794 snap 277 seq_num 794) 2024-04-12T11:23:59.670 INFO:tasks.rados.rados.0.smithi029.stdout:2185: write oid 15 current snap is 285 2024-04-12T11:23:59.670 INFO:tasks.rados.rados.0.smithi029.stdout:2180: expect (ObjNum 787 snap 274 seq_num 787) 2024-04-12T11:23:59.814 INFO:tasks.rados.rados.0.smithi029.stdout:2182: finishing write tid 1 to smithi02979277-32 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:2182: finishing write tid 2 to smithi02979277-32 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:2182: finishing write tid 3 to smithi02979277-32 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:2182: finishing write tid 4 to smithi02979277-32 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:2182: finishing write tid 5 to smithi02979277-32 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:2182: oid 32 updating version 0 to 864 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:2182: oid 32 version 864 is already newer than 862 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:2182: oid 32 version 864 is already newer than 863 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:2182: oid 32 version 864 is already newer than 861 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 864 (ObjNum 817 snap 285 seq_num 817) dirty exists 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:2182: left oid 32 (ObjNum 817 snap 285 seq_num 817) 2024-04-12T11:23:59.815 INFO:tasks.rados.rados.0.smithi029.stdout:2184: expect (ObjNum 794 snap 277 seq_num 794) 2024-04-12T11:23:59.926 INFO:tasks.rados.rados.0.smithi029.stdout:2185: seq_num 818 ranges {548126=403453,1417447=744619,2680361=447683,3680450=67676} 2024-04-12T11:23:59.931 INFO:tasks.rados.rados.0.smithi029.stdout:2185: writing smithi02979277-15 from 548126 to 951579 tid 1 2024-04-12T11:23:59.937 INFO:tasks.rados.rados.0.smithi029.stdout:2185: writing smithi02979277-15 from 1417447 to 2162066 tid 2 2024-04-12T11:23:59.942 INFO:tasks.rados.rados.0.smithi029.stdout:2185: writing smithi02979277-15 from 2680361 to 3128044 tid 3 2024-04-12T11:23:59.947 INFO:tasks.rados.rados.0.smithi029.stdout:2185: writing smithi02979277-15 from 3680450 to 3748126 tid 4 2024-04-12T11:23:59.947 INFO:tasks.rados.rados.0.smithi029.stdout:2180: done (3 left) 2024-04-12T11:23:59.947 INFO:tasks.rados.rados.0.smithi029.stdout:2182: done (2 left) 2024-04-12T11:23:59.947 INFO:tasks.rados.rados.0.smithi029.stdout:2184: done (1 left) 2024-04-12T11:23:59.947 INFO:tasks.rados.rados.0.smithi029.stdout:2186: read oid 10 snap -1 2024-04-12T11:23:59.947 INFO:tasks.rados.rados.0.smithi029.stdout:2186: expect (ObjNum 771 snap 270 seq_num 771) 2024-04-12T11:23:59.947 INFO:tasks.rados.rados.0.smithi029.stdout:2187: write oid 44 current snap is 285 2024-04-12T11:23:59.947 INFO:tasks.rados.rados.0.smithi029.stdout:2185: finishing write tid 1 to smithi02979277-15 2024-04-12T11:23:59.947 INFO:tasks.rados.rados.0.smithi029.stdout:2185: finishing write tid 2 to smithi02979277-15 2024-04-12T11:23:59.947 INFO:tasks.rados.rados.0.smithi029.stdout:2187: seq_num 819 ranges {564933=478702,1759576=488277,2991431=658955} 2024-04-12T11:23:59.952 INFO:tasks.rados.rados.0.smithi029.stdout:2187: writing smithi02979277-44 from 564933 to 1043635 tid 1 2024-04-12T11:23:59.958 INFO:tasks.rados.rados.0.smithi029.stdout:2187: writing smithi02979277-44 from 1759576 to 2247853 tid 2 2024-04-12T11:23:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:23:59 smithi087 ceph-mon[89014]: osdmap e634: 8 total, 8 up, 8 in 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2187: writing smithi02979277-44 from 2991431 to 3650386 tid 3 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2188: delete oid 42 current snap is 285 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2185: finishing write tid 3 to smithi02979277-15 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2185: finishing write tid 4 to smithi02979277-15 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2185: finishing write tid 5 to smithi02979277-15 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2185: finishing write tid 6 to smithi02979277-15 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2185: oid 15 updating version 0 to 742 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2185: oid 15 version 742 is already newer than 741 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2185: oid 15 updating version 742 to 743 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2185: oid 15 version 743 is already newer than 740 2024-04-12T11:23:59.967 INFO:tasks.rados.rados.0.smithi029.stdout:2185: oid 15 version 743 is already newer than 739 2024-04-12T11:23:59.968 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 743 (ObjNum 818 snap 285 seq_num 818) dirty exists 2024-04-12T11:23:59.968 INFO:tasks.rados.rados.0.smithi029.stdout:2185: left oid 15 (ObjNum 818 snap 285 seq_num 818) 2024-04-12T11:23:59.968 INFO:tasks.rados.rados.0.smithi029.stdout:2186: expect (ObjNum 771 snap 270 seq_num 771) 2024-04-12T11:24:00.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:23:59 smithi029 ceph-mon[97787]: osdmap e634: 8 total, 8 up, 8 in 2024-04-12T11:24:00.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:23:59 smithi029 ceph-mon[93650]: osdmap e634: 8 total, 8 up, 8 in 2024-04-12T11:24:00.150 INFO:tasks.rados.rados.0.smithi029.stdout:2187: finishing write tid 1 to smithi02979277-44 2024-04-12T11:24:00.150 INFO:tasks.rados.rados.0.smithi029.stdout:2187: finishing write tid 2 to smithi02979277-44 2024-04-12T11:24:00.150 INFO:tasks.rados.rados.0.smithi029.stdout:2187: finishing write tid 3 to smithi02979277-44 2024-04-12T11:24:00.150 INFO:tasks.rados.rados.0.smithi029.stdout:2187: finishing write tid 4 to smithi02979277-44 2024-04-12T11:24:00.150 INFO:tasks.rados.rados.0.smithi029.stdout:2187: finishing write tid 5 to smithi02979277-44 2024-04-12T11:24:00.150 INFO:tasks.rados.rados.0.smithi029.stdout:2187: oid 44 updating version 0 to 1058 2024-04-12T11:24:00.150 INFO:tasks.rados.rados.0.smithi029.stdout:2187: oid 44 updating version 1058 to 1059 2024-04-12T11:24:00.150 INFO:tasks.rados.rados.0.smithi029.stdout:2187: oid 44 updating version 1059 to 1061 2024-04-12T11:24:00.150 INFO:tasks.rados.rados.0.smithi029.stdout:2187: oid 44 version 1061 is already newer than 1060 2024-04-12T11:24:00.150 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1061 (ObjNum 819 snap 285 seq_num 819) dirty exists 2024-04-12T11:24:00.151 INFO:tasks.rados.rados.0.smithi029.stdout:2187: left oid 44 (ObjNum 819 snap 285 seq_num 819) 2024-04-12T11:24:00.151 INFO:tasks.rados.rados.0.smithi029.stdout:2185: done (3 left) 2024-04-12T11:24:00.151 INFO:tasks.rados.rados.0.smithi029.stdout:2186: done (2 left) 2024-04-12T11:24:00.151 INFO:tasks.rados.rados.0.smithi029.stdout:2187: done (1 left) 2024-04-12T11:24:00.151 INFO:tasks.rados.rados.0.smithi029.stdout:2188: done (0 left) 2024-04-12T11:24:00.151 INFO:tasks.rados.rados.0.smithi029.stdout:2189: snap_remove snap 242 2024-04-12T11:24:00.672 INFO:tasks.rados.rados.0.smithi029.stdout:2189: done (0 left) 2024-04-12T11:24:00.673 INFO:tasks.rados.rados.0.smithi029.stdout:2190: write oid 25 current snap is 285 2024-04-12T11:24:00.673 INFO:tasks.rados.rados.0.smithi029.stdout:2190: seq_num 821 ranges {598547=629200,1876540=491946,3046277=681499} 2024-04-12T11:24:00.681 INFO:tasks.rados.rados.0.smithi029.stdout:2190: writing smithi02979277-25 from 598547 to 1227747 tid 1 2024-04-12T11:24:00.686 INFO:tasks.rados.rados.0.smithi029.stdout:2190: writing smithi02979277-25 from 1876540 to 2368486 tid 2 2024-04-12T11:24:00.695 INFO:tasks.rados.rados.0.smithi029.stdout:2190: writing smithi02979277-25 from 3046277 to 3727776 tid 3 2024-04-12T11:24:00.695 INFO:tasks.rados.rados.0.smithi029.stdout:2191: read oid 33 snap -1 2024-04-12T11:24:00.695 INFO:tasks.rados.rados.0.smithi029.stdout:2191: expect deleted 2024-04-12T11:24:00.695 INFO:tasks.rados.rados.0.smithi029.stdout:2192: write oid 50 current snap is 285 2024-04-12T11:24:00.695 INFO:tasks.rados.rados.0.smithi029.stdout:2190: finishing write tid 1 to smithi02979277-25 2024-04-12T11:24:00.695 INFO:tasks.rados.rados.0.smithi029.stdout:2190: finishing write tid 2 to smithi02979277-25 2024-04-12T11:24:00.695 INFO:tasks.rados.rados.0.smithi029.stdout:2192: seq_num 822 ranges {615354=420802,1651375=751957,3190053=492771} 2024-04-12T11:24:00.700 INFO:tasks.rados.rados.0.smithi029.stdout:2192: writing smithi02979277-50 from 615354 to 1036156 tid 1 2024-04-12T11:24:00.708 INFO:tasks.rados.rados.0.smithi029.stdout:2192: writing smithi02979277-50 from 1651375 to 2403332 tid 2 2024-04-12T11:24:00.715 INFO:tasks.rados.rados.0.smithi029.stdout:2192: writing smithi02979277-50 from 3190053 to 3682824 tid 3 2024-04-12T11:24:00.715 INFO:tasks.rados.rados.0.smithi029.stdout:2193: delete oid 42 current snap is 285 2024-04-12T11:24:00.715 INFO:tasks.rados.rados.0.smithi029.stdout:2190: finishing write tid 3 to smithi02979277-25 2024-04-12T11:24:00.715 INFO:tasks.rados.rados.0.smithi029.stdout:2190: finishing write tid 4 to smithi02979277-25 2024-04-12T11:24:00.715 INFO:tasks.rados.rados.0.smithi029.stdout:2190: finishing write tid 5 to smithi02979277-25 2024-04-12T11:24:00.715 INFO:tasks.rados.rados.0.smithi029.stdout:2190: oid 25 updating version 0 to 1056 2024-04-12T11:24:00.715 INFO:tasks.rados.rados.0.smithi029.stdout:2190: oid 25 version 1056 is already newer than 1054 2024-04-12T11:24:00.715 INFO:tasks.rados.rados.0.smithi029.stdout:2190: oid 25 version 1056 is already newer than 1055 2024-04-12T11:24:00.716 INFO:tasks.rados.rados.0.smithi029.stdout:2190: oid 25 version 1056 is already newer than 1049 2024-04-12T11:24:00.716 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 1056 (ObjNum 821 snap 285 seq_num 821) dirty exists 2024-04-12T11:24:00.716 INFO:tasks.rados.rados.0.smithi029.stdout:2190: left oid 25 (ObjNum 821 snap 285 seq_num 821) 2024-04-12T11:24:00.716 INFO:tasks.rados.rados.0.smithi029.stdout:2192: finishing write tid 1 to smithi02979277-50 2024-04-12T11:24:00.716 INFO:tasks.rados.rados.0.smithi029.stdout:2192: finishing write tid 2 to smithi02979277-50 2024-04-12T11:24:00.718 INFO:tasks.rados.rados.0.smithi029.stdout:2190: done (3 left) 2024-04-12T11:24:00.718 INFO:tasks.rados.rados.0.smithi029.stdout:2191: done (2 left) 2024-04-12T11:24:00.718 INFO:tasks.rados.rados.0.smithi029.stdout:2193: done (1 left) 2024-04-12T11:24:00.718 INFO:tasks.rados.rados.0.smithi029.stdout:2194: write oid 6 current snap is 285 2024-04-12T11:24:00.718 INFO:tasks.rados.rados.0.smithi029.stdout:2194: seq_num 824 ranges {648968=571300,1651986=639273,2728546=631668} 2024-04-12T11:24:00.724 INFO:tasks.rados.rados.0.smithi029.stdout:2194: writing smithi02979277-6 from 648968 to 1220268 tid 1 2024-04-12T11:24:00.730 INFO:tasks.rados.rados.0.smithi029.stdout:2194: writing smithi02979277-6 from 1651986 to 2291259 tid 2 2024-04-12T11:24:00.736 INFO:tasks.rados.rados.0.smithi029.stdout:2194: writing smithi02979277-6 from 2728546 to 3360214 tid 3 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2195: rollback oid 12 current snap is 285 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 284 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2192: finishing write tid 3 to smithi02979277-50 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2192: finishing write tid 4 to smithi02979277-50 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2192: finishing write tid 5 to smithi02979277-50 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2192: oid 50 updating version 0 to 815 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2192: oid 50 updating version 815 to 817 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2192: oid 50 version 817 is already newer than 816 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2192: oid 50 version 817 is already newer than 814 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 817 (ObjNum 822 snap 285 seq_num 822) dirty exists 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2192: left oid 50 (ObjNum 822 snap 285 seq_num 822) 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2194: finishing write tid 1 to smithi02979277-6 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2194: finishing write tid 2 to smithi02979277-6 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2192: done (2 left) 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2196: read oid 34 snap 280 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2196: expect deleted 2024-04-12T11:24:00.737 INFO:tasks.rados.rados.0.smithi029.stdout:2197: snap_create 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2195: finishing rollback tid 0 to smithi02979277-12 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2194: finishing write tid 3 to smithi02979277-6 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2195: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2194: finishing write tid 4 to smithi02979277-6 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2195: finishing rollback tid 2 to smithi02979277-12 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1059 (ObjNum 670 snap 215 seq_num 670) dirty exists 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2194: finishing write tid 5 to smithi02979277-6 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2194: oid 6 updating version 0 to 545 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2194: oid 6 updating version 545 to 546 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2194: oid 6 version 546 is already newer than 544 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2194: oid 6 updating version 546 to 547 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 547 (ObjNum 824 snap 285 seq_num 824) dirty exists 2024-04-12T11:24:00.740 INFO:tasks.rados.rados.0.smithi029.stdout:2194: left oid 6 (ObjNum 824 snap 285 seq_num 824) 2024-04-12T11:24:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:00 smithi087 ceph-mon[89014]: pgmap v588: 89 pgs: 89 active+clean; 230 MiB data, 739 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.2 MiB/s wr, 19 op/s 2024-04-12T11:24:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:00 smithi087 ceph-mon[89014]: osdmap e635: 8 total, 8 up, 8 in 2024-04-12T11:24:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:00 smithi029 ceph-mon[93650]: pgmap v588: 89 pgs: 89 active+clean; 230 MiB data, 739 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.2 MiB/s wr, 19 op/s 2024-04-12T11:24:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:00 smithi029 ceph-mon[93650]: osdmap e635: 8 total, 8 up, 8 in 2024-04-12T11:24:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:00 smithi029 ceph-mon[97787]: pgmap v588: 89 pgs: 89 active+clean; 230 MiB data, 739 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.2 MiB/s wr, 19 op/s 2024-04-12T11:24:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:00 smithi029 ceph-mon[97787]: osdmap e635: 8 total, 8 up, 8 in 2024-04-12T11:24:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:2194: done (3 left) 2024-04-12T11:24:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:2195: done (2 left) 2024-04-12T11:24:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:2196: done (1 left) 2024-04-12T11:24:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:2197: done (0 left) 2024-04-12T11:24:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:2198: rollback oid 48 current snap is 286 2024-04-12T11:24:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 283 2024-04-12T11:24:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:2199: snap_remove snap 277 2024-04-12T11:24:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:01 smithi087 ceph-mon[89014]: osdmap e636: 8 total, 8 up, 8 in 2024-04-12T11:24:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:01 smithi029 ceph-mon[93650]: osdmap e636: 8 total, 8 up, 8 in 2024-04-12T11:24:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:01 smithi029 ceph-mon[97787]: osdmap e636: 8 total, 8 up, 8 in 2024-04-12T11:24:02.592 INFO:tasks.rados.rados.0.smithi029.stdout:2199: done (1 left) 2024-04-12T11:24:02.592 INFO:tasks.rados.rados.0.smithi029.stdout:2200: delete oid 33 current snap is 286 2024-04-12T11:24:02.592 INFO:tasks.rados.rados.0.smithi029.stdout:2198: finishing rollback tid 0 to smithi02979277-48 2024-04-12T11:24:02.592 INFO:tasks.rados.rados.0.smithi029.stdout:2198: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:24:02.592 INFO:tasks.rados.rados.0.smithi029.stdout:2198: finishing rollback tid 2 to smithi02979277-48 2024-04-12T11:24:02.592 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 897 (ObjNum 804 snap 282 seq_num 804) dirty exists 2024-04-12T11:24:02.601 INFO:tasks.rados.rados.0.smithi029.stdout:2198: done (1 left) 2024-04-12T11:24:02.601 INFO:tasks.rados.rados.0.smithi029.stdout:2200: done (0 left) 2024-04-12T11:24:02.601 INFO:tasks.rados.rados.0.smithi029.stdout:2201: read oid 35 snap -1 2024-04-12T11:24:02.601 INFO:tasks.rados.rados.0.smithi029.stdout:2201: expect deleted 2024-04-12T11:24:02.601 INFO:tasks.rados.rados.0.smithi029.stdout:2202: read oid 30 snap -1 2024-04-12T11:24:02.601 INFO:tasks.rados.rados.0.smithi029.stdout:2202: expect deleted 2024-04-12T11:24:02.602 INFO:tasks.rados.rados.0.smithi029.stdout:2203: snap_create 2024-04-12T11:24:02.680 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:24:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:24:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:24:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:02 smithi087 ceph-mon[89014]: pgmap v591: 89 pgs: 89 active+clean; 229 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:24:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:02 smithi087 ceph-mon[89014]: osdmap e637: 8 total, 8 up, 8 in 2024-04-12T11:24:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:02 smithi087 ceph-mon[89014]: osdmap e638: 8 total, 8 up, 8 in 2024-04-12T11:24:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:02 smithi029 ceph-mon[93650]: pgmap v591: 89 pgs: 89 active+clean; 229 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:24:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:02 smithi029 ceph-mon[93650]: osdmap e637: 8 total, 8 up, 8 in 2024-04-12T11:24:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:02 smithi029 ceph-mon[93650]: osdmap e638: 8 total, 8 up, 8 in 2024-04-12T11:24:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:02 smithi029 ceph-mon[97787]: pgmap v591: 89 pgs: 89 active+clean; 229 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:24:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:02 smithi029 ceph-mon[97787]: osdmap e637: 8 total, 8 up, 8 in 2024-04-12T11:24:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:02 smithi029 ceph-mon[97787]: osdmap e638: 8 total, 8 up, 8 in 2024-04-12T11:24:03.592 INFO:tasks.rados.rados.0.smithi029.stdout:2201: done (2 left) 2024-04-12T11:24:03.593 INFO:tasks.rados.rados.0.smithi029.stdout:2202: done (1 left) 2024-04-12T11:24:03.593 INFO:tasks.rados.rados.0.smithi029.stdout:2203: done (0 left) 2024-04-12T11:24:03.593 INFO:tasks.rados.rados.0.smithi029.stdout:2204: snap_create 2024-04-12T11:24:04.595 INFO:tasks.rados.rados.0.smithi029.stdout:2204: done (0 left) 2024-04-12T11:24:04.595 INFO:tasks.rados.rados.0.smithi029.stdout:2205: rollback oid 14 current snap is 288 2024-04-12T11:24:04.595 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 285 2024-04-12T11:24:04.595 INFO:tasks.rados.rados.0.smithi029.stdout:2206: snap_create 2024-04-12T11:24:04.598 INFO:tasks.rados.rados.0.smithi029.stdout:2205: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:24:04.598 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 0 (ObjNum 65 snap 0 seq_num 861934552) dirty dne 2024-04-12T11:24:04.889 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:04 smithi087 ceph-mon[89014]: pgmap v594: 89 pgs: 89 active+clean; 222 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-04-12T11:24:04.889 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:04 smithi087 ceph-mon[89014]: osdmap e639: 8 total, 8 up, 8 in 2024-04-12T11:24:04.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:04 smithi029 ceph-mon[97787]: pgmap v594: 89 pgs: 89 active+clean; 222 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-04-12T11:24:04.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:04 smithi029 ceph-mon[97787]: osdmap e639: 8 total, 8 up, 8 in 2024-04-12T11:24:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:04 smithi029 ceph-mon[93650]: pgmap v594: 89 pgs: 89 active+clean; 222 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-04-12T11:24:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:04 smithi029 ceph-mon[93650]: osdmap e639: 8 total, 8 up, 8 in 2024-04-12T11:24:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:24:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:24:04] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:24:05.597 INFO:tasks.rados.rados.0.smithi029.stdout:2205: done (1 left) 2024-04-12T11:24:05.597 INFO:tasks.rados.rados.0.smithi029.stdout:2206: done (0 left) 2024-04-12T11:24:05.597 INFO:tasks.rados.rados.0.smithi029.stdout:2207: rollback oid 41 current snap is 289 2024-04-12T11:24:05.597 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 284 2024-04-12T11:24:05.597 INFO:tasks.rados.rados.0.smithi029.stdout:2208: snap_remove snap 279 2024-04-12T11:24:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:05 smithi087 ceph-mon[89014]: osdmap e640: 8 total, 8 up, 8 in 2024-04-12T11:24:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:05 smithi087 ceph-mon[89014]: osdmap e641: 8 total, 8 up, 8 in 2024-04-12T11:24:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:05 smithi029 ceph-mon[97787]: osdmap e640: 8 total, 8 up, 8 in 2024-04-12T11:24:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:05 smithi029 ceph-mon[97787]: osdmap e641: 8 total, 8 up, 8 in 2024-04-12T11:24:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:05 smithi029 ceph-mon[93650]: osdmap e640: 8 total, 8 up, 8 in 2024-04-12T11:24:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:05 smithi029 ceph-mon[93650]: osdmap e641: 8 total, 8 up, 8 in 2024-04-12T11:24:06.598 INFO:tasks.rados.rados.0.smithi029.stdout:2208: done (1 left) 2024-04-12T11:24:06.598 INFO:tasks.rados.rados.0.smithi029.stdout:2209: read oid 21 snap -1 2024-04-12T11:24:06.598 INFO:tasks.rados.rados.0.smithi029.stdout:2209: expect (ObjNum 672 snap 215 seq_num 672) 2024-04-12T11:24:06.598 INFO:tasks.rados.rados.0.smithi029.stdout:2210: write oid 44 current snap is 289 2024-04-12T11:24:06.598 INFO:tasks.rados.rados.0.smithi029.stdout:2207: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:24:06.598 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 993 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-04-12T11:24:06.598 INFO:tasks.rados.rados.0.smithi029.stdout:2210: seq_num 826 ranges {682582=721798,2168950=642942,3583392=299190} 2024-04-12T11:24:06.606 INFO:tasks.rados.rados.0.smithi029.stdout:2210: writing smithi02979277-44 from 682582 to 1404380 tid 1 2024-04-12T11:24:06.614 INFO:tasks.rados.rados.0.smithi029.stdout:2210: writing smithi02979277-44 from 2168950 to 2811892 tid 2 2024-04-12T11:24:06.620 INFO:tasks.rados.rados.0.smithi029.stdout:2210: writing smithi02979277-44 from 3583392 to 3882582 tid 3 2024-04-12T11:24:06.620 INFO:tasks.rados.rados.0.smithi029.stdout:2207: done (2 left) 2024-04-12T11:24:06.621 INFO:tasks.rados.rados.0.smithi029.stdout:2211: rollback oid 49 current snap is 289 2024-04-12T11:24:06.621 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 49 to 286 2024-04-12T11:24:06.621 INFO:tasks.rados.rados.0.smithi029.stdout:2209: expect (ObjNum 672 snap 215 seq_num 672) 2024-04-12T11:24:06.729 INFO:tasks.rados.rados.0.smithi029.stdout:2210: finishing write tid 1 to smithi02979277-44 2024-04-12T11:24:06.729 INFO:tasks.rados.rados.0.smithi029.stdout:2210: finishing write tid 2 to smithi02979277-44 2024-04-12T11:24:06.729 INFO:tasks.rados.rados.0.smithi029.stdout:2210: finishing write tid 3 to smithi02979277-44 2024-04-12T11:24:06.729 INFO:tasks.rados.rados.0.smithi029.stdout:2210: finishing write tid 4 to smithi02979277-44 2024-04-12T11:24:06.729 INFO:tasks.rados.rados.0.smithi029.stdout:2210: finishing write tid 5 to smithi02979277-44 2024-04-12T11:24:06.729 INFO:tasks.rados.rados.0.smithi029.stdout:2210: oid 44 updating version 0 to 1072 2024-04-12T11:24:06.729 INFO:tasks.rados.rados.0.smithi029.stdout:2210: oid 44 updating version 1072 to 1073 2024-04-12T11:24:06.729 INFO:tasks.rados.rados.0.smithi029.stdout:2210: oid 44 updating version 1073 to 1075 2024-04-12T11:24:06.729 INFO:tasks.rados.rados.0.smithi029.stdout:2210: oid 44 version 1075 is already newer than 1074 2024-04-12T11:24:06.729 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1075 (ObjNum 826 snap 289 seq_num 826) dirty exists 2024-04-12T11:24:06.730 INFO:tasks.rados.rados.0.smithi029.stdout:2210: left oid 44 (ObjNum 826 snap 289 seq_num 826) 2024-04-12T11:24:06.730 INFO:tasks.rados.rados.0.smithi029.stdout:2211: finishing rollback tid 0 to smithi02979277-49 2024-04-12T11:24:06.730 INFO:tasks.rados.rados.0.smithi029.stdout:2211: finishing rollback tid 1 to smithi02979277-49 2024-04-12T11:24:06.730 INFO:tasks.rados.rados.0.smithi029.stdout:2211: finishing rollback tid 2 to smithi02979277-49 2024-04-12T11:24:06.730 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 801 (ObjNum 791 snap 275 seq_num 791) dirty exists 2024-04-12T11:24:06.730 INFO:tasks.rados.rados.0.smithi029.stdout:2209: done (2 left) 2024-04-12T11:24:06.730 INFO:tasks.rados.rados.0.smithi029.stdout:2210: done (1 left) 2024-04-12T11:24:06.730 INFO:tasks.rados.rados.0.smithi029.stdout:2211: done (0 left) 2024-04-12T11:24:06.730 INFO:tasks.rados.rados.0.smithi029.stdout:2212: write oid 28 current snap is 289 2024-04-12T11:24:06.730 INFO:tasks.rados.rados.0.smithi029.stdout:2212: seq_num 827 ranges {699389=797047,2111079=502953,3094462=581837} 2024-04-12T11:24:06.736 INFO:tasks.rados.rados.0.smithi029.stdout:2212: writing smithi02979277-28 from 699389 to 1496436 tid 1 2024-04-12T11:24:06.743 INFO:tasks.rados.rados.0.smithi029.stdout:2212: writing smithi02979277-28 from 2111079 to 2614032 tid 2 2024-04-12T11:24:06.749 INFO:tasks.rados.rados.0.smithi029.stdout:2212: writing smithi02979277-28 from 3094462 to 3676299 tid 3 2024-04-12T11:24:06.749 INFO:tasks.rados.rados.0.smithi029.stdout:2213: read oid 32 snap -1 2024-04-12T11:24:06.749 INFO:tasks.rados.rados.0.smithi029.stdout:2213: expect (ObjNum 817 snap 285 seq_num 817) 2024-04-12T11:24:06.749 INFO:tasks.rados.rados.0.smithi029.stdout:2214: snap_create 2024-04-12T11:24:06.749 INFO:tasks.rados.rados.0.smithi029.stdout:2212: finishing write tid 1 to smithi02979277-28 2024-04-12T11:24:06.749 INFO:tasks.rados.rados.0.smithi029.stdout:2212: finishing write tid 2 to smithi02979277-28 2024-04-12T11:24:06.752 INFO:tasks.rados.rados.0.smithi029.stdout:2212: finishing write tid 3 to smithi02979277-28 2024-04-12T11:24:06.752 INFO:tasks.rados.rados.0.smithi029.stdout:2212: finishing write tid 4 to smithi02979277-28 2024-04-12T11:24:06.752 INFO:tasks.rados.rados.0.smithi029.stdout:2212: finishing write tid 5 to smithi02979277-28 2024-04-12T11:24:06.752 INFO:tasks.rados.rados.0.smithi029.stdout:2212: oid 28 updating version 0 to 902 2024-04-12T11:24:06.752 INFO:tasks.rados.rados.0.smithi029.stdout:2212: oid 28 updating version 902 to 903 2024-04-12T11:24:06.752 INFO:tasks.rados.rados.0.smithi029.stdout:2212: oid 28 updating version 903 to 904 2024-04-12T11:24:06.752 INFO:tasks.rados.rados.0.smithi029.stdout:2212: oid 28 updating version 904 to 905 2024-04-12T11:24:06.752 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 905 (ObjNum 827 snap 289 seq_num 827) dirty exists 2024-04-12T11:24:06.752 INFO:tasks.rados.rados.0.smithi029.stdout:2212: left oid 28 (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:24:06.753 INFO:tasks.rados.rados.0.smithi029.stdout:2213: expect (ObjNum 817 snap 285 seq_num 817) 2024-04-12T11:24:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:06 smithi087 ceph-mon[89014]: pgmap v597: 89 pgs: 89 active+clean; 208 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-04-12T11:24:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:06 smithi087 ceph-mon[89014]: osdmap e642: 8 total, 8 up, 8 in 2024-04-12T11:24:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:06 smithi029 ceph-mon[93650]: pgmap v597: 89 pgs: 89 active+clean; 208 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-04-12T11:24:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:06 smithi029 ceph-mon[93650]: osdmap e642: 8 total, 8 up, 8 in 2024-04-12T11:24:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:06 smithi029 ceph-mon[97787]: pgmap v597: 89 pgs: 89 active+clean; 208 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-04-12T11:24:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:06 smithi029 ceph-mon[97787]: osdmap e642: 8 total, 8 up, 8 in 2024-04-12T11:24:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:2212: done (2 left) 2024-04-12T11:24:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:2213: done (1 left) 2024-04-12T11:24:07.592 INFO:tasks.rados.rados.0.smithi029.stdout:2214: done (0 left) 2024-04-12T11:24:07.593 INFO:tasks.rados.rados.0.smithi029.stdout:2215: write oid 44 current snap is 290 2024-04-12T11:24:07.593 INFO:tasks.rados.rados.0.smithi029.stdout:2215: seq_num 828 ranges {716196=472296,1769561=646611,3121885=793109} 2024-04-12T11:24:07.598 INFO:tasks.rados.rados.0.smithi029.stdout:2215: writing smithi02979277-44 from 716196 to 1188492 tid 1 2024-04-12T11:24:07.605 INFO:tasks.rados.rados.0.smithi029.stdout:2215: writing smithi02979277-44 from 1769561 to 2416172 tid 2 2024-04-12T11:24:07.614 INFO:tasks.rados.rados.0.smithi029.stdout:2215: writing smithi02979277-44 from 3121885 to 3914994 tid 3 2024-04-12T11:24:07.614 INFO:tasks.rados.rados.0.smithi029.stdout:2216: delete oid 13 current snap is 290 2024-04-12T11:24:07.614 INFO:tasks.rados.rados.0.smithi029.stdout:2215: finishing write tid 1 to smithi02979277-44 2024-04-12T11:24:07.614 INFO:tasks.rados.rados.0.smithi029.stdout:2215: finishing write tid 2 to smithi02979277-44 2024-04-12T11:24:07.619 INFO:tasks.rados.rados.0.smithi029.stdout:2215: finishing write tid 3 to smithi02979277-44 2024-04-12T11:24:07.619 INFO:tasks.rados.rados.0.smithi029.stdout:2215: finishing write tid 4 to smithi02979277-44 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:2215: finishing write tid 5 to smithi02979277-44 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:2215: oid 44 updating version 0 to 1080 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:2215: oid 44 version 1080 is already newer than 1079 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:2215: oid 44 version 1080 is already newer than 1078 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:2215: oid 44 version 1080 is already newer than 1077 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1080 (ObjNum 828 snap 290 seq_num 828) dirty exists 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:2215: left oid 44 (ObjNum 828 snap 290 seq_num 828) 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:2215: done (1 left) 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:2216: done (0 left) 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:2217: read oid 47 snap -1 2024-04-12T11:24:07.620 INFO:tasks.rados.rados.0.smithi029.stdout:2217: expect deleted 2024-04-12T11:24:07.621 INFO:tasks.rados.rados.0.smithi029.stdout:2218: read oid 37 snap -1 2024-04-12T11:24:07.621 INFO:tasks.rados.rados.0.smithi029.stdout:2218: expect (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:24:07.621 INFO:tasks.rados.rados.0.smithi029.stdout:2219: snap_remove snap 282 2024-04-12T11:24:08.612 INFO:tasks.rados.rados.0.smithi029.stdout:2219: done (2 left) 2024-04-12T11:24:08.612 INFO:tasks.rados.rados.0.smithi029.stdout:2220: snap_remove snap 249 2024-04-12T11:24:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:08 smithi087 ceph-mon[89014]: pgmap v600: 89 pgs: 89 active+clean; 210 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:24:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:08 smithi087 ceph-mon[89014]: osdmap e643: 8 total, 8 up, 8 in 2024-04-12T11:24:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:08 smithi029 ceph-mon[97787]: pgmap v600: 89 pgs: 89 active+clean; 210 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:24:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:08 smithi029 ceph-mon[97787]: osdmap e643: 8 total, 8 up, 8 in 2024-04-12T11:24:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:08 smithi029 ceph-mon[93650]: pgmap v600: 89 pgs: 89 active+clean; 210 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:24:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:08 smithi029 ceph-mon[93650]: osdmap e643: 8 total, 8 up, 8 in 2024-04-12T11:24:09.618 INFO:tasks.rados.rados.0.smithi029.stdout:2220: done (2 left) 2024-04-12T11:24:09.618 INFO:tasks.rados.rados.0.smithi029.stdout:2221: write oid 34 current snap is 290 2024-04-12T11:24:09.618 INFO:tasks.rados.rados.0.smithi029.stdout:2221: seq_num 830 ranges {749810=739147,2002878=650280,3293084=415653} 2024-04-12T11:24:09.626 INFO:tasks.rados.rados.0.smithi029.stdout:2221: writing smithi02979277-34 from 749810 to 1488957 tid 1 2024-04-12T11:24:09.634 INFO:tasks.rados.rados.0.smithi029.stdout:2221: writing smithi02979277-34 from 2002878 to 2653158 tid 2 2024-04-12T11:24:09.640 INFO:tasks.rados.rados.0.smithi029.stdout:2221: writing smithi02979277-34 from 3293084 to 3708737 tid 3 2024-04-12T11:24:09.640 INFO:tasks.rados.rados.0.smithi029.stdout:2222: snap_remove snap 280 2024-04-12T11:24:09.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:09 smithi087 ceph-mon[89014]: osdmap e644: 8 total, 8 up, 8 in 2024-04-12T11:24:10.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:09 smithi029 ceph-mon[97787]: osdmap e644: 8 total, 8 up, 8 in 2024-04-12T11:24:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:09 smithi029 ceph-mon[93650]: osdmap e644: 8 total, 8 up, 8 in 2024-04-12T11:24:10.623 INFO:tasks.rados.rados.0.smithi029.stdout:2222: done (3 left) 2024-04-12T11:24:10.623 INFO:tasks.rados.rados.0.smithi029.stdout:2223: write oid 27 current snap is 290 2024-04-12T11:24:10.623 INFO:tasks.rados.rados.0.smithi029.stdout:2223: seq_num 831 ranges {766617=414396,1945007=510291,3204154=626925} 2024-04-12T11:24:10.629 INFO:tasks.rados.rados.0.smithi029.stdout:2223: writing smithi02979277-27 from 766617 to 1181013 tid 1 2024-04-12T11:24:10.636 INFO:tasks.rados.rados.0.smithi029.stdout:2223: writing smithi02979277-27 from 1945007 to 2455298 tid 2 2024-04-12T11:24:10.644 INFO:tasks.rados.rados.0.smithi029.stdout:2223: writing smithi02979277-27 from 3204154 to 3831079 tid 3 2024-04-12T11:24:10.644 INFO:tasks.rados.rados.0.smithi029.stdout:2224: delete oid 35 current snap is 290 2024-04-12T11:24:10.644 INFO:tasks.rados.rados.0.smithi029.stdout:2218: expect (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:24:10.857 INFO:tasks.rados.rados.0.smithi029.stdout:2221: finishing write tid 1 to smithi02979277-34 2024-04-12T11:24:10.857 INFO:tasks.rados.rados.0.smithi029.stdout:2221: finishing write tid 2 to smithi02979277-34 2024-04-12T11:24:10.857 INFO:tasks.rados.rados.0.smithi029.stdout:2217: done (4 left) 2024-04-12T11:24:10.857 INFO:tasks.rados.rados.0.smithi029.stdout:2218: done (3 left) 2024-04-12T11:24:10.857 INFO:tasks.rados.rados.0.smithi029.stdout:2224: done (2 left) 2024-04-12T11:24:10.857 INFO:tasks.rados.rados.0.smithi029.stdout:2225: read oid 10 snap -1 2024-04-12T11:24:10.857 INFO:tasks.rados.rados.0.smithi029.stdout:2225: expect (ObjNum 771 snap 270 seq_num 771) 2024-04-12T11:24:10.858 INFO:tasks.rados.rados.0.smithi029.stdout:2226: write oid 37 current snap is 290 2024-04-12T11:24:10.858 INFO:tasks.rados.rados.0.smithi029.stdout:2221: finishing write tid 3 to smithi02979277-34 2024-04-12T11:24:10.858 INFO:tasks.rados.rados.0.smithi029.stdout:2226: seq_num 833 ranges {400231=564894,1661971=338260} 2024-04-12T11:24:10.862 INFO:tasks.rados.rados.0.smithi029.stdout:2226: writing smithi02979277-37 from 400231 to 965125 tid 1 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2226: writing smithi02979277-37 from 1661971 to 2000231 tid 2 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2227: snap_create 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2221: finishing write tid 4 to smithi02979277-34 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2221: finishing write tid 5 to smithi02979277-34 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2221: oid 34 updating version 0 to 895 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2221: oid 34 updating version 895 to 898 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2221: oid 34 version 898 is already newer than 890 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2221: oid 34 updating version 898 to 899 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 899 (ObjNum 830 snap 290 seq_num 830) dirty exists 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2221: left oid 34 (ObjNum 830 snap 290 seq_num 830) 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2223: finishing write tid 1 to smithi02979277-27 2024-04-12T11:24:10.868 INFO:tasks.rados.rados.0.smithi029.stdout:2223: finishing write tid 2 to smithi02979277-27 2024-04-12T11:24:10.869 INFO:tasks.rados.rados.0.smithi029.stdout:2223: finishing write tid 3 to smithi02979277-27 2024-04-12T11:24:10.869 INFO:tasks.rados.rados.0.smithi029.stdout:2223: finishing write tid 4 to smithi02979277-27 2024-04-12T11:24:10.869 INFO:tasks.rados.rados.0.smithi029.stdout:2223: finishing write tid 5 to smithi02979277-27 2024-04-12T11:24:10.869 INFO:tasks.rados.rados.0.smithi029.stdout:2223: oid 27 updating version 0 to 748 2024-04-12T11:24:10.869 INFO:tasks.rados.rados.0.smithi029.stdout:2223: oid 27 updating version 748 to 749 2024-04-12T11:24:10.869 INFO:tasks.rados.rados.0.smithi029.stdout:2223: oid 27 updating version 749 to 751 2024-04-12T11:24:10.869 INFO:tasks.rados.rados.0.smithi029.stdout:2223: oid 27 version 751 is already newer than 750 2024-04-12T11:24:10.869 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 751 (ObjNum 831 snap 290 seq_num 831) dirty exists 2024-04-12T11:24:10.869 INFO:tasks.rados.rados.0.smithi029.stdout:2223: left oid 27 (ObjNum 831 snap 290 seq_num 831) 2024-04-12T11:24:10.869 INFO:tasks.rados.rados.0.smithi029.stdout:2225: expect (ObjNum 771 snap 270 seq_num 771) 2024-04-12T11:24:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:10 smithi087 ceph-mon[89014]: pgmap v603: 89 pgs: 89 active+clean; 214 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 866 KiB/s wr, 7 op/s 2024-04-12T11:24:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:10 smithi087 ceph-mon[89014]: osdmap e645: 8 total, 8 up, 8 in 2024-04-12T11:24:11.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:10 smithi029 ceph-mon[97787]: pgmap v603: 89 pgs: 89 active+clean; 214 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 866 KiB/s wr, 7 op/s 2024-04-12T11:24:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:10 smithi029 ceph-mon[97787]: osdmap e645: 8 total, 8 up, 8 in 2024-04-12T11:24:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:10 smithi029 ceph-mon[93650]: pgmap v603: 89 pgs: 89 active+clean; 214 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 866 KiB/s wr, 7 op/s 2024-04-12T11:24:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:10 smithi029 ceph-mon[93650]: osdmap e645: 8 total, 8 up, 8 in 2024-04-12T11:24:11.050 INFO:tasks.rados.rados.0.smithi029.stdout:2226: finishing write tid 1 to smithi02979277-37 2024-04-12T11:24:11.051 INFO:tasks.rados.rados.0.smithi029.stdout:2226: finishing write tid 2 to smithi02979277-37 2024-04-12T11:24:11.051 INFO:tasks.rados.rados.0.smithi029.stdout:2226: finishing write tid 3 to smithi02979277-37 2024-04-12T11:24:11.051 INFO:tasks.rados.rados.0.smithi029.stdout:2226: finishing write tid 4 to smithi02979277-37 2024-04-12T11:24:11.051 INFO:tasks.rados.rados.0.smithi029.stdout:2226: oid 37 updating version 0 to 817 2024-04-12T11:24:11.051 INFO:tasks.rados.rados.0.smithi029.stdout:2226: oid 37 version 817 is already newer than 816 2024-04-12T11:24:11.051 INFO:tasks.rados.rados.0.smithi029.stdout:2226: oid 37 version 817 is already newer than 815 2024-04-12T11:24:11.051 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 817 (ObjNum 833 snap 290 seq_num 833) dirty exists 2024-04-12T11:24:11.051 INFO:tasks.rados.rados.0.smithi029.stdout:2226: left oid 37 (ObjNum 833 snap 290 seq_num 833) 2024-04-12T11:24:11.628 INFO:tasks.rados.rados.0.smithi029.stdout:2221: done (4 left) 2024-04-12T11:24:11.629 INFO:tasks.rados.rados.0.smithi029.stdout:2223: done (3 left) 2024-04-12T11:24:11.629 INFO:tasks.rados.rados.0.smithi029.stdout:2225: done (2 left) 2024-04-12T11:24:11.629 INFO:tasks.rados.rados.0.smithi029.stdout:2226: done (1 left) 2024-04-12T11:24:11.629 INFO:tasks.rados.rados.0.smithi029.stdout:2227: done (0 left) 2024-04-12T11:24:11.629 INFO:tasks.rados.rados.0.smithi029.stdout:2228: write oid 12 current snap is 291 2024-04-12T11:24:11.629 INFO:tasks.rados.rados.0.smithi029.stdout:2228: seq_num 834 ranges {417038=640143,1720453=296585} 2024-04-12T11:24:11.634 INFO:tasks.rados.rados.0.smithi029.stdout:2228: writing smithi02979277-12 from 417038 to 1057181 tid 1 2024-04-12T11:24:11.639 INFO:tasks.rados.rados.0.smithi029.stdout:2228: writing smithi02979277-12 from 1720453 to 2017038 tid 2 2024-04-12T11:24:11.640 INFO:tasks.rados.rados.0.smithi029.stdout:2229: write oid 38 current snap is 291 2024-04-12T11:24:11.640 INFO:tasks.rados.rados.0.smithi029.stdout:2229: seq_num 835 ranges {433845=715392,1662582=371263} 2024-04-12T11:24:11.646 INFO:tasks.rados.rados.0.smithi029.stdout:2229: writing smithi02979277-38 from 433845 to 1149237 tid 1 2024-04-12T11:24:11.651 INFO:tasks.rados.rados.0.smithi029.stdout:2229: writing smithi02979277-38 from 1662582 to 2033845 tid 2 2024-04-12T11:24:11.651 INFO:tasks.rados.rados.0.smithi029.stdout:2230: delete oid 39 current snap is 291 2024-04-12T11:24:11.652 INFO:tasks.rados.rados.0.smithi029.stdout:2228: finishing write tid 1 to smithi02979277-12 2024-04-12T11:24:11.652 INFO:tasks.rados.rados.0.smithi029.stdout:2228: finishing write tid 2 to smithi02979277-12 2024-04-12T11:24:11.652 INFO:tasks.rados.rados.0.smithi029.stdout:2228: finishing write tid 3 to smithi02979277-12 2024-04-12T11:24:11.652 INFO:tasks.rados.rados.0.smithi029.stdout:2228: finishing write tid 4 to smithi02979277-12 2024-04-12T11:24:11.652 INFO:tasks.rados.rados.0.smithi029.stdout:2228: oid 12 updating version 0 to 1079 2024-04-12T11:24:11.652 INFO:tasks.rados.rados.0.smithi029.stdout:2228: oid 12 version 1079 is already newer than 1078 2024-04-12T11:24:11.652 INFO:tasks.rados.rados.0.smithi029.stdout:2228: oid 12 version 1079 is already newer than 1077 2024-04-12T11:24:11.652 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1079 (ObjNum 834 snap 291 seq_num 834) dirty exists 2024-04-12T11:24:11.652 INFO:tasks.rados.rados.0.smithi029.stdout:2228: left oid 12 (ObjNum 834 snap 291 seq_num 834) 2024-04-12T11:24:11.653 INFO:tasks.rados.rados.0.smithi029.stdout:2229: finishing write tid 1 to smithi02979277-38 2024-04-12T11:24:11.653 INFO:tasks.rados.rados.0.smithi029.stdout:2228: done (2 left) 2024-04-12T11:24:11.654 INFO:tasks.rados.rados.0.smithi029.stdout:2230: done (1 left) 2024-04-12T11:24:11.654 INFO:tasks.rados.rados.0.smithi029.stdout:2231: write oid 30 current snap is 291 2024-04-12T11:24:11.654 INFO:tasks.rados.rados.0.smithi029.stdout:2231: seq_num 837 ranges {467459=582243,1495899=404945} 2024-04-12T11:24:11.659 INFO:tasks.rados.rados.0.smithi029.stdout:2231: writing smithi02979277-30 from 467459 to 1049702 tid 1 2024-04-12T11:24:11.664 INFO:tasks.rados.rados.0.smithi029.stdout:2231: writing smithi02979277-30 from 1495899 to 1900844 tid 2 2024-04-12T11:24:11.664 INFO:tasks.rados.rados.0.smithi029.stdout:2232: snap_create 2024-04-12T11:24:11.664 INFO:tasks.rados.rados.0.smithi029.stdout:2229: finishing write tid 2 to smithi02979277-38 2024-04-12T11:24:11.664 INFO:tasks.rados.rados.0.smithi029.stdout:2229: finishing write tid 3 to smithi02979277-38 2024-04-12T11:24:11.664 INFO:tasks.rados.rados.0.smithi029.stdout:2229: finishing write tid 4 to smithi02979277-38 2024-04-12T11:24:11.664 INFO:tasks.rados.rados.0.smithi029.stdout:2229: oid 38 updating version 0 to 688 2024-04-12T11:24:11.664 INFO:tasks.rados.rados.0.smithi029.stdout:2229: oid 38 version 688 is already newer than 687 2024-04-12T11:24:11.665 INFO:tasks.rados.rados.0.smithi029.stdout:2229: oid 38 updating version 688 to 689 2024-04-12T11:24:11.665 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 689 (ObjNum 835 snap 291 seq_num 835) dirty exists 2024-04-12T11:24:11.665 INFO:tasks.rados.rados.0.smithi029.stdout:2229: left oid 38 (ObjNum 835 snap 291 seq_num 835) 2024-04-12T11:24:11.665 INFO:tasks.rados.rados.0.smithi029.stdout:2231: finishing write tid 1 to smithi02979277-30 2024-04-12T11:24:11.667 INFO:tasks.rados.rados.0.smithi029.stdout:2231: finishing write tid 2 to smithi02979277-30 2024-04-12T11:24:11.667 INFO:tasks.rados.rados.0.smithi029.stdout:2231: finishing write tid 3 to smithi02979277-30 2024-04-12T11:24:11.667 INFO:tasks.rados.rados.0.smithi029.stdout:2231: finishing write tid 4 to smithi02979277-30 2024-04-12T11:24:11.667 INFO:tasks.rados.rados.0.smithi029.stdout:2231: oid 30 updating version 0 to 752 2024-04-12T11:24:11.667 INFO:tasks.rados.rados.0.smithi029.stdout:2231: oid 30 updating version 752 to 753 2024-04-12T11:24:11.667 INFO:tasks.rados.rados.0.smithi029.stdout:2231: oid 30 updating version 753 to 754 2024-04-12T11:24:11.667 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 754 (ObjNum 837 snap 291 seq_num 837) dirty exists 2024-04-12T11:24:11.667 INFO:tasks.rados.rados.0.smithi029.stdout:2231: left oid 30 (ObjNum 837 snap 291 seq_num 837) 2024-04-12T11:24:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:11 smithi087 ceph-mon[89014]: osdmap e646: 8 total, 8 up, 8 in 2024-04-12T11:24:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:11 smithi029 ceph-mon[93650]: osdmap e646: 8 total, 8 up, 8 in 2024-04-12T11:24:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:11 smithi029 ceph-mon[97787]: osdmap e646: 8 total, 8 up, 8 in 2024-04-12T11:24:12.594 INFO:tasks.rados.rados.0.smithi029.stdout:2229: done (2 left) 2024-04-12T11:24:12.594 INFO:tasks.rados.rados.0.smithi029.stdout:2231: done (1 left) 2024-04-12T11:24:12.594 INFO:tasks.rados.rados.0.smithi029.stdout:2232: done (0 left) 2024-04-12T11:24:12.594 INFO:tasks.rados.rados.0.smithi029.stdout:2233: read oid 46 snap -1 2024-04-12T11:24:12.594 INFO:tasks.rados.rados.0.smithi029.stdout:2233: expect deleted 2024-04-12T11:24:12.595 INFO:tasks.rados.rados.0.smithi029.stdout:2234: write oid 5 current snap is 292 2024-04-12T11:24:12.595 INFO:tasks.rados.rados.0.smithi029.stdout:2234: seq_num 838 ranges {484266=657492,1554381=529885} 2024-04-12T11:24:12.600 INFO:tasks.rados.rados.0.smithi029.stdout:2234: writing smithi02979277-5 from 484266 to 1141758 tid 1 2024-04-12T11:24:12.606 INFO:tasks.rados.rados.0.smithi029.stdout:2234: writing smithi02979277-5 from 1554381 to 2084266 tid 2 2024-04-12T11:24:12.606 INFO:tasks.rados.rados.0.smithi029.stdout:2235: write oid 37 current snap is 292 2024-04-12T11:24:12.606 INFO:tasks.rados.rados.0.smithi029.stdout:2235: seq_num 839 ranges {501073=732741,1896510=204563} 2024-04-12T11:24:12.613 INFO:tasks.rados.rados.0.smithi029.stdout:2235: writing smithi02979277-37 from 501073 to 1233814 tid 1 2024-04-12T11:24:12.618 INFO:tasks.rados.rados.0.smithi029.stdout:2235: writing smithi02979277-37 from 1896510 to 2101073 tid 2 2024-04-12T11:24:12.618 INFO:tasks.rados.rados.0.smithi029.stdout:2236: delete oid 16 current snap is 292 2024-04-12T11:24:12.618 INFO:tasks.rados.rados.0.smithi029.stdout:2234: finishing write tid 1 to smithi02979277-5 2024-04-12T11:24:12.618 INFO:tasks.rados.rados.0.smithi029.stdout:2234: finishing write tid 2 to smithi02979277-5 2024-04-12T11:24:12.618 INFO:tasks.rados.rados.0.smithi029.stdout:2234: finishing write tid 3 to smithi02979277-5 2024-04-12T11:24:12.618 INFO:tasks.rados.rados.0.smithi029.stdout:2234: finishing write tid 4 to smithi02979277-5 2024-04-12T11:24:12.618 INFO:tasks.rados.rados.0.smithi029.stdout:2234: oid 5 updating version 0 to 1044 2024-04-12T11:24:12.618 INFO:tasks.rados.rados.0.smithi029.stdout:2234: oid 5 version 1044 is already newer than 1043 2024-04-12T11:24:12.619 INFO:tasks.rados.rados.0.smithi029.stdout:2234: oid 5 version 1044 is already newer than 1042 2024-04-12T11:24:12.619 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1044 (ObjNum 838 snap 292 seq_num 838) dirty exists 2024-04-12T11:24:12.619 INFO:tasks.rados.rados.0.smithi029.stdout:2234: left oid 5 (ObjNum 838 snap 292 seq_num 838) 2024-04-12T11:24:12.619 INFO:tasks.rados.rados.0.smithi029.stdout:2235: finishing write tid 1 to smithi02979277-37 2024-04-12T11:24:12.620 INFO:tasks.rados.rados.0.smithi029.stdout:2233: done (3 left) 2024-04-12T11:24:12.620 INFO:tasks.rados.rados.0.smithi029.stdout:2234: done (2 left) 2024-04-12T11:24:12.620 INFO:tasks.rados.rados.0.smithi029.stdout:2236: done (1 left) 2024-04-12T11:24:12.620 INFO:tasks.rados.rados.0.smithi029.stdout:2237: read oid 48 snap 254 2024-04-12T11:24:12.620 INFO:tasks.rados.rados.0.smithi029.stdout:2237: expect (ObjNum 726 snap 248 seq_num 726) 2024-04-12T11:24:12.621 INFO:tasks.rados.rados.0.smithi029.stdout:2238: snap_remove snap 259 2024-04-12T11:24:12.625 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:24:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:24:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:24:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:12 smithi087 ceph-mon[89014]: pgmap v606: 89 pgs: 89 active+clean; 214 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 460 KiB/s wr, 3 op/s 2024-04-12T11:24:12.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:12 smithi087 ceph-mon[89014]: osdmap e647: 8 total, 8 up, 8 in 2024-04-12T11:24:12.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:12 smithi087 ceph-mon[89014]: osdmap e648: 8 total, 8 up, 8 in 2024-04-12T11:24:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:12 smithi029 ceph-mon[93650]: pgmap v606: 89 pgs: 89 active+clean; 214 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 460 KiB/s wr, 3 op/s 2024-04-12T11:24:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:12 smithi029 ceph-mon[93650]: osdmap e647: 8 total, 8 up, 8 in 2024-04-12T11:24:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:12 smithi029 ceph-mon[93650]: osdmap e648: 8 total, 8 up, 8 in 2024-04-12T11:24:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:12 smithi029 ceph-mon[97787]: pgmap v606: 89 pgs: 89 active+clean; 214 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 460 KiB/s wr, 3 op/s 2024-04-12T11:24:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:12 smithi029 ceph-mon[97787]: osdmap e647: 8 total, 8 up, 8 in 2024-04-12T11:24:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:12 smithi029 ceph-mon[97787]: osdmap e648: 8 total, 8 up, 8 in 2024-04-12T11:24:13.596 INFO:tasks.rados.rados.0.smithi029.stdout:2238: done (2 left) 2024-04-12T11:24:13.596 INFO:tasks.rados.rados.0.smithi029.stdout:2239: delete oid 49 current snap is 292 2024-04-12T11:24:13.596 INFO:tasks.rados.rados.0.smithi029.stdout:2235: finishing write tid 2 to smithi02979277-37 2024-04-12T11:24:13.596 INFO:tasks.rados.rados.0.smithi029.stdout:2235: finishing write tid 3 to smithi02979277-37 2024-04-12T11:24:13.596 INFO:tasks.rados.rados.0.smithi029.stdout:2235: finishing write tid 4 to smithi02979277-37 2024-04-12T11:24:13.596 INFO:tasks.rados.rados.0.smithi029.stdout:2235: oid 37 updating version 0 to 820 2024-04-12T11:24:13.596 INFO:tasks.rados.rados.0.smithi029.stdout:2235: oid 37 updating version 820 to 821 2024-04-12T11:24:13.597 INFO:tasks.rados.rados.0.smithi029.stdout:2235: oid 37 version 821 is already newer than 819 2024-04-12T11:24:13.597 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 821 (ObjNum 839 snap 292 seq_num 839) dirty exists 2024-04-12T11:24:13.597 INFO:tasks.rados.rados.0.smithi029.stdout:2235: left oid 37 (ObjNum 839 snap 292 seq_num 839) 2024-04-12T11:24:13.597 INFO:tasks.rados.rados.0.smithi029.stdout:2237: expect (ObjNum 726 snap 248 seq_num 726) 2024-04-12T11:24:13.707 INFO:tasks.rados.rados.0.smithi029.stdout:2235: done (2 left) 2024-04-12T11:24:13.707 INFO:tasks.rados.rados.0.smithi029.stdout:2237: done (1 left) 2024-04-12T11:24:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:2239: done (0 left) 2024-04-12T11:24:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:2240: rollback oid 10 current snap is 292 2024-04-12T11:24:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 10 to 287 2024-04-12T11:24:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:2241: read oid 16 snap 254 2024-04-12T11:24:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:2241: expect (ObjNum 702 snap 231 seq_num 702) 2024-04-12T11:24:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:2242: snap_remove snap 291 2024-04-12T11:24:14.632 INFO:tasks.rados.rados.0.smithi029.stdout:2242: done (2 left) 2024-04-12T11:24:14.632 INFO:tasks.rados.rados.0.smithi029.stdout:2243: delete oid 33 current snap is 292 2024-04-12T11:24:14.632 INFO:tasks.rados.rados.0.smithi029.stdout:2240: finishing rollback tid 0 to smithi02979277-10 2024-04-12T11:24:14.632 INFO:tasks.rados.rados.0.smithi029.stdout:2240: finishing rollback tid 1 to smithi02979277-10 2024-04-12T11:24:14.633 INFO:tasks.rados.rados.0.smithi029.stdout:2240: finishing rollback tid 2 to smithi02979277-10 2024-04-12T11:24:14.633 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 921 (ObjNum 771 snap 270 seq_num 771) dirty exists 2024-04-12T11:24:14.633 INFO:tasks.rados.rados.0.smithi029.stdout:2241: expect (ObjNum 702 snap 231 seq_num 702) 2024-04-12T11:24:14.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:14 smithi029 ceph-mon[97787]: pgmap v609: 89 pgs: 89 active+clean; 223 MiB data, 751 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-04-12T11:24:14.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:14 smithi029 ceph-mon[97787]: osdmap e649: 8 total, 8 up, 8 in 2024-04-12T11:24:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:14 smithi029 ceph-mon[93650]: pgmap v609: 89 pgs: 89 active+clean; 223 MiB data, 751 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-04-12T11:24:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:14 smithi029 ceph-mon[93650]: osdmap e649: 8 total, 8 up, 8 in 2024-04-12T11:24:14.947 INFO:tasks.rados.rados.0.smithi029.stdout:2240: done (2 left) 2024-04-12T11:24:14.947 INFO:tasks.rados.rados.0.smithi029.stdout:2241: done (1 left) 2024-04-12T11:24:14.947 INFO:tasks.rados.rados.0.smithi029.stdout:2243: done (0 left) 2024-04-12T11:24:14.947 INFO:tasks.rados.rados.0.smithi029.stdout:2244: write oid 39 current snap is 292 2024-04-12T11:24:14.947 INFO:tasks.rados.rados.0.smithi029.stdout:2244: seq_num 843 ranges {568301=633737,1614085=415952} 2024-04-12T11:24:14.953 INFO:tasks.rados.rados.0.smithi029.stdout:2244: writing smithi02979277-39 from 568301 to 1202038 tid 1 2024-04-12T11:24:14.958 INFO:tasks.rados.rados.0.smithi029.stdout:2244: writing smithi02979277-39 from 1614085 to 2030037 tid 2 2024-04-12T11:24:14.958 INFO:tasks.rados.rados.0.smithi029.stdout:2245: rollback oid 17 current snap is 292 2024-04-12T11:24:14.958 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 290 2024-04-12T11:24:14.958 INFO:tasks.rados.rados.0.smithi029.stdout:2244: finishing write tid 1 to smithi02979277-39 2024-04-12T11:24:14.958 INFO:tasks.rados.rados.0.smithi029.stdout:2246: snap_remove snap 284 2024-04-12T11:24:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:14 smithi087 ceph-mon[89014]: pgmap v609: 89 pgs: 89 active+clean; 223 MiB data, 751 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-04-12T11:24:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:14 smithi087 ceph-mon[89014]: osdmap e649: 8 total, 8 up, 8 in 2024-04-12T11:24:15.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:24:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:24:14] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2024-04-12T11:24:15.637 INFO:tasks.rados.rados.0.smithi029.stdout:2246: done (2 left) 2024-04-12T11:24:15.641 INFO:tasks.rados.rados.0.smithi029.stdout:2247: read oid 2 snap -1 2024-04-12T11:24:15.641 INFO:tasks.rados.rados.0.smithi029.stdout:2247: expect deleted 2024-04-12T11:24:15.641 INFO:tasks.rados.rados.0.smithi029.stdout:2248: write oid 9 current snap is 292 2024-04-12T11:24:15.641 INFO:tasks.rados.rados.0.smithi029.stdout:2244: finishing write tid 2 to smithi02979277-39 2024-04-12T11:24:15.641 INFO:tasks.rados.rados.0.smithi029.stdout:2244: finishing write tid 3 to smithi02979277-39 2024-04-12T11:24:15.641 INFO:tasks.rados.rados.0.smithi029.stdout:2244: finishing write tid 4 to smithi02979277-39 2024-04-12T11:24:15.641 INFO:tasks.rados.rados.0.smithi029.stdout:2244: oid 39 updating version 0 to 274 2024-04-12T11:24:15.641 INFO:tasks.rados.rados.0.smithi029.stdout:2244: oid 39 updating version 274 to 275 2024-04-12T11:24:15.641 INFO:tasks.rados.rados.0.smithi029.stdout:2244: oid 39 updating version 275 to 276 2024-04-12T11:24:15.642 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 276 (ObjNum 843 snap 292 seq_num 843) dirty exists 2024-04-12T11:24:15.642 INFO:tasks.rados.rados.0.smithi029.stdout:2244: left oid 39 (ObjNum 843 snap 292 seq_num 843) 2024-04-12T11:24:15.642 INFO:tasks.rados.rados.0.smithi029.stdout:2245: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:24:15.642 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1035 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-04-12T11:24:15.642 INFO:tasks.rados.rados.0.smithi029.stdout:2248: seq_num 844 ranges {585108=425339,1788920=396188} 2024-04-12T11:24:15.644 INFO:tasks.rados.rados.0.smithi029.stdout:2248: writing smithi02979277-9 from 585108 to 1010447 tid 1 2024-04-12T11:24:15.651 INFO:tasks.rados.rados.0.smithi029.stdout:2248: writing smithi02979277-9 from 1788920 to 2185108 tid 2 2024-04-12T11:24:15.651 INFO:tasks.rados.rados.0.smithi029.stdout:2244: done (3 left) 2024-04-12T11:24:15.651 INFO:tasks.rados.rados.0.smithi029.stdout:2245: done (2 left) 2024-04-12T11:24:15.651 INFO:tasks.rados.rados.0.smithi029.stdout:2249: write oid 36 current snap is 292 2024-04-12T11:24:15.652 INFO:tasks.rados.rados.0.smithi029.stdout:2249: seq_num 845 ranges {601915=500588,1847402=354513} 2024-04-12T11:24:15.657 INFO:tasks.rados.rados.0.smithi029.stdout:2249: writing smithi02979277-36 from 601915 to 1102503 tid 1 2024-04-12T11:24:15.665 INFO:tasks.rados.rados.0.smithi029.stdout:2249: writing smithi02979277-36 from 1847402 to 2201915 tid 2 2024-04-12T11:24:15.665 INFO:tasks.rados.rados.0.smithi029.stdout:2250: read oid 48 snap 287 2024-04-12T11:24:15.665 INFO:tasks.rados.rados.0.smithi029.stdout:2250: expect (ObjNum 804 snap 282 seq_num 804) 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2251: rollback oid 30 current snap is 292 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 290 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2248: finishing write tid 1 to smithi02979277-9 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2248: finishing write tid 2 to smithi02979277-9 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2248: finishing write tid 3 to smithi02979277-9 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2247: done (4 left) 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2252: read oid 24 snap -1 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2252: expect (ObjNum 735 snap 253 seq_num 735) 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2253: read oid 29 snap -1 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2253: expect (ObjNum 813 snap 284 seq_num 813) 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2254: read oid 50 snap 275 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2254: expect (ObjNum 755 snap 259 seq_num 755) 2024-04-12T11:24:15.666 INFO:tasks.rados.rados.0.smithi029.stdout:2255: rollback oid 23 current snap is 292 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 23 to 285 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:2248: finishing write tid 4 to smithi02979277-9 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:2248: oid 9 updating version 0 to 837 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:2248: oid 9 updating version 837 to 838 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:2248: oid 9 version 838 is already newer than 836 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 838 (ObjNum 844 snap 292 seq_num 844) dirty exists 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:2248: left oid 9 (ObjNum 844 snap 292 seq_num 844) 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:2249: finishing write tid 1 to smithi02979277-36 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:2248: done (7 left) 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:2256: write oid 6 current snap is 292 2024-04-12T11:24:15.667 INFO:tasks.rados.rados.0.smithi029.stdout:2256: seq_num 846 ranges {618722=575837,1905884=312838} 2024-04-12T11:24:15.671 INFO:tasks.rados.rados.0.smithi029.stdout:2256: writing smithi02979277-6 from 618722 to 1194559 tid 1 2024-04-12T11:24:15.677 INFO:tasks.rados.rados.0.smithi029.stdout:2256: writing smithi02979277-6 from 1905884 to 2218722 tid 2 2024-04-12T11:24:15.678 INFO:tasks.rados.rados.0.smithi029.stdout:2257: read oid 27 snap 287 2024-04-12T11:24:15.678 INFO:tasks.rados.rados.0.smithi029.stdout:2257: expect deleted 2024-04-12T11:24:15.678 INFO:tasks.rados.rados.0.smithi029.stdout:2258: read oid 16 snap -1 2024-04-12T11:24:15.678 INFO:tasks.rados.rados.0.smithi029.stdout:2258: expect deleted 2024-04-12T11:24:15.678 INFO:tasks.rados.rados.0.smithi029.stdout:2259: delete oid 33 current snap is 292 2024-04-12T11:24:15.678 INFO:tasks.rados.rados.0.smithi029.stdout:2252: expect (ObjNum 735 snap 253 seq_num 735) 2024-04-12T11:24:15.858 INFO:tasks.rados.rados.0.smithi029.stdout:2250: expect (ObjNum 804 snap 282 seq_num 804) 2024-04-12T11:24:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:15 smithi087 ceph-mon[89014]: osdmap e650: 8 total, 8 up, 8 in 2024-04-12T11:24:16.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:15 smithi029 ceph-mon[97787]: osdmap e650: 8 total, 8 up, 8 in 2024-04-12T11:24:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:15 smithi029 ceph-mon[93650]: osdmap e650: 8 total, 8 up, 8 in 2024-04-12T11:24:16.040 INFO:tasks.rados.rados.0.smithi029.stdout:2251: finishing rollback tid 0 to smithi02979277-30 2024-04-12T11:24:16.040 INFO:tasks.rados.rados.0.smithi029.stdout:2251: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:24:16.040 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 760 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:24:16.040 INFO:tasks.rados.rados.0.smithi029.stdout:2249: finishing write tid 2 to smithi02979277-36 2024-04-12T11:24:16.040 INFO:tasks.rados.rados.0.smithi029.stdout:2249: finishing write tid 3 to smithi02979277-36 2024-04-12T11:24:16.040 INFO:tasks.rados.rados.0.smithi029.stdout:2249: finishing write tid 4 to smithi02979277-36 2024-04-12T11:24:16.040 INFO:tasks.rados.rados.0.smithi029.stdout:2249: oid 36 updating version 0 to 567 2024-04-12T11:24:16.040 INFO:tasks.rados.rados.0.smithi029.stdout:2249: oid 36 updating version 567 to 569 2024-04-12T11:24:16.040 INFO:tasks.rados.rados.0.smithi029.stdout:2249: oid 36 version 569 is already newer than 568 2024-04-12T11:24:16.041 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 569 (ObjNum 845 snap 292 seq_num 845) dirty exists 2024-04-12T11:24:16.041 INFO:tasks.rados.rados.0.smithi029.stdout:2249: left oid 36 (ObjNum 845 snap 292 seq_num 845) 2024-04-12T11:24:16.041 INFO:tasks.rados.rados.0.smithi029.stdout:2253: expect (ObjNum 813 snap 284 seq_num 813) 2024-04-12T11:24:16.215 INFO:tasks.rados.rados.0.smithi029.stdout:2254: expect (ObjNum 755 snap 259 seq_num 755) 2024-04-12T11:24:16.379 INFO:tasks.rados.rados.0.smithi029.stdout:2255: finishing rollback tid 0 to smithi02979277-23 2024-04-12T11:24:16.379 INFO:tasks.rados.rados.0.smithi029.stdout:2255: finishing rollback tid 1 to smithi02979277-23 2024-04-12T11:24:16.379 INFO:tasks.rados.rados.0.smithi029.stdout:2249: done (10 left) 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:2250: done (9 left) 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:2251: done (8 left) 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:2252: done (7 left) 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:2253: done (6 left) 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:2254: done (5 left) 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:2259: done (4 left) 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:2260: write oid 12 current snap is 292 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:2255: finishing rollback tid 2 to smithi02979277-23 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 863 (ObjNum 773 snap 271 seq_num 773) dirty exists 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:2256: finishing write tid 1 to smithi02979277-6 2024-04-12T11:24:16.380 INFO:tasks.rados.rados.0.smithi029.stdout:2260: seq_num 848 ranges {652336=726335,1906495=345841} 2024-04-12T11:24:16.386 INFO:tasks.rados.rados.0.smithi029.stdout:2260: writing smithi02979277-12 from 652336 to 1378671 tid 1 2024-04-12T11:24:16.391 INFO:tasks.rados.rados.0.smithi029.stdout:2260: writing smithi02979277-12 from 1906495 to 2252336 tid 2 2024-04-12T11:24:16.392 INFO:tasks.rados.rados.0.smithi029.stdout:2255: done (4 left) 2024-04-12T11:24:16.392 INFO:tasks.rados.rados.0.smithi029.stdout:2261: delete oid 42 current snap is 292 2024-04-12T11:24:16.392 INFO:tasks.rados.rados.0.smithi029.stdout:2256: finishing write tid 2 to smithi02979277-6 2024-04-12T11:24:16.392 INFO:tasks.rados.rados.0.smithi029.stdout:2256: finishing write tid 3 to smithi02979277-6 2024-04-12T11:24:16.392 INFO:tasks.rados.rados.0.smithi029.stdout:2256: finishing write tid 4 to smithi02979277-6 2024-04-12T11:24:16.392 INFO:tasks.rados.rados.0.smithi029.stdout:2256: oid 6 updating version 0 to 573 2024-04-12T11:24:16.392 INFO:tasks.rados.rados.0.smithi029.stdout:2256: oid 6 version 573 is already newer than 571 2024-04-12T11:24:16.392 INFO:tasks.rados.rados.0.smithi029.stdout:2256: oid 6 version 573 is already newer than 572 2024-04-12T11:24:16.392 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 573 (ObjNum 846 snap 292 seq_num 846) dirty exists 2024-04-12T11:24:16.392 INFO:tasks.rados.rados.0.smithi029.stdout:2256: left oid 6 (ObjNum 846 snap 292 seq_num 846) 2024-04-12T11:24:16.393 INFO:tasks.rados.rados.0.smithi029.stdout:2260: finishing write tid 1 to smithi02979277-12 2024-04-12T11:24:16.394 INFO:tasks.rados.rados.0.smithi029.stdout:2256: done (4 left) 2024-04-12T11:24:16.394 INFO:tasks.rados.rados.0.smithi029.stdout:2257: done (3 left) 2024-04-12T11:24:16.394 INFO:tasks.rados.rados.0.smithi029.stdout:2258: done (2 left) 2024-04-12T11:24:16.394 INFO:tasks.rados.rados.0.smithi029.stdout:2261: done (1 left) 2024-04-12T11:24:16.394 INFO:tasks.rados.rados.0.smithi029.stdout:2262: read oid 46 snap -1 2024-04-12T11:24:16.394 INFO:tasks.rados.rados.0.smithi029.stdout:2262: expect deleted 2024-04-12T11:24:16.395 INFO:tasks.rados.rados.0.smithi029.stdout:2263: rollback oid 43 current snap is 292 2024-04-12T11:24:16.395 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 290 2024-04-12T11:24:16.395 INFO:tasks.rados.rados.0.smithi029.stdout:2264: read oid 14 snap 288 2024-04-12T11:24:16.395 INFO:tasks.rados.rados.0.smithi029.stdout:2264: expect deleted 2024-04-12T11:24:16.395 INFO:tasks.rados.rados.0.smithi029.stdout:2265: write oid 27 current snap is 292 2024-04-12T11:24:16.395 INFO:tasks.rados.rados.0.smithi029.stdout:2265: seq_num 850 ranges {685950=476833,1623459=570617} 2024-04-12T11:24:16.399 INFO:tasks.rados.rados.0.smithi029.stdout:2265: writing smithi02979277-27 from 685950 to 1162783 tid 1 2024-04-12T11:24:16.405 INFO:tasks.rados.rados.0.smithi029.stdout:2265: writing smithi02979277-27 from 1623459 to 2194076 tid 2 2024-04-12T11:24:16.405 INFO:tasks.rados.rados.0.smithi029.stdout:2266: write oid 29 current snap is 292 2024-04-12T11:24:16.405 INFO:tasks.rados.rados.0.smithi029.stdout:2266: seq_num 851 ranges {702757=552082,1681941=430628} 2024-04-12T11:24:16.412 INFO:tasks.rados.rados.0.smithi029.stdout:2266: writing smithi02979277-29 from 702757 to 1254839 tid 1 2024-04-12T11:24:16.417 INFO:tasks.rados.rados.0.smithi029.stdout:2266: writing smithi02979277-29 from 1681941 to 2112569 tid 2 2024-04-12T11:24:16.417 INFO:tasks.rados.rados.0.smithi029.stdout:2267: snap_create 2024-04-12T11:24:16.417 INFO:tasks.rados.rados.0.smithi029.stdout:2260: finishing write tid 2 to smithi02979277-12 2024-04-12T11:24:16.417 INFO:tasks.rados.rados.0.smithi029.stdout:2260: finishing write tid 3 to smithi02979277-12 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2260: finishing write tid 4 to smithi02979277-12 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2260: oid 12 updating version 0 to 1092 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2260: oid 12 version 1092 is already newer than 1090 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2260: oid 12 version 1092 is already newer than 1091 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1092 (ObjNum 848 snap 292 seq_num 848) dirty exists 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2260: left oid 12 (ObjNum 848 snap 292 seq_num 848) 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2263: finishing rollback tid 0 to smithi02979277-43 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2263: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2263: finishing rollback tid 2 to smithi02979277-43 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 919 (ObjNum 427 snap 135 seq_num 427) dirty exists 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2265: finishing write tid 1 to smithi02979277-27 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2265: finishing write tid 2 to smithi02979277-27 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2265: finishing write tid 3 to smithi02979277-27 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2265: finishing write tid 4 to smithi02979277-27 2024-04-12T11:24:16.418 INFO:tasks.rados.rados.0.smithi029.stdout:2265: oid 27 updating version 0 to 762 2024-04-12T11:24:16.419 INFO:tasks.rados.rados.0.smithi029.stdout:2265: oid 27 updating version 762 to 763 2024-04-12T11:24:16.419 INFO:tasks.rados.rados.0.smithi029.stdout:2265: oid 27 updating version 763 to 764 2024-04-12T11:24:16.419 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 764 (ObjNum 850 snap 292 seq_num 850) dirty exists 2024-04-12T11:24:16.419 INFO:tasks.rados.rados.0.smithi029.stdout:2265: left oid 27 (ObjNum 850 snap 292 seq_num 850) 2024-04-12T11:24:16.419 INFO:tasks.rados.rados.0.smithi029.stdout:2266: finishing write tid 1 to smithi02979277-29 2024-04-12T11:24:16.420 INFO:tasks.rados.rados.0.smithi029.stdout:2266: finishing write tid 2 to smithi02979277-29 2024-04-12T11:24:16.420 INFO:tasks.rados.rados.0.smithi029.stdout:2266: finishing write tid 3 to smithi02979277-29 2024-04-12T11:24:16.420 INFO:tasks.rados.rados.0.smithi029.stdout:2266: finishing write tid 4 to smithi02979277-29 2024-04-12T11:24:16.420 INFO:tasks.rados.rados.0.smithi029.stdout:2266: oid 29 updating version 0 to 923 2024-04-12T11:24:16.420 INFO:tasks.rados.rados.0.smithi029.stdout:2266: oid 29 version 923 is already newer than 921 2024-04-12T11:24:16.420 INFO:tasks.rados.rados.0.smithi029.stdout:2266: oid 29 version 923 is already newer than 922 2024-04-12T11:24:16.420 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 923 (ObjNum 851 snap 292 seq_num 851) dirty exists 2024-04-12T11:24:16.420 INFO:tasks.rados.rados.0.smithi029.stdout:2266: left oid 29 (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:24:16.641 INFO:tasks.rados.rados.0.smithi029.stdout:2260: done (6 left) 2024-04-12T11:24:16.641 INFO:tasks.rados.rados.0.smithi029.stdout:2262: done (5 left) 2024-04-12T11:24:16.641 INFO:tasks.rados.rados.0.smithi029.stdout:2263: done (4 left) 2024-04-12T11:24:16.641 INFO:tasks.rados.rados.0.smithi029.stdout:2264: done (3 left) 2024-04-12T11:24:16.641 INFO:tasks.rados.rados.0.smithi029.stdout:2265: done (2 left) 2024-04-12T11:24:16.641 INFO:tasks.rados.rados.0.smithi029.stdout:2266: done (1 left) 2024-04-12T11:24:16.641 INFO:tasks.rados.rados.0.smithi029.stdout:2267: done (0 left) 2024-04-12T11:24:16.642 INFO:tasks.rados.rados.0.smithi029.stdout:2268: snap_create 2024-04-12T11:24:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:16 smithi087 ceph-mon[89014]: pgmap v612: 89 pgs: 89 active+clean; 216 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-04-12T11:24:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:16 smithi087 ceph-mon[89014]: osdmap e651: 8 total, 8 up, 8 in 2024-04-12T11:24:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:16 smithi029 ceph-mon[93650]: pgmap v612: 89 pgs: 89 active+clean; 216 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-04-12T11:24:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:16 smithi029 ceph-mon[93650]: osdmap e651: 8 total, 8 up, 8 in 2024-04-12T11:24:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:16 smithi029 ceph-mon[97787]: pgmap v612: 89 pgs: 89 active+clean; 216 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-04-12T11:24:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:16 smithi029 ceph-mon[97787]: osdmap e651: 8 total, 8 up, 8 in 2024-04-12T11:24:17.596 INFO:tasks.rados.rados.0.smithi029.stdout:2268: done (0 left) 2024-04-12T11:24:17.597 INFO:tasks.rados.rados.0.smithi029.stdout:2269: delete oid 9 current snap is 294 2024-04-12T11:24:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:2269: done (0 left) 2024-04-12T11:24:17.601 INFO:tasks.rados.rados.0.smithi029.stdout:2270: write oid 48 current snap is 294 2024-04-12T11:24:17.602 INFO:tasks.rados.rados.0.smithi029.stdout:2270: seq_num 853 ranges {736371=418933,1798905=434297} 2024-04-12T11:24:17.608 INFO:tasks.rados.rados.0.smithi029.stdout:2270: writing smithi02979277-48 from 736371 to 1155304 tid 1 2024-04-12T11:24:17.614 INFO:tasks.rados.rados.0.smithi029.stdout:2270: writing smithi02979277-48 from 1798905 to 2233202 tid 2 2024-04-12T11:24:17.614 INFO:tasks.rados.rados.0.smithi029.stdout:2271: snap_remove snap 287 2024-04-12T11:24:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:17 smithi087 ceph-mon[89014]: osdmap e652: 8 total, 8 up, 8 in 2024-04-12T11:24:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:17 smithi087 ceph-mon[89014]: osdmap e653: 8 total, 8 up, 8 in 2024-04-12T11:24:18.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:17 smithi029 ceph-mon[93650]: osdmap e652: 8 total, 8 up, 8 in 2024-04-12T11:24:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:17 smithi029 ceph-mon[93650]: osdmap e653: 8 total, 8 up, 8 in 2024-04-12T11:24:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:17 smithi029 ceph-mon[97787]: osdmap e652: 8 total, 8 up, 8 in 2024-04-12T11:24:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:17 smithi029 ceph-mon[97787]: osdmap e653: 8 total, 8 up, 8 in 2024-04-12T11:24:18.598 INFO:tasks.rados.rados.0.smithi029.stdout:2270: finishing write tid 1 to smithi02979277-48 2024-04-12T11:24:18.598 INFO:tasks.rados.rados.0.smithi029.stdout:2270: finishing write tid 2 to smithi02979277-48 2024-04-12T11:24:18.598 INFO:tasks.rados.rados.0.smithi029.stdout:2270: finishing write tid 3 to smithi02979277-48 2024-04-12T11:24:18.598 INFO:tasks.rados.rados.0.smithi029.stdout:2270: finishing write tid 4 to smithi02979277-48 2024-04-12T11:24:18.599 INFO:tasks.rados.rados.0.smithi029.stdout:2270: oid 48 updating version 0 to 929 2024-04-12T11:24:18.599 INFO:tasks.rados.rados.0.smithi029.stdout:2270: oid 48 updating version 929 to 930 2024-04-12T11:24:18.599 INFO:tasks.rados.rados.0.smithi029.stdout:2270: oid 48 updating version 930 to 931 2024-04-12T11:24:18.599 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 931 (ObjNum 853 snap 294 seq_num 853) dirty exists 2024-04-12T11:24:18.599 INFO:tasks.rados.rados.0.smithi029.stdout:2270: left oid 48 (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:24:18.599 INFO:tasks.rados.rados.0.smithi029.stdout:2270: done (1 left) 2024-04-12T11:24:18.599 INFO:tasks.rados.rados.0.smithi029.stdout:2271: done (0 left) 2024-04-12T11:24:18.599 INFO:tasks.rados.rados.0.smithi029.stdout:2272: read oid 7 snap -1 2024-04-12T11:24:18.599 INFO:tasks.rados.rados.0.smithi029.stdout:2272: expect (ObjNum 761 snap 262 seq_num 761) 2024-04-12T11:24:18.599 INFO:tasks.rados.rados.0.smithi029.stdout:2273: snap_remove snap 281 2024-04-12T11:24:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:18 smithi087 ceph-mon[89014]: pgmap v615: 89 pgs: 89 active+clean; 218 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 997 KiB/s wr, 13 op/s 2024-04-12T11:24:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:18 smithi087 ceph-mon[89014]: osdmap e654: 8 total, 8 up, 8 in 2024-04-12T11:24:19.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:18 smithi029 ceph-mon[93650]: pgmap v615: 89 pgs: 89 active+clean; 218 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 997 KiB/s wr, 13 op/s 2024-04-12T11:24:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:18 smithi029 ceph-mon[93650]: osdmap e654: 8 total, 8 up, 8 in 2024-04-12T11:24:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:18 smithi029 ceph-mon[97787]: pgmap v615: 89 pgs: 89 active+clean; 218 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 997 KiB/s wr, 13 op/s 2024-04-12T11:24:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:18 smithi029 ceph-mon[97787]: osdmap e654: 8 total, 8 up, 8 in 2024-04-12T11:24:19.601 INFO:tasks.rados.rados.0.smithi029.stdout:2272: expect (ObjNum 761 snap 262 seq_num 761) 2024-04-12T11:24:19.697 INFO:tasks.rados.rados.0.smithi029.stdout:2272: done (1 left) 2024-04-12T11:24:19.697 INFO:tasks.rados.rados.0.smithi029.stdout:2273: done (0 left) 2024-04-12T11:24:19.697 INFO:tasks.rados.rados.0.smithi029.stdout:2274: snap_remove snap 286 2024-04-12T11:24:20.650 INFO:tasks.rados.rados.0.smithi029.stdout:2274: done (0 left) 2024-04-12T11:24:20.651 INFO:tasks.rados.rados.0.smithi029.stdout:2275: write oid 25 current snap is 294 2024-04-12T11:24:20.651 INFO:tasks.rados.rados.0.smithi029.stdout:2275: seq_num 854 ranges {753178=494182,1857387=495791} 2024-04-12T11:24:20.657 INFO:tasks.rados.rados.0.smithi029.stdout:2275: writing smithi02979277-25 from 753178 to 1247360 tid 1 2024-04-12T11:24:20.664 INFO:tasks.rados.rados.0.smithi029.stdout:2275: writing smithi02979277-25 from 1857387 to 2353178 tid 2 2024-04-12T11:24:20.664 INFO:tasks.rados.rados.0.smithi029.stdout:2276: snap_remove snap 254 2024-04-12T11:24:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:20 smithi087 ceph-mon[89014]: pgmap v618: 89 pgs: 89 active+clean; 220 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2024-04-12T11:24:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:20 smithi087 ceph-mon[89014]: osdmap e655: 8 total, 8 up, 8 in 2024-04-12T11:24:21.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:20 smithi029 ceph-mon[97787]: pgmap v618: 89 pgs: 89 active+clean; 220 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2024-04-12T11:24:21.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:20 smithi029 ceph-mon[97787]: osdmap e655: 8 total, 8 up, 8 in 2024-04-12T11:24:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:20 smithi029 ceph-mon[93650]: pgmap v618: 89 pgs: 89 active+clean; 220 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2024-04-12T11:24:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:20 smithi029 ceph-mon[93650]: osdmap e655: 8 total, 8 up, 8 in 2024-04-12T11:24:21.657 INFO:tasks.rados.rados.0.smithi029.stdout:2276: done (1 left) 2024-04-12T11:24:21.657 INFO:tasks.rados.rados.0.smithi029.stdout:2277: write oid 17 current snap is 294 2024-04-12T11:24:21.657 INFO:tasks.rados.rados.0.smithi029.stdout:2277: seq_num 855 ranges {769985=569431,1915869=437966} 2024-04-12T11:24:21.665 INFO:tasks.rados.rados.0.smithi029.stdout:2277: writing smithi02979277-17 from 769985 to 1339416 tid 1 2024-04-12T11:24:21.672 INFO:tasks.rados.rados.0.smithi029.stdout:2277: writing smithi02979277-17 from 1915869 to 2353835 tid 2 2024-04-12T11:24:21.672 INFO:tasks.rados.rados.0.smithi029.stdout:2275: finishing write tid 1 to smithi02979277-25 2024-04-12T11:24:21.672 INFO:tasks.rados.rados.0.smithi029.stdout:2275: finishing write tid 2 to smithi02979277-25 2024-04-12T11:24:21.672 INFO:tasks.rados.rados.0.smithi029.stdout:2275: finishing write tid 3 to smithi02979277-25 2024-04-12T11:24:21.672 INFO:tasks.rados.rados.0.smithi029.stdout:2275: finishing write tid 4 to smithi02979277-25 2024-04-12T11:24:21.672 INFO:tasks.rados.rados.0.smithi029.stdout:2275: oid 25 updating version 0 to 1105 2024-04-12T11:24:21.672 INFO:tasks.rados.rados.0.smithi029.stdout:2275: oid 25 updating version 1105 to 1109 2024-04-12T11:24:21.672 INFO:tasks.rados.rados.0.smithi029.stdout:2275: oid 25 version 1109 is already newer than 1108 2024-04-12T11:24:21.672 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 1109 (ObjNum 854 snap 294 seq_num 854) dirty exists 2024-04-12T11:24:21.672 INFO:tasks.rados.rados.0.smithi029.stdout:2275: left oid 25 (ObjNum 854 snap 294 seq_num 854) 2024-04-12T11:24:21.673 INFO:tasks.rados.rados.0.smithi029.stdout:2275: done (1 left) 2024-04-12T11:24:21.673 INFO:tasks.rados.rados.0.smithi029.stdout:2278: snap_create 2024-04-12T11:24:21.673 INFO:tasks.rados.rados.0.smithi029.stdout:2277: finishing write tid 1 to smithi02979277-17 2024-04-12T11:24:21.675 INFO:tasks.rados.rados.0.smithi029.stdout:2277: finishing write tid 2 to smithi02979277-17 2024-04-12T11:24:21.675 INFO:tasks.rados.rados.0.smithi029.stdout:2277: finishing write tid 3 to smithi02979277-17 2024-04-12T11:24:21.675 INFO:tasks.rados.rados.0.smithi029.stdout:2277: finishing write tid 4 to smithi02979277-17 2024-04-12T11:24:21.675 INFO:tasks.rados.rados.0.smithi029.stdout:2277: oid 17 updating version 0 to 1118 2024-04-12T11:24:21.675 INFO:tasks.rados.rados.0.smithi029.stdout:2277: oid 17 version 1118 is already newer than 1117 2024-04-12T11:24:21.675 INFO:tasks.rados.rados.0.smithi029.stdout:2277: oid 17 version 1118 is already newer than 1116 2024-04-12T11:24:21.675 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1118 (ObjNum 855 snap 294 seq_num 855) dirty exists 2024-04-12T11:24:21.676 INFO:tasks.rados.rados.0.smithi029.stdout:2277: left oid 17 (ObjNum 855 snap 294 seq_num 855) 2024-04-12T11:24:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:21 smithi087 ceph-mon[89014]: osdmap e656: 8 total, 8 up, 8 in 2024-04-12T11:24:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:22.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:21 smithi029 ceph-mon[97787]: osdmap e656: 8 total, 8 up, 8 in 2024-04-12T11:24:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:21 smithi029 ceph-mon[93650]: osdmap e656: 8 total, 8 up, 8 in 2024-04-12T11:24:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:22.597 INFO:tasks.rados.rados.0.smithi029.stdout:2277: done (1 left) 2024-04-12T11:24:22.597 INFO:tasks.rados.rados.0.smithi029.stdout:2278: done (0 left) 2024-04-12T11:24:22.597 INFO:tasks.rados.rados.0.smithi029.stdout:2279: delete oid 11 current snap is 295 2024-04-12T11:24:22.602 INFO:tasks.rados.rados.0.smithi029.stdout:2279: done (0 left) 2024-04-12T11:24:22.602 INFO:tasks.rados.rados.0.smithi029.stdout:2280: rollback oid 44 current snap is 295 2024-04-12T11:24:22.602 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 44 to 275 2024-04-12T11:24:22.602 INFO:tasks.rados.rados.0.smithi029.stdout:2281: write oid 45 current snap is 295 2024-04-12T11:24:22.602 INFO:tasks.rados.rados.0.smithi029.stdout:2281: seq_num 857 ranges {403599=719929,1916480=441635} 2024-04-12T11:24:22.608 INFO:tasks.rados.rados.0.smithi029.stdout:2281: writing smithi02979277-45 from 403599 to 1123528 tid 1 2024-04-12T11:24:22.616 INFO:tasks.rados.rados.0.smithi029.stdout:2281: writing smithi02979277-45 from 1916480 to 2358115 tid 2 2024-04-12T11:24:22.616 INFO:tasks.rados.rados.0.smithi029.stdout:2282: rollback oid 41 current snap is 295 2024-04-12T11:24:22.616 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 288 2024-04-12T11:24:22.616 INFO:tasks.rados.rados.0.smithi029.stdout:2280: finishing rollback tid 0 to smithi02979277-44 2024-04-12T11:24:22.616 INFO:tasks.rados.rados.0.smithi029.stdout:2280: finishing rollback tid 1 to smithi02979277-44 2024-04-12T11:24:22.616 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1097 (ObjNum 657 snap 0 seq_num 2925506400) dirty dne 2024-04-12T11:24:22.616 INFO:tasks.rados.rados.0.smithi029.stdout:2281: finishing write tid 1 to smithi02979277-45 2024-04-12T11:24:22.616 INFO:tasks.rados.rados.0.smithi029.stdout:2280: done (2 left) 2024-04-12T11:24:22.616 INFO:tasks.rados.rados.0.smithi029.stdout:2283: delete oid 17 current snap is 295 2024-04-12T11:24:22.617 INFO:tasks.rados.rados.0.smithi029.stdout:2282: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:24:22.617 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 0 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-04-12T11:24:22.618 INFO:tasks.rados.rados.0.smithi029.stdout:2282: done (2 left) 2024-04-12T11:24:22.618 INFO:tasks.rados.rados.0.smithi029.stdout:2283: done (1 left) 2024-04-12T11:24:22.618 INFO:tasks.rados.rados.0.smithi029.stdout:2284: snap_create 2024-04-12T11:24:22.619 INFO:tasks.rados.rados.0.smithi029.stdout:2281: finishing write tid 2 to smithi02979277-45 2024-04-12T11:24:22.619 INFO:tasks.rados.rados.0.smithi029.stdout:2281: finishing write tid 3 to smithi02979277-45 2024-04-12T11:24:22.619 INFO:tasks.rados.rados.0.smithi029.stdout:2281: finishing write tid 4 to smithi02979277-45 2024-04-12T11:24:22.620 INFO:tasks.rados.rados.0.smithi029.stdout:2281: oid 45 updating version 0 to 1108 2024-04-12T11:24:22.620 INFO:tasks.rados.rados.0.smithi029.stdout:2281: oid 45 updating version 1108 to 1109 2024-04-12T11:24:22.620 INFO:tasks.rados.rados.0.smithi029.stdout:2281: oid 45 updating version 1109 to 1110 2024-04-12T11:24:22.620 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1110 (ObjNum 857 snap 295 seq_num 857) dirty exists 2024-04-12T11:24:22.620 INFO:tasks.rados.rados.0.smithi029.stdout:2281: left oid 45 (ObjNum 857 snap 295 seq_num 857) 2024-04-12T11:24:22.652 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:24:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:24:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:24:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:22 smithi087 ceph-mon[89014]: pgmap v621: 89 pgs: 89 active+clean; 222 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-04-12T11:24:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:22 smithi087 ceph-mon[89014]: osdmap e657: 8 total, 8 up, 8 in 2024-04-12T11:24:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:22 smithi087 ceph-mon[89014]: osdmap e658: 8 total, 8 up, 8 in 2024-04-12T11:24:23.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:22 smithi029 ceph-mon[97787]: pgmap v621: 89 pgs: 89 active+clean; 222 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-04-12T11:24:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:22 smithi029 ceph-mon[97787]: osdmap e657: 8 total, 8 up, 8 in 2024-04-12T11:24:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:22 smithi029 ceph-mon[97787]: osdmap e658: 8 total, 8 up, 8 in 2024-04-12T11:24:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:22 smithi029 ceph-mon[93650]: pgmap v621: 89 pgs: 89 active+clean; 222 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 2.0 MiB/s wr, 19 op/s 2024-04-12T11:24:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:22 smithi029 ceph-mon[93650]: osdmap e657: 8 total, 8 up, 8 in 2024-04-12T11:24:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:22 smithi029 ceph-mon[93650]: osdmap e658: 8 total, 8 up, 8 in 2024-04-12T11:24:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:2281: done (1 left) 2024-04-12T11:24:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:2284: done (0 left) 2024-04-12T11:24:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:2285: write oid 22 current snap is 296 2024-04-12T11:24:23.599 INFO:tasks.rados.rados.0.smithi029.stdout:2285: seq_num 859 ranges {437213=470427,1633444=728951} 2024-04-12T11:24:23.603 INFO:tasks.rados.rados.0.smithi029.stdout:2285: writing smithi02979277-22 from 437213 to 907640 tid 1 2024-04-12T11:24:23.612 INFO:tasks.rados.rados.0.smithi029.stdout:2285: writing smithi02979277-22 from 1633444 to 2362395 tid 2 2024-04-12T11:24:23.612 INFO:tasks.rados.rados.0.smithi029.stdout:2286: rollback oid 4 current snap is 296 2024-04-12T11:24:23.612 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 4 to 275 2024-04-12T11:24:23.612 INFO:tasks.rados.rados.0.smithi029.stdout:2285: finishing write tid 1 to smithi02979277-22 2024-04-12T11:24:23.612 INFO:tasks.rados.rados.0.smithi029.stdout:2287: delete oid 42 current snap is 296 2024-04-12T11:24:23.616 INFO:tasks.rados.rados.0.smithi029.stdout:2287: done (2 left) 2024-04-12T11:24:23.616 INFO:tasks.rados.rados.0.smithi029.stdout:2288: write oid 18 current snap is 296 2024-04-12T11:24:23.616 INFO:tasks.rados.rados.0.smithi029.stdout:2288: seq_num 861 ranges {470827=737278,1750408=720419} 2024-04-12T11:24:23.623 INFO:tasks.rados.rados.0.smithi029.stdout:2288: writing smithi02979277-18 from 470827 to 1208105 tid 1 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2288: writing smithi02979277-18 from 1750408 to 2470827 tid 2 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2289: snap_create 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2285: finishing write tid 2 to smithi02979277-22 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2285: finishing write tid 3 to smithi02979277-22 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2285: finishing write tid 4 to smithi02979277-22 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2285: oid 22 updating version 0 to 888 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2285: oid 22 version 888 is already newer than 886 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2285: oid 22 version 888 is already newer than 887 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 888 (ObjNum 859 snap 296 seq_num 859) dirty exists 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2285: left oid 22 (ObjNum 859 snap 296 seq_num 859) 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2286: finishing rollback tid 0 to smithi02979277-4 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2286: finishing rollback tid 1 to smithi02979277-4 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1069 (ObjNum 129 snap 0 seq_num 861971220) dirty dne 2024-04-12T11:24:23.631 INFO:tasks.rados.rados.0.smithi029.stdout:2288: finishing write tid 1 to smithi02979277-18 2024-04-12T11:24:23.634 INFO:tasks.rados.rados.0.smithi029.stdout:2288: finishing write tid 2 to smithi02979277-18 2024-04-12T11:24:23.634 INFO:tasks.rados.rados.0.smithi029.stdout:2288: finishing write tid 3 to smithi02979277-18 2024-04-12T11:24:23.634 INFO:tasks.rados.rados.0.smithi029.stdout:2288: finishing write tid 4 to smithi02979277-18 2024-04-12T11:24:23.634 INFO:tasks.rados.rados.0.smithi029.stdout:2288: oid 18 updating version 0 to 1123 2024-04-12T11:24:23.634 INFO:tasks.rados.rados.0.smithi029.stdout:2288: oid 18 updating version 1123 to 1124 2024-04-12T11:24:23.634 INFO:tasks.rados.rados.0.smithi029.stdout:2288: oid 18 version 1124 is already newer than 1122 2024-04-12T11:24:23.634 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1124 (ObjNum 861 snap 296 seq_num 861) dirty exists 2024-04-12T11:24:23.634 INFO:tasks.rados.rados.0.smithi029.stdout:2288: left oid 18 (ObjNum 861 snap 296 seq_num 861) 2024-04-12T11:24:24.605 INFO:tasks.rados.rados.0.smithi029.stdout:2285: done (3 left) 2024-04-12T11:24:24.605 INFO:tasks.rados.rados.0.smithi029.stdout:2286: done (2 left) 2024-04-12T11:24:24.605 INFO:tasks.rados.rados.0.smithi029.stdout:2288: done (1 left) 2024-04-12T11:24:24.605 INFO:tasks.rados.rados.0.smithi029.stdout:2289: done (0 left) 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2290: read oid 3 snap -1 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2290: expect (ObjNum 779 snap 272 seq_num 779) 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2291: read oid 22 snap -1 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2291: expect (ObjNum 859 snap 296 seq_num 859) 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2292: rollback oid 35 current snap is 297 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 35 to 289 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2293: read oid 21 snap -1 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2293: expect (ObjNum 672 snap 215 seq_num 672) 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2294: read oid 8 snap 283 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2294: expect (ObjNum 776 snap 272 seq_num 776) 2024-04-12T11:24:24.606 INFO:tasks.rados.rados.0.smithi029.stdout:2295: snap_create 2024-04-12T11:24:24.613 INFO:tasks.rados.rados.0.smithi029.stdout:2290: expect (ObjNum 779 snap 272 seq_num 779) 2024-04-12T11:24:24.802 INFO:tasks.rados.rados.0.smithi029.stdout:2292: finishing rollback tid 1 to smithi02979277-35 2024-04-12T11:24:24.803 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 0 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-04-12T11:24:24.803 INFO:tasks.rados.rados.0.smithi029.stdout:2293: expect (ObjNum 672 snap 215 seq_num 672) 2024-04-12T11:24:24.852 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:24 smithi087 ceph-mon[89014]: pgmap v624: 89 pgs: 89 active+clean; 195 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 914 KiB/s wr, 7 op/s 2024-04-12T11:24:24.852 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:24 smithi087 ceph-mon[89014]: osdmap e659: 8 total, 8 up, 8 in 2024-04-12T11:24:24.909 INFO:tasks.rados.rados.0.smithi029.stdout:2294: expect (ObjNum 776 snap 272 seq_num 776) 2024-04-12T11:24:24.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:24 smithi029 ceph-mon[93650]: pgmap v624: 89 pgs: 89 active+clean; 195 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 914 KiB/s wr, 7 op/s 2024-04-12T11:24:24.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:24 smithi029 ceph-mon[93650]: osdmap e659: 8 total, 8 up, 8 in 2024-04-12T11:24:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:24 smithi029 ceph-mon[97787]: pgmap v624: 89 pgs: 89 active+clean; 195 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 914 KiB/s wr, 7 op/s 2024-04-12T11:24:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:24 smithi029 ceph-mon[97787]: osdmap e659: 8 total, 8 up, 8 in 2024-04-12T11:24:25.194 INFO:tasks.rados.rados.0.smithi029.stdout:2291: expect (ObjNum 859 snap 296 seq_num 859) 2024-04-12T11:24:25.252 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:24:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:24:24] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-12T11:24:25.609 INFO:tasks.rados.rados.0.smithi029.stdout:2290: done (5 left) 2024-04-12T11:24:25.610 INFO:tasks.rados.rados.0.smithi029.stdout:2291: done (4 left) 2024-04-12T11:24:25.610 INFO:tasks.rados.rados.0.smithi029.stdout:2292: done (3 left) 2024-04-12T11:24:25.610 INFO:tasks.rados.rados.0.smithi029.stdout:2293: done (2 left) 2024-04-12T11:24:25.610 INFO:tasks.rados.rados.0.smithi029.stdout:2294: done (1 left) 2024-04-12T11:24:25.610 INFO:tasks.rados.rados.0.smithi029.stdout:2295: done (0 left) 2024-04-12T11:24:25.610 INFO:tasks.rados.rados.0.smithi029.stdout:2296: read oid 48 snap 288 2024-04-12T11:24:25.610 INFO:tasks.rados.rados.0.smithi029.stdout:2296: expect (ObjNum 804 snap 282 seq_num 804) 2024-04-12T11:24:25.610 INFO:tasks.rados.rados.0.smithi029.stdout:2297: delete oid 12 current snap is 298 2024-04-12T11:24:25.612 INFO:tasks.rados.rados.0.smithi029.stdout:2297: done (1 left) 2024-04-12T11:24:25.612 INFO:tasks.rados.rados.0.smithi029.stdout:2298: snap_create 2024-04-12T11:24:25.616 INFO:tasks.rados.rados.0.smithi029.stdout:2296: expect (ObjNum 804 snap 282 seq_num 804) 2024-04-12T11:24:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:25 smithi087 ceph-mon[89014]: osdmap e660: 8 total, 8 up, 8 in 2024-04-12T11:24:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:25 smithi029 ceph-mon[97787]: osdmap e660: 8 total, 8 up, 8 in 2024-04-12T11:24:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:25 smithi029 ceph-mon[93650]: osdmap e660: 8 total, 8 up, 8 in 2024-04-12T11:24:26.614 INFO:tasks.rados.rados.0.smithi029.stdout:2296: done (1 left) 2024-04-12T11:24:26.615 INFO:tasks.rados.rados.0.smithi029.stdout:2298: done (0 left) 2024-04-12T11:24:26.615 INFO:tasks.rados.rados.0.smithi029.stdout:2299: read oid 45 snap 283 2024-04-12T11:24:26.615 INFO:tasks.rados.rados.0.smithi029.stdout:2299: expect (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:24:26.615 INFO:tasks.rados.rados.0.smithi029.stdout:2300: delete oid 47 current snap is 299 2024-04-12T11:24:26.621 INFO:tasks.rados.rados.0.smithi029.stdout:2299: expect (ObjNum 537 snap 173 seq_num 537) 2024-04-12T11:24:26.725 INFO:tasks.rados.rados.0.smithi029.stdout:2299: done (1 left) 2024-04-12T11:24:26.725 INFO:tasks.rados.rados.0.smithi029.stdout:2300: done (0 left) 2024-04-12T11:24:26.725 INFO:tasks.rados.rados.0.smithi029.stdout:2301: snap_create 2024-04-12T11:24:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:26 smithi087 ceph-mon[89014]: pgmap v627: 89 pgs: 89 active+clean; 170 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 488 KiB/s wr, 4 op/s 2024-04-12T11:24:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:26 smithi087 ceph-mon[89014]: osdmap e661: 8 total, 8 up, 8 in 2024-04-12T11:24:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:26 smithi029 ceph-mon[93650]: pgmap v627: 89 pgs: 89 active+clean; 170 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 488 KiB/s wr, 4 op/s 2024-04-12T11:24:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:26 smithi029 ceph-mon[93650]: osdmap e661: 8 total, 8 up, 8 in 2024-04-12T11:24:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:26 smithi029 ceph-mon[97787]: pgmap v627: 89 pgs: 89 active+clean; 170 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 488 KiB/s wr, 4 op/s 2024-04-12T11:24:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:26 smithi029 ceph-mon[97787]: osdmap e661: 8 total, 8 up, 8 in 2024-04-12T11:24:27.599 INFO:tasks.rados.rados.0.smithi029.stdout:2301: done (0 left) 2024-04-12T11:24:27.600 INFO:tasks.rados.rados.0.smithi029.stdout:2302: snap_create 2024-04-12T11:24:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:27 smithi087 ceph-mon[89014]: osdmap e662: 8 total, 8 up, 8 in 2024-04-12T11:24:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:27 smithi087 ceph-mon[89014]: osdmap e663: 8 total, 8 up, 8 in 2024-04-12T11:24:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:27 smithi029 ceph-mon[97787]: osdmap e662: 8 total, 8 up, 8 in 2024-04-12T11:24:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:27 smithi029 ceph-mon[97787]: osdmap e663: 8 total, 8 up, 8 in 2024-04-12T11:24:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:27 smithi029 ceph-mon[93650]: osdmap e662: 8 total, 8 up, 8 in 2024-04-12T11:24:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:27 smithi029 ceph-mon[93650]: osdmap e663: 8 total, 8 up, 8 in 2024-04-12T11:24:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:2302: done (0 left) 2024-04-12T11:24:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:2303: read oid 34 snap -1 2024-04-12T11:24:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:2303: expect (ObjNum 830 snap 290 seq_num 830) 2024-04-12T11:24:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:2304: read oid 49 snap -1 2024-04-12T11:24:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:2304: expect deleted 2024-04-12T11:24:28.600 INFO:tasks.rados.rados.0.smithi029.stdout:2305: snap_create 2024-04-12T11:24:28.610 INFO:tasks.rados.rados.0.smithi029.stdout:2303: expect (ObjNum 830 snap 290 seq_num 830) 2024-04-12T11:24:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:28 smithi087 ceph-mon[89014]: pgmap v630: 89 pgs: 89 active+clean; 172 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 649 KiB/s wr, 9 op/s 2024-04-12T11:24:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:28 smithi087 ceph-mon[89014]: osdmap e664: 8 total, 8 up, 8 in 2024-04-12T11:24:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:28 smithi029 ceph-mon[97787]: pgmap v630: 89 pgs: 89 active+clean; 172 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 649 KiB/s wr, 9 op/s 2024-04-12T11:24:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:28 smithi029 ceph-mon[97787]: osdmap e664: 8 total, 8 up, 8 in 2024-04-12T11:24:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:28 smithi029 ceph-mon[93650]: pgmap v630: 89 pgs: 89 active+clean; 172 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 649 KiB/s wr, 9 op/s 2024-04-12T11:24:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:28 smithi029 ceph-mon[93650]: osdmap e664: 8 total, 8 up, 8 in 2024-04-12T11:24:29.602 INFO:tasks.rados.rados.0.smithi029.stdout:2303: done (2 left) 2024-04-12T11:24:29.602 INFO:tasks.rados.rados.0.smithi029.stdout:2304: done (1 left) 2024-04-12T11:24:29.603 INFO:tasks.rados.rados.0.smithi029.stdout:2305: done (0 left) 2024-04-12T11:24:29.603 INFO:tasks.rados.rados.0.smithi029.stdout:2306: read oid 40 snap 299 2024-04-12T11:24:29.603 INFO:tasks.rados.rados.0.smithi029.stdout:2306: expect (ObjNum 787 snap 274 seq_num 787) 2024-04-12T11:24:29.603 INFO:tasks.rados.rados.0.smithi029.stdout:2307: write oid 38 current snap is 302 2024-04-12T11:24:29.603 INFO:tasks.rados.rados.0.smithi029.stdout:2307: seq_num 864 ranges {521248=563025,1809501=596300} 2024-04-12T11:24:29.608 INFO:tasks.rados.rados.0.smithi029.stdout:2307: writing smithi02979277-38 from 521248 to 1084273 tid 1 2024-04-12T11:24:29.616 INFO:tasks.rados.rados.0.smithi029.stdout:2307: writing smithi02979277-38 from 1809501 to 2405801 tid 2 2024-04-12T11:24:29.616 INFO:tasks.rados.rados.0.smithi029.stdout:2308: read oid 44 snap 298 2024-04-12T11:24:29.616 INFO:tasks.rados.rados.0.smithi029.stdout:2308: expect deleted 2024-04-12T11:24:29.616 INFO:tasks.rados.rados.0.smithi029.stdout:2309: snap_create 2024-04-12T11:24:29.616 INFO:tasks.rados.rados.0.smithi029.stdout:2306: expect (ObjNum 787 snap 274 seq_num 787) 2024-04-12T11:24:29.759 INFO:tasks.rados.rados.0.smithi029.stdout:2307: finishing write tid 1 to smithi02979277-38 2024-04-12T11:24:29.760 INFO:tasks.rados.rados.0.smithi029.stdout:2307: finishing write tid 2 to smithi02979277-38 2024-04-12T11:24:29.760 INFO:tasks.rados.rados.0.smithi029.stdout:2307: finishing write tid 3 to smithi02979277-38 2024-04-12T11:24:29.760 INFO:tasks.rados.rados.0.smithi029.stdout:2307: finishing write tid 4 to smithi02979277-38 2024-04-12T11:24:29.760 INFO:tasks.rados.rados.0.smithi029.stdout:2307: oid 38 updating version 0 to 700 2024-04-12T11:24:29.760 INFO:tasks.rados.rados.0.smithi029.stdout:2307: oid 38 version 700 is already newer than 699 2024-04-12T11:24:29.760 INFO:tasks.rados.rados.0.smithi029.stdout:2307: oid 38 updating version 700 to 701 2024-04-12T11:24:29.760 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 701 (ObjNum 864 snap 302 seq_num 864) dirty exists 2024-04-12T11:24:29.760 INFO:tasks.rados.rados.0.smithi029.stdout:2307: left oid 38 (ObjNum 864 snap 302 seq_num 864) 2024-04-12T11:24:30.609 INFO:tasks.rados.rados.0.smithi029.stdout:2306: done (3 left) 2024-04-12T11:24:30.609 INFO:tasks.rados.rados.0.smithi029.stdout:2307: done (2 left) 2024-04-12T11:24:30.609 INFO:tasks.rados.rados.0.smithi029.stdout:2308: done (1 left) 2024-04-12T11:24:30.609 INFO:tasks.rados.rados.0.smithi029.stdout:2309: done (0 left) 2024-04-12T11:24:30.609 INFO:tasks.rados.rados.0.smithi029.stdout:2310: snap_create 2024-04-12T11:24:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:30 smithi087 ceph-mon[89014]: pgmap v633: 89 pgs: 89 active+clean; 174 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 933 KiB/s wr, 19 op/s 2024-04-12T11:24:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:30 smithi087 ceph-mon[89014]: osdmap e665: 8 total, 8 up, 8 in 2024-04-12T11:24:31.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:30 smithi029 ceph-mon[93650]: pgmap v633: 89 pgs: 89 active+clean; 174 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 933 KiB/s wr, 19 op/s 2024-04-12T11:24:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:30 smithi029 ceph-mon[93650]: osdmap e665: 8 total, 8 up, 8 in 2024-04-12T11:24:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:30 smithi029 ceph-mon[97787]: pgmap v633: 89 pgs: 89 active+clean; 174 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 933 KiB/s wr, 19 op/s 2024-04-12T11:24:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:30 smithi029 ceph-mon[97787]: osdmap e665: 8 total, 8 up, 8 in 2024-04-12T11:24:31.616 INFO:tasks.rados.rados.0.smithi029.stdout:2310: done (0 left) 2024-04-12T11:24:31.616 INFO:tasks.rados.rados.0.smithi029.stdout:2311: read oid 36 snap 297 2024-04-12T11:24:31.616 INFO:tasks.rados.rados.0.smithi029.stdout:2311: expect (ObjNum 845 snap 292 seq_num 845) 2024-04-12T11:24:31.616 INFO:tasks.rados.rados.0.smithi029.stdout:2312: write oid 21 current snap is 304 2024-04-12T11:24:31.616 INFO:tasks.rados.rados.0.smithi029.stdout:2312: seq_num 865 ranges {538055=638274,1867983=670072} 2024-04-12T11:24:31.622 INFO:tasks.rados.rados.0.smithi029.stdout:2312: writing smithi02979277-21 from 538055 to 1176329 tid 1 2024-04-12T11:24:31.630 INFO:tasks.rados.rados.0.smithi029.stdout:2312: writing smithi02979277-21 from 1867983 to 2538055 tid 2 2024-04-12T11:24:31.630 INFO:tasks.rados.rados.0.smithi029.stdout:2313: snap_create 2024-04-12T11:24:31.630 INFO:tasks.rados.rados.0.smithi029.stdout:2311: expect (ObjNum 845 snap 292 seq_num 845) 2024-04-12T11:24:31.750 INFO:tasks.rados.rados.0.smithi029.stdout:2312: finishing write tid 1 to smithi02979277-21 2024-04-12T11:24:31.750 INFO:tasks.rados.rados.0.smithi029.stdout:2312: finishing write tid 2 to smithi02979277-21 2024-04-12T11:24:31.750 INFO:tasks.rados.rados.0.smithi029.stdout:2312: finishing write tid 3 to smithi02979277-21 2024-04-12T11:24:31.751 INFO:tasks.rados.rados.0.smithi029.stdout:2312: finishing write tid 4 to smithi02979277-21 2024-04-12T11:24:31.751 INFO:tasks.rados.rados.0.smithi029.stdout:2312: oid 21 updating version 0 to 1099 2024-04-12T11:24:31.751 INFO:tasks.rados.rados.0.smithi029.stdout:2312: oid 21 updating version 1099 to 1100 2024-04-12T11:24:31.751 INFO:tasks.rados.rados.0.smithi029.stdout:2312: oid 21 updating version 1100 to 1101 2024-04-12T11:24:31.751 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1101 (ObjNum 865 snap 304 seq_num 865) dirty exists 2024-04-12T11:24:31.751 INFO:tasks.rados.rados.0.smithi029.stdout:2312: left oid 21 (ObjNum 865 snap 304 seq_num 865) 2024-04-12T11:24:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:31 smithi087 ceph-mon[89014]: osdmap e666: 8 total, 8 up, 8 in 2024-04-12T11:24:32.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:31 smithi029 ceph-mon[97787]: osdmap e666: 8 total, 8 up, 8 in 2024-04-12T11:24:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:31 smithi029 ceph-mon[93650]: osdmap e666: 8 total, 8 up, 8 in 2024-04-12T11:24:32.613 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:24:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:24:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:24:32.621 INFO:tasks.rados.rados.0.smithi029.stdout:2311: done (2 left) 2024-04-12T11:24:32.621 INFO:tasks.rados.rados.0.smithi029.stdout:2312: done (1 left) 2024-04-12T11:24:32.621 INFO:tasks.rados.rados.0.smithi029.stdout:2313: done (0 left) 2024-04-12T11:24:32.621 INFO:tasks.rados.rados.0.smithi029.stdout:2314: snap_create 2024-04-12T11:24:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:32 smithi087 ceph-mon[89014]: pgmap v636: 89 pgs: 89 active+clean; 174 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 284 KiB/s wr, 10 op/s 2024-04-12T11:24:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:32 smithi087 ceph-mon[89014]: osdmap e667: 8 total, 8 up, 8 in 2024-04-12T11:24:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:32 smithi029 ceph-mon[93650]: pgmap v636: 89 pgs: 89 active+clean; 174 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 284 KiB/s wr, 10 op/s 2024-04-12T11:24:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:32 smithi029 ceph-mon[93650]: osdmap e667: 8 total, 8 up, 8 in 2024-04-12T11:24:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:32 smithi029 ceph-mon[97787]: pgmap v636: 89 pgs: 89 active+clean; 174 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 284 KiB/s wr, 10 op/s 2024-04-12T11:24:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:32 smithi029 ceph-mon[97787]: osdmap e667: 8 total, 8 up, 8 in 2024-04-12T11:24:33.626 INFO:tasks.rados.rados.0.smithi029.stdout:2314: done (0 left) 2024-04-12T11:24:33.626 INFO:tasks.rados.rados.0.smithi029.stdout:2315: snap_remove snap 292 2024-04-12T11:24:33.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:33 smithi087 ceph-mon[89014]: osdmap e668: 8 total, 8 up, 8 in 2024-04-12T11:24:34.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:33 smithi029 ceph-mon[93650]: osdmap e668: 8 total, 8 up, 8 in 2024-04-12T11:24:34.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:33 smithi029 ceph-mon[97787]: osdmap e668: 8 total, 8 up, 8 in 2024-04-12T11:24:34.629 INFO:tasks.rados.rados.0.smithi029.stdout:2315: done (0 left) 2024-04-12T11:24:34.629 INFO:tasks.rados.rados.0.smithi029.stdout:2316: snap_remove snap 295 2024-04-12T11:24:34.920 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:34 smithi087 ceph-mon[89014]: pgmap v639: 89 pgs: 89 active+clean; 175 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 320 KiB/s wr, 1 op/s 2024-04-12T11:24:34.920 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:34 smithi087 ceph-mon[89014]: osdmap e669: 8 total, 8 up, 8 in 2024-04-12T11:24:34.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:34 smithi029 ceph-mon[93650]: pgmap v639: 89 pgs: 89 active+clean; 175 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 320 KiB/s wr, 1 op/s 2024-04-12T11:24:34.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:34 smithi029 ceph-mon[93650]: osdmap e669: 8 total, 8 up, 8 in 2024-04-12T11:24:34.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:34 smithi029 ceph-mon[97787]: pgmap v639: 89 pgs: 89 active+clean; 175 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 320 KiB/s wr, 1 op/s 2024-04-12T11:24:34.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:34 smithi029 ceph-mon[97787]: osdmap e669: 8 total, 8 up, 8 in 2024-04-12T11:24:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:24:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:24:34] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-12T11:24:35.639 INFO:tasks.rados.rados.0.smithi029.stdout:2316: done (0 left) 2024-04-12T11:24:35.639 INFO:tasks.rados.rados.0.smithi029.stdout:2317: snap_create 2024-04-12T11:24:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:35 smithi087 ceph-mon[89014]: osdmap e670: 8 total, 8 up, 8 in 2024-04-12T11:24:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:35 smithi029 ceph-mon[93650]: osdmap e670: 8 total, 8 up, 8 in 2024-04-12T11:24:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:35 smithi029 ceph-mon[97787]: osdmap e670: 8 total, 8 up, 8 in 2024-04-12T11:24:36.640 INFO:tasks.rados.rados.0.smithi029.stdout:2317: done (0 left) 2024-04-12T11:24:36.641 INFO:tasks.rados.rados.0.smithi029.stdout:2318: snap_remove snap 290 2024-04-12T11:24:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:36 smithi087 ceph-mon[89014]: pgmap v642: 89 pgs: 89 active+clean; 176 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 603 KiB/s wr, 6 op/s 2024-04-12T11:24:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:36 smithi087 ceph-mon[89014]: osdmap e671: 8 total, 8 up, 8 in 2024-04-12T11:24:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:36 smithi029 ceph-mon[93650]: pgmap v642: 89 pgs: 89 active+clean; 176 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 603 KiB/s wr, 6 op/s 2024-04-12T11:24:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:36 smithi029 ceph-mon[93650]: osdmap e671: 8 total, 8 up, 8 in 2024-04-12T11:24:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:36 smithi029 ceph-mon[97787]: pgmap v642: 89 pgs: 89 active+clean; 176 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 603 KiB/s wr, 6 op/s 2024-04-12T11:24:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:36 smithi029 ceph-mon[97787]: osdmap e671: 8 total, 8 up, 8 in 2024-04-12T11:24:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:37.642 INFO:tasks.rados.rados.0.smithi029.stdout:2318: done (0 left) 2024-04-12T11:24:37.643 INFO:tasks.rados.rados.0.smithi029.stdout:2319: write oid 50 current snap is 307 2024-04-12T11:24:37.643 INFO:tasks.rados.rados.0.smithi029.stdout:2319: seq_num 866 ranges {554862=713523,1926465=599969} 2024-04-12T11:24:37.650 INFO:tasks.rados.rados.0.smithi029.stdout:2319: writing smithi02979277-50 from 554862 to 1268385 tid 1 2024-04-12T11:24:37.658 INFO:tasks.rados.rados.0.smithi029.stdout:2319: writing smithi02979277-50 from 1926465 to 2526434 tid 2 2024-04-12T11:24:37.658 INFO:tasks.rados.rados.0.smithi029.stdout:2320: read oid 4 snap -1 2024-04-12T11:24:37.658 INFO:tasks.rados.rados.0.smithi029.stdout:2320: expect deleted 2024-04-12T11:24:37.658 INFO:tasks.rados.rados.0.smithi029.stdout:2321: delete oid 12 current snap is 307 2024-04-12T11:24:37.661 INFO:tasks.rados.rados.0.smithi029.stdout:2319: finishing write tid 1 to smithi02979277-50 2024-04-12T11:24:37.661 INFO:tasks.rados.rados.0.smithi029.stdout:2320: done (2 left) 2024-04-12T11:24:37.661 INFO:tasks.rados.rados.0.smithi029.stdout:2321: done (1 left) 2024-04-12T11:24:37.662 INFO:tasks.rados.rados.0.smithi029.stdout:2322: read oid 23 snap -1 2024-04-12T11:24:37.662 INFO:tasks.rados.rados.0.smithi029.stdout:2322: expect (ObjNum 773 snap 271 seq_num 773) 2024-04-12T11:24:37.662 INFO:tasks.rados.rados.0.smithi029.stdout:2323: read oid 49 snap -1 2024-04-12T11:24:37.662 INFO:tasks.rados.rados.0.smithi029.stdout:2323: expect deleted 2024-04-12T11:24:37.662 INFO:tasks.rados.rados.0.smithi029.stdout:2324: read oid 21 snap -1 2024-04-12T11:24:37.662 INFO:tasks.rados.rados.0.smithi029.stdout:2324: expect (ObjNum 865 snap 304 seq_num 865) 2024-04-12T11:24:37.662 INFO:tasks.rados.rados.0.smithi029.stdout:2325: write oid 8 current snap is 307 2024-04-12T11:24:37.662 INFO:tasks.rados.rados.0.smithi029.stdout:2325: seq_num 868 ranges {588476=580374,1643429=603638} 2024-04-12T11:24:37.667 INFO:tasks.rados.rados.0.smithi029.stdout:2325: writing smithi02979277-8 from 588476 to 1168850 tid 1 2024-04-12T11:24:37.675 INFO:tasks.rados.rados.0.smithi029.stdout:2325: writing smithi02979277-8 from 1643429 to 2247067 tid 2 2024-04-12T11:24:37.675 INFO:tasks.rados.rados.0.smithi029.stdout:2326: write oid 9 current snap is 307 2024-04-12T11:24:37.675 INFO:tasks.rados.rados.0.smithi029.stdout:2326: seq_num 869 ranges {605283=655623,1701911=747296} 2024-04-12T11:24:37.682 INFO:tasks.rados.rados.0.smithi029.stdout:2326: writing smithi02979277-9 from 605283 to 1260906 tid 1 2024-04-12T11:24:37.688 INFO:tasks.rados.rados.0.smithi029.stdout:2326: writing smithi02979277-9 from 1701911 to 2449207 tid 2 2024-04-12T11:24:37.688 INFO:tasks.rados.rados.0.smithi029.stdout:2327: snap_remove snap 305 2024-04-12T11:24:37.689 INFO:tasks.rados.rados.0.smithi029.stdout:2325: finishing write tid 1 to smithi02979277-8 2024-04-12T11:24:37.689 INFO:tasks.rados.rados.0.smithi029.stdout:2324: expect (ObjNum 865 snap 304 seq_num 865) 2024-04-12T11:24:37.797 INFO:tasks.rados.rados.0.smithi029.stdout:2319: finishing write tid 2 to smithi02979277-50 2024-04-12T11:24:37.797 INFO:tasks.rados.rados.0.smithi029.stdout:2319: finishing write tid 3 to smithi02979277-50 2024-04-12T11:24:37.798 INFO:tasks.rados.rados.0.smithi029.stdout:2319: finishing write tid 4 to smithi02979277-50 2024-04-12T11:24:37.798 INFO:tasks.rados.rados.0.smithi029.stdout:2319: oid 50 updating version 0 to 894 2024-04-12T11:24:37.798 INFO:tasks.rados.rados.0.smithi029.stdout:2319: oid 50 updating version 894 to 899 2024-04-12T11:24:37.798 INFO:tasks.rados.rados.0.smithi029.stdout:2319: oid 50 updating version 899 to 900 2024-04-12T11:24:37.798 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 900 (ObjNum 866 snap 307 seq_num 866) dirty exists 2024-04-12T11:24:37.798 INFO:tasks.rados.rados.0.smithi029.stdout:2319: left oid 50 (ObjNum 866 snap 307 seq_num 866) 2024-04-12T11:24:37.798 INFO:tasks.rados.rados.0.smithi029.stdout:2322: expect (ObjNum 773 snap 271 seq_num 773) 2024-04-12T11:24:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:37 smithi087 ceph-mon[89014]: osdmap e672: 8 total, 8 up, 8 in 2024-04-12T11:24:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:37 smithi087 ceph-mon[89014]: osdmap e673: 8 total, 8 up, 8 in 2024-04-12T11:24:37.992 INFO:tasks.rados.rados.0.smithi029.stdout:2325: finishing write tid 2 to smithi02979277-8 2024-04-12T11:24:37.992 INFO:tasks.rados.rados.0.smithi029.stdout:2325: finishing write tid 3 to smithi02979277-8 2024-04-12T11:24:37.992 INFO:tasks.rados.rados.0.smithi029.stdout:2325: finishing write tid 4 to smithi02979277-8 2024-04-12T11:24:37.993 INFO:tasks.rados.rados.0.smithi029.stdout:2325: oid 8 updating version 0 to 237 2024-04-12T11:24:37.993 INFO:tasks.rados.rados.0.smithi029.stdout:2325: oid 8 updating version 237 to 239 2024-04-12T11:24:37.993 INFO:tasks.rados.rados.0.smithi029.stdout:2325: oid 8 version 239 is already newer than 238 2024-04-12T11:24:37.993 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 239 (ObjNum 868 snap 307 seq_num 868) dirty exists 2024-04-12T11:24:37.993 INFO:tasks.rados.rados.0.smithi029.stdout:2325: left oid 8 (ObjNum 868 snap 307 seq_num 868) 2024-04-12T11:24:37.993 INFO:tasks.rados.rados.0.smithi029.stdout:2326: finishing write tid 1 to smithi02979277-9 2024-04-12T11:24:37.993 INFO:tasks.rados.rados.0.smithi029.stdout:2326: finishing write tid 2 to smithi02979277-9 2024-04-12T11:24:38.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:37 smithi029 ceph-mon[93650]: osdmap e672: 8 total, 8 up, 8 in 2024-04-12T11:24:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:37 smithi029 ceph-mon[93650]: osdmap e673: 8 total, 8 up, 8 in 2024-04-12T11:24:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:37 smithi029 ceph-mon[97787]: osdmap e672: 8 total, 8 up, 8 in 2024-04-12T11:24:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:37 smithi029 ceph-mon[97787]: osdmap e673: 8 total, 8 up, 8 in 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:2319: done (6 left) 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:2322: done (5 left) 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:2323: done (4 left) 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:2324: done (3 left) 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:2325: done (2 left) 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:2327: done (1 left) 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:2328: rollback oid 14 current snap is 307 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 293 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:2326: finishing write tid 3 to smithi02979277-9 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:2326: finishing write tid 4 to smithi02979277-9 2024-04-12T11:24:38.659 INFO:tasks.rados.rados.0.smithi029.stdout:2326: oid 9 updating version 0 to 869 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2326: oid 9 version 869 is already newer than 867 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2326: oid 9 version 869 is already newer than 868 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 869 (ObjNum 869 snap 307 seq_num 869) dirty exists 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2326: left oid 9 (ObjNum 869 snap 307 seq_num 869) 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2326: done (1 left) 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2329: rollback oid 43 current snap is 307 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 304 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2330: rollback oid 6 current snap is 307 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 6 to 301 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2331: read oid 8 snap 296 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2331: expect (ObjNum 776 snap 272 seq_num 776) 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2332: read oid 48 snap 294 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2332: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:2333: rollback oid 13 current snap is 307 2024-04-12T11:24:38.660 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 13 to 293 2024-04-12T11:24:38.661 INFO:tasks.rados.rados.0.smithi029.stdout:2334: delete oid 35 current snap is 307 2024-04-12T11:24:38.666 INFO:tasks.rados.rados.0.smithi029.stdout:2332: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:24:38.836 INFO:tasks.rados.rados.0.smithi029.stdout:2333: finishing rollback tid 1 to smithi02979277-13 2024-04-12T11:24:38.836 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1070 (ObjNum 129 snap 0 seq_num 1800664536) dirty dne 2024-04-12T11:24:38.836 INFO:tasks.rados.rados.0.smithi029.stdout:2329: finishing rollback tid 0 to smithi02979277-43 2024-04-12T11:24:38.836 INFO:tasks.rados.rados.0.smithi029.stdout:2329: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:2332: done (6 left) 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:2333: done (5 left) 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:2334: done (4 left) 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:2335: write oid 5 current snap is 307 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:2330: finishing rollback tid 0 to smithi02979277-6 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:2330: finishing rollback tid 1 to smithi02979277-6 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:2330: finishing rollback tid 2 to smithi02979277-6 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 597 (ObjNum 846 snap 292 seq_num 846) dirty exists 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:2329: finishing rollback tid 2 to smithi02979277-43 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 955 (ObjNum 427 snap 135 seq_num 427) dirty exists 2024-04-12T11:24:38.837 INFO:tasks.rados.rados.0.smithi029.stdout:2331: expect (ObjNum 776 snap 272 seq_num 776) 2024-04-12T11:24:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:38 smithi087 ceph-mon[89014]: pgmap v645: 89 pgs: 89 active+clean; 176 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 283 KiB/s wr, 5 op/s 2024-04-12T11:24:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:38 smithi029 ceph-mon[97787]: pgmap v645: 89 pgs: 89 active+clean; 176 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 283 KiB/s wr, 5 op/s 2024-04-12T11:24:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:38 smithi029 ceph-mon[93650]: pgmap v645: 89 pgs: 89 active+clean; 176 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 283 KiB/s wr, 5 op/s 2024-04-12T11:24:39.122 INFO:tasks.rados.rados.0.smithi029.stdout:2328: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:24:39.122 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-04-12T11:24:39.122 INFO:tasks.rados.rados.0.smithi029.stdout:2335: seq_num 871 ranges {638897=406121,1818875=750965} 2024-04-12T11:24:39.127 INFO:tasks.rados.rados.0.smithi029.stdout:2335: writing smithi02979277-5 from 638897 to 1045018 tid 1 2024-04-12T11:24:39.136 INFO:tasks.rados.rados.0.smithi029.stdout:2335: writing smithi02979277-5 from 1818875 to 2569840 tid 2 2024-04-12T11:24:39.136 INFO:tasks.rados.rados.0.smithi029.stdout:2328: done (4 left) 2024-04-12T11:24:39.136 INFO:tasks.rados.rados.0.smithi029.stdout:2329: done (3 left) 2024-04-12T11:24:39.136 INFO:tasks.rados.rados.0.smithi029.stdout:2330: done (2 left) 2024-04-12T11:24:39.136 INFO:tasks.rados.rados.0.smithi029.stdout:2331: done (1 left) 2024-04-12T11:24:39.136 INFO:tasks.rados.rados.0.smithi029.stdout:2336: rollback oid 24 current snap is 307 2024-04-12T11:24:39.136 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 24 to 293 2024-04-12T11:24:39.136 INFO:tasks.rados.rados.0.smithi029.stdout:2335: finishing write tid 1 to smithi02979277-5 2024-04-12T11:24:39.136 INFO:tasks.rados.rados.0.smithi029.stdout:2337: snap_remove snap 300 2024-04-12T11:24:39.662 INFO:tasks.rados.rados.0.smithi029.stdout:2337: done (2 left) 2024-04-12T11:24:39.662 INFO:tasks.rados.rados.0.smithi029.stdout:2338: write oid 6 current snap is 307 2024-04-12T11:24:39.662 INFO:tasks.rados.rados.0.smithi029.stdout:2336: finishing rollback tid 0 to smithi02979277-24 2024-04-12T11:24:39.662 INFO:tasks.rados.rados.0.smithi029.stdout:2338: seq_num 872 ranges {655704=481370,1761004=494623} 2024-04-12T11:24:39.668 INFO:tasks.rados.rados.0.smithi029.stdout:2338: writing smithi02979277-6 from 655704 to 1137074 tid 1 2024-04-12T11:24:39.674 INFO:tasks.rados.rados.0.smithi029.stdout:2338: writing smithi02979277-6 from 1761004 to 2255627 tid 2 2024-04-12T11:24:39.674 INFO:tasks.rados.rados.0.smithi029.stdout:2339: snap_remove snap 299 2024-04-12T11:24:39.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:39 smithi087 ceph-mon[89014]: osdmap e674: 8 total, 8 up, 8 in 2024-04-12T11:24:39.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:39 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:24:40.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:39 smithi029 ceph-mon[97787]: osdmap e674: 8 total, 8 up, 8 in 2024-04-12T11:24:40.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:39 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:24:40.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:39 smithi029 ceph-mon[93650]: osdmap e674: 8 total, 8 up, 8 in 2024-04-12T11:24:40.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:39 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:24:40.660 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:40 smithi087 ceph-mon[89014]: pgmap v648: 89 pgs: 89 active+clean; 176 MiB data, 687 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:24:40.660 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:40 smithi087 ceph-mon[89014]: osdmap e675: 8 total, 8 up, 8 in 2024-04-12T11:24:40.668 INFO:tasks.rados.rados.0.smithi029.stdout:2339: done (3 left) 2024-04-12T11:24:40.668 INFO:tasks.rados.rados.0.smithi029.stdout:2340: snap_create 2024-04-12T11:24:40.668 INFO:tasks.rados.rados.0.smithi029.stdout:2336: finishing rollback tid 1 to smithi02979277-24 2024-04-12T11:24:40.668 INFO:tasks.rados.rados.0.smithi029.stdout:2336: finishing rollback tid 2 to smithi02979277-24 2024-04-12T11:24:40.668 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1366 (ObjNum 735 snap 253 seq_num 735) dirty exists 2024-04-12T11:24:40.668 INFO:tasks.rados.rados.0.smithi029.stdout:2335: finishing write tid 2 to smithi02979277-5 2024-04-12T11:24:40.668 INFO:tasks.rados.rados.0.smithi029.stdout:2335: finishing write tid 3 to smithi02979277-5 2024-04-12T11:24:40.668 INFO:tasks.rados.rados.0.smithi029.stdout:2335: finishing write tid 4 to smithi02979277-5 2024-04-12T11:24:40.668 INFO:tasks.rados.rados.0.smithi029.stdout:2335: oid 5 updating version 0 to 1081 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2335: oid 5 updating version 1081 to 1082 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2335: oid 5 version 1082 is already newer than 1080 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1082 (ObjNum 871 snap 307 seq_num 871) dirty exists 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2335: left oid 5 (ObjNum 871 snap 307 seq_num 871) 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2338: finishing write tid 1 to smithi02979277-6 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2338: finishing write tid 2 to smithi02979277-6 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2338: finishing write tid 3 to smithi02979277-6 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2338: finishing write tid 4 to smithi02979277-6 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2338: oid 6 updating version 0 to 601 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2338: oid 6 version 601 is already newer than 600 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2338: oid 6 updating version 601 to 602 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 602 (ObjNum 872 snap 307 seq_num 872) dirty exists 2024-04-12T11:24:40.669 INFO:tasks.rados.rados.0.smithi029.stdout:2338: left oid 6 (ObjNum 872 snap 307 seq_num 872) 2024-04-12T11:24:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:40 smithi029 ceph-mon[97787]: pgmap v648: 89 pgs: 89 active+clean; 176 MiB data, 687 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:24:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:40 smithi029 ceph-mon[97787]: osdmap e675: 8 total, 8 up, 8 in 2024-04-12T11:24:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:40 smithi029 ceph-mon[93650]: pgmap v648: 89 pgs: 89 active+clean; 176 MiB data, 687 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:24:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:40 smithi029 ceph-mon[93650]: osdmap e675: 8 total, 8 up, 8 in 2024-04-12T11:24:41.673 INFO:tasks.rados.rados.0.smithi029.stdout:2335: done (3 left) 2024-04-12T11:24:41.673 INFO:tasks.rados.rados.0.smithi029.stdout:2336: done (2 left) 2024-04-12T11:24:41.673 INFO:tasks.rados.rados.0.smithi029.stdout:2338: done (1 left) 2024-04-12T11:24:41.673 INFO:tasks.rados.rados.0.smithi029.stdout:2340: done (0 left) 2024-04-12T11:24:41.673 INFO:tasks.rados.rados.0.smithi029.stdout:2341: delete oid 31 current snap is 308 2024-04-12T11:24:41.674 INFO:tasks.rados.rados.0.smithi029.stdout:2341: done (0 left) 2024-04-12T11:24:41.674 INFO:tasks.rados.rados.0.smithi029.stdout:2342: write oid 40 current snap is 308 2024-04-12T11:24:41.674 INFO:tasks.rados.rados.0.smithi029.stdout:2342: seq_num 874 ranges {689318=631868,1877968=498292} 2024-04-12T11:24:41.681 INFO:tasks.rados.rados.0.smithi029.stdout:2342: writing smithi02979277-40 from 689318 to 1321186 tid 1 2024-04-12T11:24:41.688 INFO:tasks.rados.rados.0.smithi029.stdout:2342: writing smithi02979277-40 from 1877968 to 2376260 tid 2 2024-04-12T11:24:41.688 INFO:tasks.rados.rados.0.smithi029.stdout:2343: read oid 6 snap -1 2024-04-12T11:24:41.688 INFO:tasks.rados.rados.0.smithi029.stdout:2343: expect (ObjNum 872 snap 307 seq_num 872) 2024-04-12T11:24:41.688 INFO:tasks.rados.rados.0.smithi029.stdout:2344: write oid 34 current snap is 308 2024-04-12T11:24:41.688 INFO:tasks.rados.rados.0.smithi029.stdout:2344: seq_num 875 ranges {706125=423470,1652803=758303} 2024-04-12T11:24:41.694 INFO:tasks.rados.rados.0.smithi029.stdout:2344: writing smithi02979277-34 from 706125 to 1129595 tid 1 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2344: writing smithi02979277-34 from 1652803 to 2411106 tid 2 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2345: read oid 29 snap 294 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2345: expect (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2346: rollback oid 30 current snap is 308 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 304 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2342: finishing write tid 1 to smithi02979277-40 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2342: finishing write tid 2 to smithi02979277-40 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2342: finishing write tid 3 to smithi02979277-40 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2342: finishing write tid 4 to smithi02979277-40 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2342: oid 40 updating version 0 to 1374 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2342: oid 40 version 1374 is already newer than 1372 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2342: oid 40 version 1374 is already newer than 1373 2024-04-12T11:24:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 1374 (ObjNum 874 snap 308 seq_num 874) dirty exists 2024-04-12T11:24:41.704 INFO:tasks.rados.rados.0.smithi029.stdout:2342: left oid 40 (ObjNum 874 snap 308 seq_num 874) 2024-04-12T11:24:41.704 INFO:tasks.rados.rados.0.smithi029.stdout:2343: expect (ObjNum 872 snap 307 seq_num 872) 2024-04-12T11:24:41.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:41 smithi029 ceph-mon[93650]: osdmap e676: 8 total, 8 up, 8 in 2024-04-12T11:24:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:41 smithi029 ceph-mon[97787]: osdmap e676: 8 total, 8 up, 8 in 2024-04-12T11:24:41.923 INFO:tasks.rados.rados.0.smithi029.stdout:2344: finishing write tid 1 to smithi02979277-34 2024-04-12T11:24:41.923 INFO:tasks.rados.rados.0.smithi029.stdout:2346: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 0 (ObjNum 129 snap 0 seq_num 1800876032) dirty dne 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2344: finishing write tid 2 to smithi02979277-34 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2342: done (4 left) 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2343: done (3 left) 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2346: done (2 left) 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2347: delete oid 7 current snap is 308 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2344: finishing write tid 3 to smithi02979277-34 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2344: finishing write tid 4 to smithi02979277-34 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2344: oid 34 updating version 0 to 962 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2344: oid 34 version 962 is already newer than 961 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2344: oid 34 updating version 962 to 963 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 963 (ObjNum 875 snap 308 seq_num 875) dirty exists 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2344: left oid 34 (ObjNum 875 snap 308 seq_num 875) 2024-04-12T11:24:41.924 INFO:tasks.rados.rados.0.smithi029.stdout:2345: expect (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:24:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:41 smithi087 ceph-mon[89014]: osdmap e676: 8 total, 8 up, 8 in 2024-04-12T11:24:42.055 INFO:tasks.rados.rados.0.smithi029.stdout:2344: done (2 left) 2024-04-12T11:24:42.055 INFO:tasks.rados.rados.0.smithi029.stdout:2345: done (1 left) 2024-04-12T11:24:42.055 INFO:tasks.rados.rados.0.smithi029.stdout:2347: done (0 left) 2024-04-12T11:24:42.055 INFO:tasks.rados.rados.0.smithi029.stdout:2348: snap_create 2024-04-12T11:24:42.601 INFO:tasks.rados.rados.0.smithi029.stdout:2348: done (0 left) 2024-04-12T11:24:42.601 INFO:tasks.rados.rados.0.smithi029.stdout:2349: read oid 3 snap -1 2024-04-12T11:24:42.601 INFO:tasks.rados.rados.0.smithi029.stdout:2349: expect (ObjNum 779 snap 272 seq_num 779) 2024-04-12T11:24:42.601 INFO:tasks.rados.rados.0.smithi029.stdout:2350: snap_remove snap 288 2024-04-12T11:24:42.601 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:24:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:24:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:24:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:42 smithi087 ceph-mon[89014]: pgmap v651: 89 pgs: 89 active+clean; 176 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:24:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:42 smithi087 ceph-mon[89014]: osdmap e677: 8 total, 8 up, 8 in 2024-04-12T11:24:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:42 smithi087 ceph-mon[89014]: osdmap e678: 8 total, 8 up, 8 in 2024-04-12T11:24:43.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:42 smithi029 ceph-mon[97787]: pgmap v651: 89 pgs: 89 active+clean; 176 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:24:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:42 smithi029 ceph-mon[97787]: osdmap e677: 8 total, 8 up, 8 in 2024-04-12T11:24:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:42 smithi029 ceph-mon[97787]: osdmap e678: 8 total, 8 up, 8 in 2024-04-12T11:24:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:42 smithi029 ceph-mon[93650]: pgmap v651: 89 pgs: 89 active+clean; 176 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:24:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:42 smithi029 ceph-mon[93650]: osdmap e677: 8 total, 8 up, 8 in 2024-04-12T11:24:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:42 smithi029 ceph-mon[93650]: osdmap e678: 8 total, 8 up, 8 in 2024-04-12T11:24:43.602 INFO:tasks.rados.rados.0.smithi029.stdout:2350: done (1 left) 2024-04-12T11:24:43.602 INFO:tasks.rados.rados.0.smithi029.stdout:2351: read oid 10 snap 308 2024-04-12T11:24:43.602 INFO:tasks.rados.rados.0.smithi029.stdout:2351: expect (ObjNum 771 snap 270 seq_num 771) 2024-04-12T11:24:43.602 INFO:tasks.rados.rados.0.smithi029.stdout:2352: snap_remove snap 294 2024-04-12T11:24:44.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-mon[97787]: osdmap e679: 8 total, 8 up, 8 in 2024-04-12T11:24:44.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-mon[93650]: osdmap e679: 8 total, 8 up, 8 in 2024-04-12T11:24:44.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:44 smithi087 ceph-mon[89014]: osdmap e679: 8 total, 8 up, 8 in 2024-04-12T11:24:44.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:44.603 INFO:tasks.rados.rados.0.smithi029.stdout:2352: done (2 left) 2024-04-12T11:24:44.603 INFO:tasks.rados.rados.0.smithi029.stdout:2353: snap_remove snap 301 2024-04-12T11:24:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:45 smithi029 ceph-mon[93650]: pgmap v654: 89 pgs: 89 active+clean; 179 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-04-12T11:24:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:45 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:24:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:45 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:24:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:45 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:45 smithi029 ceph-mon[93650]: osdmap e680: 8 total, 8 up, 8 in 2024-04-12T11:24:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:45 smithi029 ceph-mon[97787]: pgmap v654: 89 pgs: 89 active+clean; 179 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-04-12T11:24:45.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:45 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:24:45.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:45 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:24:45.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:45 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:45.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:45 smithi029 ceph-mon[97787]: osdmap e680: 8 total, 8 up, 8 in 2024-04-12T11:24:45.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:24:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:24:44] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2024-04-12T11:24:45.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:45 smithi087 ceph-mon[89014]: pgmap v654: 89 pgs: 89 active+clean; 179 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-04-12T11:24:45.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:45 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:24:45.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:45 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:24:45.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:45 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:24:45.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:45 smithi087 ceph-mon[89014]: osdmap e680: 8 total, 8 up, 8 in 2024-04-12T11:24:45.605 INFO:tasks.rados.rados.0.smithi029.stdout:2353: done (2 left) 2024-04-12T11:24:45.605 INFO:tasks.rados.rados.0.smithi029.stdout:2354: snap_create 2024-04-12T11:24:45.605 INFO:tasks.rados.rados.0.smithi029.stdout:2349: expect (ObjNum 779 snap 272 seq_num 779) 2024-04-12T11:24:45.795 INFO:tasks.rados.rados.0.smithi029.stdout:2351: expect (ObjNum 771 snap 270 seq_num 771) 2024-04-12T11:24:46.610 INFO:tasks.rados.rados.0.smithi029.stdout:2349: done (2 left) 2024-04-12T11:24:46.610 INFO:tasks.rados.rados.0.smithi029.stdout:2351: done (1 left) 2024-04-12T11:24:46.610 INFO:tasks.rados.rados.0.smithi029.stdout:2354: done (0 left) 2024-04-12T11:24:46.610 INFO:tasks.rados.rados.0.smithi029.stdout:2355: write oid 41 current snap is 310 2024-04-12T11:24:46.611 INFO:tasks.rados.rados.0.smithi029.stdout:2355: seq_num 877 ranges {739739=573968,2053414=686325} 2024-04-12T11:24:46.617 INFO:tasks.rados.rados.0.smithi029.stdout:2355: writing smithi02979277-41 from 739739 to 1313707 tid 1 2024-04-12T11:24:46.625 INFO:tasks.rados.rados.0.smithi029.stdout:2355: writing smithi02979277-41 from 2053414 to 2739739 tid 2 2024-04-12T11:24:46.625 INFO:tasks.rados.rados.0.smithi029.stdout:2356: write oid 32 current snap is 310 2024-04-12T11:24:46.625 INFO:tasks.rados.rados.0.smithi029.stdout:2355: finishing write tid 1 to smithi02979277-41 2024-04-12T11:24:46.625 INFO:tasks.rados.rados.0.smithi029.stdout:2356: seq_num 878 ranges {756546=649217,2111896=505630} 2024-04-12T11:24:46.633 INFO:tasks.rados.rados.0.smithi029.stdout:2356: writing smithi02979277-32 from 756546 to 1405763 tid 1 2024-04-12T11:24:46.640 INFO:tasks.rados.rados.0.smithi029.stdout:2356: writing smithi02979277-32 from 2111896 to 2617526 tid 2 2024-04-12T11:24:46.640 INFO:tasks.rados.rados.0.smithi029.stdout:2357: write oid 15 current snap is 310 2024-04-12T11:24:46.640 INFO:tasks.rados.rados.0.smithi029.stdout:2355: finishing write tid 2 to smithi02979277-41 2024-04-12T11:24:46.640 INFO:tasks.rados.rados.0.smithi029.stdout:2355: finishing write tid 3 to smithi02979277-41 2024-04-12T11:24:46.640 INFO:tasks.rados.rados.0.smithi029.stdout:2355: finishing write tid 4 to smithi02979277-41 2024-04-12T11:24:46.640 INFO:tasks.rados.rados.0.smithi029.stdout:2355: oid 41 updating version 0 to 1126 2024-04-12T11:24:46.640 INFO:tasks.rados.rados.0.smithi029.stdout:2355: oid 41 updating version 1126 to 1127 2024-04-12T11:24:46.640 INFO:tasks.rados.rados.0.smithi029.stdout:2355: oid 41 updating version 1127 to 1128 2024-04-12T11:24:46.641 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1128 (ObjNum 877 snap 310 seq_num 877) dirty exists 2024-04-12T11:24:46.641 INFO:tasks.rados.rados.0.smithi029.stdout:2355: left oid 41 (ObjNum 877 snap 310 seq_num 877) 2024-04-12T11:24:46.641 INFO:tasks.rados.rados.0.smithi029.stdout:2356: finishing write tid 1 to smithi02979277-32 2024-04-12T11:24:46.641 INFO:tasks.rados.rados.0.smithi029.stdout:2357: seq_num 879 ranges {773353=724466,2170378=602975} 2024-04-12T11:24:46.648 INFO:tasks.rados.rados.0.smithi029.stdout:2357: writing smithi02979277-15 from 773353 to 1497819 tid 1 2024-04-12T11:24:46.655 INFO:tasks.rados.rados.0.smithi029.stdout:2357: writing smithi02979277-15 from 2170378 to 2773353 tid 2 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2355: done (2 left) 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2358: read oid 5 snap -1 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2358: expect (ObjNum 871 snap 307 seq_num 871) 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2359: snap_create 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2356: finishing write tid 2 to smithi02979277-32 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2356: finishing write tid 3 to smithi02979277-32 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2356: finishing write tid 4 to smithi02979277-32 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2356: oid 32 updating version 0 to 981 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2356: oid 32 version 981 is already newer than 979 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2356: oid 32 version 981 is already newer than 980 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 981 (ObjNum 878 snap 310 seq_num 878) dirty exists 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2356: left oid 32 (ObjNum 878 snap 310 seq_num 878) 2024-04-12T11:24:46.656 INFO:tasks.rados.rados.0.smithi029.stdout:2357: finishing write tid 1 to smithi02979277-15 2024-04-12T11:24:46.658 INFO:tasks.rados.rados.0.smithi029.stdout:2357: finishing write tid 2 to smithi02979277-15 2024-04-12T11:24:46.658 INFO:tasks.rados.rados.0.smithi029.stdout:2357: finishing write tid 3 to smithi02979277-15 2024-04-12T11:24:46.658 INFO:tasks.rados.rados.0.smithi029.stdout:2357: finishing write tid 4 to smithi02979277-15 2024-04-12T11:24:46.659 INFO:tasks.rados.rados.0.smithi029.stdout:2357: oid 15 updating version 0 to 770 2024-04-12T11:24:46.659 INFO:tasks.rados.rados.0.smithi029.stdout:2357: oid 15 updating version 770 to 772 2024-04-12T11:24:46.659 INFO:tasks.rados.rados.0.smithi029.stdout:2357: oid 15 version 772 is already newer than 771 2024-04-12T11:24:46.659 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 772 (ObjNum 879 snap 310 seq_num 879) dirty exists 2024-04-12T11:24:46.659 INFO:tasks.rados.rados.0.smithi029.stdout:2357: left oid 15 (ObjNum 879 snap 310 seq_num 879) 2024-04-12T11:24:46.665 INFO:tasks.rados.rados.0.smithi029.stdout:2358: expect (ObjNum 871 snap 307 seq_num 871) 2024-04-12T11:24:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:46 smithi087 ceph-mon[89014]: pgmap v657: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 182 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-04-12T11:24:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:46 smithi087 ceph-mon[89014]: osdmap e681: 8 total, 8 up, 8 in 2024-04-12T11:24:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:46 smithi029 ceph-mon[93650]: pgmap v657: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 182 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-04-12T11:24:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:46 smithi029 ceph-mon[93650]: osdmap e681: 8 total, 8 up, 8 in 2024-04-12T11:24:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:46 smithi029 ceph-mon[97787]: pgmap v657: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 182 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-04-12T11:24:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:46 smithi029 ceph-mon[97787]: osdmap e681: 8 total, 8 up, 8 in 2024-04-12T11:24:47.604 INFO:tasks.rados.rados.0.smithi029.stdout:2356: done (3 left) 2024-04-12T11:24:47.604 INFO:tasks.rados.rados.0.smithi029.stdout:2357: done (2 left) 2024-04-12T11:24:47.604 INFO:tasks.rados.rados.0.smithi029.stdout:2358: done (1 left) 2024-04-12T11:24:47.604 INFO:tasks.rados.rados.0.smithi029.stdout:2359: done (0 left) 2024-04-12T11:24:47.604 INFO:tasks.rados.rados.0.smithi029.stdout:2360: write oid 34 current snap is 311 2024-04-12T11:24:47.604 INFO:tasks.rados.rados.0.smithi029.stdout:2360: seq_num 880 ranges {790160=799715,2112507=509299} 2024-04-12T11:24:47.613 INFO:tasks.rados.rados.0.smithi029.stdout:2360: writing smithi02979277-34 from 790160 to 1589875 tid 1 2024-04-12T11:24:47.618 INFO:tasks.rados.rados.0.smithi029.stdout:2360: writing smithi02979277-34 from 2112507 to 2621806 tid 2 2024-04-12T11:24:47.619 INFO:tasks.rados.rados.0.smithi029.stdout:2361: write oid 13 current snap is 311 2024-04-12T11:24:47.619 INFO:tasks.rados.rados.0.smithi029.stdout:2360: finishing write tid 1 to smithi02979277-34 2024-04-12T11:24:47.619 INFO:tasks.rados.rados.0.smithi029.stdout:2361: seq_num 881 ranges {406967=474964,1370989=769310,2677186=129781} 2024-04-12T11:24:47.623 INFO:tasks.rados.rados.0.smithi029.stdout:2361: writing smithi02979277-13 from 406967 to 881931 tid 1 2024-04-12T11:24:47.630 INFO:tasks.rados.rados.0.smithi029.stdout:2361: writing smithi02979277-13 from 1370989 to 2140299 tid 2 2024-04-12T11:24:47.634 INFO:tasks.rados.rados.0.smithi029.stdout:2361: writing smithi02979277-13 from 2677186 to 2806967 tid 3 2024-04-12T11:24:47.634 INFO:tasks.rados.rados.0.smithi029.stdout:2362: write oid 36 current snap is 311 2024-04-12T11:24:47.634 INFO:tasks.rados.rados.0.smithi029.stdout:2362: seq_num 882 ranges {423774=666566,1545824=512968,2704609=119165} 2024-04-12T11:24:47.639 INFO:tasks.rados.rados.0.smithi029.stdout:2362: writing smithi02979277-36 from 423774 to 1090340 tid 1 2024-04-12T11:24:47.645 INFO:tasks.rados.rados.0.smithi029.stdout:2362: writing smithi02979277-36 from 1545824 to 2058792 tid 2 2024-04-12T11:24:47.650 INFO:tasks.rados.rados.0.smithi029.stdout:2362: writing smithi02979277-36 from 2704609 to 2823774 tid 3 2024-04-12T11:24:47.650 INFO:tasks.rados.rados.0.smithi029.stdout:2363: read oid 15 snap -1 2024-04-12T11:24:47.650 INFO:tasks.rados.rados.0.smithi029.stdout:2363: expect (ObjNum 879 snap 310 seq_num 879) 2024-04-12T11:24:47.650 INFO:tasks.rados.rados.0.smithi029.stdout:2364: write oid 17 current snap is 311 2024-04-12T11:24:47.650 INFO:tasks.rados.rados.0.smithi029.stdout:2360: finishing write tid 2 to smithi02979277-34 2024-04-12T11:24:47.650 INFO:tasks.rados.rados.0.smithi029.stdout:2364: seq_num 883 ranges {440581=741815,1604306=656626,2732032=108549} 2024-04-12T11:24:47.656 INFO:tasks.rados.rados.0.smithi029.stdout:2364: writing smithi02979277-17 from 440581 to 1182396 tid 1 2024-04-12T11:24:47.662 INFO:tasks.rados.rados.0.smithi029.stdout:2364: writing smithi02979277-17 from 1604306 to 2260932 tid 2 2024-04-12T11:24:47.666 INFO:tasks.rados.rados.0.smithi029.stdout:2364: writing smithi02979277-17 from 2732032 to 2840581 tid 3 2024-04-12T11:24:47.666 INFO:tasks.rados.rados.0.smithi029.stdout:2365: write oid 47 current snap is 311 2024-04-12T11:24:47.666 INFO:tasks.rados.rados.0.smithi029.stdout:2360: finishing write tid 3 to smithi02979277-34 2024-04-12T11:24:47.666 INFO:tasks.rados.rados.0.smithi029.stdout:2360: finishing write tid 4 to smithi02979277-34 2024-04-12T11:24:47.666 INFO:tasks.rados.rados.0.smithi029.stdout:2360: oid 34 updating version 0 to 985 2024-04-12T11:24:47.666 INFO:tasks.rados.rados.0.smithi029.stdout:2360: oid 34 version 985 is already newer than 984 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2360: oid 34 version 985 is already newer than 983 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 985 (ObjNum 880 snap 311 seq_num 880) dirty exists 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2360: left oid 34 (ObjNum 880 snap 311 seq_num 880) 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2361: finishing write tid 1 to smithi02979277-13 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2361: finishing write tid 2 to smithi02979277-13 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2361: finishing write tid 3 to smithi02979277-13 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2361: finishing write tid 4 to smithi02979277-13 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2361: finishing write tid 5 to smithi02979277-13 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2361: oid 13 updating version 0 to 1149 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2361: oid 13 version 1149 is already newer than 1146 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2361: oid 13 version 1149 is already newer than 1148 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2361: oid 13 version 1149 is already newer than 1147 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1149 (ObjNum 881 snap 311 seq_num 881) dirty exists 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2361: left oid 13 (ObjNum 881 snap 311 seq_num 881) 2024-04-12T11:24:47.667 INFO:tasks.rados.rados.0.smithi029.stdout:2362: finishing write tid 1 to smithi02979277-36 2024-04-12T11:24:47.668 INFO:tasks.rados.rados.0.smithi029.stdout:2365: seq_num 884 ranges {457388=417064,1546435=516637,2643102=214286} 2024-04-12T11:24:47.670 INFO:tasks.rados.rados.0.smithi029.stdout:2365: writing smithi02979277-47 from 457388 to 874452 tid 1 2024-04-12T11:24:47.677 INFO:tasks.rados.rados.0.smithi029.stdout:2365: writing smithi02979277-47 from 1546435 to 2063072 tid 2 2024-04-12T11:24:47.681 INFO:tasks.rados.rados.0.smithi029.stdout:2365: writing smithi02979277-47 from 2643102 to 2857388 tid 3 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2360: done (5 left) 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2361: done (4 left) 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2366: snap_create 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2362: finishing write tid 2 to smithi02979277-36 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2362: finishing write tid 3 to smithi02979277-36 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2362: finishing write tid 4 to smithi02979277-36 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2362: finishing write tid 5 to smithi02979277-36 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2362: oid 36 updating version 0 to 615 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2362: oid 36 updating version 615 to 617 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2362: oid 36 version 617 is already newer than 614 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2362: oid 36 version 617 is already newer than 616 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 617 (ObjNum 882 snap 311 seq_num 882) dirty exists 2024-04-12T11:24:47.682 INFO:tasks.rados.rados.0.smithi029.stdout:2362: left oid 36 (ObjNum 882 snap 311 seq_num 882) 2024-04-12T11:24:47.683 INFO:tasks.rados.rados.0.smithi029.stdout:2363: expect (ObjNum 879 snap 310 seq_num 879) 2024-04-12T11:24:47.798 INFO:tasks.rados.rados.0.smithi029.stdout:2364: finishing write tid 1 to smithi02979277-17 2024-04-12T11:24:47.798 INFO:tasks.rados.rados.0.smithi029.stdout:2364: finishing write tid 2 to smithi02979277-17 2024-04-12T11:24:47.798 INFO:tasks.rados.rados.0.smithi029.stdout:2364: finishing write tid 3 to smithi02979277-17 2024-04-12T11:24:47.798 INFO:tasks.rados.rados.0.smithi029.stdout:2364: finishing write tid 4 to smithi02979277-17 2024-04-12T11:24:47.798 INFO:tasks.rados.rados.0.smithi029.stdout:2364: finishing write tid 5 to smithi02979277-17 2024-04-12T11:24:47.798 INFO:tasks.rados.rados.0.smithi029.stdout:2364: oid 17 updating version 0 to 1151 2024-04-12T11:24:47.798 INFO:tasks.rados.rados.0.smithi029.stdout:2364: oid 17 version 1151 is already newer than 1150 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2364: oid 17 updating version 1151 to 1152 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2364: oid 17 updating version 1152 to 1153 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1153 (ObjNum 883 snap 311 seq_num 883) dirty exists 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2364: left oid 17 (ObjNum 883 snap 311 seq_num 883) 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2365: finishing write tid 1 to smithi02979277-47 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2365: finishing write tid 2 to smithi02979277-47 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2365: finishing write tid 3 to smithi02979277-47 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2365: finishing write tid 4 to smithi02979277-47 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2365: finishing write tid 5 to smithi02979277-47 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2365: oid 47 updating version 0 to 1144 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2365: oid 47 version 1144 is already newer than 1142 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2365: oid 47 version 1144 is already newer than 1143 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2365: oid 47 version 1144 is already newer than 1141 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1144 (ObjNum 884 snap 311 seq_num 884) dirty exists 2024-04-12T11:24:47.799 INFO:tasks.rados.rados.0.smithi029.stdout:2365: left oid 47 (ObjNum 884 snap 311 seq_num 884) 2024-04-12T11:24:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:47 smithi087 ceph-mon[89014]: osdmap e682: 8 total, 8 up, 8 in 2024-04-12T11:24:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:47 smithi087 ceph-mon[89014]: osdmap e683: 8 total, 8 up, 8 in 2024-04-12T11:24:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:47 smithi029 ceph-mon[93650]: osdmap e682: 8 total, 8 up, 8 in 2024-04-12T11:24:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:47 smithi029 ceph-mon[93650]: osdmap e683: 8 total, 8 up, 8 in 2024-04-12T11:24:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:47 smithi029 ceph-mon[97787]: osdmap e682: 8 total, 8 up, 8 in 2024-04-12T11:24:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:47 smithi029 ceph-mon[97787]: osdmap e683: 8 total, 8 up, 8 in 2024-04-12T11:24:48.620 INFO:tasks.rados.rados.0.smithi029.stdout:2362: done (4 left) 2024-04-12T11:24:48.620 INFO:tasks.rados.rados.0.smithi029.stdout:2363: done (3 left) 2024-04-12T11:24:48.620 INFO:tasks.rados.rados.0.smithi029.stdout:2364: done (2 left) 2024-04-12T11:24:48.620 INFO:tasks.rados.rados.0.smithi029.stdout:2365: done (1 left) 2024-04-12T11:24:48.621 INFO:tasks.rados.rados.0.smithi029.stdout:2366: done (0 left) 2024-04-12T11:24:48.621 INFO:tasks.rados.rados.0.smithi029.stdout:2367: read oid 25 snap 304 2024-04-12T11:24:48.621 INFO:tasks.rados.rados.0.smithi029.stdout:2367: expect (ObjNum 854 snap 294 seq_num 854) 2024-04-12T11:24:48.621 INFO:tasks.rados.rados.0.smithi029.stdout:2368: write oid 39 current snap is 312 2024-04-12T11:24:48.621 INFO:tasks.rados.rados.0.smithi029.stdout:2368: seq_num 885 ranges {474195=492313,1604917=660295,2670525=203670} 2024-04-12T11:24:48.625 INFO:tasks.rados.rados.0.smithi029.stdout:2368: writing smithi02979277-39 from 474195 to 966508 tid 1 2024-04-12T11:24:48.633 INFO:tasks.rados.rados.0.smithi029.stdout:2368: writing smithi02979277-39 from 1604917 to 2265212 tid 2 2024-04-12T11:24:48.637 INFO:tasks.rados.rados.0.smithi029.stdout:2368: writing smithi02979277-39 from 2670525 to 2874195 tid 3 2024-04-12T11:24:48.637 INFO:tasks.rados.rados.0.smithi029.stdout:2369: read oid 42 snap -1 2024-04-12T11:24:48.637 INFO:tasks.rados.rados.0.smithi029.stdout:2369: expect deleted 2024-04-12T11:24:48.637 INFO:tasks.rados.rados.0.smithi029.stdout:2370: write oid 33 current snap is 312 2024-04-12T11:24:48.637 INFO:tasks.rados.rados.0.smithi029.stdout:2370: seq_num 886 ranges {491002=567562,1663399=520306,2814301=76701} 2024-04-12T11:24:48.642 INFO:tasks.rados.rados.0.smithi029.stdout:2370: writing smithi02979277-33 from 491002 to 1058564 tid 1 2024-04-12T11:24:48.649 INFO:tasks.rados.rados.0.smithi029.stdout:2370: writing smithi02979277-33 from 1663399 to 2183705 tid 2 2024-04-12T11:24:48.653 INFO:tasks.rados.rados.0.smithi029.stdout:2370: writing smithi02979277-33 from 2814301 to 2891002 tid 3 2024-04-12T11:24:48.653 INFO:tasks.rados.rados.0.smithi029.stdout:2371: write oid 23 current snap is 312 2024-04-12T11:24:48.653 INFO:tasks.rados.rados.0.smithi029.stdout:2371: seq_num 887 ranges {507809=642811,1721881=663964} 2024-04-12T11:24:48.659 INFO:tasks.rados.rados.0.smithi029.stdout:2371: writing smithi02979277-23 from 507809 to 1150620 tid 1 2024-04-12T11:24:48.666 INFO:tasks.rados.rados.0.smithi029.stdout:2371: writing smithi02979277-23 from 1721881 to 2385845 tid 2 2024-04-12T11:24:48.666 INFO:tasks.rados.rados.0.smithi029.stdout:2372: read oid 8 snap -1 2024-04-12T11:24:48.666 INFO:tasks.rados.rados.0.smithi029.stdout:2372: expect (ObjNum 868 snap 307 seq_num 868) 2024-04-12T11:24:48.666 INFO:tasks.rados.rados.0.smithi029.stdout:2373: delete oid 21 current snap is 312 2024-04-12T11:24:48.666 INFO:tasks.rados.rados.0.smithi029.stdout:2367: expect (ObjNum 854 snap 294 seq_num 854) 2024-04-12T11:24:48.800 INFO:tasks.rados.rados.0.smithi029.stdout:2368: finishing write tid 1 to smithi02979277-39 2024-04-12T11:24:48.800 INFO:tasks.rados.rados.0.smithi029.stdout:2368: finishing write tid 2 to smithi02979277-39 2024-04-12T11:24:48.800 INFO:tasks.rados.rados.0.smithi029.stdout:2368: finishing write tid 3 to smithi02979277-39 2024-04-12T11:24:48.800 INFO:tasks.rados.rados.0.smithi029.stdout:2368: finishing write tid 4 to smithi02979277-39 2024-04-12T11:24:48.800 INFO:tasks.rados.rados.0.smithi029.stdout:2368: finishing write tid 5 to smithi02979277-39 2024-04-12T11:24:48.800 INFO:tasks.rados.rados.0.smithi029.stdout:2368: oid 39 updating version 0 to 292 2024-04-12T11:24:48.800 INFO:tasks.rados.rados.0.smithi029.stdout:2368: oid 39 updating version 292 to 293 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2368: oid 39 updating version 293 to 295 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2368: oid 39 version 295 is already newer than 294 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 295 (ObjNum 885 snap 312 seq_num 885) dirty exists 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2368: left oid 39 (ObjNum 885 snap 312 seq_num 885) 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2370: finishing write tid 1 to smithi02979277-33 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2370: finishing write tid 2 to smithi02979277-33 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2370: finishing write tid 3 to smithi02979277-33 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2367: done (6 left) 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2368: done (5 left) 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2369: done (4 left) 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2373: done (3 left) 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2374: rollback oid 20 current snap is 312 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 20 to 293 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2370: finishing write tid 4 to smithi02979277-33 2024-04-12T11:24:48.801 INFO:tasks.rados.rados.0.smithi029.stdout:2370: finishing write tid 5 to smithi02979277-33 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2370: oid 33 updating version 0 to 920 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2370: oid 33 version 920 is already newer than 919 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2370: oid 33 updating version 920 to 922 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2370: oid 33 version 922 is already newer than 921 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 922 (ObjNum 886 snap 312 seq_num 886) dirty exists 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2370: left oid 33 (ObjNum 886 snap 312 seq_num 886) 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2371: finishing write tid 1 to smithi02979277-23 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2371: finishing write tid 2 to smithi02979277-23 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2371: finishing write tid 3 to smithi02979277-23 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2371: finishing write tid 4 to smithi02979277-23 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2371: oid 23 updating version 0 to 925 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2371: oid 23 updating version 925 to 926 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:2371: oid 23 version 926 is already newer than 924 2024-04-12T11:24:48.802 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 926 (ObjNum 887 snap 312 seq_num 887) dirty exists 2024-04-12T11:24:48.803 INFO:tasks.rados.rados.0.smithi029.stdout:2371: left oid 23 (ObjNum 887 snap 312 seq_num 887) 2024-04-12T11:24:48.803 INFO:tasks.rados.rados.0.smithi029.stdout:2372: expect (ObjNum 868 snap 307 seq_num 868) 2024-04-12T11:24:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:48 smithi087 ceph-mon[89014]: pgmap v660: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 183 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 18 op/s 2024-04-12T11:24:49.011 INFO:tasks.rados.rados.0.smithi029.stdout:2374: finishing rollback tid 0 to smithi02979277-20 2024-04-12T11:24:49.011 INFO:tasks.rados.rados.0.smithi029.stdout:2374: finishing rollback tid 1 to smithi02979277-20 2024-04-12T11:24:49.011 INFO:tasks.rados.rados.0.smithi029.stdout:2374: finishing rollback tid 2 to smithi02979277-20 2024-04-12T11:24:49.011 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 717 (ObjNum 801 snap 282 seq_num 801) dirty exists 2024-04-12T11:24:49.011 INFO:tasks.rados.rados.0.smithi029.stdout:2370: done (3 left) 2024-04-12T11:24:49.011 INFO:tasks.rados.rados.0.smithi029.stdout:2371: done (2 left) 2024-04-12T11:24:49.012 INFO:tasks.rados.rados.0.smithi029.stdout:2372: done (1 left) 2024-04-12T11:24:49.012 INFO:tasks.rados.rados.0.smithi029.stdout:2374: done (0 left) 2024-04-12T11:24:49.012 INFO:tasks.rados.rados.0.smithi029.stdout:2375: rollback oid 33 current snap is 312 2024-04-12T11:24:49.012 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 308 2024-04-12T11:24:49.012 INFO:tasks.rados.rados.0.smithi029.stdout:2376: write oid 10 current snap is 312 2024-04-12T11:24:49.012 INFO:tasks.rados.rados.0.smithi029.stdout:2376: seq_num 889 ranges {541423=793309,2122492=667633} 2024-04-12T11:24:49.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:48 smithi029 ceph-mon[93650]: pgmap v660: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 183 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 18 op/s 2024-04-12T11:24:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:48 smithi029 ceph-mon[97787]: pgmap v660: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 183 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 18 op/s 2024-04-12T11:24:49.018 INFO:tasks.rados.rados.0.smithi029.stdout:2376: writing smithi02979277-10 from 541423 to 1334732 tid 1 2024-04-12T11:24:49.026 INFO:tasks.rados.rados.0.smithi029.stdout:2376: writing smithi02979277-10 from 2122492 to 2790125 tid 2 2024-04-12T11:24:49.026 INFO:tasks.rados.rados.0.smithi029.stdout:2377: delete oid 16 current snap is 312 2024-04-12T11:24:49.026 INFO:tasks.rados.rados.0.smithi029.stdout:2375: finishing rollback tid 0 to smithi02979277-33 2024-04-12T11:24:49.027 INFO:tasks.rados.rados.0.smithi029.stdout:2375: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:24:49.027 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 928 (ObjNum 33 snap 0 seq_num 1801197120) dirty dne 2024-04-12T11:24:49.027 INFO:tasks.rados.rados.0.smithi029.stdout:2376: finishing write tid 1 to smithi02979277-10 2024-04-12T11:24:49.028 INFO:tasks.rados.rados.0.smithi029.stdout:2375: done (2 left) 2024-04-12T11:24:49.029 INFO:tasks.rados.rados.0.smithi029.stdout:2377: done (1 left) 2024-04-12T11:24:49.029 INFO:tasks.rados.rados.0.smithi029.stdout:2378: write oid 19 current snap is 312 2024-04-12T11:24:49.029 INFO:tasks.rados.rados.0.smithi029.stdout:2378: seq_num 891 ranges {575037=660160,1955809=671302} 2024-04-12T11:24:49.035 INFO:tasks.rados.rados.0.smithi029.stdout:2378: writing smithi02979277-19 from 575037 to 1235197 tid 1 2024-04-12T11:24:49.043 INFO:tasks.rados.rados.0.smithi029.stdout:2378: writing smithi02979277-19 from 1955809 to 2627111 tid 2 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2379: read oid 41 snap -1 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2379: expect (ObjNum 877 snap 310 seq_num 877) 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2380: write oid 22 current snap is 312 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2376: finishing write tid 2 to smithi02979277-10 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2376: finishing write tid 3 to smithi02979277-10 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2376: finishing write tid 4 to smithi02979277-10 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2376: oid 10 updating version 0 to 970 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2376: oid 10 version 970 is already newer than 969 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2376: oid 10 updating version 970 to 971 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 971 (ObjNum 889 snap 312 seq_num 889) dirty exists 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2376: left oid 10 (ObjNum 889 snap 312 seq_num 889) 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2378: finishing write tid 1 to smithi02979277-19 2024-04-12T11:24:49.044 INFO:tasks.rados.rados.0.smithi029.stdout:2380: seq_num 892 ranges {591844=735409,1897938=531313,2862486=129358} 2024-04-12T11:24:49.050 INFO:tasks.rados.rados.0.smithi029.stdout:2380: writing smithi02979277-22 from 591844 to 1327253 tid 1 2024-04-12T11:24:49.056 INFO:tasks.rados.rados.0.smithi029.stdout:2380: writing smithi02979277-22 from 1897938 to 2429251 tid 2 2024-04-12T11:24:49.060 INFO:tasks.rados.rados.0.smithi029.stdout:2380: writing smithi02979277-22 from 2862486 to 2991844 tid 3 2024-04-12T11:24:49.060 INFO:tasks.rados.rados.0.smithi029.stdout:2376: done (3 left) 2024-04-12T11:24:49.060 INFO:tasks.rados.rados.0.smithi029.stdout:2381: snap_create 2024-04-12T11:24:49.060 INFO:tasks.rados.rados.0.smithi029.stdout:2378: finishing write tid 2 to smithi02979277-19 2024-04-12T11:24:49.060 INFO:tasks.rados.rados.0.smithi029.stdout:2378: finishing write tid 3 to smithi02979277-19 2024-04-12T11:24:49.060 INFO:tasks.rados.rados.0.smithi029.stdout:2378: finishing write tid 4 to smithi02979277-19 2024-04-12T11:24:49.060 INFO:tasks.rados.rados.0.smithi029.stdout:2378: oid 19 updating version 0 to 1099 2024-04-12T11:24:49.060 INFO:tasks.rados.rados.0.smithi029.stdout:2378: oid 19 updating version 1099 to 1100 2024-04-12T11:24:49.060 INFO:tasks.rados.rados.0.smithi029.stdout:2378: oid 19 version 1100 is already newer than 1098 2024-04-12T11:24:49.061 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1100 (ObjNum 891 snap 312 seq_num 891) dirty exists 2024-04-12T11:24:49.061 INFO:tasks.rados.rados.0.smithi029.stdout:2378: left oid 19 (ObjNum 891 snap 312 seq_num 891) 2024-04-12T11:24:49.061 INFO:tasks.rados.rados.0.smithi029.stdout:2379: expect (ObjNum 877 snap 310 seq_num 877) 2024-04-12T11:24:49.151 INFO:tasks.rados.rados.0.smithi029.stdout:2380: finishing write tid 1 to smithi02979277-22 2024-04-12T11:24:49.151 INFO:tasks.rados.rados.0.smithi029.stdout:2380: finishing write tid 2 to smithi02979277-22 2024-04-12T11:24:49.151 INFO:tasks.rados.rados.0.smithi029.stdout:2380: finishing write tid 3 to smithi02979277-22 2024-04-12T11:24:49.151 INFO:tasks.rados.rados.0.smithi029.stdout:2380: finishing write tid 4 to smithi02979277-22 2024-04-12T11:24:49.151 INFO:tasks.rados.rados.0.smithi029.stdout:2380: finishing write tid 5 to smithi02979277-22 2024-04-12T11:24:49.152 INFO:tasks.rados.rados.0.smithi029.stdout:2380: oid 22 updating version 0 to 930 2024-04-12T11:24:49.152 INFO:tasks.rados.rados.0.smithi029.stdout:2380: oid 22 updating version 930 to 933 2024-04-12T11:24:49.152 INFO:tasks.rados.rados.0.smithi029.stdout:2380: oid 22 version 933 is already newer than 931 2024-04-12T11:24:49.152 INFO:tasks.rados.rados.0.smithi029.stdout:2380: oid 22 version 933 is already newer than 932 2024-04-12T11:24:49.152 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 933 (ObjNum 892 snap 312 seq_num 892) dirty exists 2024-04-12T11:24:49.152 INFO:tasks.rados.rados.0.smithi029.stdout:2380: left oid 22 (ObjNum 892 snap 312 seq_num 892) 2024-04-12T11:24:49.628 INFO:tasks.rados.rados.0.smithi029.stdout:2378: done (3 left) 2024-04-12T11:24:49.628 INFO:tasks.rados.rados.0.smithi029.stdout:2379: done (2 left) 2024-04-12T11:24:49.628 INFO:tasks.rados.rados.0.smithi029.stdout:2380: done (1 left) 2024-04-12T11:24:49.628 INFO:tasks.rados.rados.0.smithi029.stdout:2381: done (0 left) 2024-04-12T11:24:49.628 INFO:tasks.rados.rados.0.smithi029.stdout:2382: write oid 9 current snap is 313 2024-04-12T11:24:49.628 INFO:tasks.rados.rados.0.smithi029.stdout:2382: seq_num 893 ranges {608651=410658,1556420=674971,2773556=235095} 2024-04-12T11:24:49.633 INFO:tasks.rados.rados.0.smithi029.stdout:2382: writing smithi02979277-9 from 608651 to 1019309 tid 1 2024-04-12T11:24:49.640 INFO:tasks.rados.rados.0.smithi029.stdout:2382: writing smithi02979277-9 from 1556420 to 2231391 tid 2 2024-04-12T11:24:49.644 INFO:tasks.rados.rados.0.smithi029.stdout:2382: writing smithi02979277-9 from 2773556 to 3008651 tid 3 2024-04-12T11:24:49.645 INFO:tasks.rados.rados.0.smithi029.stdout:2383: delete oid 25 current snap is 313 2024-04-12T11:24:49.645 INFO:tasks.rados.rados.0.smithi029.stdout:2382: finishing write tid 1 to smithi02979277-9 2024-04-12T11:24:49.645 INFO:tasks.rados.rados.0.smithi029.stdout:2382: finishing write tid 2 to smithi02979277-9 2024-04-12T11:24:49.647 INFO:tasks.rados.rados.0.smithi029.stdout:2382: finishing write tid 3 to smithi02979277-9 2024-04-12T11:24:49.647 INFO:tasks.rados.rados.0.smithi029.stdout:2382: finishing write tid 4 to smithi02979277-9 2024-04-12T11:24:49.648 INFO:tasks.rados.rados.0.smithi029.stdout:2382: finishing write tid 5 to smithi02979277-9 2024-04-12T11:24:49.648 INFO:tasks.rados.rados.0.smithi029.stdout:2382: oid 9 updating version 0 to 878 2024-04-12T11:24:49.648 INFO:tasks.rados.rados.0.smithi029.stdout:2382: oid 9 version 878 is already newer than 875 2024-04-12T11:24:49.648 INFO:tasks.rados.rados.0.smithi029.stdout:2382: oid 9 version 878 is already newer than 876 2024-04-12T11:24:49.648 INFO:tasks.rados.rados.0.smithi029.stdout:2382: oid 9 version 878 is already newer than 877 2024-04-12T11:24:49.648 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 878 (ObjNum 893 snap 313 seq_num 893) dirty exists 2024-04-12T11:24:49.648 INFO:tasks.rados.rados.0.smithi029.stdout:2382: left oid 9 (ObjNum 893 snap 313 seq_num 893) 2024-04-12T11:24:49.650 INFO:tasks.rados.rados.0.smithi029.stdout:2382: done (1 left) 2024-04-12T11:24:49.650 INFO:tasks.rados.rados.0.smithi029.stdout:2383: done (0 left) 2024-04-12T11:24:49.650 INFO:tasks.rados.rados.0.smithi029.stdout:2384: read oid 14 snap -1 2024-04-12T11:24:49.650 INFO:tasks.rados.rados.0.smithi029.stdout:2384: expect deleted 2024-04-12T11:24:49.650 INFO:tasks.rados.rados.0.smithi029.stdout:2385: write oid 2 current snap is 313 2024-04-12T11:24:49.650 INFO:tasks.rados.rados.0.smithi029.stdout:2385: seq_num 895 ranges {642265=561156,1673384=678640,2828402=213863} 2024-04-12T11:24:49.656 INFO:tasks.rados.rados.0.smithi029.stdout:2385: writing smithi02979277-2 from 642265 to 1203421 tid 1 2024-04-12T11:24:49.662 INFO:tasks.rados.rados.0.smithi029.stdout:2385: writing smithi02979277-2 from 1673384 to 2352024 tid 2 2024-04-12T11:24:49.667 INFO:tasks.rados.rados.0.smithi029.stdout:2385: writing smithi02979277-2 from 2828402 to 3042265 tid 3 2024-04-12T11:24:49.667 INFO:tasks.rados.rados.0.smithi029.stdout:2386: write oid 19 current snap is 313 2024-04-12T11:24:49.667 INFO:tasks.rados.rados.0.smithi029.stdout:2386: seq_num 896 ranges {659072=636405,1731866=422298,2855825=203247} 2024-04-12T11:24:49.674 INFO:tasks.rados.rados.0.smithi029.stdout:2386: writing smithi02979277-19 from 659072 to 1295477 tid 1 2024-04-12T11:24:49.678 INFO:tasks.rados.rados.0.smithi029.stdout:2386: writing smithi02979277-19 from 1731866 to 2154164 tid 2 2024-04-12T11:24:49.684 INFO:tasks.rados.rados.0.smithi029.stdout:2386: writing smithi02979277-19 from 2855825 to 3059072 tid 3 2024-04-12T11:24:49.684 INFO:tasks.rados.rados.0.smithi029.stdout:2387: read oid 46 snap -1 2024-04-12T11:24:49.684 INFO:tasks.rados.rados.0.smithi029.stdout:2387: expect deleted 2024-04-12T11:24:49.684 INFO:tasks.rados.rados.0.smithi029.stdout:2388: read oid 32 snap -1 2024-04-12T11:24:49.684 INFO:tasks.rados.rados.0.smithi029.stdout:2388: expect (ObjNum 878 snap 310 seq_num 878) 2024-04-12T11:24:49.684 INFO:tasks.rados.rados.0.smithi029.stdout:2389: read oid 45 snap -1 2024-04-12T11:24:49.684 INFO:tasks.rados.rados.0.smithi029.stdout:2389: expect (ObjNum 857 snap 295 seq_num 857) 2024-04-12T11:24:49.684 INFO:tasks.rados.rados.0.smithi029.stdout:2390: write oid 8 current snap is 313 2024-04-12T11:24:49.684 INFO:tasks.rados.rados.0.smithi029.stdout:2390: seq_num 897 ranges {675879=711654,2073995=682309} 2024-04-12T11:24:49.691 INFO:tasks.rados.rados.0.smithi029.stdout:2390: writing smithi02979277-8 from 675879 to 1387533 tid 1 2024-04-12T11:24:49.699 INFO:tasks.rados.rados.0.smithi029.stdout:2390: writing smithi02979277-8 from 2073995 to 2756304 tid 2 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2391: delete oid 31 current snap is 313 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2385: finishing write tid 1 to smithi02979277-2 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2385: finishing write tid 2 to smithi02979277-2 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2385: finishing write tid 3 to smithi02979277-2 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2385: finishing write tid 4 to smithi02979277-2 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2385: finishing write tid 5 to smithi02979277-2 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2385: oid 2 updating version 0 to 254 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2385: oid 2 updating version 254 to 256 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2385: oid 2 updating version 256 to 257 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2385: oid 2 version 257 is already newer than 255 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 257 (ObjNum 895 snap 313 seq_num 895) dirty exists 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2385: left oid 2 (ObjNum 895 snap 313 seq_num 895) 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2386: finishing write tid 1 to smithi02979277-19 2024-04-12T11:24:49.700 INFO:tasks.rados.rados.0.smithi029.stdout:2386: finishing write tid 2 to smithi02979277-19 2024-04-12T11:24:49.701 INFO:tasks.rados.rados.0.smithi029.stdout:2386: finishing write tid 3 to smithi02979277-19 2024-04-12T11:24:49.701 INFO:tasks.rados.rados.0.smithi029.stdout:2386: finishing write tid 4 to smithi02979277-19 2024-04-12T11:24:49.701 INFO:tasks.rados.rados.0.smithi029.stdout:2386: finishing write tid 5 to smithi02979277-19 2024-04-12T11:24:49.701 INFO:tasks.rados.rados.0.smithi029.stdout:2386: oid 19 updating version 0 to 1102 2024-04-12T11:24:49.701 INFO:tasks.rados.rados.0.smithi029.stdout:2386: oid 19 updating version 1102 to 1103 2024-04-12T11:24:49.701 INFO:tasks.rados.rados.0.smithi029.stdout:2386: oid 19 updating version 1103 to 1104 2024-04-12T11:24:49.701 INFO:tasks.rados.rados.0.smithi029.stdout:2386: oid 19 updating version 1104 to 1105 2024-04-12T11:24:49.701 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1105 (ObjNum 896 snap 313 seq_num 896) dirty exists 2024-04-12T11:24:49.701 INFO:tasks.rados.rados.0.smithi029.stdout:2386: left oid 19 (ObjNum 896 snap 313 seq_num 896) 2024-04-12T11:24:49.701 INFO:tasks.rados.rados.0.smithi029.stdout:2388: expect (ObjNum 878 snap 310 seq_num 878) 2024-04-12T11:24:49.870 INFO:tasks.rados.rados.0.smithi029.stdout:2389: expect (ObjNum 857 snap 295 seq_num 857) 2024-04-12T11:24:49.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:49 smithi087 ceph-mon[89014]: osdmap e684: 8 total, 8 up, 8 in 2024-04-12T11:24:49.985 INFO:tasks.rados.rados.0.smithi029.stdout:2390: finishing write tid 1 to smithi02979277-8 2024-04-12T11:24:49.985 INFO:tasks.rados.rados.0.smithi029.stdout:2390: finishing write tid 2 to smithi02979277-8 2024-04-12T11:24:49.985 INFO:tasks.rados.rados.0.smithi029.stdout:2390: finishing write tid 3 to smithi02979277-8 2024-04-12T11:24:49.985 INFO:tasks.rados.rados.0.smithi029.stdout:2390: finishing write tid 4 to smithi02979277-8 2024-04-12T11:24:49.985 INFO:tasks.rados.rados.0.smithi029.stdout:2390: oid 8 updating version 0 to 255 2024-04-12T11:24:49.985 INFO:tasks.rados.rados.0.smithi029.stdout:2390: oid 8 version 255 is already newer than 253 2024-04-12T11:24:49.986 INFO:tasks.rados.rados.0.smithi029.stdout:2390: oid 8 version 255 is already newer than 254 2024-04-12T11:24:49.986 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 255 (ObjNum 897 snap 313 seq_num 897) dirty exists 2024-04-12T11:24:49.986 INFO:tasks.rados.rados.0.smithi029.stdout:2390: left oid 8 (ObjNum 897 snap 313 seq_num 897) 2024-04-12T11:24:49.986 INFO:tasks.rados.rados.0.smithi029.stdout:2384: done (7 left) 2024-04-12T11:24:49.986 INFO:tasks.rados.rados.0.smithi029.stdout:2385: done (6 left) 2024-04-12T11:24:49.986 INFO:tasks.rados.rados.0.smithi029.stdout:2386: done (5 left) 2024-04-12T11:24:49.986 INFO:tasks.rados.rados.0.smithi029.stdout:2387: done (4 left) 2024-04-12T11:24:49.986 INFO:tasks.rados.rados.0.smithi029.stdout:2388: done (3 left) 2024-04-12T11:24:49.986 INFO:tasks.rados.rados.0.smithi029.stdout:2389: done (2 left) 2024-04-12T11:24:49.987 INFO:tasks.rados.rados.0.smithi029.stdout:2390: done (1 left) 2024-04-12T11:24:49.987 INFO:tasks.rados.rados.0.smithi029.stdout:2391: done (0 left) 2024-04-12T11:24:49.987 INFO:tasks.rados.rados.0.smithi029.stdout:2392: snap_remove snap 283 2024-04-12T11:24:50.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:49 smithi029 ceph-mon[93650]: osdmap e684: 8 total, 8 up, 8 in 2024-04-12T11:24:50.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:49 smithi029 ceph-mon[97787]: osdmap e684: 8 total, 8 up, 8 in 2024-04-12T11:24:50.627 INFO:tasks.rados.rados.0.smithi029.stdout:2392: done (0 left) 2024-04-12T11:24:50.627 INFO:tasks.rados.rados.0.smithi029.stdout:2393: read oid 48 snap -1 2024-04-12T11:24:50.627 INFO:tasks.rados.rados.0.smithi029.stdout:2393: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:24:50.627 INFO:tasks.rados.rados.0.smithi029.stdout:2394: read oid 37 snap -1 2024-04-12T11:24:50.627 INFO:tasks.rados.rados.0.smithi029.stdout:2394: expect (ObjNum 839 snap 292 seq_num 839) 2024-04-12T11:24:50.627 INFO:tasks.rados.rados.0.smithi029.stdout:2395: read oid 31 snap 302 2024-04-12T11:24:50.627 INFO:tasks.rados.rados.0.smithi029.stdout:2395: expect (ObjNum 794 snap 277 seq_num 794) 2024-04-12T11:24:50.627 INFO:tasks.rados.rados.0.smithi029.stdout:2396: delete oid 14 current snap is 313 2024-04-12T11:24:50.636 INFO:tasks.rados.rados.0.smithi029.stdout:2393: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:24:50.806 INFO:tasks.rados.rados.0.smithi029.stdout:2394: expect (ObjNum 839 snap 292 seq_num 839) 2024-04-12T11:24:50.952 INFO:tasks.rados.rados.0.smithi029.stdout:2395: expect (ObjNum 794 snap 277 seq_num 794) 2024-04-12T11:24:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:50 smithi087 ceph-mon[89014]: pgmap v663: 89 pgs: 89 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-04-12T11:24:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:50 smithi087 ceph-mon[89014]: osdmap e685: 8 total, 8 up, 8 in 2024-04-12T11:24:51.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:50 smithi029 ceph-mon[97787]: pgmap v663: 89 pgs: 89 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-04-12T11:24:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:50 smithi029 ceph-mon[97787]: osdmap e685: 8 total, 8 up, 8 in 2024-04-12T11:24:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:50 smithi029 ceph-mon[93650]: pgmap v663: 89 pgs: 89 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-04-12T11:24:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:50 smithi029 ceph-mon[93650]: osdmap e685: 8 total, 8 up, 8 in 2024-04-12T11:24:51.062 INFO:tasks.rados.rados.0.smithi029.stdout:2393: done (3 left) 2024-04-12T11:24:51.063 INFO:tasks.rados.rados.0.smithi029.stdout:2394: done (2 left) 2024-04-12T11:24:51.063 INFO:tasks.rados.rados.0.smithi029.stdout:2395: done (1 left) 2024-04-12T11:24:51.063 INFO:tasks.rados.rados.0.smithi029.stdout:2396: done (0 left) 2024-04-12T11:24:51.063 INFO:tasks.rados.rados.0.smithi029.stdout:2397: write oid 33 current snap is 313 2024-04-12T11:24:51.063 INFO:tasks.rados.rados.0.smithi029.stdout:2397: seq_num 900 ranges {726300=653754,1965794=429636,2965517=160783} 2024-04-12T11:24:51.069 INFO:tasks.rados.rados.0.smithi029.stdout:2397: writing smithi02979277-33 from 726300 to 1380054 tid 1 2024-04-12T11:24:51.075 INFO:tasks.rados.rados.0.smithi029.stdout:2397: writing smithi02979277-33 from 1965794 to 2395430 tid 2 2024-04-12T11:24:51.080 INFO:tasks.rados.rados.0.smithi029.stdout:2397: writing smithi02979277-33 from 2965517 to 3126300 tid 3 2024-04-12T11:24:51.080 INFO:tasks.rados.rados.0.smithi029.stdout:2398: read oid 26 snap 306 2024-04-12T11:24:51.080 INFO:tasks.rados.rados.0.smithi029.stdout:2398: expect (ObjNum 748 snap 258 seq_num 748) 2024-04-12T11:24:51.080 INFO:tasks.rados.rados.0.smithi029.stdout:2399: write oid 12 current snap is 313 2024-04-12T11:24:51.080 INFO:tasks.rados.rados.0.smithi029.stdout:2397: finishing write tid 1 to smithi02979277-33 2024-04-12T11:24:51.080 INFO:tasks.rados.rados.0.smithi029.stdout:2397: finishing write tid 2 to smithi02979277-33 2024-04-12T11:24:51.080 INFO:tasks.rados.rados.0.smithi029.stdout:2399: seq_num 901 ranges {743107=729003,1907923=689647} 2024-04-12T11:24:51.087 INFO:tasks.rados.rados.0.smithi029.stdout:2399: writing smithi02979277-12 from 743107 to 1472110 tid 1 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:2399: writing smithi02979277-12 from 1907923 to 2597570 tid 2 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:2400: rollback oid 50 current snap is 313 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 50 to 298 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:2397: finishing write tid 3 to smithi02979277-33 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:2397: finishing write tid 4 to smithi02979277-33 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:2397: finishing write tid 5 to smithi02979277-33 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:2397: oid 33 updating version 0 to 944 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:2397: oid 33 version 944 is already newer than 943 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:2397: oid 33 updating version 944 to 945 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:2397: oid 33 version 945 is already newer than 942 2024-04-12T11:24:51.094 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 945 (ObjNum 900 snap 313 seq_num 900) dirty exists 2024-04-12T11:24:51.095 INFO:tasks.rados.rados.0.smithi029.stdout:2397: left oid 33 (ObjNum 900 snap 313 seq_num 900) 2024-04-12T11:24:51.095 INFO:tasks.rados.rados.0.smithi029.stdout:2398: expect (ObjNum 748 snap 258 seq_num 748) 2024-04-12T11:24:51.246 INFO:tasks.rados.rados.0.smithi029.stdout:2399: finishing write tid 1 to smithi02979277-12 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2399: finishing write tid 2 to smithi02979277-12 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2399: finishing write tid 3 to smithi02979277-12 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2399: finishing write tid 4 to smithi02979277-12 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2399: oid 12 updating version 0 to 1153 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2399: oid 12 version 1153 is already newer than 1152 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2399: oid 12 version 1153 is already newer than 1151 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1153 (ObjNum 901 snap 313 seq_num 901) dirty exists 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2399: left oid 12 (ObjNum 901 snap 313 seq_num 901) 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2400: finishing rollback tid 0 to smithi02979277-50 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2400: finishing rollback tid 1 to smithi02979277-50 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2400: finishing rollback tid 2 to smithi02979277-50 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 949 (ObjNum 822 snap 285 seq_num 822) dirty exists 2024-04-12T11:24:51.247 INFO:tasks.rados.rados.0.smithi029.stdout:2397: done (3 left) 2024-04-12T11:24:51.248 INFO:tasks.rados.rados.0.smithi029.stdout:2398: done (2 left) 2024-04-12T11:24:51.248 INFO:tasks.rados.rados.0.smithi029.stdout:2399: done (1 left) 2024-04-12T11:24:51.248 INFO:tasks.rados.rados.0.smithi029.stdout:2400: done (0 left) 2024-04-12T11:24:51.248 INFO:tasks.rados.rados.0.smithi029.stdout:2401: read oid 22 snap -1 2024-04-12T11:24:51.248 INFO:tasks.rados.rados.0.smithi029.stdout:2401: expect (ObjNum 892 snap 312 seq_num 892) 2024-04-12T11:24:51.248 INFO:tasks.rados.rados.0.smithi029.stdout:2402: delete oid 16 current snap is 313 2024-04-12T11:24:51.248 INFO:tasks.rados.rados.0.smithi029.stdout:2402: done (1 left) 2024-04-12T11:24:51.248 INFO:tasks.rados.rados.0.smithi029.stdout:2403: snap_remove snap 293 2024-04-12T11:24:51.637 INFO:tasks.rados.rados.0.smithi029.stdout:2403: done (1 left) 2024-04-12T11:24:51.637 INFO:tasks.rados.rados.0.smithi029.stdout:2404: read oid 5 snap 311 2024-04-12T11:24:51.637 INFO:tasks.rados.rados.0.smithi029.stdout:2404: expect (ObjNum 871 snap 307 seq_num 871) 2024-04-12T11:24:51.637 INFO:tasks.rados.rados.0.smithi029.stdout:2405: delete oid 36 current snap is 313 2024-04-12T11:24:51.637 INFO:tasks.rados.rados.0.smithi029.stdout:2401: expect (ObjNum 892 snap 312 seq_num 892) 2024-04-12T11:24:51.840 INFO:tasks.rados.rados.0.smithi029.stdout:2404: expect (ObjNum 871 snap 307 seq_num 871) 2024-04-12T11:24:51.964 INFO:tasks.rados.rados.0.smithi029.stdout:2401: done (2 left) 2024-04-12T11:24:51.964 INFO:tasks.rados.rados.0.smithi029.stdout:2404: done (1 left) 2024-04-12T11:24:51.964 INFO:tasks.rados.rados.0.smithi029.stdout:2405: done (0 left) 2024-04-12T11:24:51.965 INFO:tasks.rados.rados.0.smithi029.stdout:2406: rollback oid 37 current snap is 313 2024-04-12T11:24:51.965 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 289 2024-04-12T11:24:51.965 INFO:tasks.rados.rados.0.smithi029.stdout:2407: rollback oid 34 current snap is 313 2024-04-12T11:24:51.965 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 312 2024-04-12T11:24:51.965 INFO:tasks.rados.rados.0.smithi029.stdout:2408: write oid 50 current snap is 313 2024-04-12T11:24:51.965 INFO:tasks.rados.rados.0.smithi029.stdout:2408: seq_num 904 ranges {793528=554750,2083369=436974,2958856=234672} 2024-04-12T11:24:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:51 smithi087 ceph-mon[89014]: osdmap e686: 8 total, 8 up, 8 in 2024-04-12T11:24:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:51.971 INFO:tasks.rados.rados.0.smithi029.stdout:2408: writing smithi02979277-50 from 793528 to 1348278 tid 1 2024-04-12T11:24:51.977 INFO:tasks.rados.rados.0.smithi029.stdout:2408: writing smithi02979277-50 from 2083369 to 2520343 tid 2 2024-04-12T11:24:51.981 INFO:tasks.rados.rados.0.smithi029.stdout:2408: writing smithi02979277-50 from 2958856 to 3193528 tid 3 2024-04-12T11:24:51.982 INFO:tasks.rados.rados.0.smithi029.stdout:2409: snap_create 2024-04-12T11:24:51.982 INFO:tasks.rados.rados.0.smithi029.stdout:2407: finishing rollback tid 0 to smithi02979277-34 2024-04-12T11:24:51.982 INFO:tasks.rados.rados.0.smithi029.stdout:2406: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:24:51.982 INFO:tasks.rados.rados.0.smithi029.stdout:2406: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:24:51.982 INFO:tasks.rados.rados.0.smithi029.stdout:2407: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:24:51.982 INFO:tasks.rados.rados.0.smithi029.stdout:2407: finishing rollback tid 2 to smithi02979277-34 2024-04-12T11:24:51.982 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1001 (ObjNum 880 snap 311 seq_num 880) dirty exists 2024-04-12T11:24:51.983 INFO:tasks.rados.rados.0.smithi029.stdout:2406: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:24:51.983 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 888 (ObjNum 692 snap 224 seq_num 692) dirty exists 2024-04-12T11:24:51.983 INFO:tasks.rados.rados.0.smithi029.stdout:2408: finishing write tid 1 to smithi02979277-50 2024-04-12T11:24:51.983 INFO:tasks.rados.rados.0.smithi029.stdout:2408: finishing write tid 2 to smithi02979277-50 2024-04-12T11:24:51.984 INFO:tasks.rados.rados.0.smithi029.stdout:2408: finishing write tid 3 to smithi02979277-50 2024-04-12T11:24:51.984 INFO:tasks.rados.rados.0.smithi029.stdout:2408: finishing write tid 4 to smithi02979277-50 2024-04-12T11:24:51.984 INFO:tasks.rados.rados.0.smithi029.stdout:2408: finishing write tid 5 to smithi02979277-50 2024-04-12T11:24:51.984 INFO:tasks.rados.rados.0.smithi029.stdout:2408: oid 50 updating version 0 to 956 2024-04-12T11:24:51.985 INFO:tasks.rados.rados.0.smithi029.stdout:2408: oid 50 updating version 956 to 958 2024-04-12T11:24:51.985 INFO:tasks.rados.rados.0.smithi029.stdout:2408: oid 50 updating version 958 to 959 2024-04-12T11:24:51.985 INFO:tasks.rados.rados.0.smithi029.stdout:2408: oid 50 version 959 is already newer than 957 2024-04-12T11:24:51.985 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 959 (ObjNum 904 snap 313 seq_num 904) dirty exists 2024-04-12T11:24:51.985 INFO:tasks.rados.rados.0.smithi029.stdout:2408: left oid 50 (ObjNum 904 snap 313 seq_num 904) 2024-04-12T11:24:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:51 smithi029 ceph-mon[93650]: osdmap e686: 8 total, 8 up, 8 in 2024-04-12T11:24:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:51 smithi029 ceph-mon[97787]: osdmap e686: 8 total, 8 up, 8 in 2024-04-12T11:24:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:24:52.607 INFO:tasks.rados.rados.0.smithi029.stdout:2406: done (3 left) 2024-04-12T11:24:52.607 INFO:tasks.rados.rados.0.smithi029.stdout:2407: done (2 left) 2024-04-12T11:24:52.607 INFO:tasks.rados.rados.0.smithi029.stdout:2408: done (1 left) 2024-04-12T11:24:52.608 INFO:tasks.rados.rados.0.smithi029.stdout:2409: done (0 left) 2024-04-12T11:24:52.608 INFO:tasks.rados.rados.0.smithi029.stdout:2410: delete oid 24 current snap is 314 2024-04-12T11:24:52.618 INFO:tasks.rados.rados.0.smithi029.stdout:2410: done (0 left) 2024-04-12T11:24:52.618 INFO:tasks.rados.rados.0.smithi029.stdout:2411: rollback oid 18 current snap is 314 2024-04-12T11:24:52.619 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 18 to 310 2024-04-12T11:24:52.619 INFO:tasks.rados.rados.0.smithi029.stdout:2412: snap_remove snap 302 2024-04-12T11:24:52.646 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:24:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:24:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:24:52.908 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:52 smithi087 ceph-mon[89014]: pgmap v666: 89 pgs: 89 active+clean; 188 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 500 KiB/s rd, 1.2 MiB/s wr, 5 op/s 2024-04-12T11:24:52.909 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:52 smithi087 ceph-mon[89014]: osdmap e687: 8 total, 8 up, 8 in 2024-04-12T11:24:52.909 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:52 smithi087 ceph-mon[89014]: osdmap e688: 8 total, 8 up, 8 in 2024-04-12T11:24:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:52 smithi029 ceph-mon[93650]: pgmap v666: 89 pgs: 89 active+clean; 188 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 500 KiB/s rd, 1.2 MiB/s wr, 5 op/s 2024-04-12T11:24:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:52 smithi029 ceph-mon[93650]: osdmap e687: 8 total, 8 up, 8 in 2024-04-12T11:24:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:52 smithi029 ceph-mon[93650]: osdmap e688: 8 total, 8 up, 8 in 2024-04-12T11:24:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:52 smithi029 ceph-mon[97787]: pgmap v666: 89 pgs: 89 active+clean; 188 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 500 KiB/s rd, 1.2 MiB/s wr, 5 op/s 2024-04-12T11:24:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:52 smithi029 ceph-mon[97787]: osdmap e687: 8 total, 8 up, 8 in 2024-04-12T11:24:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:52 smithi029 ceph-mon[97787]: osdmap e688: 8 total, 8 up, 8 in 2024-04-12T11:24:53.617 INFO:tasks.rados.rados.0.smithi029.stdout:2412: done (1 left) 2024-04-12T11:24:53.617 INFO:tasks.rados.rados.0.smithi029.stdout:2413: write oid 19 current snap is 314 2024-04-12T11:24:53.618 INFO:tasks.rados.rados.0.smithi029.stdout:2413: seq_num 906 ranges {427142=421601,1400333=440643,2613702=450326} 2024-04-12T11:24:53.622 INFO:tasks.rados.rados.0.smithi029.stdout:2413: writing smithi02979277-19 from 427142 to 848743 tid 1 2024-04-12T11:24:53.628 INFO:tasks.rados.rados.0.smithi029.stdout:2413: writing smithi02979277-19 from 1400333 to 1840976 tid 2 2024-04-12T11:24:53.635 INFO:tasks.rados.rados.0.smithi029.stdout:2413: writing smithi02979277-19 from 2613702 to 3064028 tid 3 2024-04-12T11:24:53.635 INFO:tasks.rados.rados.0.smithi029.stdout:2414: write oid 43 current snap is 314 2024-04-12T11:24:53.635 INFO:tasks.rados.rados.0.smithi029.stdout:2411: finishing rollback tid 0 to smithi02979277-18 2024-04-12T11:24:53.635 INFO:tasks.rados.rados.0.smithi029.stdout:2414: seq_num 907 ranges {443949=496850,1458815=584301,2641125=602824} 2024-04-12T11:24:53.640 INFO:tasks.rados.rados.0.smithi029.stdout:2414: writing smithi02979277-43 from 443949 to 940799 tid 1 2024-04-12T11:24:53.646 INFO:tasks.rados.rados.0.smithi029.stdout:2414: writing smithi02979277-43 from 1458815 to 2043116 tid 2 2024-04-12T11:24:53.653 INFO:tasks.rados.rados.0.smithi029.stdout:2414: writing smithi02979277-43 from 2641125 to 3243949 tid 3 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2415: read oid 47 snap 298 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2415: expect deleted 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2416: rollback oid 35 current snap is 314 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 35 to 297 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2411: finishing rollback tid 1 to smithi02979277-18 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2411: finishing rollback tid 2 to smithi02979277-18 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1169 (ObjNum 861 snap 296 seq_num 861) dirty exists 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2413: finishing write tid 1 to smithi02979277-19 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2413: finishing write tid 2 to smithi02979277-19 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2413: finishing write tid 3 to smithi02979277-19 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2413: finishing write tid 4 to smithi02979277-19 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2413: finishing write tid 5 to smithi02979277-19 2024-04-12T11:24:53.654 INFO:tasks.rados.rados.0.smithi029.stdout:2413: oid 19 updating version 0 to 1121 2024-04-12T11:24:53.655 INFO:tasks.rados.rados.0.smithi029.stdout:2413: oid 19 version 1121 is already newer than 1119 2024-04-12T11:24:53.655 INFO:tasks.rados.rados.0.smithi029.stdout:2413: oid 19 updating version 1121 to 1122 2024-04-12T11:24:53.655 INFO:tasks.rados.rados.0.smithi029.stdout:2413: oid 19 version 1122 is already newer than 1120 2024-04-12T11:24:53.655 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1122 (ObjNum 906 snap 314 seq_num 906) dirty exists 2024-04-12T11:24:53.655 INFO:tasks.rados.rados.0.smithi029.stdout:2413: left oid 19 (ObjNum 906 snap 314 seq_num 906) 2024-04-12T11:24:53.655 INFO:tasks.rados.rados.0.smithi029.stdout:2414: finishing write tid 1 to smithi02979277-43 2024-04-12T11:24:53.655 INFO:tasks.rados.rados.0.smithi029.stdout:2411: done (4 left) 2024-04-12T11:24:53.655 INFO:tasks.rados.rados.0.smithi029.stdout:2413: done (3 left) 2024-04-12T11:24:53.655 INFO:tasks.rados.rados.0.smithi029.stdout:2417: snap_remove snap 304 2024-04-12T11:24:54.653 INFO:tasks.rados.rados.0.smithi029.stdout:2417: done (3 left) 2024-04-12T11:24:54.653 INFO:tasks.rados.rados.0.smithi029.stdout:2418: snap_remove snap 311 2024-04-12T11:24:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:54 smithi029 ceph-mon[97787]: pgmap v669: 89 pgs: 89 active+clean; 210 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 4.4 MiB/s wr, 38 op/s 2024-04-12T11:24:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:54 smithi029 ceph-mon[97787]: osdmap e689: 8 total, 8 up, 8 in 2024-04-12T11:24:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:54 smithi029 ceph-mon[93650]: pgmap v669: 89 pgs: 89 active+clean; 210 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 4.4 MiB/s wr, 38 op/s 2024-04-12T11:24:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:54 smithi029 ceph-mon[93650]: osdmap e689: 8 total, 8 up, 8 in 2024-04-12T11:24:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:54 smithi087 ceph-mon[89014]: pgmap v669: 89 pgs: 89 active+clean; 210 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 4.4 MiB/s wr, 38 op/s 2024-04-12T11:24:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:54 smithi087 ceph-mon[89014]: osdmap e689: 8 total, 8 up, 8 in 2024-04-12T11:24:55.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:24:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:24:54] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-12T11:24:55.657 INFO:tasks.rados.rados.0.smithi029.stdout:2418: done (3 left) 2024-04-12T11:24:55.657 INFO:tasks.rados.rados.0.smithi029.stdout:2419: rollback oid 3 current snap is 314 2024-04-12T11:24:55.657 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 312 2024-04-12T11:24:55.657 INFO:tasks.rados.rados.0.smithi029.stdout:2414: finishing write tid 2 to smithi02979277-43 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2416: finishing rollback tid 1 to smithi02979277-35 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 0 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2414: finishing write tid 3 to smithi02979277-43 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2414: finishing write tid 4 to smithi02979277-43 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2414: finishing write tid 5 to smithi02979277-43 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2414: oid 43 updating version 0 to 1009 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2414: oid 43 updating version 1009 to 1011 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2414: oid 43 version 1011 is already newer than 1010 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2414: oid 43 updating version 1011 to 1012 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1012 (ObjNum 907 snap 314 seq_num 907) dirty exists 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2414: left oid 43 (ObjNum 907 snap 314 seq_num 907) 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2414: done (3 left) 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2415: done (2 left) 2024-04-12T11:24:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2416: done (1 left) 2024-04-12T11:24:55.659 INFO:tasks.rados.rados.0.smithi029.stdout:2420: snap_create 2024-04-12T11:24:55.667 INFO:tasks.rados.rados.0.smithi029.stdout:2419: finishing rollback tid 0 to smithi02979277-3 2024-04-12T11:24:55.668 INFO:tasks.rados.rados.0.smithi029.stdout:2419: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:24:55.668 INFO:tasks.rados.rados.0.smithi029.stdout:2419: finishing rollback tid 2 to smithi02979277-3 2024-04-12T11:24:55.668 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1409 (ObjNum 779 snap 272 seq_num 779) dirty exists 2024-04-12T11:24:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:55 smithi087 ceph-mon[89014]: osdmap e690: 8 total, 8 up, 8 in 2024-04-12T11:24:56.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:55 smithi029 ceph-mon[93650]: osdmap e690: 8 total, 8 up, 8 in 2024-04-12T11:24:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:55 smithi029 ceph-mon[97787]: osdmap e690: 8 total, 8 up, 8 in 2024-04-12T11:24:56.658 INFO:tasks.rados.rados.0.smithi029.stdout:2419: done (1 left) 2024-04-12T11:24:56.658 INFO:tasks.rados.rados.0.smithi029.stdout:2420: done (0 left) 2024-04-12T11:24:56.658 INFO:tasks.rados.rados.0.smithi029.stdout:2421: snap_create 2024-04-12T11:24:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:56 smithi087 ceph-mon[89014]: pgmap v672: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 211 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 5.1 MiB/s wr, 51 op/s 2024-04-12T11:24:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:56 smithi087 ceph-mon[89014]: osdmap e691: 8 total, 8 up, 8 in 2024-04-12T11:24:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:56 smithi029 ceph-mon[93650]: pgmap v672: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 211 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 5.1 MiB/s wr, 51 op/s 2024-04-12T11:24:57.055 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:56 smithi029 ceph-mon[93650]: osdmap e691: 8 total, 8 up, 8 in 2024-04-12T11:24:57.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:56 smithi029 ceph-mon[97787]: pgmap v672: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 211 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 5.1 MiB/s wr, 51 op/s 2024-04-12T11:24:57.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:56 smithi029 ceph-mon[97787]: osdmap e691: 8 total, 8 up, 8 in 2024-04-12T11:24:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2421: done (0 left) 2024-04-12T11:24:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2422: read oid 23 snap -1 2024-04-12T11:24:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2422: expect (ObjNum 887 snap 312 seq_num 887) 2024-04-12T11:24:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2423: rollback oid 29 current snap is 316 2024-04-12T11:24:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 29 to 306 2024-04-12T11:24:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2424: write oid 33 current snap is 316 2024-04-12T11:24:57.609 INFO:tasks.rados.rados.0.smithi029.stdout:2424: seq_num 908 ranges {460756=572099,1517297=444312,2668548=472870} 2024-04-12T11:24:57.615 INFO:tasks.rados.rados.0.smithi029.stdout:2424: writing smithi02979277-33 from 460756 to 1032855 tid 1 2024-04-12T11:24:57.620 INFO:tasks.rados.rados.0.smithi029.stdout:2424: writing smithi02979277-33 from 1517297 to 1961609 tid 2 2024-04-12T11:24:57.626 INFO:tasks.rados.rados.0.smithi029.stdout:2424: writing smithi02979277-33 from 2668548 to 3141418 tid 3 2024-04-12T11:24:57.626 INFO:tasks.rados.rados.0.smithi029.stdout:2425: write oid 3 current snap is 316 2024-04-12T11:24:57.627 INFO:tasks.rados.rados.0.smithi029.stdout:2425: seq_num 909 ranges {477563=647348,1859426=587970,2979618=297945} 2024-04-12T11:24:57.632 INFO:tasks.rados.rados.0.smithi029.stdout:2425: writing smithi02979277-3 from 477563 to 1124911 tid 1 2024-04-12T11:24:57.640 INFO:tasks.rados.rados.0.smithi029.stdout:2425: writing smithi02979277-3 from 1859426 to 2447396 tid 2 2024-04-12T11:24:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2425: writing smithi02979277-3 from 2979618 to 3277563 tid 3 2024-04-12T11:24:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2426: rollback oid 36 current snap is 316 2024-04-12T11:24:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 36 to 312 2024-04-12T11:24:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2423: finishing rollback tid 0 to smithi02979277-29 2024-04-12T11:24:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2423: finishing rollback tid 1 to smithi02979277-29 2024-04-12T11:24:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2423: finishing rollback tid 2 to smithi02979277-29 2024-04-12T11:24:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1026 (ObjNum 851 snap 292 seq_num 851) dirty exists 2024-04-12T11:24:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2422: expect (ObjNum 887 snap 312 seq_num 887) 2024-04-12T11:24:57.884 INFO:tasks.rados.rados.0.smithi029.stdout:2424: finishing write tid 1 to smithi02979277-33 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2424: finishing write tid 2 to smithi02979277-33 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2424: finishing write tid 3 to smithi02979277-33 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2424: finishing write tid 4 to smithi02979277-33 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2424: finishing write tid 5 to smithi02979277-33 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2424: oid 33 updating version 0 to 979 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2424: oid 33 updating version 979 to 981 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2424: oid 33 updating version 981 to 982 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2424: oid 33 version 982 is already newer than 980 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 982 (ObjNum 908 snap 316 seq_num 908) dirty exists 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2424: left oid 33 (ObjNum 908 snap 316 seq_num 908) 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2425: finishing write tid 1 to smithi02979277-3 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2425: finishing write tid 2 to smithi02979277-3 2024-04-12T11:24:57.885 INFO:tasks.rados.rados.0.smithi029.stdout:2422: done (4 left) 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2423: done (3 left) 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2424: done (2 left) 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2427: snap_create 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2426: finishing rollback tid 1 to smithi02979277-36 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2426: finishing rollback tid 2 to smithi02979277-36 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 639 (ObjNum 882 snap 311 seq_num 882) dirty exists 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2425: finishing write tid 3 to smithi02979277-3 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2425: finishing write tid 4 to smithi02979277-3 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2425: finishing write tid 5 to smithi02979277-3 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2425: oid 3 updating version 0 to 1415 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2425: oid 3 version 1415 is already newer than 1414 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2425: oid 3 updating version 1415 to 1416 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2425: oid 3 version 1416 is already newer than 1413 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1416 (ObjNum 909 snap 316 seq_num 909) dirty exists 2024-04-12T11:24:57.886 INFO:tasks.rados.rados.0.smithi029.stdout:2425: left oid 3 (ObjNum 909 snap 316 seq_num 909) 2024-04-12T11:24:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:57 smithi087 ceph-mon[89014]: osdmap e692: 8 total, 8 up, 8 in 2024-04-12T11:24:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:57 smithi087 ceph-mon[89014]: osdmap e693: 8 total, 8 up, 8 in 2024-04-12T11:24:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:57 smithi029 ceph-mon[97787]: osdmap e692: 8 total, 8 up, 8 in 2024-04-12T11:24:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:57 smithi029 ceph-mon[97787]: osdmap e693: 8 total, 8 up, 8 in 2024-04-12T11:24:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:57 smithi029 ceph-mon[93650]: osdmap e692: 8 total, 8 up, 8 in 2024-04-12T11:24:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:57 smithi029 ceph-mon[93650]: osdmap e693: 8 total, 8 up, 8 in 2024-04-12T11:24:58.672 INFO:tasks.rados.rados.0.smithi029.stdout:2425: done (2 left) 2024-04-12T11:24:58.672 INFO:tasks.rados.rados.0.smithi029.stdout:2426: done (1 left) 2024-04-12T11:24:58.672 INFO:tasks.rados.rados.0.smithi029.stdout:2427: done (0 left) 2024-04-12T11:24:58.672 INFO:tasks.rados.rados.0.smithi029.stdout:2428: read oid 24 snap -1 2024-04-12T11:24:58.672 INFO:tasks.rados.rados.0.smithi029.stdout:2428: expect deleted 2024-04-12T11:24:58.672 INFO:tasks.rados.rados.0.smithi029.stdout:2429: delete oid 2 current snap is 317 2024-04-12T11:24:58.676 INFO:tasks.rados.rados.0.smithi029.stdout:2428: done (1 left) 2024-04-12T11:24:58.676 INFO:tasks.rados.rados.0.smithi029.stdout:2429: done (0 left) 2024-04-12T11:24:58.676 INFO:tasks.rados.rados.0.smithi029.stdout:2430: delete oid 40 current snap is 317 2024-04-12T11:24:58.680 INFO:tasks.rados.rados.0.smithi029.stdout:2430: done (0 left) 2024-04-12T11:24:58.680 INFO:tasks.rados.rados.0.smithi029.stdout:2431: read oid 48 snap -1 2024-04-12T11:24:58.680 INFO:tasks.rados.rados.0.smithi029.stdout:2431: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:24:58.680 INFO:tasks.rados.rados.0.smithi029.stdout:2432: write oid 41 current snap is 317 2024-04-12T11:24:58.680 INFO:tasks.rados.rados.0.smithi029.stdout:2432: seq_num 912 ranges {527984=473095,1634872=451650,2661887=634311} 2024-04-12T11:24:58.685 INFO:tasks.rados.rados.0.smithi029.stdout:2432: writing smithi02979277-41 from 527984 to 1001079 tid 1 2024-04-12T11:24:58.691 INFO:tasks.rados.rados.0.smithi029.stdout:2432: writing smithi02979277-41 from 1634872 to 2086522 tid 2 2024-04-12T11:24:58.698 INFO:tasks.rados.rados.0.smithi029.stdout:2432: writing smithi02979277-41 from 2661887 to 3296198 tid 3 2024-04-12T11:24:58.698 INFO:tasks.rados.rados.0.smithi029.stdout:2433: write oid 3 current snap is 317 2024-04-12T11:24:58.699 INFO:tasks.rados.rados.0.smithi029.stdout:2433: seq_num 913 ranges {544791=664697,1693354=595308,2689310=445583} 2024-04-12T11:24:58.705 INFO:tasks.rados.rados.0.smithi029.stdout:2433: writing smithi02979277-3 from 544791 to 1209488 tid 1 2024-04-12T11:24:58.711 INFO:tasks.rados.rados.0.smithi029.stdout:2433: writing smithi02979277-3 from 1693354 to 2288662 tid 2 2024-04-12T11:24:58.716 INFO:tasks.rados.rados.0.smithi029.stdout:2433: writing smithi02979277-3 from 2689310 to 3134893 tid 3 2024-04-12T11:24:58.716 INFO:tasks.rados.rados.0.smithi029.stdout:2434: read oid 33 snap -1 2024-04-12T11:24:58.716 INFO:tasks.rados.rados.0.smithi029.stdout:2434: expect (ObjNum 908 snap 316 seq_num 908) 2024-04-12T11:24:58.717 INFO:tasks.rados.rados.0.smithi029.stdout:2435: read oid 32 snap -1 2024-04-12T11:24:58.717 INFO:tasks.rados.rados.0.smithi029.stdout:2435: expect (ObjNum 878 snap 310 seq_num 878) 2024-04-12T11:24:58.717 INFO:tasks.rados.rados.0.smithi029.stdout:2436: rollback oid 43 current snap is 317 2024-04-12T11:24:58.717 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 289 2024-04-12T11:24:58.717 INFO:tasks.rados.rados.0.smithi029.stdout:2431: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:24:58.888 INFO:tasks.rados.rados.0.smithi029.stdout:2432: finishing write tid 1 to smithi02979277-41 2024-04-12T11:24:58.888 INFO:tasks.rados.rados.0.smithi029.stdout:2432: finishing write tid 2 to smithi02979277-41 2024-04-12T11:24:58.888 INFO:tasks.rados.rados.0.smithi029.stdout:2431: done (5 left) 2024-04-12T11:24:58.888 INFO:tasks.rados.rados.0.smithi029.stdout:2437: delete oid 50 current snap is 317 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2432: finishing write tid 3 to smithi02979277-41 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2432: finishing write tid 4 to smithi02979277-41 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2432: finishing write tid 5 to smithi02979277-41 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2432: oid 41 updating version 0 to 1144 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2432: oid 41 updating version 1144 to 1145 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2432: oid 41 updating version 1145 to 1146 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2432: oid 41 updating version 1146 to 1147 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1147 (ObjNum 912 snap 317 seq_num 912) dirty exists 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2432: left oid 41 (ObjNum 912 snap 317 seq_num 912) 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2433: finishing write tid 1 to smithi02979277-3 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2433: finishing write tid 2 to smithi02979277-3 2024-04-12T11:24:58.889 INFO:tasks.rados.rados.0.smithi029.stdout:2435: expect (ObjNum 878 snap 310 seq_num 878) 2024-04-12T11:24:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:58 smithi087 ceph-mon[89014]: pgmap v675: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 212 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-04-12T11:24:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:58 smithi029 ceph-mon[93650]: pgmap v675: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 212 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-04-12T11:24:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:58 smithi029 ceph-mon[97787]: pgmap v675: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 212 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:2433: finishing write tid 3 to smithi02979277-3 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:2433: finishing write tid 4 to smithi02979277-3 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:2433: finishing write tid 5 to smithi02979277-3 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:2433: oid 3 updating version 0 to 1420 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:2433: oid 3 updating version 1420 to 1421 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:2433: oid 3 updating version 1421 to 1423 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:2433: oid 3 version 1423 is already newer than 1422 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1423 (ObjNum 913 snap 317 seq_num 913) dirty exists 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:2433: left oid 3 (ObjNum 913 snap 317 seq_num 913) 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:2436: finishing rollback tid 0 to smithi02979277-43 2024-04-12T11:24:59.059 INFO:tasks.rados.rados.0.smithi029.stdout:2436: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2436: finishing rollback tid 2 to smithi02979277-43 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1030 (ObjNum 427 snap 135 seq_num 427) dirty exists 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2432: done (5 left) 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2433: done (4 left) 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2435: done (3 left) 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2436: done (2 left) 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2437: done (1 left) 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2438: read oid 13 snap -1 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2438: expect (ObjNum 881 snap 311 seq_num 881) 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2439: read oid 50 snap 316 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2439: expect (ObjNum 904 snap 313 seq_num 904) 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2440: snap_create 2024-04-12T11:24:59.060 INFO:tasks.rados.rados.0.smithi029.stdout:2434: expect (ObjNum 908 snap 316 seq_num 908) 2024-04-12T11:24:59.210 INFO:tasks.rados.rados.0.smithi029.stdout:2438: expect (ObjNum 881 snap 311 seq_num 881) 2024-04-12T11:24:59.301 INFO:tasks.rados.rados.0.smithi029.stdout:2439: expect (ObjNum 904 snap 313 seq_num 904) 2024-04-12T11:24:59.676 INFO:tasks.rados.rados.0.smithi029.stdout:2434: done (3 left) 2024-04-12T11:24:59.676 INFO:tasks.rados.rados.0.smithi029.stdout:2438: done (2 left) 2024-04-12T11:24:59.677 INFO:tasks.rados.rados.0.smithi029.stdout:2439: done (1 left) 2024-04-12T11:24:59.677 INFO:tasks.rados.rados.0.smithi029.stdout:2440: done (0 left) 2024-04-12T11:24:59.677 INFO:tasks.rados.rados.0.smithi029.stdout:2441: write oid 17 current snap is 318 2024-04-12T11:24:59.677 INFO:tasks.rados.rados.0.smithi029.stdout:2441: seq_num 915 ranges {578405=415195,1410318=598977,2744156=584480} 2024-04-12T11:24:59.681 INFO:tasks.rados.rados.0.smithi029.stdout:2441: writing smithi02979277-17 from 578405 to 993600 tid 1 2024-04-12T11:24:59.687 INFO:tasks.rados.rados.0.smithi029.stdout:2441: writing smithi02979277-17 from 1410318 to 2009295 tid 2 2024-04-12T11:24:59.695 INFO:tasks.rados.rados.0.smithi029.stdout:2441: writing smithi02979277-17 from 2744156 to 3328636 tid 3 2024-04-12T11:24:59.695 INFO:tasks.rados.rados.0.smithi029.stdout:2442: read oid 39 snap -1 2024-04-12T11:24:59.695 INFO:tasks.rados.rados.0.smithi029.stdout:2442: expect (ObjNum 885 snap 312 seq_num 885) 2024-04-12T11:24:59.695 INFO:tasks.rados.rados.0.smithi029.stdout:2443: read oid 42 snap -1 2024-04-12T11:24:59.695 INFO:tasks.rados.rados.0.smithi029.stdout:2443: expect deleted 2024-04-12T11:24:59.696 INFO:tasks.rados.rados.0.smithi029.stdout:2444: delete oid 1 current snap is 318 2024-04-12T11:24:59.696 INFO:tasks.rados.rados.0.smithi029.stdout:2441: finishing write tid 1 to smithi02979277-17 2024-04-12T11:24:59.696 INFO:tasks.rados.rados.0.smithi029.stdout:2441: finishing write tid 2 to smithi02979277-17 2024-04-12T11:24:59.698 INFO:tasks.rados.rados.0.smithi029.stdout:2443: done (3 left) 2024-04-12T11:24:59.698 INFO:tasks.rados.rados.0.smithi029.stdout:2444: done (2 left) 2024-04-12T11:24:59.698 INFO:tasks.rados.rados.0.smithi029.stdout:2445: delete oid 9 current snap is 318 2024-04-12T11:24:59.699 INFO:tasks.rados.rados.0.smithi029.stdout:2442: expect (ObjNum 885 snap 312 seq_num 885) 2024-04-12T11:24:59.828 INFO:tasks.rados.rados.0.smithi029.stdout:2441: finishing write tid 3 to smithi02979277-17 2024-04-12T11:24:59.828 INFO:tasks.rados.rados.0.smithi029.stdout:2441: finishing write tid 4 to smithi02979277-17 2024-04-12T11:24:59.828 INFO:tasks.rados.rados.0.smithi029.stdout:2442: done (2 left) 2024-04-12T11:24:59.828 INFO:tasks.rados.rados.0.smithi029.stdout:2445: done (1 left) 2024-04-12T11:24:59.829 INFO:tasks.rados.rados.0.smithi029.stdout:2446: snap_remove snap 285 2024-04-12T11:24:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:24:59 smithi087 ceph-mon[89014]: osdmap e694: 8 total, 8 up, 8 in 2024-04-12T11:25:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:24:59 smithi029 ceph-mon[97787]: osdmap e694: 8 total, 8 up, 8 in 2024-04-12T11:25:00.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:24:59 smithi029 ceph-mon[93650]: osdmap e694: 8 total, 8 up, 8 in 2024-04-12T11:25:00.683 INFO:tasks.rados.rados.0.smithi029.stdout:2446: done (1 left) 2024-04-12T11:25:00.683 INFO:tasks.rados.rados.0.smithi029.stdout:2447: snap_remove snap 313 2024-04-12T11:25:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:00 smithi087 ceph-mon[89014]: pgmap v678: 89 pgs: 89 active+clean; 215 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-04-12T11:25:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:00 smithi087 ceph-mon[89014]: osdmap e695: 8 total, 8 up, 8 in 2024-04-12T11:25:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:00 smithi029 ceph-mon[93650]: pgmap v678: 89 pgs: 89 active+clean; 215 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-04-12T11:25:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:00 smithi029 ceph-mon[93650]: osdmap e695: 8 total, 8 up, 8 in 2024-04-12T11:25:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:00 smithi029 ceph-mon[97787]: pgmap v678: 89 pgs: 89 active+clean; 215 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 5 op/s 2024-04-12T11:25:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:00 smithi029 ceph-mon[97787]: osdmap e695: 8 total, 8 up, 8 in 2024-04-12T11:25:01.689 INFO:tasks.rados.rados.0.smithi029.stdout:2447: done (1 left) 2024-04-12T11:25:01.690 INFO:tasks.rados.rados.0.smithi029.stdout:2448: read oid 38 snap -1 2024-04-12T11:25:01.690 INFO:tasks.rados.rados.0.smithi029.stdout:2448: expect (ObjNum 864 snap 302 seq_num 864) 2024-04-12T11:25:01.690 INFO:tasks.rados.rados.0.smithi029.stdout:2449: read oid 31 snap -1 2024-04-12T11:25:01.690 INFO:tasks.rados.rados.0.smithi029.stdout:2449: expect deleted 2024-04-12T11:25:01.690 INFO:tasks.rados.rados.0.smithi029.stdout:2450: read oid 7 snap 312 2024-04-12T11:25:01.690 INFO:tasks.rados.rados.0.smithi029.stdout:2450: expect deleted 2024-04-12T11:25:01.690 INFO:tasks.rados.rados.0.smithi029.stdout:2451: write oid 47 current snap is 318 2024-04-12T11:25:01.690 INFO:tasks.rados.rados.0.smithi029.stdout:2451: seq_num 918 ranges {628826=640942,1869411=462657,2826425=534649} 2024-04-12T11:25:01.697 INFO:tasks.rados.rados.0.smithi029.stdout:2451: writing smithi02979277-47 from 628826 to 1269768 tid 1 2024-04-12T11:25:01.703 INFO:tasks.rados.rados.0.smithi029.stdout:2451: writing smithi02979277-47 from 1869411 to 2332068 tid 2 2024-04-12T11:25:01.709 INFO:tasks.rados.rados.0.smithi029.stdout:2451: writing smithi02979277-47 from 2826425 to 3361074 tid 3 2024-04-12T11:25:01.709 INFO:tasks.rados.rados.0.smithi029.stdout:2452: snap_remove snap 308 2024-04-12T11:25:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:01 smithi087 ceph-mon[89014]: osdmap e696: 8 total, 8 up, 8 in 2024-04-12T11:25:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:01 smithi029 ceph-mon[93650]: osdmap e696: 8 total, 8 up, 8 in 2024-04-12T11:25:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:01 smithi029 ceph-mon[97787]: osdmap e696: 8 total, 8 up, 8 in 2024-04-12T11:25:02.610 INFO:tasks.rados.rados.0.smithi029.stdout:2452: done (5 left) 2024-04-12T11:25:02.610 INFO:tasks.rados.rados.0.smithi029.stdout:2453: delete oid 34 current snap is 318 2024-04-12T11:25:02.611 INFO:tasks.rados.rados.0.smithi029.stdout:2441: finishing write tid 5 to smithi02979277-17 2024-04-12T11:25:02.611 INFO:tasks.rados.rados.0.smithi029.stdout:2441: oid 17 updating version 0 to 1186 2024-04-12T11:25:02.611 INFO:tasks.rados.rados.0.smithi029.stdout:2441: oid 17 version 1186 is already newer than 1183 2024-04-12T11:25:02.611 INFO:tasks.rados.rados.0.smithi029.stdout:2441: oid 17 version 1186 is already newer than 1185 2024-04-12T11:25:02.611 INFO:tasks.rados.rados.0.smithi029.stdout:2441: oid 17 version 1186 is already newer than 1184 2024-04-12T11:25:02.611 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1186 (ObjNum 915 snap 318 seq_num 915) dirty exists 2024-04-12T11:25:02.611 INFO:tasks.rados.rados.0.smithi029.stdout:2441: left oid 17 (ObjNum 915 snap 318 seq_num 915) 2024-04-12T11:25:02.611 INFO:tasks.rados.rados.0.smithi029.stdout:2448: expect (ObjNum 864 snap 302 seq_num 864) 2024-04-12T11:25:02.685 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:25:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:25:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:2451: finishing write tid 1 to smithi02979277-47 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:2451: finishing write tid 2 to smithi02979277-47 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:2451: finishing write tid 3 to smithi02979277-47 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:2451: finishing write tid 4 to smithi02979277-47 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:2451: finishing write tid 5 to smithi02979277-47 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:2451: oid 47 updating version 0 to 1171 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:2451: oid 47 updating version 1171 to 1173 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:2451: oid 47 updating version 1173 to 1174 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:2451: oid 47 version 1174 is already newer than 1172 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1174 (ObjNum 918 snap 318 seq_num 918) dirty exists 2024-04-12T11:25:02.734 INFO:tasks.rados.rados.0.smithi029.stdout:2451: left oid 47 (ObjNum 918 snap 318 seq_num 918) 2024-04-12T11:25:02.735 INFO:tasks.rados.rados.0.smithi029.stdout:2441: done (5 left) 2024-04-12T11:25:02.735 INFO:tasks.rados.rados.0.smithi029.stdout:2448: done (4 left) 2024-04-12T11:25:02.735 INFO:tasks.rados.rados.0.smithi029.stdout:2449: done (3 left) 2024-04-12T11:25:02.735 INFO:tasks.rados.rados.0.smithi029.stdout:2450: done (2 left) 2024-04-12T11:25:02.735 INFO:tasks.rados.rados.0.smithi029.stdout:2451: done (1 left) 2024-04-12T11:25:02.735 INFO:tasks.rados.rados.0.smithi029.stdout:2453: done (0 left) 2024-04-12T11:25:02.735 INFO:tasks.rados.rados.0.smithi029.stdout:2454: write oid 15 current snap is 318 2024-04-12T11:25:02.735 INFO:tasks.rados.rados.0.smithi029.stdout:2454: seq_num 920 ranges {662440=507793,1702728=749973,2881271=557193} 2024-04-12T11:25:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:2454: writing smithi02979277-15 from 662440 to 1170233 tid 1 2024-04-12T11:25:02.747 INFO:tasks.rados.rados.0.smithi029.stdout:2454: writing smithi02979277-15 from 1702728 to 2452701 tid 2 2024-04-12T11:25:02.752 INFO:tasks.rados.rados.0.smithi029.stdout:2454: writing smithi02979277-15 from 2881271 to 3438464 tid 3 2024-04-12T11:25:02.752 INFO:tasks.rados.rados.0.smithi029.stdout:2455: snap_remove snap 315 2024-04-12T11:25:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:02 smithi087 ceph-mon[89014]: pgmap v681: 89 pgs: 89 active+clean; 218 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 786 KiB/s wr, 4 op/s 2024-04-12T11:25:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:02 smithi087 ceph-mon[89014]: osdmap e697: 8 total, 8 up, 8 in 2024-04-12T11:25:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:02 smithi087 ceph-mon[89014]: osdmap e698: 8 total, 8 up, 8 in 2024-04-12T11:25:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:02 smithi029 ceph-mon[93650]: pgmap v681: 89 pgs: 89 active+clean; 218 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 786 KiB/s wr, 4 op/s 2024-04-12T11:25:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:02 smithi029 ceph-mon[93650]: osdmap e697: 8 total, 8 up, 8 in 2024-04-12T11:25:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:02 smithi029 ceph-mon[93650]: osdmap e698: 8 total, 8 up, 8 in 2024-04-12T11:25:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:02 smithi029 ceph-mon[97787]: pgmap v681: 89 pgs: 89 active+clean; 218 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 786 KiB/s wr, 4 op/s 2024-04-12T11:25:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:02 smithi029 ceph-mon[97787]: osdmap e697: 8 total, 8 up, 8 in 2024-04-12T11:25:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:02 smithi029 ceph-mon[97787]: osdmap e698: 8 total, 8 up, 8 in 2024-04-12T11:25:03.694 INFO:tasks.rados.rados.0.smithi029.stdout:2454: finishing write tid 1 to smithi02979277-15 2024-04-12T11:25:03.694 INFO:tasks.rados.rados.0.smithi029.stdout:2454: finishing write tid 2 to smithi02979277-15 2024-04-12T11:25:03.694 INFO:tasks.rados.rados.0.smithi029.stdout:2454: finishing write tid 3 to smithi02979277-15 2024-04-12T11:25:03.694 INFO:tasks.rados.rados.0.smithi029.stdout:2454: finishing write tid 4 to smithi02979277-15 2024-04-12T11:25:03.694 INFO:tasks.rados.rados.0.smithi029.stdout:2454: finishing write tid 5 to smithi02979277-15 2024-04-12T11:25:03.694 INFO:tasks.rados.rados.0.smithi029.stdout:2454: oid 15 updating version 0 to 785 2024-04-12T11:25:03.694 INFO:tasks.rados.rados.0.smithi029.stdout:2454: oid 15 version 785 is already newer than 784 2024-04-12T11:25:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:2454: oid 15 updating version 785 to 786 2024-04-12T11:25:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:2454: oid 15 updating version 786 to 787 2024-04-12T11:25:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 787 (ObjNum 920 snap 318 seq_num 920) dirty exists 2024-04-12T11:25:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:2454: left oid 15 (ObjNum 920 snap 318 seq_num 920) 2024-04-12T11:25:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:2454: done (1 left) 2024-04-12T11:25:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:2455: done (0 left) 2024-04-12T11:25:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:2456: read oid 7 snap -1 2024-04-12T11:25:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:2456: expect deleted 2024-04-12T11:25:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:2457: delete oid 36 current snap is 318 2024-04-12T11:25:03.697 INFO:tasks.rados.rados.0.smithi029.stdout:2456: done (1 left) 2024-04-12T11:25:03.697 INFO:tasks.rados.rados.0.smithi029.stdout:2457: done (0 left) 2024-04-12T11:25:03.697 INFO:tasks.rados.rados.0.smithi029.stdout:2458: write oid 2 current snap is 318 2024-04-12T11:25:03.697 INFO:tasks.rados.rados.0.smithi029.stdout:2458: seq_num 922 ranges {696054=658291,2103339=753642} 2024-04-12T11:25:03.704 INFO:tasks.rados.rados.0.smithi029.stdout:2458: writing smithi02979277-2 from 696054 to 1354345 tid 1 2024-04-12T11:25:03.713 INFO:tasks.rados.rados.0.smithi029.stdout:2458: writing smithi02979277-2 from 2103339 to 2856981 tid 2 2024-04-12T11:25:03.713 INFO:tasks.rados.rados.0.smithi029.stdout:2459: delete oid 4 current snap is 318 2024-04-12T11:25:03.713 INFO:tasks.rados.rados.0.smithi029.stdout:2458: finishing write tid 1 to smithi02979277-2 2024-04-12T11:25:03.717 INFO:tasks.rados.rados.0.smithi029.stdout:2458: finishing write tid 2 to smithi02979277-2 2024-04-12T11:25:03.717 INFO:tasks.rados.rados.0.smithi029.stdout:2458: finishing write tid 3 to smithi02979277-2 2024-04-12T11:25:03.717 INFO:tasks.rados.rados.0.smithi029.stdout:2458: finishing write tid 4 to smithi02979277-2 2024-04-12T11:25:03.717 INFO:tasks.rados.rados.0.smithi029.stdout:2458: oid 2 updating version 0 to 265 2024-04-12T11:25:03.717 INFO:tasks.rados.rados.0.smithi029.stdout:2458: oid 2 updating version 265 to 266 2024-04-12T11:25:03.717 INFO:tasks.rados.rados.0.smithi029.stdout:2458: oid 2 version 266 is already newer than 262 2024-04-12T11:25:03.717 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 266 (ObjNum 922 snap 318 seq_num 922) dirty exists 2024-04-12T11:25:03.717 INFO:tasks.rados.rados.0.smithi029.stdout:2458: left oid 2 (ObjNum 922 snap 318 seq_num 922) 2024-04-12T11:25:03.718 INFO:tasks.rados.rados.0.smithi029.stdout:2458: done (1 left) 2024-04-12T11:25:03.718 INFO:tasks.rados.rados.0.smithi029.stdout:2459: done (0 left) 2024-04-12T11:25:03.718 INFO:tasks.rados.rados.0.smithi029.stdout:2460: read oid 2 snap -1 2024-04-12T11:25:03.718 INFO:tasks.rados.rados.0.smithi029.stdout:2460: expect (ObjNum 922 snap 318 seq_num 922) 2024-04-12T11:25:03.718 INFO:tasks.rados.rados.0.smithi029.stdout:2461: delete oid 35 current snap is 318 2024-04-12T11:25:03.719 INFO:tasks.rados.rados.0.smithi029.stdout:2461: done (1 left) 2024-04-12T11:25:03.719 INFO:tasks.rados.rados.0.smithi029.stdout:2462: delete oid 45 current snap is 318 2024-04-12T11:25:03.722 INFO:tasks.rados.rados.0.smithi029.stdout:2460: expect (ObjNum 922 snap 318 seq_num 922) 2024-04-12T11:25:03.841 INFO:tasks.rados.rados.0.smithi029.stdout:2460: done (1 left) 2024-04-12T11:25:03.841 INFO:tasks.rados.rados.0.smithi029.stdout:2462: done (0 left) 2024-04-12T11:25:03.841 INFO:tasks.rados.rados.0.smithi029.stdout:2463: rollback oid 15 current snap is 318 2024-04-12T11:25:03.841 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 15 to 298 2024-04-12T11:25:03.841 INFO:tasks.rados.rados.0.smithi029.stdout:2464: delete oid 33 current snap is 318 2024-04-12T11:25:03.843 INFO:tasks.rados.rados.0.smithi029.stdout:2463: finishing rollback tid 0 to smithi02979277-15 2024-04-12T11:25:03.843 INFO:tasks.rados.rados.0.smithi029.stdout:2464: done (1 left) 2024-04-12T11:25:03.843 INFO:tasks.rados.rados.0.smithi029.stdout:2465: write oid 8 current snap is 318 2024-04-12T11:25:03.843 INFO:tasks.rados.rados.0.smithi029.stdout:2465: seq_num 927 ranges {780089=634536,1879396=620991,3240526=339563} 2024-04-12T11:25:03.851 INFO:tasks.rados.rados.0.smithi029.stdout:2465: writing smithi02979277-8 from 780089 to 1414625 tid 1 2024-04-12T11:25:03.857 INFO:tasks.rados.rados.0.smithi029.stdout:2465: writing smithi02979277-8 from 1879396 to 2500387 tid 2 2024-04-12T11:25:03.863 INFO:tasks.rados.rados.0.smithi029.stdout:2465: writing smithi02979277-8 from 3240526 to 3580089 tid 3 2024-04-12T11:25:03.863 INFO:tasks.rados.rados.0.smithi029.stdout:2466: delete oid 5 current snap is 318 2024-04-12T11:25:03.863 INFO:tasks.rados.rados.0.smithi029.stdout:2463: finishing rollback tid 1 to smithi02979277-15 2024-04-12T11:25:03.863 INFO:tasks.rados.rados.0.smithi029.stdout:2463: finishing rollback tid 2 to smithi02979277-15 2024-04-12T11:25:03.863 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 792 (ObjNum 818 snap 285 seq_num 818) dirty exists 2024-04-12T11:25:03.863 INFO:tasks.rados.rados.0.smithi029.stdout:2465: finishing write tid 1 to smithi02979277-8 2024-04-12T11:25:03.863 INFO:tasks.rados.rados.0.smithi029.stdout:2465: finishing write tid 2 to smithi02979277-8 2024-04-12T11:25:03.867 INFO:tasks.rados.rados.0.smithi029.stdout:2465: finishing write tid 3 to smithi02979277-8 2024-04-12T11:25:03.867 INFO:tasks.rados.rados.0.smithi029.stdout:2465: finishing write tid 4 to smithi02979277-8 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2465: finishing write tid 5 to smithi02979277-8 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2465: oid 8 updating version 0 to 274 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2465: oid 8 version 274 is already newer than 273 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2465: oid 8 version 274 is already newer than 271 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2465: oid 8 version 274 is already newer than 272 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 274 (ObjNum 927 snap 318 seq_num 927) dirty exists 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2465: left oid 8 (ObjNum 927 snap 318 seq_num 927) 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2463: done (2 left) 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2465: done (1 left) 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2466: done (0 left) 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2467: rollback oid 12 current snap is 318 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 309 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2468: read oid 45 snap 296 2024-04-12T11:25:03.868 INFO:tasks.rados.rados.0.smithi029.stdout:2468: expect (ObjNum 857 snap 295 seq_num 857) 2024-04-12T11:25:03.869 INFO:tasks.rados.rados.0.smithi029.stdout:2469: read oid 13 snap 298 2024-04-12T11:25:03.869 INFO:tasks.rados.rados.0.smithi029.stdout:2469: expect deleted 2024-04-12T11:25:03.869 INFO:tasks.rados.rados.0.smithi029.stdout:2470: write oid 42 current snap is 318 2024-04-12T11:25:03.869 INFO:tasks.rados.rados.0.smithi029.stdout:2470: seq_num 929 ranges {413703=501387,1596360=624660,2895372=691347} 2024-04-12T11:25:03.872 INFO:tasks.rados.rados.0.smithi029.stdout:2470: writing smithi02979277-42 from 413703 to 915090 tid 1 2024-04-12T11:25:03.879 INFO:tasks.rados.rados.0.smithi029.stdout:2470: writing smithi02979277-42 from 1596360 to 2221020 tid 2 2024-04-12T11:25:03.887 INFO:tasks.rados.rados.0.smithi029.stdout:2470: writing smithi02979277-42 from 2895372 to 3586719 tid 3 2024-04-12T11:25:03.887 INFO:tasks.rados.rados.0.smithi029.stdout:2471: snap_remove snap 297 2024-04-12T11:25:04.700 INFO:tasks.rados.rados.0.smithi029.stdout:2471: done (4 left) 2024-04-12T11:25:04.700 INFO:tasks.rados.rados.0.smithi029.stdout:2472: snap_create 2024-04-12T11:25:04.700 INFO:tasks.rados.rados.0.smithi029.stdout:2467: finishing rollback tid 0 to smithi02979277-12 2024-04-12T11:25:04.701 INFO:tasks.rados.rados.0.smithi029.stdout:2467: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:25:04.701 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1181 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-04-12T11:25:04.701 INFO:tasks.rados.rados.0.smithi029.stdout:2468: expect (ObjNum 857 snap 295 seq_num 857) 2024-04-12T11:25:04.819 INFO:tasks.rados.rados.0.smithi029.stdout:2470: finishing write tid 1 to smithi02979277-42 2024-04-12T11:25:04.819 INFO:tasks.rados.rados.0.smithi029.stdout:2470: finishing write tid 2 to smithi02979277-42 2024-04-12T11:25:04.819 INFO:tasks.rados.rados.0.smithi029.stdout:2470: finishing write tid 3 to smithi02979277-42 2024-04-12T11:25:04.820 INFO:tasks.rados.rados.0.smithi029.stdout:2470: finishing write tid 4 to smithi02979277-42 2024-04-12T11:25:04.820 INFO:tasks.rados.rados.0.smithi029.stdout:2470: finishing write tid 5 to smithi02979277-42 2024-04-12T11:25:04.820 INFO:tasks.rados.rados.0.smithi029.stdout:2470: oid 42 updating version 0 to 322 2024-04-12T11:25:04.820 INFO:tasks.rados.rados.0.smithi029.stdout:2470: oid 42 version 322 is already newer than 321 2024-04-12T11:25:04.820 INFO:tasks.rados.rados.0.smithi029.stdout:2470: oid 42 version 322 is already newer than 320 2024-04-12T11:25:04.820 INFO:tasks.rados.rados.0.smithi029.stdout:2470: oid 42 version 322 is already newer than 319 2024-04-12T11:25:04.820 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 322 (ObjNum 929 snap 318 seq_num 929) dirty exists 2024-04-12T11:25:04.820 INFO:tasks.rados.rados.0.smithi029.stdout:2470: left oid 42 (ObjNum 929 snap 318 seq_num 929) 2024-04-12T11:25:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:04 smithi087 ceph-mon[89014]: pgmap v684: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 222 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-04-12T11:25:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:04 smithi087 ceph-mon[89014]: osdmap e699: 8 total, 8 up, 8 in 2024-04-12T11:25:05.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:04 smithi029 ceph-mon[97787]: pgmap v684: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 222 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-04-12T11:25:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:04 smithi029 ceph-mon[97787]: osdmap e699: 8 total, 8 up, 8 in 2024-04-12T11:25:05.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:04 smithi029 ceph-mon[93650]: pgmap v684: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 222 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-04-12T11:25:05.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:04 smithi029 ceph-mon[93650]: osdmap e699: 8 total, 8 up, 8 in 2024-04-12T11:25:05.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:25:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:25:04] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-12T11:25:05.706 INFO:tasks.rados.rados.0.smithi029.stdout:2467: done (4 left) 2024-04-12T11:25:05.706 INFO:tasks.rados.rados.0.smithi029.stdout:2468: done (3 left) 2024-04-12T11:25:05.706 INFO:tasks.rados.rados.0.smithi029.stdout:2469: done (2 left) 2024-04-12T11:25:05.706 INFO:tasks.rados.rados.0.smithi029.stdout:2470: done (1 left) 2024-04-12T11:25:05.706 INFO:tasks.rados.rados.0.smithi029.stdout:2472: done (0 left) 2024-04-12T11:25:05.706 INFO:tasks.rados.rados.0.smithi029.stdout:2473: read oid 7 snap -1 2024-04-12T11:25:05.706 INFO:tasks.rados.rados.0.smithi029.stdout:2473: expect deleted 2024-04-12T11:25:05.706 INFO:tasks.rados.rados.0.smithi029.stdout:2474: rollback oid 40 current snap is 319 2024-04-12T11:25:05.706 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 40 to 312 2024-04-12T11:25:05.706 INFO:tasks.rados.rados.0.smithi029.stdout:2475: snap_remove snap 298 2024-04-12T11:25:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:05 smithi087 ceph-mon[89014]: osdmap e700: 8 total, 8 up, 8 in 2024-04-12T11:25:06.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:05 smithi029 ceph-mon[93650]: osdmap e700: 8 total, 8 up, 8 in 2024-04-12T11:25:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:05 smithi029 ceph-mon[97787]: osdmap e700: 8 total, 8 up, 8 in 2024-04-12T11:25:06.710 INFO:tasks.rados.rados.0.smithi029.stdout:2475: done (2 left) 2024-04-12T11:25:06.711 INFO:tasks.rados.rados.0.smithi029.stdout:2476: snap_remove snap 289 2024-04-12T11:25:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:06 smithi087 ceph-mon[89014]: pgmap v687: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 229 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.3 MiB/s wr, 29 op/s 2024-04-12T11:25:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:06 smithi087 ceph-mon[89014]: osdmap e701: 8 total, 8 up, 8 in 2024-04-12T11:25:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:06 smithi029 ceph-mon[97787]: pgmap v687: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 229 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.3 MiB/s wr, 29 op/s 2024-04-12T11:25:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:06 smithi029 ceph-mon[97787]: osdmap e701: 8 total, 8 up, 8 in 2024-04-12T11:25:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:06 smithi029 ceph-mon[93650]: pgmap v687: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 229 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.3 MiB/s wr, 29 op/s 2024-04-12T11:25:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:06 smithi029 ceph-mon[93650]: osdmap e701: 8 total, 8 up, 8 in 2024-04-12T11:25:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:07.611 INFO:tasks.rados.rados.0.smithi029.stdout:2476: done (2 left) 2024-04-12T11:25:07.612 INFO:tasks.rados.rados.0.smithi029.stdout:2477: rollback oid 41 current snap is 319 2024-04-12T11:25:07.612 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 314 2024-04-12T11:25:07.612 INFO:tasks.rados.rados.0.smithi029.stdout:2474: finishing rollback tid 1 to smithi02979277-40 2024-04-12T11:25:07.612 INFO:tasks.rados.rados.0.smithi029.stdout:2474: finishing rollback tid 2 to smithi02979277-40 2024-04-12T11:25:07.612 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 1454 (ObjNum 874 snap 308 seq_num 874) dirty exists 2024-04-12T11:25:07.612 INFO:tasks.rados.rados.0.smithi029.stdout:2473: done (2 left) 2024-04-12T11:25:07.612 INFO:tasks.rados.rados.0.smithi029.stdout:2474: done (1 left) 2024-04-12T11:25:07.612 INFO:tasks.rados.rados.0.smithi029.stdout:2478: read oid 3 snap -1 2024-04-12T11:25:07.612 INFO:tasks.rados.rados.0.smithi029.stdout:2478: expect (ObjNum 913 snap 317 seq_num 913) 2024-04-12T11:25:07.612 INFO:tasks.rados.rados.0.smithi029.stdout:2479: snap_remove snap 310 2024-04-12T11:25:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:07 smithi087 ceph-mon[89014]: osdmap e702: 8 total, 8 up, 8 in 2024-04-12T11:25:07.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:07 smithi087 ceph-mon[89014]: osdmap e703: 8 total, 8 up, 8 in 2024-04-12T11:25:08.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:07 smithi029 ceph-mon[97787]: osdmap e702: 8 total, 8 up, 8 in 2024-04-12T11:25:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:07 smithi029 ceph-mon[97787]: osdmap e703: 8 total, 8 up, 8 in 2024-04-12T11:25:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:07 smithi029 ceph-mon[93650]: osdmap e702: 8 total, 8 up, 8 in 2024-04-12T11:25:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:07 smithi029 ceph-mon[93650]: osdmap e703: 8 total, 8 up, 8 in 2024-04-12T11:25:08.613 INFO:tasks.rados.rados.0.smithi029.stdout:2479: done (2 left) 2024-04-12T11:25:08.613 INFO:tasks.rados.rados.0.smithi029.stdout:2480: snap_create 2024-04-12T11:25:08.613 INFO:tasks.rados.rados.0.smithi029.stdout:2477: finishing rollback tid 0 to smithi02979277-41 2024-04-12T11:25:08.613 INFO:tasks.rados.rados.0.smithi029.stdout:2477: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:25:08.613 INFO:tasks.rados.rados.0.smithi029.stdout:2477: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:25:08.614 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1179 (ObjNum 877 snap 310 seq_num 877) dirty exists 2024-04-12T11:25:08.614 INFO:tasks.rados.rados.0.smithi029.stdout:2478: expect (ObjNum 913 snap 317 seq_num 913) 2024-04-12T11:25:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:08 smithi087 ceph-mon[89014]: pgmap v690: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 230 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-04-12T11:25:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:08 smithi087 ceph-mon[89014]: osdmap e704: 8 total, 8 up, 8 in 2024-04-12T11:25:09.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:08 smithi029 ceph-mon[93650]: pgmap v690: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 230 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-04-12T11:25:09.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:08 smithi029 ceph-mon[93650]: osdmap e704: 8 total, 8 up, 8 in 2024-04-12T11:25:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:08 smithi029 ceph-mon[97787]: pgmap v690: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 230 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-04-12T11:25:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:08 smithi029 ceph-mon[97787]: osdmap e704: 8 total, 8 up, 8 in 2024-04-12T11:25:09.614 INFO:tasks.rados.rados.0.smithi029.stdout:2477: done (2 left) 2024-04-12T11:25:09.614 INFO:tasks.rados.rados.0.smithi029.stdout:2478: done (1 left) 2024-04-12T11:25:09.614 INFO:tasks.rados.rados.0.smithi029.stdout:2480: done (0 left) 2024-04-12T11:25:09.614 INFO:tasks.rados.rados.0.smithi029.stdout:2481: write oid 39 current snap is 320 2024-04-12T11:25:09.615 INFO:tasks.rados.rados.0.smithi029.stdout:2481: seq_num 930 ranges {430510=576636,1654842=768318,2922795=502619} 2024-04-12T11:25:09.619 INFO:tasks.rados.rados.0.smithi029.stdout:2481: writing smithi02979277-39 from 430510 to 1007146 tid 1 2024-04-12T11:25:09.627 INFO:tasks.rados.rados.0.smithi029.stdout:2481: writing smithi02979277-39 from 1654842 to 2423160 tid 2 2024-04-12T11:25:09.633 INFO:tasks.rados.rados.0.smithi029.stdout:2481: writing smithi02979277-39 from 2922795 to 3425414 tid 3 2024-04-12T11:25:09.633 INFO:tasks.rados.rados.0.smithi029.stdout:2482: snap_remove snap 312 2024-04-12T11:25:10.620 INFO:tasks.rados.rados.0.smithi029.stdout:2482: done (1 left) 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2483: snap_create 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2481: finishing write tid 1 to smithi02979277-39 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2481: finishing write tid 2 to smithi02979277-39 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2481: finishing write tid 3 to smithi02979277-39 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2481: finishing write tid 4 to smithi02979277-39 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2481: finishing write tid 5 to smithi02979277-39 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2481: oid 39 updating version 0 to 319 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2481: oid 39 updating version 319 to 322 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2481: oid 39 version 322 is already newer than 321 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2481: oid 39 version 322 is already newer than 320 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 322 (ObjNum 930 snap 320 seq_num 930) dirty exists 2024-04-12T11:25:10.621 INFO:tasks.rados.rados.0.smithi029.stdout:2481: left oid 39 (ObjNum 930 snap 320 seq_num 930) 2024-04-12T11:25:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:10 smithi087 ceph-mon[89014]: pgmap v693: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 232 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:25:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:10 smithi087 ceph-mon[89014]: osdmap e705: 8 total, 8 up, 8 in 2024-04-12T11:25:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:10 smithi029 ceph-mon[93650]: pgmap v693: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 232 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:25:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:10 smithi029 ceph-mon[93650]: osdmap e705: 8 total, 8 up, 8 in 2024-04-12T11:25:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:10 smithi029 ceph-mon[97787]: pgmap v693: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 232 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:25:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:10 smithi029 ceph-mon[97787]: osdmap e705: 8 total, 8 up, 8 in 2024-04-12T11:25:11.625 INFO:tasks.rados.rados.0.smithi029.stdout:2481: done (1 left) 2024-04-12T11:25:11.625 INFO:tasks.rados.rados.0.smithi029.stdout:2483: done (0 left) 2024-04-12T11:25:11.625 INFO:tasks.rados.rados.0.smithi029.stdout:2484: write oid 13 current snap is 321 2024-04-12T11:25:11.625 INFO:tasks.rados.rados.0.smithi029.stdout:2484: seq_num 931 ranges {447317=651885,1713324=628329,3066571=580746} 2024-04-12T11:25:11.631 INFO:tasks.rados.rados.0.smithi029.stdout:2484: writing smithi02979277-13 from 447317 to 1099202 tid 1 2024-04-12T11:25:11.638 INFO:tasks.rados.rados.0.smithi029.stdout:2484: writing smithi02979277-13 from 1713324 to 2341653 tid 2 2024-04-12T11:25:11.645 INFO:tasks.rados.rados.0.smithi029.stdout:2484: writing smithi02979277-13 from 3066571 to 3647317 tid 3 2024-04-12T11:25:11.645 INFO:tasks.rados.rados.0.smithi029.stdout:2485: delete oid 35 current snap is 321 2024-04-12T11:25:11.645 INFO:tasks.rados.rados.0.smithi029.stdout:2484: finishing write tid 1 to smithi02979277-13 2024-04-12T11:25:11.645 INFO:tasks.rados.rados.0.smithi029.stdout:2484: finishing write tid 2 to smithi02979277-13 2024-04-12T11:25:11.647 INFO:tasks.rados.rados.0.smithi029.stdout:2485: done (1 left) 2024-04-12T11:25:11.647 INFO:tasks.rados.rados.0.smithi029.stdout:2486: snap_create 2024-04-12T11:25:11.649 INFO:tasks.rados.rados.0.smithi029.stdout:2484: finishing write tid 3 to smithi02979277-13 2024-04-12T11:25:11.649 INFO:tasks.rados.rados.0.smithi029.stdout:2484: finishing write tid 4 to smithi02979277-13 2024-04-12T11:25:11.649 INFO:tasks.rados.rados.0.smithi029.stdout:2484: finishing write tid 5 to smithi02979277-13 2024-04-12T11:25:11.649 INFO:tasks.rados.rados.0.smithi029.stdout:2484: oid 13 updating version 0 to 1229 2024-04-12T11:25:11.649 INFO:tasks.rados.rados.0.smithi029.stdout:2484: oid 13 updating version 1229 to 1230 2024-04-12T11:25:11.649 INFO:tasks.rados.rados.0.smithi029.stdout:2484: oid 13 version 1230 is already newer than 1228 2024-04-12T11:25:11.649 INFO:tasks.rados.rados.0.smithi029.stdout:2484: oid 13 updating version 1230 to 1231 2024-04-12T11:25:11.649 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1231 (ObjNum 931 snap 321 seq_num 931) dirty exists 2024-04-12T11:25:11.649 INFO:tasks.rados.rados.0.smithi029.stdout:2484: left oid 13 (ObjNum 931 snap 321 seq_num 931) 2024-04-12T11:25:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:11 smithi087 ceph-mon[89014]: osdmap e706: 8 total, 8 up, 8 in 2024-04-12T11:25:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:11 smithi029 ceph-mon[93650]: osdmap e706: 8 total, 8 up, 8 in 2024-04-12T11:25:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:11 smithi029 ceph-mon[97787]: osdmap e706: 8 total, 8 up, 8 in 2024-04-12T11:25:12.614 INFO:tasks.rados.rados.0.smithi029.stdout:2484: done (1 left) 2024-04-12T11:25:12.614 INFO:tasks.rados.rados.0.smithi029.stdout:2486: done (0 left) 2024-04-12T11:25:12.614 INFO:tasks.rados.rados.0.smithi029.stdout:2487: snap_remove snap 318 2024-04-12T11:25:12.622 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:25:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:25:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:25:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:12 smithi087 ceph-mon[89014]: pgmap v696: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 230 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 788 KiB/s wr, 11 op/s 2024-04-12T11:25:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:12 smithi087 ceph-mon[89014]: osdmap e707: 8 total, 8 up, 8 in 2024-04-12T11:25:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:12 smithi087 ceph-mon[89014]: osdmap e708: 8 total, 8 up, 8 in 2024-04-12T11:25:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:12 smithi029 ceph-mon[93650]: pgmap v696: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 230 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 788 KiB/s wr, 11 op/s 2024-04-12T11:25:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:12 smithi029 ceph-mon[93650]: osdmap e707: 8 total, 8 up, 8 in 2024-04-12T11:25:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:12 smithi029 ceph-mon[93650]: osdmap e708: 8 total, 8 up, 8 in 2024-04-12T11:25:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:12 smithi029 ceph-mon[97787]: pgmap v696: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 230 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 788 KiB/s wr, 11 op/s 2024-04-12T11:25:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:12 smithi029 ceph-mon[97787]: osdmap e707: 8 total, 8 up, 8 in 2024-04-12T11:25:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:12 smithi029 ceph-mon[97787]: osdmap e708: 8 total, 8 up, 8 in 2024-04-12T11:25:13.615 INFO:tasks.rados.rados.0.smithi029.stdout:2487: done (0 left) 2024-04-12T11:25:13.615 INFO:tasks.rados.rados.0.smithi029.stdout:2488: read oid 30 snap -1 2024-04-12T11:25:13.615 INFO:tasks.rados.rados.0.smithi029.stdout:2488: expect deleted 2024-04-12T11:25:13.615 INFO:tasks.rados.rados.0.smithi029.stdout:2489: read oid 44 snap -1 2024-04-12T11:25:13.615 INFO:tasks.rados.rados.0.smithi029.stdout:2489: expect deleted 2024-04-12T11:25:13.615 INFO:tasks.rados.rados.0.smithi029.stdout:2490: rollback oid 20 current snap is 322 2024-04-12T11:25:13.615 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 20 to 303 2024-04-12T11:25:13.615 INFO:tasks.rados.rados.0.smithi029.stdout:2491: delete oid 23 current snap is 322 2024-04-12T11:25:13.621 INFO:tasks.rados.rados.0.smithi029.stdout:2488: done (3 left) 2024-04-12T11:25:13.621 INFO:tasks.rados.rados.0.smithi029.stdout:2489: done (2 left) 2024-04-12T11:25:13.621 INFO:tasks.rados.rados.0.smithi029.stdout:2491: done (1 left) 2024-04-12T11:25:13.621 INFO:tasks.rados.rados.0.smithi029.stdout:2492: read oid 26 snap 303 2024-04-12T11:25:13.621 INFO:tasks.rados.rados.0.smithi029.stdout:2492: expect (ObjNum 748 snap 258 seq_num 748) 2024-04-12T11:25:13.621 INFO:tasks.rados.rados.0.smithi029.stdout:2493: snap_create 2024-04-12T11:25:13.622 INFO:tasks.rados.rados.0.smithi029.stdout:2490: finishing rollback tid 0 to smithi02979277-20 2024-04-12T11:25:13.623 INFO:tasks.rados.rados.0.smithi029.stdout:2490: finishing rollback tid 1 to smithi02979277-20 2024-04-12T11:25:13.623 INFO:tasks.rados.rados.0.smithi029.stdout:2490: finishing rollback tid 2 to smithi02979277-20 2024-04-12T11:25:13.623 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 749 (ObjNum 801 snap 282 seq_num 801) dirty exists 2024-04-12T11:25:13.625 INFO:tasks.rados.rados.0.smithi029.stdout:2492: expect (ObjNum 748 snap 258 seq_num 748) 2024-04-12T11:25:14.620 INFO:tasks.rados.rados.0.smithi029.stdout:2490: done (2 left) 2024-04-12T11:25:14.620 INFO:tasks.rados.rados.0.smithi029.stdout:2492: done (1 left) 2024-04-12T11:25:14.620 INFO:tasks.rados.rados.0.smithi029.stdout:2493: done (0 left) 2024-04-12T11:25:14.620 INFO:tasks.rados.rados.0.smithi029.stdout:2494: write oid 21 current snap is 323 2024-04-12T11:25:14.620 INFO:tasks.rados.rados.0.smithi029.stdout:2494: seq_num 934 ranges {497738=477632,1772417=775656,3316134=381604} 2024-04-12T11:25:14.625 INFO:tasks.rados.rados.0.smithi029.stdout:2494: writing smithi02979277-21 from 497738 to 975370 tid 1 2024-04-12T11:25:14.634 INFO:tasks.rados.rados.0.smithi029.stdout:2494: writing smithi02979277-21 from 1772417 to 2548073 tid 2 2024-04-12T11:25:14.640 INFO:tasks.rados.rados.0.smithi029.stdout:2494: writing smithi02979277-21 from 3316134 to 3697738 tid 3 2024-04-12T11:25:14.640 INFO:tasks.rados.rados.0.smithi029.stdout:2495: write oid 30 current snap is 323 2024-04-12T11:25:14.640 INFO:tasks.rados.rados.0.smithi029.stdout:2494: finishing write tid 1 to smithi02979277-21 2024-04-12T11:25:14.641 INFO:tasks.rados.rados.0.smithi029.stdout:2494: finishing write tid 2 to smithi02979277-21 2024-04-12T11:25:14.641 INFO:tasks.rados.rados.0.smithi029.stdout:2495: seq_num 935 ranges {514545=552881,1830899=519314,2943557=475332} 2024-04-12T11:25:14.646 INFO:tasks.rados.rados.0.smithi029.stdout:2495: writing smithi02979277-30 from 514545 to 1067426 tid 1 2024-04-12T11:25:14.653 INFO:tasks.rados.rados.0.smithi029.stdout:2495: writing smithi02979277-30 from 1830899 to 2350213 tid 2 2024-04-12T11:25:14.659 INFO:tasks.rados.rados.0.smithi029.stdout:2495: writing smithi02979277-30 from 2943557 to 3418889 tid 3 2024-04-12T11:25:14.659 INFO:tasks.rados.rados.0.smithi029.stdout:2496: write oid 17 current snap is 323 2024-04-12T11:25:14.660 INFO:tasks.rados.rados.0.smithi029.stdout:2496: seq_num 936 ranges {531352=744483,2005734=779325,3487333=244019} 2024-04-12T11:25:14.666 INFO:tasks.rados.rados.0.smithi029.stdout:2496: writing smithi02979277-17 from 531352 to 1275835 tid 1 2024-04-12T11:25:14.675 INFO:tasks.rados.rados.0.smithi029.stdout:2496: writing smithi02979277-17 from 2005734 to 2785059 tid 2 2024-04-12T11:25:14.681 INFO:tasks.rados.rados.0.smithi029.stdout:2496: writing smithi02979277-17 from 3487333 to 3731352 tid 3 2024-04-12T11:25:14.681 INFO:tasks.rados.rados.0.smithi029.stdout:2497: rollback oid 46 current snap is 323 2024-04-12T11:25:14.681 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 46 to 316 2024-04-12T11:25:14.681 INFO:tasks.rados.rados.0.smithi029.stdout:2494: finishing write tid 3 to smithi02979277-21 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2494: finishing write tid 4 to smithi02979277-21 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2494: finishing write tid 5 to smithi02979277-21 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2494: oid 21 updating version 0 to 1199 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2494: oid 21 updating version 1199 to 1201 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2494: oid 21 version 1201 is already newer than 1200 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2494: oid 21 version 1201 is already newer than 1198 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1201 (ObjNum 934 snap 323 seq_num 934) dirty exists 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2494: left oid 21 (ObjNum 934 snap 323 seq_num 934) 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2494: done (3 left) 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2498: snap_create 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2495: finishing write tid 1 to smithi02979277-30 2024-04-12T11:25:14.682 INFO:tasks.rados.rados.0.smithi029.stdout:2495: finishing write tid 2 to smithi02979277-30 2024-04-12T11:25:14.683 INFO:tasks.rados.rados.0.smithi029.stdout:2495: finishing write tid 3 to smithi02979277-30 2024-04-12T11:25:14.683 INFO:tasks.rados.rados.0.smithi029.stdout:2495: finishing write tid 4 to smithi02979277-30 2024-04-12T11:25:14.683 INFO:tasks.rados.rados.0.smithi029.stdout:2495: finishing write tid 5 to smithi02979277-30 2024-04-12T11:25:14.683 INFO:tasks.rados.rados.0.smithi029.stdout:2495: oid 30 updating version 0 to 804 2024-04-12T11:25:14.683 INFO:tasks.rados.rados.0.smithi029.stdout:2495: oid 30 version 804 is already newer than 803 2024-04-12T11:25:14.683 INFO:tasks.rados.rados.0.smithi029.stdout:2495: oid 30 updating version 804 to 806 2024-04-12T11:25:14.683 INFO:tasks.rados.rados.0.smithi029.stdout:2495: oid 30 version 806 is already newer than 805 2024-04-12T11:25:14.683 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 806 (ObjNum 935 snap 323 seq_num 935) dirty exists 2024-04-12T11:25:14.684 INFO:tasks.rados.rados.0.smithi029.stdout:2495: left oid 30 (ObjNum 935 snap 323 seq_num 935) 2024-04-12T11:25:14.684 INFO:tasks.rados.rados.0.smithi029.stdout:2496: finishing write tid 1 to smithi02979277-17 2024-04-12T11:25:14.684 INFO:tasks.rados.rados.0.smithi029.stdout:2496: finishing write tid 2 to smithi02979277-17 2024-04-12T11:25:14.684 INFO:tasks.rados.rados.0.smithi029.stdout:2497: finishing rollback tid 1 to smithi02979277-46 2024-04-12T11:25:14.684 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 0 (ObjNum 305 snap 0 seq_num 1) dirty dne 2024-04-12T11:25:14.685 INFO:tasks.rados.rados.0.smithi029.stdout:2496: finishing write tid 3 to smithi02979277-17 2024-04-12T11:25:14.685 INFO:tasks.rados.rados.0.smithi029.stdout:2496: finishing write tid 4 to smithi02979277-17 2024-04-12T11:25:14.685 INFO:tasks.rados.rados.0.smithi029.stdout:2496: finishing write tid 5 to smithi02979277-17 2024-04-12T11:25:14.685 INFO:tasks.rados.rados.0.smithi029.stdout:2496: oid 17 updating version 0 to 1235 2024-04-12T11:25:14.685 INFO:tasks.rados.rados.0.smithi029.stdout:2496: oid 17 updating version 1235 to 1236 2024-04-12T11:25:14.685 INFO:tasks.rados.rados.0.smithi029.stdout:2496: oid 17 updating version 1236 to 1238 2024-04-12T11:25:14.685 INFO:tasks.rados.rados.0.smithi029.stdout:2496: oid 17 version 1238 is already newer than 1237 2024-04-12T11:25:14.685 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1238 (ObjNum 936 snap 323 seq_num 936) dirty exists 2024-04-12T11:25:14.685 INFO:tasks.rados.rados.0.smithi029.stdout:2496: left oid 17 (ObjNum 936 snap 323 seq_num 936) 2024-04-12T11:25:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:14 smithi029 ceph-mon[97787]: pgmap v699: 89 pgs: 89 active+clean; 218 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 455 KiB/s wr, 1 op/s 2024-04-12T11:25:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:14 smithi029 ceph-mon[97787]: osdmap e709: 8 total, 8 up, 8 in 2024-04-12T11:25:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:14 smithi029 ceph-mon[93650]: pgmap v699: 89 pgs: 89 active+clean; 218 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 455 KiB/s wr, 1 op/s 2024-04-12T11:25:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:14 smithi029 ceph-mon[93650]: osdmap e709: 8 total, 8 up, 8 in 2024-04-12T11:25:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:14 smithi087 ceph-mon[89014]: pgmap v699: 89 pgs: 89 active+clean; 218 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 455 KiB/s wr, 1 op/s 2024-04-12T11:25:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:14 smithi087 ceph-mon[89014]: osdmap e709: 8 total, 8 up, 8 in 2024-04-12T11:25:15.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:25:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:25:14] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-12T11:25:15.624 INFO:tasks.rados.rados.0.smithi029.stdout:2495: done (3 left) 2024-04-12T11:25:15.625 INFO:tasks.rados.rados.0.smithi029.stdout:2496: done (2 left) 2024-04-12T11:25:15.625 INFO:tasks.rados.rados.0.smithi029.stdout:2497: done (1 left) 2024-04-12T11:25:15.625 INFO:tasks.rados.rados.0.smithi029.stdout:2498: done (0 left) 2024-04-12T11:25:15.625 INFO:tasks.rados.rados.0.smithi029.stdout:2499: snap_create 2024-04-12T11:25:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:15 smithi087 ceph-mon[89014]: osdmap e710: 8 total, 8 up, 8 in 2024-04-12T11:25:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:15 smithi029 ceph-mon[97787]: osdmap e710: 8 total, 8 up, 8 in 2024-04-12T11:25:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:15 smithi029 ceph-mon[93650]: osdmap e710: 8 total, 8 up, 8 in 2024-04-12T11:25:16.626 INFO:tasks.rados.rados.0.smithi029.stdout:2499: done (0 left) 2024-04-12T11:25:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:2500: write oid 35 current snap is 325 2024-04-12T11:25:16.627 INFO:tasks.rados.rados.0.smithi029.stdout:2500: seq_num 937 ranges {548159=419732,1664216=522983,2714756=614229} 2024-04-12T11:25:16.632 INFO:tasks.rados.rados.0.smithi029.stdout:2500: writing smithi02979277-35 from 548159 to 967891 tid 1 2024-04-12T11:25:16.639 INFO:tasks.rados.rados.0.smithi029.stdout:2500: writing smithi02979277-35 from 1664216 to 2187199 tid 2 2024-04-12T11:25:16.645 INFO:tasks.rados.rados.0.smithi029.stdout:2500: writing smithi02979277-35 from 2714756 to 3328985 tid 3 2024-04-12T11:25:16.645 INFO:tasks.rados.rados.0.smithi029.stdout:2501: write oid 26 current snap is 325 2024-04-12T11:25:16.645 INFO:tasks.rados.rados.0.smithi029.stdout:2501: seq_num 938 ranges {564966=494981,1606345=782994,3025826=425501} 2024-04-12T11:25:16.651 INFO:tasks.rados.rados.0.smithi029.stdout:2501: writing smithi02979277-26 from 564966 to 1059947 tid 1 2024-04-12T11:25:16.658 INFO:tasks.rados.rados.0.smithi029.stdout:2501: writing smithi02979277-26 from 1606345 to 2389339 tid 2 2024-04-12T11:25:16.665 INFO:tasks.rados.rados.0.smithi029.stdout:2501: writing smithi02979277-26 from 3025826 to 3451327 tid 3 2024-04-12T11:25:16.665 INFO:tasks.rados.rados.0.smithi029.stdout:2502: read oid 34 snap 309 2024-04-12T11:25:16.665 INFO:tasks.rados.rados.0.smithi029.stdout:2502: expect (ObjNum 875 snap 308 seq_num 875) 2024-04-12T11:25:16.665 INFO:tasks.rados.rados.0.smithi029.stdout:2503: snap_remove snap 319 2024-04-12T11:25:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:16 smithi087 ceph-mon[89014]: pgmap v702: 89 pgs: 89 active+clean; 218 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 906 KiB/s wr, 5 op/s 2024-04-12T11:25:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:16 smithi087 ceph-mon[89014]: osdmap e711: 8 total, 8 up, 8 in 2024-04-12T11:25:17.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:16 smithi029 ceph-mon[97787]: pgmap v702: 89 pgs: 89 active+clean; 218 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 906 KiB/s wr, 5 op/s 2024-04-12T11:25:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:16 smithi029 ceph-mon[97787]: osdmap e711: 8 total, 8 up, 8 in 2024-04-12T11:25:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:16 smithi029 ceph-mon[93650]: pgmap v702: 89 pgs: 89 active+clean; 218 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 906 KiB/s wr, 5 op/s 2024-04-12T11:25:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:16 smithi029 ceph-mon[93650]: osdmap e711: 8 total, 8 up, 8 in 2024-04-12T11:25:17.616 INFO:tasks.rados.rados.0.smithi029.stdout:2503: done (3 left) 2024-04-12T11:25:17.616 INFO:tasks.rados.rados.0.smithi029.stdout:2504: rollback oid 14 current snap is 325 2024-04-12T11:25:17.616 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 317 2024-04-12T11:25:17.617 INFO:tasks.rados.rados.0.smithi029.stdout:2500: finishing write tid 1 to smithi02979277-35 2024-04-12T11:25:17.617 INFO:tasks.rados.rados.0.smithi029.stdout:2500: finishing write tid 2 to smithi02979277-35 2024-04-12T11:25:17.617 INFO:tasks.rados.rados.0.smithi029.stdout:2500: finishing write tid 3 to smithi02979277-35 2024-04-12T11:25:17.617 INFO:tasks.rados.rados.0.smithi029.stdout:2500: finishing write tid 4 to smithi02979277-35 2024-04-12T11:25:17.617 INFO:tasks.rados.rados.0.smithi029.stdout:2500: finishing write tid 5 to smithi02979277-35 2024-04-12T11:25:17.617 INFO:tasks.rados.rados.0.smithi029.stdout:2500: oid 35 updating version 0 to 1483 2024-04-12T11:25:17.617 INFO:tasks.rados.rados.0.smithi029.stdout:2500: oid 35 version 1483 is already newer than 1482 2024-04-12T11:25:17.617 INFO:tasks.rados.rados.0.smithi029.stdout:2500: oid 35 version 1483 is already newer than 1481 2024-04-12T11:25:17.617 INFO:tasks.rados.rados.0.smithi029.stdout:2500: oid 35 version 1483 is already newer than 1480 2024-04-12T11:25:17.617 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 1483 (ObjNum 937 snap 325 seq_num 937) dirty exists 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:2500: left oid 35 (ObjNum 937 snap 325 seq_num 937) 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:2501: finishing write tid 1 to smithi02979277-26 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:2501: finishing write tid 2 to smithi02979277-26 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:2500: done (3 left) 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:2505: read oid 21 snap -1 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:2505: expect (ObjNum 934 snap 323 seq_num 934) 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:2506: rollback oid 19 current snap is 325 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 19 to 321 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:2501: finishing write tid 3 to smithi02979277-26 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:2501: finishing write tid 4 to smithi02979277-26 2024-04-12T11:25:17.618 INFO:tasks.rados.rados.0.smithi029.stdout:2502: expect (ObjNum 875 snap 308 seq_num 875) 2024-04-12T11:25:17.753 INFO:tasks.rados.rados.0.smithi029.stdout:2501: finishing write tid 5 to smithi02979277-26 2024-04-12T11:25:17.753 INFO:tasks.rados.rados.0.smithi029.stdout:2501: oid 26 updating version 0 to 1486 2024-04-12T11:25:17.753 INFO:tasks.rados.rados.0.smithi029.stdout:2501: oid 26 updating version 1486 to 1488 2024-04-12T11:25:17.754 INFO:tasks.rados.rados.0.smithi029.stdout:2501: oid 26 version 1488 is already newer than 1487 2024-04-12T11:25:17.754 INFO:tasks.rados.rados.0.smithi029.stdout:2501: oid 26 version 1488 is already newer than 1485 2024-04-12T11:25:17.754 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 1488 (ObjNum 938 snap 325 seq_num 938) dirty exists 2024-04-12T11:25:17.754 INFO:tasks.rados.rados.0.smithi029.stdout:2501: left oid 26 (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:25:17.754 INFO:tasks.rados.rados.0.smithi029.stdout:2501: done (4 left) 2024-04-12T11:25:17.755 INFO:tasks.rados.rados.0.smithi029.stdout:2502: done (3 left) 2024-04-12T11:25:17.755 INFO:tasks.rados.rados.0.smithi029.stdout:2507: write oid 18 current snap is 325 2024-04-12T11:25:17.756 INFO:tasks.rados.rados.0.smithi029.stdout:2504: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:25:17.756 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 0 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-04-12T11:25:17.756 INFO:tasks.rados.rados.0.smithi029.stdout:2506: finishing rollback tid 0 to smithi02979277-19 2024-04-12T11:25:17.756 INFO:tasks.rados.rados.0.smithi029.stdout:2507: seq_num 939 ranges {581773=570230,1664827=526652,2653249=636773} 2024-04-12T11:25:17.759 INFO:tasks.rados.rados.0.smithi029.stdout:2507: writing smithi02979277-18 from 581773 to 1152003 tid 1 2024-04-12T11:25:17.765 INFO:tasks.rados.rados.0.smithi029.stdout:2507: writing smithi02979277-18 from 1664827 to 2191479 tid 2 2024-04-12T11:25:17.772 INFO:tasks.rados.rados.0.smithi029.stdout:2507: writing smithi02979277-18 from 2653249 to 3290022 tid 3 2024-04-12T11:25:17.772 INFO:tasks.rados.rados.0.smithi029.stdout:2504: done (3 left) 2024-04-12T11:25:17.772 INFO:tasks.rados.rados.0.smithi029.stdout:2508: read oid 28 snap -1 2024-04-12T11:25:17.772 INFO:tasks.rados.rados.0.smithi029.stdout:2508: expect (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:25:17.773 INFO:tasks.rados.rados.0.smithi029.stdout:2509: read oid 49 snap -1 2024-04-12T11:25:17.773 INFO:tasks.rados.rados.0.smithi029.stdout:2509: expect deleted 2024-04-12T11:25:17.773 INFO:tasks.rados.rados.0.smithi029.stdout:2510: delete oid 1 current snap is 325 2024-04-12T11:25:17.773 INFO:tasks.rados.rados.0.smithi029.stdout:2506: finishing rollback tid 1 to smithi02979277-19 2024-04-12T11:25:17.774 INFO:tasks.rados.rados.0.smithi029.stdout:2506: finishing rollback tid 2 to smithi02979277-19 2024-04-12T11:25:17.774 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1152 (ObjNum 906 snap 314 seq_num 906) dirty exists 2024-04-12T11:25:17.774 INFO:tasks.rados.rados.0.smithi029.stdout:2505: expect (ObjNum 934 snap 323 seq_num 934) 2024-04-12T11:25:17.897 INFO:tasks.rados.rados.0.smithi029.stdout:2507: finishing write tid 1 to smithi02979277-18 2024-04-12T11:25:17.897 INFO:tasks.rados.rados.0.smithi029.stdout:2507: finishing write tid 2 to smithi02979277-18 2024-04-12T11:25:17.897 INFO:tasks.rados.rados.0.smithi029.stdout:2505: done (5 left) 2024-04-12T11:25:17.897 INFO:tasks.rados.rados.0.smithi029.stdout:2506: done (4 left) 2024-04-12T11:25:17.897 INFO:tasks.rados.rados.0.smithi029.stdout:2510: done (3 left) 2024-04-12T11:25:17.897 INFO:tasks.rados.rados.0.smithi029.stdout:2511: write oid 21 current snap is 325 2024-04-12T11:25:17.897 INFO:tasks.rados.rados.0.smithi029.stdout:2507: finishing write tid 3 to smithi02979277-18 2024-04-12T11:25:17.897 INFO:tasks.rados.rados.0.smithi029.stdout:2507: finishing write tid 4 to smithi02979277-18 2024-04-12T11:25:17.897 INFO:tasks.rados.rados.0.smithi029.stdout:2511: seq_num 941 ranges {615387=720728,1781791=530321,3108095=707292} 2024-04-12T11:25:17.903 INFO:tasks.rados.rados.0.smithi029.stdout:2511: writing smithi02979277-21 from 615387 to 1336115 tid 1 2024-04-12T11:25:17.909 INFO:tasks.rados.rados.0.smithi029.stdout:2511: writing smithi02979277-21 from 1781791 to 2312112 tid 2 2024-04-12T11:25:17.918 INFO:tasks.rados.rados.0.smithi029.stdout:2511: writing smithi02979277-21 from 3108095 to 3815387 tid 3 2024-04-12T11:25:17.918 INFO:tasks.rados.rados.0.smithi029.stdout:2509: done (3 left) 2024-04-12T11:25:17.918 INFO:tasks.rados.rados.0.smithi029.stdout:2512: delete oid 10 current snap is 325 2024-04-12T11:25:17.918 INFO:tasks.rados.rados.0.smithi029.stdout:2507: finishing write tid 5 to smithi02979277-18 2024-04-12T11:25:17.918 INFO:tasks.rados.rados.0.smithi029.stdout:2507: oid 18 updating version 0 to 1245 2024-04-12T11:25:17.918 INFO:tasks.rados.rados.0.smithi029.stdout:2507: oid 18 updating version 1245 to 1246 2024-04-12T11:25:17.918 INFO:tasks.rados.rados.0.smithi029.stdout:2507: oid 18 updating version 1246 to 1247 2024-04-12T11:25:17.919 INFO:tasks.rados.rados.0.smithi029.stdout:2507: oid 18 version 1247 is already newer than 1240 2024-04-12T11:25:17.919 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1247 (ObjNum 939 snap 325 seq_num 939) dirty exists 2024-04-12T11:25:17.919 INFO:tasks.rados.rados.0.smithi029.stdout:2507: left oid 18 (ObjNum 939 snap 325 seq_num 939) 2024-04-12T11:25:17.919 INFO:tasks.rados.rados.0.smithi029.stdout:2508: expect (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:25:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:17 smithi087 ceph-mon[89014]: osdmap e712: 8 total, 8 up, 8 in 2024-04-12T11:25:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:17 smithi087 ceph-mon[89014]: osdmap e713: 8 total, 8 up, 8 in 2024-04-12T11:25:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:17 smithi029 ceph-mon[97787]: osdmap e712: 8 total, 8 up, 8 in 2024-04-12T11:25:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:17 smithi029 ceph-mon[97787]: osdmap e713: 8 total, 8 up, 8 in 2024-04-12T11:25:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:17 smithi029 ceph-mon[93650]: osdmap e712: 8 total, 8 up, 8 in 2024-04-12T11:25:18.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:17 smithi029 ceph-mon[93650]: osdmap e713: 8 total, 8 up, 8 in 2024-04-12T11:25:18.046 INFO:tasks.rados.rados.0.smithi029.stdout:2511: finishing write tid 1 to smithi02979277-21 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2511: finishing write tid 2 to smithi02979277-21 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2507: done (3 left) 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2508: done (2 left) 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2512: done (1 left) 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2513: delete oid 50 current snap is 325 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2511: finishing write tid 3 to smithi02979277-21 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2511: finishing write tid 4 to smithi02979277-21 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2511: finishing write tid 5 to smithi02979277-21 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2511: oid 21 updating version 0 to 1206 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2511: oid 21 updating version 1206 to 1207 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2511: oid 21 version 1207 is already newer than 1204 2024-04-12T11:25:18.047 INFO:tasks.rados.rados.0.smithi029.stdout:2511: oid 21 version 1207 is already newer than 1205 2024-04-12T11:25:18.048 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1207 (ObjNum 941 snap 325 seq_num 941) dirty exists 2024-04-12T11:25:18.048 INFO:tasks.rados.rados.0.smithi029.stdout:2511: left oid 21 (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:25:18.049 INFO:tasks.rados.rados.0.smithi029.stdout:2511: done (1 left) 2024-04-12T11:25:18.049 INFO:tasks.rados.rados.0.smithi029.stdout:2513: done (0 left) 2024-04-12T11:25:18.050 INFO:tasks.rados.rados.0.smithi029.stdout:2514: write oid 36 current snap is 325 2024-04-12T11:25:18.050 INFO:tasks.rados.rados.0.smithi029.stdout:2514: seq_num 944 ranges {665808=662828,1957237=677648,3074011=609486} 2024-04-12T11:25:18.056 INFO:tasks.rados.rados.0.smithi029.stdout:2514: writing smithi02979277-36 from 665808 to 1328636 tid 1 2024-04-12T11:25:18.063 INFO:tasks.rados.rados.0.smithi029.stdout:2514: writing smithi02979277-36 from 1957237 to 2634885 tid 2 2024-04-12T11:25:18.069 INFO:tasks.rados.rados.0.smithi029.stdout:2514: writing smithi02979277-36 from 3074011 to 3683497 tid 3 2024-04-12T11:25:18.069 INFO:tasks.rados.rados.0.smithi029.stdout:2515: delete oid 13 current snap is 325 2024-04-12T11:25:18.069 INFO:tasks.rados.rados.0.smithi029.stdout:2514: finishing write tid 1 to smithi02979277-36 2024-04-12T11:25:18.069 INFO:tasks.rados.rados.0.smithi029.stdout:2514: finishing write tid 2 to smithi02979277-36 2024-04-12T11:25:18.072 INFO:tasks.rados.rados.0.smithi029.stdout:2515: done (1 left) 2024-04-12T11:25:18.072 INFO:tasks.rados.rados.0.smithi029.stdout:2516: read oid 45 snap -1 2024-04-12T11:25:18.072 INFO:tasks.rados.rados.0.smithi029.stdout:2516: expect deleted 2024-04-12T11:25:18.072 INFO:tasks.rados.rados.0.smithi029.stdout:2517: snap_remove snap 303 2024-04-12T11:25:18.642 INFO:tasks.rados.rados.0.smithi029.stdout:2517: done (2 left) 2024-04-12T11:25:18.643 INFO:tasks.rados.rados.0.smithi029.stdout:2518: delete oid 34 current snap is 325 2024-04-12T11:25:18.643 INFO:tasks.rados.rados.0.smithi029.stdout:2514: finishing write tid 3 to smithi02979277-36 2024-04-12T11:25:18.643 INFO:tasks.rados.rados.0.smithi029.stdout:2514: finishing write tid 4 to smithi02979277-36 2024-04-12T11:25:18.643 INFO:tasks.rados.rados.0.smithi029.stdout:2514: finishing write tid 5 to smithi02979277-36 2024-04-12T11:25:18.643 INFO:tasks.rados.rados.0.smithi029.stdout:2514: oid 36 updating version 0 to 665 2024-04-12T11:25:18.643 INFO:tasks.rados.rados.0.smithi029.stdout:2514: oid 36 version 665 is already newer than 664 2024-04-12T11:25:18.643 INFO:tasks.rados.rados.0.smithi029.stdout:2514: oid 36 updating version 665 to 666 2024-04-12T11:25:18.643 INFO:tasks.rados.rados.0.smithi029.stdout:2514: oid 36 updating version 666 to 667 2024-04-12T11:25:18.643 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 667 (ObjNum 944 snap 325 seq_num 944) dirty exists 2024-04-12T11:25:18.643 INFO:tasks.rados.rados.0.smithi029.stdout:2514: left oid 36 (ObjNum 944 snap 325 seq_num 944) 2024-04-12T11:25:18.650 INFO:tasks.rados.rados.0.smithi029.stdout:2514: done (2 left) 2024-04-12T11:25:18.650 INFO:tasks.rados.rados.0.smithi029.stdout:2516: done (1 left) 2024-04-12T11:25:18.651 INFO:tasks.rados.rados.0.smithi029.stdout:2518: done (0 left) 2024-04-12T11:25:18.651 INFO:tasks.rados.rados.0.smithi029.stdout:2519: snap_create 2024-04-12T11:25:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:18 smithi087 ceph-mon[89014]: pgmap v705: 89 pgs: 89 active+clean; 223 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-04-12T11:25:19.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:18 smithi029 ceph-mon[93650]: pgmap v705: 89 pgs: 89 active+clean; 223 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-04-12T11:25:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:18 smithi029 ceph-mon[97787]: pgmap v705: 89 pgs: 89 active+clean; 223 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-04-12T11:25:19.648 INFO:tasks.rados.rados.0.smithi029.stdout:2519: done (0 left) 2024-04-12T11:25:19.648 INFO:tasks.rados.rados.0.smithi029.stdout:2520: delete oid 11 current snap is 326 2024-04-12T11:25:19.652 INFO:tasks.rados.rados.0.smithi029.stdout:2520: done (0 left) 2024-04-12T11:25:19.652 INFO:tasks.rados.rados.0.smithi029.stdout:2521: read oid 13 snap -1 2024-04-12T11:25:19.652 INFO:tasks.rados.rados.0.smithi029.stdout:2521: expect deleted 2024-04-12T11:25:19.653 INFO:tasks.rados.rados.0.smithi029.stdout:2522: snap_create 2024-04-12T11:25:19.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:19 smithi087 ceph-mon[89014]: osdmap e714: 8 total, 8 up, 8 in 2024-04-12T11:25:20.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:19 smithi029 ceph-mon[93650]: osdmap e714: 8 total, 8 up, 8 in 2024-04-12T11:25:20.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:19 smithi029 ceph-mon[97787]: osdmap e714: 8 total, 8 up, 8 in 2024-04-12T11:25:20.655 INFO:tasks.rados.rados.0.smithi029.stdout:2521: done (1 left) 2024-04-12T11:25:20.655 INFO:tasks.rados.rados.0.smithi029.stdout:2522: done (0 left) 2024-04-12T11:25:20.655 INFO:tasks.rados.rados.0.smithi029.stdout:2523: write oid 7 current snap is 327 2024-04-12T11:25:20.655 INFO:tasks.rados.rados.0.smithi029.stdout:2523: seq_num 948 ranges {733036=563824,2074812=684986,3183703=749333} 2024-04-12T11:25:20.662 INFO:tasks.rados.rados.0.smithi029.stdout:2523: writing smithi02979277-7 from 733036 to 1296860 tid 1 2024-04-12T11:25:20.670 INFO:tasks.rados.rados.0.smithi029.stdout:2523: writing smithi02979277-7 from 2074812 to 2759798 tid 2 2024-04-12T11:25:20.677 INFO:tasks.rados.rados.0.smithi029.stdout:2523: writing smithi02979277-7 from 3183703 to 3933036 tid 3 2024-04-12T11:25:20.677 INFO:tasks.rados.rados.0.smithi029.stdout:2524: read oid 10 snap -1 2024-04-12T11:25:20.677 INFO:tasks.rados.rados.0.smithi029.stdout:2524: expect deleted 2024-04-12T11:25:20.677 INFO:tasks.rados.rados.0.smithi029.stdout:2525: snap_remove snap 324 2024-04-12T11:25:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:20 smithi087 ceph-mon[89014]: pgmap v708: 89 pgs: 89 active+clean; 233 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-04-12T11:25:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:20 smithi087 ceph-mon[89014]: osdmap e715: 8 total, 8 up, 8 in 2024-04-12T11:25:21.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:20 smithi029 ceph-mon[93650]: pgmap v708: 89 pgs: 89 active+clean; 233 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-04-12T11:25:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:20 smithi029 ceph-mon[93650]: osdmap e715: 8 total, 8 up, 8 in 2024-04-12T11:25:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:20 smithi029 ceph-mon[97787]: pgmap v708: 89 pgs: 89 active+clean; 233 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-04-12T11:25:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:20 smithi029 ceph-mon[97787]: osdmap e715: 8 total, 8 up, 8 in 2024-04-12T11:25:21.659 INFO:tasks.rados.rados.0.smithi029.stdout:2525: done (2 left) 2024-04-12T11:25:21.659 INFO:tasks.rados.rados.0.smithi029.stdout:2526: snap_remove snap 275 2024-04-12T11:25:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:21 smithi087 ceph-mon[89014]: osdmap e716: 8 total, 8 up, 8 in 2024-04-12T11:25:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:22.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:21 smithi029 ceph-mon[93650]: osdmap e716: 8 total, 8 up, 8 in 2024-04-12T11:25:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:21 smithi029 ceph-mon[97787]: osdmap e716: 8 total, 8 up, 8 in 2024-04-12T11:25:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:22.618 INFO:tasks.rados.rados.0.smithi029.stdout:2526: done (2 left) 2024-04-12T11:25:22.618 INFO:tasks.rados.rados.0.smithi029.stdout:2527: write oid 34 current snap is 327 2024-04-12T11:25:22.618 INFO:tasks.rados.rados.0.smithi029.stdout:2527: seq_num 949 ranges {749843=639073,2133294=544997,3211126=582199} 2024-04-12T11:25:22.626 INFO:tasks.rados.rados.0.smithi029.stdout:2527: writing smithi02979277-34 from 749843 to 1388916 tid 1 2024-04-12T11:25:22.633 INFO:tasks.rados.rados.0.smithi029.stdout:2527: writing smithi02979277-34 from 2133294 to 2678291 tid 2 2024-04-12T11:25:22.640 INFO:tasks.rados.rados.0.smithi029.stdout:2527: writing smithi02979277-34 from 3211126 to 3793325 tid 3 2024-04-12T11:25:22.641 INFO:tasks.rados.rados.0.smithi029.stdout:2528: write oid 20 current snap is 327 2024-04-12T11:25:22.641 INFO:tasks.rados.rados.0.smithi029.stdout:2523: finishing write tid 1 to smithi02979277-7 2024-04-12T11:25:22.641 INFO:tasks.rados.rados.0.smithi029.stdout:2528: seq_num 950 ranges {766650=714322,2075423=688655,3522196=444454} 2024-04-12T11:25:22.648 INFO:tasks.rados.rados.0.smithi029.stdout:2528: writing smithi02979277-20 from 766650 to 1480972 tid 1 2024-04-12T11:25:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2528: writing smithi02979277-20 from 2075423 to 2764078 tid 2 2024-04-12T11:25:22.657 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:25:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:25:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:25:22.663 INFO:tasks.rados.rados.0.smithi029.stdout:2528: writing smithi02979277-20 from 3522196 to 3966650 tid 3 2024-04-12T11:25:22.663 INFO:tasks.rados.rados.0.smithi029.stdout:2529: read oid 48 snap -1 2024-04-12T11:25:22.663 INFO:tasks.rados.rados.0.smithi029.stdout:2529: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:25:22.663 INFO:tasks.rados.rados.0.smithi029.stdout:2530: snap_remove snap 314 2024-04-12T11:25:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:22 smithi087 ceph-mon[89014]: pgmap v711: 89 pgs: 89 active+clean; 237 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-04-12T11:25:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:22 smithi087 ceph-mon[89014]: osdmap e717: 8 total, 8 up, 8 in 2024-04-12T11:25:22.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:22 smithi087 ceph-mon[89014]: osdmap e718: 8 total, 8 up, 8 in 2024-04-12T11:25:23.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:22 smithi029 ceph-mon[93650]: pgmap v711: 89 pgs: 89 active+clean; 237 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-04-12T11:25:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:22 smithi029 ceph-mon[93650]: osdmap e717: 8 total, 8 up, 8 in 2024-04-12T11:25:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:22 smithi029 ceph-mon[93650]: osdmap e718: 8 total, 8 up, 8 in 2024-04-12T11:25:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:22 smithi029 ceph-mon[97787]: pgmap v711: 89 pgs: 89 active+clean; 237 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-04-12T11:25:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:22 smithi029 ceph-mon[97787]: osdmap e717: 8 total, 8 up, 8 in 2024-04-12T11:25:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:22 smithi029 ceph-mon[97787]: osdmap e718: 8 total, 8 up, 8 in 2024-04-12T11:25:23.665 INFO:tasks.rados.rados.0.smithi029.stdout:2530: done (5 left) 2024-04-12T11:25:23.665 INFO:tasks.rados.rados.0.smithi029.stdout:2531: write oid 16 current snap is 327 2024-04-12T11:25:23.666 INFO:tasks.rados.rados.0.smithi029.stdout:2531: seq_num 951 ranges {783457=505924,1850258=548666,2865972=721096} 2024-04-12T11:25:23.673 INFO:tasks.rados.rados.0.smithi029.stdout:2531: writing smithi02979277-16 from 783457 to 1289381 tid 1 2024-04-12T11:25:23.679 INFO:tasks.rados.rados.0.smithi029.stdout:2531: writing smithi02979277-16 from 1850258 to 2398924 tid 2 2024-04-12T11:25:23.686 INFO:tasks.rados.rados.0.smithi029.stdout:2531: writing smithi02979277-16 from 2865972 to 3587068 tid 3 2024-04-12T11:25:23.686 INFO:tasks.rados.rados.0.smithi029.stdout:2532: write oid 30 current snap is 327 2024-04-12T11:25:23.686 INFO:tasks.rados.rados.0.smithi029.stdout:2532: seq_num 952 ranges {400264=581173,1508740=491524} 2024-04-12T11:25:23.691 INFO:tasks.rados.rados.0.smithi029.stdout:2532: writing smithi02979277-30 from 400264 to 981437 tid 1 2024-04-12T11:25:23.697 INFO:tasks.rados.rados.0.smithi029.stdout:2532: writing smithi02979277-30 from 1508740 to 2000264 tid 2 2024-04-12T11:25:23.697 INFO:tasks.rados.rados.0.smithi029.stdout:2533: snap_remove snap 320 2024-04-12T11:25:24.672 INFO:tasks.rados.rados.0.smithi029.stdout:2533: done (7 left) 2024-04-12T11:25:24.672 INFO:tasks.rados.rados.0.smithi029.stdout:2534: read oid 37 snap -1 2024-04-12T11:25:24.672 INFO:tasks.rados.rados.0.smithi029.stdout:2534: expect (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:25:24.672 INFO:tasks.rados.rados.0.smithi029.stdout:2535: read oid 1 snap 307 2024-04-12T11:25:24.672 INFO:tasks.rados.rados.0.smithi029.stdout:2535: expect (ObjNum 805 snap 282 seq_num 805) 2024-04-12T11:25:24.672 INFO:tasks.rados.rados.0.smithi029.stdout:2536: write oid 43 current snap is 327 2024-04-12T11:25:24.672 INFO:tasks.rados.rados.0.smithi029.stdout:2536: seq_num 953 ranges {417071=656422,1567222=449849} 2024-04-12T11:25:24.677 INFO:tasks.rados.rados.0.smithi029.stdout:2536: writing smithi02979277-43 from 417071 to 1073493 tid 1 2024-04-12T11:25:24.684 INFO:tasks.rados.rados.0.smithi029.stdout:2536: writing smithi02979277-43 from 1567222 to 2017071 tid 2 2024-04-12T11:25:24.684 INFO:tasks.rados.rados.0.smithi029.stdout:2537: write oid 50 current snap is 327 2024-04-12T11:25:24.684 INFO:tasks.rados.rados.0.smithi029.stdout:2537: seq_num 954 ranges {433878=731671,1909351=124527} 2024-04-12T11:25:24.691 INFO:tasks.rados.rados.0.smithi029.stdout:2537: writing smithi02979277-50 from 433878 to 1165549 tid 1 2024-04-12T11:25:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2537: writing smithi02979277-50 from 1909351 to 2033878 tid 2 2024-04-12T11:25:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2538: delete oid 12 current snap is 327 2024-04-12T11:25:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2523: finishing write tid 2 to smithi02979277-7 2024-04-12T11:25:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2523: finishing write tid 3 to smithi02979277-7 2024-04-12T11:25:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2523: finishing write tid 4 to smithi02979277-7 2024-04-12T11:25:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2523: finishing write tid 5 to smithi02979277-7 2024-04-12T11:25:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2523: oid 7 updating version 0 to 1209 2024-04-12T11:25:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2523: oid 7 version 1209 is already newer than 1208 2024-04-12T11:25:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2523: oid 7 updating version 1209 to 1210 2024-04-12T11:25:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2523: oid 7 updating version 1210 to 1211 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1211 (ObjNum 948 snap 327 seq_num 948) dirty exists 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2523: left oid 7 (ObjNum 948 snap 327 seq_num 948) 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2527: finishing write tid 1 to smithi02979277-34 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2527: finishing write tid 2 to smithi02979277-34 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2527: finishing write tid 3 to smithi02979277-34 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2527: finishing write tid 4 to smithi02979277-34 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2527: finishing write tid 5 to smithi02979277-34 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2527: oid 34 updating version 0 to 1110 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2527: oid 34 version 1110 is already newer than 1109 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2527: oid 34 updating version 1110 to 1111 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2527: oid 34 version 1111 is already newer than 1104 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1111 (ObjNum 949 snap 327 seq_num 949) dirty exists 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2527: left oid 34 (ObjNum 949 snap 327 seq_num 949) 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2528: finishing write tid 1 to smithi02979277-20 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2528: finishing write tid 2 to smithi02979277-20 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2528: finishing write tid 3 to smithi02979277-20 2024-04-12T11:25:24.697 INFO:tasks.rados.rados.0.smithi029.stdout:2528: finishing write tid 4 to smithi02979277-20 2024-04-12T11:25:24.698 INFO:tasks.rados.rados.0.smithi029.stdout:2528: finishing write tid 5 to smithi02979277-20 2024-04-12T11:25:24.698 INFO:tasks.rados.rados.0.smithi029.stdout:2528: oid 20 updating version 0 to 758 2024-04-12T11:25:24.698 INFO:tasks.rados.rados.0.smithi029.stdout:2528: oid 20 version 758 is already newer than 757 2024-04-12T11:25:24.698 INFO:tasks.rados.rados.0.smithi029.stdout:2528: oid 20 version 758 is already newer than 756 2024-04-12T11:25:24.698 INFO:tasks.rados.rados.0.smithi029.stdout:2528: oid 20 version 758 is already newer than 755 2024-04-12T11:25:24.698 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 758 (ObjNum 950 snap 327 seq_num 950) dirty exists 2024-04-12T11:25:24.698 INFO:tasks.rados.rados.0.smithi029.stdout:2528: left oid 20 (ObjNum 950 snap 327 seq_num 950) 2024-04-12T11:25:24.698 INFO:tasks.rados.rados.0.smithi029.stdout:2529: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:2531: finishing write tid 1 to smithi02979277-16 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:2531: finishing write tid 2 to smithi02979277-16 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:2531: finishing write tid 3 to smithi02979277-16 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:2531: finishing write tid 4 to smithi02979277-16 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:2531: finishing write tid 5 to smithi02979277-16 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:2531: oid 16 updating version 0 to 1166 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:2531: oid 16 updating version 1166 to 1168 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:2531: oid 16 version 1168 is already newer than 1165 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:2531: oid 16 version 1168 is already newer than 1167 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1168 (ObjNum 951 snap 327 seq_num 951) dirty exists 2024-04-12T11:25:24.865 INFO:tasks.rados.rados.0.smithi029.stdout:2531: left oid 16 (ObjNum 951 snap 327 seq_num 951) 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2532: finishing write tid 1 to smithi02979277-30 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2532: finishing write tid 2 to smithi02979277-30 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2532: finishing write tid 3 to smithi02979277-30 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2523: done (11 left) 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2524: done (10 left) 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2527: done (9 left) 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2528: done (8 left) 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2529: done (7 left) 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2531: done (6 left) 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2538: done (5 left) 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2539: read oid 28 snap 309 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2539: expect (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:25:24.866 INFO:tasks.rados.rados.0.smithi029.stdout:2540: delete oid 25 current snap is 327 2024-04-12T11:25:24.867 INFO:tasks.rados.rados.0.smithi029.stdout:2532: finishing write tid 4 to smithi02979277-30 2024-04-12T11:25:24.867 INFO:tasks.rados.rados.0.smithi029.stdout:2532: oid 30 updating version 0 to 812 2024-04-12T11:25:24.867 INFO:tasks.rados.rados.0.smithi029.stdout:2532: oid 30 updating version 812 to 814 2024-04-12T11:25:24.867 INFO:tasks.rados.rados.0.smithi029.stdout:2532: oid 30 version 814 is already newer than 813 2024-04-12T11:25:24.867 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 814 (ObjNum 952 snap 327 seq_num 952) dirty exists 2024-04-12T11:25:24.867 INFO:tasks.rados.rados.0.smithi029.stdout:2532: left oid 30 (ObjNum 952 snap 327 seq_num 952) 2024-04-12T11:25:24.867 INFO:tasks.rados.rados.0.smithi029.stdout:2534: expect (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:25:24.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:24 smithi029 ceph-mon[97787]: pgmap v714: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-04-12T11:25:24.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:24 smithi029 ceph-mon[97787]: osdmap e719: 8 total, 8 up, 8 in 2024-04-12T11:25:24.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:24 smithi029 ceph-mon[93650]: pgmap v714: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-04-12T11:25:24.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:24 smithi029 ceph-mon[93650]: osdmap e719: 8 total, 8 up, 8 in 2024-04-12T11:25:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:24 smithi087 ceph-mon[89014]: pgmap v714: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-04-12T11:25:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:24 smithi087 ceph-mon[89014]: osdmap e719: 8 total, 8 up, 8 in 2024-04-12T11:25:25.075 INFO:tasks.rados.rados.0.smithi029.stdout:2535: expect (ObjNum 805 snap 282 seq_num 805) 2024-04-12T11:25:25.192 INFO:tasks.rados.rados.0.smithi029.stdout:2536: finishing write tid 1 to smithi02979277-43 2024-04-12T11:25:25.192 INFO:tasks.rados.rados.0.smithi029.stdout:2536: finishing write tid 2 to smithi02979277-43 2024-04-12T11:25:25.192 INFO:tasks.rados.rados.0.smithi029.stdout:2536: finishing write tid 3 to smithi02979277-43 2024-04-12T11:25:25.192 INFO:tasks.rados.rados.0.smithi029.stdout:2536: finishing write tid 4 to smithi02979277-43 2024-04-12T11:25:25.192 INFO:tasks.rados.rados.0.smithi029.stdout:2536: oid 43 updating version 0 to 1121 2024-04-12T11:25:25.192 INFO:tasks.rados.rados.0.smithi029.stdout:2536: oid 43 version 1121 is already newer than 1120 2024-04-12T11:25:25.192 INFO:tasks.rados.rados.0.smithi029.stdout:2536: oid 43 version 1121 is already newer than 1119 2024-04-12T11:25:25.192 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1121 (ObjNum 953 snap 327 seq_num 953) dirty exists 2024-04-12T11:25:25.192 INFO:tasks.rados.rados.0.smithi029.stdout:2536: left oid 43 (ObjNum 953 snap 327 seq_num 953) 2024-04-12T11:25:25.193 INFO:tasks.rados.rados.0.smithi029.stdout:2537: finishing write tid 1 to smithi02979277-50 2024-04-12T11:25:25.193 INFO:tasks.rados.rados.0.smithi029.stdout:2532: done (6 left) 2024-04-12T11:25:25.193 INFO:tasks.rados.rados.0.smithi029.stdout:2534: done (5 left) 2024-04-12T11:25:25.193 INFO:tasks.rados.rados.0.smithi029.stdout:2535: done (4 left) 2024-04-12T11:25:25.193 INFO:tasks.rados.rados.0.smithi029.stdout:2536: done (3 left) 2024-04-12T11:25:25.193 INFO:tasks.rados.rados.0.smithi029.stdout:2540: done (2 left) 2024-04-12T11:25:25.193 INFO:tasks.rados.rados.0.smithi029.stdout:2541: read oid 44 snap 323 2024-04-12T11:25:25.193 INFO:tasks.rados.rados.0.smithi029.stdout:2541: expect deleted 2024-04-12T11:25:25.193 INFO:tasks.rados.rados.0.smithi029.stdout:2542: snap_remove snap 306 2024-04-12T11:25:25.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:25:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:25:24] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2024-04-12T11:25:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:2542: done (3 left) 2024-04-12T11:25:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:2543: snap_create 2024-04-12T11:25:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:2537: finishing write tid 2 to smithi02979277-50 2024-04-12T11:25:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:2537: finishing write tid 3 to smithi02979277-50 2024-04-12T11:25:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:2537: finishing write tid 4 to smithi02979277-50 2024-04-12T11:25:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:2537: oid 50 updating version 0 to 1061 2024-04-12T11:25:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:2537: oid 50 version 1061 is already newer than 1060 2024-04-12T11:25:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:2537: oid 50 updating version 1061 to 1062 2024-04-12T11:25:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1062 (ObjNum 954 snap 327 seq_num 954) dirty exists 2024-04-12T11:25:25.677 INFO:tasks.rados.rados.0.smithi029.stdout:2537: left oid 50 (ObjNum 954 snap 327 seq_num 954) 2024-04-12T11:25:25.678 INFO:tasks.rados.rados.0.smithi029.stdout:2539: expect (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:25:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:25 smithi087 ceph-mon[89014]: osdmap e720: 8 total, 8 up, 8 in 2024-04-12T11:25:26.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:25 smithi029 ceph-mon[93650]: osdmap e720: 8 total, 8 up, 8 in 2024-04-12T11:25:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:25 smithi029 ceph-mon[97787]: osdmap e720: 8 total, 8 up, 8 in 2024-04-12T11:25:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:2537: done (3 left) 2024-04-12T11:25:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:2539: done (2 left) 2024-04-12T11:25:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:2541: done (1 left) 2024-04-12T11:25:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:2543: done (0 left) 2024-04-12T11:25:26.677 INFO:tasks.rados.rados.0.smithi029.stdout:2544: snap_create 2024-04-12T11:25:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:26 smithi087 ceph-mon[89014]: pgmap v717: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 240 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-04-12T11:25:26.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:26 smithi087 ceph-mon[89014]: osdmap e721: 8 total, 8 up, 8 in 2024-04-12T11:25:27.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:26 smithi029 ceph-mon[93650]: pgmap v717: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 240 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-04-12T11:25:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:26 smithi029 ceph-mon[93650]: osdmap e721: 8 total, 8 up, 8 in 2024-04-12T11:25:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:26 smithi029 ceph-mon[97787]: pgmap v717: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 240 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-04-12T11:25:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:26 smithi029 ceph-mon[97787]: osdmap e721: 8 total, 8 up, 8 in 2024-04-12T11:25:27.617 INFO:tasks.rados.rados.0.smithi029.stdout:2544: done (0 left) 2024-04-12T11:25:27.617 INFO:tasks.rados.rados.0.smithi029.stdout:2545: snap_remove snap 307 2024-04-12T11:25:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:27 smithi087 ceph-mon[89014]: osdmap e722: 8 total, 8 up, 8 in 2024-04-12T11:25:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:27 smithi087 ceph-mon[89014]: osdmap e723: 8 total, 8 up, 8 in 2024-04-12T11:25:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:27 smithi029 ceph-mon[93650]: osdmap e722: 8 total, 8 up, 8 in 2024-04-12T11:25:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:27 smithi029 ceph-mon[93650]: osdmap e723: 8 total, 8 up, 8 in 2024-04-12T11:25:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:27 smithi029 ceph-mon[97787]: osdmap e722: 8 total, 8 up, 8 in 2024-04-12T11:25:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:27 smithi029 ceph-mon[97787]: osdmap e723: 8 total, 8 up, 8 in 2024-04-12T11:25:28.620 INFO:tasks.rados.rados.0.smithi029.stdout:2545: done (0 left) 2024-04-12T11:25:28.620 INFO:tasks.rados.rados.0.smithi029.stdout:2546: delete oid 4 current snap is 329 2024-04-12T11:25:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:2546: done (0 left) 2024-04-12T11:25:28.624 INFO:tasks.rados.rados.0.smithi029.stdout:2547: snap_remove snap 296 2024-04-12T11:25:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:28 smithi087 ceph-mon[89014]: pgmap v720: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 222 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-04-12T11:25:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:28 smithi087 ceph-mon[89014]: osdmap e724: 8 total, 8 up, 8 in 2024-04-12T11:25:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:28 smithi029 ceph-mon[97787]: pgmap v720: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 222 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-04-12T11:25:29.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:28 smithi029 ceph-mon[97787]: osdmap e724: 8 total, 8 up, 8 in 2024-04-12T11:25:29.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:28 smithi029 ceph-mon[93650]: pgmap v720: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 222 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-04-12T11:25:29.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:28 smithi029 ceph-mon[93650]: osdmap e724: 8 total, 8 up, 8 in 2024-04-12T11:25:29.624 INFO:tasks.rados.rados.0.smithi029.stdout:2547: done (0 left) 2024-04-12T11:25:29.625 INFO:tasks.rados.rados.0.smithi029.stdout:2548: rollback oid 35 current snap is 329 2024-04-12T11:25:29.625 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 35 to 326 2024-04-12T11:25:29.625 INFO:tasks.rados.rados.0.smithi029.stdout:2549: read oid 29 snap -1 2024-04-12T11:25:29.625 INFO:tasks.rados.rados.0.smithi029.stdout:2549: expect (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:25:29.625 INFO:tasks.rados.rados.0.smithi029.stdout:2550: write oid 44 current snap is 329 2024-04-12T11:25:29.625 INFO:tasks.rados.rados.0.smithi029.stdout:2550: seq_num 958 ranges {501106=749020,1743279=357827} 2024-04-12T11:25:29.634 INFO:tasks.rados.rados.0.smithi029.stdout:2550: writing smithi02979277-44 from 501106 to 1250126 tid 1 2024-04-12T11:25:29.637 INFO:tasks.rados.rados.0.smithi029.stdout:2550: writing smithi02979277-44 from 1743279 to 2101106 tid 2 2024-04-12T11:25:29.638 INFO:tasks.rados.rados.0.smithi029.stdout:2551: write oid 7 current snap is 329 2024-04-12T11:25:29.638 INFO:tasks.rados.rados.0.smithi029.stdout:2551: seq_num 959 ranges {517913=424269,1401761=446989} 2024-04-12T11:25:29.643 INFO:tasks.rados.rados.0.smithi029.stdout:2551: writing smithi02979277-7 from 517913 to 942182 tid 1 2024-04-12T11:25:29.648 INFO:tasks.rados.rados.0.smithi029.stdout:2551: writing smithi02979277-7 from 1401761 to 1848750 tid 2 2024-04-12T11:25:29.648 INFO:tasks.rados.rados.0.smithi029.stdout:2552: delete oid 15 current snap is 329 2024-04-12T11:25:29.648 INFO:tasks.rados.rados.0.smithi029.stdout:2549: expect (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:25:29.780 INFO:tasks.rados.rados.0.smithi029.stdout:2548: finishing rollback tid 0 to smithi02979277-35 2024-04-12T11:25:29.780 INFO:tasks.rados.rados.0.smithi029.stdout:2548: finishing rollback tid 1 to smithi02979277-35 2024-04-12T11:25:29.780 INFO:tasks.rados.rados.0.smithi029.stdout:2548: finishing rollback tid 2 to smithi02979277-35 2024-04-12T11:25:29.780 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 1541 (ObjNum 937 snap 325 seq_num 937) dirty exists 2024-04-12T11:25:29.780 INFO:tasks.rados.rados.0.smithi029.stdout:2548: done (4 left) 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2549: done (3 left) 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2552: done (2 left) 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2553: snap_create 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2550: finishing write tid 1 to smithi02979277-44 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2550: finishing write tid 2 to smithi02979277-44 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2550: finishing write tid 3 to smithi02979277-44 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2550: finishing write tid 4 to smithi02979277-44 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2550: oid 44 updating version 0 to 1232 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2550: oid 44 updating version 1232 to 1234 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2550: oid 44 version 1234 is already newer than 1233 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1234 (ObjNum 958 snap 329 seq_num 958) dirty exists 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2550: left oid 44 (ObjNum 958 snap 329 seq_num 958) 2024-04-12T11:25:29.781 INFO:tasks.rados.rados.0.smithi029.stdout:2551: finishing write tid 1 to smithi02979277-7 2024-04-12T11:25:29.782 INFO:tasks.rados.rados.0.smithi029.stdout:2551: finishing write tid 2 to smithi02979277-7 2024-04-12T11:25:29.782 INFO:tasks.rados.rados.0.smithi029.stdout:2551: finishing write tid 3 to smithi02979277-7 2024-04-12T11:25:29.782 INFO:tasks.rados.rados.0.smithi029.stdout:2551: finishing write tid 4 to smithi02979277-7 2024-04-12T11:25:29.782 INFO:tasks.rados.rados.0.smithi029.stdout:2551: oid 7 updating version 0 to 1241 2024-04-12T11:25:29.782 INFO:tasks.rados.rados.0.smithi029.stdout:2551: oid 7 version 1241 is already newer than 1240 2024-04-12T11:25:29.782 INFO:tasks.rados.rados.0.smithi029.stdout:2551: oid 7 updating version 1241 to 1242 2024-04-12T11:25:29.782 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1242 (ObjNum 959 snap 329 seq_num 959) dirty exists 2024-04-12T11:25:29.782 INFO:tasks.rados.rados.0.smithi029.stdout:2551: left oid 7 (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:25:30.691 INFO:tasks.rados.rados.0.smithi029.stdout:2550: done (2 left) 2024-04-12T11:25:30.691 INFO:tasks.rados.rados.0.smithi029.stdout:2551: done (1 left) 2024-04-12T11:25:30.691 INFO:tasks.rados.rados.0.smithi029.stdout:2553: done (0 left) 2024-04-12T11:25:30.691 INFO:tasks.rados.rados.0.smithi029.stdout:2554: write oid 4 current snap is 330 2024-04-12T11:25:30.691 INFO:tasks.rados.rados.0.smithi029.stdout:2554: seq_num 961 ranges {551527=574767,1918725=232802} 2024-04-12T11:25:30.697 INFO:tasks.rados.rados.0.smithi029.stdout:2554: writing smithi02979277-4 from 551527 to 1126294 tid 1 2024-04-12T11:25:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:2554: writing smithi02979277-4 from 1918725 to 2151527 tid 2 2024-04-12T11:25:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:2555: snap_remove snap 326 2024-04-12T11:25:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:30 smithi087 ceph-mon[89014]: pgmap v723: 89 pgs: 89 active+clean; 211 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:25:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:30 smithi087 ceph-mon[89014]: osdmap e725: 8 total, 8 up, 8 in 2024-04-12T11:25:31.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:30 smithi029 ceph-mon[93650]: pgmap v723: 89 pgs: 89 active+clean; 211 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:25:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:30 smithi029 ceph-mon[93650]: osdmap e725: 8 total, 8 up, 8 in 2024-04-12T11:25:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:30 smithi029 ceph-mon[97787]: pgmap v723: 89 pgs: 89 active+clean; 211 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:25:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:30 smithi029 ceph-mon[97787]: osdmap e725: 8 total, 8 up, 8 in 2024-04-12T11:25:31.697 INFO:tasks.rados.rados.0.smithi029.stdout:2555: done (1 left) 2024-04-12T11:25:31.697 INFO:tasks.rados.rados.0.smithi029.stdout:2556: snap_remove snap 321 2024-04-12T11:25:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:31 smithi087 ceph-mon[89014]: osdmap e726: 8 total, 8 up, 8 in 2024-04-12T11:25:32.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:31 smithi029 ceph-mon[93650]: osdmap e726: 8 total, 8 up, 8 in 2024-04-12T11:25:32.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:31 smithi029 ceph-mon[97787]: osdmap e726: 8 total, 8 up, 8 in 2024-04-12T11:25:32.619 INFO:tasks.rados.rados.0.smithi029.stdout:2556: done (1 left) 2024-04-12T11:25:32.620 INFO:tasks.rados.rados.0.smithi029.stdout:2557: snap_remove snap 316 2024-04-12T11:25:32.694 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:25:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:25:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:25:32.694 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:32 smithi087 ceph-mon[89014]: pgmap v726: 89 pgs: 89 active+clean; 211 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 533 KiB/s wr, 6 op/s 2024-04-12T11:25:32.694 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:32 smithi087 ceph-mon[89014]: osdmap e727: 8 total, 8 up, 8 in 2024-04-12T11:25:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:32 smithi087 ceph-mon[89014]: osdmap e728: 8 total, 8 up, 8 in 2024-04-12T11:25:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:32 smithi029 ceph-mon[97787]: pgmap v726: 89 pgs: 89 active+clean; 211 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 533 KiB/s wr, 6 op/s 2024-04-12T11:25:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:32 smithi029 ceph-mon[97787]: osdmap e727: 8 total, 8 up, 8 in 2024-04-12T11:25:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:32 smithi029 ceph-mon[97787]: osdmap e728: 8 total, 8 up, 8 in 2024-04-12T11:25:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:32 smithi029 ceph-mon[93650]: pgmap v726: 89 pgs: 89 active+clean; 211 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 533 KiB/s wr, 6 op/s 2024-04-12T11:25:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:32 smithi029 ceph-mon[93650]: osdmap e727: 8 total, 8 up, 8 in 2024-04-12T11:25:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:32 smithi029 ceph-mon[93650]: osdmap e728: 8 total, 8 up, 8 in 2024-04-12T11:25:33.621 INFO:tasks.rados.rados.0.smithi029.stdout:2557: done (1 left) 2024-04-12T11:25:33.621 INFO:tasks.rados.rados.0.smithi029.stdout:2558: write oid 45 current snap is 330 2024-04-12T11:25:33.621 INFO:tasks.rados.rados.0.smithi029.stdout:2554: finishing write tid 1 to smithi02979277-4 2024-04-12T11:25:33.621 INFO:tasks.rados.rados.0.smithi029.stdout:2558: seq_num 962 ranges {568334=650016,1860854=307480} 2024-04-12T11:25:33.628 INFO:tasks.rados.rados.0.smithi029.stdout:2558: writing smithi02979277-45 from 568334 to 1218350 tid 1 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:2558: writing smithi02979277-45 from 1860854 to 2168334 tid 2 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:2559: snap_create 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:2554: finishing write tid 2 to smithi02979277-4 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:2554: finishing write tid 3 to smithi02979277-4 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:2554: finishing write tid 4 to smithi02979277-4 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:2554: oid 4 updating version 0 to 1180 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:2554: oid 4 version 1180 is already newer than 1179 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:2554: oid 4 version 1180 is already newer than 1178 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1180 (ObjNum 961 snap 330 seq_num 961) dirty exists 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:2554: left oid 4 (ObjNum 961 snap 330 seq_num 961) 2024-04-12T11:25:33.634 INFO:tasks.rados.rados.0.smithi029.stdout:2558: finishing write tid 1 to smithi02979277-45 2024-04-12T11:25:33.636 INFO:tasks.rados.rados.0.smithi029.stdout:2558: finishing write tid 2 to smithi02979277-45 2024-04-12T11:25:33.636 INFO:tasks.rados.rados.0.smithi029.stdout:2558: finishing write tid 3 to smithi02979277-45 2024-04-12T11:25:33.636 INFO:tasks.rados.rados.0.smithi029.stdout:2558: finishing write tid 4 to smithi02979277-45 2024-04-12T11:25:33.637 INFO:tasks.rados.rados.0.smithi029.stdout:2558: oid 45 updating version 0 to 1251 2024-04-12T11:25:33.637 INFO:tasks.rados.rados.0.smithi029.stdout:2558: oid 45 version 1251 is already newer than 1250 2024-04-12T11:25:33.637 INFO:tasks.rados.rados.0.smithi029.stdout:2558: oid 45 version 1251 is already newer than 1249 2024-04-12T11:25:33.637 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1251 (ObjNum 962 snap 330 seq_num 962) dirty exists 2024-04-12T11:25:33.637 INFO:tasks.rados.rados.0.smithi029.stdout:2558: left oid 45 (ObjNum 962 snap 330 seq_num 962) 2024-04-12T11:25:34.628 INFO:tasks.rados.rados.0.smithi029.stdout:2554: done (2 left) 2024-04-12T11:25:34.628 INFO:tasks.rados.rados.0.smithi029.stdout:2558: done (1 left) 2024-04-12T11:25:34.628 INFO:tasks.rados.rados.0.smithi029.stdout:2559: done (0 left) 2024-04-12T11:25:34.628 INFO:tasks.rados.rados.0.smithi029.stdout:2560: delete oid 44 current snap is 331 2024-04-12T11:25:34.634 INFO:tasks.rados.rados.0.smithi029.stdout:2560: done (0 left) 2024-04-12T11:25:34.634 INFO:tasks.rados.rados.0.smithi029.stdout:2561: write oid 25 current snap is 331 2024-04-12T11:25:34.634 INFO:tasks.rados.rados.0.smithi029.stdout:2561: seq_num 964 ranges {601948=400514,1577818=624130} 2024-04-12T11:25:34.639 INFO:tasks.rados.rados.0.smithi029.stdout:2561: writing smithi02979277-25 from 601948 to 1002462 tid 1 2024-04-12T11:25:34.646 INFO:tasks.rados.rados.0.smithi029.stdout:2561: writing smithi02979277-25 from 1577818 to 2201948 tid 2 2024-04-12T11:25:34.646 INFO:tasks.rados.rados.0.smithi029.stdout:2562: read oid 49 snap -1 2024-04-12T11:25:34.647 INFO:tasks.rados.rados.0.smithi029.stdout:2562: expect deleted 2024-04-12T11:25:34.647 INFO:tasks.rados.rados.0.smithi029.stdout:2563: snap_create 2024-04-12T11:25:34.647 INFO:tasks.rados.rados.0.smithi029.stdout:2561: finishing write tid 1 to smithi02979277-25 2024-04-12T11:25:34.650 INFO:tasks.rados.rados.0.smithi029.stdout:2561: finishing write tid 2 to smithi02979277-25 2024-04-12T11:25:34.650 INFO:tasks.rados.rados.0.smithi029.stdout:2561: finishing write tid 3 to smithi02979277-25 2024-04-12T11:25:34.650 INFO:tasks.rados.rados.0.smithi029.stdout:2561: finishing write tid 4 to smithi02979277-25 2024-04-12T11:25:34.650 INFO:tasks.rados.rados.0.smithi029.stdout:2561: oid 25 updating version 0 to 1305 2024-04-12T11:25:34.650 INFO:tasks.rados.rados.0.smithi029.stdout:2561: oid 25 version 1305 is already newer than 1303 2024-04-12T11:25:34.651 INFO:tasks.rados.rados.0.smithi029.stdout:2561: oid 25 version 1305 is already newer than 1304 2024-04-12T11:25:34.651 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 1305 (ObjNum 964 snap 331 seq_num 964) dirty exists 2024-04-12T11:25:34.651 INFO:tasks.rados.rados.0.smithi029.stdout:2561: left oid 25 (ObjNum 964 snap 331 seq_num 964) 2024-04-12T11:25:34.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:34 smithi029 ceph-mon[97787]: pgmap v729: 89 pgs: 89 active+clean; 204 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 681 KiB/s wr, 4 op/s 2024-04-12T11:25:34.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:34 smithi029 ceph-mon[97787]: osdmap e729: 8 total, 8 up, 8 in 2024-04-12T11:25:34.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:34 smithi029 ceph-mon[93650]: pgmap v729: 89 pgs: 89 active+clean; 204 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 681 KiB/s wr, 4 op/s 2024-04-12T11:25:34.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:34 smithi029 ceph-mon[93650]: osdmap e729: 8 total, 8 up, 8 in 2024-04-12T11:25:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:34 smithi087 ceph-mon[89014]: pgmap v729: 89 pgs: 89 active+clean; 204 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 681 KiB/s wr, 4 op/s 2024-04-12T11:25:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:34 smithi087 ceph-mon[89014]: osdmap e729: 8 total, 8 up, 8 in 2024-04-12T11:25:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:25:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:25:34] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2024-04-12T11:25:35.634 INFO:tasks.rados.rados.0.smithi029.stdout:2561: done (2 left) 2024-04-12T11:25:35.634 INFO:tasks.rados.rados.0.smithi029.stdout:2562: done (1 left) 2024-04-12T11:25:35.634 INFO:tasks.rados.rados.0.smithi029.stdout:2563: done (0 left) 2024-04-12T11:25:35.634 INFO:tasks.rados.rados.0.smithi029.stdout:2564: snap_create 2024-04-12T11:25:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:35 smithi087 ceph-mon[89014]: osdmap e730: 8 total, 8 up, 8 in 2024-04-12T11:25:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:35 smithi029 ceph-mon[93650]: osdmap e730: 8 total, 8 up, 8 in 2024-04-12T11:25:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:35 smithi029 ceph-mon[97787]: osdmap e730: 8 total, 8 up, 8 in 2024-04-12T11:25:36.635 INFO:tasks.rados.rados.0.smithi029.stdout:2564: done (0 left) 2024-04-12T11:25:36.636 INFO:tasks.rados.rados.0.smithi029.stdout:2565: snap_create 2024-04-12T11:25:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:36 smithi087 ceph-mon[89014]: pgmap v732: 89 pgs: 89 active+clean; 196 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 681 KiB/s wr, 6 op/s 2024-04-12T11:25:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:36 smithi087 ceph-mon[89014]: osdmap e731: 8 total, 8 up, 8 in 2024-04-12T11:25:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:36 smithi029 ceph-mon[93650]: pgmap v732: 89 pgs: 89 active+clean; 196 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 681 KiB/s wr, 6 op/s 2024-04-12T11:25:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:36 smithi029 ceph-mon[93650]: osdmap e731: 8 total, 8 up, 8 in 2024-04-12T11:25:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:36 smithi029 ceph-mon[97787]: pgmap v732: 89 pgs: 89 active+clean; 196 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 681 KiB/s wr, 6 op/s 2024-04-12T11:25:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:36 smithi029 ceph-mon[97787]: osdmap e731: 8 total, 8 up, 8 in 2024-04-12T11:25:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:37.621 INFO:tasks.rados.rados.0.smithi029.stdout:2565: done (0 left) 2024-04-12T11:25:37.621 INFO:tasks.rados.rados.0.smithi029.stdout:2566: read oid 32 snap -1 2024-04-12T11:25:37.621 INFO:tasks.rados.rados.0.smithi029.stdout:2566: expect (ObjNum 878 snap 310 seq_num 878) 2024-04-12T11:25:37.622 INFO:tasks.rados.rados.0.smithi029.stdout:2567: write oid 41 current snap is 334 2024-04-12T11:25:37.622 INFO:tasks.rados.rados.0.smithi029.stdout:2567: seq_num 965 ranges {618755=475763,1636300=457996} 2024-04-12T11:25:37.627 INFO:tasks.rados.rados.0.smithi029.stdout:2567: writing smithi02979277-41 from 618755 to 1094518 tid 1 2024-04-12T11:25:37.633 INFO:tasks.rados.rados.0.smithi029.stdout:2567: writing smithi02979277-41 from 1636300 to 2094296 tid 2 2024-04-12T11:25:37.633 INFO:tasks.rados.rados.0.smithi029.stdout:2568: snap_create 2024-04-12T11:25:37.633 INFO:tasks.rados.rados.0.smithi029.stdout:2566: expect (ObjNum 878 snap 310 seq_num 878) 2024-04-12T11:25:37.804 INFO:tasks.rados.rados.0.smithi029.stdout:2567: finishing write tid 1 to smithi02979277-41 2024-04-12T11:25:37.804 INFO:tasks.rados.rados.0.smithi029.stdout:2567: finishing write tid 2 to smithi02979277-41 2024-04-12T11:25:37.804 INFO:tasks.rados.rados.0.smithi029.stdout:2567: finishing write tid 3 to smithi02979277-41 2024-04-12T11:25:37.804 INFO:tasks.rados.rados.0.smithi029.stdout:2567: finishing write tid 4 to smithi02979277-41 2024-04-12T11:25:37.804 INFO:tasks.rados.rados.0.smithi029.stdout:2567: oid 41 updating version 0 to 1243 2024-04-12T11:25:37.804 INFO:tasks.rados.rados.0.smithi029.stdout:2567: oid 41 updating version 1243 to 1244 2024-04-12T11:25:37.804 INFO:tasks.rados.rados.0.smithi029.stdout:2567: oid 41 version 1244 is already newer than 1242 2024-04-12T11:25:37.804 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1244 (ObjNum 965 snap 334 seq_num 965) dirty exists 2024-04-12T11:25:37.804 INFO:tasks.rados.rados.0.smithi029.stdout:2567: left oid 41 (ObjNum 965 snap 334 seq_num 965) 2024-04-12T11:25:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:37 smithi087 ceph-mon[89014]: osdmap e732: 8 total, 8 up, 8 in 2024-04-12T11:25:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:37 smithi087 ceph-mon[89014]: osdmap e733: 8 total, 8 up, 8 in 2024-04-12T11:25:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:37 smithi029 ceph-mon[93650]: osdmap e732: 8 total, 8 up, 8 in 2024-04-12T11:25:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:37 smithi029 ceph-mon[93650]: osdmap e733: 8 total, 8 up, 8 in 2024-04-12T11:25:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:37 smithi029 ceph-mon[97787]: osdmap e732: 8 total, 8 up, 8 in 2024-04-12T11:25:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:37 smithi029 ceph-mon[97787]: osdmap e733: 8 total, 8 up, 8 in 2024-04-12T11:25:38.623 INFO:tasks.rados.rados.0.smithi029.stdout:2566: done (2 left) 2024-04-12T11:25:38.623 INFO:tasks.rados.rados.0.smithi029.stdout:2567: done (1 left) 2024-04-12T11:25:38.623 INFO:tasks.rados.rados.0.smithi029.stdout:2568: done (0 left) 2024-04-12T11:25:38.623 INFO:tasks.rados.rados.0.smithi029.stdout:2569: write oid 16 current snap is 335 2024-04-12T11:25:38.623 INFO:tasks.rados.rados.0.smithi029.stdout:2569: seq_num 966 ranges {635562=667365,2094782=140780} 2024-04-12T11:25:38.630 INFO:tasks.rados.rados.0.smithi029.stdout:2569: writing smithi02979277-16 from 635562 to 1302927 tid 1 2024-04-12T11:25:38.635 INFO:tasks.rados.rados.0.smithi029.stdout:2569: writing smithi02979277-16 from 2094782 to 2235562 tid 2 2024-04-12T11:25:38.635 INFO:tasks.rados.rados.0.smithi029.stdout:2570: read oid 5 snap -1 2024-04-12T11:25:38.635 INFO:tasks.rados.rados.0.smithi029.stdout:2570: expect deleted 2024-04-12T11:25:38.635 INFO:tasks.rados.rados.0.smithi029.stdout:2571: snap_remove snap 327 2024-04-12T11:25:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:38 smithi087 ceph-mon[89014]: pgmap v735: 89 pgs: 89 active+clean; 195 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 250 KiB/s wr, 3 op/s 2024-04-12T11:25:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:38 smithi087 ceph-mon[89014]: osdmap e734: 8 total, 8 up, 8 in 2024-04-12T11:25:39.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:38 smithi029 ceph-mon[93650]: pgmap v735: 89 pgs: 89 active+clean; 195 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 250 KiB/s wr, 3 op/s 2024-04-12T11:25:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:38 smithi029 ceph-mon[93650]: osdmap e734: 8 total, 8 up, 8 in 2024-04-12T11:25:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:38 smithi029 ceph-mon[97787]: pgmap v735: 89 pgs: 89 active+clean; 195 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 250 KiB/s wr, 3 op/s 2024-04-12T11:25:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:38 smithi029 ceph-mon[97787]: osdmap e734: 8 total, 8 up, 8 in 2024-04-12T11:25:39.625 INFO:tasks.rados.rados.0.smithi029.stdout:2571: done (2 left) 2024-04-12T11:25:39.625 INFO:tasks.rados.rados.0.smithi029.stdout:2572: write oid 34 current snap is 335 2024-04-12T11:25:39.625 INFO:tasks.rados.rados.0.smithi029.stdout:2572: seq_num 967 ranges {652369=742614,2153264=99105} 2024-04-12T11:25:39.632 INFO:tasks.rados.rados.0.smithi029.stdout:2572: writing smithi02979277-34 from 652369 to 1394983 tid 1 2024-04-12T11:25:39.637 INFO:tasks.rados.rados.0.smithi029.stdout:2572: writing smithi02979277-34 from 2153264 to 2252369 tid 2 2024-04-12T11:25:39.638 INFO:tasks.rados.rados.0.smithi029.stdout:2573: snap_create 2024-04-12T11:25:39.638 INFO:tasks.rados.rados.0.smithi029.stdout:2569: finishing write tid 1 to smithi02979277-16 2024-04-12T11:25:39.638 INFO:tasks.rados.rados.0.smithi029.stdout:2569: finishing write tid 2 to smithi02979277-16 2024-04-12T11:25:39.638 INFO:tasks.rados.rados.0.smithi029.stdout:2569: finishing write tid 3 to smithi02979277-16 2024-04-12T11:25:39.638 INFO:tasks.rados.rados.0.smithi029.stdout:2569: finishing write tid 4 to smithi02979277-16 2024-04-12T11:25:39.638 INFO:tasks.rados.rados.0.smithi029.stdout:2569: oid 16 updating version 0 to 1190 2024-04-12T11:25:39.638 INFO:tasks.rados.rados.0.smithi029.stdout:2569: oid 16 version 1190 is already newer than 1188 2024-04-12T11:25:39.638 INFO:tasks.rados.rados.0.smithi029.stdout:2569: oid 16 version 1190 is already newer than 1189 2024-04-12T11:25:39.638 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1190 (ObjNum 966 snap 335 seq_num 966) dirty exists 2024-04-12T11:25:39.638 INFO:tasks.rados.rados.0.smithi029.stdout:2569: left oid 16 (ObjNum 966 snap 335 seq_num 966) 2024-04-12T11:25:39.640 INFO:tasks.rados.rados.0.smithi029.stdout:2572: finishing write tid 1 to smithi02979277-34 2024-04-12T11:25:39.640 INFO:tasks.rados.rados.0.smithi029.stdout:2572: finishing write tid 2 to smithi02979277-34 2024-04-12T11:25:39.641 INFO:tasks.rados.rados.0.smithi029.stdout:2572: finishing write tid 3 to smithi02979277-34 2024-04-12T11:25:39.641 INFO:tasks.rados.rados.0.smithi029.stdout:2572: finishing write tid 4 to smithi02979277-34 2024-04-12T11:25:39.641 INFO:tasks.rados.rados.0.smithi029.stdout:2572: oid 34 updating version 0 to 1156 2024-04-12T11:25:39.641 INFO:tasks.rados.rados.0.smithi029.stdout:2572: oid 34 updating version 1156 to 1157 2024-04-12T11:25:39.641 INFO:tasks.rados.rados.0.smithi029.stdout:2572: oid 34 version 1157 is already newer than 1155 2024-04-12T11:25:39.641 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1157 (ObjNum 967 snap 335 seq_num 967) dirty exists 2024-04-12T11:25:39.642 INFO:tasks.rados.rados.0.smithi029.stdout:2572: left oid 34 (ObjNum 967 snap 335 seq_num 967) 2024-04-12T11:25:40.631 INFO:tasks.rados.rados.0.smithi029.stdout:2569: done (3 left) 2024-04-12T11:25:40.631 INFO:tasks.rados.rados.0.smithi029.stdout:2570: done (2 left) 2024-04-12T11:25:40.631 INFO:tasks.rados.rados.0.smithi029.stdout:2572: done (1 left) 2024-04-12T11:25:40.631 INFO:tasks.rados.rados.0.smithi029.stdout:2573: done (0 left) 2024-04-12T11:25:40.631 INFO:tasks.rados.rados.0.smithi029.stdout:2574: write oid 33 current snap is 336 2024-04-12T11:25:40.631 INFO:tasks.rados.rados.0.smithi029.stdout:2574: seq_num 968 ranges {669176=417863,1811746=457430} 2024-04-12T11:25:40.636 INFO:tasks.rados.rados.0.smithi029.stdout:2574: writing smithi02979277-33 from 669176 to 1087039 tid 1 2024-04-12T11:25:40.643 INFO:tasks.rados.rados.0.smithi029.stdout:2574: writing smithi02979277-33 from 1811746 to 2269176 tid 2 2024-04-12T11:25:40.643 INFO:tasks.rados.rados.0.smithi029.stdout:2575: delete oid 20 current snap is 336 2024-04-12T11:25:40.643 INFO:tasks.rados.rados.0.smithi029.stdout:2574: finishing write tid 1 to smithi02979277-33 2024-04-12T11:25:40.647 INFO:tasks.rados.rados.0.smithi029.stdout:2574: finishing write tid 2 to smithi02979277-33 2024-04-12T11:25:40.647 INFO:tasks.rados.rados.0.smithi029.stdout:2574: finishing write tid 3 to smithi02979277-33 2024-04-12T11:25:40.647 INFO:tasks.rados.rados.0.smithi029.stdout:2574: finishing write tid 4 to smithi02979277-33 2024-04-12T11:25:40.649 INFO:tasks.rados.rados.0.smithi029.stdout:2574: oid 33 updating version 0 to 1091 2024-04-12T11:25:40.650 INFO:tasks.rados.rados.0.smithi029.stdout:2574: oid 33 version 1091 is already newer than 1089 2024-04-12T11:25:40.650 INFO:tasks.rados.rados.0.smithi029.stdout:2574: oid 33 version 1091 is already newer than 1090 2024-04-12T11:25:40.650 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 1091 (ObjNum 968 snap 336 seq_num 968) dirty exists 2024-04-12T11:25:40.650 INFO:tasks.rados.rados.0.smithi029.stdout:2574: left oid 33 (ObjNum 968 snap 336 seq_num 968) 2024-04-12T11:25:40.651 INFO:tasks.rados.rados.0.smithi029.stdout:2574: done (1 left) 2024-04-12T11:25:40.651 INFO:tasks.rados.rados.0.smithi029.stdout:2575: done (0 left) 2024-04-12T11:25:40.651 INFO:tasks.rados.rados.0.smithi029.stdout:2576: snap_create 2024-04-12T11:25:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:40 smithi087 ceph-mon[89014]: pgmap v738: 89 pgs: 89 active+clean; 192 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 484 KiB/s wr, 4 op/s 2024-04-12T11:25:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:40 smithi087 ceph-mon[89014]: osdmap e735: 8 total, 8 up, 8 in 2024-04-12T11:25:41.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:40 smithi029 ceph-mon[93650]: pgmap v738: 89 pgs: 89 active+clean; 192 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 484 KiB/s wr, 4 op/s 2024-04-12T11:25:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:40 smithi029 ceph-mon[93650]: osdmap e735: 8 total, 8 up, 8 in 2024-04-12T11:25:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:40 smithi029 ceph-mon[97787]: pgmap v738: 89 pgs: 89 active+clean; 192 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 484 KiB/s wr, 4 op/s 2024-04-12T11:25:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:40 smithi029 ceph-mon[97787]: osdmap e735: 8 total, 8 up, 8 in 2024-04-12T11:25:41.637 INFO:tasks.rados.rados.0.smithi029.stdout:2576: done (0 left) 2024-04-12T11:25:41.638 INFO:tasks.rados.rados.0.smithi029.stdout:2577: rollback oid 15 current snap is 337 2024-04-12T11:25:41.638 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 15 to 336 2024-04-12T11:25:41.638 INFO:tasks.rados.rados.0.smithi029.stdout:2578: read oid 28 snap -1 2024-04-12T11:25:41.638 INFO:tasks.rados.rados.0.smithi029.stdout:2578: expect (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:25:41.638 INFO:tasks.rados.rados.0.smithi029.stdout:2579: read oid 11 snap -1 2024-04-12T11:25:41.638 INFO:tasks.rados.rados.0.smithi029.stdout:2579: expect deleted 2024-04-12T11:25:41.638 INFO:tasks.rados.rados.0.smithi029.stdout:2580: snap_create 2024-04-12T11:25:41.640 INFO:tasks.rados.rados.0.smithi029.stdout:2577: finishing rollback tid 1 to smithi02979277-15 2024-04-12T11:25:41.640 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 822 (ObjNum 657 snap 0 seq_num 2925506400) dirty dne 2024-04-12T11:25:41.648 INFO:tasks.rados.rados.0.smithi029.stdout:2578: expect (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:25:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:41 smithi087 ceph-mon[89014]: osdmap e736: 8 total, 8 up, 8 in 2024-04-12T11:25:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:41 smithi029 ceph-mon[93650]: osdmap e736: 8 total, 8 up, 8 in 2024-04-12T11:25:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:41 smithi029 ceph-mon[97787]: osdmap e736: 8 total, 8 up, 8 in 2024-04-12T11:25:42.622 INFO:tasks.rados.rados.0.smithi029.stdout:2577: done (3 left) 2024-04-12T11:25:42.622 INFO:tasks.rados.rados.0.smithi029.stdout:2578: done (2 left) 2024-04-12T11:25:42.622 INFO:tasks.rados.rados.0.smithi029.stdout:2579: done (1 left) 2024-04-12T11:25:42.622 INFO:tasks.rados.rados.0.smithi029.stdout:2580: done (0 left) 2024-04-12T11:25:42.622 INFO:tasks.rados.rados.0.smithi029.stdout:2581: snap_create 2024-04-12T11:25:42.634 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:25:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:25:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:25:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:42 smithi087 ceph-mon[89014]: pgmap v741: 89 pgs: 89 active+clean; 192 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 234 KiB/s wr, 3 op/s 2024-04-12T11:25:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:42 smithi087 ceph-mon[89014]: osdmap e737: 8 total, 8 up, 8 in 2024-04-12T11:25:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:42 smithi087 ceph-mon[89014]: osdmap e738: 8 total, 8 up, 8 in 2024-04-12T11:25:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:42 smithi029 ceph-mon[97787]: pgmap v741: 89 pgs: 89 active+clean; 192 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 234 KiB/s wr, 3 op/s 2024-04-12T11:25:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:42 smithi029 ceph-mon[97787]: osdmap e737: 8 total, 8 up, 8 in 2024-04-12T11:25:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:42 smithi029 ceph-mon[97787]: osdmap e738: 8 total, 8 up, 8 in 2024-04-12T11:25:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:42 smithi029 ceph-mon[93650]: pgmap v741: 89 pgs: 89 active+clean; 192 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 234 KiB/s wr, 3 op/s 2024-04-12T11:25:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:42 smithi029 ceph-mon[93650]: osdmap e737: 8 total, 8 up, 8 in 2024-04-12T11:25:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:42 smithi029 ceph-mon[93650]: osdmap e738: 8 total, 8 up, 8 in 2024-04-12T11:25:43.624 INFO:tasks.rados.rados.0.smithi029.stdout:2581: done (0 left) 2024-04-12T11:25:43.624 INFO:tasks.rados.rados.0.smithi029.stdout:2582: read oid 7 snap -1 2024-04-12T11:25:43.624 INFO:tasks.rados.rados.0.smithi029.stdout:2582: expect (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:25:43.624 INFO:tasks.rados.rados.0.smithi029.stdout:2583: read oid 5 snap -1 2024-04-12T11:25:43.624 INFO:tasks.rados.rados.0.smithi029.stdout:2583: expect deleted 2024-04-12T11:25:43.625 INFO:tasks.rados.rados.0.smithi029.stdout:2584: write oid 35 current snap is 339 2024-04-12T11:25:43.625 INFO:tasks.rados.rados.0.smithi029.stdout:2584: seq_num 970 ranges {702790=568361,1812357=490433} 2024-04-12T11:25:43.631 INFO:tasks.rados.rados.0.smithi029.stdout:2584: writing smithi02979277-35 from 702790 to 1271151 tid 1 2024-04-12T11:25:43.637 INFO:tasks.rados.rados.0.smithi029.stdout:2584: writing smithi02979277-35 from 1812357 to 2302790 tid 2 2024-04-12T11:25:43.637 INFO:tasks.rados.rados.0.smithi029.stdout:2585: write oid 17 current snap is 339 2024-04-12T11:25:43.637 INFO:tasks.rados.rados.0.smithi029.stdout:2585: seq_num 971 ranges {719597=643610,1870839=448758} 2024-04-12T11:25:43.644 INFO:tasks.rados.rados.0.smithi029.stdout:2585: writing smithi02979277-17 from 719597 to 1363207 tid 1 2024-04-12T11:25:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2585: writing smithi02979277-17 from 1870839 to 2319597 tid 2 2024-04-12T11:25:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2586: delete oid 33 current snap is 339 2024-04-12T11:25:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2582: expect (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:25:43.752 INFO:tasks.rados.rados.0.smithi029.stdout:2584: finishing write tid 1 to smithi02979277-35 2024-04-12T11:25:43.752 INFO:tasks.rados.rados.0.smithi029.stdout:2584: finishing write tid 2 to smithi02979277-35 2024-04-12T11:25:43.752 INFO:tasks.rados.rados.0.smithi029.stdout:2584: finishing write tid 3 to smithi02979277-35 2024-04-12T11:25:43.752 INFO:tasks.rados.rados.0.smithi029.stdout:2584: finishing write tid 4 to smithi02979277-35 2024-04-12T11:25:43.752 INFO:tasks.rados.rados.0.smithi029.stdout:2584: oid 35 updating version 0 to 1560 2024-04-12T11:25:43.752 INFO:tasks.rados.rados.0.smithi029.stdout:2584: oid 35 updating version 1560 to 1561 2024-04-12T11:25:43.752 INFO:tasks.rados.rados.0.smithi029.stdout:2584: oid 35 version 1561 is already newer than 1559 2024-04-12T11:25:43.752 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 1561 (ObjNum 970 snap 339 seq_num 970) dirty exists 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2584: left oid 35 (ObjNum 970 snap 339 seq_num 970) 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2585: finishing write tid 1 to smithi02979277-17 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2585: finishing write tid 2 to smithi02979277-17 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2582: done (4 left) 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2583: done (3 left) 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2584: done (2 left) 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2586: done (1 left) 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2587: read oid 44 snap -1 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2587: expect deleted 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2588: read oid 24 snap -1 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2588: expect deleted 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2589: read oid 19 snap -1 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2589: expect (ObjNum 906 snap 314 seq_num 906) 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2590: read oid 47 snap 325 2024-04-12T11:25:43.753 INFO:tasks.rados.rados.0.smithi029.stdout:2590: expect (ObjNum 918 snap 318 seq_num 918) 2024-04-12T11:25:43.754 INFO:tasks.rados.rados.0.smithi029.stdout:2591: read oid 34 snap 333 2024-04-12T11:25:43.754 INFO:tasks.rados.rados.0.smithi029.stdout:2591: expect (ObjNum 949 snap 327 seq_num 949) 2024-04-12T11:25:43.754 INFO:tasks.rados.rados.0.smithi029.stdout:2592: read oid 21 snap -1 2024-04-12T11:25:43.754 INFO:tasks.rados.rados.0.smithi029.stdout:2592: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:25:43.754 INFO:tasks.rados.rados.0.smithi029.stdout:2593: snap_remove snap 332 2024-04-12T11:25:44.640 INFO:tasks.rados.rados.0.smithi029.stdout:2593: done (7 left) 2024-04-12T11:25:44.640 INFO:tasks.rados.rados.0.smithi029.stdout:2594: read oid 7 snap -1 2024-04-12T11:25:44.641 INFO:tasks.rados.rados.0.smithi029.stdout:2594: expect (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:25:44.641 INFO:tasks.rados.rados.0.smithi029.stdout:2595: write oid 9 current snap is 339 2024-04-12T11:25:44.641 INFO:tasks.rados.rados.0.smithi029.stdout:2595: seq_num 973 ranges {753211=794108,1987803=365408} 2024-04-12T11:25:44.649 INFO:tasks.rados.rados.0.smithi029.stdout:2595: writing smithi02979277-9 from 753211 to 1547319 tid 1 2024-04-12T11:25:44.655 INFO:tasks.rados.rados.0.smithi029.stdout:2595: writing smithi02979277-9 from 1987803 to 2353211 tid 2 2024-04-12T11:25:44.655 INFO:tasks.rados.rados.0.smithi029.stdout:2596: read oid 49 snap -1 2024-04-12T11:25:44.655 INFO:tasks.rados.rados.0.smithi029.stdout:2596: expect deleted 2024-04-12T11:25:44.655 INFO:tasks.rados.rados.0.smithi029.stdout:2597: rollback oid 30 current snap is 339 2024-04-12T11:25:44.655 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 325 2024-04-12T11:25:44.655 INFO:tasks.rados.rados.0.smithi029.stdout:2585: finishing write tid 3 to smithi02979277-17 2024-04-12T11:25:44.655 INFO:tasks.rados.rados.0.smithi029.stdout:2585: finishing write tid 4 to smithi02979277-17 2024-04-12T11:25:44.655 INFO:tasks.rados.rados.0.smithi029.stdout:2585: oid 17 updating version 0 to 1308 2024-04-12T11:25:44.656 INFO:tasks.rados.rados.0.smithi029.stdout:2585: oid 17 updating version 1308 to 1309 2024-04-12T11:25:44.656 INFO:tasks.rados.rados.0.smithi029.stdout:2585: oid 17 version 1309 is already newer than 1307 2024-04-12T11:25:44.656 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1309 (ObjNum 971 snap 339 seq_num 971) dirty exists 2024-04-12T11:25:44.656 INFO:tasks.rados.rados.0.smithi029.stdout:2585: left oid 17 (ObjNum 971 snap 339 seq_num 971) 2024-04-12T11:25:44.656 INFO:tasks.rados.rados.0.smithi029.stdout:2585: done (10 left) 2024-04-12T11:25:44.656 INFO:tasks.rados.rados.0.smithi029.stdout:2598: snap_remove snap 338 2024-04-12T11:25:44.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:44 smithi029 ceph-mon[97787]: pgmap v744: 89 pgs: 89 active+clean; 196 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 640 KiB/s wr, 3 op/s 2024-04-12T11:25:44.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:44 smithi029 ceph-mon[97787]: osdmap e739: 8 total, 8 up, 8 in 2024-04-12T11:25:44.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:25:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:44 smithi029 ceph-mon[93650]: pgmap v744: 89 pgs: 89 active+clean; 196 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 640 KiB/s wr, 3 op/s 2024-04-12T11:25:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:44 smithi029 ceph-mon[93650]: osdmap e739: 8 total, 8 up, 8 in 2024-04-12T11:25:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:25:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:44 smithi087 ceph-mon[89014]: pgmap v744: 89 pgs: 89 active+clean; 196 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 640 KiB/s wr, 3 op/s 2024-04-12T11:25:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:44 smithi087 ceph-mon[89014]: osdmap e739: 8 total, 8 up, 8 in 2024-04-12T11:25:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:25:45.210 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:25:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:25:44] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.43.0" 2024-04-12T11:25:45.646 INFO:tasks.rados.rados.0.smithi029.stdout:2598: done (10 left) 2024-04-12T11:25:45.646 INFO:tasks.rados.rados.0.smithi029.stdout:2599: read oid 41 snap 323 2024-04-12T11:25:45.646 INFO:tasks.rados.rados.0.smithi029.stdout:2599: expect (ObjNum 877 snap 310 seq_num 877) 2024-04-12T11:25:45.646 INFO:tasks.rados.rados.0.smithi029.stdout:2600: rollback oid 12 current snap is 339 2024-04-12T11:25:45.646 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 333 2024-04-12T11:25:45.646 INFO:tasks.rados.rados.0.smithi029.stdout:2591: expect (ObjNum 949 snap 327 seq_num 949) 2024-04-12T11:25:45.789 INFO:tasks.rados.rados.0.smithi029.stdout:2590: expect (ObjNum 918 snap 318 seq_num 918) 2024-04-12T11:25:45.925 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:45 smithi087 ceph-mon[89014]: osdmap e740: 8 total, 8 up, 8 in 2024-04-12T11:25:45.965 INFO:tasks.rados.rados.0.smithi029.stdout:2589: expect (ObjNum 906 snap 314 seq_num 906) 2024-04-12T11:25:46.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:45 smithi029 ceph-mon[93650]: osdmap e740: 8 total, 8 up, 8 in 2024-04-12T11:25:46.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:45 smithi029 ceph-mon[97787]: osdmap e740: 8 total, 8 up, 8 in 2024-04-12T11:25:46.151 INFO:tasks.rados.rados.0.smithi029.stdout:2592: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:25:46.316 INFO:tasks.rados.rados.0.smithi029.stdout:2594: expect (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2595: finishing write tid 1 to smithi02979277-9 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2595: finishing write tid 2 to smithi02979277-9 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2597: finishing rollback tid 0 to smithi02979277-30 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2595: finishing write tid 3 to smithi02979277-9 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2597: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2595: finishing write tid 4 to smithi02979277-9 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2595: oid 9 updating version 0 to 949 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2595: oid 9 version 949 is already newer than 948 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2595: oid 9 version 949 is already newer than 947 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 949 (ObjNum 973 snap 339 seq_num 973) dirty exists 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2595: left oid 9 (ObjNum 973 snap 339 seq_num 973) 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:2597: finishing rollback tid 2 to smithi02979277-30 2024-04-12T11:25:46.417 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 837 (ObjNum 935 snap 323 seq_num 935) dirty exists 2024-04-12T11:25:46.418 INFO:tasks.rados.rados.0.smithi029.stdout:2600: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:25:46.418 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1181 (ObjNum 177 snap 0 seq_num 1798700912) dirty dne 2024-04-12T11:25:46.418 INFO:tasks.rados.rados.0.smithi029.stdout:2599: expect (ObjNum 877 snap 310 seq_num 877) 2024-04-12T11:25:46.508 INFO:tasks.rados.rados.0.smithi029.stdout:2587: done (11 left) 2024-04-12T11:25:46.508 INFO:tasks.rados.rados.0.smithi029.stdout:2588: done (10 left) 2024-04-12T11:25:46.508 INFO:tasks.rados.rados.0.smithi029.stdout:2589: done (9 left) 2024-04-12T11:25:46.508 INFO:tasks.rados.rados.0.smithi029.stdout:2590: done (8 left) 2024-04-12T11:25:46.508 INFO:tasks.rados.rados.0.smithi029.stdout:2591: done (7 left) 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2592: done (6 left) 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2594: done (5 left) 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2595: done (4 left) 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2596: done (3 left) 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2597: done (2 left) 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2599: done (1 left) 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2600: done (0 left) 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2601: read oid 4 snap -1 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2601: expect (ObjNum 961 snap 330 seq_num 961) 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2602: read oid 26 snap -1 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2602: expect (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2603: rollback oid 5 current snap is 339 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 5 to 325 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2604: write oid 25 current snap is 339 2024-04-12T11:25:46.509 INFO:tasks.rados.rados.0.smithi029.stdout:2604: seq_num 974 ranges {770018=585710,2046285=323733} 2024-04-12T11:25:46.515 INFO:tasks.rados.rados.0.smithi029.stdout:2604: writing smithi02979277-25 from 770018 to 1355728 tid 1 2024-04-12T11:25:46.521 INFO:tasks.rados.rados.0.smithi029.stdout:2604: writing smithi02979277-25 from 2046285 to 2370018 tid 2 2024-04-12T11:25:46.521 INFO:tasks.rados.rados.0.smithi029.stdout:2605: snap_create 2024-04-12T11:25:46.521 INFO:tasks.rados.rados.0.smithi029.stdout:2602: expect (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:25:46.745 INFO:tasks.rados.rados.0.smithi029.stdout:2601: expect (ObjNum 961 snap 330 seq_num 961) 2024-04-12T11:25:46.819 INFO:tasks.rados.rados.0.smithi029.stdout:2603: finishing rollback tid 1 to smithi02979277-5 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1133 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2604: finishing write tid 1 to smithi02979277-25 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2604: finishing write tid 2 to smithi02979277-25 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2604: finishing write tid 3 to smithi02979277-25 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2604: finishing write tid 4 to smithi02979277-25 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2604: oid 25 updating version 0 to 1316 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2604: oid 25 updating version 1316 to 1317 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2604: oid 25 version 1317 is already newer than 1315 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 1317 (ObjNum 974 snap 339 seq_num 974) dirty exists 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2604: left oid 25 (ObjNum 974 snap 339 seq_num 974) 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2601: done (4 left) 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2602: done (3 left) 2024-04-12T11:25:46.820 INFO:tasks.rados.rados.0.smithi029.stdout:2603: done (2 left) 2024-04-12T11:25:46.821 INFO:tasks.rados.rados.0.smithi029.stdout:2604: done (1 left) 2024-04-12T11:25:46.821 INFO:tasks.rados.rados.0.smithi029.stdout:2605: done (0 left) 2024-04-12T11:25:46.821 INFO:tasks.rados.rados.0.smithi029.stdout:2606: write oid 10 current snap is 340 2024-04-12T11:25:46.821 INFO:tasks.rados.rados.0.smithi029.stdout:2606: seq_num 975 ranges {786825=660959,2104767=282058} 2024-04-12T11:25:46.827 INFO:tasks.rados.rados.0.smithi029.stdout:2606: writing smithi02979277-10 from 786825 to 1447784 tid 1 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:2606: writing smithi02979277-10 from 2104767 to 2386825 tid 2 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:2607: read oid 7 snap 333 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:2607: expect (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:2608: rollback oid 15 current snap is 340 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 15 to 333 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:2606: finishing write tid 1 to smithi02979277-10 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:2609: read oid 29 snap 323 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:2609: expect (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:2610: read oid 35 snap 322 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:2610: expect deleted 2024-04-12T11:25:46.833 INFO:tasks.rados.rados.0.smithi029.stdout:2611: delete oid 13 current snap is 340 2024-04-12T11:25:46.836 INFO:tasks.rados.rados.0.smithi029.stdout:2606: finishing write tid 2 to smithi02979277-10 2024-04-12T11:25:46.837 INFO:tasks.rados.rados.0.smithi029.stdout:2606: finishing write tid 3 to smithi02979277-10 2024-04-12T11:25:46.837 INFO:tasks.rados.rados.0.smithi029.stdout:2606: finishing write tid 4 to smithi02979277-10 2024-04-12T11:25:46.837 INFO:tasks.rados.rados.0.smithi029.stdout:2606: oid 10 updating version 0 to 1053 2024-04-12T11:25:46.837 INFO:tasks.rados.rados.0.smithi029.stdout:2606: oid 10 version 1053 is already newer than 1052 2024-04-12T11:25:46.837 INFO:tasks.rados.rados.0.smithi029.stdout:2606: oid 10 version 1053 is already newer than 1051 2024-04-12T11:25:46.837 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 1053 (ObjNum 975 snap 340 seq_num 975) dirty exists 2024-04-12T11:25:46.837 INFO:tasks.rados.rados.0.smithi029.stdout:2606: left oid 10 (ObjNum 975 snap 340 seq_num 975) 2024-04-12T11:25:46.838 INFO:tasks.rados.rados.0.smithi029.stdout:2606: done (5 left) 2024-04-12T11:25:46.838 INFO:tasks.rados.rados.0.smithi029.stdout:2610: done (4 left) 2024-04-12T11:25:46.838 INFO:tasks.rados.rados.0.smithi029.stdout:2611: done (3 left) 2024-04-12T11:25:46.838 INFO:tasks.rados.rados.0.smithi029.stdout:2612: snap_remove snap 335 2024-04-12T11:25:46.904 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:46 smithi087 ceph-mon[89014]: pgmap v747: 89 pgs: 89 active+clean; 196 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 845 KiB/s wr, 7 op/s 2024-04-12T11:25:46.904 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:46 smithi087 ceph-mon[89014]: osdmap e741: 8 total, 8 up, 8 in 2024-04-12T11:25:47.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:46 smithi029 ceph-mon[93650]: pgmap v747: 89 pgs: 89 active+clean; 196 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 845 KiB/s wr, 7 op/s 2024-04-12T11:25:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:46 smithi029 ceph-mon[93650]: osdmap e741: 8 total, 8 up, 8 in 2024-04-12T11:25:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:46 smithi029 ceph-mon[97787]: pgmap v747: 89 pgs: 89 active+clean; 196 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 845 KiB/s wr, 7 op/s 2024-04-12T11:25:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:46 smithi029 ceph-mon[97787]: osdmap e741: 8 total, 8 up, 8 in 2024-04-12T11:25:47.622 INFO:tasks.rados.rados.0.smithi029.stdout:2612: done (3 left) 2024-04-12T11:25:47.623 INFO:tasks.rados.rados.0.smithi029.stdout:2613: delete oid 16 current snap is 340 2024-04-12T11:25:47.623 INFO:tasks.rados.rados.0.smithi029.stdout:2608: finishing rollback tid 1 to smithi02979277-15 2024-04-12T11:25:47.623 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 822 (ObjNum 129 snap 0 seq_num 1801166800) dirty dne 2024-04-12T11:25:47.623 INFO:tasks.rados.rados.0.smithi029.stdout:2607: expect (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:25:47.731 INFO:tasks.rados.rados.0.smithi029.stdout:2609: expect (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:25:47.862 INFO:tasks.rados.rados.0.smithi029.stdout:2607: done (3 left) 2024-04-12T11:25:47.862 INFO:tasks.rados.rados.0.smithi029.stdout:2608: done (2 left) 2024-04-12T11:25:47.862 INFO:tasks.rados.rados.0.smithi029.stdout:2609: done (1 left) 2024-04-12T11:25:47.863 INFO:tasks.rados.rados.0.smithi029.stdout:2613: done (0 left) 2024-04-12T11:25:47.863 INFO:tasks.rados.rados.0.smithi029.stdout:2614: read oid 39 snap -1 2024-04-12T11:25:47.863 INFO:tasks.rados.rados.0.smithi029.stdout:2614: expect (ObjNum 930 snap 320 seq_num 930) 2024-04-12T11:25:47.863 INFO:tasks.rados.rados.0.smithi029.stdout:2615: delete oid 24 current snap is 340 2024-04-12T11:25:47.870 INFO:tasks.rados.rados.0.smithi029.stdout:2614: expect (ObjNum 930 snap 320 seq_num 930) 2024-04-12T11:25:47.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:47 smithi029 ceph-mon[97787]: osdmap e742: 8 total, 8 up, 8 in 2024-04-12T11:25:47.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:47 smithi029 ceph-mon[97787]: osdmap e743: 8 total, 8 up, 8 in 2024-04-12T11:25:47.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:47 smithi029 ceph-mon[93650]: osdmap e742: 8 total, 8 up, 8 in 2024-04-12T11:25:47.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:47 smithi029 ceph-mon[93650]: osdmap e743: 8 total, 8 up, 8 in 2024-04-12T11:25:47.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:47 smithi087 ceph-mon[89014]: osdmap e742: 8 total, 8 up, 8 in 2024-04-12T11:25:47.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:47 smithi087 ceph-mon[89014]: osdmap e743: 8 total, 8 up, 8 in 2024-04-12T11:25:48.079 INFO:tasks.rados.rados.0.smithi029.stdout:2614: done (1 left) 2024-04-12T11:25:48.079 INFO:tasks.rados.rados.0.smithi029.stdout:2615: done (0 left) 2024-04-12T11:25:48.079 INFO:tasks.rados.rados.0.smithi029.stdout:2616: delete oid 32 current snap is 340 2024-04-12T11:25:48.081 INFO:tasks.rados.rados.0.smithi029.stdout:2616: done (0 left) 2024-04-12T11:25:48.081 INFO:tasks.rados.rados.0.smithi029.stdout:2617: snap_remove snap 323 2024-04-12T11:25:48.660 INFO:tasks.rados.rados.0.smithi029.stdout:2617: done (0 left) 2024-04-12T11:25:48.660 INFO:tasks.rados.rados.0.smithi029.stdout:2618: snap_create 2024-04-12T11:25:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:48 smithi087 ceph-mon[89014]: pgmap v750: 89 pgs: 89 active+clean; 198 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 695 KiB/s wr, 14 op/s 2024-04-12T11:25:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:48.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:48.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:48 smithi029 ceph-mon[93650]: pgmap v750: 89 pgs: 89 active+clean; 198 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 695 KiB/s wr, 14 op/s 2024-04-12T11:25:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:48 smithi029 ceph-mon[97787]: pgmap v750: 89 pgs: 89 active+clean; 198 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 695 KiB/s wr, 14 op/s 2024-04-12T11:25:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:49.661 INFO:tasks.rados.rados.0.smithi029.stdout:2618: done (0 left) 2024-04-12T11:25:49.661 INFO:tasks.rados.rados.0.smithi029.stdout:2619: snap_remove snap 328 2024-04-12T11:25:49.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:49 smithi087 ceph-mon[89014]: osdmap e744: 8 total, 8 up, 8 in 2024-04-12T11:25:49.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:49 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:25:49.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:49 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:25:49.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:49 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:50.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:49 smithi029 ceph-mon[93650]: osdmap e744: 8 total, 8 up, 8 in 2024-04-12T11:25:50.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:49 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:25:50.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:49 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:25:50.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:49 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:50.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:49 smithi029 ceph-mon[97787]: osdmap e744: 8 total, 8 up, 8 in 2024-04-12T11:25:50.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:49 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:25:50.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:49 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:25:50.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:49 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:25:50.673 INFO:tasks.rados.rados.0.smithi029.stdout:2619: done (0 left) 2024-04-12T11:25:50.673 INFO:tasks.rados.rados.0.smithi029.stdout:2620: rollback oid 42 current snap is 341 2024-04-12T11:25:50.673 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 309 2024-04-12T11:25:50.673 INFO:tasks.rados.rados.0.smithi029.stdout:2621: rollback oid 5 current snap is 341 2024-04-12T11:25:50.673 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 5 to 331 2024-04-12T11:25:50.673 INFO:tasks.rados.rados.0.smithi029.stdout:2622: delete oid 47 current snap is 341 2024-04-12T11:25:50.680 INFO:tasks.rados.rados.0.smithi029.stdout:2620: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:25:50.681 INFO:tasks.rados.rados.0.smithi029.stdout:2620: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:25:50.681 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 325 (ObjNum 33 snap 0 seq_num 1800965984) dirty dne 2024-04-12T11:25:50.681 INFO:tasks.rados.rados.0.smithi029.stdout:2621: finishing rollback tid 1 to smithi02979277-5 2024-04-12T11:25:50.681 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1133 (ObjNum 129 snap 0 seq_num 1801066840) dirty dne 2024-04-12T11:25:50.682 INFO:tasks.rados.rados.0.smithi029.stdout:2620: done (2 left) 2024-04-12T11:25:50.682 INFO:tasks.rados.rados.0.smithi029.stdout:2621: done (1 left) 2024-04-12T11:25:50.682 INFO:tasks.rados.rados.0.smithi029.stdout:2622: done (0 left) 2024-04-12T11:25:50.682 INFO:tasks.rados.rados.0.smithi029.stdout:2623: write oid 1 current snap is 341 2024-04-12T11:25:50.682 INFO:tasks.rados.rados.0.smithi029.stdout:2623: seq_num 981 ranges {487667=428806,1655659=770995} 2024-04-12T11:25:50.687 INFO:tasks.rados.rados.0.smithi029.stdout:2623: writing smithi02979277-1 from 487667 to 916473 tid 1 2024-04-12T11:25:50.696 INFO:tasks.rados.rados.0.smithi029.stdout:2623: writing smithi02979277-1 from 1655659 to 2426654 tid 2 2024-04-12T11:25:50.696 INFO:tasks.rados.rados.0.smithi029.stdout:2624: rollback oid 35 current snap is 341 2024-04-12T11:25:50.696 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 35 to 336 2024-04-12T11:25:50.696 INFO:tasks.rados.rados.0.smithi029.stdout:2623: finishing write tid 1 to smithi02979277-1 2024-04-12T11:25:50.696 INFO:tasks.rados.rados.0.smithi029.stdout:2625: rollback oid 14 current snap is 341 2024-04-12T11:25:50.696 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 337 2024-04-12T11:25:50.696 INFO:tasks.rados.rados.0.smithi029.stdout:2626: delete oid 16 current snap is 341 2024-04-12T11:25:50.698 INFO:tasks.rados.rados.0.smithi029.stdout:2625: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:25:50.698 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:25:50.700 INFO:tasks.rados.rados.0.smithi029.stdout:2625: done (3 left) 2024-04-12T11:25:50.700 INFO:tasks.rados.rados.0.smithi029.stdout:2626: done (2 left) 2024-04-12T11:25:50.700 INFO:tasks.rados.rados.0.smithi029.stdout:2627: read oid 38 snap -1 2024-04-12T11:25:50.700 INFO:tasks.rados.rados.0.smithi029.stdout:2627: expect (ObjNum 864 snap 302 seq_num 864) 2024-04-12T11:25:50.700 INFO:tasks.rados.rados.0.smithi029.stdout:2628: write oid 2 current snap is 341 2024-04-12T11:25:50.700 INFO:tasks.rados.rados.0.smithi029.stdout:2628: seq_num 983 ranges {521281=579304,1656270=774664} 2024-04-12T11:25:50.706 INFO:tasks.rados.rados.0.smithi029.stdout:2628: writing smithi02979277-2 from 521281 to 1100585 tid 1 2024-04-12T11:25:50.714 INFO:tasks.rados.rados.0.smithi029.stdout:2628: writing smithi02979277-2 from 1656270 to 2430934 tid 2 2024-04-12T11:25:50.714 INFO:tasks.rados.rados.0.smithi029.stdout:2629: read oid 44 snap -1 2024-04-12T11:25:50.714 INFO:tasks.rados.rados.0.smithi029.stdout:2629: expect deleted 2024-04-12T11:25:50.714 INFO:tasks.rados.rados.0.smithi029.stdout:2630: read oid 18 snap -1 2024-04-12T11:25:50.714 INFO:tasks.rados.rados.0.smithi029.stdout:2630: expect (ObjNum 939 snap 325 seq_num 939) 2024-04-12T11:25:50.714 INFO:tasks.rados.rados.0.smithi029.stdout:2631: snap_create 2024-04-12T11:25:50.714 INFO:tasks.rados.rados.0.smithi029.stdout:2623: finishing write tid 2 to smithi02979277-1 2024-04-12T11:25:50.714 INFO:tasks.rados.rados.0.smithi029.stdout:2623: finishing write tid 3 to smithi02979277-1 2024-04-12T11:25:50.714 INFO:tasks.rados.rados.0.smithi029.stdout:2624: finishing rollback tid 0 to smithi02979277-35 2024-04-12T11:25:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:2623: finishing write tid 4 to smithi02979277-1 2024-04-12T11:25:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:2623: oid 1 updating version 0 to 1257 2024-04-12T11:25:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:2623: oid 1 version 1257 is already newer than 1256 2024-04-12T11:25:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:2623: oid 1 version 1257 is already newer than 1255 2024-04-12T11:25:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1257 (ObjNum 981 snap 341 seq_num 981) dirty exists 2024-04-12T11:25:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:2623: left oid 1 (ObjNum 981 snap 341 seq_num 981) 2024-04-12T11:25:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:2624: finishing rollback tid 1 to smithi02979277-35 2024-04-12T11:25:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:2624: finishing rollback tid 2 to smithi02979277-35 2024-04-12T11:25:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 1576 (ObjNum 937 snap 325 seq_num 937) dirty exists 2024-04-12T11:25:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:2627: expect (ObjNum 864 snap 302 seq_num 864) 2024-04-12T11:25:50.833 INFO:tasks.rados.rados.0.smithi029.stdout:2628: finishing write tid 1 to smithi02979277-2 2024-04-12T11:25:50.833 INFO:tasks.rados.rados.0.smithi029.stdout:2628: finishing write tid 2 to smithi02979277-2 2024-04-12T11:25:50.833 INFO:tasks.rados.rados.0.smithi029.stdout:2628: finishing write tid 3 to smithi02979277-2 2024-04-12T11:25:50.833 INFO:tasks.rados.rados.0.smithi029.stdout:2628: finishing write tid 4 to smithi02979277-2 2024-04-12T11:25:50.833 INFO:tasks.rados.rados.0.smithi029.stdout:2628: oid 2 updating version 0 to 273 2024-04-12T11:25:50.834 INFO:tasks.rados.rados.0.smithi029.stdout:2628: oid 2 updating version 273 to 274 2024-04-12T11:25:50.834 INFO:tasks.rados.rados.0.smithi029.stdout:2628: oid 2 version 274 is already newer than 272 2024-04-12T11:25:50.834 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 274 (ObjNum 983 snap 341 seq_num 983) dirty exists 2024-04-12T11:25:50.834 INFO:tasks.rados.rados.0.smithi029.stdout:2628: left oid 2 (ObjNum 983 snap 341 seq_num 983) 2024-04-12T11:25:50.834 INFO:tasks.rados.rados.0.smithi029.stdout:2630: expect (ObjNum 939 snap 325 seq_num 939) 2024-04-12T11:25:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:50 smithi087 ceph-mon[89014]: pgmap v753: 89 pgs: 1 active+clean+snaptrim_wait, 88 active+clean; 207 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-04-12T11:25:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:50 smithi087 ceph-mon[89014]: osdmap e745: 8 total, 8 up, 8 in 2024-04-12T11:25:51.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:50 smithi029 ceph-mon[97787]: pgmap v753: 89 pgs: 1 active+clean+snaptrim_wait, 88 active+clean; 207 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-04-12T11:25:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:50 smithi029 ceph-mon[97787]: osdmap e745: 8 total, 8 up, 8 in 2024-04-12T11:25:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:50 smithi029 ceph-mon[93650]: pgmap v753: 89 pgs: 1 active+clean+snaptrim_wait, 88 active+clean; 207 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-04-12T11:25:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:50 smithi029 ceph-mon[93650]: osdmap e745: 8 total, 8 up, 8 in 2024-04-12T11:25:51.676 INFO:tasks.rados.rados.0.smithi029.stdout:2623: done (6 left) 2024-04-12T11:25:51.677 INFO:tasks.rados.rados.0.smithi029.stdout:2624: done (5 left) 2024-04-12T11:25:51.677 INFO:tasks.rados.rados.0.smithi029.stdout:2627: done (4 left) 2024-04-12T11:25:51.677 INFO:tasks.rados.rados.0.smithi029.stdout:2628: done (3 left) 2024-04-12T11:25:51.677 INFO:tasks.rados.rados.0.smithi029.stdout:2629: done (2 left) 2024-04-12T11:25:51.677 INFO:tasks.rados.rados.0.smithi029.stdout:2630: done (1 left) 2024-04-12T11:25:51.677 INFO:tasks.rados.rados.0.smithi029.stdout:2631: done (0 left) 2024-04-12T11:25:51.677 INFO:tasks.rados.rados.0.smithi029.stdout:2632: write oid 20 current snap is 342 2024-04-12T11:25:51.677 INFO:tasks.rados.rados.0.smithi029.stdout:2632: seq_num 984 ranges {538088=654553,1714752=634675} 2024-04-12T11:25:51.683 INFO:tasks.rados.rados.0.smithi029.stdout:2632: writing smithi02979277-20 from 538088 to 1192641 tid 1 2024-04-12T11:25:51.689 INFO:tasks.rados.rados.0.smithi029.stdout:2632: writing smithi02979277-20 from 1714752 to 2349427 tid 2 2024-04-12T11:25:51.689 INFO:tasks.rados.rados.0.smithi029.stdout:2633: rollback oid 39 current snap is 342 2024-04-12T11:25:51.689 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 340 2024-04-12T11:25:51.689 INFO:tasks.rados.rados.0.smithi029.stdout:2632: finishing write tid 1 to smithi02979277-20 2024-04-12T11:25:51.689 INFO:tasks.rados.rados.0.smithi029.stdout:2634: delete oid 9 current snap is 342 2024-04-12T11:25:51.694 INFO:tasks.rados.rados.0.smithi029.stdout:2632: finishing write tid 2 to smithi02979277-20 2024-04-12T11:25:51.694 INFO:tasks.rados.rados.0.smithi029.stdout:2632: finishing write tid 3 to smithi02979277-20 2024-04-12T11:25:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:2632: finishing write tid 4 to smithi02979277-20 2024-04-12T11:25:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:2632: oid 20 updating version 0 to 788 2024-04-12T11:25:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:2632: oid 20 updating version 788 to 789 2024-04-12T11:25:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:2632: oid 20 version 789 is already newer than 787 2024-04-12T11:25:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 789 (ObjNum 984 snap 342 seq_num 984) dirty exists 2024-04-12T11:25:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:2632: left oid 20 (ObjNum 984 snap 342 seq_num 984) 2024-04-12T11:25:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:2633: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:25:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:2633: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:25:51.696 INFO:tasks.rados.rados.0.smithi029.stdout:2632: done (2 left) 2024-04-12T11:25:51.696 INFO:tasks.rados.rados.0.smithi029.stdout:2634: done (1 left) 2024-04-12T11:25:51.696 INFO:tasks.rados.rados.0.smithi029.stdout:2635: read oid 12 snap -1 2024-04-12T11:25:51.696 INFO:tasks.rados.rados.0.smithi029.stdout:2635: expect deleted 2024-04-12T11:25:51.696 INFO:tasks.rados.rados.0.smithi029.stdout:2636: write oid 30 current snap is 342 2024-04-12T11:25:51.696 INFO:tasks.rados.rados.0.smithi029.stdout:2633: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:25:51.696 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 346 (ObjNum 930 snap 320 seq_num 930) dirty exists 2024-04-12T11:25:51.696 INFO:tasks.rados.rados.0.smithi029.stdout:2636: seq_num 986 ranges {571702=405051,1431716=638344,2560365=11337} 2024-04-12T11:25:51.701 INFO:tasks.rados.rados.0.smithi029.stdout:2636: writing smithi02979277-30 from 571702 to 976753 tid 1 2024-04-12T11:25:51.707 INFO:tasks.rados.rados.0.smithi029.stdout:2636: writing smithi02979277-30 from 1431716 to 2070060 tid 2 2024-04-12T11:25:51.710 INFO:tasks.rados.rados.0.smithi029.stdout:2636: writing smithi02979277-30 from 2560365 to 2571702 tid 3 2024-04-12T11:25:51.710 INFO:tasks.rados.rados.0.smithi029.stdout:2633: done (2 left) 2024-04-12T11:25:51.710 INFO:tasks.rados.rados.0.smithi029.stdout:2637: read oid 15 snap -1 2024-04-12T11:25:51.710 INFO:tasks.rados.rados.0.smithi029.stdout:2637: expect deleted 2024-04-12T11:25:51.710 INFO:tasks.rados.rados.0.smithi029.stdout:2638: snap_remove snap 329 2024-04-12T11:25:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:51 smithi087 ceph-mon[89014]: osdmap e746: 8 total, 8 up, 8 in 2024-04-12T11:25:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:52.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:51 smithi029 ceph-mon[93650]: osdmap e746: 8 total, 8 up, 8 in 2024-04-12T11:25:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:51 smithi029 ceph-mon[97787]: osdmap e746: 8 total, 8 up, 8 in 2024-04-12T11:25:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:25:52.625 INFO:tasks.rados.rados.0.smithi029.stdout:2638: done (3 left) 2024-04-12T11:25:52.625 INFO:tasks.rados.rados.0.smithi029.stdout:2639: write oid 34 current snap is 342 2024-04-12T11:25:52.625 INFO:tasks.rados.rados.0.smithi029.stdout:2639: seq_num 987 ranges {588509=480300,1773845=782002} 2024-04-12T11:25:52.631 INFO:tasks.rados.rados.0.smithi029.stdout:2639: writing smithi02979277-34 from 588509 to 1068809 tid 1 2024-04-12T11:25:52.640 INFO:tasks.rados.rados.0.smithi029.stdout:2639: writing smithi02979277-34 from 1773845 to 2555847 tid 2 2024-04-12T11:25:52.640 INFO:tasks.rados.rados.0.smithi029.stdout:2640: write oid 11 current snap is 342 2024-04-12T11:25:52.640 INFO:tasks.rados.rados.0.smithi029.stdout:2640: seq_num 988 ranges {605316=555549,1832327=642013} 2024-04-12T11:25:52.647 INFO:tasks.rados.rados.0.smithi029.stdout:2640: writing smithi02979277-11 from 605316 to 1160865 tid 1 2024-04-12T11:25:52.655 INFO:tasks.rados.rados.0.smithi029.stdout:2640: writing smithi02979277-11 from 1832327 to 2474340 tid 2 2024-04-12T11:25:52.655 INFO:tasks.rados.rados.0.smithi029.stdout:2641: snap_create 2024-04-12T11:25:52.655 INFO:tasks.rados.rados.0.smithi029.stdout:2636: finishing write tid 1 to smithi02979277-30 2024-04-12T11:25:52.655 INFO:tasks.rados.rados.0.smithi029.stdout:2636: finishing write tid 2 to smithi02979277-30 2024-04-12T11:25:52.655 INFO:tasks.rados.rados.0.smithi029.stdout:2636: finishing write tid 3 to smithi02979277-30 2024-04-12T11:25:52.655 INFO:tasks.rados.rados.0.smithi029.stdout:2636: finishing write tid 4 to smithi02979277-30 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2636: finishing write tid 5 to smithi02979277-30 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2636: oid 30 updating version 0 to 854 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2636: oid 30 version 854 is already newer than 852 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2636: oid 30 updating version 854 to 855 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2636: oid 30 version 855 is already newer than 853 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 855 (ObjNum 986 snap 342 seq_num 986) dirty exists 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2636: left oid 30 (ObjNum 986 snap 342 seq_num 986) 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2639: finishing write tid 1 to smithi02979277-34 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2639: finishing write tid 2 to smithi02979277-34 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2639: finishing write tid 3 to smithi02979277-34 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2639: finishing write tid 4 to smithi02979277-34 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2639: oid 34 updating version 0 to 1176 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2639: oid 34 updating version 1176 to 1177 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:2639: oid 34 version 1177 is already newer than 1171 2024-04-12T11:25:52.656 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1177 (ObjNum 987 snap 342 seq_num 987) dirty exists 2024-04-12T11:25:52.657 INFO:tasks.rados.rados.0.smithi029.stdout:2639: left oid 34 (ObjNum 987 snap 342 seq_num 987) 2024-04-12T11:25:52.657 INFO:tasks.rados.rados.0.smithi029.stdout:2640: finishing write tid 1 to smithi02979277-11 2024-04-12T11:25:52.658 INFO:tasks.rados.rados.0.smithi029.stdout:2640: finishing write tid 2 to smithi02979277-11 2024-04-12T11:25:52.659 INFO:tasks.rados.rados.0.smithi029.stdout:2640: finishing write tid 3 to smithi02979277-11 2024-04-12T11:25:52.659 INFO:tasks.rados.rados.0.smithi029.stdout:2640: finishing write tid 4 to smithi02979277-11 2024-04-12T11:25:52.659 INFO:tasks.rados.rados.0.smithi029.stdout:2640: oid 11 updating version 0 to 1579 2024-04-12T11:25:52.659 INFO:tasks.rados.rados.0.smithi029.stdout:2640: oid 11 updating version 1579 to 1580 2024-04-12T11:25:52.659 INFO:tasks.rados.rados.0.smithi029.stdout:2640: oid 11 updating version 1580 to 1581 2024-04-12T11:25:52.659 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 1581 (ObjNum 988 snap 342 seq_num 988) dirty exists 2024-04-12T11:25:52.659 INFO:tasks.rados.rados.0.smithi029.stdout:2640: left oid 11 (ObjNum 988 snap 342 seq_num 988) 2024-04-12T11:25:52.674 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:25:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:25:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:25:52.940 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:52 smithi087 ceph-mon[89014]: pgmap v756: 89 pgs: 1 active+clean+snaptrim_wait, 88 active+clean; 207 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 772 KiB/s wr, 21 op/s 2024-04-12T11:25:52.940 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:52 smithi087 ceph-mon[89014]: osdmap e747: 8 total, 8 up, 8 in 2024-04-12T11:25:52.940 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:52 smithi087 ceph-mon[89014]: osdmap e748: 8 total, 8 up, 8 in 2024-04-12T11:25:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:52 smithi029 ceph-mon[93650]: pgmap v756: 89 pgs: 1 active+clean+snaptrim_wait, 88 active+clean; 207 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 772 KiB/s wr, 21 op/s 2024-04-12T11:25:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:52 smithi029 ceph-mon[93650]: osdmap e747: 8 total, 8 up, 8 in 2024-04-12T11:25:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:52 smithi029 ceph-mon[93650]: osdmap e748: 8 total, 8 up, 8 in 2024-04-12T11:25:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:52 smithi029 ceph-mon[97787]: pgmap v756: 89 pgs: 1 active+clean+snaptrim_wait, 88 active+clean; 207 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 772 KiB/s wr, 21 op/s 2024-04-12T11:25:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:52 smithi029 ceph-mon[97787]: osdmap e747: 8 total, 8 up, 8 in 2024-04-12T11:25:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:52 smithi029 ceph-mon[97787]: osdmap e748: 8 total, 8 up, 8 in 2024-04-12T11:25:53.626 INFO:tasks.rados.rados.0.smithi029.stdout:2635: done (5 left) 2024-04-12T11:25:53.626 INFO:tasks.rados.rados.0.smithi029.stdout:2636: done (4 left) 2024-04-12T11:25:53.626 INFO:tasks.rados.rados.0.smithi029.stdout:2637: done (3 left) 2024-04-12T11:25:53.626 INFO:tasks.rados.rados.0.smithi029.stdout:2639: done (2 left) 2024-04-12T11:25:53.627 INFO:tasks.rados.rados.0.smithi029.stdout:2640: done (1 left) 2024-04-12T11:25:53.627 INFO:tasks.rados.rados.0.smithi029.stdout:2641: done (0 left) 2024-04-12T11:25:53.627 INFO:tasks.rados.rados.0.smithi029.stdout:2642: delete oid 37 current snap is 343 2024-04-12T11:25:53.631 INFO:tasks.rados.rados.0.smithi029.stdout:2642: done (0 left) 2024-04-12T11:25:53.631 INFO:tasks.rados.rados.0.smithi029.stdout:2643: snap_create 2024-04-12T11:25:54.632 INFO:tasks.rados.rados.0.smithi029.stdout:2643: done (0 left) 2024-04-12T11:25:54.632 INFO:tasks.rados.rados.0.smithi029.stdout:2644: write oid 43 current snap is 344 2024-04-12T11:25:54.632 INFO:tasks.rados.rados.0.smithi029.stdout:2644: seq_num 990 ranges {638930=422400,1665644=645682} 2024-04-12T11:25:54.638 INFO:tasks.rados.rados.0.smithi029.stdout:2644: writing smithi02979277-43 from 638930 to 1061330 tid 1 2024-04-12T11:25:54.645 INFO:tasks.rados.rados.0.smithi029.stdout:2644: writing smithi02979277-43 from 1665644 to 2311326 tid 2 2024-04-12T11:25:54.645 INFO:tasks.rados.rados.0.smithi029.stdout:2645: delete oid 14 current snap is 344 2024-04-12T11:25:54.645 INFO:tasks.rados.rados.0.smithi029.stdout:2644: finishing write tid 1 to smithi02979277-43 2024-04-12T11:25:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:2645: done (1 left) 2024-04-12T11:25:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:2646: read oid 49 snap -1 2024-04-12T11:25:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:2646: expect deleted 2024-04-12T11:25:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:2647: read oid 27 snap -1 2024-04-12T11:25:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:2647: expect (ObjNum 850 snap 292 seq_num 850) 2024-04-12T11:25:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:2648: write oid 3 current snap is 344 2024-04-12T11:25:54.648 INFO:tasks.rados.rados.0.smithi029.stdout:2648: seq_num 992 ranges {672544=572898,1666255=532998} 2024-04-12T11:25:54.654 INFO:tasks.rados.rados.0.smithi029.stdout:2648: writing smithi02979277-3 from 672544 to 1245442 tid 1 2024-04-12T11:25:54.660 INFO:tasks.rados.rados.0.smithi029.stdout:2648: writing smithi02979277-3 from 1666255 to 2199253 tid 2 2024-04-12T11:25:54.660 INFO:tasks.rados.rados.0.smithi029.stdout:2649: delete oid 12 current snap is 344 2024-04-12T11:25:54.660 INFO:tasks.rados.rados.0.smithi029.stdout:2644: finishing write tid 2 to smithi02979277-43 2024-04-12T11:25:54.661 INFO:tasks.rados.rados.0.smithi029.stdout:2644: finishing write tid 3 to smithi02979277-43 2024-04-12T11:25:54.661 INFO:tasks.rados.rados.0.smithi029.stdout:2644: finishing write tid 4 to smithi02979277-43 2024-04-12T11:25:54.661 INFO:tasks.rados.rados.0.smithi029.stdout:2644: oid 43 updating version 0 to 1179 2024-04-12T11:25:54.661 INFO:tasks.rados.rados.0.smithi029.stdout:2644: oid 43 updating version 1179 to 1180 2024-04-12T11:25:54.661 INFO:tasks.rados.rados.0.smithi029.stdout:2644: oid 43 updating version 1180 to 1181 2024-04-12T11:25:54.661 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1181 (ObjNum 990 snap 344 seq_num 990) dirty exists 2024-04-12T11:25:54.661 INFO:tasks.rados.rados.0.smithi029.stdout:2644: left oid 43 (ObjNum 990 snap 344 seq_num 990) 2024-04-12T11:25:54.661 INFO:tasks.rados.rados.0.smithi029.stdout:2647: expect (ObjNum 850 snap 292 seq_num 850) 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:2648: finishing write tid 1 to smithi02979277-3 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:2648: finishing write tid 2 to smithi02979277-3 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:2648: finishing write tid 3 to smithi02979277-3 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:2648: finishing write tid 4 to smithi02979277-3 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:2648: oid 3 updating version 0 to 1585 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:2648: oid 3 version 1585 is already newer than 1584 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:2648: oid 3 version 1585 is already newer than 1583 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1585 (ObjNum 992 snap 344 seq_num 992) dirty exists 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:2648: left oid 3 (ObjNum 992 snap 344 seq_num 992) 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:2644: done (4 left) 2024-04-12T11:25:54.773 INFO:tasks.rados.rados.0.smithi029.stdout:2646: done (3 left) 2024-04-12T11:25:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2647: done (2 left) 2024-04-12T11:25:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2648: done (1 left) 2024-04-12T11:25:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2649: done (0 left) 2024-04-12T11:25:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2650: rollback oid 5 current snap is 344 2024-04-12T11:25:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 5 to 325 2024-04-12T11:25:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2651: snap_create 2024-04-12T11:25:54.775 INFO:tasks.rados.rados.0.smithi029.stdout:2650: finishing rollback tid 1 to smithi02979277-5 2024-04-12T11:25:54.775 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1133 (ObjNum 49 snap 0 seq_num 1777349616) dirty dne 2024-04-12T11:25:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:54 smithi029 ceph-mon[97787]: pgmap v759: 89 pgs: 89 active+clean; 206 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 551 KiB/s wr, 5 op/s 2024-04-12T11:25:54.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:54 smithi029 ceph-mon[97787]: osdmap e749: 8 total, 8 up, 8 in 2024-04-12T11:25:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:54 smithi029 ceph-mon[93650]: pgmap v759: 89 pgs: 89 active+clean; 206 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 551 KiB/s wr, 5 op/s 2024-04-12T11:25:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:54 smithi029 ceph-mon[93650]: osdmap e749: 8 total, 8 up, 8 in 2024-04-12T11:25:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:54 smithi087 ceph-mon[89014]: pgmap v759: 89 pgs: 89 active+clean; 206 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 551 KiB/s wr, 5 op/s 2024-04-12T11:25:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:54 smithi087 ceph-mon[89014]: osdmap e749: 8 total, 8 up, 8 in 2024-04-12T11:25:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:25:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:25:54] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.43.0" 2024-04-12T11:25:55.638 INFO:tasks.rados.rados.0.smithi029.stdout:2650: done (1 left) 2024-04-12T11:25:55.638 INFO:tasks.rados.rados.0.smithi029.stdout:2651: done (0 left) 2024-04-12T11:25:55.638 INFO:tasks.rados.rados.0.smithi029.stdout:2652: read oid 46 snap -1 2024-04-12T11:25:55.638 INFO:tasks.rados.rados.0.smithi029.stdout:2652: expect deleted 2024-04-12T11:25:55.638 INFO:tasks.rados.rados.0.smithi029.stdout:2653: delete oid 17 current snap is 345 2024-04-12T11:25:55.642 INFO:tasks.rados.rados.0.smithi029.stdout:2652: done (1 left) 2024-04-12T11:25:55.642 INFO:tasks.rados.rados.0.smithi029.stdout:2653: done (0 left) 2024-04-12T11:25:55.643 INFO:tasks.rados.rados.0.smithi029.stdout:2654: write oid 38 current snap is 345 2024-04-12T11:25:55.643 INFO:tasks.rados.rados.0.smithi029.stdout:2654: seq_num 995 ranges {722965=798645,2125348=597617} 2024-04-12T11:25:55.651 INFO:tasks.rados.rados.0.smithi029.stdout:2654: writing smithi02979277-38 from 722965 to 1521610 tid 1 2024-04-12T11:25:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2654: writing smithi02979277-38 from 2125348 to 2722965 tid 2 2024-04-12T11:25:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2655: write oid 16 current snap is 345 2024-04-12T11:25:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2654: finishing write tid 1 to smithi02979277-38 2024-04-12T11:25:55.658 INFO:tasks.rados.rados.0.smithi029.stdout:2655: seq_num 996 ranges {739772=590247,1900183=540336} 2024-04-12T11:25:55.665 INFO:tasks.rados.rados.0.smithi029.stdout:2655: writing smithi02979277-16 from 739772 to 1330019 tid 1 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2655: writing smithi02979277-16 from 1900183 to 2440519 tid 2 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2656: rollback oid 4 current snap is 345 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 4 to 330 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2654: finishing write tid 2 to smithi02979277-38 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2657: read oid 34 snap 343 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2657: expect (ObjNum 987 snap 342 seq_num 987) 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2658: delete oid 7 current snap is 345 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2654: finishing write tid 3 to smithi02979277-38 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2654: finishing write tid 4 to smithi02979277-38 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2654: oid 38 updating version 0 to 793 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2654: oid 38 updating version 793 to 795 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2654: oid 38 version 795 is already newer than 794 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 795 (ObjNum 995 snap 345 seq_num 995) dirty exists 2024-04-12T11:25:55.672 INFO:tasks.rados.rados.0.smithi029.stdout:2654: left oid 38 (ObjNum 995 snap 345 seq_num 995) 2024-04-12T11:25:55.673 INFO:tasks.rados.rados.0.smithi029.stdout:2655: finishing write tid 1 to smithi02979277-16 2024-04-12T11:25:55.675 INFO:tasks.rados.rados.0.smithi029.stdout:2654: done (4 left) 2024-04-12T11:25:55.675 INFO:tasks.rados.rados.0.smithi029.stdout:2658: done (3 left) 2024-04-12T11:25:55.675 INFO:tasks.rados.rados.0.smithi029.stdout:2659: read oid 20 snap -1 2024-04-12T11:25:55.675 INFO:tasks.rados.rados.0.smithi029.stdout:2659: expect (ObjNum 984 snap 342 seq_num 984) 2024-04-12T11:25:55.675 INFO:tasks.rados.rados.0.smithi029.stdout:2660: read oid 47 snap -1 2024-04-12T11:25:55.675 INFO:tasks.rados.rados.0.smithi029.stdout:2660: expect deleted 2024-04-12T11:25:55.675 INFO:tasks.rados.rados.0.smithi029.stdout:2661: write oid 41 current snap is 345 2024-04-12T11:25:55.675 INFO:tasks.rados.rados.0.smithi029.stdout:2661: seq_num 998 ranges {773386=740745,2017147=544005} 2024-04-12T11:25:55.683 INFO:tasks.rados.rados.0.smithi029.stdout:2661: writing smithi02979277-41 from 773386 to 1514131 tid 1 2024-04-12T11:25:55.689 INFO:tasks.rados.rados.0.smithi029.stdout:2661: writing smithi02979277-41 from 2017147 to 2561152 tid 2 2024-04-12T11:25:55.689 INFO:tasks.rados.rados.0.smithi029.stdout:2662: write oid 50 current snap is 345 2024-04-12T11:25:55.689 INFO:tasks.rados.rados.0.smithi029.stdout:2657: expect (ObjNum 987 snap 342 seq_num 987) 2024-04-12T11:25:55.812 INFO:tasks.rados.rados.0.smithi029.stdout:2655: finishing write tid 2 to smithi02979277-16 2024-04-12T11:25:55.812 INFO:tasks.rados.rados.0.smithi029.stdout:2655: finishing write tid 3 to smithi02979277-16 2024-04-12T11:25:55.812 INFO:tasks.rados.rados.0.smithi029.stdout:2655: finishing write tid 4 to smithi02979277-16 2024-04-12T11:25:55.812 INFO:tasks.rados.rados.0.smithi029.stdout:2655: oid 16 updating version 0 to 1211 2024-04-12T11:25:55.813 INFO:tasks.rados.rados.0.smithi029.stdout:2655: oid 16 version 1211 is already newer than 1209 2024-04-12T11:25:55.813 INFO:tasks.rados.rados.0.smithi029.stdout:2655: oid 16 version 1211 is already newer than 1210 2024-04-12T11:25:55.813 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1211 (ObjNum 996 snap 345 seq_num 996) dirty exists 2024-04-12T11:25:55.813 INFO:tasks.rados.rados.0.smithi029.stdout:2655: left oid 16 (ObjNum 996 snap 345 seq_num 996) 2024-04-12T11:25:55.813 INFO:tasks.rados.rados.0.smithi029.stdout:2656: finishing rollback tid 0 to smithi02979277-4 2024-04-12T11:25:55.813 INFO:tasks.rados.rados.0.smithi029.stdout:2656: finishing rollback tid 1 to smithi02979277-4 2024-04-12T11:25:55.813 INFO:tasks.rados.rados.0.smithi029.stdout:2656: finishing rollback tid 2 to smithi02979277-4 2024-04-12T11:25:55.813 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1215 (ObjNum 961 snap 330 seq_num 961) dirty exists 2024-04-12T11:25:55.813 INFO:tasks.rados.rados.0.smithi029.stdout:2659: expect (ObjNum 984 snap 342 seq_num 984) 2024-04-12T11:25:55.895 INFO:tasks.rados.rados.0.smithi029.stdout:2661: finishing write tid 1 to smithi02979277-41 2024-04-12T11:25:55.895 INFO:tasks.rados.rados.0.smithi029.stdout:2661: finishing write tid 2 to smithi02979277-41 2024-04-12T11:25:55.895 INFO:tasks.rados.rados.0.smithi029.stdout:2661: finishing write tid 3 to smithi02979277-41 2024-04-12T11:25:55.896 INFO:tasks.rados.rados.0.smithi029.stdout:2661: finishing write tid 4 to smithi02979277-41 2024-04-12T11:25:55.896 INFO:tasks.rados.rados.0.smithi029.stdout:2661: oid 41 updating version 0 to 1263 2024-04-12T11:25:55.896 INFO:tasks.rados.rados.0.smithi029.stdout:2661: oid 41 updating version 1263 to 1264 2024-04-12T11:25:55.896 INFO:tasks.rados.rados.0.smithi029.stdout:2661: oid 41 updating version 1264 to 1265 2024-04-12T11:25:55.896 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1265 (ObjNum 998 snap 345 seq_num 998) dirty exists 2024-04-12T11:25:55.896 INFO:tasks.rados.rados.0.smithi029.stdout:2661: left oid 41 (ObjNum 998 snap 345 seq_num 998) 2024-04-12T11:25:55.896 INFO:tasks.rados.rados.0.smithi029.stdout:2662: seq_num 999 ranges {790193=415994,1959276=404016} 2024-04-12T11:25:55.902 INFO:tasks.rados.rados.0.smithi029.stdout:2662: writing smithi02979277-50 from 790193 to 1206187 tid 1 2024-04-12T11:25:55.908 INFO:tasks.rados.rados.0.smithi029.stdout:2662: writing smithi02979277-50 from 1959276 to 2363292 tid 2 2024-04-12T11:25:55.908 INFO:tasks.rados.rados.0.smithi029.stdout:2655: done (6 left) 2024-04-12T11:25:55.908 INFO:tasks.rados.rados.0.smithi029.stdout:2656: done (5 left) 2024-04-12T11:25:55.909 INFO:tasks.rados.rados.0.smithi029.stdout:2657: done (4 left) 2024-04-12T11:25:55.909 INFO:tasks.rados.rados.0.smithi029.stdout:2659: done (3 left) 2024-04-12T11:25:55.909 INFO:tasks.rados.rados.0.smithi029.stdout:2660: done (2 left) 2024-04-12T11:25:55.909 INFO:tasks.rados.rados.0.smithi029.stdout:2661: done (1 left) 2024-04-12T11:25:55.909 INFO:tasks.rados.rados.0.smithi029.stdout:2663: write oid 1 current snap is 345 2024-04-12T11:25:55.909 INFO:tasks.rados.rados.0.smithi029.stdout:2662: finishing write tid 1 to smithi02979277-50 2024-04-12T11:25:55.909 INFO:tasks.rados.rados.0.smithi029.stdout:2663: seq_num 1000 ranges {407000=491243,1617758=547674,2595228=211772} 2024-04-12T11:25:55.913 INFO:tasks.rados.rados.0.smithi029.stdout:2663: writing smithi02979277-1 from 407000 to 898243 tid 1 2024-04-12T11:25:55.920 INFO:tasks.rados.rados.0.smithi029.stdout:2663: writing smithi02979277-1 from 1617758 to 2165432 tid 2 2024-04-12T11:25:55.924 INFO:tasks.rados.rados.0.smithi029.stdout:2663: writing smithi02979277-1 from 2595228 to 2807000 tid 3 2024-04-12T11:25:55.924 INFO:tasks.rados.rados.0.smithi029.stdout:2664: snap_remove snap 336 2024-04-12T11:25:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:55 smithi087 ceph-mon[89014]: osdmap e750: 8 total, 8 up, 8 in 2024-04-12T11:25:56.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:55 smithi029 ceph-mon[97787]: osdmap e750: 8 total, 8 up, 8 in 2024-04-12T11:25:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:55 smithi029 ceph-mon[93650]: osdmap e750: 8 total, 8 up, 8 in 2024-04-12T11:25:56.642 INFO:tasks.rados.rados.0.smithi029.stdout:2664: done (2 left) 2024-04-12T11:25:56.642 INFO:tasks.rados.rados.0.smithi029.stdout:2665: rollback oid 32 current snap is 345 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 32 to 344 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2662: finishing write tid 2 to smithi02979277-50 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2662: finishing write tid 3 to smithi02979277-50 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2662: finishing write tid 4 to smithi02979277-50 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2662: oid 50 updating version 0 to 1099 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2662: oid 50 version 1099 is already newer than 1098 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2662: oid 50 version 1099 is already newer than 1097 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1099 (ObjNum 999 snap 345 seq_num 999) dirty exists 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2662: left oid 50 (ObjNum 999 snap 345 seq_num 999) 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2663: finishing write tid 1 to smithi02979277-1 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2663: finishing write tid 2 to smithi02979277-1 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2663: finishing write tid 3 to smithi02979277-1 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2663: finishing write tid 4 to smithi02979277-1 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2663: finishing write tid 5 to smithi02979277-1 2024-04-12T11:25:56.643 INFO:tasks.rados.rados.0.smithi029.stdout:2663: oid 1 updating version 0 to 1268 2024-04-12T11:25:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:2663: oid 1 version 1268 is already newer than 1267 2024-04-12T11:25:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:2663: oid 1 updating version 1268 to 1269 2024-04-12T11:25:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:2663: oid 1 updating version 1269 to 1270 2024-04-12T11:25:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1270 (ObjNum 1000 snap 345 seq_num 1000) dirty exists 2024-04-12T11:25:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:2663: left oid 1 (ObjNum 1000 snap 345 seq_num 1000) 2024-04-12T11:25:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:2662: done (2 left) 2024-04-12T11:25:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:2663: done (1 left) 2024-04-12T11:25:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:2666: write oid 36 current snap is 345 2024-04-12T11:25:56.644 INFO:tasks.rados.rados.0.smithi029.stdout:2666: seq_num 1001 ranges {423807=566492,1676240=407685,2739004=84803} 2024-04-12T11:25:56.648 INFO:tasks.rados.rados.0.smithi029.stdout:2666: writing smithi02979277-36 from 423807 to 990299 tid 1 2024-04-12T11:25:56.654 INFO:tasks.rados.rados.0.smithi029.stdout:2666: writing smithi02979277-36 from 1676240 to 2083925 tid 2 2024-04-12T11:25:56.659 INFO:tasks.rados.rados.0.smithi029.stdout:2666: writing smithi02979277-36 from 2739004 to 2823807 tid 3 2024-04-12T11:25:56.659 INFO:tasks.rados.rados.0.smithi029.stdout:2667: snap_remove snap 337 2024-04-12T11:25:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:56 smithi087 ceph-mon[89014]: pgmap v762: 89 pgs: 89 active+clean; 215 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-04-12T11:25:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:56 smithi087 ceph-mon[89014]: osdmap e751: 8 total, 8 up, 8 in 2024-04-12T11:25:57.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:56 smithi029 ceph-mon[93650]: pgmap v762: 89 pgs: 89 active+clean; 215 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-04-12T11:25:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:56 smithi029 ceph-mon[93650]: osdmap e751: 8 total, 8 up, 8 in 2024-04-12T11:25:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:56 smithi029 ceph-mon[97787]: pgmap v762: 89 pgs: 89 active+clean; 215 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-04-12T11:25:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:56 smithi029 ceph-mon[97787]: osdmap e751: 8 total, 8 up, 8 in 2024-04-12T11:25:57.626 INFO:tasks.rados.rados.0.smithi029.stdout:2667: done (2 left) 2024-04-12T11:25:57.626 INFO:tasks.rados.rados.0.smithi029.stdout:2668: read oid 19 snap -1 2024-04-12T11:25:57.626 INFO:tasks.rados.rados.0.smithi029.stdout:2668: expect (ObjNum 906 snap 314 seq_num 906) 2024-04-12T11:25:57.627 INFO:tasks.rados.rados.0.smithi029.stdout:2669: write oid 25 current snap is 345 2024-04-12T11:25:57.627 INFO:tasks.rados.rados.0.smithi029.stdout:2665: finishing rollback tid 1 to smithi02979277-32 2024-04-12T11:25:57.627 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 1161 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-04-12T11:25:57.627 INFO:tasks.rados.rados.0.smithi029.stdout:2666: finishing write tid 1 to smithi02979277-36 2024-04-12T11:25:57.627 INFO:tasks.rados.rados.0.smithi029.stdout:2666: finishing write tid 2 to smithi02979277-36 2024-04-12T11:25:57.627 INFO:tasks.rados.rados.0.smithi029.stdout:2666: finishing write tid 3 to smithi02979277-36 2024-04-12T11:25:57.627 INFO:tasks.rados.rados.0.smithi029.stdout:2669: seq_num 1002 ranges {440614=641741,1734722=551343} 2024-04-12T11:25:57.632 INFO:tasks.rados.rados.0.smithi029.stdout:2669: writing smithi02979277-25 from 440614 to 1082355 tid 1 2024-04-12T11:25:57.641 INFO:tasks.rados.rados.0.smithi029.stdout:2669: writing smithi02979277-25 from 1734722 to 2286065 tid 2 2024-04-12T11:25:57.641 INFO:tasks.rados.rados.0.smithi029.stdout:2665: done (3 left) 2024-04-12T11:25:57.641 INFO:tasks.rados.rados.0.smithi029.stdout:2670: read oid 20 snap 342 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2670: expect (ObjNum 984 snap 342 seq_num 984) 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2671: write oid 30 current snap is 345 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2666: finishing write tid 4 to smithi02979277-36 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2666: finishing write tid 5 to smithi02979277-36 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2666: oid 36 updating version 0 to 689 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2666: oid 36 updating version 689 to 690 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2666: oid 36 updating version 690 to 692 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2666: oid 36 version 692 is already newer than 691 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 692 (ObjNum 1001 snap 345 seq_num 1001) dirty exists 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2666: left oid 36 (ObjNum 1001 snap 345 seq_num 1001) 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2669: finishing write tid 1 to smithi02979277-25 2024-04-12T11:25:57.642 INFO:tasks.rados.rados.0.smithi029.stdout:2671: seq_num 1003 ranges {457421=716990,1676851=411354,2677497=179924} 2024-04-12T11:25:57.649 INFO:tasks.rados.rados.0.smithi029.stdout:2671: writing smithi02979277-30 from 457421 to 1174411 tid 1 2024-04-12T11:25:57.654 INFO:tasks.rados.rados.0.smithi029.stdout:2671: writing smithi02979277-30 from 1676851 to 2088205 tid 2 2024-04-12T11:25:57.659 INFO:tasks.rados.rados.0.smithi029.stdout:2671: writing smithi02979277-30 from 2677497 to 2857421 tid 3 2024-04-12T11:25:57.659 INFO:tasks.rados.rados.0.smithi029.stdout:2666: done (4 left) 2024-04-12T11:25:57.659 INFO:tasks.rados.rados.0.smithi029.stdout:2672: snap_create 2024-04-12T11:25:57.659 INFO:tasks.rados.rados.0.smithi029.stdout:2668: expect (ObjNum 906 snap 314 seq_num 906) 2024-04-12T11:25:57.845 INFO:tasks.rados.rados.0.smithi029.stdout:2670: expect (ObjNum 984 snap 342 seq_num 984) 2024-04-12T11:25:57.928 INFO:tasks.rados.rados.0.smithi029.stdout:2669: finishing write tid 2 to smithi02979277-25 2024-04-12T11:25:57.928 INFO:tasks.rados.rados.0.smithi029.stdout:2669: finishing write tid 3 to smithi02979277-25 2024-04-12T11:25:57.928 INFO:tasks.rados.rados.0.smithi029.stdout:2669: finishing write tid 4 to smithi02979277-25 2024-04-12T11:25:57.928 INFO:tasks.rados.rados.0.smithi029.stdout:2669: oid 25 updating version 0 to 1346 2024-04-12T11:25:57.928 INFO:tasks.rados.rados.0.smithi029.stdout:2669: oid 25 version 1346 is already newer than 1344 2024-04-12T11:25:57.928 INFO:tasks.rados.rados.0.smithi029.stdout:2669: oid 25 version 1346 is already newer than 1345 2024-04-12T11:25:57.928 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 1346 (ObjNum 1002 snap 345 seq_num 1002) dirty exists 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2669: left oid 25 (ObjNum 1002 snap 345 seq_num 1002) 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2671: finishing write tid 1 to smithi02979277-30 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2671: finishing write tid 2 to smithi02979277-30 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2671: finishing write tid 3 to smithi02979277-30 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2671: finishing write tid 4 to smithi02979277-30 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2671: finishing write tid 5 to smithi02979277-30 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2671: oid 30 updating version 0 to 865 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2671: oid 30 version 865 is already newer than 863 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2671: oid 30 version 865 is already newer than 864 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2671: oid 30 updating version 865 to 866 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 866 (ObjNum 1003 snap 345 seq_num 1003) dirty exists 2024-04-12T11:25:57.929 INFO:tasks.rados.rados.0.smithi029.stdout:2671: left oid 30 (ObjNum 1003 snap 345 seq_num 1003) 2024-04-12T11:25:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:57 smithi087 ceph-mon[89014]: osdmap e752: 8 total, 8 up, 8 in 2024-04-12T11:25:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:57 smithi087 ceph-mon[89014]: osdmap e753: 8 total, 8 up, 8 in 2024-04-12T11:25:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:57 smithi029 ceph-mon[93650]: osdmap e752: 8 total, 8 up, 8 in 2024-04-12T11:25:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:57 smithi029 ceph-mon[93650]: osdmap e753: 8 total, 8 up, 8 in 2024-04-12T11:25:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:57 smithi029 ceph-mon[97787]: osdmap e752: 8 total, 8 up, 8 in 2024-04-12T11:25:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:57 smithi029 ceph-mon[97787]: osdmap e753: 8 total, 8 up, 8 in 2024-04-12T11:25:58.654 INFO:tasks.rados.rados.0.smithi029.stdout:2668: done (4 left) 2024-04-12T11:25:58.654 INFO:tasks.rados.rados.0.smithi029.stdout:2669: done (3 left) 2024-04-12T11:25:58.654 INFO:tasks.rados.rados.0.smithi029.stdout:2670: done (2 left) 2024-04-12T11:25:58.654 INFO:tasks.rados.rados.0.smithi029.stdout:2671: done (1 left) 2024-04-12T11:25:58.654 INFO:tasks.rados.rados.0.smithi029.stdout:2672: done (0 left) 2024-04-12T11:25:58.654 INFO:tasks.rados.rados.0.smithi029.stdout:2673: delete oid 4 current snap is 346 2024-04-12T11:25:58.656 INFO:tasks.rados.rados.0.smithi029.stdout:2673: done (0 left) 2024-04-12T11:25:58.656 INFO:tasks.rados.rados.0.smithi029.stdout:2674: rollback oid 2 current snap is 346 2024-04-12T11:25:58.656 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 2 to 339 2024-04-12T11:25:58.656 INFO:tasks.rados.rados.0.smithi029.stdout:2675: snap_create 2024-04-12T11:25:58.660 INFO:tasks.rados.rados.0.smithi029.stdout:2674: finishing rollback tid 0 to smithi02979277-2 2024-04-12T11:25:58.661 INFO:tasks.rados.rados.0.smithi029.stdout:2674: finishing rollback tid 1 to smithi02979277-2 2024-04-12T11:25:58.661 INFO:tasks.rados.rados.0.smithi029.stdout:2674: finishing rollback tid 2 to smithi02979277-2 2024-04-12T11:25:58.661 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 284 (ObjNum 922 snap 318 seq_num 922) dirty exists 2024-04-12T11:25:58.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:58 smithi087 ceph-mon[89014]: pgmap v765: 89 pgs: 89 active+clean; 222 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 14 op/s 2024-04-12T11:25:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:58 smithi029 ceph-mon[93650]: pgmap v765: 89 pgs: 89 active+clean; 222 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 14 op/s 2024-04-12T11:25:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:58 smithi029 ceph-mon[97787]: pgmap v765: 89 pgs: 89 active+clean; 222 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 14 op/s 2024-04-12T11:25:59.660 INFO:tasks.rados.rados.0.smithi029.stdout:2674: done (1 left) 2024-04-12T11:25:59.661 INFO:tasks.rados.rados.0.smithi029.stdout:2675: done (0 left) 2024-04-12T11:25:59.661 INFO:tasks.rados.rados.0.smithi029.stdout:2676: read oid 35 snap -1 2024-04-12T11:25:59.661 INFO:tasks.rados.rados.0.smithi029.stdout:2676: expect (ObjNum 937 snap 325 seq_num 937) 2024-04-12T11:25:59.661 INFO:tasks.rados.rados.0.smithi029.stdout:2677: write oid 26 current snap is 347 2024-04-12T11:25:59.661 INFO:tasks.rados.rados.0.smithi029.stdout:2677: seq_num 1005 ranges {491035=583841,1510168=698670,2732343=158692} 2024-04-12T11:25:59.666 INFO:tasks.rados.rados.0.smithi029.stdout:2677: writing smithi02979277-26 from 491035 to 1074876 tid 1 2024-04-12T11:25:59.673 INFO:tasks.rados.rados.0.smithi029.stdout:2677: writing smithi02979277-26 from 1510168 to 2208838 tid 2 2024-04-12T11:25:59.677 INFO:tasks.rados.rados.0.smithi029.stdout:2677: writing smithi02979277-26 from 2732343 to 2891035 tid 3 2024-04-12T11:25:59.677 INFO:tasks.rados.rados.0.smithi029.stdout:2678: write oid 41 current snap is 347 2024-04-12T11:25:59.677 INFO:tasks.rados.rados.0.smithi029.stdout:2678: seq_num 1006 ranges {507842=659090,1568650=558681,2759766=148076} 2024-04-12T11:25:59.683 INFO:tasks.rados.rados.0.smithi029.stdout:2678: writing smithi02979277-41 from 507842 to 1166932 tid 1 2024-04-12T11:25:59.689 INFO:tasks.rados.rados.0.smithi029.stdout:2678: writing smithi02979277-41 from 1568650 to 2127331 tid 2 2024-04-12T11:25:59.693 INFO:tasks.rados.rados.0.smithi029.stdout:2678: writing smithi02979277-41 from 2759766 to 2907842 tid 3 2024-04-12T11:25:59.693 INFO:tasks.rados.rados.0.smithi029.stdout:2679: read oid 32 snap 342 2024-04-12T11:25:59.694 INFO:tasks.rados.rados.0.smithi029.stdout:2679: expect deleted 2024-04-12T11:25:59.694 INFO:tasks.rados.rados.0.smithi029.stdout:2680: write oid 19 current snap is 347 2024-04-12T11:25:59.694 INFO:tasks.rados.rados.0.smithi029.stdout:2680: seq_num 1007 ranges {524649=734339,1910779=702339} 2024-04-12T11:25:59.700 INFO:tasks.rados.rados.0.smithi029.stdout:2680: writing smithi02979277-19 from 524649 to 1258988 tid 1 2024-04-12T11:25:59.708 INFO:tasks.rados.rados.0.smithi029.stdout:2680: writing smithi02979277-19 from 1910779 to 2613118 tid 2 2024-04-12T11:25:59.708 INFO:tasks.rados.rados.0.smithi029.stdout:2681: read oid 39 snap -1 2024-04-12T11:25:59.708 INFO:tasks.rados.rados.0.smithi029.stdout:2681: expect (ObjNum 930 snap 320 seq_num 930) 2024-04-12T11:25:59.708 INFO:tasks.rados.rados.0.smithi029.stdout:2682: rollback oid 7 current snap is 347 2024-04-12T11:25:59.708 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 331 2024-04-12T11:25:59.708 INFO:tasks.rados.rados.0.smithi029.stdout:2676: expect (ObjNum 937 snap 325 seq_num 937) 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:2677: finishing write tid 1 to smithi02979277-26 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:2677: finishing write tid 2 to smithi02979277-26 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:2677: finishing write tid 3 to smithi02979277-26 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:2677: finishing write tid 4 to smithi02979277-26 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:2677: finishing write tid 5 to smithi02979277-26 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:2677: oid 26 updating version 0 to 1598 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:2677: oid 26 version 1598 is already newer than 1597 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:2677: oid 26 version 1598 is already newer than 1596 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:2677: oid 26 version 1598 is already newer than 1595 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 1598 (ObjNum 1005 snap 347 seq_num 1005) dirty exists 2024-04-12T11:25:59.837 INFO:tasks.rados.rados.0.smithi029.stdout:2677: left oid 26 (ObjNum 1005 snap 347 seq_num 1005) 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2678: finishing write tid 1 to smithi02979277-41 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2676: done (6 left) 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2677: done (5 left) 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2683: read oid 47 snap -1 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2683: expect deleted 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2678: finishing write tid 2 to smithi02979277-41 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2684: read oid 38 snap 331 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2684: expect (ObjNum 864 snap 302 seq_num 864) 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2685: read oid 20 snap -1 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2685: expect (ObjNum 984 snap 342 seq_num 984) 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2686: read oid 43 snap 325 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2686: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2687: write oid 44 current snap is 347 2024-04-12T11:25:59.838 INFO:tasks.rados.rados.0.smithi029.stdout:2687: seq_num 1008 ranges {541456=409588,1569261=562350,2814612=126844} 2024-04-12T11:25:59.841 INFO:tasks.rados.rados.0.smithi029.stdout:2687: writing smithi02979277-44 from 541456 to 951044 tid 1 2024-04-12T11:25:59.848 INFO:tasks.rados.rados.0.smithi029.stdout:2687: writing smithi02979277-44 from 1569261 to 2131611 tid 2 2024-04-12T11:25:59.853 INFO:tasks.rados.rados.0.smithi029.stdout:2687: writing smithi02979277-44 from 2814612 to 2941456 tid 3 2024-04-12T11:25:59.853 INFO:tasks.rados.rados.0.smithi029.stdout:2679: done (9 left) 2024-04-12T11:25:59.853 INFO:tasks.rados.rados.0.smithi029.stdout:2688: snap_remove snap 342 2024-04-12T11:25:59.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:25:59 smithi087 ceph-mon[89014]: osdmap e754: 8 total, 8 up, 8 in 2024-04-12T11:26:00.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:25:59 smithi029 ceph-mon[93650]: osdmap e754: 8 total, 8 up, 8 in 2024-04-12T11:26:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:25:59 smithi029 ceph-mon[97787]: osdmap e754: 8 total, 8 up, 8 in 2024-04-12T11:26:00.668 INFO:tasks.rados.rados.0.smithi029.stdout:2688: done (9 left) 2024-04-12T11:26:00.668 INFO:tasks.rados.rados.0.smithi029.stdout:2689: read oid 46 snap -1 2024-04-12T11:26:00.668 INFO:tasks.rados.rados.0.smithi029.stdout:2689: expect deleted 2024-04-12T11:26:00.668 INFO:tasks.rados.rados.0.smithi029.stdout:2690: read oid 23 snap -1 2024-04-12T11:26:00.668 INFO:tasks.rados.rados.0.smithi029.stdout:2690: expect deleted 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2691: snap_create 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2678: finishing write tid 3 to smithi02979277-41 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2678: finishing write tid 4 to smithi02979277-41 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2678: finishing write tid 5 to smithi02979277-41 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2678: oid 41 updating version 0 to 1278 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2678: oid 41 version 1278 is already newer than 1276 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2678: oid 41 version 1278 is already newer than 1277 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2678: oid 41 updating version 1278 to 1279 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1279 (ObjNum 1006 snap 347 seq_num 1006) dirty exists 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2678: left oid 41 (ObjNum 1006 snap 347 seq_num 1006) 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2680: finishing write tid 1 to smithi02979277-19 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2682: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2682: finishing rollback tid 2 to smithi02979277-7 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1273 (ObjNum 959 snap 329 seq_num 959) dirty exists 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2680: finishing write tid 2 to smithi02979277-19 2024-04-12T11:26:00.669 INFO:tasks.rados.rados.0.smithi029.stdout:2680: finishing write tid 3 to smithi02979277-19 2024-04-12T11:26:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:2680: finishing write tid 4 to smithi02979277-19 2024-04-12T11:26:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:2680: oid 19 updating version 0 to 1227 2024-04-12T11:26:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:2680: oid 19 updating version 1227 to 1228 2024-04-12T11:26:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:2680: oid 19 updating version 1228 to 1229 2024-04-12T11:26:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1229 (ObjNum 1007 snap 347 seq_num 1007) dirty exists 2024-04-12T11:26:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:2680: left oid 19 (ObjNum 1007 snap 347 seq_num 1007) 2024-04-12T11:26:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:2681: expect (ObjNum 930 snap 320 seq_num 930) 2024-04-12T11:26:00.878 INFO:tasks.rados.rados.0.smithi029.stdout:2686: expect (ObjNum 427 snap 135 seq_num 427) 2024-04-12T11:26:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:00 smithi087 ceph-mon[89014]: pgmap v768: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 226 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-04-12T11:26:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:00 smithi087 ceph-mon[89014]: osdmap e755: 8 total, 8 up, 8 in 2024-04-12T11:26:00.986 INFO:tasks.rados.rados.0.smithi029.stdout:2684: expect (ObjNum 864 snap 302 seq_num 864) 2024-04-12T11:26:01.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:00 smithi029 ceph-mon[97787]: pgmap v768: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 226 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-04-12T11:26:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:00 smithi029 ceph-mon[97787]: osdmap e755: 8 total, 8 up, 8 in 2024-04-12T11:26:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:00 smithi029 ceph-mon[93650]: pgmap v768: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 226 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2024-04-12T11:26:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:00 smithi029 ceph-mon[93650]: osdmap e755: 8 total, 8 up, 8 in 2024-04-12T11:26:01.106 INFO:tasks.rados.rados.0.smithi029.stdout:2685: expect (ObjNum 984 snap 342 seq_num 984) 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:2687: finishing write tid 1 to smithi02979277-44 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:2687: finishing write tid 2 to smithi02979277-44 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:2687: finishing write tid 3 to smithi02979277-44 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:2687: finishing write tid 4 to smithi02979277-44 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:2687: finishing write tid 5 to smithi02979277-44 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:2687: oid 44 updating version 0 to 1283 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:2687: oid 44 version 1283 is already newer than 1282 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:2687: oid 44 version 1283 is already newer than 1280 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:2687: oid 44 version 1283 is already newer than 1281 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1283 (ObjNum 1008 snap 347 seq_num 1008) dirty exists 2024-04-12T11:26:01.188 INFO:tasks.rados.rados.0.smithi029.stdout:2687: left oid 44 (ObjNum 1008 snap 347 seq_num 1008) 2024-04-12T11:26:01.670 INFO:tasks.rados.rados.0.smithi029.stdout:2678: done (11 left) 2024-04-12T11:26:01.670 INFO:tasks.rados.rados.0.smithi029.stdout:2680: done (10 left) 2024-04-12T11:26:01.670 INFO:tasks.rados.rados.0.smithi029.stdout:2681: done (9 left) 2024-04-12T11:26:01.670 INFO:tasks.rados.rados.0.smithi029.stdout:2682: done (8 left) 2024-04-12T11:26:01.670 INFO:tasks.rados.rados.0.smithi029.stdout:2683: done (7 left) 2024-04-12T11:26:01.671 INFO:tasks.rados.rados.0.smithi029.stdout:2684: done (6 left) 2024-04-12T11:26:01.671 INFO:tasks.rados.rados.0.smithi029.stdout:2685: done (5 left) 2024-04-12T11:26:01.671 INFO:tasks.rados.rados.0.smithi029.stdout:2686: done (4 left) 2024-04-12T11:26:01.671 INFO:tasks.rados.rados.0.smithi029.stdout:2687: done (3 left) 2024-04-12T11:26:01.671 INFO:tasks.rados.rados.0.smithi029.stdout:2689: done (2 left) 2024-04-12T11:26:01.671 INFO:tasks.rados.rados.0.smithi029.stdout:2690: done (1 left) 2024-04-12T11:26:01.671 INFO:tasks.rados.rados.0.smithi029.stdout:2691: done (0 left) 2024-04-12T11:26:01.671 INFO:tasks.rados.rados.0.smithi029.stdout:2692: write oid 44 current snap is 348 2024-04-12T11:26:01.671 INFO:tasks.rados.rados.0.smithi029.stdout:2692: seq_num 1009 ranges {558263=484837,1627743=706008} 2024-04-12T11:26:01.676 INFO:tasks.rados.rados.0.smithi029.stdout:2692: writing smithi02979277-44 from 558263 to 1043100 tid 1 2024-04-12T11:26:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:2692: writing smithi02979277-44 from 1627743 to 2333751 tid 2 2024-04-12T11:26:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:2693: write oid 47 current snap is 348 2024-04-12T11:26:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:2692: finishing write tid 1 to smithi02979277-44 2024-04-12T11:26:01.683 INFO:tasks.rados.rados.0.smithi029.stdout:2693: seq_num 1010 ranges {575070=560086,1686225=566019,2869458=105612} 2024-04-12T11:26:01.689 INFO:tasks.rados.rados.0.smithi029.stdout:2693: writing smithi02979277-47 from 575070 to 1135156 tid 1 2024-04-12T11:26:01.695 INFO:tasks.rados.rados.0.smithi029.stdout:2693: writing smithi02979277-47 from 1686225 to 2252244 tid 2 2024-04-12T11:26:01.699 INFO:tasks.rados.rados.0.smithi029.stdout:2693: writing smithi02979277-47 from 2869458 to 2975070 tid 3 2024-04-12T11:26:01.699 INFO:tasks.rados.rados.0.smithi029.stdout:2694: read oid 12 snap -1 2024-04-12T11:26:01.699 INFO:tasks.rados.rados.0.smithi029.stdout:2694: expect deleted 2024-04-12T11:26:01.699 INFO:tasks.rados.rados.0.smithi029.stdout:2695: write oid 14 current snap is 348 2024-04-12T11:26:01.699 INFO:tasks.rados.rados.0.smithi029.stdout:2695: seq_num 1011 ranges {591877=635335,1628354=709677} 2024-04-12T11:26:01.706 INFO:tasks.rados.rados.0.smithi029.stdout:2695: writing smithi02979277-14 from 591877 to 1227212 tid 1 2024-04-12T11:26:01.712 INFO:tasks.rados.rados.0.smithi029.stdout:2695: writing smithi02979277-14 from 1628354 to 2338031 tid 2 2024-04-12T11:26:01.712 INFO:tasks.rados.rados.0.smithi029.stdout:2696: rollback oid 16 current snap is 348 2024-04-12T11:26:01.713 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 16 to 317 2024-04-12T11:26:01.713 INFO:tasks.rados.rados.0.smithi029.stdout:2692: finishing write tid 2 to smithi02979277-44 2024-04-12T11:26:01.713 INFO:tasks.rados.rados.0.smithi029.stdout:2697: snap_remove snap 346 2024-04-12T11:26:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:01 smithi087 ceph-mon[89014]: osdmap e756: 8 total, 8 up, 8 in 2024-04-12T11:26:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:01 smithi029 ceph-mon[93650]: osdmap e756: 8 total, 8 up, 8 in 2024-04-12T11:26:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:01 smithi029 ceph-mon[97787]: osdmap e756: 8 total, 8 up, 8 in 2024-04-12T11:26:02.629 INFO:tasks.rados.rados.0.smithi029.stdout:2697: done (5 left) 2024-04-12T11:26:02.629 INFO:tasks.rados.rados.0.smithi029.stdout:2698: rollback oid 41 current snap is 348 2024-04-12T11:26:02.629 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 347 2024-04-12T11:26:02.629 INFO:tasks.rados.rados.0.smithi029.stdout:2692: finishing write tid 3 to smithi02979277-44 2024-04-12T11:26:02.629 INFO:tasks.rados.rados.0.smithi029.stdout:2692: finishing write tid 4 to smithi02979277-44 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2692: oid 44 updating version 0 to 1291 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2692: oid 44 version 1291 is already newer than 1290 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2692: oid 44 version 1291 is already newer than 1289 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1291 (ObjNum 1009 snap 348 seq_num 1009) dirty exists 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2692: left oid 44 (ObjNum 1009 snap 348 seq_num 1009) 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2693: finishing write tid 1 to smithi02979277-47 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2693: finishing write tid 2 to smithi02979277-47 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2693: finishing write tid 3 to smithi02979277-47 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2693: finishing write tid 4 to smithi02979277-47 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2693: finishing write tid 5 to smithi02979277-47 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2693: oid 47 updating version 0 to 1279 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2693: oid 47 version 1279 is already newer than 1278 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2693: oid 47 version 1279 is already newer than 1276 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:2693: oid 47 version 1279 is already newer than 1277 2024-04-12T11:26:02.630 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1279 (ObjNum 1010 snap 348 seq_num 1010) dirty exists 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2693: left oid 47 (ObjNum 1010 snap 348 seq_num 1010) 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2695: finishing write tid 1 to smithi02979277-14 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2695: finishing write tid 2 to smithi02979277-14 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2695: finishing write tid 3 to smithi02979277-14 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2696: finishing rollback tid 0 to smithi02979277-16 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2696: finishing rollback tid 1 to smithi02979277-16 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1239 (ObjNum 65 snap 0 seq_num 1800984352) dirty dne 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2695: finishing write tid 4 to smithi02979277-14 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2695: oid 14 updating version 0 to 1234 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2695: oid 14 updating version 1234 to 1236 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2695: oid 14 version 1236 is already newer than 1235 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 1236 (ObjNum 1011 snap 348 seq_num 1011) dirty exists 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2695: left oid 14 (ObjNum 1011 snap 348 seq_num 1011) 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2692: done (5 left) 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2693: done (4 left) 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2694: done (3 left) 2024-04-12T11:26:02.631 INFO:tasks.rados.rados.0.smithi029.stdout:2695: done (2 left) 2024-04-12T11:26:02.632 INFO:tasks.rados.rados.0.smithi029.stdout:2696: done (1 left) 2024-04-12T11:26:02.632 INFO:tasks.rados.rados.0.smithi029.stdout:2699: read oid 12 snap -1 2024-04-12T11:26:02.632 INFO:tasks.rados.rados.0.smithi029.stdout:2699: expect deleted 2024-04-12T11:26:02.632 INFO:tasks.rados.rados.0.smithi029.stdout:2700: read oid 8 snap -1 2024-04-12T11:26:02.632 INFO:tasks.rados.rados.0.smithi029.stdout:2700: expect (ObjNum 927 snap 318 seq_num 927) 2024-04-12T11:26:02.632 INFO:tasks.rados.rados.0.smithi029.stdout:2701: rollback oid 26 current snap is 348 2024-04-12T11:26:02.632 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 330 2024-04-12T11:26:02.632 INFO:tasks.rados.rados.0.smithi029.stdout:2702: snap_remove snap 345 2024-04-12T11:26:02.668 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:26:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:26:02.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:26:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:02 smithi087 ceph-mon[89014]: pgmap v771: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 227 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.1 MiB/s wr, 13 op/s 2024-04-12T11:26:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:02 smithi087 ceph-mon[89014]: osdmap e757: 8 total, 8 up, 8 in 2024-04-12T11:26:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:02 smithi087 ceph-mon[89014]: osdmap e758: 8 total, 8 up, 8 in 2024-04-12T11:26:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:02 smithi029 ceph-mon[93650]: pgmap v771: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 227 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.1 MiB/s wr, 13 op/s 2024-04-12T11:26:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:02 smithi029 ceph-mon[93650]: osdmap e757: 8 total, 8 up, 8 in 2024-04-12T11:26:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:02 smithi029 ceph-mon[93650]: osdmap e758: 8 total, 8 up, 8 in 2024-04-12T11:26:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:02 smithi029 ceph-mon[97787]: pgmap v771: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 227 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.1 MiB/s wr, 13 op/s 2024-04-12T11:26:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:02 smithi029 ceph-mon[97787]: osdmap e757: 8 total, 8 up, 8 in 2024-04-12T11:26:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:02 smithi029 ceph-mon[97787]: osdmap e758: 8 total, 8 up, 8 in 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:2702: done (4 left) 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:2703: rollback oid 18 current snap is 348 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 18 to 322 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:2701: finishing rollback tid 0 to smithi02979277-26 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:2701: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:2701: finishing rollback tid 2 to smithi02979277-26 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 1606 (ObjNum 938 snap 325 seq_num 938) dirty exists 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:2698: finishing rollback tid 0 to smithi02979277-41 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:2698: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:2698: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1297 (ObjNum 1006 snap 347 seq_num 1006) dirty exists 2024-04-12T11:26:03.631 INFO:tasks.rados.rados.0.smithi029.stdout:2700: expect (ObjNum 927 snap 318 seq_num 927) 2024-04-12T11:26:03.929 INFO:tasks.rados.rados.0.smithi029.stdout:2703: finishing rollback tid 0 to smithi02979277-18 2024-04-12T11:26:03.929 INFO:tasks.rados.rados.0.smithi029.stdout:2703: finishing rollback tid 1 to smithi02979277-18 2024-04-12T11:26:03.929 INFO:tasks.rados.rados.0.smithi029.stdout:2698: done (4 left) 2024-04-12T11:26:03.929 INFO:tasks.rados.rados.0.smithi029.stdout:2699: done (3 left) 2024-04-12T11:26:03.929 INFO:tasks.rados.rados.0.smithi029.stdout:2700: done (2 left) 2024-04-12T11:26:03.929 INFO:tasks.rados.rados.0.smithi029.stdout:2701: done (1 left) 2024-04-12T11:26:03.930 INFO:tasks.rados.rados.0.smithi029.stdout:2704: read oid 21 snap -1 2024-04-12T11:26:03.930 INFO:tasks.rados.rados.0.smithi029.stdout:2704: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:26:03.930 INFO:tasks.rados.rados.0.smithi029.stdout:2705: write oid 40 current snap is 348 2024-04-12T11:26:03.930 INFO:tasks.rados.rados.0.smithi029.stdout:2703: finishing rollback tid 2 to smithi02979277-18 2024-04-12T11:26:03.930 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1354 (ObjNum 861 snap 296 seq_num 861) dirty exists 2024-04-12T11:26:03.930 INFO:tasks.rados.rados.0.smithi029.stdout:2705: seq_num 1012 ranges {608684=426937,1803189=569688,2924304=84380} 2024-04-12T11:26:03.934 INFO:tasks.rados.rados.0.smithi029.stdout:2705: writing smithi02979277-40 from 608684 to 1035621 tid 1 2024-04-12T11:26:03.942 INFO:tasks.rados.rados.0.smithi029.stdout:2705: writing smithi02979277-40 from 1803189 to 2372877 tid 2 2024-04-12T11:26:03.946 INFO:tasks.rados.rados.0.smithi029.stdout:2705: writing smithi02979277-40 from 2924304 to 3008684 tid 3 2024-04-12T11:26:03.946 INFO:tasks.rados.rados.0.smithi029.stdout:2703: done (2 left) 2024-04-12T11:26:03.946 INFO:tasks.rados.rados.0.smithi029.stdout:2706: snap_remove snap 340 2024-04-12T11:26:04.675 INFO:tasks.rados.rados.0.smithi029.stdout:2706: done (2 left) 2024-04-12T11:26:04.675 INFO:tasks.rados.rados.0.smithi029.stdout:2707: delete oid 50 current snap is 348 2024-04-12T11:26:04.675 INFO:tasks.rados.rados.0.smithi029.stdout:2705: finishing write tid 1 to smithi02979277-40 2024-04-12T11:26:04.675 INFO:tasks.rados.rados.0.smithi029.stdout:2704: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:26:04.839 INFO:tasks.rados.rados.0.smithi029.stdout:2705: finishing write tid 2 to smithi02979277-40 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2705: finishing write tid 3 to smithi02979277-40 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2704: done (2 left) 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2707: done (1 left) 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2708: read oid 37 snap 333 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2708: expect (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2709: write oid 43 current snap is 348 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2705: finishing write tid 4 to smithi02979277-40 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2705: finishing write tid 5 to smithi02979277-40 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2705: oid 40 updating version 0 to 1613 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2705: oid 40 version 1613 is already newer than 1612 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2705: oid 40 updating version 1613 to 1614 2024-04-12T11:26:04.840 INFO:tasks.rados.rados.0.smithi029.stdout:2705: oid 40 updating version 1614 to 1615 2024-04-12T11:26:04.841 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 1615 (ObjNum 1012 snap 348 seq_num 1012) dirty exists 2024-04-12T11:26:04.841 INFO:tasks.rados.rados.0.smithi029.stdout:2705: left oid 40 (ObjNum 1012 snap 348 seq_num 1012) 2024-04-12T11:26:04.841 INFO:tasks.rados.rados.0.smithi029.stdout:2709: seq_num 1014 ranges {642298=577435,1920153=573357,2979150=63148} 2024-04-12T11:26:04.846 INFO:tasks.rados.rados.0.smithi029.stdout:2709: writing smithi02979277-43 from 642298 to 1219733 tid 1 2024-04-12T11:26:04.853 INFO:tasks.rados.rados.0.smithi029.stdout:2709: writing smithi02979277-43 from 1920153 to 2493510 tid 2 2024-04-12T11:26:04.856 INFO:tasks.rados.rados.0.smithi029.stdout:2709: writing smithi02979277-43 from 2979150 to 3042298 tid 3 2024-04-12T11:26:04.857 INFO:tasks.rados.rados.0.smithi029.stdout:2705: done (2 left) 2024-04-12T11:26:04.857 INFO:tasks.rados.rados.0.smithi029.stdout:2710: read oid 35 snap -1 2024-04-12T11:26:04.857 INFO:tasks.rados.rados.0.smithi029.stdout:2710: expect (ObjNum 937 snap 325 seq_num 937) 2024-04-12T11:26:04.857 INFO:tasks.rados.rados.0.smithi029.stdout:2711: snap_create 2024-04-12T11:26:04.857 INFO:tasks.rados.rados.0.smithi029.stdout:2708: expect (ObjNum 692 snap 224 seq_num 692) 2024-04-12T11:26:04.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:04 smithi029 ceph-mon[93650]: pgmap v774: 89 pgs: 89 active+clean; 242 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-04-12T11:26:04.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:04 smithi029 ceph-mon[93650]: osdmap e759: 8 total, 8 up, 8 in 2024-04-12T11:26:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:04 smithi029 ceph-mon[97787]: pgmap v774: 89 pgs: 89 active+clean; 242 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-04-12T11:26:04.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:04 smithi029 ceph-mon[97787]: osdmap e759: 8 total, 8 up, 8 in 2024-04-12T11:26:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:04 smithi087 ceph-mon[89014]: pgmap v774: 89 pgs: 89 active+clean; 242 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-04-12T11:26:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:04 smithi087 ceph-mon[89014]: osdmap e759: 8 total, 8 up, 8 in 2024-04-12T11:26:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:2709: finishing write tid 1 to smithi02979277-43 2024-04-12T11:26:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:2709: finishing write tid 2 to smithi02979277-43 2024-04-12T11:26:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:2709: finishing write tid 3 to smithi02979277-43 2024-04-12T11:26:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:2709: finishing write tid 4 to smithi02979277-43 2024-04-12T11:26:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:2709: finishing write tid 5 to smithi02979277-43 2024-04-12T11:26:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:2709: oid 43 updating version 0 to 1203 2024-04-12T11:26:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:2709: oid 43 updating version 1203 to 1205 2024-04-12T11:26:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:2709: oid 43 version 1205 is already newer than 1204 2024-04-12T11:26:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:2709: oid 43 version 1205 is already newer than 1202 2024-04-12T11:26:05.068 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1205 (ObjNum 1014 snap 348 seq_num 1014) dirty exists 2024-04-12T11:26:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:2709: left oid 43 (ObjNum 1014 snap 348 seq_num 1014) 2024-04-12T11:26:05.069 INFO:tasks.rados.rados.0.smithi029.stdout:2710: expect (ObjNum 937 snap 325 seq_num 937) 2024-04-12T11:26:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:26:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:26:04] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.43.0" 2024-04-12T11:26:05.680 INFO:tasks.rados.rados.0.smithi029.stdout:2708: done (3 left) 2024-04-12T11:26:05.680 INFO:tasks.rados.rados.0.smithi029.stdout:2709: done (2 left) 2024-04-12T11:26:05.680 INFO:tasks.rados.rados.0.smithi029.stdout:2710: done (1 left) 2024-04-12T11:26:05.681 INFO:tasks.rados.rados.0.smithi029.stdout:2711: done (0 left) 2024-04-12T11:26:05.681 INFO:tasks.rados.rados.0.smithi029.stdout:2712: read oid 4 snap -1 2024-04-12T11:26:05.681 INFO:tasks.rados.rados.0.smithi029.stdout:2712: expect deleted 2024-04-12T11:26:05.681 INFO:tasks.rados.rados.0.smithi029.stdout:2713: snap_remove snap 344 2024-04-12T11:26:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:05 smithi087 ceph-mon[89014]: osdmap e760: 8 total, 8 up, 8 in 2024-04-12T11:26:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:05 smithi029 ceph-mon[93650]: osdmap e760: 8 total, 8 up, 8 in 2024-04-12T11:26:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:05 smithi029 ceph-mon[97787]: osdmap e760: 8 total, 8 up, 8 in 2024-04-12T11:26:06.681 INFO:tasks.rados.rados.0.smithi029.stdout:2713: done (1 left) 2024-04-12T11:26:06.681 INFO:tasks.rados.rados.0.smithi029.stdout:2714: snap_remove snap 343 2024-04-12T11:26:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:06 smithi029 ceph-mon[93650]: pgmap v777: 89 pgs: 89 active+clean; 248 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 27 op/s 2024-04-12T11:26:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:06 smithi029 ceph-mon[93650]: osdmap e761: 8 total, 8 up, 8 in 2024-04-12T11:26:06.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:06.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:06 smithi029 ceph-mon[97787]: pgmap v777: 89 pgs: 89 active+clean; 248 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 27 op/s 2024-04-12T11:26:06.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:06 smithi029 ceph-mon[97787]: osdmap e761: 8 total, 8 up, 8 in 2024-04-12T11:26:06.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:06 smithi087 ceph-mon[89014]: pgmap v777: 89 pgs: 89 active+clean; 248 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 27 op/s 2024-04-12T11:26:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:06 smithi087 ceph-mon[89014]: osdmap e761: 8 total, 8 up, 8 in 2024-04-12T11:26:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:07.631 INFO:tasks.rados.rados.0.smithi029.stdout:2714: done (1 left) 2024-04-12T11:26:07.631 INFO:tasks.rados.rados.0.smithi029.stdout:2715: write oid 16 current snap is 349 2024-04-12T11:26:07.631 INFO:tasks.rados.rados.0.smithi029.stdout:2715: seq_num 1015 ranges {659105=652684,1862282=717015} 2024-04-12T11:26:07.638 INFO:tasks.rados.rados.0.smithi029.stdout:2715: writing smithi02979277-16 from 659105 to 1311789 tid 1 2024-04-12T11:26:07.646 INFO:tasks.rados.rados.0.smithi029.stdout:2715: writing smithi02979277-16 from 1862282 to 2579297 tid 2 2024-04-12T11:26:07.646 INFO:tasks.rados.rados.0.smithi029.stdout:2716: snap_create 2024-04-12T11:26:07.646 INFO:tasks.rados.rados.0.smithi029.stdout:2715: finishing write tid 1 to smithi02979277-16 2024-04-12T11:26:07.650 INFO:tasks.rados.rados.0.smithi029.stdout:2715: finishing write tid 2 to smithi02979277-16 2024-04-12T11:26:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:2715: finishing write tid 3 to smithi02979277-16 2024-04-12T11:26:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:2715: finishing write tid 4 to smithi02979277-16 2024-04-12T11:26:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:2715: oid 16 updating version 0 to 1264 2024-04-12T11:26:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:2715: oid 16 version 1264 is already newer than 1263 2024-04-12T11:26:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:2715: oid 16 version 1264 is already newer than 1262 2024-04-12T11:26:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1264 (ObjNum 1015 snap 349 seq_num 1015) dirty exists 2024-04-12T11:26:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:2715: left oid 16 (ObjNum 1015 snap 349 seq_num 1015) 2024-04-12T11:26:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:07 smithi087 ceph-mon[89014]: osdmap e762: 8 total, 8 up, 8 in 2024-04-12T11:26:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:07 smithi087 ceph-mon[89014]: osdmap e763: 8 total, 8 up, 8 in 2024-04-12T11:26:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:07 smithi029 ceph-mon[97787]: osdmap e762: 8 total, 8 up, 8 in 2024-04-12T11:26:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:07 smithi029 ceph-mon[97787]: osdmap e763: 8 total, 8 up, 8 in 2024-04-12T11:26:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:07 smithi029 ceph-mon[93650]: osdmap e762: 8 total, 8 up, 8 in 2024-04-12T11:26:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:07 smithi029 ceph-mon[93650]: osdmap e763: 8 total, 8 up, 8 in 2024-04-12T11:26:08.637 INFO:tasks.rados.rados.0.smithi029.stdout:2712: done (2 left) 2024-04-12T11:26:08.637 INFO:tasks.rados.rados.0.smithi029.stdout:2715: done (1 left) 2024-04-12T11:26:08.638 INFO:tasks.rados.rados.0.smithi029.stdout:2716: done (0 left) 2024-04-12T11:26:08.638 INFO:tasks.rados.rados.0.smithi029.stdout:2717: read oid 27 snap 334 2024-04-12T11:26:08.638 INFO:tasks.rados.rados.0.smithi029.stdout:2717: expect (ObjNum 850 snap 292 seq_num 850) 2024-04-12T11:26:08.638 INFO:tasks.rados.rados.0.smithi029.stdout:2718: write oid 30 current snap is 350 2024-04-12T11:26:08.638 INFO:tasks.rados.rados.0.smithi029.stdout:2718: seq_num 1016 ranges {675912=727933,1920764=460673,2801290=274622} 2024-04-12T11:26:08.645 INFO:tasks.rados.rados.0.smithi029.stdout:2718: writing smithi02979277-30 from 675912 to 1403845 tid 1 2024-04-12T11:26:08.651 INFO:tasks.rados.rados.0.smithi029.stdout:2718: writing smithi02979277-30 from 1920764 to 2381437 tid 2 2024-04-12T11:26:08.656 INFO:tasks.rados.rados.0.smithi029.stdout:2718: writing smithi02979277-30 from 2801290 to 3075912 tid 3 2024-04-12T11:26:08.656 INFO:tasks.rados.rados.0.smithi029.stdout:2719: write oid 24 current snap is 350 2024-04-12T11:26:08.656 INFO:tasks.rados.rados.0.smithi029.stdout:2719: seq_num 1017 ranges {692719=403182,1579246=720684,2828713=264006} 2024-04-12T11:26:08.661 INFO:tasks.rados.rados.0.smithi029.stdout:2719: writing smithi02979277-24 from 692719 to 1095901 tid 1 2024-04-12T11:26:08.668 INFO:tasks.rados.rados.0.smithi029.stdout:2719: writing smithi02979277-24 from 1579246 to 2299930 tid 2 2024-04-12T11:26:08.673 INFO:tasks.rados.rados.0.smithi029.stdout:2719: writing smithi02979277-24 from 2828713 to 3092719 tid 3 2024-04-12T11:26:08.673 INFO:tasks.rados.rados.0.smithi029.stdout:2720: snap_remove snap 333 2024-04-12T11:26:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:08 smithi087 ceph-mon[89014]: pgmap v780: 89 pgs: 89 active+clean; 248 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 352 KiB/s wr, 18 op/s 2024-04-12T11:26:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:08 smithi087 ceph-mon[89014]: osdmap e764: 8 total, 8 up, 8 in 2024-04-12T11:26:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:08 smithi029 ceph-mon[93650]: pgmap v780: 89 pgs: 89 active+clean; 248 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 352 KiB/s wr, 18 op/s 2024-04-12T11:26:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:08 smithi029 ceph-mon[93650]: osdmap e764: 8 total, 8 up, 8 in 2024-04-12T11:26:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:08 smithi029 ceph-mon[97787]: pgmap v780: 89 pgs: 89 active+clean; 248 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 352 KiB/s wr, 18 op/s 2024-04-12T11:26:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:08 smithi029 ceph-mon[97787]: osdmap e764: 8 total, 8 up, 8 in 2024-04-12T11:26:09.639 INFO:tasks.rados.rados.0.smithi029.stdout:2720: done (3 left) 2024-04-12T11:26:09.639 INFO:tasks.rados.rados.0.smithi029.stdout:2721: read oid 19 snap -1 2024-04-12T11:26:09.639 INFO:tasks.rados.rados.0.smithi029.stdout:2721: expect (ObjNum 1007 snap 347 seq_num 1007) 2024-04-12T11:26:09.639 INFO:tasks.rados.rados.0.smithi029.stdout:2722: read oid 34 snap 317 2024-04-12T11:26:09.640 INFO:tasks.rados.rados.0.smithi029.stdout:2722: expect (ObjNum 880 snap 311 seq_num 880) 2024-04-12T11:26:09.640 INFO:tasks.rados.rados.0.smithi029.stdout:2723: rollback oid 5 current snap is 350 2024-04-12T11:26:09.640 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 5 to 339 2024-04-12T11:26:09.640 INFO:tasks.rados.rados.0.smithi029.stdout:2724: snap_create 2024-04-12T11:26:09.640 INFO:tasks.rados.rados.0.smithi029.stdout:2717: expect (ObjNum 850 snap 292 seq_num 850) 2024-04-12T11:26:09.756 INFO:tasks.rados.rados.0.smithi029.stdout:2718: finishing write tid 1 to smithi02979277-30 2024-04-12T11:26:09.756 INFO:tasks.rados.rados.0.smithi029.stdout:2718: finishing write tid 2 to smithi02979277-30 2024-04-12T11:26:09.756 INFO:tasks.rados.rados.0.smithi029.stdout:2718: finishing write tid 3 to smithi02979277-30 2024-04-12T11:26:09.756 INFO:tasks.rados.rados.0.smithi029.stdout:2718: finishing write tid 4 to smithi02979277-30 2024-04-12T11:26:09.756 INFO:tasks.rados.rados.0.smithi029.stdout:2718: finishing write tid 5 to smithi02979277-30 2024-04-12T11:26:09.756 INFO:tasks.rados.rados.0.smithi029.stdout:2718: oid 30 updating version 0 to 879 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2718: oid 30 version 879 is already newer than 877 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2718: oid 30 version 879 is already newer than 878 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2718: oid 30 version 879 is already newer than 876 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 879 (ObjNum 1016 snap 350 seq_num 1016) dirty exists 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2718: left oid 30 (ObjNum 1016 snap 350 seq_num 1016) 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2719: finishing write tid 1 to smithi02979277-24 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2719: finishing write tid 2 to smithi02979277-24 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2719: finishing write tid 3 to smithi02979277-24 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2719: finishing write tid 4 to smithi02979277-24 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2719: finishing write tid 5 to smithi02979277-24 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2719: oid 24 updating version 0 to 1635 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2719: oid 24 updating version 1635 to 1636 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2719: oid 24 updating version 1636 to 1637 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:2719: oid 24 version 1637 is already newer than 1634 2024-04-12T11:26:09.757 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1637 (ObjNum 1017 snap 350 seq_num 1017) dirty exists 2024-04-12T11:26:09.758 INFO:tasks.rados.rados.0.smithi029.stdout:2719: left oid 24 (ObjNum 1017 snap 350 seq_num 1017) 2024-04-12T11:26:09.758 INFO:tasks.rados.rados.0.smithi029.stdout:2722: expect (ObjNum 880 snap 311 seq_num 880) 2024-04-12T11:26:09.937 INFO:tasks.rados.rados.0.smithi029.stdout:2721: expect (ObjNum 1007 snap 347 seq_num 1007) 2024-04-12T11:26:10.141 INFO:tasks.rados.rados.0.smithi029.stdout:2723: finishing rollback tid 1 to smithi02979277-5 2024-04-12T11:26:10.142 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1133 (ObjNum 129 snap 0 seq_num 1801125920) dirty dne 2024-04-12T11:26:10.644 INFO:tasks.rados.rados.0.smithi029.stdout:2717: done (6 left) 2024-04-12T11:26:10.644 INFO:tasks.rados.rados.0.smithi029.stdout:2718: done (5 left) 2024-04-12T11:26:10.644 INFO:tasks.rados.rados.0.smithi029.stdout:2719: done (4 left) 2024-04-12T11:26:10.644 INFO:tasks.rados.rados.0.smithi029.stdout:2721: done (3 left) 2024-04-12T11:26:10.644 INFO:tasks.rados.rados.0.smithi029.stdout:2722: done (2 left) 2024-04-12T11:26:10.645 INFO:tasks.rados.rados.0.smithi029.stdout:2723: done (1 left) 2024-04-12T11:26:10.645 INFO:tasks.rados.rados.0.smithi029.stdout:2724: done (0 left) 2024-04-12T11:26:10.645 INFO:tasks.rados.rados.0.smithi029.stdout:2725: rollback oid 32 current snap is 351 2024-04-12T11:26:10.645 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 32 to 309 2024-04-12T11:26:10.645 INFO:tasks.rados.rados.0.smithi029.stdout:2726: delete oid 43 current snap is 351 2024-04-12T11:26:10.649 INFO:tasks.rados.rados.0.smithi029.stdout:2725: finishing rollback tid 1 to smithi02979277-32 2024-04-12T11:26:10.649 INFO:tasks.rados.rados.0.smithi029.stdout:2725: finishing rollback tid 2 to smithi02979277-32 2024-04-12T11:26:10.649 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 1217 (ObjNum 817 snap 285 seq_num 817) dirty exists 2024-04-12T11:26:10.650 INFO:tasks.rados.rados.0.smithi029.stdout:2725: done (1 left) 2024-04-12T11:26:10.650 INFO:tasks.rados.rados.0.smithi029.stdout:2726: done (0 left) 2024-04-12T11:26:10.650 INFO:tasks.rados.rados.0.smithi029.stdout:2727: read oid 8 snap -1 2024-04-12T11:26:10.650 INFO:tasks.rados.rados.0.smithi029.stdout:2727: expect (ObjNum 927 snap 318 seq_num 927) 2024-04-12T11:26:10.650 INFO:tasks.rados.rados.0.smithi029.stdout:2728: snap_create 2024-04-12T11:26:10.657 INFO:tasks.rados.rados.0.smithi029.stdout:2727: expect (ObjNum 927 snap 318 seq_num 927) 2024-04-12T11:26:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:10 smithi087 ceph-mon[89014]: pgmap v783: 89 pgs: 89 active+clean; 251 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 561 KiB/s wr, 13 op/s 2024-04-12T11:26:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:10 smithi087 ceph-mon[89014]: osdmap e765: 8 total, 8 up, 8 in 2024-04-12T11:26:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:10 smithi029 ceph-mon[93650]: pgmap v783: 89 pgs: 89 active+clean; 251 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 561 KiB/s wr, 13 op/s 2024-04-12T11:26:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:10 smithi029 ceph-mon[93650]: osdmap e765: 8 total, 8 up, 8 in 2024-04-12T11:26:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:10 smithi029 ceph-mon[97787]: pgmap v783: 89 pgs: 89 active+clean; 251 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 561 KiB/s wr, 13 op/s 2024-04-12T11:26:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:10 smithi029 ceph-mon[97787]: osdmap e765: 8 total, 8 up, 8 in 2024-04-12T11:26:11.650 INFO:tasks.rados.rados.0.smithi029.stdout:2727: done (1 left) 2024-04-12T11:26:11.650 INFO:tasks.rados.rados.0.smithi029.stdout:2728: done (0 left) 2024-04-12T11:26:11.650 INFO:tasks.rados.rados.0.smithi029.stdout:2729: read oid 7 snap 348 2024-04-12T11:26:11.650 INFO:tasks.rados.rados.0.smithi029.stdout:2729: expect (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:26:11.650 INFO:tasks.rados.rados.0.smithi029.stdout:2730: write oid 17 current snap is 352 2024-04-12T11:26:11.650 INFO:tasks.rados.rados.0.smithi029.stdout:2730: seq_num 1019 ranges {726333=670033,2096210=724353} 2024-04-12T11:26:11.657 INFO:tasks.rados.rados.0.smithi029.stdout:2730: writing smithi02979277-17 from 726333 to 1396366 tid 1 2024-04-12T11:26:11.666 INFO:tasks.rados.rados.0.smithi029.stdout:2730: writing smithi02979277-17 from 2096210 to 2820563 tid 2 2024-04-12T11:26:11.666 INFO:tasks.rados.rados.0.smithi029.stdout:2731: read oid 13 snap -1 2024-04-12T11:26:11.666 INFO:tasks.rados.rados.0.smithi029.stdout:2731: expect deleted 2024-04-12T11:26:11.666 INFO:tasks.rados.rados.0.smithi029.stdout:2732: read oid 40 snap -1 2024-04-12T11:26:11.666 INFO:tasks.rados.rados.0.smithi029.stdout:2732: expect (ObjNum 1012 snap 348 seq_num 1012) 2024-04-12T11:26:11.666 INFO:tasks.rados.rados.0.smithi029.stdout:2733: write oid 11 current snap is 352 2024-04-12T11:26:11.667 INFO:tasks.rados.rados.0.smithi029.stdout:2733: seq_num 1020 ranges {743140=745282,2154692=468011} 2024-04-12T11:26:11.674 INFO:tasks.rados.rados.0.smithi029.stdout:2733: writing smithi02979277-11 from 743140 to 1488422 tid 1 2024-04-12T11:26:11.681 INFO:tasks.rados.rados.0.smithi029.stdout:2733: writing smithi02979277-11 from 2154692 to 2622703 tid 2 2024-04-12T11:26:11.681 INFO:tasks.rados.rados.0.smithi029.stdout:2734: read oid 34 snap -1 2024-04-12T11:26:11.681 INFO:tasks.rados.rados.0.smithi029.stdout:2734: expect (ObjNum 987 snap 342 seq_num 987) 2024-04-12T11:26:11.681 INFO:tasks.rados.rados.0.smithi029.stdout:2735: read oid 35 snap 334 2024-04-12T11:26:11.681 INFO:tasks.rados.rados.0.smithi029.stdout:2735: expect (ObjNum 937 snap 325 seq_num 937) 2024-04-12T11:26:11.681 INFO:tasks.rados.rados.0.smithi029.stdout:2736: write oid 19 current snap is 352 2024-04-12T11:26:11.682 INFO:tasks.rados.rados.0.smithi029.stdout:2736: seq_num 1021 ranges {759947=420531,1813174=728022} 2024-04-12T11:26:11.687 INFO:tasks.rados.rados.0.smithi029.stdout:2736: writing smithi02979277-19 from 759947 to 1180478 tid 1 2024-04-12T11:26:11.695 INFO:tasks.rados.rados.0.smithi029.stdout:2736: writing smithi02979277-19 from 1813174 to 2541196 tid 2 2024-04-12T11:26:11.695 INFO:tasks.rados.rados.0.smithi029.stdout:2737: snap_remove snap 317 2024-04-12T11:26:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:11 smithi087 ceph-mon[89014]: osdmap e766: 8 total, 8 up, 8 in 2024-04-12T11:26:12.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:11 smithi029 ceph-mon[93650]: osdmap e766: 8 total, 8 up, 8 in 2024-04-12T11:26:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:11 smithi029 ceph-mon[97787]: osdmap e766: 8 total, 8 up, 8 in 2024-04-12T11:26:12.633 INFO:tasks.rados.rados.0.smithi029.stdout:2737: done (8 left) 2024-04-12T11:26:12.633 INFO:tasks.rados.rados.0.smithi029.stdout:2738: write oid 47 current snap is 352 2024-04-12T11:26:12.633 INFO:tasks.rados.rados.0.smithi029.stdout:2738: seq_num 1022 ranges {776754=495780,1871656=471680,2965828=210926} 2024-04-12T11:26:12.640 INFO:tasks.rados.rados.0.smithi029.stdout:2738: writing smithi02979277-47 from 776754 to 1272534 tid 1 2024-04-12T11:26:12.647 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:26:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:26:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:26:12.648 INFO:tasks.rados.rados.0.smithi029.stdout:2738: writing smithi02979277-47 from 1871656 to 2343336 tid 2 2024-04-12T11:26:12.653 INFO:tasks.rados.rados.0.smithi029.stdout:2738: writing smithi02979277-47 from 2965828 to 3176754 tid 3 2024-04-12T11:26:12.653 INFO:tasks.rados.rados.0.smithi029.stdout:2739: rollback oid 10 current snap is 352 2024-04-12T11:26:12.653 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 10 to 351 2024-04-12T11:26:12.653 INFO:tasks.rados.rados.0.smithi029.stdout:2729: expect (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:26:12.755 INFO:tasks.rados.rados.0.smithi029.stdout:2730: finishing write tid 1 to smithi02979277-17 2024-04-12T11:26:12.755 INFO:tasks.rados.rados.0.smithi029.stdout:2730: finishing write tid 2 to smithi02979277-17 2024-04-12T11:26:12.755 INFO:tasks.rados.rados.0.smithi029.stdout:2732: expect (ObjNum 1012 snap 348 seq_num 1012) 2024-04-12T11:26:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:12 smithi087 ceph-mon[89014]: pgmap v786: 89 pgs: 89 active+clean; 251 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 561 KiB/s wr, 7 op/s 2024-04-12T11:26:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:12 smithi087 ceph-mon[89014]: osdmap e767: 8 total, 8 up, 8 in 2024-04-12T11:26:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:12 smithi087 ceph-mon[89014]: osdmap e768: 8 total, 8 up, 8 in 2024-04-12T11:26:12.975 INFO:tasks.rados.rados.0.smithi029.stdout:2730: finishing write tid 3 to smithi02979277-17 2024-04-12T11:26:12.975 INFO:tasks.rados.rados.0.smithi029.stdout:2730: finishing write tid 4 to smithi02979277-17 2024-04-12T11:26:12.975 INFO:tasks.rados.rados.0.smithi029.stdout:2730: oid 17 updating version 0 to 1380 2024-04-12T11:26:12.975 INFO:tasks.rados.rados.0.smithi029.stdout:2730: oid 17 updating version 1380 to 1381 2024-04-12T11:26:12.975 INFO:tasks.rados.rados.0.smithi029.stdout:2730: oid 17 version 1381 is already newer than 1379 2024-04-12T11:26:12.975 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1381 (ObjNum 1019 snap 352 seq_num 1019) dirty exists 2024-04-12T11:26:12.975 INFO:tasks.rados.rados.0.smithi029.stdout:2730: left oid 17 (ObjNum 1019 snap 352 seq_num 1019) 2024-04-12T11:26:12.975 INFO:tasks.rados.rados.0.smithi029.stdout:2733: finishing write tid 1 to smithi02979277-11 2024-04-12T11:26:12.975 INFO:tasks.rados.rados.0.smithi029.stdout:2729: done (9 left) 2024-04-12T11:26:12.976 INFO:tasks.rados.rados.0.smithi029.stdout:2730: done (8 left) 2024-04-12T11:26:12.976 INFO:tasks.rados.rados.0.smithi029.stdout:2731: done (7 left) 2024-04-12T11:26:12.976 INFO:tasks.rados.rados.0.smithi029.stdout:2732: done (6 left) 2024-04-12T11:26:12.976 INFO:tasks.rados.rados.0.smithi029.stdout:2740: read oid 36 snap -1 2024-04-12T11:26:12.976 INFO:tasks.rados.rados.0.smithi029.stdout:2740: expect (ObjNum 1001 snap 345 seq_num 1001) 2024-04-12T11:26:12.976 INFO:tasks.rados.rados.0.smithi029.stdout:2741: snap_remove snap 348 2024-04-12T11:26:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:12 smithi029 ceph-mon[93650]: pgmap v786: 89 pgs: 89 active+clean; 251 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 561 KiB/s wr, 7 op/s 2024-04-12T11:26:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:12 smithi029 ceph-mon[93650]: osdmap e767: 8 total, 8 up, 8 in 2024-04-12T11:26:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:12 smithi029 ceph-mon[93650]: osdmap e768: 8 total, 8 up, 8 in 2024-04-12T11:26:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:12 smithi029 ceph-mon[97787]: pgmap v786: 89 pgs: 89 active+clean; 251 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 561 KiB/s wr, 7 op/s 2024-04-12T11:26:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:12 smithi029 ceph-mon[97787]: osdmap e767: 8 total, 8 up, 8 in 2024-04-12T11:26:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:12 smithi029 ceph-mon[97787]: osdmap e768: 8 total, 8 up, 8 in 2024-04-12T11:26:13.657 INFO:tasks.rados.rados.0.smithi029.stdout:2741: done (7 left) 2024-04-12T11:26:13.657 INFO:tasks.rados.rados.0.smithi029.stdout:2742: delete oid 31 current snap is 352 2024-04-12T11:26:13.657 INFO:tasks.rados.rados.0.smithi029.stdout:2734: expect (ObjNum 987 snap 342 seq_num 987) 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:2733: finishing write tid 2 to smithi02979277-11 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:2733: finishing write tid 3 to smithi02979277-11 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:2734: done (7 left) 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:2742: done (6 left) 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:2743: write oid 34 current snap is 352 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:2733: finishing write tid 4 to smithi02979277-11 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:2733: oid 11 updating version 0 to 1648 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:2733: oid 11 updating version 1648 to 1649 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:2733: oid 11 version 1649 is already newer than 1647 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 1649 (ObjNum 1020 snap 352 seq_num 1020) dirty exists 2024-04-12T11:26:13.782 INFO:tasks.rados.rados.0.smithi029.stdout:2733: left oid 11 (ObjNum 1020 snap 352 seq_num 1020) 2024-04-12T11:26:13.783 INFO:tasks.rados.rados.0.smithi029.stdout:2735: expect (ObjNum 937 snap 325 seq_num 937) 2024-04-12T11:26:13.908 INFO:tasks.rados.rados.0.smithi029.stdout:2736: finishing write tid 1 to smithi02979277-19 2024-04-12T11:26:13.908 INFO:tasks.rados.rados.0.smithi029.stdout:2736: finishing write tid 2 to smithi02979277-19 2024-04-12T11:26:13.908 INFO:tasks.rados.rados.0.smithi029.stdout:2736: finishing write tid 3 to smithi02979277-19 2024-04-12T11:26:13.908 INFO:tasks.rados.rados.0.smithi029.stdout:2736: finishing write tid 4 to smithi02979277-19 2024-04-12T11:26:13.908 INFO:tasks.rados.rados.0.smithi029.stdout:2736: oid 19 updating version 0 to 1274 2024-04-12T11:26:13.908 INFO:tasks.rados.rados.0.smithi029.stdout:2736: oid 19 version 1274 is already newer than 1273 2024-04-12T11:26:13.908 INFO:tasks.rados.rados.0.smithi029.stdout:2736: oid 19 updating version 1274 to 1275 2024-04-12T11:26:13.908 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1275 (ObjNum 1021 snap 352 seq_num 1021) dirty exists 2024-04-12T11:26:13.909 INFO:tasks.rados.rados.0.smithi029.stdout:2736: left oid 19 (ObjNum 1021 snap 352 seq_num 1021) 2024-04-12T11:26:13.909 INFO:tasks.rados.rados.0.smithi029.stdout:2738: finishing write tid 1 to smithi02979277-47 2024-04-12T11:26:13.909 INFO:tasks.rados.rados.0.smithi029.stdout:2743: seq_num 1024 ranges {410368=646278,1472267=475349,2504321=405600} 2024-04-12T11:26:13.913 INFO:tasks.rados.rados.0.smithi029.stdout:2743: writing smithi02979277-34 from 410368 to 1056646 tid 1 2024-04-12T11:26:13.919 INFO:tasks.rados.rados.0.smithi029.stdout:2743: writing smithi02979277-34 from 1472267 to 1947616 tid 2 2024-04-12T11:26:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:2743: writing smithi02979277-34 from 2504321 to 2909921 tid 3 2024-04-12T11:26:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:2733: done (6 left) 2024-04-12T11:26:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:2735: done (5 left) 2024-04-12T11:26:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:2736: done (4 left) 2024-04-12T11:26:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:2744: write oid 49 current snap is 352 2024-04-12T11:26:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:2738: finishing write tid 2 to smithi02979277-47 2024-04-12T11:26:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:2738: finishing write tid 3 to smithi02979277-47 2024-04-12T11:26:13.925 INFO:tasks.rados.rados.0.smithi029.stdout:2744: seq_num 1025 ranges {427175=721527,1930749=735360} 2024-04-12T11:26:13.931 INFO:tasks.rados.rados.0.smithi029.stdout:2744: writing smithi02979277-49 from 427175 to 1148702 tid 1 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2744: writing smithi02979277-49 from 1930749 to 2666109 tid 2 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2745: delete oid 9 current snap is 352 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2739: finishing rollback tid 0 to smithi02979277-10 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2738: finishing write tid 4 to smithi02979277-47 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2739: finishing rollback tid 1 to smithi02979277-10 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2739: finishing rollback tid 2 to smithi02979277-10 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 1061 (ObjNum 975 snap 340 seq_num 975) dirty exists 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2738: finishing write tid 5 to smithi02979277-47 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2738: oid 47 updating version 0 to 1297 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2738: oid 47 updating version 1297 to 1300 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2738: oid 47 updating version 1300 to 1302 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:2738: oid 47 version 1302 is already newer than 1301 2024-04-12T11:26:13.940 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1302 (ObjNum 1022 snap 352 seq_num 1022) dirty exists 2024-04-12T11:26:13.941 INFO:tasks.rados.rados.0.smithi029.stdout:2738: left oid 47 (ObjNum 1022 snap 352 seq_num 1022) 2024-04-12T11:26:13.941 INFO:tasks.rados.rados.0.smithi029.stdout:2740: expect (ObjNum 1001 snap 345 seq_num 1001) 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2743: finishing write tid 1 to smithi02979277-34 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2743: finishing write tid 2 to smithi02979277-34 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2743: finishing write tid 3 to smithi02979277-34 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2743: finishing write tid 4 to smithi02979277-34 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2743: finishing write tid 5 to smithi02979277-34 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2743: oid 34 updating version 0 to 1232 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2743: oid 34 version 1232 is already newer than 1227 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2743: oid 34 version 1232 is already newer than 1230 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2743: oid 34 version 1232 is already newer than 1231 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1232 (ObjNum 1024 snap 352 seq_num 1024) dirty exists 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2743: left oid 34 (ObjNum 1024 snap 352 seq_num 1024) 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2744: finishing write tid 1 to smithi02979277-49 2024-04-12T11:26:14.069 INFO:tasks.rados.rados.0.smithi029.stdout:2738: done (5 left) 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2739: done (4 left) 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2740: done (3 left) 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2743: done (2 left) 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2745: done (1 left) 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2746: delete oid 19 current snap is 352 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2744: finishing write tid 2 to smithi02979277-49 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2744: finishing write tid 3 to smithi02979277-49 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2744: finishing write tid 4 to smithi02979277-49 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2744: oid 49 updating version 0 to 970 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2744: oid 49 updating version 970 to 971 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2744: oid 49 updating version 971 to 972 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 972 (ObjNum 1025 snap 352 seq_num 1025) dirty exists 2024-04-12T11:26:14.070 INFO:tasks.rados.rados.0.smithi029.stdout:2744: left oid 49 (ObjNum 1025 snap 352 seq_num 1025) 2024-04-12T11:26:14.071 INFO:tasks.rados.rados.0.smithi029.stdout:2744: done (1 left) 2024-04-12T11:26:14.071 INFO:tasks.rados.rados.0.smithi029.stdout:2746: done (0 left) 2024-04-12T11:26:14.071 INFO:tasks.rados.rados.0.smithi029.stdout:2747: snap_remove snap 347 2024-04-12T11:26:14.658 INFO:tasks.rados.rados.0.smithi029.stdout:2747: done (0 left) 2024-04-12T11:26:14.658 INFO:tasks.rados.rados.0.smithi029.stdout:2748: snap_remove snap 325 2024-04-12T11:26:14.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:14 smithi029 ceph-mon[97787]: pgmap v789: 89 pgs: 89 active+clean; 259 MiB data, 854 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-04-12T11:26:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:14 smithi029 ceph-mon[97787]: osdmap e769: 8 total, 8 up, 8 in 2024-04-12T11:26:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:14 smithi029 ceph-mon[93650]: pgmap v789: 89 pgs: 89 active+clean; 259 MiB data, 854 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-04-12T11:26:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:14 smithi029 ceph-mon[93650]: osdmap e769: 8 total, 8 up, 8 in 2024-04-12T11:26:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:14 smithi087 ceph-mon[89014]: pgmap v789: 89 pgs: 89 active+clean; 259 MiB data, 854 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-04-12T11:26:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:14 smithi087 ceph-mon[89014]: osdmap e769: 8 total, 8 up, 8 in 2024-04-12T11:26:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:26:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:26:14] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-12T11:26:15.661 INFO:tasks.rados.rados.0.smithi029.stdout:2748: done (0 left) 2024-04-12T11:26:15.661 INFO:tasks.rados.rados.0.smithi029.stdout:2749: snap_create 2024-04-12T11:26:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:15 smithi087 ceph-mon[89014]: osdmap e770: 8 total, 8 up, 8 in 2024-04-12T11:26:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:15 smithi029 ceph-mon[93650]: osdmap e770: 8 total, 8 up, 8 in 2024-04-12T11:26:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:15 smithi029 ceph-mon[97787]: osdmap e770: 8 total, 8 up, 8 in 2024-04-12T11:26:16.661 INFO:tasks.rados.rados.0.smithi029.stdout:2749: done (0 left) 2024-04-12T11:26:16.661 INFO:tasks.rados.rados.0.smithi029.stdout:2750: read oid 24 snap -1 2024-04-12T11:26:16.661 INFO:tasks.rados.rados.0.smithi029.stdout:2750: expect (ObjNum 1017 snap 350 seq_num 1017) 2024-04-12T11:26:16.661 INFO:tasks.rados.rados.0.smithi029.stdout:2751: snap_remove snap 331 2024-04-12T11:26:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:16 smithi087 ceph-mon[89014]: pgmap v792: 89 pgs: 89 active+clean; 262 MiB data, 847 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 1.9 MiB/s wr, 27 op/s 2024-04-12T11:26:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:16 smithi087 ceph-mon[89014]: osdmap e771: 8 total, 8 up, 8 in 2024-04-12T11:26:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:16 smithi087 ceph-mon[89014]: osdmap e772: 8 total, 8 up, 8 in 2024-04-12T11:26:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:16 smithi029 ceph-mon[93650]: pgmap v792: 89 pgs: 89 active+clean; 262 MiB data, 847 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 1.9 MiB/s wr, 27 op/s 2024-04-12T11:26:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:16 smithi029 ceph-mon[93650]: osdmap e771: 8 total, 8 up, 8 in 2024-04-12T11:26:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:16 smithi029 ceph-mon[93650]: osdmap e772: 8 total, 8 up, 8 in 2024-04-12T11:26:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:16 smithi029 ceph-mon[97787]: pgmap v792: 89 pgs: 89 active+clean; 262 MiB data, 847 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 1.9 MiB/s wr, 27 op/s 2024-04-12T11:26:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:16 smithi029 ceph-mon[97787]: osdmap e771: 8 total, 8 up, 8 in 2024-04-12T11:26:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:16 smithi029 ceph-mon[97787]: osdmap e772: 8 total, 8 up, 8 in 2024-04-12T11:26:17.633 INFO:tasks.rados.rados.0.smithi029.stdout:2751: done (1 left) 2024-04-12T11:26:17.634 INFO:tasks.rados.rados.0.smithi029.stdout:2752: write oid 41 current snap is 353 2024-04-12T11:26:17.634 INFO:tasks.rados.rados.0.smithi029.stdout:2752: seq_num 1028 ranges {477596=663627,1706195=482687,2614013=567041} 2024-04-12T11:26:17.640 INFO:tasks.rados.rados.0.smithi029.stdout:2752: writing smithi02979277-41 from 477596 to 1141223 tid 1 2024-04-12T11:26:17.647 INFO:tasks.rados.rados.0.smithi029.stdout:2752: writing smithi02979277-41 from 1706195 to 2188882 tid 2 2024-04-12T11:26:17.653 INFO:tasks.rados.rados.0.smithi029.stdout:2752: writing smithi02979277-41 from 2614013 to 3181054 tid 3 2024-04-12T11:26:17.653 INFO:tasks.rados.rados.0.smithi029.stdout:2753: delete oid 8 current snap is 353 2024-04-12T11:26:17.653 INFO:tasks.rados.rados.0.smithi029.stdout:2750: expect (ObjNum 1017 snap 350 seq_num 1017) 2024-04-12T11:26:17.757 INFO:tasks.rados.rados.0.smithi029.stdout:2752: finishing write tid 1 to smithi02979277-41 2024-04-12T11:26:17.757 INFO:tasks.rados.rados.0.smithi029.stdout:2752: finishing write tid 2 to smithi02979277-41 2024-04-12T11:26:17.757 INFO:tasks.rados.rados.0.smithi029.stdout:2752: finishing write tid 3 to smithi02979277-41 2024-04-12T11:26:17.757 INFO:tasks.rados.rados.0.smithi029.stdout:2752: finishing write tid 4 to smithi02979277-41 2024-04-12T11:26:17.757 INFO:tasks.rados.rados.0.smithi029.stdout:2752: finishing write tid 5 to smithi02979277-41 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2752: oid 41 updating version 0 to 1327 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2752: oid 41 version 1327 is already newer than 1325 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2752: oid 41 updating version 1327 to 1328 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2752: oid 41 version 1328 is already newer than 1326 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1328 (ObjNum 1028 snap 353 seq_num 1028) dirty exists 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2752: left oid 41 (ObjNum 1028 snap 353 seq_num 1028) 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2750: done (2 left) 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2752: done (1 left) 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2753: done (0 left) 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2754: read oid 41 snap -1 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2754: expect (ObjNum 1028 snap 353 seq_num 1028) 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2755: read oid 4 snap -1 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2755: expect deleted 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2756: rollback oid 45 current snap is 353 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 45 to 351 2024-04-12T11:26:17.758 INFO:tasks.rados.rados.0.smithi029.stdout:2757: snap_create 2024-04-12T11:26:17.760 INFO:tasks.rados.rados.0.smithi029.stdout:2756: finishing rollback tid 0 to smithi02979277-45 2024-04-12T11:26:17.760 INFO:tasks.rados.rados.0.smithi029.stdout:2756: finishing rollback tid 1 to smithi02979277-45 2024-04-12T11:26:17.760 INFO:tasks.rados.rados.0.smithi029.stdout:2756: finishing rollback tid 2 to smithi02979277-45 2024-04-12T11:26:17.761 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1314 (ObjNum 962 snap 330 seq_num 962) dirty exists 2024-04-12T11:26:17.769 INFO:tasks.rados.rados.0.smithi029.stdout:2754: expect (ObjNum 1028 snap 353 seq_num 1028) 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2754: done (3 left) 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2755: done (2 left) 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2756: done (1 left) 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2757: done (0 left) 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2758: read oid 42 snap -1 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2758: expect deleted 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2759: read oid 29 snap -1 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2759: expect (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2760: read oid 21 snap -1 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2760: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2761: read oid 45 snap -1 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2761: expect (ObjNum 962 snap 330 seq_num 962) 2024-04-12T11:26:18.677 INFO:tasks.rados.rados.0.smithi029.stdout:2762: rollback oid 34 current snap is 354 2024-04-12T11:26:18.678 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 350 2024-04-12T11:26:18.678 INFO:tasks.rados.rados.0.smithi029.stdout:2763: snap_remove snap 351 2024-04-12T11:26:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:18 smithi087 ceph-mon[89014]: pgmap v795: 89 pgs: 89 active+clean; 246 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 635 KiB/s wr, 14 op/s 2024-04-12T11:26:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:18 smithi087 ceph-mon[89014]: osdmap e773: 8 total, 8 up, 8 in 2024-04-12T11:26:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:18 smithi029 ceph-mon[93650]: pgmap v795: 89 pgs: 89 active+clean; 246 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 635 KiB/s wr, 14 op/s 2024-04-12T11:26:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:18 smithi029 ceph-mon[93650]: osdmap e773: 8 total, 8 up, 8 in 2024-04-12T11:26:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:18 smithi029 ceph-mon[97787]: pgmap v795: 89 pgs: 89 active+clean; 246 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 635 KiB/s wr, 14 op/s 2024-04-12T11:26:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:18 smithi029 ceph-mon[97787]: osdmap e773: 8 total, 8 up, 8 in 2024-04-12T11:26:19.684 INFO:tasks.rados.rados.0.smithi029.stdout:2763: done (5 left) 2024-04-12T11:26:19.684 INFO:tasks.rados.rados.0.smithi029.stdout:2764: read oid 28 snap -1 2024-04-12T11:26:19.684 INFO:tasks.rados.rados.0.smithi029.stdout:2764: expect (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:26:19.684 INFO:tasks.rados.rados.0.smithi029.stdout:2765: write oid 41 current snap is 354 2024-04-12T11:26:19.684 INFO:tasks.rados.rados.0.smithi029.stdout:2765: seq_num 1030 ranges {511210=414125,1423159=486356,2668859=589585} 2024-04-12T11:26:19.689 INFO:tasks.rados.rados.0.smithi029.stdout:2765: writing smithi02979277-41 from 511210 to 925335 tid 1 2024-04-12T11:26:19.695 INFO:tasks.rados.rados.0.smithi029.stdout:2765: writing smithi02979277-41 from 1423159 to 1909515 tid 2 2024-04-12T11:26:19.702 INFO:tasks.rados.rados.0.smithi029.stdout:2765: writing smithi02979277-41 from 2668859 to 3258444 tid 3 2024-04-12T11:26:19.702 INFO:tasks.rados.rados.0.smithi029.stdout:2766: write oid 47 current snap is 354 2024-04-12T11:26:19.702 INFO:tasks.rados.rados.0.smithi029.stdout:2759: expect (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:26:19.833 INFO:tasks.rados.rados.0.smithi029.stdout:2761: expect (ObjNum 962 snap 330 seq_num 962) 2024-04-12T11:26:19.905 INFO:tasks.rados.rados.0.smithi029.stdout:2762: finishing rollback tid 0 to smithi02979277-34 2024-04-12T11:26:19.905 INFO:tasks.rados.rados.0.smithi029.stdout:2762: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:26:19.906 INFO:tasks.rados.rados.0.smithi029.stdout:2766: seq_num 1031 ranges {528017=489374,1481641=630014,2696282=400857} 2024-04-12T11:26:19.911 INFO:tasks.rados.rados.0.smithi029.stdout:2766: writing smithi02979277-47 from 528017 to 1017391 tid 1 2024-04-12T11:26:19.917 INFO:tasks.rados.rados.0.smithi029.stdout:2766: writing smithi02979277-47 from 1481641 to 2111655 tid 2 2024-04-12T11:26:19.923 INFO:tasks.rados.rados.0.smithi029.stdout:2766: writing smithi02979277-47 from 2696282 to 3097139 tid 3 2024-04-12T11:26:19.923 INFO:tasks.rados.rados.0.smithi029.stdout:2758: done (7 left) 2024-04-12T11:26:19.923 INFO:tasks.rados.rados.0.smithi029.stdout:2759: done (6 left) 2024-04-12T11:26:19.923 INFO:tasks.rados.rados.0.smithi029.stdout:2761: done (5 left) 2024-04-12T11:26:19.923 INFO:tasks.rados.rados.0.smithi029.stdout:2767: rollback oid 6 current snap is 354 2024-04-12T11:26:19.923 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 6 to 349 2024-04-12T11:26:19.923 INFO:tasks.rados.rados.0.smithi029.stdout:2762: finishing rollback tid 2 to smithi02979277-34 2024-04-12T11:26:19.923 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1250 (ObjNum 987 snap 342 seq_num 987) dirty exists 2024-04-12T11:26:19.923 INFO:tasks.rados.rados.0.smithi029.stdout:2760: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:26:19.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:19 smithi087 ceph-mon[89014]: osdmap e774: 8 total, 8 up, 8 in 2024-04-12T11:26:20.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:19 smithi029 ceph-mon[93650]: osdmap e774: 8 total, 8 up, 8 in 2024-04-12T11:26:20.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:19 smithi029 ceph-mon[97787]: osdmap e774: 8 total, 8 up, 8 in 2024-04-12T11:26:20.083 INFO:tasks.rados.rados.0.smithi029.stdout:2764: expect (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:26:20.213 INFO:tasks.rados.rados.0.smithi029.stdout:2765: finishing write tid 1 to smithi02979277-41 2024-04-12T11:26:20.213 INFO:tasks.rados.rados.0.smithi029.stdout:2765: finishing write tid 2 to smithi02979277-41 2024-04-12T11:26:20.213 INFO:tasks.rados.rados.0.smithi029.stdout:2760: done (5 left) 2024-04-12T11:26:20.213 INFO:tasks.rados.rados.0.smithi029.stdout:2762: done (4 left) 2024-04-12T11:26:20.213 INFO:tasks.rados.rados.0.smithi029.stdout:2764: done (3 left) 2024-04-12T11:26:20.213 INFO:tasks.rados.rados.0.smithi029.stdout:2768: rollback oid 49 current snap is 354 2024-04-12T11:26:20.213 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 49 to 341 2024-04-12T11:26:20.213 INFO:tasks.rados.rados.0.smithi029.stdout:2765: finishing write tid 3 to smithi02979277-41 2024-04-12T11:26:20.213 INFO:tasks.rados.rados.0.smithi029.stdout:2765: finishing write tid 4 to smithi02979277-41 2024-04-12T11:26:20.213 INFO:tasks.rados.rados.0.smithi029.stdout:2765: finishing write tid 5 to smithi02979277-41 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2765: oid 41 updating version 0 to 1335 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2765: oid 41 version 1335 is already newer than 1334 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2765: oid 41 version 1335 is already newer than 1333 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2765: oid 41 version 1335 is already newer than 1332 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1335 (ObjNum 1030 snap 354 seq_num 1030) dirty exists 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2765: left oid 41 (ObjNum 1030 snap 354 seq_num 1030) 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2766: finishing write tid 1 to smithi02979277-47 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2766: finishing write tid 2 to smithi02979277-47 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2766: finishing write tid 3 to smithi02979277-47 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2766: finishing write tid 4 to smithi02979277-47 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2766: finishing write tid 5 to smithi02979277-47 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2766: oid 47 updating version 0 to 1323 2024-04-12T11:26:20.214 INFO:tasks.rados.rados.0.smithi029.stdout:2766: oid 47 version 1323 is already newer than 1320 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:2766: oid 47 version 1323 is already newer than 1321 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:2766: oid 47 version 1323 is already newer than 1322 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1323 (ObjNum 1031 snap 354 seq_num 1031) dirty exists 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:2766: left oid 47 (ObjNum 1031 snap 354 seq_num 1031) 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:2767: finishing rollback tid 0 to smithi02979277-6 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:2767: finishing rollback tid 1 to smithi02979277-6 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:2767: finishing rollback tid 2 to smithi02979277-6 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 714 (ObjNum 872 snap 307 seq_num 872) dirty exists 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:2765: done (3 left) 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:2766: done (2 left) 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:2767: done (1 left) 2024-04-12T11:26:20.215 INFO:tasks.rados.rados.0.smithi029.stdout:2769: read oid 11 snap -1 2024-04-12T11:26:20.216 INFO:tasks.rados.rados.0.smithi029.stdout:2769: expect (ObjNum 1020 snap 352 seq_num 1020) 2024-04-12T11:26:20.216 INFO:tasks.rados.rados.0.smithi029.stdout:2770: write oid 2 current snap is 354 2024-04-12T11:26:20.216 INFO:tasks.rados.rados.0.smithi029.stdout:2770: seq_num 1032 ranges {544824=564623,1823770=490025,3007352=337472} 2024-04-12T11:26:20.217 INFO:tasks.rados.rados.0.smithi029.stdout:2770: writing smithi02979277-2 from 544824 to 1109447 tid 1 2024-04-12T11:26:20.224 INFO:tasks.rados.rados.0.smithi029.stdout:2770: writing smithi02979277-2 from 1823770 to 2313795 tid 2 2024-04-12T11:26:20.230 INFO:tasks.rados.rados.0.smithi029.stdout:2770: writing smithi02979277-2 from 3007352 to 3344824 tid 3 2024-04-12T11:26:20.230 INFO:tasks.rados.rados.0.smithi029.stdout:2771: read oid 20 snap -1 2024-04-12T11:26:20.230 INFO:tasks.rados.rados.0.smithi029.stdout:2771: expect (ObjNum 984 snap 342 seq_num 984) 2024-04-12T11:26:20.230 INFO:tasks.rados.rados.0.smithi029.stdout:2772: write oid 39 current snap is 354 2024-04-12T11:26:20.231 INFO:tasks.rados.rados.0.smithi029.stdout:2772: seq_num 1033 ranges {561631=639872,1882252=633683,3034775=326856} 2024-04-12T11:26:20.237 INFO:tasks.rados.rados.0.smithi029.stdout:2772: writing smithi02979277-39 from 561631 to 1201503 tid 1 2024-04-12T11:26:20.244 INFO:tasks.rados.rados.0.smithi029.stdout:2772: writing smithi02979277-39 from 1882252 to 2515935 tid 2 2024-04-12T11:26:20.249 INFO:tasks.rados.rados.0.smithi029.stdout:2772: writing smithi02979277-39 from 3034775 to 3361631 tid 3 2024-04-12T11:26:20.249 INFO:tasks.rados.rados.0.smithi029.stdout:2773: write oid 37 current snap is 354 2024-04-12T11:26:20.249 INFO:tasks.rados.rados.0.smithi029.stdout:2773: seq_num 1034 ranges {578438=431474,1657087=493694,2778551=599887} 2024-04-12T11:26:20.254 INFO:tasks.rados.rados.0.smithi029.stdout:2773: writing smithi02979277-37 from 578438 to 1009912 tid 1 2024-04-12T11:26:20.260 INFO:tasks.rados.rados.0.smithi029.stdout:2773: writing smithi02979277-37 from 1657087 to 2150781 tid 2 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:2773: writing smithi02979277-37 from 2778551 to 3378438 tid 3 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:2774: read oid 35 snap -1 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:2774: expect (ObjNum 937 snap 325 seq_num 937) 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:2775: rollback oid 47 current snap is 354 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 352 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:2768: finishing rollback tid 0 to smithi02979277-49 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:2776: read oid 26 snap -1 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:2776: expect (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:2777: snap_create 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:2768: finishing rollback tid 1 to smithi02979277-49 2024-04-12T11:26:20.268 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 980 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-04-12T11:26:20.269 INFO:tasks.rados.rados.0.smithi029.stdout:2769: expect (ObjNum 1020 snap 352 seq_num 1020) 2024-04-12T11:26:20.413 INFO:tasks.rados.rados.0.smithi029.stdout:2770: finishing write tid 1 to smithi02979277-2 2024-04-12T11:26:20.413 INFO:tasks.rados.rados.0.smithi029.stdout:2770: finishing write tid 2 to smithi02979277-2 2024-04-12T11:26:20.413 INFO:tasks.rados.rados.0.smithi029.stdout:2770: finishing write tid 3 to smithi02979277-2 2024-04-12T11:26:20.413 INFO:tasks.rados.rados.0.smithi029.stdout:2771: expect (ObjNum 984 snap 342 seq_num 984) 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2770: finishing write tid 4 to smithi02979277-2 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2770: finishing write tid 5 to smithi02979277-2 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2770: oid 2 updating version 0 to 302 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2770: oid 2 updating version 302 to 303 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2770: oid 2 updating version 303 to 305 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2770: oid 2 version 305 is already newer than 304 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 305 (ObjNum 1032 snap 354 seq_num 1032) dirty exists 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2770: left oid 2 (ObjNum 1032 snap 354 seq_num 1032) 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2772: finishing write tid 1 to smithi02979277-39 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2772: finishing write tid 2 to smithi02979277-39 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2772: finishing write tid 3 to smithi02979277-39 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2772: finishing write tid 4 to smithi02979277-39 2024-04-12T11:26:20.496 INFO:tasks.rados.rados.0.smithi029.stdout:2772: finishing write tid 5 to smithi02979277-39 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2772: oid 39 updating version 0 to 364 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2772: oid 39 updating version 364 to 367 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2772: oid 39 version 367 is already newer than 365 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2772: oid 39 version 367 is already newer than 366 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 367 (ObjNum 1033 snap 354 seq_num 1033) dirty exists 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2772: left oid 39 (ObjNum 1033 snap 354 seq_num 1033) 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2773: finishing write tid 1 to smithi02979277-37 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2773: finishing write tid 2 to smithi02979277-37 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2773: finishing write tid 3 to smithi02979277-37 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2773: finishing write tid 4 to smithi02979277-37 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2773: finishing write tid 5 to smithi02979277-37 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2773: oid 37 updating version 0 to 981 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2773: oid 37 updating version 981 to 983 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2773: oid 37 version 983 is already newer than 982 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:2773: oid 37 updating version 983 to 984 2024-04-12T11:26:20.497 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 984 (ObjNum 1034 snap 354 seq_num 1034) dirty exists 2024-04-12T11:26:20.498 INFO:tasks.rados.rados.0.smithi029.stdout:2773: left oid 37 (ObjNum 1034 snap 354 seq_num 1034) 2024-04-12T11:26:20.498 INFO:tasks.rados.rados.0.smithi029.stdout:2775: finishing rollback tid 0 to smithi02979277-47 2024-04-12T11:26:20.498 INFO:tasks.rados.rados.0.smithi029.stdout:2775: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:26:20.498 INFO:tasks.rados.rados.0.smithi029.stdout:2774: expect (ObjNum 937 snap 325 seq_num 937) 2024-04-12T11:26:20.622 INFO:tasks.rados.rados.0.smithi029.stdout:2775: finishing rollback tid 2 to smithi02979277-47 2024-04-12T11:26:20.622 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1326 (ObjNum 1022 snap 352 seq_num 1022) dirty exists 2024-04-12T11:26:20.622 INFO:tasks.rados.rados.0.smithi029.stdout:2776: expect (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:26:20.840 INFO:tasks.rados.rados.0.smithi029.stdout:2768: done (9 left) 2024-04-12T11:26:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:2769: done (8 left) 2024-04-12T11:26:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:2770: done (7 left) 2024-04-12T11:26:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:2771: done (6 left) 2024-04-12T11:26:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:2772: done (5 left) 2024-04-12T11:26:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:2773: done (4 left) 2024-04-12T11:26:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:2774: done (3 left) 2024-04-12T11:26:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:2775: done (2 left) 2024-04-12T11:26:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:2776: done (1 left) 2024-04-12T11:26:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:2777: done (0 left) 2024-04-12T11:26:20.841 INFO:tasks.rados.rados.0.smithi029.stdout:2778: delete oid 30 current snap is 355 2024-04-12T11:26:20.843 INFO:tasks.rados.rados.0.smithi029.stdout:2778: done (0 left) 2024-04-12T11:26:20.843 INFO:tasks.rados.rados.0.smithi029.stdout:2779: rollback oid 50 current snap is 355 2024-04-12T11:26:20.843 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 50 to 322 2024-04-12T11:26:20.843 INFO:tasks.rados.rados.0.smithi029.stdout:2780: read oid 9 snap -1 2024-04-12T11:26:20.843 INFO:tasks.rados.rados.0.smithi029.stdout:2780: expect deleted 2024-04-12T11:26:20.843 INFO:tasks.rados.rados.0.smithi029.stdout:2781: write oid 23 current snap is 355 2024-04-12T11:26:20.844 INFO:tasks.rados.rados.0.smithi029.stdout:2781: seq_num 1036 ranges {612052=581972,1657698=497363,2717044=695008} 2024-04-12T11:26:20.849 INFO:tasks.rados.rados.0.smithi029.stdout:2781: writing smithi02979277-23 from 612052 to 1194024 tid 1 2024-04-12T11:26:20.854 INFO:tasks.rados.rados.0.smithi029.stdout:2781: writing smithi02979277-23 from 1657698 to 2155061 tid 2 2024-04-12T11:26:20.862 INFO:tasks.rados.rados.0.smithi029.stdout:2781: writing smithi02979277-23 from 2717044 to 3412052 tid 3 2024-04-12T11:26:20.862 INFO:tasks.rados.rados.0.smithi029.stdout:2782: write oid 13 current snap is 355 2024-04-12T11:26:20.862 INFO:tasks.rados.rados.0.smithi029.stdout:2779: finishing rollback tid 1 to smithi02979277-50 2024-04-12T11:26:20.862 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1111 (ObjNum 129 snap 0 seq_num 1801166800) dirty dne 2024-04-12T11:26:20.862 INFO:tasks.rados.rados.0.smithi029.stdout:2781: finishing write tid 1 to smithi02979277-23 2024-04-12T11:26:20.862 INFO:tasks.rados.rados.0.smithi029.stdout:2781: finishing write tid 2 to smithi02979277-23 2024-04-12T11:26:20.862 INFO:tasks.rados.rados.0.smithi029.stdout:2782: seq_num 1037 ranges {628859=657221,1716180=641021,3144467=284392} 2024-04-12T11:26:20.868 INFO:tasks.rados.rados.0.smithi029.stdout:2782: writing smithi02979277-13 from 628859 to 1286080 tid 1 2024-04-12T11:26:20.874 INFO:tasks.rados.rados.0.smithi029.stdout:2782: writing smithi02979277-13 from 1716180 to 2357201 tid 2 2024-04-12T11:26:20.881 INFO:tasks.rados.rados.0.smithi029.stdout:2782: writing smithi02979277-13 from 3144467 to 3428859 tid 3 2024-04-12T11:26:20.881 INFO:tasks.rados.rados.0.smithi029.stdout:2779: done (3 left) 2024-04-12T11:26:20.881 INFO:tasks.rados.rados.0.smithi029.stdout:2780: done (2 left) 2024-04-12T11:26:20.881 INFO:tasks.rados.rados.0.smithi029.stdout:2783: rollback oid 50 current snap is 355 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 50 to 352 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2781: finishing write tid 3 to smithi02979277-23 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2784: write oid 39 current snap is 355 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2781: finishing write tid 4 to smithi02979277-23 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2781: finishing write tid 5 to smithi02979277-23 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2781: oid 23 updating version 0 to 1129 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2781: oid 23 updating version 1129 to 1130 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2781: oid 23 updating version 1130 to 1132 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2781: oid 23 version 1132 is already newer than 1131 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1132 (ObjNum 1036 snap 355 seq_num 1036) dirty exists 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2781: left oid 23 (ObjNum 1036 snap 355 seq_num 1036) 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2782: finishing write tid 1 to smithi02979277-13 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2782: finishing write tid 2 to smithi02979277-13 2024-04-12T11:26:20.882 INFO:tasks.rados.rados.0.smithi029.stdout:2784: seq_num 1038 ranges {645666=732470,2174662=501032,3288243=157423} 2024-04-12T11:26:20.888 INFO:tasks.rados.rados.0.smithi029.stdout:2784: writing smithi02979277-39 from 645666 to 1378136 tid 1 2024-04-12T11:26:20.895 INFO:tasks.rados.rados.0.smithi029.stdout:2784: writing smithi02979277-39 from 2174662 to 2675694 tid 2 2024-04-12T11:26:20.899 INFO:tasks.rados.rados.0.smithi029.stdout:2784: writing smithi02979277-39 from 3288243 to 3445666 tid 3 2024-04-12T11:26:20.899 INFO:tasks.rados.rados.0.smithi029.stdout:2781: done (3 left) 2024-04-12T11:26:20.899 INFO:tasks.rados.rados.0.smithi029.stdout:2785: snap_remove snap 322 2024-04-12T11:26:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:20 smithi087 ceph-mon[89014]: pgmap v798: 89 pgs: 89 active+clean; 241 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1017 KiB/s wr, 6 op/s 2024-04-12T11:26:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:20 smithi087 ceph-mon[89014]: osdmap e775: 8 total, 8 up, 8 in 2024-04-12T11:26:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:20 smithi029 ceph-mon[93650]: pgmap v798: 89 pgs: 89 active+clean; 241 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1017 KiB/s wr, 6 op/s 2024-04-12T11:26:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:20 smithi029 ceph-mon[93650]: osdmap e775: 8 total, 8 up, 8 in 2024-04-12T11:26:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:20 smithi029 ceph-mon[97787]: pgmap v798: 89 pgs: 89 active+clean; 241 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1017 KiB/s wr, 6 op/s 2024-04-12T11:26:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:20 smithi029 ceph-mon[97787]: osdmap e775: 8 total, 8 up, 8 in 2024-04-12T11:26:21.694 INFO:tasks.rados.rados.0.smithi029.stdout:2785: done (3 left) 2024-04-12T11:26:21.694 INFO:tasks.rados.rados.0.smithi029.stdout:2786: snap_remove snap 349 2024-04-12T11:26:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:21 smithi087 ceph-mon[89014]: osdmap e776: 8 total, 8 up, 8 in 2024-04-12T11:26:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:22.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:21 smithi029 ceph-mon[97787]: osdmap e776: 8 total, 8 up, 8 in 2024-04-12T11:26:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:21 smithi029 ceph-mon[93650]: osdmap e776: 8 total, 8 up, 8 in 2024-04-12T11:26:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:22.635 INFO:tasks.rados.rados.0.smithi029.stdout:2786: done (3 left) 2024-04-12T11:26:22.635 INFO:tasks.rados.rados.0.smithi029.stdout:2787: write oid 14 current snap is 355 2024-04-12T11:26:22.636 INFO:tasks.rados.rados.0.smithi029.stdout:2787: seq_num 1039 ranges {662473=407719,1833144=644690,3199313=263160} 2024-04-12T11:26:22.641 INFO:tasks.rados.rados.0.smithi029.stdout:2787: writing smithi02979277-14 from 662473 to 1070192 tid 1 2024-04-12T11:26:22.649 INFO:tasks.rados.rados.0.smithi029.stdout:2787: writing smithi02979277-14 from 1833144 to 2477834 tid 2 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2787: writing smithi02979277-14 from 3199313 to 3462473 tid 3 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2788: snap_create 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2783: finishing rollback tid 1 to smithi02979277-50 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1111 (ObjNum 129 snap 0 seq_num 1800993304) dirty dne 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2782: finishing write tid 3 to smithi02979277-13 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2782: finishing write tid 4 to smithi02979277-13 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2782: finishing write tid 5 to smithi02979277-13 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2782: oid 13 updating version 0 to 1402 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2782: oid 13 updating version 1402 to 1403 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2782: oid 13 version 1403 is already newer than 1400 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2782: oid 13 version 1403 is already newer than 1401 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1403 (ObjNum 1037 snap 355 seq_num 1037) dirty exists 2024-04-12T11:26:22.656 INFO:tasks.rados.rados.0.smithi029.stdout:2782: left oid 13 (ObjNum 1037 snap 355 seq_num 1037) 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2784: finishing write tid 1 to smithi02979277-39 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2784: finishing write tid 2 to smithi02979277-39 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2784: finishing write tid 3 to smithi02979277-39 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2784: finishing write tid 4 to smithi02979277-39 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2784: finishing write tid 5 to smithi02979277-39 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2784: oid 39 updating version 0 to 371 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2784: oid 39 version 371 is already newer than 369 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2784: oid 39 updating version 371 to 372 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2784: oid 39 version 372 is already newer than 370 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 372 (ObjNum 1038 snap 355 seq_num 1038) dirty exists 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2784: left oid 39 (ObjNum 1038 snap 355 seq_num 1038) 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2787: finishing write tid 1 to smithi02979277-14 2024-04-12T11:26:22.657 INFO:tasks.rados.rados.0.smithi029.stdout:2787: finishing write tid 2 to smithi02979277-14 2024-04-12T11:26:22.658 INFO:tasks.rados.rados.0.smithi029.stdout:2787: finishing write tid 3 to smithi02979277-14 2024-04-12T11:26:22.658 INFO:tasks.rados.rados.0.smithi029.stdout:2787: finishing write tid 4 to smithi02979277-14 2024-04-12T11:26:22.658 INFO:tasks.rados.rados.0.smithi029.stdout:2787: finishing write tid 5 to smithi02979277-14 2024-04-12T11:26:22.658 INFO:tasks.rados.rados.0.smithi029.stdout:2787: oid 14 updating version 0 to 1304 2024-04-12T11:26:22.658 INFO:tasks.rados.rados.0.smithi029.stdout:2787: oid 14 version 1304 is already newer than 1302 2024-04-12T11:26:22.658 INFO:tasks.rados.rados.0.smithi029.stdout:2787: oid 14 version 1304 is already newer than 1303 2024-04-12T11:26:22.659 INFO:tasks.rados.rados.0.smithi029.stdout:2787: oid 14 updating version 1304 to 1305 2024-04-12T11:26:22.659 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 1305 (ObjNum 1039 snap 355 seq_num 1039) dirty exists 2024-04-12T11:26:22.659 INFO:tasks.rados.rados.0.smithi029.stdout:2787: left oid 14 (ObjNum 1039 snap 355 seq_num 1039) 2024-04-12T11:26:22.690 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:26:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:26:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:26:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:22 smithi087 ceph-mon[89014]: pgmap v801: 89 pgs: 89 active+clean; 241 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1017 KiB/s wr, 6 op/s 2024-04-12T11:26:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:22 smithi087 ceph-mon[89014]: osdmap e777: 8 total, 8 up, 8 in 2024-04-12T11:26:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:22 smithi087 ceph-mon[89014]: osdmap e778: 8 total, 8 up, 8 in 2024-04-12T11:26:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:22 smithi029 ceph-mon[93650]: pgmap v801: 89 pgs: 89 active+clean; 241 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1017 KiB/s wr, 6 op/s 2024-04-12T11:26:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:22 smithi029 ceph-mon[93650]: osdmap e777: 8 total, 8 up, 8 in 2024-04-12T11:26:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:22 smithi029 ceph-mon[93650]: osdmap e778: 8 total, 8 up, 8 in 2024-04-12T11:26:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:22 smithi029 ceph-mon[97787]: pgmap v801: 89 pgs: 89 active+clean; 241 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1017 KiB/s wr, 6 op/s 2024-04-12T11:26:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:22 smithi029 ceph-mon[97787]: osdmap e777: 8 total, 8 up, 8 in 2024-04-12T11:26:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:22 smithi029 ceph-mon[97787]: osdmap e778: 8 total, 8 up, 8 in 2024-04-12T11:26:23.636 INFO:tasks.rados.rados.0.smithi029.stdout:2782: done (4 left) 2024-04-12T11:26:23.636 INFO:tasks.rados.rados.0.smithi029.stdout:2783: done (3 left) 2024-04-12T11:26:23.637 INFO:tasks.rados.rados.0.smithi029.stdout:2784: done (2 left) 2024-04-12T11:26:23.637 INFO:tasks.rados.rados.0.smithi029.stdout:2787: done (1 left) 2024-04-12T11:26:23.637 INFO:tasks.rados.rados.0.smithi029.stdout:2788: done (0 left) 2024-04-12T11:26:23.637 INFO:tasks.rados.rados.0.smithi029.stdout:2789: delete oid 8 current snap is 356 2024-04-12T11:26:23.639 INFO:tasks.rados.rados.0.smithi029.stdout:2789: done (0 left) 2024-04-12T11:26:23.639 INFO:tasks.rados.rados.0.smithi029.stdout:2790: read oid 29 snap -1 2024-04-12T11:26:23.639 INFO:tasks.rados.rados.0.smithi029.stdout:2790: expect (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:26:23.639 INFO:tasks.rados.rados.0.smithi029.stdout:2791: read oid 43 snap -1 2024-04-12T11:26:23.639 INFO:tasks.rados.rados.0.smithi029.stdout:2791: expect deleted 2024-04-12T11:26:23.639 INFO:tasks.rados.rados.0.smithi029.stdout:2792: rollback oid 3 current snap is 356 2024-04-12T11:26:23.639 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 309 2024-04-12T11:26:23.639 INFO:tasks.rados.rados.0.smithi029.stdout:2793: write oid 44 current snap is 356 2024-04-12T11:26:23.639 INFO:tasks.rados.rados.0.smithi029.stdout:2793: seq_num 1041 ranges {696087=558217,1833755=648359,3137806=358281} 2024-04-12T11:26:23.645 INFO:tasks.rados.rados.0.smithi029.stdout:2793: writing smithi02979277-44 from 696087 to 1254304 tid 1 2024-04-12T11:26:23.652 INFO:tasks.rados.rados.0.smithi029.stdout:2793: writing smithi02979277-44 from 1833755 to 2482114 tid 2 2024-04-12T11:26:23.658 INFO:tasks.rados.rados.0.smithi029.stdout:2793: writing smithi02979277-44 from 3137806 to 3496087 tid 3 2024-04-12T11:26:23.658 INFO:tasks.rados.rados.0.smithi029.stdout:2794: delete oid 37 current snap is 356 2024-04-12T11:26:23.658 INFO:tasks.rados.rados.0.smithi029.stdout:2790: expect (ObjNum 851 snap 292 seq_num 851) 2024-04-12T11:26:23.791 INFO:tasks.rados.rados.0.smithi029.stdout:2792: finishing rollback tid 0 to smithi02979277-3 2024-04-12T11:26:23.791 INFO:tasks.rados.rados.0.smithi029.stdout:2792: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2792: finishing rollback tid 2 to smithi02979277-3 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1691 (ObjNum 779 snap 272 seq_num 779) dirty exists 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2790: done (4 left) 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2791: done (3 left) 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2792: done (2 left) 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2794: done (1 left) 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2795: write oid 32 current snap is 356 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2793: finishing write tid 1 to smithi02979277-44 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2793: finishing write tid 2 to smithi02979277-44 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2793: finishing write tid 3 to smithi02979277-44 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2793: finishing write tid 4 to smithi02979277-44 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2793: finishing write tid 5 to smithi02979277-44 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2793: oid 44 updating version 0 to 1341 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2793: oid 44 updating version 1341 to 1343 2024-04-12T11:26:23.792 INFO:tasks.rados.rados.0.smithi029.stdout:2793: oid 44 version 1343 is already newer than 1342 2024-04-12T11:26:23.793 INFO:tasks.rados.rados.0.smithi029.stdout:2793: oid 44 updating version 1343 to 1344 2024-04-12T11:26:23.793 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1344 (ObjNum 1041 snap 356 seq_num 1041) dirty exists 2024-04-12T11:26:23.793 INFO:tasks.rados.rados.0.smithi029.stdout:2793: left oid 44 (ObjNum 1041 snap 356 seq_num 1041) 2024-04-12T11:26:23.793 INFO:tasks.rados.rados.0.smithi029.stdout:2795: seq_num 1043 ranges {729701=425068,1667072=652028,2909005=485180} 2024-04-12T11:26:23.797 INFO:tasks.rados.rados.0.smithi029.stdout:2795: writing smithi02979277-32 from 729701 to 1154769 tid 1 2024-04-12T11:26:23.804 INFO:tasks.rados.rados.0.smithi029.stdout:2795: writing smithi02979277-32 from 1667072 to 2319100 tid 2 2024-04-12T11:26:23.810 INFO:tasks.rados.rados.0.smithi029.stdout:2795: writing smithi02979277-32 from 2909005 to 3394185 tid 3 2024-04-12T11:26:23.810 INFO:tasks.rados.rados.0.smithi029.stdout:2793: done (1 left) 2024-04-12T11:26:23.810 INFO:tasks.rados.rados.0.smithi029.stdout:2796: write oid 13 current snap is 356 2024-04-12T11:26:23.810 INFO:tasks.rados.rados.0.smithi029.stdout:2796: seq_num 1044 ranges {746508=500317,2009201=795686,3220075=326433} 2024-04-12T11:26:23.816 INFO:tasks.rados.rados.0.smithi029.stdout:2796: writing smithi02979277-13 from 746508 to 1246825 tid 1 2024-04-12T11:26:23.825 INFO:tasks.rados.rados.0.smithi029.stdout:2796: writing smithi02979277-13 from 2009201 to 2804887 tid 2 2024-04-12T11:26:23.829 INFO:tasks.rados.rados.0.smithi029.stdout:2796: writing smithi02979277-13 from 3220075 to 3546508 tid 3 2024-04-12T11:26:23.829 INFO:tasks.rados.rados.0.smithi029.stdout:2797: delete oid 49 current snap is 356 2024-04-12T11:26:23.829 INFO:tasks.rados.rados.0.smithi029.stdout:2795: finishing write tid 1 to smithi02979277-32 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2795: finishing write tid 2 to smithi02979277-32 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2795: finishing write tid 3 to smithi02979277-32 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2795: finishing write tid 4 to smithi02979277-32 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2795: finishing write tid 5 to smithi02979277-32 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2795: oid 32 updating version 0 to 1262 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2795: oid 32 updating version 1262 to 1263 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2795: oid 32 updating version 1263 to 1264 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2795: oid 32 updating version 1264 to 1265 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 1265 (ObjNum 1043 snap 356 seq_num 1043) dirty exists 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2795: left oid 32 (ObjNum 1043 snap 356 seq_num 1043) 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2796: finishing write tid 1 to smithi02979277-13 2024-04-12T11:26:23.830 INFO:tasks.rados.rados.0.smithi029.stdout:2796: finishing write tid 2 to smithi02979277-13 2024-04-12T11:26:23.831 INFO:tasks.rados.rados.0.smithi029.stdout:2795: done (2 left) 2024-04-12T11:26:23.831 INFO:tasks.rados.rados.0.smithi029.stdout:2797: done (1 left) 2024-04-12T11:26:23.831 INFO:tasks.rados.rados.0.smithi029.stdout:2798: read oid 48 snap -1 2024-04-12T11:26:23.831 INFO:tasks.rados.rados.0.smithi029.stdout:2798: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:26:23.831 INFO:tasks.rados.rados.0.smithi029.stdout:2799: delete oid 35 current snap is 356 2024-04-12T11:26:23.833 INFO:tasks.rados.rados.0.smithi029.stdout:2796: finishing write tid 3 to smithi02979277-13 2024-04-12T11:26:23.833 INFO:tasks.rados.rados.0.smithi029.stdout:2796: finishing write tid 4 to smithi02979277-13 2024-04-12T11:26:23.833 INFO:tasks.rados.rados.0.smithi029.stdout:2796: finishing write tid 5 to smithi02979277-13 2024-04-12T11:26:23.833 INFO:tasks.rados.rados.0.smithi029.stdout:2796: oid 13 updating version 0 to 1414 2024-04-12T11:26:23.833 INFO:tasks.rados.rados.0.smithi029.stdout:2796: oid 13 version 1414 is already newer than 1413 2024-04-12T11:26:23.833 INFO:tasks.rados.rados.0.smithi029.stdout:2796: oid 13 version 1414 is already newer than 1412 2024-04-12T11:26:23.833 INFO:tasks.rados.rados.0.smithi029.stdout:2796: oid 13 version 1414 is already newer than 1411 2024-04-12T11:26:23.833 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1414 (ObjNum 1044 snap 356 seq_num 1044) dirty exists 2024-04-12T11:26:23.833 INFO:tasks.rados.rados.0.smithi029.stdout:2796: left oid 13 (ObjNum 1044 snap 356 seq_num 1044) 2024-04-12T11:26:23.834 INFO:tasks.rados.rados.0.smithi029.stdout:2796: done (2 left) 2024-04-12T11:26:23.834 INFO:tasks.rados.rados.0.smithi029.stdout:2799: done (1 left) 2024-04-12T11:26:23.834 INFO:tasks.rados.rados.0.smithi029.stdout:2800: snap_create 2024-04-12T11:26:23.834 INFO:tasks.rados.rados.0.smithi029.stdout:2798: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:26:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2798: done (1 left) 2024-04-12T11:26:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2800: done (0 left) 2024-04-12T11:26:24.696 INFO:tasks.rados.rados.0.smithi029.stdout:2801: snap_remove snap 352 2024-04-12T11:26:24.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:24 smithi029 ceph-mon[93650]: pgmap v804: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 247 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2024-04-12T11:26:24.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:24 smithi029 ceph-mon[93650]: osdmap e779: 8 total, 8 up, 8 in 2024-04-12T11:26:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:24 smithi029 ceph-mon[97787]: pgmap v804: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 247 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2024-04-12T11:26:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:24 smithi029 ceph-mon[97787]: osdmap e779: 8 total, 8 up, 8 in 2024-04-12T11:26:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:24 smithi087 ceph-mon[89014]: pgmap v804: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 247 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.3 MiB/s wr, 17 op/s 2024-04-12T11:26:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:24 smithi087 ceph-mon[89014]: osdmap e779: 8 total, 8 up, 8 in 2024-04-12T11:26:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:26:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:26:24] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2024-04-12T11:26:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:2801: done (0 left) 2024-04-12T11:26:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:2802: write oid 4 current snap is 357 2024-04-12T11:26:25.702 INFO:tasks.rados.rados.0.smithi029.stdout:2802: seq_num 1047 ranges {796929=726064,2184647=659366,3302344=294585} 2024-04-12T11:26:25.710 INFO:tasks.rados.rados.0.smithi029.stdout:2802: writing smithi02979277-4 from 796929 to 1522993 tid 1 2024-04-12T11:26:25.717 INFO:tasks.rados.rados.0.smithi029.stdout:2802: writing smithi02979277-4 from 2184647 to 2844013 tid 2 2024-04-12T11:26:25.722 INFO:tasks.rados.rados.0.smithi029.stdout:2802: writing smithi02979277-4 from 3302344 to 3596929 tid 3 2024-04-12T11:26:25.722 INFO:tasks.rados.rados.0.smithi029.stdout:2803: write oid 11 current snap is 357 2024-04-12T11:26:25.722 INFO:tasks.rados.rados.0.smithi029.stdout:2802: finishing write tid 1 to smithi02979277-4 2024-04-12T11:26:25.722 INFO:tasks.rados.rados.0.smithi029.stdout:2803: seq_num 1048 ranges {413736=401313,1326776=403024,2413414=457893,3438716=175020} 2024-04-12T11:26:25.726 INFO:tasks.rados.rados.0.smithi029.stdout:2803: writing smithi02979277-11 from 413736 to 815049 tid 1 2024-04-12T11:26:25.732 INFO:tasks.rados.rados.0.smithi029.stdout:2803: writing smithi02979277-11 from 1326776 to 1729800 tid 2 2024-04-12T11:26:25.738 INFO:tasks.rados.rados.0.smithi029.stdout:2803: writing smithi02979277-11 from 2413414 to 2871307 tid 3 2024-04-12T11:26:25.743 INFO:tasks.rados.rados.0.smithi029.stdout:2803: writing smithi02979277-11 from 3438716 to 3613736 tid 4 2024-04-12T11:26:25.743 INFO:tasks.rados.rados.0.smithi029.stdout:2804: write oid 39 current snap is 357 2024-04-12T11:26:25.743 INFO:tasks.rados.rados.0.smithi029.stdout:2804: seq_num 1049 ranges {430543=476562,1385258=663035,2840837=669165} 2024-04-12T11:26:25.748 INFO:tasks.rados.rados.0.smithi029.stdout:2804: writing smithi02979277-39 from 430543 to 907105 tid 1 2024-04-12T11:26:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:2804: writing smithi02979277-39 from 1385258 to 2048293 tid 2 2024-04-12T11:26:25.763 INFO:tasks.rados.rados.0.smithi029.stdout:2804: writing smithi02979277-39 from 2840837 to 3510002 tid 3 2024-04-12T11:26:25.763 INFO:tasks.rados.rados.0.smithi029.stdout:2805: delete oid 36 current snap is 357 2024-04-12T11:26:25.763 INFO:tasks.rados.rados.0.smithi029.stdout:2802: finishing write tid 2 to smithi02979277-4 2024-04-12T11:26:25.763 INFO:tasks.rados.rados.0.smithi029.stdout:2802: finishing write tid 3 to smithi02979277-4 2024-04-12T11:26:25.763 INFO:tasks.rados.rados.0.smithi029.stdout:2802: finishing write tid 4 to smithi02979277-4 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2802: finishing write tid 5 to smithi02979277-4 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2802: oid 4 updating version 0 to 1310 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2802: oid 4 updating version 1310 to 1311 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2802: oid 4 version 1311 is already newer than 1309 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2802: oid 4 version 1311 is already newer than 1308 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1311 (ObjNum 1047 snap 357 seq_num 1047) dirty exists 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2802: left oid 4 (ObjNum 1047 snap 357 seq_num 1047) 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2803: finishing write tid 1 to smithi02979277-11 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2803: finishing write tid 2 to smithi02979277-11 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2803: finishing write tid 3 to smithi02979277-11 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2803: finishing write tid 4 to smithi02979277-11 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2803: finishing write tid 5 to smithi02979277-11 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2803: finishing write tid 6 to smithi02979277-11 2024-04-12T11:26:25.764 INFO:tasks.rados.rados.0.smithi029.stdout:2803: oid 11 updating version 0 to 1695 2024-04-12T11:26:25.765 INFO:tasks.rados.rados.0.smithi029.stdout:2803: oid 11 updating version 1695 to 1703 2024-04-12T11:26:25.765 INFO:tasks.rados.rados.0.smithi029.stdout:2803: oid 11 version 1703 is already newer than 1700 2024-04-12T11:26:25.765 INFO:tasks.rados.rados.0.smithi029.stdout:2803: oid 11 version 1703 is already newer than 1702 2024-04-12T11:26:25.765 INFO:tasks.rados.rados.0.smithi029.stdout:2803: oid 11 version 1703 is already newer than 1701 2024-04-12T11:26:25.765 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 1703 (ObjNum 1048 snap 357 seq_num 1048) dirty exists 2024-04-12T11:26:25.765 INFO:tasks.rados.rados.0.smithi029.stdout:2803: left oid 11 (ObjNum 1048 snap 357 seq_num 1048) 2024-04-12T11:26:25.765 INFO:tasks.rados.rados.0.smithi029.stdout:2804: finishing write tid 1 to smithi02979277-39 2024-04-12T11:26:25.765 INFO:tasks.rados.rados.0.smithi029.stdout:2804: finishing write tid 2 to smithi02979277-39 2024-04-12T11:26:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:2802: done (3 left) 2024-04-12T11:26:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:2803: done (2 left) 2024-04-12T11:26:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:2805: done (1 left) 2024-04-12T11:26:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:2806: read oid 50 snap -1 2024-04-12T11:26:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:2806: expect deleted 2024-04-12T11:26:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:2807: write oid 24 current snap is 357 2024-04-12T11:26:25.766 INFO:tasks.rados.rados.0.smithi029.stdout:2807: seq_num 1051 ranges {464157=743413,1618575=666704,3012036=408062} 2024-04-12T11:26:25.772 INFO:tasks.rados.rados.0.smithi029.stdout:2807: writing smithi02979277-24 from 464157 to 1207570 tid 1 2024-04-12T11:26:25.778 INFO:tasks.rados.rados.0.smithi029.stdout:2807: writing smithi02979277-24 from 1618575 to 2285279 tid 2 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2807: writing smithi02979277-24 from 3012036 to 3420098 tid 3 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2808: write oid 33 current snap is 357 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2804: finishing write tid 3 to smithi02979277-39 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2804: finishing write tid 4 to smithi02979277-39 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2804: finishing write tid 5 to smithi02979277-39 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2804: oid 39 updating version 0 to 381 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2804: oid 39 updating version 381 to 382 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2804: oid 39 version 382 is already newer than 380 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2804: oid 39 updating version 382 to 383 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 383 (ObjNum 1049 snap 357 seq_num 1049) dirty exists 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2804: left oid 39 (ObjNum 1049 snap 357 seq_num 1049) 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2807: finishing write tid 1 to smithi02979277-24 2024-04-12T11:26:25.785 INFO:tasks.rados.rados.0.smithi029.stdout:2807: finishing write tid 2 to smithi02979277-24 2024-04-12T11:26:25.786 INFO:tasks.rados.rados.0.smithi029.stdout:2808: seq_num 1052 ranges {480964=418662,1560704=410362,2523106=619334} 2024-04-12T11:26:25.789 INFO:tasks.rados.rados.0.smithi029.stdout:2808: writing smithi02979277-33 from 480964 to 899626 tid 1 2024-04-12T11:26:25.795 INFO:tasks.rados.rados.0.smithi029.stdout:2808: writing smithi02979277-33 from 1560704 to 1971066 tid 2 2024-04-12T11:26:25.802 INFO:tasks.rados.rados.0.smithi029.stdout:2808: writing smithi02979277-33 from 2523106 to 3142440 tid 3 2024-04-12T11:26:25.802 INFO:tasks.rados.rados.0.smithi029.stdout:2804: done (3 left) 2024-04-12T11:26:25.802 INFO:tasks.rados.rados.0.smithi029.stdout:2806: done (2 left) 2024-04-12T11:26:25.802 INFO:tasks.rados.rados.0.smithi029.stdout:2809: read oid 32 snap -1 2024-04-12T11:26:25.802 INFO:tasks.rados.rados.0.smithi029.stdout:2809: expect (ObjNum 1043 snap 356 seq_num 1043) 2024-04-12T11:26:25.802 INFO:tasks.rados.rados.0.smithi029.stdout:2810: rollback oid 21 current snap is 357 2024-04-12T11:26:25.802 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 21 to 341 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2807: finishing write tid 3 to smithi02979277-24 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2807: finishing write tid 4 to smithi02979277-24 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2807: finishing write tid 5 to smithi02979277-24 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2807: oid 24 updating version 0 to 1707 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2807: oid 24 updating version 1707 to 1708 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2807: oid 24 version 1708 is already newer than 1706 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2807: oid 24 version 1708 is already newer than 1705 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1708 (ObjNum 1051 snap 357 seq_num 1051) dirty exists 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2807: left oid 24 (ObjNum 1051 snap 357 seq_num 1051) 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2808: finishing write tid 1 to smithi02979277-33 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2808: finishing write tid 2 to smithi02979277-33 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2807: done (3 left) 2024-04-12T11:26:25.803 INFO:tasks.rados.rados.0.smithi029.stdout:2811: snap_create 2024-04-12T11:26:25.805 INFO:tasks.rados.rados.0.smithi029.stdout:2808: finishing write tid 3 to smithi02979277-33 2024-04-12T11:26:25.806 INFO:tasks.rados.rados.0.smithi029.stdout:2810: finishing rollback tid 0 to smithi02979277-21 2024-04-12T11:26:25.806 INFO:tasks.rados.rados.0.smithi029.stdout:2810: finishing rollback tid 1 to smithi02979277-21 2024-04-12T11:26:25.806 INFO:tasks.rados.rados.0.smithi029.stdout:2810: finishing rollback tid 2 to smithi02979277-21 2024-04-12T11:26:25.806 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1352 (ObjNum 941 snap 325 seq_num 941) dirty exists 2024-04-12T11:26:25.807 INFO:tasks.rados.rados.0.smithi029.stdout:2809: expect (ObjNum 1043 snap 356 seq_num 1043) 2024-04-12T11:26:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:25 smithi087 ceph-mon[89014]: osdmap e780: 8 total, 8 up, 8 in 2024-04-12T11:26:26.013 INFO:tasks.rados.rados.0.smithi029.stdout:2808: finishing write tid 4 to smithi02979277-33 2024-04-12T11:26:26.013 INFO:tasks.rados.rados.0.smithi029.stdout:2808: finishing write tid 5 to smithi02979277-33 2024-04-12T11:26:26.014 INFO:tasks.rados.rados.0.smithi029.stdout:2808: oid 33 updating version 0 to 1135 2024-04-12T11:26:26.014 INFO:tasks.rados.rados.0.smithi029.stdout:2808: oid 33 updating version 1135 to 1136 2024-04-12T11:26:26.014 INFO:tasks.rados.rados.0.smithi029.stdout:2808: oid 33 version 1136 is already newer than 1134 2024-04-12T11:26:26.014 INFO:tasks.rados.rados.0.smithi029.stdout:2808: oid 33 updating version 1136 to 1137 2024-04-12T11:26:26.014 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 1137 (ObjNum 1052 snap 357 seq_num 1052) dirty exists 2024-04-12T11:26:26.014 INFO:tasks.rados.rados.0.smithi029.stdout:2808: left oid 33 (ObjNum 1052 snap 357 seq_num 1052) 2024-04-12T11:26:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:25 smithi029 ceph-mon[93650]: osdmap e780: 8 total, 8 up, 8 in 2024-04-12T11:26:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:25 smithi029 ceph-mon[97787]: osdmap e780: 8 total, 8 up, 8 in 2024-04-12T11:26:26.702 INFO:tasks.rados.rados.0.smithi029.stdout:2808: done (3 left) 2024-04-12T11:26:26.703 INFO:tasks.rados.rados.0.smithi029.stdout:2809: done (2 left) 2024-04-12T11:26:26.703 INFO:tasks.rados.rados.0.smithi029.stdout:2810: done (1 left) 2024-04-12T11:26:26.703 INFO:tasks.rados.rados.0.smithi029.stdout:2811: done (0 left) 2024-04-12T11:26:26.703 INFO:tasks.rados.rados.0.smithi029.stdout:2812: snap_remove snap 353 2024-04-12T11:26:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:26 smithi087 ceph-mon[89014]: pgmap v807: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 250 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 3.3 MiB/s wr, 35 op/s 2024-04-12T11:26:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:26 smithi087 ceph-mon[89014]: osdmap e781: 8 total, 8 up, 8 in 2024-04-12T11:26:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:26 smithi029 ceph-mon[97787]: pgmap v807: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 250 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 3.3 MiB/s wr, 35 op/s 2024-04-12T11:26:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:26 smithi029 ceph-mon[97787]: osdmap e781: 8 total, 8 up, 8 in 2024-04-12T11:26:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:26 smithi029 ceph-mon[93650]: pgmap v807: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 250 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 3.3 MiB/s wr, 35 op/s 2024-04-12T11:26:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:26 smithi029 ceph-mon[93650]: osdmap e781: 8 total, 8 up, 8 in 2024-04-12T11:26:27.636 INFO:tasks.rados.rados.0.smithi029.stdout:2812: done (0 left) 2024-04-12T11:26:27.636 INFO:tasks.rados.rados.0.smithi029.stdout:2813: rollback oid 26 current snap is 358 2024-04-12T11:26:27.637 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 350 2024-04-12T11:26:27.637 INFO:tasks.rados.rados.0.smithi029.stdout:2814: delete oid 4 current snap is 358 2024-04-12T11:26:27.647 INFO:tasks.rados.rados.0.smithi029.stdout:2814: done (1 left) 2024-04-12T11:26:27.647 INFO:tasks.rados.rados.0.smithi029.stdout:2815: read oid 4 snap 350 2024-04-12T11:26:27.647 INFO:tasks.rados.rados.0.smithi029.stdout:2815: expect deleted 2024-04-12T11:26:27.647 INFO:tasks.rados.rados.0.smithi029.stdout:2816: write oid 29 current snap is 358 2024-04-12T11:26:27.647 INFO:tasks.rados.rados.0.smithi029.stdout:2816: seq_num 1054 ranges {514578=569160,1677668=414031,2577952=641878} 2024-04-12T11:26:27.653 INFO:tasks.rados.rados.0.smithi029.stdout:2816: writing smithi02979277-29 from 514578 to 1083738 tid 1 2024-04-12T11:26:27.659 INFO:tasks.rados.rados.0.smithi029.stdout:2816: writing smithi02979277-29 from 1677668 to 2091699 tid 2 2024-04-12T11:26:27.665 INFO:tasks.rados.rados.0.smithi029.stdout:2816: writing smithi02979277-29 from 2577952 to 3219830 tid 3 2024-04-12T11:26:27.665 INFO:tasks.rados.rados.0.smithi029.stdout:2817: snap_create 2024-04-12T11:26:27.665 INFO:tasks.rados.rados.0.smithi029.stdout:2813: finishing rollback tid 0 to smithi02979277-26 2024-04-12T11:26:27.665 INFO:tasks.rados.rados.0.smithi029.stdout:2813: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:26:27.665 INFO:tasks.rados.rados.0.smithi029.stdout:2813: finishing rollback tid 2 to smithi02979277-26 2024-04-12T11:26:27.665 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 1712 (ObjNum 938 snap 325 seq_num 938) dirty exists 2024-04-12T11:26:27.665 INFO:tasks.rados.rados.0.smithi029.stdout:2816: finishing write tid 1 to smithi02979277-29 2024-04-12T11:26:27.666 INFO:tasks.rados.rados.0.smithi029.stdout:2816: finishing write tid 2 to smithi02979277-29 2024-04-12T11:26:27.668 INFO:tasks.rados.rados.0.smithi029.stdout:2816: finishing write tid 3 to smithi02979277-29 2024-04-12T11:26:27.669 INFO:tasks.rados.rados.0.smithi029.stdout:2816: finishing write tid 4 to smithi02979277-29 2024-04-12T11:26:27.669 INFO:tasks.rados.rados.0.smithi029.stdout:2816: finishing write tid 5 to smithi02979277-29 2024-04-12T11:26:27.669 INFO:tasks.rados.rados.0.smithi029.stdout:2816: oid 29 updating version 0 to 1277 2024-04-12T11:26:27.669 INFO:tasks.rados.rados.0.smithi029.stdout:2816: oid 29 version 1277 is already newer than 1276 2024-04-12T11:26:27.669 INFO:tasks.rados.rados.0.smithi029.stdout:2816: oid 29 updating version 1277 to 1278 2024-04-12T11:26:27.669 INFO:tasks.rados.rados.0.smithi029.stdout:2816: oid 29 version 1278 is already newer than 1275 2024-04-12T11:26:27.669 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1278 (ObjNum 1054 snap 358 seq_num 1054) dirty exists 2024-04-12T11:26:27.669 INFO:tasks.rados.rados.0.smithi029.stdout:2816: left oid 29 (ObjNum 1054 snap 358 seq_num 1054) 2024-04-12T11:26:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:27 smithi087 ceph-mon[89014]: osdmap e782: 8 total, 8 up, 8 in 2024-04-12T11:26:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:27 smithi087 ceph-mon[89014]: osdmap e783: 8 total, 8 up, 8 in 2024-04-12T11:26:28.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:27 smithi029 ceph-mon[93650]: osdmap e782: 8 total, 8 up, 8 in 2024-04-12T11:26:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:27 smithi029 ceph-mon[93650]: osdmap e783: 8 total, 8 up, 8 in 2024-04-12T11:26:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:27 smithi029 ceph-mon[97787]: osdmap e782: 8 total, 8 up, 8 in 2024-04-12T11:26:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:27 smithi029 ceph-mon[97787]: osdmap e783: 8 total, 8 up, 8 in 2024-04-12T11:26:28.638 INFO:tasks.rados.rados.0.smithi029.stdout:2813: done (3 left) 2024-04-12T11:26:28.638 INFO:tasks.rados.rados.0.smithi029.stdout:2815: done (2 left) 2024-04-12T11:26:28.638 INFO:tasks.rados.rados.0.smithi029.stdout:2816: done (1 left) 2024-04-12T11:26:28.638 INFO:tasks.rados.rados.0.smithi029.stdout:2817: done (0 left) 2024-04-12T11:26:28.638 INFO:tasks.rados.rados.0.smithi029.stdout:2818: delete oid 25 current snap is 359 2024-04-12T11:26:28.642 INFO:tasks.rados.rados.0.smithi029.stdout:2818: done (0 left) 2024-04-12T11:26:28.643 INFO:tasks.rados.rados.0.smithi029.stdout:2819: snap_create 2024-04-12T11:26:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:28 smithi087 ceph-mon[89014]: pgmap v810: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 261 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 2.9 MiB/s wr, 25 op/s 2024-04-12T11:26:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:28 smithi087 ceph-mon[89014]: osdmap e784: 8 total, 8 up, 8 in 2024-04-12T11:26:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:28 smithi029 ceph-mon[93650]: pgmap v810: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 261 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 2.9 MiB/s wr, 25 op/s 2024-04-12T11:26:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:28 smithi029 ceph-mon[93650]: osdmap e784: 8 total, 8 up, 8 in 2024-04-12T11:26:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:28 smithi029 ceph-mon[97787]: pgmap v810: 89 pgs: 1 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 87 active+clean; 261 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 2.9 MiB/s wr, 25 op/s 2024-04-12T11:26:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:28 smithi029 ceph-mon[97787]: osdmap e784: 8 total, 8 up, 8 in 2024-04-12T11:26:29.640 INFO:tasks.rados.rados.0.smithi029.stdout:2819: done (0 left) 2024-04-12T11:26:29.640 INFO:tasks.rados.rados.0.smithi029.stdout:2820: rollback oid 38 current snap is 360 2024-04-12T11:26:29.640 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 38 to 330 2024-04-12T11:26:29.640 INFO:tasks.rados.rados.0.smithi029.stdout:2821: read oid 16 snap 354 2024-04-12T11:26:29.641 INFO:tasks.rados.rados.0.smithi029.stdout:2821: expect (ObjNum 1015 snap 349 seq_num 1015) 2024-04-12T11:26:29.641 INFO:tasks.rados.rados.0.smithi029.stdout:2822: read oid 33 snap -1 2024-04-12T11:26:29.641 INFO:tasks.rados.rados.0.smithi029.stdout:2822: expect (ObjNum 1052 snap 357 seq_num 1052) 2024-04-12T11:26:29.641 INFO:tasks.rados.rados.0.smithi029.stdout:2823: rollback oid 34 current snap is 360 2024-04-12T11:26:29.641 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 350 2024-04-12T11:26:29.641 INFO:tasks.rados.rados.0.smithi029.stdout:2824: write oid 3 current snap is 360 2024-04-12T11:26:29.641 INFO:tasks.rados.rados.0.smithi029.stdout:2824: seq_num 1056 ranges {548192=719658,1678279=417700,2516445=780775} 2024-04-12T11:26:29.647 INFO:tasks.rados.rados.0.smithi029.stdout:2824: writing smithi02979277-3 from 548192 to 1267850 tid 1 2024-04-12T11:26:29.654 INFO:tasks.rados.rados.0.smithi029.stdout:2824: writing smithi02979277-3 from 1678279 to 2095979 tid 2 2024-04-12T11:26:29.659 INFO:tasks.rados.rados.0.smithi029.stdout:2824: writing smithi02979277-3 from 2516445 to 3297220 tid 3 2024-04-12T11:26:29.659 INFO:tasks.rados.rados.0.smithi029.stdout:2825: write oid 37 current snap is 360 2024-04-12T11:26:29.659 INFO:tasks.rados.rados.0.smithi029.stdout:2820: finishing rollback tid 0 to smithi02979277-38 2024-04-12T11:26:29.659 INFO:tasks.rados.rados.0.smithi029.stdout:2823: finishing rollback tid 0 to smithi02979277-34 2024-04-12T11:26:29.659 INFO:tasks.rados.rados.0.smithi029.stdout:2825: seq_num 1057 ranges {564999=511260,1853114=561358,3060221=592047} 2024-04-12T11:26:29.665 INFO:tasks.rados.rados.0.smithi029.stdout:2825: writing smithi02979277-37 from 564999 to 1076259 tid 1 2024-04-12T11:26:29.672 INFO:tasks.rados.rados.0.smithi029.stdout:2825: writing smithi02979277-37 from 1853114 to 2414472 tid 2 2024-04-12T11:26:29.679 INFO:tasks.rados.rados.0.smithi029.stdout:2825: writing smithi02979277-37 from 3060221 to 3652268 tid 3 2024-04-12T11:26:29.679 INFO:tasks.rados.rados.0.smithi029.stdout:2826: write oid 35 current snap is 360 2024-04-12T11:26:29.680 INFO:tasks.rados.rados.0.smithi029.stdout:2820: finishing rollback tid 1 to smithi02979277-38 2024-04-12T11:26:29.680 INFO:tasks.rados.rados.0.smithi029.stdout:2823: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:26:29.680 INFO:tasks.rados.rados.0.smithi029.stdout:2823: finishing rollback tid 2 to smithi02979277-34 2024-04-12T11:26:29.680 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1282 (ObjNum 987 snap 342 seq_num 987) dirty exists 2024-04-12T11:26:29.680 INFO:tasks.rados.rados.0.smithi029.stdout:2820: finishing rollback tid 2 to smithi02979277-38 2024-04-12T11:26:29.680 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 831 (ObjNum 864 snap 302 seq_num 864) dirty exists 2024-04-12T11:26:29.680 INFO:tasks.rados.rados.0.smithi029.stdout:2826: seq_num 1058 ranges {581806=586509,1911596=421369,3087644=403319} 2024-04-12T11:26:29.686 INFO:tasks.rados.rados.0.smithi029.stdout:2826: writing smithi02979277-35 from 581806 to 1168315 tid 1 2024-04-12T11:26:29.692 INFO:tasks.rados.rados.0.smithi029.stdout:2826: writing smithi02979277-35 from 1911596 to 2332965 tid 2 2024-04-12T11:26:29.699 INFO:tasks.rados.rados.0.smithi029.stdout:2826: writing smithi02979277-35 from 3087644 to 3490963 tid 3 2024-04-12T11:26:29.699 INFO:tasks.rados.rados.0.smithi029.stdout:2820: done (6 left) 2024-04-12T11:26:29.699 INFO:tasks.rados.rados.0.smithi029.stdout:2823: done (5 left) 2024-04-12T11:26:29.699 INFO:tasks.rados.rados.0.smithi029.stdout:2827: snap_create 2024-04-12T11:26:29.700 INFO:tasks.rados.rados.0.smithi029.stdout:2824: finishing write tid 1 to smithi02979277-3 2024-04-12T11:26:29.700 INFO:tasks.rados.rados.0.smithi029.stdout:2824: finishing write tid 2 to smithi02979277-3 2024-04-12T11:26:29.700 INFO:tasks.rados.rados.0.smithi029.stdout:2822: expect (ObjNum 1052 snap 357 seq_num 1052) 2024-04-12T11:26:29.825 INFO:tasks.rados.rados.0.smithi029.stdout:2821: expect (ObjNum 1015 snap 349 seq_num 1015) 2024-04-12T11:26:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:2824: finishing write tid 3 to smithi02979277-3 2024-04-12T11:26:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:2824: finishing write tid 4 to smithi02979277-3 2024-04-12T11:26:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:2824: finishing write tid 5 to smithi02979277-3 2024-04-12T11:26:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:2824: oid 3 updating version 0 to 1723 2024-04-12T11:26:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:2824: oid 3 version 1723 is already newer than 1722 2024-04-12T11:26:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:2824: oid 3 updating version 1723 to 1725 2024-04-12T11:26:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:2824: oid 3 version 1725 is already newer than 1724 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1725 (ObjNum 1056 snap 360 seq_num 1056) dirty exists 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:2824: left oid 3 (ObjNum 1056 snap 360 seq_num 1056) 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:2825: finishing write tid 1 to smithi02979277-37 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:2825: finishing write tid 2 to smithi02979277-37 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:2825: finishing write tid 3 to smithi02979277-37 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:2825: finishing write tid 4 to smithi02979277-37 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:2825: finishing write tid 5 to smithi02979277-37 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:2825: oid 37 updating version 0 to 994 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:2825: oid 37 updating version 994 to 995 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:2825: oid 37 updating version 995 to 997 2024-04-12T11:26:29.928 INFO:tasks.rados.rados.0.smithi029.stdout:2825: oid 37 version 997 is already newer than 996 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 997 (ObjNum 1057 snap 360 seq_num 1057) dirty exists 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:2825: left oid 37 (ObjNum 1057 snap 360 seq_num 1057) 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:2826: finishing write tid 1 to smithi02979277-35 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:2826: finishing write tid 2 to smithi02979277-35 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:2826: finishing write tid 3 to smithi02979277-35 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:2826: finishing write tid 4 to smithi02979277-35 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:2826: finishing write tid 5 to smithi02979277-35 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:2826: oid 35 updating version 0 to 1727 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:2826: oid 35 version 1727 is already newer than 1726 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:2826: oid 35 updating version 1727 to 1729 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:2826: oid 35 version 1729 is already newer than 1728 2024-04-12T11:26:29.929 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 1729 (ObjNum 1058 snap 360 seq_num 1058) dirty exists 2024-04-12T11:26:29.930 INFO:tasks.rados.rados.0.smithi029.stdout:2826: left oid 35 (ObjNum 1058 snap 360 seq_num 1058) 2024-04-12T11:26:30.644 INFO:tasks.rados.rados.0.smithi029.stdout:2821: done (5 left) 2024-04-12T11:26:30.645 INFO:tasks.rados.rados.0.smithi029.stdout:2822: done (4 left) 2024-04-12T11:26:30.645 INFO:tasks.rados.rados.0.smithi029.stdout:2824: done (3 left) 2024-04-12T11:26:30.645 INFO:tasks.rados.rados.0.smithi029.stdout:2825: done (2 left) 2024-04-12T11:26:30.645 INFO:tasks.rados.rados.0.smithi029.stdout:2826: done (1 left) 2024-04-12T11:26:30.645 INFO:tasks.rados.rados.0.smithi029.stdout:2827: done (0 left) 2024-04-12T11:26:30.645 INFO:tasks.rados.rados.0.smithi029.stdout:2828: read oid 45 snap -1 2024-04-12T11:26:30.645 INFO:tasks.rados.rados.0.smithi029.stdout:2828: expect (ObjNum 962 snap 330 seq_num 962) 2024-04-12T11:26:30.645 INFO:tasks.rados.rados.0.smithi029.stdout:2829: write oid 8 current snap is 361 2024-04-12T11:26:30.645 INFO:tasks.rados.rados.0.smithi029.stdout:2829: seq_num 1059 ranges {598613=661758,1970078=565027,3115067=614591} 2024-04-12T11:26:30.651 INFO:tasks.rados.rados.0.smithi029.stdout:2829: writing smithi02979277-8 from 598613 to 1260371 tid 1 2024-04-12T11:26:30.658 INFO:tasks.rados.rados.0.smithi029.stdout:2829: writing smithi02979277-8 from 1970078 to 2535105 tid 2 2024-04-12T11:26:30.665 INFO:tasks.rados.rados.0.smithi029.stdout:2829: writing smithi02979277-8 from 3115067 to 3729658 tid 3 2024-04-12T11:26:30.665 INFO:tasks.rados.rados.0.smithi029.stdout:2830: read oid 39 snap 341 2024-04-12T11:26:30.665 INFO:tasks.rados.rados.0.smithi029.stdout:2830: expect (ObjNum 930 snap 320 seq_num 930) 2024-04-12T11:26:30.666 INFO:tasks.rados.rados.0.smithi029.stdout:2831: rollback oid 40 current snap is 361 2024-04-12T11:26:30.666 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 40 to 356 2024-04-12T11:26:30.666 INFO:tasks.rados.rados.0.smithi029.stdout:2828: expect (ObjNum 962 snap 330 seq_num 962) 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:2829: finishing write tid 1 to smithi02979277-8 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:2829: finishing write tid 2 to smithi02979277-8 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:2829: finishing write tid 3 to smithi02979277-8 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:2829: finishing write tid 4 to smithi02979277-8 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:2829: finishing write tid 5 to smithi02979277-8 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:2829: oid 8 updating version 0 to 315 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:2829: oid 8 version 315 is already newer than 312 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:2829: oid 8 version 315 is already newer than 314 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:2829: oid 8 version 315 is already newer than 313 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 315 (ObjNum 1059 snap 361 seq_num 1059) dirty exists 2024-04-12T11:26:30.739 INFO:tasks.rados.rados.0.smithi029.stdout:2829: left oid 8 (ObjNum 1059 snap 361 seq_num 1059) 2024-04-12T11:26:30.740 INFO:tasks.rados.rados.0.smithi029.stdout:2831: finishing rollback tid 0 to smithi02979277-40 2024-04-12T11:26:30.740 INFO:tasks.rados.rados.0.smithi029.stdout:2831: finishing rollback tid 1 to smithi02979277-40 2024-04-12T11:26:30.740 INFO:tasks.rados.rados.0.smithi029.stdout:2831: finishing rollback tid 2 to smithi02979277-40 2024-04-12T11:26:30.740 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 1733 (ObjNum 1012 snap 348 seq_num 1012) dirty exists 2024-04-12T11:26:30.740 INFO:tasks.rados.rados.0.smithi029.stdout:2830: expect (ObjNum 930 snap 320 seq_num 930) 2024-04-12T11:26:30.947 INFO:tasks.rados.rados.0.smithi029.stdout:2828: done (3 left) 2024-04-12T11:26:30.947 INFO:tasks.rados.rados.0.smithi029.stdout:2829: done (2 left) 2024-04-12T11:26:30.947 INFO:tasks.rados.rados.0.smithi029.stdout:2830: done (1 left) 2024-04-12T11:26:30.947 INFO:tasks.rados.rados.0.smithi029.stdout:2831: done (0 left) 2024-04-12T11:26:30.947 INFO:tasks.rados.rados.0.smithi029.stdout:2832: read oid 14 snap -1 2024-04-12T11:26:30.948 INFO:tasks.rados.rados.0.smithi029.stdout:2832: expect (ObjNum 1039 snap 355 seq_num 1039) 2024-04-12T11:26:30.948 INFO:tasks.rados.rados.0.smithi029.stdout:2833: read oid 44 snap -1 2024-04-12T11:26:30.948 INFO:tasks.rados.rados.0.smithi029.stdout:2833: expect (ObjNum 1041 snap 356 seq_num 1041) 2024-04-12T11:26:30.948 INFO:tasks.rados.rados.0.smithi029.stdout:2834: rollback oid 34 current snap is 361 2024-04-12T11:26:30.948 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 355 2024-04-12T11:26:30.948 INFO:tasks.rados.rados.0.smithi029.stdout:2835: read oid 43 snap 355 2024-04-12T11:26:30.948 INFO:tasks.rados.rados.0.smithi029.stdout:2835: expect deleted 2024-04-12T11:26:30.948 INFO:tasks.rados.rados.0.smithi029.stdout:2836: read oid 11 snap -1 2024-04-12T11:26:30.948 INFO:tasks.rados.rados.0.smithi029.stdout:2836: expect (ObjNum 1048 snap 357 seq_num 1048) 2024-04-12T11:26:30.948 INFO:tasks.rados.rados.0.smithi029.stdout:2837: snap_remove snap 350 2024-04-12T11:26:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:30 smithi087 ceph-mon[89014]: pgmap v813: 89 pgs: 89 active+clean; 269 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2024-04-12T11:26:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:30 smithi087 ceph-mon[89014]: osdmap e785: 8 total, 8 up, 8 in 2024-04-12T11:26:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:30 smithi029 ceph-mon[97787]: pgmap v813: 89 pgs: 89 active+clean; 269 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2024-04-12T11:26:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:30 smithi029 ceph-mon[97787]: osdmap e785: 8 total, 8 up, 8 in 2024-04-12T11:26:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:30 smithi029 ceph-mon[93650]: pgmap v813: 89 pgs: 89 active+clean; 269 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2024-04-12T11:26:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:30 smithi029 ceph-mon[93650]: osdmap e785: 8 total, 8 up, 8 in 2024-04-12T11:26:31.651 INFO:tasks.rados.rados.0.smithi029.stdout:2834: finishing rollback tid 0 to smithi02979277-34 2024-04-12T11:26:31.651 INFO:tasks.rados.rados.0.smithi029.stdout:2834: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:26:31.651 INFO:tasks.rados.rados.0.smithi029.stdout:2834: finishing rollback tid 2 to smithi02979277-34 2024-04-12T11:26:31.652 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1286 (ObjNum 987 snap 342 seq_num 987) dirty exists 2024-04-12T11:26:31.652 INFO:tasks.rados.rados.0.smithi029.stdout:2836: expect (ObjNum 1048 snap 357 seq_num 1048) 2024-04-12T11:26:31.862 INFO:tasks.rados.rados.0.smithi029.stdout:2832: expect (ObjNum 1039 snap 355 seq_num 1039) 2024-04-12T11:26:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:31 smithi087 ceph-mon[89014]: osdmap e786: 8 total, 8 up, 8 in 2024-04-12T11:26:32.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:31 smithi029 ceph-mon[93650]: osdmap e786: 8 total, 8 up, 8 in 2024-04-12T11:26:32.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:31 smithi029 ceph-mon[97787]: osdmap e786: 8 total, 8 up, 8 in 2024-04-12T11:26:32.023 INFO:tasks.rados.rados.0.smithi029.stdout:2833: expect (ObjNum 1041 snap 356 seq_num 1041) 2024-04-12T11:26:32.229 INFO:tasks.rados.rados.0.smithi029.stdout:2832: done (5 left) 2024-04-12T11:26:32.229 INFO:tasks.rados.rados.0.smithi029.stdout:2833: done (4 left) 2024-04-12T11:26:32.229 INFO:tasks.rados.rados.0.smithi029.stdout:2834: done (3 left) 2024-04-12T11:26:32.229 INFO:tasks.rados.rados.0.smithi029.stdout:2835: done (2 left) 2024-04-12T11:26:32.229 INFO:tasks.rados.rados.0.smithi029.stdout:2836: done (1 left) 2024-04-12T11:26:32.229 INFO:tasks.rados.rados.0.smithi029.stdout:2837: done (0 left) 2024-04-12T11:26:32.229 INFO:tasks.rados.rados.0.smithi029.stdout:2838: snap_remove snap 360 2024-04-12T11:26:32.637 INFO:tasks.rados.rados.0.smithi029.stdout:2838: done (0 left) 2024-04-12T11:26:32.638 INFO:tasks.rados.rados.0.smithi029.stdout:2839: read oid 41 snap -1 2024-04-12T11:26:32.638 INFO:tasks.rados.rados.0.smithi029.stdout:2839: expect (ObjNum 1030 snap 354 seq_num 1030) 2024-04-12T11:26:32.638 INFO:tasks.rados.rados.0.smithi029.stdout:2840: write oid 5 current snap is 361 2024-04-12T11:26:32.638 INFO:tasks.rados.rados.0.smithi029.stdout:2840: seq_num 1060 ranges {615420=737007,1912207=425038,3026137=542216} 2024-04-12T11:26:32.645 INFO:tasks.rados.rados.0.smithi029.stdout:2840: writing smithi02979277-5 from 615420 to 1352427 tid 1 2024-04-12T11:26:32.648 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:26:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:26:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:26:32.651 INFO:tasks.rados.rados.0.smithi029.stdout:2840: writing smithi02979277-5 from 1912207 to 2337245 tid 2 2024-04-12T11:26:32.659 INFO:tasks.rados.rados.0.smithi029.stdout:2840: writing smithi02979277-5 from 3026137 to 3568353 tid 3 2024-04-12T11:26:32.659 INFO:tasks.rados.rados.0.smithi029.stdout:2841: snap_create 2024-04-12T11:26:32.659 INFO:tasks.rados.rados.0.smithi029.stdout:2839: expect (ObjNum 1030 snap 354 seq_num 1030) 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:2840: finishing write tid 1 to smithi02979277-5 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:2840: finishing write tid 2 to smithi02979277-5 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:2840: finishing write tid 3 to smithi02979277-5 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:2840: finishing write tid 4 to smithi02979277-5 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:2840: finishing write tid 5 to smithi02979277-5 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:2840: oid 5 updating version 0 to 1321 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:2840: oid 5 updating version 1321 to 1323 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:2840: oid 5 version 1323 is already newer than 1322 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:2840: oid 5 version 1323 is already newer than 1320 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1323 (ObjNum 1060 snap 361 seq_num 1060) dirty exists 2024-04-12T11:26:32.946 INFO:tasks.rados.rados.0.smithi029.stdout:2840: left oid 5 (ObjNum 1060 snap 361 seq_num 1060) 2024-04-12T11:26:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:32 smithi087 ceph-mon[89014]: pgmap v816: 89 pgs: 89 active+clean; 269 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-04-12T11:26:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:32 smithi087 ceph-mon[89014]: osdmap e787: 8 total, 8 up, 8 in 2024-04-12T11:26:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:32 smithi087 ceph-mon[89014]: osdmap e788: 8 total, 8 up, 8 in 2024-04-12T11:26:33.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:32 smithi029 ceph-mon[93650]: pgmap v816: 89 pgs: 89 active+clean; 269 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-04-12T11:26:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:32 smithi029 ceph-mon[93650]: osdmap e787: 8 total, 8 up, 8 in 2024-04-12T11:26:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:32 smithi029 ceph-mon[93650]: osdmap e788: 8 total, 8 up, 8 in 2024-04-12T11:26:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:32 smithi029 ceph-mon[97787]: pgmap v816: 89 pgs: 89 active+clean; 269 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-04-12T11:26:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:32 smithi029 ceph-mon[97787]: osdmap e787: 8 total, 8 up, 8 in 2024-04-12T11:26:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:32 smithi029 ceph-mon[97787]: osdmap e788: 8 total, 8 up, 8 in 2024-04-12T11:26:33.656 INFO:tasks.rados.rados.0.smithi029.stdout:2839: done (2 left) 2024-04-12T11:26:33.656 INFO:tasks.rados.rados.0.smithi029.stdout:2840: done (1 left) 2024-04-12T11:26:33.656 INFO:tasks.rados.rados.0.smithi029.stdout:2841: done (0 left) 2024-04-12T11:26:33.656 INFO:tasks.rados.rados.0.smithi029.stdout:2842: write oid 12 current snap is 362 2024-04-12T11:26:33.656 INFO:tasks.rados.rados.0.smithi029.stdout:2842: seq_num 1061 ranges {632227=412256,1570689=568696,2653560=753488} 2024-04-12T11:26:33.661 INFO:tasks.rados.rados.0.smithi029.stdout:2842: writing smithi02979277-12 from 632227 to 1044483 tid 1 2024-04-12T11:26:33.668 INFO:tasks.rados.rados.0.smithi029.stdout:2842: writing smithi02979277-12 from 1570689 to 2139385 tid 2 2024-04-12T11:26:33.675 INFO:tasks.rados.rados.0.smithi029.stdout:2842: writing smithi02979277-12 from 2653560 to 3407048 tid 3 2024-04-12T11:26:33.675 INFO:tasks.rados.rados.0.smithi029.stdout:2843: rollback oid 13 current snap is 362 2024-04-12T11:26:33.675 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 13 to 309 2024-04-12T11:26:33.675 INFO:tasks.rados.rados.0.smithi029.stdout:2842: finishing write tid 1 to smithi02979277-12 2024-04-12T11:26:33.675 INFO:tasks.rados.rados.0.smithi029.stdout:2842: finishing write tid 2 to smithi02979277-12 2024-04-12T11:26:33.675 INFO:tasks.rados.rados.0.smithi029.stdout:2844: delete oid 41 current snap is 362 2024-04-12T11:26:33.679 INFO:tasks.rados.rados.0.smithi029.stdout:2844: done (2 left) 2024-04-12T11:26:33.679 INFO:tasks.rados.rados.0.smithi029.stdout:2845: snap_create 2024-04-12T11:26:33.680 INFO:tasks.rados.rados.0.smithi029.stdout:2842: finishing write tid 3 to smithi02979277-12 2024-04-12T11:26:33.680 INFO:tasks.rados.rados.0.smithi029.stdout:2842: finishing write tid 4 to smithi02979277-12 2024-04-12T11:26:33.680 INFO:tasks.rados.rados.0.smithi029.stdout:2842: finishing write tid 5 to smithi02979277-12 2024-04-12T11:26:33.680 INFO:tasks.rados.rados.0.smithi029.stdout:2842: oid 12 updating version 0 to 1343 2024-04-12T11:26:33.681 INFO:tasks.rados.rados.0.smithi029.stdout:2842: oid 12 updating version 1343 to 1344 2024-04-12T11:26:33.681 INFO:tasks.rados.rados.0.smithi029.stdout:2842: oid 12 updating version 1344 to 1345 2024-04-12T11:26:33.681 INFO:tasks.rados.rados.0.smithi029.stdout:2842: oid 12 updating version 1345 to 1346 2024-04-12T11:26:33.681 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1346 (ObjNum 1061 snap 362 seq_num 1061) dirty exists 2024-04-12T11:26:33.681 INFO:tasks.rados.rados.0.smithi029.stdout:2842: left oid 12 (ObjNum 1061 snap 362 seq_num 1061) 2024-04-12T11:26:33.681 INFO:tasks.rados.rados.0.smithi029.stdout:2843: finishing rollback tid 0 to smithi02979277-13 2024-04-12T11:26:33.681 INFO:tasks.rados.rados.0.smithi029.stdout:2843: finishing rollback tid 1 to smithi02979277-13 2024-04-12T11:26:33.681 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1421 (ObjNum 49 snap 0 seq_num 1800915824) dirty dne 2024-04-12T11:26:34.661 INFO:tasks.rados.rados.0.smithi029.stdout:2842: done (2 left) 2024-04-12T11:26:34.661 INFO:tasks.rados.rados.0.smithi029.stdout:2843: done (1 left) 2024-04-12T11:26:34.661 INFO:tasks.rados.rados.0.smithi029.stdout:2845: done (0 left) 2024-04-12T11:26:34.661 INFO:tasks.rados.rados.0.smithi029.stdout:2846: read oid 48 snap -1 2024-04-12T11:26:34.661 INFO:tasks.rados.rados.0.smithi029.stdout:2846: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:26:34.661 INFO:tasks.rados.rados.0.smithi029.stdout:2847: delete oid 23 current snap is 363 2024-04-12T11:26:34.664 INFO:tasks.rados.rados.0.smithi029.stdout:2846: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:26:34.832 INFO:tasks.rados.rados.0.smithi029.stdout:2846: done (1 left) 2024-04-12T11:26:34.832 INFO:tasks.rados.rados.0.smithi029.stdout:2847: done (0 left) 2024-04-12T11:26:34.832 INFO:tasks.rados.rados.0.smithi029.stdout:2848: read oid 47 snap -1 2024-04-12T11:26:34.832 INFO:tasks.rados.rados.0.smithi029.stdout:2848: expect (ObjNum 1022 snap 352 seq_num 1022) 2024-04-12T11:26:34.832 INFO:tasks.rados.rados.0.smithi029.stdout:2849: read oid 2 snap 341 2024-04-12T11:26:34.832 INFO:tasks.rados.rados.0.smithi029.stdout:2849: expect (ObjNum 983 snap 341 seq_num 983) 2024-04-12T11:26:34.833 INFO:tasks.rados.rados.0.smithi029.stdout:2850: snap_remove snap 334 2024-04-12T11:26:34.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:34 smithi029 ceph-mon[97787]: pgmap v819: 89 pgs: 89 active+clean; 268 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 857 KiB/s wr, 14 op/s 2024-04-12T11:26:34.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:34 smithi029 ceph-mon[97787]: osdmap e789: 8 total, 8 up, 8 in 2024-04-12T11:26:34.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:34 smithi029 ceph-mon[93650]: pgmap v819: 89 pgs: 89 active+clean; 268 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 857 KiB/s wr, 14 op/s 2024-04-12T11:26:34.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:34 smithi029 ceph-mon[93650]: osdmap e789: 8 total, 8 up, 8 in 2024-04-12T11:26:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:34 smithi087 ceph-mon[89014]: pgmap v819: 89 pgs: 89 active+clean; 268 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 857 KiB/s wr, 14 op/s 2024-04-12T11:26:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:34 smithi087 ceph-mon[89014]: osdmap e789: 8 total, 8 up, 8 in 2024-04-12T11:26:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:26:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:26:34] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2024-04-12T11:26:35.668 INFO:tasks.rados.rados.0.smithi029.stdout:2850: done (2 left) 2024-04-12T11:26:35.668 INFO:tasks.rados.rados.0.smithi029.stdout:2851: read oid 25 snap -1 2024-04-12T11:26:35.668 INFO:tasks.rados.rados.0.smithi029.stdout:2851: expect deleted 2024-04-12T11:26:35.668 INFO:tasks.rados.rados.0.smithi029.stdout:2852: write oid 35 current snap is 363 2024-04-12T11:26:35.668 INFO:tasks.rados.rados.0.smithi029.stdout:2852: seq_num 1064 ranges {682648=638003,2029782=432376,3019476=587304} 2024-04-12T11:26:35.676 INFO:tasks.rados.rados.0.smithi029.stdout:2852: writing smithi02979277-35 from 682648 to 1320651 tid 1 2024-04-12T11:26:35.683 INFO:tasks.rados.rados.0.smithi029.stdout:2852: writing smithi02979277-35 from 2029782 to 2462158 tid 2 2024-04-12T11:26:35.690 INFO:tasks.rados.rados.0.smithi029.stdout:2852: writing smithi02979277-35 from 3019476 to 3606780 tid 3 2024-04-12T11:26:35.690 INFO:tasks.rados.rados.0.smithi029.stdout:2853: read oid 1 snap -1 2024-04-12T11:26:35.690 INFO:tasks.rados.rados.0.smithi029.stdout:2853: expect (ObjNum 1000 snap 345 seq_num 1000) 2024-04-12T11:26:35.690 INFO:tasks.rados.rados.0.smithi029.stdout:2854: snap_remove snap 309 2024-04-12T11:26:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:35 smithi087 ceph-mon[89014]: osdmap e790: 8 total, 8 up, 8 in 2024-04-12T11:26:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:35 smithi029 ceph-mon[93650]: osdmap e790: 8 total, 8 up, 8 in 2024-04-12T11:26:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:35 smithi029 ceph-mon[97787]: osdmap e790: 8 total, 8 up, 8 in 2024-04-12T11:26:36.674 INFO:tasks.rados.rados.0.smithi029.stdout:2854: done (5 left) 2024-04-12T11:26:36.674 INFO:tasks.rados.rados.0.smithi029.stdout:2855: write oid 7 current snap is 363 2024-04-12T11:26:36.674 INFO:tasks.rados.rados.0.smithi029.stdout:2855: seq_num 1065 ranges {699455=429605,1804617=576034,3163252=514929} 2024-04-12T11:26:36.680 INFO:tasks.rados.rados.0.smithi029.stdout:2855: writing smithi02979277-7 from 699455 to 1129060 tid 1 2024-04-12T11:26:36.687 INFO:tasks.rados.rados.0.smithi029.stdout:2855: writing smithi02979277-7 from 1804617 to 2380651 tid 2 2024-04-12T11:26:36.695 INFO:tasks.rados.rados.0.smithi029.stdout:2855: writing smithi02979277-7 from 3163252 to 3678181 tid 3 2024-04-12T11:26:36.695 INFO:tasks.rados.rados.0.smithi029.stdout:2856: read oid 30 snap -1 2024-04-12T11:26:36.695 INFO:tasks.rados.rados.0.smithi029.stdout:2856: expect deleted 2024-04-12T11:26:36.695 INFO:tasks.rados.rados.0.smithi029.stdout:2857: snap_create 2024-04-12T11:26:36.695 INFO:tasks.rados.rados.0.smithi029.stdout:2849: expect (ObjNum 983 snap 341 seq_num 983) 2024-04-12T11:26:36.813 INFO:tasks.rados.rados.0.smithi029.stdout:2848: expect (ObjNum 1022 snap 352 seq_num 1022) 2024-04-12T11:26:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:36 smithi087 ceph-mon[89014]: pgmap v822: 89 pgs: 89 active+clean; 271 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-04-12T11:26:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:36 smithi087 ceph-mon[89014]: osdmap e791: 8 total, 8 up, 8 in 2024-04-12T11:26:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:36.967 INFO:tasks.rados.rados.0.smithi029.stdout:2852: finishing write tid 1 to smithi02979277-35 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:2852: finishing write tid 2 to smithi02979277-35 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:2852: finishing write tid 3 to smithi02979277-35 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:2852: finishing write tid 4 to smithi02979277-35 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:2852: finishing write tid 5 to smithi02979277-35 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:2852: oid 35 updating version 0 to 1753 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:2852: oid 35 updating version 1753 to 1759 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:2852: oid 35 updating version 1759 to 1760 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:2852: oid 35 version 1760 is already newer than 1758 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 1760 (ObjNum 1064 snap 363 seq_num 1064) dirty exists 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:2852: left oid 35 (ObjNum 1064 snap 363 seq_num 1064) 2024-04-12T11:26:36.968 INFO:tasks.rados.rados.0.smithi029.stdout:2853: expect (ObjNum 1000 snap 345 seq_num 1000) 2024-04-12T11:26:37.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:36 smithi029 ceph-mon[93650]: pgmap v822: 89 pgs: 89 active+clean; 271 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-04-12T11:26:37.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:36 smithi029 ceph-mon[93650]: osdmap e791: 8 total, 8 up, 8 in 2024-04-12T11:26:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:36 smithi029 ceph-mon[97787]: pgmap v822: 89 pgs: 89 active+clean; 271 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-04-12T11:26:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:36 smithi029 ceph-mon[97787]: osdmap e791: 8 total, 8 up, 8 in 2024-04-12T11:26:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:37.098 INFO:tasks.rados.rados.0.smithi029.stdout:2855: finishing write tid 1 to smithi02979277-7 2024-04-12T11:26:37.098 INFO:tasks.rados.rados.0.smithi029.stdout:2855: finishing write tid 2 to smithi02979277-7 2024-04-12T11:26:37.098 INFO:tasks.rados.rados.0.smithi029.stdout:2855: finishing write tid 3 to smithi02979277-7 2024-04-12T11:26:37.098 INFO:tasks.rados.rados.0.smithi029.stdout:2855: finishing write tid 4 to smithi02979277-7 2024-04-12T11:26:37.098 INFO:tasks.rados.rados.0.smithi029.stdout:2855: finishing write tid 5 to smithi02979277-7 2024-04-12T11:26:37.098 INFO:tasks.rados.rados.0.smithi029.stdout:2855: oid 7 updating version 0 to 1359 2024-04-12T11:26:37.098 INFO:tasks.rados.rados.0.smithi029.stdout:2855: oid 7 version 1359 is already newer than 1358 2024-04-12T11:26:37.098 INFO:tasks.rados.rados.0.smithi029.stdout:2855: oid 7 version 1359 is already newer than 1357 2024-04-12T11:26:37.099 INFO:tasks.rados.rados.0.smithi029.stdout:2855: oid 7 version 1359 is already newer than 1356 2024-04-12T11:26:37.099 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1359 (ObjNum 1065 snap 363 seq_num 1065) dirty exists 2024-04-12T11:26:37.099 INFO:tasks.rados.rados.0.smithi029.stdout:2855: left oid 7 (ObjNum 1065 snap 363 seq_num 1065) 2024-04-12T11:26:37.639 INFO:tasks.rados.rados.0.smithi029.stdout:2848: done (7 left) 2024-04-12T11:26:37.639 INFO:tasks.rados.rados.0.smithi029.stdout:2849: done (6 left) 2024-04-12T11:26:37.639 INFO:tasks.rados.rados.0.smithi029.stdout:2851: done (5 left) 2024-04-12T11:26:37.639 INFO:tasks.rados.rados.0.smithi029.stdout:2852: done (4 left) 2024-04-12T11:26:37.639 INFO:tasks.rados.rados.0.smithi029.stdout:2853: done (3 left) 2024-04-12T11:26:37.639 INFO:tasks.rados.rados.0.smithi029.stdout:2855: done (2 left) 2024-04-12T11:26:37.639 INFO:tasks.rados.rados.0.smithi029.stdout:2856: done (1 left) 2024-04-12T11:26:37.639 INFO:tasks.rados.rados.0.smithi029.stdout:2857: done (0 left) 2024-04-12T11:26:37.639 INFO:tasks.rados.rados.0.smithi029.stdout:2858: delete oid 12 current snap is 364 2024-04-12T11:26:37.644 INFO:tasks.rados.rados.0.smithi029.stdout:2858: done (0 left) 2024-04-12T11:26:37.645 INFO:tasks.rados.rados.0.smithi029.stdout:2859: write oid 33 current snap is 364 2024-04-12T11:26:37.645 INFO:tasks.rados.rados.0.smithi029.stdout:2859: seq_num 1067 ranges {733069=580103,1921581=579703,3218098=537473} 2024-04-12T11:26:37.651 INFO:tasks.rados.rados.0.smithi029.stdout:2859: writing smithi02979277-33 from 733069 to 1313172 tid 1 2024-04-12T11:26:37.658 INFO:tasks.rados.rados.0.smithi029.stdout:2859: writing smithi02979277-33 from 1921581 to 2501284 tid 2 2024-04-12T11:26:37.665 INFO:tasks.rados.rados.0.smithi029.stdout:2859: writing smithi02979277-33 from 3218098 to 3755571 tid 3 2024-04-12T11:26:37.665 INFO:tasks.rados.rados.0.smithi029.stdout:2860: read oid 21 snap -1 2024-04-12T11:26:37.665 INFO:tasks.rados.rados.0.smithi029.stdout:2860: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:26:37.665 INFO:tasks.rados.rados.0.smithi029.stdout:2861: snap_remove snap 363 2024-04-12T11:26:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:37 smithi087 ceph-mon[89014]: osdmap e792: 8 total, 8 up, 8 in 2024-04-12T11:26:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:37 smithi087 ceph-mon[89014]: osdmap e793: 8 total, 8 up, 8 in 2024-04-12T11:26:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:37 smithi029 ceph-mon[93650]: osdmap e792: 8 total, 8 up, 8 in 2024-04-12T11:26:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:37 smithi029 ceph-mon[93650]: osdmap e793: 8 total, 8 up, 8 in 2024-04-12T11:26:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:37 smithi029 ceph-mon[97787]: osdmap e792: 8 total, 8 up, 8 in 2024-04-12T11:26:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:37 smithi029 ceph-mon[97787]: osdmap e793: 8 total, 8 up, 8 in 2024-04-12T11:26:38.640 INFO:tasks.rados.rados.0.smithi029.stdout:2861: done (2 left) 2024-04-12T11:26:38.641 INFO:tasks.rados.rados.0.smithi029.stdout:2862: read oid 38 snap -1 2024-04-12T11:26:38.641 INFO:tasks.rados.rados.0.smithi029.stdout:2862: expect (ObjNum 864 snap 302 seq_num 864) 2024-04-12T11:26:38.641 INFO:tasks.rados.rados.0.smithi029.stdout:2863: write oid 27 current snap is 364 2024-04-12T11:26:38.641 INFO:tasks.rados.rados.0.smithi029.stdout:2859: finishing write tid 1 to smithi02979277-33 2024-04-12T11:26:38.641 INFO:tasks.rados.rados.0.smithi029.stdout:2863: seq_num 1068 ranges {749876=655352,1863710=723361,3012815=748745} 2024-04-12T11:26:38.648 INFO:tasks.rados.rados.0.smithi029.stdout:2863: writing smithi02979277-27 from 749876 to 1405228 tid 1 2024-04-12T11:26:38.655 INFO:tasks.rados.rados.0.smithi029.stdout:2863: writing smithi02979277-27 from 1863710 to 2587071 tid 2 2024-04-12T11:26:38.662 INFO:tasks.rados.rados.0.smithi029.stdout:2863: writing smithi02979277-27 from 3012815 to 3761560 tid 3 2024-04-12T11:26:38.662 INFO:tasks.rados.rados.0.smithi029.stdout:2864: read oid 49 snap 356 2024-04-12T11:26:38.662 INFO:tasks.rados.rados.0.smithi029.stdout:2864: expect deleted 2024-04-12T11:26:38.662 INFO:tasks.rados.rados.0.smithi029.stdout:2865: read oid 19 snap -1 2024-04-12T11:26:38.662 INFO:tasks.rados.rados.0.smithi029.stdout:2865: expect deleted 2024-04-12T11:26:38.662 INFO:tasks.rados.rados.0.smithi029.stdout:2866: rollback oid 35 current snap is 364 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 35 to 358 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:2859: finishing write tid 2 to smithi02979277-33 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:2859: finishing write tid 3 to smithi02979277-33 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:2859: finishing write tid 4 to smithi02979277-33 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:2859: finishing write tid 5 to smithi02979277-33 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:2859: oid 33 updating version 0 to 1149 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:2859: oid 33 updating version 1149 to 1152 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:2859: oid 33 version 1152 is already newer than 1150 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:2859: oid 33 version 1152 is already newer than 1151 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 1152 (ObjNum 1067 snap 364 seq_num 1067) dirty exists 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:2859: left oid 33 (ObjNum 1067 snap 364 seq_num 1067) 2024-04-12T11:26:38.663 INFO:tasks.rados.rados.0.smithi029.stdout:2860: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:26:38.825 INFO:tasks.rados.rados.0.smithi029.stdout:2862: expect (ObjNum 864 snap 302 seq_num 864) 2024-04-12T11:26:38.944 INFO:tasks.rados.rados.0.smithi029.stdout:2863: finishing write tid 1 to smithi02979277-27 2024-04-12T11:26:38.944 INFO:tasks.rados.rados.0.smithi029.stdout:2863: finishing write tid 2 to smithi02979277-27 2024-04-12T11:26:38.944 INFO:tasks.rados.rados.0.smithi029.stdout:2866: finishing rollback tid 0 to smithi02979277-35 2024-04-12T11:26:38.944 INFO:tasks.rados.rados.0.smithi029.stdout:2863: finishing write tid 3 to smithi02979277-27 2024-04-12T11:26:38.944 INFO:tasks.rados.rados.0.smithi029.stdout:2863: finishing write tid 4 to smithi02979277-27 2024-04-12T11:26:38.944 INFO:tasks.rados.rados.0.smithi029.stdout:2859: done (6 left) 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2860: done (5 left) 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2862: done (4 left) 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2864: done (3 left) 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2865: done (2 left) 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2867: write oid 23 current snap is 364 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2866: finishing rollback tid 1 to smithi02979277-35 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 1770 (ObjNum 49 snap 0 seq_num 1800738864) dirty dne 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2863: finishing write tid 5 to smithi02979277-27 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2863: oid 27 updating version 0 to 913 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2863: oid 27 version 913 is already newer than 912 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2863: oid 27 version 913 is already newer than 911 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2863: oid 27 updating version 913 to 914 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 914 (ObjNum 1068 snap 364 seq_num 1068) dirty exists 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2863: left oid 27 (ObjNum 1068 snap 364 seq_num 1068) 2024-04-12T11:26:38.945 INFO:tasks.rados.rados.0.smithi029.stdout:2867: seq_num 1069 ranges {766683=730601,1922192=583372,3156591=676370} 2024-04-12T11:26:38.952 INFO:tasks.rados.rados.0.smithi029.stdout:2867: writing smithi02979277-23 from 766683 to 1497284 tid 1 2024-04-12T11:26:38.957 INFO:tasks.rados.rados.0.smithi029.stdout:2867: writing smithi02979277-23 from 1922192 to 2505564 tid 2 2024-04-12T11:26:38.965 INFO:tasks.rados.rados.0.smithi029.stdout:2867: writing smithi02979277-23 from 3156591 to 3832961 tid 3 2024-04-12T11:26:38.965 INFO:tasks.rados.rados.0.smithi029.stdout:2863: done (2 left) 2024-04-12T11:26:38.965 INFO:tasks.rados.rados.0.smithi029.stdout:2866: done (1 left) 2024-04-12T11:26:38.965 INFO:tasks.rados.rados.0.smithi029.stdout:2868: read oid 22 snap -1 2024-04-12T11:26:38.965 INFO:tasks.rados.rados.0.smithi029.stdout:2868: expect (ObjNum 892 snap 312 seq_num 892) 2024-04-12T11:26:38.965 INFO:tasks.rados.rados.0.smithi029.stdout:2869: delete oid 29 current snap is 364 2024-04-12T11:26:38.965 INFO:tasks.rados.rados.0.smithi029.stdout:2867: finishing write tid 1 to smithi02979277-23 2024-04-12T11:26:38.965 INFO:tasks.rados.rados.0.smithi029.stdout:2867: finishing write tid 2 to smithi02979277-23 2024-04-12T11:26:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:38 smithi087 ceph-mon[89014]: pgmap v825: 89 pgs: 89 active+clean; 254 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-04-12T11:26:38.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:38 smithi087 ceph-mon[89014]: osdmap e794: 8 total, 8 up, 8 in 2024-04-12T11:26:38.968 INFO:tasks.rados.rados.0.smithi029.stdout:2869: done (2 left) 2024-04-12T11:26:38.968 INFO:tasks.rados.rados.0.smithi029.stdout:2870: rollback oid 45 current snap is 364 2024-04-12T11:26:38.968 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 45 to 356 2024-04-12T11:26:38.968 INFO:tasks.rados.rados.0.smithi029.stdout:2871: write oid 7 current snap is 364 2024-04-12T11:26:38.968 INFO:tasks.rados.rados.0.smithi029.stdout:2871: seq_num 1071 ranges {400297=481099,1639156=361141} 2024-04-12T11:26:38.972 INFO:tasks.rados.rados.0.smithi029.stdout:2871: writing smithi02979277-7 from 400297 to 881396 tid 1 2024-04-12T11:26:38.978 INFO:tasks.rados.rados.0.smithi029.stdout:2871: writing smithi02979277-7 from 1639156 to 2000297 tid 2 2024-04-12T11:26:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:2872: snap_remove snap 339 2024-04-12T11:26:39.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:38 smithi029 ceph-mon[93650]: pgmap v825: 89 pgs: 89 active+clean; 254 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-04-12T11:26:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:38 smithi029 ceph-mon[93650]: osdmap e794: 8 total, 8 up, 8 in 2024-04-12T11:26:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:38 smithi029 ceph-mon[97787]: pgmap v825: 89 pgs: 89 active+clean; 254 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-04-12T11:26:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:38 smithi029 ceph-mon[97787]: osdmap e794: 8 total, 8 up, 8 in 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2872: done (4 left) 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2873: delete oid 44 current snap is 364 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2870: finishing rollback tid 0 to smithi02979277-45 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2870: finishing rollback tid 1 to smithi02979277-45 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2870: finishing rollback tid 2 to smithi02979277-45 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1367 (ObjNum 962 snap 330 seq_num 962) dirty exists 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2867: finishing write tid 3 to smithi02979277-23 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2867: finishing write tid 4 to smithi02979277-23 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2867: finishing write tid 5 to smithi02979277-23 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2867: oid 23 updating version 0 to 1156 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2867: oid 23 updating version 1156 to 1158 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2867: oid 23 version 1158 is already newer than 1157 2024-04-12T11:26:39.688 INFO:tasks.rados.rados.0.smithi029.stdout:2867: oid 23 version 1158 is already newer than 1155 2024-04-12T11:26:39.689 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1158 (ObjNum 1069 snap 364 seq_num 1069) dirty exists 2024-04-12T11:26:39.689 INFO:tasks.rados.rados.0.smithi029.stdout:2867: left oid 23 (ObjNum 1069 snap 364 seq_num 1069) 2024-04-12T11:26:39.689 INFO:tasks.rados.rados.0.smithi029.stdout:2871: finishing write tid 1 to smithi02979277-7 2024-04-12T11:26:39.689 INFO:tasks.rados.rados.0.smithi029.stdout:2868: expect (ObjNum 892 snap 312 seq_num 892) 2024-04-12T11:26:39.891 INFO:tasks.rados.rados.0.smithi029.stdout:2871: finishing write tid 2 to smithi02979277-7 2024-04-12T11:26:39.891 INFO:tasks.rados.rados.0.smithi029.stdout:2871: finishing write tid 3 to smithi02979277-7 2024-04-12T11:26:39.891 INFO:tasks.rados.rados.0.smithi029.stdout:2871: finishing write tid 4 to smithi02979277-7 2024-04-12T11:26:39.891 INFO:tasks.rados.rados.0.smithi029.stdout:2871: oid 7 updating version 0 to 1370 2024-04-12T11:26:39.891 INFO:tasks.rados.rados.0.smithi029.stdout:2871: oid 7 version 1370 is already newer than 1368 2024-04-12T11:26:39.891 INFO:tasks.rados.rados.0.smithi029.stdout:2871: oid 7 version 1370 is already newer than 1369 2024-04-12T11:26:39.891 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1370 (ObjNum 1071 snap 364 seq_num 1071) dirty exists 2024-04-12T11:26:39.891 INFO:tasks.rados.rados.0.smithi029.stdout:2871: left oid 7 (ObjNum 1071 snap 364 seq_num 1071) 2024-04-12T11:26:39.891 INFO:tasks.rados.rados.0.smithi029.stdout:2867: done (4 left) 2024-04-12T11:26:39.892 INFO:tasks.rados.rados.0.smithi029.stdout:2868: done (3 left) 2024-04-12T11:26:39.892 INFO:tasks.rados.rados.0.smithi029.stdout:2870: done (2 left) 2024-04-12T11:26:39.892 INFO:tasks.rados.rados.0.smithi029.stdout:2871: done (1 left) 2024-04-12T11:26:39.892 INFO:tasks.rados.rados.0.smithi029.stdout:2873: done (0 left) 2024-04-12T11:26:39.892 INFO:tasks.rados.rados.0.smithi029.stdout:2874: snap_remove snap 359 2024-04-12T11:26:40.692 INFO:tasks.rados.rados.0.smithi029.stdout:2874: done (0 left) 2024-04-12T11:26:40.693 INFO:tasks.rados.rados.0.smithi029.stdout:2875: delete oid 43 current snap is 364 2024-04-12T11:26:40.698 INFO:tasks.rados.rados.0.smithi029.stdout:2875: done (0 left) 2024-04-12T11:26:40.698 INFO:tasks.rados.rados.0.smithi029.stdout:2876: write oid 30 current snap is 364 2024-04-12T11:26:40.698 INFO:tasks.rados.rados.0.smithi029.stdout:2876: seq_num 1074 ranges {450718=423199,1414602=636116} 2024-04-12T11:26:40.703 INFO:tasks.rados.rados.0.smithi029.stdout:2876: writing smithi02979277-30 from 450718 to 873917 tid 1 2024-04-12T11:26:40.710 INFO:tasks.rados.rados.0.smithi029.stdout:2876: writing smithi02979277-30 from 1414602 to 2050718 tid 2 2024-04-12T11:26:40.710 INFO:tasks.rados.rados.0.smithi029.stdout:2877: write oid 26 current snap is 364 2024-04-12T11:26:40.710 INFO:tasks.rados.rados.0.smithi029.stdout:2877: seq_num 1075 ranges {467525=498448,1473084=594379} 2024-04-12T11:26:40.716 INFO:tasks.rados.rados.0.smithi029.stdout:2877: writing smithi02979277-26 from 467525 to 965973 tid 1 2024-04-12T11:26:40.722 INFO:tasks.rados.rados.0.smithi029.stdout:2877: writing smithi02979277-26 from 1473084 to 2067463 tid 2 2024-04-12T11:26:40.722 INFO:tasks.rados.rados.0.smithi029.stdout:2878: snap_create 2024-04-12T11:26:40.723 INFO:tasks.rados.rados.0.smithi029.stdout:2876: finishing write tid 1 to smithi02979277-30 2024-04-12T11:26:40.723 INFO:tasks.rados.rados.0.smithi029.stdout:2876: finishing write tid 2 to smithi02979277-30 2024-04-12T11:26:40.723 INFO:tasks.rados.rados.0.smithi029.stdout:2876: finishing write tid 3 to smithi02979277-30 2024-04-12T11:26:40.723 INFO:tasks.rados.rados.0.smithi029.stdout:2876: finishing write tid 4 to smithi02979277-30 2024-04-12T11:26:40.723 INFO:tasks.rados.rados.0.smithi029.stdout:2876: oid 30 updating version 0 to 922 2024-04-12T11:26:40.723 INFO:tasks.rados.rados.0.smithi029.stdout:2876: oid 30 version 922 is already newer than 921 2024-04-12T11:26:40.723 INFO:tasks.rados.rados.0.smithi029.stdout:2876: oid 30 updating version 922 to 923 2024-04-12T11:26:40.723 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 923 (ObjNum 1074 snap 364 seq_num 1074) dirty exists 2024-04-12T11:26:40.723 INFO:tasks.rados.rados.0.smithi029.stdout:2876: left oid 30 (ObjNum 1074 snap 364 seq_num 1074) 2024-04-12T11:26:40.723 INFO:tasks.rados.rados.0.smithi029.stdout:2877: finishing write tid 1 to smithi02979277-26 2024-04-12T11:26:40.726 INFO:tasks.rados.rados.0.smithi029.stdout:2877: finishing write tid 2 to smithi02979277-26 2024-04-12T11:26:40.726 INFO:tasks.rados.rados.0.smithi029.stdout:2877: finishing write tid 3 to smithi02979277-26 2024-04-12T11:26:40.727 INFO:tasks.rados.rados.0.smithi029.stdout:2877: finishing write tid 4 to smithi02979277-26 2024-04-12T11:26:40.727 INFO:tasks.rados.rados.0.smithi029.stdout:2877: oid 26 updating version 0 to 1786 2024-04-12T11:26:40.727 INFO:tasks.rados.rados.0.smithi029.stdout:2877: oid 26 version 1786 is already newer than 1784 2024-04-12T11:26:40.727 INFO:tasks.rados.rados.0.smithi029.stdout:2877: oid 26 version 1786 is already newer than 1785 2024-04-12T11:26:40.727 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 1786 (ObjNum 1075 snap 364 seq_num 1075) dirty exists 2024-04-12T11:26:40.727 INFO:tasks.rados.rados.0.smithi029.stdout:2877: left oid 26 (ObjNum 1075 snap 364 seq_num 1075) 2024-04-12T11:26:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:40 smithi087 ceph-mon[89014]: pgmap v828: 89 pgs: 89 active+clean; 232 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-04-12T11:26:40.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:40 smithi087 ceph-mon[89014]: osdmap e795: 8 total, 8 up, 8 in 2024-04-12T11:26:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:40 smithi029 ceph-mon[93650]: pgmap v828: 89 pgs: 89 active+clean; 232 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-04-12T11:26:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:40 smithi029 ceph-mon[93650]: osdmap e795: 8 total, 8 up, 8 in 2024-04-12T11:26:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:40 smithi029 ceph-mon[97787]: pgmap v828: 89 pgs: 89 active+clean; 232 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-04-12T11:26:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:40 smithi029 ceph-mon[97787]: osdmap e795: 8 total, 8 up, 8 in 2024-04-12T11:26:41.698 INFO:tasks.rados.rados.0.smithi029.stdout:2876: done (2 left) 2024-04-12T11:26:41.699 INFO:tasks.rados.rados.0.smithi029.stdout:2877: done (1 left) 2024-04-12T11:26:41.699 INFO:tasks.rados.rados.0.smithi029.stdout:2878: done (0 left) 2024-04-12T11:26:41.699 INFO:tasks.rados.rados.0.smithi029.stdout:2879: delete oid 34 current snap is 365 2024-04-12T11:26:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2879: done (0 left) 2024-04-12T11:26:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2880: read oid 43 snap -1 2024-04-12T11:26:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2880: expect deleted 2024-04-12T11:26:41.703 INFO:tasks.rados.rados.0.smithi029.stdout:2881: delete oid 36 current snap is 365 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:2880: done (1 left) 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:2881: done (0 left) 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:2882: rollback oid 17 current snap is 365 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 356 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:2883: read oid 28 snap 355 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:2883: expect (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:2884: rollback oid 31 current snap is 365 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 31 to 364 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:2885: rollback oid 30 current snap is 365 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 354 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:2886: write oid 42 current snap is 365 2024-04-12T11:26:41.706 INFO:tasks.rados.rados.0.smithi029.stdout:2886: seq_num 1078 ranges {517946=724195,1932177=185769} 2024-04-12T11:26:41.713 INFO:tasks.rados.rados.0.smithi029.stdout:2886: writing smithi02979277-42 from 517946 to 1242141 tid 1 2024-04-12T11:26:41.718 INFO:tasks.rados.rados.0.smithi029.stdout:2886: writing smithi02979277-42 from 1932177 to 2117946 tid 2 2024-04-12T11:26:41.718 INFO:tasks.rados.rados.0.smithi029.stdout:2887: read oid 18 snap -1 2024-04-12T11:26:41.719 INFO:tasks.rados.rados.0.smithi029.stdout:2887: expect (ObjNum 861 snap 296 seq_num 861) 2024-04-12T11:26:41.719 INFO:tasks.rados.rados.0.smithi029.stdout:2888: write oid 38 current snap is 365 2024-04-12T11:26:41.719 INFO:tasks.rados.rados.0.smithi029.stdout:2882: finishing rollback tid 0 to smithi02979277-17 2024-04-12T11:26:41.719 INFO:tasks.rados.rados.0.smithi029.stdout:2888: seq_num 1079 ranges {534753=799444,1990659=144094} 2024-04-12T11:26:41.725 INFO:tasks.rados.rados.0.smithi029.stdout:2888: writing smithi02979277-38 from 534753 to 1334197 tid 1 2024-04-12T11:26:41.731 INFO:tasks.rados.rados.0.smithi029.stdout:2888: writing smithi02979277-38 from 1990659 to 2134753 tid 2 2024-04-12T11:26:41.731 INFO:tasks.rados.rados.0.smithi029.stdout:2889: delete oid 27 current snap is 365 2024-04-12T11:26:41.731 INFO:tasks.rados.rados.0.smithi029.stdout:2882: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:26:41.731 INFO:tasks.rados.rados.0.smithi029.stdout:2882: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:26:41.731 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1443 (ObjNum 1019 snap 352 seq_num 1019) dirty exists 2024-04-12T11:26:41.731 INFO:tasks.rados.rados.0.smithi029.stdout:2885: finishing rollback tid 0 to smithi02979277-30 2024-04-12T11:26:41.731 INFO:tasks.rados.rados.0.smithi029.stdout:2885: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:26:41.731 INFO:tasks.rados.rados.0.smithi029.stdout:2885: finishing rollback tid 2 to smithi02979277-30 2024-04-12T11:26:41.731 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 927 (ObjNum 1016 snap 350 seq_num 1016) dirty exists 2024-04-12T11:26:41.731 INFO:tasks.rados.rados.0.smithi029.stdout:2883: expect (ObjNum 827 snap 289 seq_num 827) 2024-04-12T11:26:41.859 INFO:tasks.rados.rados.0.smithi029.stdout:2884: finishing rollback tid 1 to smithi02979277-31 2024-04-12T11:26:41.859 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 0 (ObjNum 81 snap 0 seq_num 862899297) dirty dne 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2886: finishing write tid 1 to smithi02979277-42 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2886: finishing write tid 2 to smithi02979277-42 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2882: done (7 left) 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2883: done (6 left) 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2884: done (5 left) 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2885: done (4 left) 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2889: done (3 left) 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2890: write oid 46 current snap is 365 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2886: finishing write tid 3 to smithi02979277-42 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2886: finishing write tid 4 to smithi02979277-42 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2886: oid 42 updating version 0 to 348 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2886: oid 42 version 348 is already newer than 346 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2886: oid 42 version 348 is already newer than 347 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 348 (ObjNum 1078 snap 365 seq_num 1078) dirty exists 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2886: left oid 42 (ObjNum 1078 snap 365 seq_num 1078) 2024-04-12T11:26:41.860 INFO:tasks.rados.rados.0.smithi029.stdout:2887: expect (ObjNum 861 snap 296 seq_num 861) 2024-04-12T11:26:41.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:41 smithi087 ceph-mon[89014]: osdmap e796: 8 total, 8 up, 8 in 2024-04-12T11:26:41.978 INFO:tasks.rados.rados.0.smithi029.stdout:2888: finishing write tid 1 to smithi02979277-38 2024-04-12T11:26:41.978 INFO:tasks.rados.rados.0.smithi029.stdout:2888: finishing write tid 2 to smithi02979277-38 2024-04-12T11:26:41.978 INFO:tasks.rados.rados.0.smithi029.stdout:2888: finishing write tid 3 to smithi02979277-38 2024-04-12T11:26:41.978 INFO:tasks.rados.rados.0.smithi029.stdout:2888: finishing write tid 4 to smithi02979277-38 2024-04-12T11:26:41.978 INFO:tasks.rados.rados.0.smithi029.stdout:2888: oid 38 updating version 0 to 849 2024-04-12T11:26:41.978 INFO:tasks.rados.rados.0.smithi029.stdout:2888: oid 38 version 849 is already newer than 848 2024-04-12T11:26:41.979 INFO:tasks.rados.rados.0.smithi029.stdout:2888: oid 38 version 849 is already newer than 847 2024-04-12T11:26:41.979 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 849 (ObjNum 1079 snap 365 seq_num 1079) dirty exists 2024-04-12T11:26:41.979 INFO:tasks.rados.rados.0.smithi029.stdout:2888: left oid 38 (ObjNum 1079 snap 365 seq_num 1079) 2024-04-12T11:26:41.979 INFO:tasks.rados.rados.0.smithi029.stdout:2890: seq_num 1081 ranges {568367=666295,1707623=460744} 2024-04-12T11:26:41.985 INFO:tasks.rados.rados.0.smithi029.stdout:2890: writing smithi02979277-46 from 568367 to 1234662 tid 1 2024-04-12T11:26:41.990 INFO:tasks.rados.rados.0.smithi029.stdout:2890: writing smithi02979277-46 from 1707623 to 2168367 tid 2 2024-04-12T11:26:41.991 INFO:tasks.rados.rados.0.smithi029.stdout:2886: done (3 left) 2024-04-12T11:26:41.991 INFO:tasks.rados.rados.0.smithi029.stdout:2887: done (2 left) 2024-04-12T11:26:41.991 INFO:tasks.rados.rados.0.smithi029.stdout:2888: done (1 left) 2024-04-12T11:26:41.991 INFO:tasks.rados.rados.0.smithi029.stdout:2891: read oid 34 snap -1 2024-04-12T11:26:41.991 INFO:tasks.rados.rados.0.smithi029.stdout:2891: expect deleted 2024-04-12T11:26:41.991 INFO:tasks.rados.rados.0.smithi029.stdout:2892: write oid 37 current snap is 365 2024-04-12T11:26:41.991 INFO:tasks.rados.rados.0.smithi029.stdout:2890: finishing write tid 1 to smithi02979277-46 2024-04-12T11:26:41.991 INFO:tasks.rados.rados.0.smithi029.stdout:2892: seq_num 1082 ranges {585174=741544,1766105=419069} 2024-04-12T11:26:41.997 INFO:tasks.rados.rados.0.smithi029.stdout:2892: writing smithi02979277-37 from 585174 to 1326718 tid 1 2024-04-12T11:26:42.002 INFO:tasks.rados.rados.0.smithi029.stdout:2892: writing smithi02979277-37 from 1766105 to 2185174 tid 2 2024-04-12T11:26:42.002 INFO:tasks.rados.rados.0.smithi029.stdout:2893: snap_remove snap 364 2024-04-12T11:26:42.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:41 smithi029 ceph-mon[93650]: osdmap e796: 8 total, 8 up, 8 in 2024-04-12T11:26:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:41 smithi029 ceph-mon[97787]: osdmap e796: 8 total, 8 up, 8 in 2024-04-12T11:26:42.641 INFO:tasks.rados.rados.0.smithi029.stdout:2893: done (3 left) 2024-04-12T11:26:42.641 INFO:tasks.rados.rados.0.smithi029.stdout:2894: snap_remove snap 358 2024-04-12T11:26:42.696 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:26:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:26:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:26:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:42 smithi087 ceph-mon[89014]: pgmap v831: 89 pgs: 89 active+clean; 230 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-04-12T11:26:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:42 smithi087 ceph-mon[89014]: osdmap e797: 8 total, 8 up, 8 in 2024-04-12T11:26:42.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:42 smithi087 ceph-mon[89014]: osdmap e798: 8 total, 8 up, 8 in 2024-04-12T11:26:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:42 smithi029 ceph-mon[93650]: pgmap v831: 89 pgs: 89 active+clean; 230 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-04-12T11:26:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:42 smithi029 ceph-mon[93650]: osdmap e797: 8 total, 8 up, 8 in 2024-04-12T11:26:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:42 smithi029 ceph-mon[93650]: osdmap e798: 8 total, 8 up, 8 in 2024-04-12T11:26:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:42 smithi029 ceph-mon[97787]: pgmap v831: 89 pgs: 89 active+clean; 230 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-04-12T11:26:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:42 smithi029 ceph-mon[97787]: osdmap e797: 8 total, 8 up, 8 in 2024-04-12T11:26:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:42 smithi029 ceph-mon[97787]: osdmap e798: 8 total, 8 up, 8 in 2024-04-12T11:26:43.642 INFO:tasks.rados.rados.0.smithi029.stdout:2894: done (3 left) 2024-04-12T11:26:43.642 INFO:tasks.rados.rados.0.smithi029.stdout:2895: delete oid 13 current snap is 365 2024-04-12T11:26:43.642 INFO:tasks.rados.rados.0.smithi029.stdout:2890: finishing write tid 2 to smithi02979277-46 2024-04-12T11:26:43.642 INFO:tasks.rados.rados.0.smithi029.stdout:2890: finishing write tid 3 to smithi02979277-46 2024-04-12T11:26:43.642 INFO:tasks.rados.rados.0.smithi029.stdout:2890: finishing write tid 4 to smithi02979277-46 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2890: oid 46 updating version 0 to 850 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2890: oid 46 updating version 850 to 852 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2890: oid 46 version 852 is already newer than 851 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 852 (ObjNum 1081 snap 365 seq_num 1081) dirty exists 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2890: left oid 46 (ObjNum 1081 snap 365 seq_num 1081) 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2892: finishing write tid 1 to smithi02979277-37 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2892: finishing write tid 2 to smithi02979277-37 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2892: finishing write tid 3 to smithi02979277-37 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2892: finishing write tid 4 to smithi02979277-37 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2892: oid 37 updating version 0 to 1011 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2892: oid 37 version 1011 is already newer than 1009 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2892: oid 37 version 1011 is already newer than 1010 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 1011 (ObjNum 1082 snap 365 seq_num 1082) dirty exists 2024-04-12T11:26:43.643 INFO:tasks.rados.rados.0.smithi029.stdout:2892: left oid 37 (ObjNum 1082 snap 365 seq_num 1082) 2024-04-12T11:26:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2890: done (3 left) 2024-04-12T11:26:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2891: done (2 left) 2024-04-12T11:26:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2892: done (1 left) 2024-04-12T11:26:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2895: done (0 left) 2024-04-12T11:26:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2896: read oid 32 snap -1 2024-04-12T11:26:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2896: expect (ObjNum 1043 snap 356 seq_num 1043) 2024-04-12T11:26:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2897: read oid 12 snap -1 2024-04-12T11:26:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2897: expect deleted 2024-04-12T11:26:43.650 INFO:tasks.rados.rados.0.smithi029.stdout:2898: read oid 39 snap -1 2024-04-12T11:26:43.651 INFO:tasks.rados.rados.0.smithi029.stdout:2898: expect (ObjNum 1049 snap 357 seq_num 1049) 2024-04-12T11:26:43.651 INFO:tasks.rados.rados.0.smithi029.stdout:2899: write oid 50 current snap is 365 2024-04-12T11:26:43.651 INFO:tasks.rados.rados.0.smithi029.stdout:2899: seq_num 1084 ranges {618788=492042,1883069=335719} 2024-04-12T11:26:43.656 INFO:tasks.rados.rados.0.smithi029.stdout:2899: writing smithi02979277-50 from 618788 to 1110830 tid 1 2024-04-12T11:26:43.663 INFO:tasks.rados.rados.0.smithi029.stdout:2899: writing smithi02979277-50 from 1883069 to 2218788 tid 2 2024-04-12T11:26:43.663 INFO:tasks.rados.rados.0.smithi029.stdout:2900: delete oid 11 current snap is 365 2024-04-12T11:26:43.663 INFO:tasks.rados.rados.0.smithi029.stdout:2896: expect (ObjNum 1043 snap 356 seq_num 1043) 2024-04-12T11:26:43.869 INFO:tasks.rados.rados.0.smithi029.stdout:2898: expect (ObjNum 1049 snap 357 seq_num 1049) 2024-04-12T11:26:44.118 INFO:tasks.rados.rados.0.smithi029.stdout:2899: finishing write tid 1 to smithi02979277-50 2024-04-12T11:26:44.118 INFO:tasks.rados.rados.0.smithi029.stdout:2899: finishing write tid 2 to smithi02979277-50 2024-04-12T11:26:44.118 INFO:tasks.rados.rados.0.smithi029.stdout:2899: finishing write tid 3 to smithi02979277-50 2024-04-12T11:26:44.118 INFO:tasks.rados.rados.0.smithi029.stdout:2899: finishing write tid 4 to smithi02979277-50 2024-04-12T11:26:44.118 INFO:tasks.rados.rados.0.smithi029.stdout:2899: oid 50 updating version 0 to 1170 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:2899: oid 50 updating version 1170 to 1171 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:2899: oid 50 version 1171 is already newer than 1169 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1171 (ObjNum 1084 snap 365 seq_num 1084) dirty exists 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:2899: left oid 50 (ObjNum 1084 snap 365 seq_num 1084) 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:2896: done (4 left) 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:2897: done (3 left) 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:2898: done (2 left) 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:2899: done (1 left) 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:2900: done (0 left) 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:2901: write oid 13 current snap is 365 2024-04-12T11:26:44.119 INFO:tasks.rados.rados.0.smithi029.stdout:2901: seq_num 1086 ranges {652402=642540,1883680=368722} 2024-04-12T11:26:44.125 INFO:tasks.rados.rados.0.smithi029.stdout:2901: writing smithi02979277-13 from 652402 to 1294942 tid 1 2024-04-12T11:26:44.130 INFO:tasks.rados.rados.0.smithi029.stdout:2901: writing smithi02979277-13 from 1883680 to 2252402 tid 2 2024-04-12T11:26:44.130 INFO:tasks.rados.rados.0.smithi029.stdout:2902: write oid 41 current snap is 365 2024-04-12T11:26:44.130 INFO:tasks.rados.rados.0.smithi029.stdout:2901: finishing write tid 1 to smithi02979277-13 2024-04-12T11:26:44.130 INFO:tasks.rados.rados.0.smithi029.stdout:2902: seq_num 1087 ranges {669209=717789,1942162=327047} 2024-04-12T11:26:44.138 INFO:tasks.rados.rados.0.smithi029.stdout:2902: writing smithi02979277-41 from 669209 to 1386998 tid 1 2024-04-12T11:26:44.143 INFO:tasks.rados.rados.0.smithi029.stdout:2902: writing smithi02979277-41 from 1942162 to 2269209 tid 2 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2903: rollback oid 12 current snap is 365 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 354 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2901: finishing write tid 2 to smithi02979277-13 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2901: finishing write tid 3 to smithi02979277-13 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2901: finishing write tid 4 to smithi02979277-13 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2901: oid 13 updating version 0 to 1454 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2901: oid 13 updating version 1454 to 1455 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2901: oid 13 version 1455 is already newer than 1453 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1455 (ObjNum 1086 snap 365 seq_num 1086) dirty exists 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2901: left oid 13 (ObjNum 1086 snap 365 seq_num 1086) 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2902: finishing write tid 1 to smithi02979277-41 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2901: done (2 left) 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2904: read oid 37 snap -1 2024-04-12T11:26:44.144 INFO:tasks.rados.rados.0.smithi029.stdout:2904: expect (ObjNum 1082 snap 365 seq_num 1082) 2024-04-12T11:26:44.145 INFO:tasks.rados.rados.0.smithi029.stdout:2905: rollback oid 15 current snap is 365 2024-04-12T11:26:44.145 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 15 to 354 2024-04-12T11:26:44.145 INFO:tasks.rados.rados.0.smithi029.stdout:2906: write oid 39 current snap is 365 2024-04-12T11:26:44.145 INFO:tasks.rados.rados.0.smithi029.stdout:2906: seq_num 1088 ranges {686016=509391,1716997=569019} 2024-04-12T11:26:44.149 INFO:tasks.rados.rados.0.smithi029.stdout:2906: writing smithi02979277-39 from 686016 to 1195407 tid 1 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:2906: writing smithi02979277-39 from 1716997 to 2286016 tid 2 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:2907: rollback oid 24 current snap is 365 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 24 to 356 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:2905: finishing rollback tid 1 to smithi02979277-15 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:2905: done (5 left) 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:2908: write oid 43 current snap is 365 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:2903: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1361 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:2902: finishing write tid 2 to smithi02979277-41 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:2902: finishing write tid 3 to smithi02979277-41 2024-04-12T11:26:44.156 INFO:tasks.rados.rados.0.smithi029.stdout:2902: finishing write tid 4 to smithi02979277-41 2024-04-12T11:26:44.157 INFO:tasks.rados.rados.0.smithi029.stdout:2902: oid 41 updating version 0 to 1391 2024-04-12T11:26:44.157 INFO:tasks.rados.rados.0.smithi029.stdout:2902: oid 41 version 1391 is already newer than 1389 2024-04-12T11:26:44.157 INFO:tasks.rados.rados.0.smithi029.stdout:2902: oid 41 version 1391 is already newer than 1390 2024-04-12T11:26:44.157 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1391 (ObjNum 1087 snap 365 seq_num 1087) dirty exists 2024-04-12T11:26:44.157 INFO:tasks.rados.rados.0.smithi029.stdout:2902: left oid 41 (ObjNum 1087 snap 365 seq_num 1087) 2024-04-12T11:26:44.157 INFO:tasks.rados.rados.0.smithi029.stdout:2904: expect (ObjNum 1082 snap 365 seq_num 1082) 2024-04-12T11:26:44.236 INFO:tasks.rados.rados.0.smithi029.stdout:2906: finishing write tid 1 to smithi02979277-39 2024-04-12T11:26:44.236 INFO:tasks.rados.rados.0.smithi029.stdout:2906: finishing write tid 2 to smithi02979277-39 2024-04-12T11:26:44.236 INFO:tasks.rados.rados.0.smithi029.stdout:2906: finishing write tid 3 to smithi02979277-39 2024-04-12T11:26:44.236 INFO:tasks.rados.rados.0.smithi029.stdout:2906: finishing write tid 4 to smithi02979277-39 2024-04-12T11:26:44.236 INFO:tasks.rados.rados.0.smithi029.stdout:2906: oid 39 updating version 0 to 395 2024-04-12T11:26:44.236 INFO:tasks.rados.rados.0.smithi029.stdout:2906: oid 39 updating version 395 to 397 2024-04-12T11:26:44.236 INFO:tasks.rados.rados.0.smithi029.stdout:2906: oid 39 version 397 is already newer than 396 2024-04-12T11:26:44.236 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 397 (ObjNum 1088 snap 365 seq_num 1088) dirty exists 2024-04-12T11:26:44.236 INFO:tasks.rados.rados.0.smithi029.stdout:2906: left oid 39 (ObjNum 1088 snap 365 seq_num 1088) 2024-04-12T11:26:44.237 INFO:tasks.rados.rados.0.smithi029.stdout:2907: finishing rollback tid 0 to smithi02979277-24 2024-04-12T11:26:44.237 INFO:tasks.rados.rados.0.smithi029.stdout:2907: finishing rollback tid 1 to smithi02979277-24 2024-04-12T11:26:44.237 INFO:tasks.rados.rados.0.smithi029.stdout:2907: finishing rollback tid 2 to smithi02979277-24 2024-04-12T11:26:44.237 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1798 (ObjNum 1017 snap 350 seq_num 1017) dirty exists 2024-04-12T11:26:44.237 INFO:tasks.rados.rados.0.smithi029.stdout:2908: seq_num 1089 ranges {702823=584640,2059126=243697} 2024-04-12T11:26:44.243 INFO:tasks.rados.rados.0.smithi029.stdout:2908: writing smithi02979277-43 from 702823 to 1287463 tid 1 2024-04-12T11:26:44.249 INFO:tasks.rados.rados.0.smithi029.stdout:2908: writing smithi02979277-43 from 2059126 to 2302823 tid 2 2024-04-12T11:26:44.249 INFO:tasks.rados.rados.0.smithi029.stdout:2902: done (5 left) 2024-04-12T11:26:44.249 INFO:tasks.rados.rados.0.smithi029.stdout:2903: done (4 left) 2024-04-12T11:26:44.249 INFO:tasks.rados.rados.0.smithi029.stdout:2904: done (3 left) 2024-04-12T11:26:44.249 INFO:tasks.rados.rados.0.smithi029.stdout:2906: done (2 left) 2024-04-12T11:26:44.249 INFO:tasks.rados.rados.0.smithi029.stdout:2907: done (1 left) 2024-04-12T11:26:44.249 INFO:tasks.rados.rados.0.smithi029.stdout:2909: write oid 44 current snap is 365 2024-04-12T11:26:44.249 INFO:tasks.rados.rados.0.smithi029.stdout:2908: finishing write tid 1 to smithi02979277-43 2024-04-12T11:26:44.249 INFO:tasks.rados.rados.0.smithi029.stdout:2909: seq_num 1090 ranges {719630=659889,2117608=202022} 2024-04-12T11:26:44.256 INFO:tasks.rados.rados.0.smithi029.stdout:2909: writing smithi02979277-44 from 719630 to 1379519 tid 1 2024-04-12T11:26:44.261 INFO:tasks.rados.rados.0.smithi029.stdout:2909: writing smithi02979277-44 from 2117608 to 2319630 tid 2 2024-04-12T11:26:44.261 INFO:tasks.rados.rados.0.smithi029.stdout:2910: rollback oid 24 current snap is 365 2024-04-12T11:26:44.261 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 24 to 357 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:2908: finishing write tid 2 to smithi02979277-43 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:2908: finishing write tid 3 to smithi02979277-43 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:2908: finishing write tid 4 to smithi02979277-43 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:2908: oid 43 updating version 0 to 1334 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:2908: oid 43 updating version 1334 to 1335 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:2908: oid 43 updating version 1335 to 1336 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1336 (ObjNum 1089 snap 365 seq_num 1089) dirty exists 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:2908: left oid 43 (ObjNum 1089 snap 365 seq_num 1089) 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:2909: finishing write tid 1 to smithi02979277-44 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:2908: done (2 left) 2024-04-12T11:26:44.262 INFO:tasks.rados.rados.0.smithi029.stdout:2911: delete oid 5 current snap is 365 2024-04-12T11:26:44.265 INFO:tasks.rados.rados.0.smithi029.stdout:2911: done (2 left) 2024-04-12T11:26:44.265 INFO:tasks.rados.rados.0.smithi029.stdout:2912: rollback oid 8 current snap is 365 2024-04-12T11:26:44.265 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 354 2024-04-12T11:26:44.265 INFO:tasks.rados.rados.0.smithi029.stdout:2913: read oid 32 snap -1 2024-04-12T11:26:44.265 INFO:tasks.rados.rados.0.smithi029.stdout:2913: expect (ObjNum 1043 snap 356 seq_num 1043) 2024-04-12T11:26:44.265 INFO:tasks.rados.rados.0.smithi029.stdout:2914: delete oid 12 current snap is 365 2024-04-12T11:26:44.265 INFO:tasks.rados.rados.0.smithi029.stdout:2909: finishing write tid 2 to smithi02979277-44 2024-04-12T11:26:44.265 INFO:tasks.rados.rados.0.smithi029.stdout:2910: finishing rollback tid 0 to smithi02979277-24 2024-04-12T11:26:44.265 INFO:tasks.rados.rados.0.smithi029.stdout:2909: finishing write tid 3 to smithi02979277-44 2024-04-12T11:26:44.266 INFO:tasks.rados.rados.0.smithi029.stdout:2909: finishing write tid 4 to smithi02979277-44 2024-04-12T11:26:44.266 INFO:tasks.rados.rados.0.smithi029.stdout:2909: oid 44 updating version 0 to 1394 2024-04-12T11:26:44.266 INFO:tasks.rados.rados.0.smithi029.stdout:2909: oid 44 version 1394 is already newer than 1392 2024-04-12T11:26:44.266 INFO:tasks.rados.rados.0.smithi029.stdout:2909: oid 44 version 1394 is already newer than 1393 2024-04-12T11:26:44.266 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1394 (ObjNum 1090 snap 365 seq_num 1090) dirty exists 2024-04-12T11:26:44.266 INFO:tasks.rados.rados.0.smithi029.stdout:2909: left oid 44 (ObjNum 1090 snap 365 seq_num 1090) 2024-04-12T11:26:44.266 INFO:tasks.rados.rados.0.smithi029.stdout:2910: finishing rollback tid 1 to smithi02979277-24 2024-04-12T11:26:44.267 INFO:tasks.rados.rados.0.smithi029.stdout:2910: finishing rollback tid 2 to smithi02979277-24 2024-04-12T11:26:44.267 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1801 (ObjNum 1051 snap 357 seq_num 1051) dirty exists 2024-04-12T11:26:44.267 INFO:tasks.rados.rados.0.smithi029.stdout:2912: finishing rollback tid 0 to smithi02979277-8 2024-04-12T11:26:44.267 INFO:tasks.rados.rados.0.smithi029.stdout:2913: expect (ObjNum 1043 snap 356 seq_num 1043) 2024-04-12T11:26:44.474 INFO:tasks.rados.rados.0.smithi029.stdout:2912: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:26:44.474 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 326 (ObjNum 129 snap 0 seq_num 1801129776) dirty dne 2024-04-12T11:26:44.474 INFO:tasks.rados.rados.0.smithi029.stdout:2909: done (4 left) 2024-04-12T11:26:44.474 INFO:tasks.rados.rados.0.smithi029.stdout:2910: done (3 left) 2024-04-12T11:26:44.474 INFO:tasks.rados.rados.0.smithi029.stdout:2912: done (2 left) 2024-04-12T11:26:44.474 INFO:tasks.rados.rados.0.smithi029.stdout:2913: done (1 left) 2024-04-12T11:26:44.474 INFO:tasks.rados.rados.0.smithi029.stdout:2914: done (0 left) 2024-04-12T11:26:44.474 INFO:tasks.rados.rados.0.smithi029.stdout:2915: snap_create 2024-04-12T11:26:44.701 INFO:tasks.rados.rados.0.smithi029.stdout:2915: done (0 left) 2024-04-12T11:26:44.701 INFO:tasks.rados.rados.0.smithi029.stdout:2916: write oid 3 current snap is 366 2024-04-12T11:26:44.701 INFO:tasks.rados.rados.0.smithi029.stdout:2916: seq_num 1093 ranges {770051=485636,1776701=511047} 2024-04-12T11:26:44.708 INFO:tasks.rados.rados.0.smithi029.stdout:2916: writing smithi02979277-3 from 770051 to 1255687 tid 1 2024-04-12T11:26:44.714 INFO:tasks.rados.rados.0.smithi029.stdout:2916: writing smithi02979277-3 from 1776701 to 2287748 tid 2 2024-04-12T11:26:44.714 INFO:tasks.rados.rados.0.smithi029.stdout:2917: snap_create 2024-04-12T11:26:44.714 INFO:tasks.rados.rados.0.smithi029.stdout:2916: finishing write tid 1 to smithi02979277-3 2024-04-12T11:26:44.717 INFO:tasks.rados.rados.0.smithi029.stdout:2916: finishing write tid 2 to smithi02979277-3 2024-04-12T11:26:44.717 INFO:tasks.rados.rados.0.smithi029.stdout:2916: finishing write tid 3 to smithi02979277-3 2024-04-12T11:26:44.718 INFO:tasks.rados.rados.0.smithi029.stdout:2916: finishing write tid 4 to smithi02979277-3 2024-04-12T11:26:44.718 INFO:tasks.rados.rados.0.smithi029.stdout:2916: oid 3 updating version 0 to 1805 2024-04-12T11:26:44.718 INFO:tasks.rados.rados.0.smithi029.stdout:2916: oid 3 version 1805 is already newer than 1804 2024-04-12T11:26:44.718 INFO:tasks.rados.rados.0.smithi029.stdout:2916: oid 3 version 1805 is already newer than 1803 2024-04-12T11:26:44.718 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1805 (ObjNum 1093 snap 366 seq_num 1093) dirty exists 2024-04-12T11:26:44.718 INFO:tasks.rados.rados.0.smithi029.stdout:2916: left oid 3 (ObjNum 1093 snap 366 seq_num 1093) 2024-04-12T11:26:44.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:44 smithi029 ceph-mon[97787]: pgmap v834: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 242 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-04-12T11:26:44.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:44 smithi029 ceph-mon[97787]: osdmap e799: 8 total, 8 up, 8 in 2024-04-12T11:26:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:44 smithi029 ceph-mon[93650]: pgmap v834: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 242 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-04-12T11:26:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:44 smithi029 ceph-mon[93650]: osdmap e799: 8 total, 8 up, 8 in 2024-04-12T11:26:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:44 smithi087 ceph-mon[89014]: pgmap v834: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 242 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-04-12T11:26:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:44 smithi087 ceph-mon[89014]: osdmap e799: 8 total, 8 up, 8 in 2024-04-12T11:26:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:26:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:26:44] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.43.0" 2024-04-12T11:26:45.710 INFO:tasks.rados.rados.0.smithi029.stdout:2916: done (1 left) 2024-04-12T11:26:45.711 INFO:tasks.rados.rados.0.smithi029.stdout:2917: done (0 left) 2024-04-12T11:26:45.711 INFO:tasks.rados.rados.0.smithi029.stdout:2918: delete oid 47 current snap is 367 2024-04-12T11:26:45.717 INFO:tasks.rados.rados.0.smithi029.stdout:2918: done (0 left) 2024-04-12T11:26:45.717 INFO:tasks.rados.rados.0.smithi029.stdout:2919: rollback oid 10 current snap is 367 2024-04-12T11:26:45.717 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 10 to 354 2024-04-12T11:26:45.717 INFO:tasks.rados.rados.0.smithi029.stdout:2920: snap_create 2024-04-12T11:26:45.721 INFO:tasks.rados.rados.0.smithi029.stdout:2919: finishing rollback tid 0 to smithi02979277-10 2024-04-12T11:26:45.722 INFO:tasks.rados.rados.0.smithi029.stdout:2919: finishing rollback tid 1 to smithi02979277-10 2024-04-12T11:26:45.722 INFO:tasks.rados.rados.0.smithi029.stdout:2919: finishing rollback tid 2 to smithi02979277-10 2024-04-12T11:26:45.722 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 1080 (ObjNum 975 snap 340 seq_num 975) dirty exists 2024-04-12T11:26:45.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:45 smithi087 ceph-mon[89014]: osdmap e800: 8 total, 8 up, 8 in 2024-04-12T11:26:46.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:45 smithi029 ceph-mon[93650]: osdmap e800: 8 total, 8 up, 8 in 2024-04-12T11:26:46.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:45 smithi029 ceph-mon[97787]: osdmap e800: 8 total, 8 up, 8 in 2024-04-12T11:26:46.713 INFO:tasks.rados.rados.0.smithi029.stdout:2919: done (1 left) 2024-04-12T11:26:46.713 INFO:tasks.rados.rados.0.smithi029.stdout:2920: done (0 left) 2024-04-12T11:26:46.713 INFO:tasks.rados.rados.0.smithi029.stdout:2921: read oid 35 snap -1 2024-04-12T11:26:46.714 INFO:tasks.rados.rados.0.smithi029.stdout:2921: expect deleted 2024-04-12T11:26:46.714 INFO:tasks.rados.rados.0.smithi029.stdout:2922: read oid 48 snap 356 2024-04-12T11:26:46.714 INFO:tasks.rados.rados.0.smithi029.stdout:2922: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:26:46.714 INFO:tasks.rados.rados.0.smithi029.stdout:2923: write oid 46 current snap is 368 2024-04-12T11:26:46.714 INFO:tasks.rados.rados.0.smithi029.stdout:2923: seq_num 1095 ranges {403665=752487,1610018=514716} 2024-04-12T11:26:46.720 INFO:tasks.rados.rados.0.smithi029.stdout:2923: writing smithi02979277-46 from 403665 to 1156152 tid 1 2024-04-12T11:26:46.726 INFO:tasks.rados.rados.0.smithi029.stdout:2923: writing smithi02979277-46 from 1610018 to 2124734 tid 2 2024-04-12T11:26:46.726 INFO:tasks.rados.rados.0.smithi029.stdout:2924: read oid 4 snap 357 2024-04-12T11:26:46.726 INFO:tasks.rados.rados.0.smithi029.stdout:2924: expect (ObjNum 1047 snap 357 seq_num 1047) 2024-04-12T11:26:46.726 INFO:tasks.rados.rados.0.smithi029.stdout:2925: write oid 22 current snap is 368 2024-04-12T11:26:46.726 INFO:tasks.rados.rados.0.smithi029.stdout:2925: seq_num 1096 ranges {420472=427736,1268500=658374,2347953=72519} 2024-04-12T11:26:46.730 INFO:tasks.rados.rados.0.smithi029.stdout:2925: writing smithi02979277-22 from 420472 to 848208 tid 1 2024-04-12T11:26:46.736 INFO:tasks.rados.rados.0.smithi029.stdout:2925: writing smithi02979277-22 from 1268500 to 1926874 tid 2 2024-04-12T11:26:46.739 INFO:tasks.rados.rados.0.smithi029.stdout:2925: writing smithi02979277-22 from 2347953 to 2420472 tid 3 2024-04-12T11:26:46.739 INFO:tasks.rados.rados.0.smithi029.stdout:2926: write oid 49 current snap is 368 2024-04-12T11:26:46.739 INFO:tasks.rados.rados.0.smithi029.stdout:2926: seq_num 1097 ranges {437279=502985,1610629=518385} 2024-04-12T11:26:46.744 INFO:tasks.rados.rados.0.smithi029.stdout:2926: writing smithi02979277-49 from 437279 to 940264 tid 1 2024-04-12T11:26:46.751 INFO:tasks.rados.rados.0.smithi029.stdout:2926: writing smithi02979277-49 from 1610629 to 2129014 tid 2 2024-04-12T11:26:46.751 INFO:tasks.rados.rados.0.smithi029.stdout:2927: read oid 44 snap -1 2024-04-12T11:26:46.751 INFO:tasks.rados.rados.0.smithi029.stdout:2927: expect (ObjNum 1090 snap 365 seq_num 1090) 2024-04-12T11:26:46.751 INFO:tasks.rados.rados.0.smithi029.stdout:2928: snap_remove snap 354 2024-04-12T11:26:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:46 smithi087 ceph-mon[89014]: pgmap v837: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 246 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2024-04-12T11:26:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:46 smithi087 ceph-mon[89014]: osdmap e801: 8 total, 8 up, 8 in 2024-04-12T11:26:47.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:46 smithi029 ceph-mon[93650]: pgmap v837: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 246 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2024-04-12T11:26:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:46 smithi029 ceph-mon[93650]: osdmap e801: 8 total, 8 up, 8 in 2024-04-12T11:26:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:46 smithi029 ceph-mon[97787]: pgmap v837: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 246 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2024-04-12T11:26:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:46 smithi029 ceph-mon[97787]: osdmap e801: 8 total, 8 up, 8 in 2024-04-12T11:26:47.642 INFO:tasks.rados.rados.0.smithi029.stdout:2928: done (7 left) 2024-04-12T11:26:47.642 INFO:tasks.rados.rados.0.smithi029.stdout:2929: rollback oid 31 current snap is 368 2024-04-12T11:26:47.642 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 31 to 341 2024-04-12T11:26:47.643 INFO:tasks.rados.rados.0.smithi029.stdout:2922: expect (ObjNum 853 snap 294 seq_num 853) 2024-04-12T11:26:47.814 INFO:tasks.rados.rados.0.smithi029.stdout:2923: finishing write tid 1 to smithi02979277-46 2024-04-12T11:26:47.815 INFO:tasks.rados.rados.0.smithi029.stdout:2923: finishing write tid 2 to smithi02979277-46 2024-04-12T11:26:47.815 INFO:tasks.rados.rados.0.smithi029.stdout:2923: finishing write tid 3 to smithi02979277-46 2024-04-12T11:26:47.815 INFO:tasks.rados.rados.0.smithi029.stdout:2921: done (7 left) 2024-04-12T11:26:47.815 INFO:tasks.rados.rados.0.smithi029.stdout:2922: done (6 left) 2024-04-12T11:26:47.815 INFO:tasks.rados.rados.0.smithi029.stdout:2930: snap_remove snap 356 2024-04-12T11:26:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:47 smithi087 ceph-mon[89014]: osdmap e802: 8 total, 8 up, 8 in 2024-04-12T11:26:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:47 smithi087 ceph-mon[89014]: osdmap e803: 8 total, 8 up, 8 in 2024-04-12T11:26:48.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:47 smithi029 ceph-mon[97787]: osdmap e802: 8 total, 8 up, 8 in 2024-04-12T11:26:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:47 smithi029 ceph-mon[97787]: osdmap e803: 8 total, 8 up, 8 in 2024-04-12T11:26:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:47 smithi029 ceph-mon[93650]: osdmap e802: 8 total, 8 up, 8 in 2024-04-12T11:26:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:47 smithi029 ceph-mon[93650]: osdmap e803: 8 total, 8 up, 8 in 2024-04-12T11:26:48.724 INFO:tasks.rados.rados.0.smithi029.stdout:2930: done (6 left) 2024-04-12T11:26:48.724 INFO:tasks.rados.rados.0.smithi029.stdout:2931: read oid 41 snap -1 2024-04-12T11:26:48.724 INFO:tasks.rados.rados.0.smithi029.stdout:2931: expect (ObjNum 1087 snap 365 seq_num 1087) 2024-04-12T11:26:48.724 INFO:tasks.rados.rados.0.smithi029.stdout:2932: delete oid 33 current snap is 368 2024-04-12T11:26:48.724 INFO:tasks.rados.rados.0.smithi029.stdout:2923: finishing write tid 4 to smithi02979277-46 2024-04-12T11:26:48.724 INFO:tasks.rados.rados.0.smithi029.stdout:2923: oid 46 updating version 0 to 858 2024-04-12T11:26:48.725 INFO:tasks.rados.rados.0.smithi029.stdout:2923: oid 46 updating version 858 to 859 2024-04-12T11:26:48.725 INFO:tasks.rados.rados.0.smithi029.stdout:2923: oid 46 updating version 859 to 860 2024-04-12T11:26:48.725 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 860 (ObjNum 1095 snap 368 seq_num 1095) dirty exists 2024-04-12T11:26:48.725 INFO:tasks.rados.rados.0.smithi029.stdout:2923: left oid 46 (ObjNum 1095 snap 368 seq_num 1095) 2024-04-12T11:26:48.725 INFO:tasks.rados.rados.0.smithi029.stdout:2925: finishing write tid 1 to smithi02979277-22 2024-04-12T11:26:48.725 INFO:tasks.rados.rados.0.smithi029.stdout:2924: expect (ObjNum 1047 snap 357 seq_num 1047) 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2925: finishing write tid 2 to smithi02979277-22 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2925: finishing write tid 3 to smithi02979277-22 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2925: finishing write tid 4 to smithi02979277-22 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2925: finishing write tid 5 to smithi02979277-22 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2925: oid 22 updating version 0 to 1174 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2925: oid 22 version 1174 is already newer than 1173 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2925: oid 22 updating version 1174 to 1176 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2925: oid 22 version 1176 is already newer than 1175 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 1176 (ObjNum 1096 snap 368 seq_num 1096) dirty exists 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2925: left oid 22 (ObjNum 1096 snap 368 seq_num 1096) 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2926: finishing write tid 1 to smithi02979277-49 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2926: finishing write tid 2 to smithi02979277-49 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2926: finishing write tid 3 to smithi02979277-49 2024-04-12T11:26:48.849 INFO:tasks.rados.rados.0.smithi029.stdout:2926: finishing write tid 4 to smithi02979277-49 2024-04-12T11:26:48.850 INFO:tasks.rados.rados.0.smithi029.stdout:2926: oid 49 updating version 0 to 1014 2024-04-12T11:26:48.850 INFO:tasks.rados.rados.0.smithi029.stdout:2926: oid 49 updating version 1014 to 1015 2024-04-12T11:26:48.850 INFO:tasks.rados.rados.0.smithi029.stdout:2926: oid 49 updating version 1015 to 1016 2024-04-12T11:26:48.850 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 1016 (ObjNum 1097 snap 368 seq_num 1097) dirty exists 2024-04-12T11:26:48.850 INFO:tasks.rados.rados.0.smithi029.stdout:2926: left oid 49 (ObjNum 1097 snap 368 seq_num 1097) 2024-04-12T11:26:48.850 INFO:tasks.rados.rados.0.smithi029.stdout:2927: expect (ObjNum 1090 snap 365 seq_num 1090) 2024-04-12T11:26:48.928 INFO:tasks.rados.rados.0.smithi029.stdout:2929: finishing rollback tid 1 to smithi02979277-31 2024-04-12T11:26:48.928 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-04-12T11:26:48.928 INFO:tasks.rados.rados.0.smithi029.stdout:2931: expect (ObjNum 1087 snap 365 seq_num 1087) 2024-04-12T11:26:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:48 smithi087 ceph-mon[89014]: pgmap v840: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 255 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 21 op/s 2024-04-12T11:26:49.003 INFO:tasks.rados.rados.0.smithi029.stdout:2923: done (7 left) 2024-04-12T11:26:49.003 INFO:tasks.rados.rados.0.smithi029.stdout:2924: done (6 left) 2024-04-12T11:26:49.004 INFO:tasks.rados.rados.0.smithi029.stdout:2925: done (5 left) 2024-04-12T11:26:49.004 INFO:tasks.rados.rados.0.smithi029.stdout:2926: done (4 left) 2024-04-12T11:26:49.004 INFO:tasks.rados.rados.0.smithi029.stdout:2927: done (3 left) 2024-04-12T11:26:49.004 INFO:tasks.rados.rados.0.smithi029.stdout:2929: done (2 left) 2024-04-12T11:26:49.004 INFO:tasks.rados.rados.0.smithi029.stdout:2931: done (1 left) 2024-04-12T11:26:49.004 INFO:tasks.rados.rados.0.smithi029.stdout:2932: done (0 left) 2024-04-12T11:26:49.004 INFO:tasks.rados.rados.0.smithi029.stdout:2933: read oid 41 snap -1 2024-04-12T11:26:49.004 INFO:tasks.rados.rados.0.smithi029.stdout:2933: expect (ObjNum 1087 snap 365 seq_num 1087) 2024-04-12T11:26:49.004 INFO:tasks.rados.rados.0.smithi029.stdout:2934: write oid 50 current snap is 368 2024-04-12T11:26:49.004 INFO:tasks.rados.rados.0.smithi029.stdout:2934: seq_num 1099 ranges {470893=653483,1727593=522054} 2024-04-12T11:26:49.010 INFO:tasks.rados.rados.0.smithi029.stdout:2934: writing smithi02979277-50 from 470893 to 1124376 tid 1 2024-04-12T11:26:49.016 INFO:tasks.rados.rados.0.smithi029.stdout:2934: writing smithi02979277-50 from 1727593 to 2249647 tid 2 2024-04-12T11:26:49.016 INFO:tasks.rados.rados.0.smithi029.stdout:2935: write oid 20 current snap is 368 2024-04-12T11:26:49.016 INFO:tasks.rados.rados.0.smithi029.stdout:2935: seq_num 1100 ranges {487700=728732,1786075=665712} 2024-04-12T11:26:49.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:48 smithi029 ceph-mon[97787]: pgmap v840: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 255 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 21 op/s 2024-04-12T11:26:49.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:48 smithi029 ceph-mon[93650]: pgmap v840: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 255 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.1 MiB/s wr, 21 op/s 2024-04-12T11:26:49.022 INFO:tasks.rados.rados.0.smithi029.stdout:2935: writing smithi02979277-20 from 487700 to 1216432 tid 1 2024-04-12T11:26:49.029 INFO:tasks.rados.rados.0.smithi029.stdout:2935: writing smithi02979277-20 from 1786075 to 2451787 tid 2 2024-04-12T11:26:49.030 INFO:tasks.rados.rados.0.smithi029.stdout:2936: delete oid 12 current snap is 368 2024-04-12T11:26:49.030 INFO:tasks.rados.rados.0.smithi029.stdout:2933: expect (ObjNum 1087 snap 365 seq_num 1087) 2024-04-12T11:26:49.105 INFO:tasks.rados.rados.0.smithi029.stdout:2934: finishing write tid 1 to smithi02979277-50 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2934: finishing write tid 2 to smithi02979277-50 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2934: finishing write tid 3 to smithi02979277-50 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2934: finishing write tid 4 to smithi02979277-50 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2934: oid 50 updating version 0 to 1186 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2934: oid 50 updating version 1186 to 1188 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2934: oid 50 version 1188 is already newer than 1187 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1188 (ObjNum 1099 snap 368 seq_num 1099) dirty exists 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2934: left oid 50 (ObjNum 1099 snap 368 seq_num 1099) 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2935: finishing write tid 1 to smithi02979277-20 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2935: finishing write tid 2 to smithi02979277-20 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2935: finishing write tid 3 to smithi02979277-20 2024-04-12T11:26:49.106 INFO:tasks.rados.rados.0.smithi029.stdout:2935: finishing write tid 4 to smithi02979277-20 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:2935: oid 20 updating version 0 to 866 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:2935: oid 20 updating version 866 to 868 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:2935: oid 20 version 868 is already newer than 867 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 868 (ObjNum 1100 snap 368 seq_num 1100) dirty exists 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:2935: left oid 20 (ObjNum 1100 snap 368 seq_num 1100) 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:2933: done (3 left) 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:2934: done (2 left) 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:2935: done (1 left) 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:2936: done (0 left) 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:2937: write oid 31 current snap is 368 2024-04-12T11:26:49.107 INFO:tasks.rados.rados.0.smithi029.stdout:2937: seq_num 1102 ranges {521314=479230,1786686=669381} 2024-04-12T11:26:49.110 INFO:tasks.rados.rados.0.smithi029.stdout:2937: writing smithi02979277-31 from 521314 to 1000544 tid 1 2024-04-12T11:26:49.119 INFO:tasks.rados.rados.0.smithi029.stdout:2937: writing smithi02979277-31 from 1786686 to 2456067 tid 2 2024-04-12T11:26:49.119 INFO:tasks.rados.rados.0.smithi029.stdout:2938: snap_create 2024-04-12T11:26:49.119 INFO:tasks.rados.rados.0.smithi029.stdout:2937: finishing write tid 1 to smithi02979277-31 2024-04-12T11:26:49.122 INFO:tasks.rados.rados.0.smithi029.stdout:2937: finishing write tid 2 to smithi02979277-31 2024-04-12T11:26:49.122 INFO:tasks.rados.rados.0.smithi029.stdout:2937: finishing write tid 3 to smithi02979277-31 2024-04-12T11:26:49.122 INFO:tasks.rados.rados.0.smithi029.stdout:2937: finishing write tid 4 to smithi02979277-31 2024-04-12T11:26:49.122 INFO:tasks.rados.rados.0.smithi029.stdout:2937: oid 31 updating version 0 to 1088 2024-04-12T11:26:49.122 INFO:tasks.rados.rados.0.smithi029.stdout:2937: oid 31 version 1088 is already newer than 1086 2024-04-12T11:26:49.122 INFO:tasks.rados.rados.0.smithi029.stdout:2937: oid 31 version 1088 is already newer than 1087 2024-04-12T11:26:49.122 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1088 (ObjNum 1102 snap 368 seq_num 1102) dirty exists 2024-04-12T11:26:49.123 INFO:tasks.rados.rados.0.smithi029.stdout:2937: left oid 31 (ObjNum 1102 snap 368 seq_num 1102) 2024-04-12T11:26:49.727 INFO:tasks.rados.rados.0.smithi029.stdout:2937: done (1 left) 2024-04-12T11:26:49.728 INFO:tasks.rados.rados.0.smithi029.stdout:2938: done (0 left) 2024-04-12T11:26:49.728 INFO:tasks.rados.rados.0.smithi029.stdout:2939: write oid 23 current snap is 369 2024-04-12T11:26:49.728 INFO:tasks.rados.rados.0.smithi029.stdout:2939: seq_num 1103 ranges {538121=670832,1961521=413039} 2024-04-12T11:26:49.735 INFO:tasks.rados.rados.0.smithi029.stdout:2939: writing smithi02979277-23 from 538121 to 1208953 tid 1 2024-04-12T11:26:49.741 INFO:tasks.rados.rados.0.smithi029.stdout:2939: writing smithi02979277-23 from 1961521 to 2374560 tid 2 2024-04-12T11:26:49.741 INFO:tasks.rados.rados.0.smithi029.stdout:2940: snap_remove snap 330 2024-04-12T11:26:50.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:49 smithi029 ceph-mon[93650]: osdmap e804: 8 total, 8 up, 8 in 2024-04-12T11:26:50.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:49 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:26:50.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:49 smithi029 ceph-mon[97787]: osdmap e804: 8 total, 8 up, 8 in 2024-04-12T11:26:50.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:49 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:26:50.055 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:49 smithi087 ceph-mon[89014]: osdmap e804: 8 total, 8 up, 8 in 2024-04-12T11:26:50.055 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:49 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:26:50.734 INFO:tasks.rados.rados.0.smithi029.stdout:2940: done (1 left) 2024-04-12T11:26:50.735 INFO:tasks.rados.rados.0.smithi029.stdout:2941: snap_create 2024-04-12T11:26:50.735 INFO:tasks.rados.rados.0.smithi029.stdout:2939: finishing write tid 1 to smithi02979277-23 2024-04-12T11:26:50.735 INFO:tasks.rados.rados.0.smithi029.stdout:2939: finishing write tid 2 to smithi02979277-23 2024-04-12T11:26:50.735 INFO:tasks.rados.rados.0.smithi029.stdout:2939: finishing write tid 3 to smithi02979277-23 2024-04-12T11:26:50.735 INFO:tasks.rados.rados.0.smithi029.stdout:2939: finishing write tid 4 to smithi02979277-23 2024-04-12T11:26:50.735 INFO:tasks.rados.rados.0.smithi029.stdout:2939: oid 23 updating version 0 to 1192 2024-04-12T11:26:50.735 INFO:tasks.rados.rados.0.smithi029.stdout:2939: oid 23 version 1192 is already newer than 1190 2024-04-12T11:26:50.735 INFO:tasks.rados.rados.0.smithi029.stdout:2939: oid 23 version 1192 is already newer than 1191 2024-04-12T11:26:50.735 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1192 (ObjNum 1103 snap 369 seq_num 1103) dirty exists 2024-04-12T11:26:50.735 INFO:tasks.rados.rados.0.smithi029.stdout:2939: left oid 23 (ObjNum 1103 snap 369 seq_num 1103) 2024-04-12T11:26:51.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:50 smithi029 ceph-mon[97787]: pgmap v843: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 257 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 2.4 MiB/s wr, 31 op/s 2024-04-12T11:26:51.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:50 smithi029 ceph-mon[97787]: osdmap e805: 8 total, 8 up, 8 in 2024-04-12T11:26:51.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:50 smithi029 ceph-mon[93650]: pgmap v843: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 257 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 2.4 MiB/s wr, 31 op/s 2024-04-12T11:26:51.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:50 smithi029 ceph-mon[93650]: osdmap e805: 8 total, 8 up, 8 in 2024-04-12T11:26:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:50 smithi087 ceph-mon[89014]: pgmap v843: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 257 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 2.4 MiB/s wr, 31 op/s 2024-04-12T11:26:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:50 smithi087 ceph-mon[89014]: osdmap e805: 8 total, 8 up, 8 in 2024-04-12T11:26:51.737 INFO:tasks.rados.rados.0.smithi029.stdout:2939: done (1 left) 2024-04-12T11:26:51.737 INFO:tasks.rados.rados.0.smithi029.stdout:2941: done (0 left) 2024-04-12T11:26:51.737 INFO:tasks.rados.rados.0.smithi029.stdout:2942: rollback oid 14 current snap is 370 2024-04-12T11:26:51.737 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 362 2024-04-12T11:26:51.737 INFO:tasks.rados.rados.0.smithi029.stdout:2943: read oid 17 snap -1 2024-04-12T11:26:51.737 INFO:tasks.rados.rados.0.smithi029.stdout:2943: expect (ObjNum 1019 snap 352 seq_num 1019) 2024-04-12T11:26:51.737 INFO:tasks.rados.rados.0.smithi029.stdout:2944: snap_remove snap 365 2024-04-12T11:26:51.836 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:51 smithi087 ceph-mon[89014]: osdmap e806: 8 total, 8 up, 8 in 2024-04-12T11:26:51.836 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:52.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:51 smithi029 ceph-mon[93650]: osdmap e806: 8 total, 8 up, 8 in 2024-04-12T11:26:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:51 smithi029 ceph-mon[97787]: osdmap e806: 8 total, 8 up, 8 in 2024-04-12T11:26:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:26:52.643 INFO:tasks.rados.rados.0.smithi029.stdout:2944: done (2 left) 2024-04-12T11:26:52.644 INFO:tasks.rados.rados.0.smithi029.stdout:2945: delete oid 46 current snap is 370 2024-04-12T11:26:52.644 INFO:tasks.rados.rados.0.smithi029.stdout:2942: finishing rollback tid 0 to smithi02979277-14 2024-04-12T11:26:52.644 INFO:tasks.rados.rados.0.smithi029.stdout:2942: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:26:52.644 INFO:tasks.rados.rados.0.smithi029.stdout:2942: finishing rollback tid 2 to smithi02979277-14 2024-04-12T11:26:52.644 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 1351 (ObjNum 1039 snap 355 seq_num 1039) dirty exists 2024-04-12T11:26:52.644 INFO:tasks.rados.rados.0.smithi029.stdout:2943: expect (ObjNum 1019 snap 352 seq_num 1019) 2024-04-12T11:26:52.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:26:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:26:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:26:52.751 INFO:tasks.rados.rados.0.smithi029.stdout:2942: done (2 left) 2024-04-12T11:26:52.751 INFO:tasks.rados.rados.0.smithi029.stdout:2943: done (1 left) 2024-04-12T11:26:52.751 INFO:tasks.rados.rados.0.smithi029.stdout:2945: done (0 left) 2024-04-12T11:26:52.751 INFO:tasks.rados.rados.0.smithi029.stdout:2946: delete oid 24 current snap is 370 2024-04-12T11:26:52.754 INFO:tasks.rados.rados.0.smithi029.stdout:2946: done (0 left) 2024-04-12T11:26:52.754 INFO:tasks.rados.rados.0.smithi029.stdout:2947: delete oid 15 current snap is 370 2024-04-12T11:26:52.756 INFO:tasks.rados.rados.0.smithi029.stdout:2947: done (0 left) 2024-04-12T11:26:52.756 INFO:tasks.rados.rados.0.smithi029.stdout:2948: rollback oid 32 current snap is 370 2024-04-12T11:26:52.756 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 32 to 362 2024-04-12T11:26:52.756 INFO:tasks.rados.rados.0.smithi029.stdout:2949: write oid 48 current snap is 370 2024-04-12T11:26:52.756 INFO:tasks.rados.rados.0.smithi029.stdout:2949: seq_num 1107 ranges {605349=571828,1679096=420377} 2024-04-12T11:26:52.762 INFO:tasks.rados.rados.0.smithi029.stdout:2949: writing smithi02979277-48 from 605349 to 1177177 tid 1 2024-04-12T11:26:52.768 INFO:tasks.rados.rados.0.smithi029.stdout:2949: writing smithi02979277-48 from 1679096 to 2099473 tid 2 2024-04-12T11:26:52.768 INFO:tasks.rados.rados.0.smithi029.stdout:2950: write oid 28 current snap is 370 2024-04-12T11:26:52.768 INFO:tasks.rados.rados.0.smithi029.stdout:2948: finishing rollback tid 0 to smithi02979277-32 2024-04-12T11:26:52.768 INFO:tasks.rados.rados.0.smithi029.stdout:2948: finishing rollback tid 1 to smithi02979277-32 2024-04-12T11:26:52.768 INFO:tasks.rados.rados.0.smithi029.stdout:2950: seq_num 1108 ranges {622156=647077,1737578=680388} 2024-04-12T11:26:52.774 INFO:tasks.rados.rados.0.smithi029.stdout:2950: writing smithi02979277-28 from 622156 to 1269233 tid 1 2024-04-12T11:26:52.781 INFO:tasks.rados.rados.0.smithi029.stdout:2950: writing smithi02979277-28 from 1737578 to 2417966 tid 2 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2951: read oid 10 snap -1 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2951: expect (ObjNum 975 snap 340 seq_num 975) 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2948: finishing rollback tid 2 to smithi02979277-32 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 1358 (ObjNum 1043 snap 356 seq_num 1043) dirty exists 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2949: finishing write tid 1 to smithi02979277-48 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2948: done (3 left) 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2952: snap_create 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2949: finishing write tid 2 to smithi02979277-48 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2949: finishing write tid 3 to smithi02979277-48 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2949: finishing write tid 4 to smithi02979277-48 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2949: oid 48 updating version 0 to 1092 2024-04-12T11:26:52.782 INFO:tasks.rados.rados.0.smithi029.stdout:2949: oid 48 updating version 1092 to 1094 2024-04-12T11:26:52.783 INFO:tasks.rados.rados.0.smithi029.stdout:2949: oid 48 version 1094 is already newer than 1093 2024-04-12T11:26:52.783 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 1094 (ObjNum 1107 snap 370 seq_num 1107) dirty exists 2024-04-12T11:26:52.783 INFO:tasks.rados.rados.0.smithi029.stdout:2949: left oid 48 (ObjNum 1107 snap 370 seq_num 1107) 2024-04-12T11:26:52.783 INFO:tasks.rados.rados.0.smithi029.stdout:2950: finishing write tid 1 to smithi02979277-28 2024-04-12T11:26:52.784 INFO:tasks.rados.rados.0.smithi029.stdout:2950: finishing write tid 2 to smithi02979277-28 2024-04-12T11:26:52.785 INFO:tasks.rados.rados.0.smithi029.stdout:2950: finishing write tid 3 to smithi02979277-28 2024-04-12T11:26:52.785 INFO:tasks.rados.rados.0.smithi029.stdout:2950: finishing write tid 4 to smithi02979277-28 2024-04-12T11:26:52.785 INFO:tasks.rados.rados.0.smithi029.stdout:2950: oid 28 updating version 0 to 1096 2024-04-12T11:26:52.785 INFO:tasks.rados.rados.0.smithi029.stdout:2950: oid 28 updating version 1096 to 1098 2024-04-12T11:26:52.785 INFO:tasks.rados.rados.0.smithi029.stdout:2950: oid 28 version 1098 is already newer than 1097 2024-04-12T11:26:52.785 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 1098 (ObjNum 1108 snap 370 seq_num 1108) dirty exists 2024-04-12T11:26:52.785 INFO:tasks.rados.rados.0.smithi029.stdout:2950: left oid 28 (ObjNum 1108 snap 370 seq_num 1108) 2024-04-12T11:26:52.786 INFO:tasks.rados.rados.0.smithi029.stdout:2951: expect (ObjNum 975 snap 340 seq_num 975) 2024-04-12T11:26:52.986 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:52 smithi087 ceph-mon[89014]: pgmap v846: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 257 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-04-12T11:26:52.986 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:52 smithi087 ceph-mon[89014]: osdmap e807: 8 total, 8 up, 8 in 2024-04-12T11:26:52.987 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:52 smithi087 ceph-mon[89014]: osdmap e808: 8 total, 8 up, 8 in 2024-04-12T11:26:53.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:52 smithi029 ceph-mon[93650]: pgmap v846: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 257 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-04-12T11:26:53.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:52 smithi029 ceph-mon[93650]: osdmap e807: 8 total, 8 up, 8 in 2024-04-12T11:26:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:52 smithi029 ceph-mon[93650]: osdmap e808: 8 total, 8 up, 8 in 2024-04-12T11:26:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:52 smithi029 ceph-mon[97787]: pgmap v846: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 257 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-04-12T11:26:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:52 smithi029 ceph-mon[97787]: osdmap e807: 8 total, 8 up, 8 in 2024-04-12T11:26:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:52 smithi029 ceph-mon[97787]: osdmap e808: 8 total, 8 up, 8 in 2024-04-12T11:26:53.737 INFO:tasks.rados.rados.0.smithi029.stdout:2949: done (3 left) 2024-04-12T11:26:53.737 INFO:tasks.rados.rados.0.smithi029.stdout:2950: done (2 left) 2024-04-12T11:26:53.737 INFO:tasks.rados.rados.0.smithi029.stdout:2951: done (1 left) 2024-04-12T11:26:53.737 INFO:tasks.rados.rados.0.smithi029.stdout:2952: done (0 left) 2024-04-12T11:26:53.738 INFO:tasks.rados.rados.0.smithi029.stdout:2953: write oid 45 current snap is 371 2024-04-12T11:26:53.738 INFO:tasks.rados.rados.0.smithi029.stdout:2953: seq_num 1109 ranges {638963=722326,2079707=424046} 2024-04-12T11:26:53.744 INFO:tasks.rados.rados.0.smithi029.stdout:2953: writing smithi02979277-45 from 638963 to 1361289 tid 1 2024-04-12T11:26:53.751 INFO:tasks.rados.rados.0.smithi029.stdout:2953: writing smithi02979277-45 from 2079707 to 2503753 tid 2 2024-04-12T11:26:53.751 INFO:tasks.rados.rados.0.smithi029.stdout:2954: write oid 4 current snap is 371 2024-04-12T11:26:53.751 INFO:tasks.rados.rados.0.smithi029.stdout:2954: seq_num 1110 ranges {655770=513928,1854542=684057} 2024-04-12T11:26:53.757 INFO:tasks.rados.rados.0.smithi029.stdout:2954: writing smithi02979277-4 from 655770 to 1169698 tid 1 2024-04-12T11:26:53.765 INFO:tasks.rados.rados.0.smithi029.stdout:2954: writing smithi02979277-4 from 1854542 to 2538599 tid 2 2024-04-12T11:26:53.765 INFO:tasks.rados.rados.0.smithi029.stdout:2955: read oid 26 snap -1 2024-04-12T11:26:53.765 INFO:tasks.rados.rados.0.smithi029.stdout:2955: expect (ObjNum 1075 snap 364 seq_num 1075) 2024-04-12T11:26:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:2956: delete oid 13 current snap is 371 2024-04-12T11:26:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:2953: finishing write tid 1 to smithi02979277-45 2024-04-12T11:26:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:2953: finishing write tid 2 to smithi02979277-45 2024-04-12T11:26:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:2953: finishing write tid 3 to smithi02979277-45 2024-04-12T11:26:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:2953: finishing write tid 4 to smithi02979277-45 2024-04-12T11:26:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:2953: oid 45 updating version 0 to 1413 2024-04-12T11:26:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:2953: oid 45 version 1413 is already newer than 1412 2024-04-12T11:26:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:2953: oid 45 version 1413 is already newer than 1411 2024-04-12T11:26:53.766 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1413 (ObjNum 1109 snap 371 seq_num 1109) dirty exists 2024-04-12T11:26:53.767 INFO:tasks.rados.rados.0.smithi029.stdout:2953: left oid 45 (ObjNum 1109 snap 371 seq_num 1109) 2024-04-12T11:26:53.767 INFO:tasks.rados.rados.0.smithi029.stdout:2954: finishing write tid 1 to smithi02979277-4 2024-04-12T11:26:53.769 INFO:tasks.rados.rados.0.smithi029.stdout:2954: finishing write tid 2 to smithi02979277-4 2024-04-12T11:26:53.770 INFO:tasks.rados.rados.0.smithi029.stdout:2954: finishing write tid 3 to smithi02979277-4 2024-04-12T11:26:53.770 INFO:tasks.rados.rados.0.smithi029.stdout:2955: expect (ObjNum 1075 snap 364 seq_num 1075) 2024-04-12T11:26:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:2954: finishing write tid 4 to smithi02979277-4 2024-04-12T11:26:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:2954: oid 4 updating version 0 to 1355 2024-04-12T11:26:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:2954: oid 4 version 1355 is already newer than 1354 2024-04-12T11:26:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:2954: oid 4 updating version 1355 to 1356 2024-04-12T11:26:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1356 (ObjNum 1110 snap 371 seq_num 1110) dirty exists 2024-04-12T11:26:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:2954: left oid 4 (ObjNum 1110 snap 371 seq_num 1110) 2024-04-12T11:26:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:2953: done (3 left) 2024-04-12T11:26:53.906 INFO:tasks.rados.rados.0.smithi029.stdout:2954: done (2 left) 2024-04-12T11:26:53.906 INFO:tasks.rados.rados.0.smithi029.stdout:2955: done (1 left) 2024-04-12T11:26:53.906 INFO:tasks.rados.rados.0.smithi029.stdout:2956: done (0 left) 2024-04-12T11:26:53.906 INFO:tasks.rados.rados.0.smithi029.stdout:2957: rollback oid 46 current snap is 371 2024-04-12T11:26:53.906 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 46 to 341 2024-04-12T11:26:53.906 INFO:tasks.rados.rados.0.smithi029.stdout:2958: delete oid 7 current snap is 371 2024-04-12T11:26:53.907 INFO:tasks.rados.rados.0.smithi029.stdout:2957: finishing rollback tid 1 to smithi02979277-46 2024-04-12T11:26:53.907 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 874 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-04-12T11:26:53.907 INFO:tasks.rados.rados.0.smithi029.stdout:2957: done (1 left) 2024-04-12T11:26:53.907 INFO:tasks.rados.rados.0.smithi029.stdout:2958: done (0 left) 2024-04-12T11:26:53.907 INFO:tasks.rados.rados.0.smithi029.stdout:2959: delete oid 2 current snap is 371 2024-04-12T11:26:53.909 INFO:tasks.rados.rados.0.smithi029.stdout:2959: done (0 left) 2024-04-12T11:26:53.909 INFO:tasks.rados.rados.0.smithi029.stdout:2960: read oid 48 snap -1 2024-04-12T11:26:53.909 INFO:tasks.rados.rados.0.smithi029.stdout:2960: expect (ObjNum 1107 snap 370 seq_num 1107) 2024-04-12T11:26:53.909 INFO:tasks.rados.rados.0.smithi029.stdout:2961: write oid 22 current snap is 371 2024-04-12T11:26:53.909 INFO:tasks.rados.rados.0.smithi029.stdout:2961: seq_num 1114 ranges {722998=414924,1572117=575042} 2024-04-12T11:26:53.915 INFO:tasks.rados.rados.0.smithi029.stdout:2961: writing smithi02979277-22 from 722998 to 1137922 tid 1 2024-04-12T11:26:53.921 INFO:tasks.rados.rados.0.smithi029.stdout:2961: writing smithi02979277-22 from 1572117 to 2147159 tid 2 2024-04-12T11:26:53.921 INFO:tasks.rados.rados.0.smithi029.stdout:2962: snap_create 2024-04-12T11:26:53.921 INFO:tasks.rados.rados.0.smithi029.stdout:2960: expect (ObjNum 1107 snap 370 seq_num 1107) 2024-04-12T11:26:54.116 INFO:tasks.rados.rados.0.smithi029.stdout:2961: finishing write tid 1 to smithi02979277-22 2024-04-12T11:26:54.116 INFO:tasks.rados.rados.0.smithi029.stdout:2961: finishing write tid 2 to smithi02979277-22 2024-04-12T11:26:54.116 INFO:tasks.rados.rados.0.smithi029.stdout:2961: finishing write tid 3 to smithi02979277-22 2024-04-12T11:26:54.116 INFO:tasks.rados.rados.0.smithi029.stdout:2961: finishing write tid 4 to smithi02979277-22 2024-04-12T11:26:54.116 INFO:tasks.rados.rados.0.smithi029.stdout:2961: oid 22 updating version 0 to 1207 2024-04-12T11:26:54.116 INFO:tasks.rados.rados.0.smithi029.stdout:2961: oid 22 updating version 1207 to 1208 2024-04-12T11:26:54.117 INFO:tasks.rados.rados.0.smithi029.stdout:2961: oid 22 version 1208 is already newer than 1206 2024-04-12T11:26:54.117 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 1208 (ObjNum 1114 snap 371 seq_num 1114) dirty exists 2024-04-12T11:26:54.117 INFO:tasks.rados.rados.0.smithi029.stdout:2961: left oid 22 (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:26:54.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:54 smithi087 ceph-mon[89014]: osdmap e809: 8 total, 8 up, 8 in 2024-04-12T11:26:54.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-mon[93650]: osdmap e809: 8 total, 8 up, 8 in 2024-04-12T11:26:54.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:54.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-mon[97787]: osdmap e809: 8 total, 8 up, 8 in 2024-04-12T11:26:54.738 INFO:tasks.rados.rados.0.smithi029.stdout:2960: done (2 left) 2024-04-12T11:26:54.739 INFO:tasks.rados.rados.0.smithi029.stdout:2961: done (1 left) 2024-04-12T11:26:54.739 INFO:tasks.rados.rados.0.smithi029.stdout:2962: done (0 left) 2024-04-12T11:26:54.739 INFO:tasks.rados.rados.0.smithi029.stdout:2963: delete oid 36 current snap is 372 2024-04-12T11:26:54.744 INFO:tasks.rados.rados.0.smithi029.stdout:2963: done (0 left) 2024-04-12T11:26:54.744 INFO:tasks.rados.rados.0.smithi029.stdout:2964: delete oid 42 current snap is 372 2024-04-12T11:26:54.746 INFO:tasks.rados.rados.0.smithi029.stdout:2964: done (0 left) 2024-04-12T11:26:54.746 INFO:tasks.rados.rados.0.smithi029.stdout:2965: write oid 47 current snap is 372 2024-04-12T11:26:54.746 INFO:tasks.rados.rados.0.smithi029.stdout:2965: seq_num 1117 ranges {773419=640671,2147563=438722} 2024-04-12T11:26:54.754 INFO:tasks.rados.rados.0.smithi029.stdout:2965: writing smithi02979277-47 from 773419 to 1414090 tid 1 2024-04-12T11:26:54.760 INFO:tasks.rados.rados.0.smithi029.stdout:2965: writing smithi02979277-47 from 2147563 to 2586285 tid 2 2024-04-12T11:26:54.760 INFO:tasks.rados.rados.0.smithi029.stdout:2966: write oid 20 current snap is 372 2024-04-12T11:26:54.760 INFO:tasks.rados.rados.0.smithi029.stdout:2966: seq_num 1118 ranges {790226=432273,1806045=582380} 2024-04-12T11:26:54.767 INFO:tasks.rados.rados.0.smithi029.stdout:2966: writing smithi02979277-20 from 790226 to 1222499 tid 1 2024-04-12T11:26:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2966: writing smithi02979277-20 from 1806045 to 2388425 tid 2 2024-04-12T11:26:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2967: read oid 50 snap -1 2024-04-12T11:26:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2967: expect (ObjNum 1099 snap 368 seq_num 1099) 2024-04-12T11:26:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2968: write oid 46 current snap is 372 2024-04-12T11:26:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2965: finishing write tid 1 to smithi02979277-47 2024-04-12T11:26:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2965: finishing write tid 2 to smithi02979277-47 2024-04-12T11:26:54.774 INFO:tasks.rados.rados.0.smithi029.stdout:2965: finishing write tid 3 to smithi02979277-47 2024-04-12T11:26:54.775 INFO:tasks.rados.rados.0.smithi029.stdout:2965: finishing write tid 4 to smithi02979277-47 2024-04-12T11:26:54.775 INFO:tasks.rados.rados.0.smithi029.stdout:2965: oid 47 updating version 0 to 1417 2024-04-12T11:26:54.775 INFO:tasks.rados.rados.0.smithi029.stdout:2965: oid 47 updating version 1417 to 1418 2024-04-12T11:26:54.775 INFO:tasks.rados.rados.0.smithi029.stdout:2965: oid 47 version 1418 is already newer than 1416 2024-04-12T11:26:54.775 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1418 (ObjNum 1117 snap 372 seq_num 1117) dirty exists 2024-04-12T11:26:54.775 INFO:tasks.rados.rados.0.smithi029.stdout:2965: left oid 47 (ObjNum 1117 snap 372 seq_num 1117) 2024-04-12T11:26:54.775 INFO:tasks.rados.rados.0.smithi029.stdout:2966: finishing write tid 1 to smithi02979277-20 2024-04-12T11:26:54.775 INFO:tasks.rados.rados.0.smithi029.stdout:2968: seq_num 1119 ranges {407033=507522,1464527=442391,2345976=461057} 2024-04-12T11:26:54.779 INFO:tasks.rados.rados.0.smithi029.stdout:2968: writing smithi02979277-46 from 407033 to 914555 tid 1 2024-04-12T11:26:54.784 INFO:tasks.rados.rados.0.smithi029.stdout:2968: writing smithi02979277-46 from 1464527 to 1906918 tid 2 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2968: writing smithi02979277-46 from 2345976 to 2807033 tid 3 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2965: done (3 left) 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2969: snap_create 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2966: finishing write tid 2 to smithi02979277-20 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2966: finishing write tid 3 to smithi02979277-20 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2966: finishing write tid 4 to smithi02979277-20 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2966: oid 20 updating version 0 to 883 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2966: oid 20 version 883 is already newer than 882 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2966: oid 20 version 883 is already newer than 881 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 883 (ObjNum 1118 snap 372 seq_num 1118) dirty exists 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2966: left oid 20 (ObjNum 1118 snap 372 seq_num 1118) 2024-04-12T11:26:54.790 INFO:tasks.rados.rados.0.smithi029.stdout:2967: expect (ObjNum 1099 snap 368 seq_num 1099) 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:2968: finishing write tid 1 to smithi02979277-46 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:2968: finishing write tid 2 to smithi02979277-46 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:2968: finishing write tid 3 to smithi02979277-46 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:2968: finishing write tid 4 to smithi02979277-46 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:2968: finishing write tid 5 to smithi02979277-46 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:2968: oid 46 updating version 0 to 887 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:2968: oid 46 version 887 is already newer than 886 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:2968: oid 46 version 887 is already newer than 885 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:2968: oid 46 version 887 is already newer than 884 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 887 (ObjNum 1119 snap 372 seq_num 1119) dirty exists 2024-04-12T11:26:54.908 INFO:tasks.rados.rados.0.smithi029.stdout:2968: left oid 46 (ObjNum 1119 snap 372 seq_num 1119) 2024-04-12T11:26:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:55 smithi029 ceph-mon[97787]: pgmap v849: 89 pgs: 89 active+clean; 245 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 552 KiB/s wr, 9 op/s 2024-04-12T11:26:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:26:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:26:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:55 smithi029 ceph-mon[97787]: osdmap e810: 8 total, 8 up, 8 in 2024-04-12T11:26:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:55 smithi029 ceph-mon[93650]: pgmap v849: 89 pgs: 89 active+clean; 245 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 552 KiB/s wr, 9 op/s 2024-04-12T11:26:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:26:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:26:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:55 smithi029 ceph-mon[93650]: osdmap e810: 8 total, 8 up, 8 in 2024-04-12T11:26:55.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:26:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:26:54] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2024-04-12T11:26:55.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:55 smithi087 ceph-mon[89014]: pgmap v849: 89 pgs: 89 active+clean; 245 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 552 KiB/s wr, 9 op/s 2024-04-12T11:26:55.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:26:55.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:26:55.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:26:55.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:55 smithi087 ceph-mon[89014]: osdmap e810: 8 total, 8 up, 8 in 2024-04-12T11:26:55.740 INFO:tasks.rados.rados.0.smithi029.stdout:2966: done (3 left) 2024-04-12T11:26:55.741 INFO:tasks.rados.rados.0.smithi029.stdout:2967: done (2 left) 2024-04-12T11:26:55.741 INFO:tasks.rados.rados.0.smithi029.stdout:2968: done (1 left) 2024-04-12T11:26:55.741 INFO:tasks.rados.rados.0.smithi029.stdout:2969: done (0 left) 2024-04-12T11:26:55.741 INFO:tasks.rados.rados.0.smithi029.stdout:2970: snap_remove snap 367 2024-04-12T11:26:56.743 INFO:tasks.rados.rados.0.smithi029.stdout:2970: done (0 left) 2024-04-12T11:26:56.743 INFO:tasks.rados.rados.0.smithi029.stdout:2971: read oid 11 snap -1 2024-04-12T11:26:56.743 INFO:tasks.rados.rados.0.smithi029.stdout:2971: expect deleted 2024-04-12T11:26:56.744 INFO:tasks.rados.rados.0.smithi029.stdout:2972: rollback oid 23 current snap is 373 2024-04-12T11:26:56.744 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 23 to 368 2024-04-12T11:26:56.744 INFO:tasks.rados.rados.0.smithi029.stdout:2973: snap_create 2024-04-12T11:26:56.751 INFO:tasks.rados.rados.0.smithi029.stdout:2972: finishing rollback tid 0 to smithi02979277-23 2024-04-12T11:26:56.753 INFO:tasks.rados.rados.0.smithi029.stdout:2972: finishing rollback tid 1 to smithi02979277-23 2024-04-12T11:26:56.753 INFO:tasks.rados.rados.0.smithi029.stdout:2972: finishing rollback tid 2 to smithi02979277-23 2024-04-12T11:26:56.753 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1212 (ObjNum 1069 snap 364 seq_num 1069) dirty exists 2024-04-12T11:26:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:56 smithi029 ceph-mon[93650]: pgmap v852: 89 pgs: 89 active+clean; 234 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-04-12T11:26:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:56 smithi029 ceph-mon[93650]: osdmap e811: 8 total, 8 up, 8 in 2024-04-12T11:26:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:56 smithi029 ceph-mon[97787]: pgmap v852: 89 pgs: 89 active+clean; 234 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-04-12T11:26:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:56 smithi029 ceph-mon[97787]: osdmap e811: 8 total, 8 up, 8 in 2024-04-12T11:26:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:56 smithi087 ceph-mon[89014]: pgmap v852: 89 pgs: 89 active+clean; 234 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-04-12T11:26:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:56 smithi087 ceph-mon[89014]: osdmap e811: 8 total, 8 up, 8 in 2024-04-12T11:26:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2971: done (2 left) 2024-04-12T11:26:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2972: done (1 left) 2024-04-12T11:26:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2973: done (0 left) 2024-04-12T11:26:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2974: rollback oid 9 current snap is 374 2024-04-12T11:26:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 9 to 370 2024-04-12T11:26:57.645 INFO:tasks.rados.rados.0.smithi029.stdout:2975: write oid 29 current snap is 374 2024-04-12T11:26:57.646 INFO:tasks.rados.rados.0.smithi029.stdout:2975: seq_num 1120 ranges {423840=582771,1523009=586049,2657046=166794} 2024-04-12T11:26:57.651 INFO:tasks.rados.rados.0.smithi029.stdout:2975: writing smithi02979277-29 from 423840 to 1006611 tid 1 2024-04-12T11:26:57.657 INFO:tasks.rados.rados.0.smithi029.stdout:2975: writing smithi02979277-29 from 1523009 to 2109058 tid 2 2024-04-12T11:26:57.662 INFO:tasks.rados.rados.0.smithi029.stdout:2975: writing smithi02979277-29 from 2657046 to 2823840 tid 3 2024-04-12T11:26:57.662 INFO:tasks.rados.rados.0.smithi029.stdout:2976: write oid 44 current snap is 374 2024-04-12T11:26:57.662 INFO:tasks.rados.rados.0.smithi029.stdout:2974: finishing rollback tid 1 to smithi02979277-9 2024-04-12T11:26:57.662 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 951 (ObjNum 33 snap 0 seq_num 1801223424) dirty dne 2024-04-12T11:26:57.662 INFO:tasks.rados.rados.0.smithi029.stdout:2976: seq_num 1121 ranges {440647=658020,1581491=446060,2800822=39825} 2024-04-12T11:26:57.668 INFO:tasks.rados.rados.0.smithi029.stdout:2976: writing smithi02979277-44 from 440647 to 1098667 tid 1 2024-04-12T11:26:57.673 INFO:tasks.rados.rados.0.smithi029.stdout:2976: writing smithi02979277-44 from 1581491 to 2027551 tid 2 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2976: writing smithi02979277-44 from 2800822 to 2840647 tid 3 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2974: done (2 left) 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2977: snap_create 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2975: finishing write tid 1 to smithi02979277-29 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2975: finishing write tid 2 to smithi02979277-29 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2975: finishing write tid 3 to smithi02979277-29 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2975: finishing write tid 4 to smithi02979277-29 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2975: finishing write tid 5 to smithi02979277-29 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2975: oid 29 updating version 0 to 1363 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2975: oid 29 updating version 1363 to 1364 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2975: oid 29 version 1364 is already newer than 1361 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2975: oid 29 version 1364 is already newer than 1362 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1364 (ObjNum 1120 snap 374 seq_num 1120) dirty exists 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2975: left oid 29 (ObjNum 1120 snap 374 seq_num 1120) 2024-04-12T11:26:57.678 INFO:tasks.rados.rados.0.smithi029.stdout:2976: finishing write tid 1 to smithi02979277-44 2024-04-12T11:26:57.679 INFO:tasks.rados.rados.0.smithi029.stdout:2976: finishing write tid 2 to smithi02979277-44 2024-04-12T11:26:57.680 INFO:tasks.rados.rados.0.smithi029.stdout:2976: finishing write tid 3 to smithi02979277-44 2024-04-12T11:26:57.680 INFO:tasks.rados.rados.0.smithi029.stdout:2976: finishing write tid 4 to smithi02979277-44 2024-04-12T11:26:57.680 INFO:tasks.rados.rados.0.smithi029.stdout:2976: finishing write tid 5 to smithi02979277-44 2024-04-12T11:26:57.680 INFO:tasks.rados.rados.0.smithi029.stdout:2976: oid 44 updating version 0 to 1414 2024-04-12T11:26:57.680 INFO:tasks.rados.rados.0.smithi029.stdout:2976: oid 44 updating version 1414 to 1416 2024-04-12T11:26:57.680 INFO:tasks.rados.rados.0.smithi029.stdout:2976: oid 44 updating version 1416 to 1417 2024-04-12T11:26:57.680 INFO:tasks.rados.rados.0.smithi029.stdout:2976: oid 44 version 1417 is already newer than 1415 2024-04-12T11:26:57.680 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1417 (ObjNum 1121 snap 374 seq_num 1121) dirty exists 2024-04-12T11:26:57.680 INFO:tasks.rados.rados.0.smithi029.stdout:2976: left oid 44 (ObjNum 1121 snap 374 seq_num 1121) 2024-04-12T11:26:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:57 smithi029 ceph-mon[93650]: osdmap e812: 8 total, 8 up, 8 in 2024-04-12T11:26:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:57 smithi029 ceph-mon[93650]: osdmap e813: 8 total, 8 up, 8 in 2024-04-12T11:26:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:57 smithi029 ceph-mon[97787]: osdmap e812: 8 total, 8 up, 8 in 2024-04-12T11:26:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:57 smithi029 ceph-mon[97787]: osdmap e813: 8 total, 8 up, 8 in 2024-04-12T11:26:58.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:57 smithi087 ceph-mon[89014]: osdmap e812: 8 total, 8 up, 8 in 2024-04-12T11:26:58.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:57 smithi087 ceph-mon[89014]: osdmap e813: 8 total, 8 up, 8 in 2024-04-12T11:26:58.646 INFO:tasks.rados.rados.0.smithi029.stdout:2975: done (2 left) 2024-04-12T11:26:58.647 INFO:tasks.rados.rados.0.smithi029.stdout:2976: done (1 left) 2024-04-12T11:26:58.647 INFO:tasks.rados.rados.0.smithi029.stdout:2977: done (0 left) 2024-04-12T11:26:58.647 INFO:tasks.rados.rados.0.smithi029.stdout:2978: snap_remove snap 370 2024-04-12T11:26:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:58 smithi029 ceph-mon[93650]: pgmap v855: 89 pgs: 89 active+clean; 240 MiB data, 827 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-04-12T11:26:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:26:58 smithi029 ceph-mon[93650]: osdmap e814: 8 total, 8 up, 8 in 2024-04-12T11:26:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:58 smithi029 ceph-mon[97787]: pgmap v855: 89 pgs: 89 active+clean; 240 MiB data, 827 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-04-12T11:26:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:26:58 smithi029 ceph-mon[97787]: osdmap e814: 8 total, 8 up, 8 in 2024-04-12T11:26:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:58 smithi087 ceph-mon[89014]: pgmap v855: 89 pgs: 89 active+clean; 240 MiB data, 827 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-04-12T11:26:59.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:26:58 smithi087 ceph-mon[89014]: osdmap e814: 8 total, 8 up, 8 in 2024-04-12T11:26:59.648 INFO:tasks.rados.rados.0.smithi029.stdout:2978: done (0 left) 2024-04-12T11:26:59.648 INFO:tasks.rados.rados.0.smithi029.stdout:2979: snap_create 2024-04-12T11:27:00.651 INFO:tasks.rados.rados.0.smithi029.stdout:2979: done (0 left) 2024-04-12T11:27:00.651 INFO:tasks.rados.rados.0.smithi029.stdout:2980: rollback oid 7 current snap is 376 2024-04-12T11:27:00.651 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 368 2024-04-12T11:27:00.651 INFO:tasks.rados.rados.0.smithi029.stdout:2981: write oid 14 current snap is 376 2024-04-12T11:27:00.651 INFO:tasks.rados.rados.0.smithi029.stdout:2981: seq_num 1122 ranges {457454=733269,1923620=589718} 2024-04-12T11:27:00.657 INFO:tasks.rados.rados.0.smithi029.stdout:2981: writing smithi02979277-14 from 457454 to 1190723 tid 1 2024-04-12T11:27:00.665 INFO:tasks.rados.rados.0.smithi029.stdout:2981: writing smithi02979277-14 from 1923620 to 2513338 tid 2 2024-04-12T11:27:00.665 INFO:tasks.rados.rados.0.smithi029.stdout:2982: read oid 39 snap -1 2024-04-12T11:27:00.665 INFO:tasks.rados.rados.0.smithi029.stdout:2982: expect (ObjNum 1088 snap 365 seq_num 1088) 2024-04-12T11:27:00.665 INFO:tasks.rados.rados.0.smithi029.stdout:2983: delete oid 41 current snap is 376 2024-04-12T11:27:00.666 INFO:tasks.rados.rados.0.smithi029.stdout:2980: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:27:00.666 INFO:tasks.rados.rados.0.smithi029.stdout:2980: finishing rollback tid 2 to smithi02979277-7 2024-04-12T11:27:00.666 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1428 (ObjNum 1071 snap 364 seq_num 1071) dirty exists 2024-04-12T11:27:00.666 INFO:tasks.rados.rados.0.smithi029.stdout:2981: finishing write tid 1 to smithi02979277-14 2024-04-12T11:27:00.668 INFO:tasks.rados.rados.0.smithi029.stdout:2980: done (3 left) 2024-04-12T11:27:00.668 INFO:tasks.rados.rados.0.smithi029.stdout:2983: done (2 left) 2024-04-12T11:27:00.668 INFO:tasks.rados.rados.0.smithi029.stdout:2984: write oid 7 current snap is 376 2024-04-12T11:27:00.668 INFO:tasks.rados.rados.0.smithi029.stdout:2984: seq_num 1124 ranges {491068=483767,1640584=593387,2650385=240683} 2024-04-12T11:27:00.673 INFO:tasks.rados.rados.0.smithi029.stdout:2984: writing smithi02979277-7 from 491068 to 974835 tid 1 2024-04-12T11:27:00.680 INFO:tasks.rados.rados.0.smithi029.stdout:2984: writing smithi02979277-7 from 1640584 to 2233971 tid 2 2024-04-12T11:27:00.684 INFO:tasks.rados.rados.0.smithi029.stdout:2984: writing smithi02979277-7 from 2650385 to 2891068 tid 3 2024-04-12T11:27:00.684 INFO:tasks.rados.rados.0.smithi029.stdout:2985: write oid 37 current snap is 376 2024-04-12T11:27:00.684 INFO:tasks.rados.rados.0.smithi029.stdout:2985: seq_num 1125 ranges {507875=559016,1699066=737045} 2024-04-12T11:27:00.690 INFO:tasks.rados.rados.0.smithi029.stdout:2985: writing smithi02979277-37 from 507875 to 1066891 tid 1 2024-04-12T11:27:00.698 INFO:tasks.rados.rados.0.smithi029.stdout:2985: writing smithi02979277-37 from 1699066 to 2436111 tid 2 2024-04-12T11:27:00.698 INFO:tasks.rados.rados.0.smithi029.stdout:2986: snap_create 2024-04-12T11:27:00.698 INFO:tasks.rados.rados.0.smithi029.stdout:2981: finishing write tid 2 to smithi02979277-14 2024-04-12T11:27:00.698 INFO:tasks.rados.rados.0.smithi029.stdout:2981: finishing write tid 3 to smithi02979277-14 2024-04-12T11:27:00.698 INFO:tasks.rados.rados.0.smithi029.stdout:2981: finishing write tid 4 to smithi02979277-14 2024-04-12T11:27:00.698 INFO:tasks.rados.rados.0.smithi029.stdout:2981: oid 14 updating version 0 to 1361 2024-04-12T11:27:00.699 INFO:tasks.rados.rados.0.smithi029.stdout:2981: oid 14 updating version 1361 to 1362 2024-04-12T11:27:00.699 INFO:tasks.rados.rados.0.smithi029.stdout:2981: oid 14 version 1362 is already newer than 1360 2024-04-12T11:27:00.699 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 1362 (ObjNum 1122 snap 376 seq_num 1122) dirty exists 2024-04-12T11:27:00.699 INFO:tasks.rados.rados.0.smithi029.stdout:2981: left oid 14 (ObjNum 1122 snap 376 seq_num 1122) 2024-04-12T11:27:00.699 INFO:tasks.rados.rados.0.smithi029.stdout:2982: expect (ObjNum 1088 snap 365 seq_num 1088) 2024-04-12T11:27:00.822 INFO:tasks.rados.rados.0.smithi029.stdout:2984: finishing write tid 1 to smithi02979277-7 2024-04-12T11:27:00.822 INFO:tasks.rados.rados.0.smithi029.stdout:2984: finishing write tid 2 to smithi02979277-7 2024-04-12T11:27:00.822 INFO:tasks.rados.rados.0.smithi029.stdout:2984: finishing write tid 3 to smithi02979277-7 2024-04-12T11:27:00.822 INFO:tasks.rados.rados.0.smithi029.stdout:2984: finishing write tid 4 to smithi02979277-7 2024-04-12T11:27:00.822 INFO:tasks.rados.rados.0.smithi029.stdout:2984: finishing write tid 5 to smithi02979277-7 2024-04-12T11:27:00.822 INFO:tasks.rados.rados.0.smithi029.stdout:2984: oid 7 updating version 0 to 1429 2024-04-12T11:27:00.822 INFO:tasks.rados.rados.0.smithi029.stdout:2984: oid 7 updating version 1429 to 1431 2024-04-12T11:27:00.822 INFO:tasks.rados.rados.0.smithi029.stdout:2984: oid 7 updating version 1431 to 1432 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:2984: oid 7 version 1432 is already newer than 1430 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1432 (ObjNum 1124 snap 376 seq_num 1124) dirty exists 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:2984: left oid 7 (ObjNum 1124 snap 376 seq_num 1124) 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:2985: finishing write tid 1 to smithi02979277-37 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:2985: finishing write tid 2 to smithi02979277-37 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:2985: finishing write tid 3 to smithi02979277-37 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:2985: finishing write tid 4 to smithi02979277-37 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:2985: oid 37 updating version 0 to 1023 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:2985: oid 37 updating version 1023 to 1025 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:2985: oid 37 version 1025 is already newer than 1024 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 1025 (ObjNum 1125 snap 376 seq_num 1125) dirty exists 2024-04-12T11:27:00.823 INFO:tasks.rados.rados.0.smithi029.stdout:2985: left oid 37 (ObjNum 1125 snap 376 seq_num 1125) 2024-04-12T11:27:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:00 smithi087 ceph-mon[89014]: pgmap v858: 89 pgs: 89 active+clean; 251 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-04-12T11:27:00.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:00 smithi087 ceph-mon[89014]: osdmap e815: 8 total, 8 up, 8 in 2024-04-12T11:27:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:00 smithi029 ceph-mon[93650]: pgmap v858: 89 pgs: 89 active+clean; 251 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-04-12T11:27:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:00 smithi029 ceph-mon[93650]: osdmap e815: 8 total, 8 up, 8 in 2024-04-12T11:27:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:00 smithi029 ceph-mon[97787]: pgmap v858: 89 pgs: 89 active+clean; 251 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-04-12T11:27:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:00 smithi029 ceph-mon[97787]: osdmap e815: 8 total, 8 up, 8 in 2024-04-12T11:27:01.664 INFO:tasks.rados.rados.0.smithi029.stdout:2981: done (4 left) 2024-04-12T11:27:01.664 INFO:tasks.rados.rados.0.smithi029.stdout:2982: done (3 left) 2024-04-12T11:27:01.664 INFO:tasks.rados.rados.0.smithi029.stdout:2984: done (2 left) 2024-04-12T11:27:01.664 INFO:tasks.rados.rados.0.smithi029.stdout:2985: done (1 left) 2024-04-12T11:27:01.664 INFO:tasks.rados.rados.0.smithi029.stdout:2986: done (0 left) 2024-04-12T11:27:01.664 INFO:tasks.rados.rados.0.smithi029.stdout:2987: rollback oid 7 current snap is 377 2024-04-12T11:27:01.664 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 375 2024-04-12T11:27:01.664 INFO:tasks.rados.rados.0.smithi029.stdout:2988: write oid 26 current snap is 377 2024-04-12T11:27:01.664 INFO:tasks.rados.rados.0.smithi029.stdout:2988: seq_num 1126 ranges {524682=750618,1757548=597056} 2024-04-12T11:27:01.670 INFO:tasks.rados.rados.0.smithi029.stdout:2988: writing smithi02979277-26 from 524682 to 1275300 tid 1 2024-04-12T11:27:01.677 INFO:tasks.rados.rados.0.smithi029.stdout:2988: writing smithi02979277-26 from 1757548 to 2354604 tid 2 2024-04-12T11:27:01.677 INFO:tasks.rados.rados.0.smithi029.stdout:2989: snap_create 2024-04-12T11:27:01.677 INFO:tasks.rados.rados.0.smithi029.stdout:2987: finishing rollback tid 0 to smithi02979277-7 2024-04-12T11:27:01.677 INFO:tasks.rados.rados.0.smithi029.stdout:2987: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:27:01.677 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1435 (ObjNum 225 snap 0 seq_num 1800669456) dirty dne 2024-04-12T11:27:01.679 INFO:tasks.rados.rados.0.smithi029.stdout:2988: finishing write tid 1 to smithi02979277-26 2024-04-12T11:27:01.681 INFO:tasks.rados.rados.0.smithi029.stdout:2988: finishing write tid 2 to smithi02979277-26 2024-04-12T11:27:01.681 INFO:tasks.rados.rados.0.smithi029.stdout:2988: finishing write tid 3 to smithi02979277-26 2024-04-12T11:27:01.681 INFO:tasks.rados.rados.0.smithi029.stdout:2988: finishing write tid 4 to smithi02979277-26 2024-04-12T11:27:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:2988: oid 26 updating version 0 to 1843 2024-04-12T11:27:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:2988: oid 26 updating version 1843 to 1844 2024-04-12T11:27:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:2988: oid 26 updating version 1844 to 1845 2024-04-12T11:27:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 1845 (ObjNum 1126 snap 377 seq_num 1126) dirty exists 2024-04-12T11:27:01.682 INFO:tasks.rados.rados.0.smithi029.stdout:2988: left oid 26 (ObjNum 1126 snap 377 seq_num 1126) 2024-04-12T11:27:01.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:01 smithi087 ceph-mon[89014]: osdmap e816: 8 total, 8 up, 8 in 2024-04-12T11:27:02.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:01 smithi029 ceph-mon[93650]: osdmap e816: 8 total, 8 up, 8 in 2024-04-12T11:27:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:01 smithi029 ceph-mon[97787]: osdmap e816: 8 total, 8 up, 8 in 2024-04-12T11:27:02.646 INFO:tasks.rados.rados.0.smithi029.stdout:2987: done (2 left) 2024-04-12T11:27:02.646 INFO:tasks.rados.rados.0.smithi029.stdout:2988: done (1 left) 2024-04-12T11:27:02.646 INFO:tasks.rados.rados.0.smithi029.stdout:2989: done (0 left) 2024-04-12T11:27:02.646 INFO:tasks.rados.rados.0.smithi029.stdout:2990: rollback oid 2 current snap is 378 2024-04-12T11:27:02.646 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 2 to 376 2024-04-12T11:27:02.646 INFO:tasks.rados.rados.0.smithi029.stdout:2991: snap_create 2024-04-12T11:27:02.649 INFO:tasks.rados.rados.0.smithi029.stdout:2990: finishing rollback tid 1 to smithi02979277-2 2024-04-12T11:27:02.649 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 323 (ObjNum 49 snap 0 seq_num 1777349616) dirty dne 2024-04-12T11:27:02.660 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:27:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:27:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:27:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:02 smithi087 ceph-mon[89014]: pgmap v861: 89 pgs: 89 active+clean; 251 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-04-12T11:27:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:02 smithi087 ceph-mon[89014]: osdmap e817: 8 total, 8 up, 8 in 2024-04-12T11:27:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:02 smithi087 ceph-mon[89014]: osdmap e818: 8 total, 8 up, 8 in 2024-04-12T11:27:03.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:02 smithi029 ceph-mon[93650]: pgmap v861: 89 pgs: 89 active+clean; 251 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-04-12T11:27:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:02 smithi029 ceph-mon[93650]: osdmap e817: 8 total, 8 up, 8 in 2024-04-12T11:27:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:02 smithi029 ceph-mon[93650]: osdmap e818: 8 total, 8 up, 8 in 2024-04-12T11:27:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:02 smithi029 ceph-mon[97787]: pgmap v861: 89 pgs: 89 active+clean; 251 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-04-12T11:27:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:02 smithi029 ceph-mon[97787]: osdmap e817: 8 total, 8 up, 8 in 2024-04-12T11:27:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:02 smithi029 ceph-mon[97787]: osdmap e818: 8 total, 8 up, 8 in 2024-04-12T11:27:03.648 INFO:tasks.rados.rados.0.smithi029.stdout:2990: done (1 left) 2024-04-12T11:27:03.648 INFO:tasks.rados.rados.0.smithi029.stdout:2991: done (0 left) 2024-04-12T11:27:03.648 INFO:tasks.rados.rados.0.smithi029.stdout:2992: write oid 18 current snap is 379 2024-04-12T11:27:03.648 INFO:tasks.rados.rados.0.smithi029.stdout:2992: seq_num 1127 ranges {541489=425867,1416030=740714,2732654=208835} 2024-04-12T11:27:03.653 INFO:tasks.rados.rados.0.smithi029.stdout:2992: writing smithi02979277-18 from 541489 to 967356 tid 1 2024-04-12T11:27:03.659 INFO:tasks.rados.rados.0.smithi029.stdout:2992: writing smithi02979277-18 from 1416030 to 2156744 tid 2 2024-04-12T11:27:03.664 INFO:tasks.rados.rados.0.smithi029.stdout:2992: writing smithi02979277-18 from 2732654 to 2941489 tid 3 2024-04-12T11:27:03.664 INFO:tasks.rados.rados.0.smithi029.stdout:2993: delete oid 49 current snap is 379 2024-04-12T11:27:03.664 INFO:tasks.rados.rados.0.smithi029.stdout:2992: finishing write tid 1 to smithi02979277-18 2024-04-12T11:27:03.664 INFO:tasks.rados.rados.0.smithi029.stdout:2992: finishing write tid 2 to smithi02979277-18 2024-04-12T11:27:03.667 INFO:tasks.rados.rados.0.smithi029.stdout:2992: finishing write tid 3 to smithi02979277-18 2024-04-12T11:27:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:2992: finishing write tid 4 to smithi02979277-18 2024-04-12T11:27:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:2992: finishing write tid 5 to smithi02979277-18 2024-04-12T11:27:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:2992: oid 18 updating version 0 to 1478 2024-04-12T11:27:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:2992: oid 18 updating version 1478 to 1480 2024-04-12T11:27:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:2992: oid 18 version 1480 is already newer than 1479 2024-04-12T11:27:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:2992: oid 18 version 1480 is already newer than 1477 2024-04-12T11:27:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1480 (ObjNum 1127 snap 379 seq_num 1127) dirty exists 2024-04-12T11:27:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:2992: left oid 18 (ObjNum 1127 snap 379 seq_num 1127) 2024-04-12T11:27:03.669 INFO:tasks.rados.rados.0.smithi029.stdout:2992: done (1 left) 2024-04-12T11:27:03.669 INFO:tasks.rados.rados.0.smithi029.stdout:2993: done (0 left) 2024-04-12T11:27:03.669 INFO:tasks.rados.rados.0.smithi029.stdout:2994: snap_remove snap 372 2024-04-12T11:27:04.667 INFO:tasks.rados.rados.0.smithi029.stdout:2994: done (0 left) 2024-04-12T11:27:04.667 INFO:tasks.rados.rados.0.smithi029.stdout:2995: write oid 42 current snap is 379 2024-04-12T11:27:04.667 INFO:tasks.rados.rados.0.smithi029.stdout:2995: seq_num 1129 ranges {575103=576365,1932994=744383} 2024-04-12T11:27:04.673 INFO:tasks.rados.rados.0.smithi029.stdout:2995: writing smithi02979277-42 from 575103 to 1151468 tid 1 2024-04-12T11:27:04.682 INFO:tasks.rados.rados.0.smithi029.stdout:2995: writing smithi02979277-42 from 1932994 to 2677377 tid 2 2024-04-12T11:27:04.683 INFO:tasks.rados.rados.0.smithi029.stdout:2996: rollback oid 32 current snap is 379 2024-04-12T11:27:04.683 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 32 to 373 2024-04-12T11:27:04.683 INFO:tasks.rados.rados.0.smithi029.stdout:2995: finishing write tid 1 to smithi02979277-42 2024-04-12T11:27:04.683 INFO:tasks.rados.rados.0.smithi029.stdout:2997: delete oid 41 current snap is 379 2024-04-12T11:27:04.685 INFO:tasks.rados.rados.0.smithi029.stdout:2997: done (2 left) 2024-04-12T11:27:04.685 INFO:tasks.rados.rados.0.smithi029.stdout:2998: read oid 25 snap -1 2024-04-12T11:27:04.685 INFO:tasks.rados.rados.0.smithi029.stdout:2998: expect deleted 2024-04-12T11:27:04.686 INFO:tasks.rados.rados.0.smithi029.stdout:2999: write oid 46 current snap is 379 2024-04-12T11:27:04.686 INFO:tasks.rados.rados.0.smithi029.stdout:2999: seq_num 1131 ranges {608717=726863,1933605=748052} 2024-04-12T11:27:04.693 INFO:tasks.rados.rados.0.smithi029.stdout:2999: writing smithi02979277-46 from 608717 to 1335580 tid 1 2024-04-12T11:27:04.700 INFO:tasks.rados.rados.0.smithi029.stdout:2999: writing smithi02979277-46 from 1933605 to 2681657 tid 2 2024-04-12T11:27:04.700 INFO:tasks.rados.rados.0.smithi029.stdout:3000: write oid 9 current snap is 379 2024-04-12T11:27:04.700 INFO:tasks.rados.rados.0.smithi029.stdout:3000: seq_num 1132 ranges {625524=402112,1592087=608063,2869769=155755} 2024-04-12T11:27:04.706 INFO:tasks.rados.rados.0.smithi029.stdout:3000: writing smithi02979277-9 from 625524 to 1027636 tid 1 2024-04-12T11:27:04.712 INFO:tasks.rados.rados.0.smithi029.stdout:3000: writing smithi02979277-9 from 1592087 to 2200150 tid 2 2024-04-12T11:27:04.718 INFO:tasks.rados.rados.0.smithi029.stdout:3000: writing smithi02979277-9 from 2869769 to 3025524 tid 3 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:3001: rollback oid 37 current snap is 379 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 378 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:2996: finishing rollback tid 0 to smithi02979277-32 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:2995: finishing write tid 2 to smithi02979277-42 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:2996: finishing rollback tid 1 to smithi02979277-32 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:2995: finishing write tid 3 to smithi02979277-42 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:2996: finishing rollback tid 2 to smithi02979277-32 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 1368 (ObjNum 1043 snap 356 seq_num 1043) dirty exists 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:2995: finishing write tid 4 to smithi02979277-42 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:2995: oid 42 updating version 0 to 358 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:2995: oid 42 updating version 358 to 359 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:2995: oid 42 version 359 is already newer than 357 2024-04-12T11:27:04.719 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 359 (ObjNum 1129 snap 379 seq_num 1129) dirty exists 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2995: left oid 42 (ObjNum 1129 snap 379 seq_num 1129) 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2999: finishing write tid 1 to smithi02979277-46 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2995: done (5 left) 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2996: done (4 left) 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2998: done (3 left) 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:3002: rollback oid 35 current snap is 379 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 35 to 371 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2999: finishing write tid 2 to smithi02979277-46 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2999: finishing write tid 3 to smithi02979277-46 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2999: finishing write tid 4 to smithi02979277-46 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2999: oid 46 updating version 0 to 896 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2999: oid 46 version 896 is already newer than 895 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:2999: oid 46 updating version 896 to 897 2024-04-12T11:27:04.720 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 897 (ObjNum 1131 snap 379 seq_num 1131) dirty exists 2024-04-12T11:27:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:2999: left oid 46 (ObjNum 1131 snap 379 seq_num 1131) 2024-04-12T11:27:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3000: finishing write tid 1 to smithi02979277-9 2024-04-12T11:27:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3000: finishing write tid 2 to smithi02979277-9 2024-04-12T11:27:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:2999: done (3 left) 2024-04-12T11:27:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3003: read oid 27 snap -1 2024-04-12T11:27:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3003: expect deleted 2024-04-12T11:27:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3004: snap_remove snap 355 2024-04-12T11:27:04.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:04 smithi029 ceph-mon[93650]: pgmap v864: 89 pgs: 89 active+clean; 258 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-04-12T11:27:04.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:04 smithi029 ceph-mon[93650]: osdmap e819: 8 total, 8 up, 8 in 2024-04-12T11:27:04.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:04 smithi029 ceph-mon[97787]: pgmap v864: 89 pgs: 89 active+clean; 258 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-04-12T11:27:04.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:04 smithi029 ceph-mon[97787]: osdmap e819: 8 total, 8 up, 8 in 2024-04-12T11:27:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:04 smithi087 ceph-mon[89014]: pgmap v864: 89 pgs: 89 active+clean; 258 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-04-12T11:27:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:04 smithi087 ceph-mon[89014]: osdmap e819: 8 total, 8 up, 8 in 2024-04-12T11:27:05.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:27:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:27:04] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2024-04-12T11:27:05.674 INFO:tasks.rados.rados.0.smithi029.stdout:3004: done (4 left) 2024-04-12T11:27:05.674 INFO:tasks.rados.rados.0.smithi029.stdout:3005: read oid 10 snap -1 2024-04-12T11:27:05.674 INFO:tasks.rados.rados.0.smithi029.stdout:3005: expect (ObjNum 975 snap 340 seq_num 975) 2024-04-12T11:27:05.674 INFO:tasks.rados.rados.0.smithi029.stdout:3006: read oid 6 snap -1 2024-04-12T11:27:05.674 INFO:tasks.rados.rados.0.smithi029.stdout:3006: expect (ObjNum 872 snap 307 seq_num 872) 2024-04-12T11:27:05.675 INFO:tasks.rados.rados.0.smithi029.stdout:3007: read oid 24 snap -1 2024-04-12T11:27:05.675 INFO:tasks.rados.rados.0.smithi029.stdout:3007: expect deleted 2024-04-12T11:27:05.675 INFO:tasks.rados.rados.0.smithi029.stdout:3008: delete oid 26 current snap is 379 2024-04-12T11:27:05.675 INFO:tasks.rados.rados.0.smithi029.stdout:3000: finishing write tid 3 to smithi02979277-9 2024-04-12T11:27:05.675 INFO:tasks.rados.rados.0.smithi029.stdout:3002: finishing rollback tid 1 to smithi02979277-35 2024-04-12T11:27:05.675 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 1770 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:27:05.675 INFO:tasks.rados.rados.0.smithi029.stdout:3000: finishing write tid 4 to smithi02979277-9 2024-04-12T11:27:05.675 INFO:tasks.rados.rados.0.smithi029.stdout:3000: finishing write tid 5 to smithi02979277-9 2024-04-12T11:27:05.675 INFO:tasks.rados.rados.0.smithi029.stdout:3000: oid 9 updating version 0 to 1035 2024-04-12T11:27:05.675 INFO:tasks.rados.rados.0.smithi029.stdout:3000: oid 9 version 1035 is already newer than 1033 2024-04-12T11:27:05.676 INFO:tasks.rados.rados.0.smithi029.stdout:3000: oid 9 version 1035 is already newer than 1034 2024-04-12T11:27:05.676 INFO:tasks.rados.rados.0.smithi029.stdout:3000: oid 9 version 1035 is already newer than 1032 2024-04-12T11:27:05.676 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 1035 (ObjNum 1132 snap 379 seq_num 1132) dirty exists 2024-04-12T11:27:05.676 INFO:tasks.rados.rados.0.smithi029.stdout:3000: left oid 9 (ObjNum 1132 snap 379 seq_num 1132) 2024-04-12T11:27:05.676 INFO:tasks.rados.rados.0.smithi029.stdout:3001: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:27:05.676 INFO:tasks.rados.rados.0.smithi029.stdout:3001: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:27:05.676 INFO:tasks.rados.rados.0.smithi029.stdout:3001: finishing rollback tid 2 to smithi02979277-37 2024-04-12T11:27:05.676 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 1039 (ObjNum 1125 snap 376 seq_num 1125) dirty exists 2024-04-12T11:27:05.683 INFO:tasks.rados.rados.0.smithi029.stdout:3005: expect (ObjNum 975 snap 340 seq_num 975) 2024-04-12T11:27:05.766 INFO:tasks.rados.rados.0.smithi029.stdout:3006: expect (ObjNum 872 snap 307 seq_num 872) 2024-04-12T11:27:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:05 smithi087 ceph-mon[89014]: osdmap e820: 8 total, 8 up, 8 in 2024-04-12T11:27:05.984 INFO:tasks.rados.rados.0.smithi029.stdout:3000: done (7 left) 2024-04-12T11:27:05.984 INFO:tasks.rados.rados.0.smithi029.stdout:3001: done (6 left) 2024-04-12T11:27:05.984 INFO:tasks.rados.rados.0.smithi029.stdout:3002: done (5 left) 2024-04-12T11:27:05.985 INFO:tasks.rados.rados.0.smithi029.stdout:3003: done (4 left) 2024-04-12T11:27:05.985 INFO:tasks.rados.rados.0.smithi029.stdout:3005: done (3 left) 2024-04-12T11:27:05.985 INFO:tasks.rados.rados.0.smithi029.stdout:3006: done (2 left) 2024-04-12T11:27:05.985 INFO:tasks.rados.rados.0.smithi029.stdout:3007: done (1 left) 2024-04-12T11:27:05.985 INFO:tasks.rados.rados.0.smithi029.stdout:3008: done (0 left) 2024-04-12T11:27:05.985 INFO:tasks.rados.rados.0.smithi029.stdout:3009: snap_create 2024-04-12T11:27:06.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:05 smithi029 ceph-mon[93650]: osdmap e820: 8 total, 8 up, 8 in 2024-04-12T11:27:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:05 smithi029 ceph-mon[97787]: osdmap e820: 8 total, 8 up, 8 in 2024-04-12T11:27:06.673 INFO:tasks.rados.rados.0.smithi029.stdout:3009: done (0 left) 2024-04-12T11:27:06.674 INFO:tasks.rados.rados.0.smithi029.stdout:3010: rollback oid 13 current snap is 380 2024-04-12T11:27:06.674 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 13 to 366 2024-04-12T11:27:06.674 INFO:tasks.rados.rados.0.smithi029.stdout:3011: snap_create 2024-04-12T11:27:06.678 INFO:tasks.rados.rados.0.smithi029.stdout:3010: finishing rollback tid 1 to smithi02979277-13 2024-04-12T11:27:06.678 INFO:tasks.rados.rados.0.smithi029.stdout:3010: finishing rollback tid 2 to smithi02979277-13 2024-04-12T11:27:06.678 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1492 (ObjNum 1086 snap 365 seq_num 1086) dirty exists 2024-04-12T11:27:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:06 smithi087 ceph-mon[89014]: pgmap v867: 89 pgs: 89 active+clean; 260 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-04-12T11:27:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:06 smithi087 ceph-mon[89014]: osdmap e821: 8 total, 8 up, 8 in 2024-04-12T11:27:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:06 smithi029 ceph-mon[97787]: pgmap v867: 89 pgs: 89 active+clean; 260 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-04-12T11:27:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:06 smithi029 ceph-mon[97787]: osdmap e821: 8 total, 8 up, 8 in 2024-04-12T11:27:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:06 smithi029 ceph-mon[93650]: pgmap v867: 89 pgs: 89 active+clean; 260 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-04-12T11:27:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:06 smithi029 ceph-mon[93650]: osdmap e821: 8 total, 8 up, 8 in 2024-04-12T11:27:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:07.650 INFO:tasks.rados.rados.0.smithi029.stdout:3010: done (1 left) 2024-04-12T11:27:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:3011: done (0 left) 2024-04-12T11:27:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:3012: read oid 9 snap -1 2024-04-12T11:27:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:3012: expect (ObjNum 1132 snap 379 seq_num 1132) 2024-04-12T11:27:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:3013: write oid 11 current snap is 381 2024-04-12T11:27:07.651 INFO:tasks.rados.rados.0.smithi029.stdout:3013: seq_num 1134 ranges {659138=668963,2109051=611732} 2024-04-12T11:27:07.658 INFO:tasks.rados.rados.0.smithi029.stdout:3013: writing smithi02979277-11 from 659138 to 1328101 tid 1 2024-04-12T11:27:07.667 INFO:tasks.rados.rados.0.smithi029.stdout:3013: writing smithi02979277-11 from 2109051 to 2720783 tid 2 2024-04-12T11:27:07.667 INFO:tasks.rados.rados.0.smithi029.stdout:3014: snap_remove snap 375 2024-04-12T11:27:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:07 smithi087 ceph-mon[89014]: osdmap e822: 8 total, 8 up, 8 in 2024-04-12T11:27:07.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:07 smithi087 ceph-mon[89014]: osdmap e823: 8 total, 8 up, 8 in 2024-04-12T11:27:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:07 smithi029 ceph-mon[93650]: osdmap e822: 8 total, 8 up, 8 in 2024-04-12T11:27:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:07 smithi029 ceph-mon[93650]: osdmap e823: 8 total, 8 up, 8 in 2024-04-12T11:27:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:07 smithi029 ceph-mon[97787]: osdmap e822: 8 total, 8 up, 8 in 2024-04-12T11:27:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:07 smithi029 ceph-mon[97787]: osdmap e823: 8 total, 8 up, 8 in 2024-04-12T11:27:08.658 INFO:tasks.rados.rados.0.smithi029.stdout:3014: done (2 left) 2024-04-12T11:27:08.658 INFO:tasks.rados.rados.0.smithi029.stdout:3015: write oid 27 current snap is 381 2024-04-12T11:27:08.658 INFO:tasks.rados.rados.0.smithi029.stdout:3012: expect (ObjNum 1132 snap 379 seq_num 1132) 2024-04-12T11:27:08.764 INFO:tasks.rados.rados.0.smithi029.stdout:3013: finishing write tid 1 to smithi02979277-11 2024-04-12T11:27:08.764 INFO:tasks.rados.rados.0.smithi029.stdout:3013: finishing write tid 2 to smithi02979277-11 2024-04-12T11:27:08.764 INFO:tasks.rados.rados.0.smithi029.stdout:3015: seq_num 1135 ranges {675945=744212,2167533=755390} 2024-04-12T11:27:08.772 INFO:tasks.rados.rados.0.smithi029.stdout:3015: writing smithi02979277-27 from 675945 to 1420157 tid 1 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3015: writing smithi02979277-27 from 2167533 to 2922923 tid 2 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3012: done (2 left) 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3016: write oid 31 current snap is 381 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3013: finishing write tid 3 to smithi02979277-11 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3013: finishing write tid 4 to smithi02979277-11 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3013: oid 11 updating version 0 to 1865 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3013: oid 11 version 1865 is already newer than 1864 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3013: oid 11 version 1865 is already newer than 1863 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 1865 (ObjNum 1134 snap 381 seq_num 1134) dirty exists 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3013: left oid 11 (ObjNum 1134 snap 381 seq_num 1134) 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3015: finishing write tid 1 to smithi02979277-27 2024-04-12T11:27:08.781 INFO:tasks.rados.rados.0.smithi029.stdout:3016: seq_num 1136 ranges {692752=419461,1826015=615401,2979461=113291} 2024-04-12T11:27:08.786 INFO:tasks.rados.rados.0.smithi029.stdout:3016: writing smithi02979277-31 from 692752 to 1112213 tid 1 2024-04-12T11:27:08.794 INFO:tasks.rados.rados.0.smithi029.stdout:3016: writing smithi02979277-31 from 1826015 to 2441416 tid 2 2024-04-12T11:27:08.798 INFO:tasks.rados.rados.0.smithi029.stdout:3016: writing smithi02979277-31 from 2979461 to 3092752 tid 3 2024-04-12T11:27:08.798 INFO:tasks.rados.rados.0.smithi029.stdout:3013: done (2 left) 2024-04-12T11:27:08.798 INFO:tasks.rados.rados.0.smithi029.stdout:3017: read oid 42 snap -1 2024-04-12T11:27:08.798 INFO:tasks.rados.rados.0.smithi029.stdout:3017: expect (ObjNum 1129 snap 379 seq_num 1129) 2024-04-12T11:27:08.798 INFO:tasks.rados.rados.0.smithi029.stdout:3018: snap_remove snap 378 2024-04-12T11:27:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:08 smithi087 ceph-mon[89014]: pgmap v870: 89 pgs: 89 active+clean; 261 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 665 KiB/s wr, 4 op/s 2024-04-12T11:27:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:08 smithi087 ceph-mon[89014]: osdmap e824: 8 total, 8 up, 8 in 2024-04-12T11:27:09.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:08 smithi029 ceph-mon[93650]: pgmap v870: 89 pgs: 89 active+clean; 261 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 665 KiB/s wr, 4 op/s 2024-04-12T11:27:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:08 smithi029 ceph-mon[93650]: osdmap e824: 8 total, 8 up, 8 in 2024-04-12T11:27:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:08 smithi029 ceph-mon[97787]: pgmap v870: 89 pgs: 89 active+clean; 261 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 665 KiB/s wr, 4 op/s 2024-04-12T11:27:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:08 smithi029 ceph-mon[97787]: osdmap e824: 8 total, 8 up, 8 in 2024-04-12T11:27:09.690 INFO:tasks.rados.rados.0.smithi029.stdout:3018: done (3 left) 2024-04-12T11:27:09.690 INFO:tasks.rados.rados.0.smithi029.stdout:3019: snap_create 2024-04-12T11:27:09.690 INFO:tasks.rados.rados.0.smithi029.stdout:3015: finishing write tid 2 to smithi02979277-27 2024-04-12T11:27:09.690 INFO:tasks.rados.rados.0.smithi029.stdout:3015: finishing write tid 3 to smithi02979277-27 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3015: finishing write tid 4 to smithi02979277-27 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3015: oid 27 updating version 0 to 950 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3015: oid 27 updating version 950 to 951 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3015: oid 27 version 951 is already newer than 949 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 951 (ObjNum 1135 snap 381 seq_num 1135) dirty exists 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3015: left oid 27 (ObjNum 1135 snap 381 seq_num 1135) 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3016: finishing write tid 1 to smithi02979277-31 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3016: finishing write tid 2 to smithi02979277-31 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3016: finishing write tid 3 to smithi02979277-31 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3016: finishing write tid 4 to smithi02979277-31 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3016: finishing write tid 5 to smithi02979277-31 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3016: oid 31 updating version 0 to 1111 2024-04-12T11:27:09.691 INFO:tasks.rados.rados.0.smithi029.stdout:3016: oid 31 updating version 1111 to 1113 2024-04-12T11:27:09.692 INFO:tasks.rados.rados.0.smithi029.stdout:3016: oid 31 version 1113 is already newer than 1112 2024-04-12T11:27:09.692 INFO:tasks.rados.rados.0.smithi029.stdout:3016: oid 31 version 1113 is already newer than 1110 2024-04-12T11:27:09.692 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1113 (ObjNum 1136 snap 381 seq_num 1136) dirty exists 2024-04-12T11:27:09.692 INFO:tasks.rados.rados.0.smithi029.stdout:3016: left oid 31 (ObjNum 1136 snap 381 seq_num 1136) 2024-04-12T11:27:09.692 INFO:tasks.rados.rados.0.smithi029.stdout:3017: expect (ObjNum 1129 snap 379 seq_num 1129) 2024-04-12T11:27:10.692 INFO:tasks.rados.rados.0.smithi029.stdout:3015: done (3 left) 2024-04-12T11:27:10.693 INFO:tasks.rados.rados.0.smithi029.stdout:3016: done (2 left) 2024-04-12T11:27:10.693 INFO:tasks.rados.rados.0.smithi029.stdout:3017: done (1 left) 2024-04-12T11:27:10.693 INFO:tasks.rados.rados.0.smithi029.stdout:3019: done (0 left) 2024-04-12T11:27:10.693 INFO:tasks.rados.rados.0.smithi029.stdout:3020: write oid 49 current snap is 382 2024-04-12T11:27:10.693 INFO:tasks.rados.rados.0.smithi029.stdout:3020: seq_num 1137 ranges {709559=494710,1884497=759059} 2024-04-12T11:27:10.699 INFO:tasks.rados.rados.0.smithi029.stdout:3020: writing smithi02979277-49 from 709559 to 1204269 tid 1 2024-04-12T11:27:10.707 INFO:tasks.rados.rados.0.smithi029.stdout:3020: writing smithi02979277-49 from 1884497 to 2643556 tid 2 2024-04-12T11:27:10.707 INFO:tasks.rados.rados.0.smithi029.stdout:3021: read oid 38 snap -1 2024-04-12T11:27:10.708 INFO:tasks.rados.rados.0.smithi029.stdout:3021: expect (ObjNum 1079 snap 365 seq_num 1079) 2024-04-12T11:27:10.708 INFO:tasks.rados.rados.0.smithi029.stdout:3022: rollback oid 21 current snap is 382 2024-04-12T11:27:10.708 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 21 to 369 2024-04-12T11:27:10.708 INFO:tasks.rados.rados.0.smithi029.stdout:3020: finishing write tid 1 to smithi02979277-49 2024-04-12T11:27:10.708 INFO:tasks.rados.rados.0.smithi029.stdout:3023: snap_create 2024-04-12T11:27:10.711 INFO:tasks.rados.rados.0.smithi029.stdout:3021: expect (ObjNum 1079 snap 365 seq_num 1079) 2024-04-12T11:27:10.839 INFO:tasks.rados.rados.0.smithi029.stdout:3020: finishing write tid 2 to smithi02979277-49 2024-04-12T11:27:10.839 INFO:tasks.rados.rados.0.smithi029.stdout:3020: finishing write tid 3 to smithi02979277-49 2024-04-12T11:27:10.839 INFO:tasks.rados.rados.0.smithi029.stdout:3020: finishing write tid 4 to smithi02979277-49 2024-04-12T11:27:10.839 INFO:tasks.rados.rados.0.smithi029.stdout:3020: oid 49 updating version 0 to 1052 2024-04-12T11:27:10.839 INFO:tasks.rados.rados.0.smithi029.stdout:3020: oid 49 version 1052 is already newer than 1051 2024-04-12T11:27:10.839 INFO:tasks.rados.rados.0.smithi029.stdout:3020: oid 49 version 1052 is already newer than 1050 2024-04-12T11:27:10.839 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 1052 (ObjNum 1137 snap 382 seq_num 1137) dirty exists 2024-04-12T11:27:10.839 INFO:tasks.rados.rados.0.smithi029.stdout:3020: left oid 49 (ObjNum 1137 snap 382 seq_num 1137) 2024-04-12T11:27:10.839 INFO:tasks.rados.rados.0.smithi029.stdout:3022: finishing rollback tid 0 to smithi02979277-21 2024-04-12T11:27:10.840 INFO:tasks.rados.rados.0.smithi029.stdout:3022: finishing rollback tid 1 to smithi02979277-21 2024-04-12T11:27:10.840 INFO:tasks.rados.rados.0.smithi029.stdout:3022: finishing rollback tid 2 to smithi02979277-21 2024-04-12T11:27:10.840 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1438 (ObjNum 941 snap 325 seq_num 941) dirty exists 2024-04-12T11:27:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:10 smithi087 ceph-mon[89014]: pgmap v873: 89 pgs: 89 active+clean; 259 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-04-12T11:27:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:10 smithi087 ceph-mon[89014]: osdmap e825: 8 total, 8 up, 8 in 2024-04-12T11:27:11.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:10 smithi029 ceph-mon[93650]: pgmap v873: 89 pgs: 89 active+clean; 259 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-04-12T11:27:11.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:10 smithi029 ceph-mon[93650]: osdmap e825: 8 total, 8 up, 8 in 2024-04-12T11:27:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:10 smithi029 ceph-mon[97787]: pgmap v873: 89 pgs: 89 active+clean; 259 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-04-12T11:27:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:10 smithi029 ceph-mon[97787]: osdmap e825: 8 total, 8 up, 8 in 2024-04-12T11:27:11.702 INFO:tasks.rados.rados.0.smithi029.stdout:3020: done (3 left) 2024-04-12T11:27:11.702 INFO:tasks.rados.rados.0.smithi029.stdout:3021: done (2 left) 2024-04-12T11:27:11.702 INFO:tasks.rados.rados.0.smithi029.stdout:3022: done (1 left) 2024-04-12T11:27:11.702 INFO:tasks.rados.rados.0.smithi029.stdout:3023: done (0 left) 2024-04-12T11:27:11.702 INFO:tasks.rados.rados.0.smithi029.stdout:3024: rollback oid 10 current snap is 383 2024-04-12T11:27:11.702 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 10 to 380 2024-04-12T11:27:11.702 INFO:tasks.rados.rados.0.smithi029.stdout:3025: snap_remove snap 377 2024-04-12T11:27:11.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:11 smithi087 ceph-mon[89014]: osdmap e826: 8 total, 8 up, 8 in 2024-04-12T11:27:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:11 smithi029 ceph-mon[93650]: osdmap e826: 8 total, 8 up, 8 in 2024-04-12T11:27:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:11 smithi029 ceph-mon[97787]: osdmap e826: 8 total, 8 up, 8 in 2024-04-12T11:27:12.652 INFO:tasks.rados.rados.0.smithi029.stdout:3025: done (1 left) 2024-04-12T11:27:12.653 INFO:tasks.rados.rados.0.smithi029.stdout:3026: read oid 47 snap -1 2024-04-12T11:27:12.653 INFO:tasks.rados.rados.0.smithi029.stdout:3026: expect (ObjNum 1117 snap 372 seq_num 1117) 2024-04-12T11:27:12.653 INFO:tasks.rados.rados.0.smithi029.stdout:3027: read oid 31 snap 341 2024-04-12T11:27:12.653 INFO:tasks.rados.rados.0.smithi029.stdout:3027: expect deleted 2024-04-12T11:27:12.653 INFO:tasks.rados.rados.0.smithi029.stdout:3028: write oid 20 current snap is 383 2024-04-12T11:27:12.653 INFO:tasks.rados.rados.0.smithi029.stdout:3028: seq_num 1138 ranges {726366=569959,1826626=502717,2801601=324765} 2024-04-12T11:27:12.660 INFO:tasks.rados.rados.0.smithi029.stdout:3028: writing smithi02979277-20 from 726366 to 1296325 tid 1 2024-04-12T11:27:12.666 INFO:tasks.rados.rados.0.smithi029.stdout:3028: writing smithi02979277-20 from 1826626 to 2329343 tid 2 2024-04-12T11:27:12.671 INFO:tasks.rados.rados.0.smithi029.stdout:3028: writing smithi02979277-20 from 2801601 to 3126366 tid 3 2024-04-12T11:27:12.671 INFO:tasks.rados.rados.0.smithi029.stdout:3029: rollback oid 8 current snap is 383 2024-04-12T11:27:12.671 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 374 2024-04-12T11:27:12.671 INFO:tasks.rados.rados.0.smithi029.stdout:3024: finishing rollback tid 0 to smithi02979277-10 2024-04-12T11:27:12.672 INFO:tasks.rados.rados.0.smithi029.stdout:3024: finishing rollback tid 1 to smithi02979277-10 2024-04-12T11:27:12.672 INFO:tasks.rados.rados.0.smithi029.stdout:3024: finishing rollback tid 2 to smithi02979277-10 2024-04-12T11:27:12.672 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 1119 (ObjNum 975 snap 340 seq_num 975) dirty exists 2024-04-12T11:27:12.672 INFO:tasks.rados.rados.0.smithi029.stdout:3024: done (4 left) 2024-04-12T11:27:12.672 INFO:tasks.rados.rados.0.smithi029.stdout:3030: rollback oid 2 current snap is 383 2024-04-12T11:27:12.672 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 2 to 362 2024-04-12T11:27:12.672 INFO:tasks.rados.rados.0.smithi029.stdout:3026: expect (ObjNum 1117 snap 372 seq_num 1117) 2024-04-12T11:27:12.699 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:27:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:27:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:3028: finishing write tid 1 to smithi02979277-20 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:3028: finishing write tid 2 to smithi02979277-20 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:3029: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 326 (ObjNum 225 snap 0 seq_num 1800669456) dirty dne 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:3030: finishing rollback tid 1 to smithi02979277-2 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:3026: done (4 left) 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:3027: done (3 left) 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:3029: done (2 left) 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:3031: delete oid 50 current snap is 383 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:3030: finishing rollback tid 2 to smithi02979277-2 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 332 (ObjNum 1032 snap 354 seq_num 1032) dirty exists 2024-04-12T11:27:12.768 INFO:tasks.rados.rados.0.smithi029.stdout:3028: finishing write tid 3 to smithi02979277-20 2024-04-12T11:27:12.769 INFO:tasks.rados.rados.0.smithi029.stdout:3028: finishing write tid 4 to smithi02979277-20 2024-04-12T11:27:12.769 INFO:tasks.rados.rados.0.smithi029.stdout:3028: finishing write tid 5 to smithi02979277-20 2024-04-12T11:27:12.769 INFO:tasks.rados.rados.0.smithi029.stdout:3028: oid 20 updating version 0 to 907 2024-04-12T11:27:12.769 INFO:tasks.rados.rados.0.smithi029.stdout:3028: oid 20 updating version 907 to 908 2024-04-12T11:27:12.769 INFO:tasks.rados.rados.0.smithi029.stdout:3028: oid 20 updating version 908 to 912 2024-04-12T11:27:12.769 INFO:tasks.rados.rados.0.smithi029.stdout:3028: oid 20 version 912 is already newer than 911 2024-04-12T11:27:12.769 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 912 (ObjNum 1138 snap 383 seq_num 1138) dirty exists 2024-04-12T11:27:12.769 INFO:tasks.rados.rados.0.smithi029.stdout:3028: left oid 20 (ObjNum 1138 snap 383 seq_num 1138) 2024-04-12T11:27:12.770 INFO:tasks.rados.rados.0.smithi029.stdout:3028: done (2 left) 2024-04-12T11:27:12.770 INFO:tasks.rados.rados.0.smithi029.stdout:3030: done (1 left) 2024-04-12T11:27:12.770 INFO:tasks.rados.rados.0.smithi029.stdout:3031: done (0 left) 2024-04-12T11:27:12.770 INFO:tasks.rados.rados.0.smithi029.stdout:3032: read oid 29 snap -1 2024-04-12T11:27:12.770 INFO:tasks.rados.rados.0.smithi029.stdout:3032: expect (ObjNum 1120 snap 374 seq_num 1120) 2024-04-12T11:27:12.770 INFO:tasks.rados.rados.0.smithi029.stdout:3033: write oid 23 current snap is 383 2024-04-12T11:27:12.770 INFO:tasks.rados.rados.0.smithi029.stdout:3033: seq_num 1140 ranges {759980=720457,1943590=506386,2856447=303533} 2024-04-12T11:27:12.778 INFO:tasks.rados.rados.0.smithi029.stdout:3033: writing smithi02979277-23 from 759980 to 1480437 tid 1 2024-04-12T11:27:12.783 INFO:tasks.rados.rados.0.smithi029.stdout:3033: writing smithi02979277-23 from 1943590 to 2449976 tid 2 2024-04-12T11:27:12.787 INFO:tasks.rados.rados.0.smithi029.stdout:3033: writing smithi02979277-23 from 2856447 to 3159980 tid 3 2024-04-12T11:27:12.787 INFO:tasks.rados.rados.0.smithi029.stdout:3034: read oid 15 snap -1 2024-04-12T11:27:12.787 INFO:tasks.rados.rados.0.smithi029.stdout:3034: expect deleted 2024-04-12T11:27:12.787 INFO:tasks.rados.rados.0.smithi029.stdout:3035: snap_create 2024-04-12T11:27:12.787 INFO:tasks.rados.rados.0.smithi029.stdout:3032: expect (ObjNum 1120 snap 374 seq_num 1120) 2024-04-12T11:27:12.880 INFO:tasks.rados.rados.0.smithi029.stdout:3033: finishing write tid 1 to smithi02979277-23 2024-04-12T11:27:12.880 INFO:tasks.rados.rados.0.smithi029.stdout:3033: finishing write tid 2 to smithi02979277-23 2024-04-12T11:27:12.880 INFO:tasks.rados.rados.0.smithi029.stdout:3033: finishing write tid 3 to smithi02979277-23 2024-04-12T11:27:12.880 INFO:tasks.rados.rados.0.smithi029.stdout:3033: finishing write tid 4 to smithi02979277-23 2024-04-12T11:27:12.881 INFO:tasks.rados.rados.0.smithi029.stdout:3033: finishing write tid 5 to smithi02979277-23 2024-04-12T11:27:12.881 INFO:tasks.rados.rados.0.smithi029.stdout:3033: oid 23 updating version 0 to 1234 2024-04-12T11:27:12.881 INFO:tasks.rados.rados.0.smithi029.stdout:3033: oid 23 updating version 1234 to 1236 2024-04-12T11:27:12.881 INFO:tasks.rados.rados.0.smithi029.stdout:3033: oid 23 version 1236 is already newer than 1235 2024-04-12T11:27:12.881 INFO:tasks.rados.rados.0.smithi029.stdout:3033: oid 23 updating version 1236 to 1237 2024-04-12T11:27:12.881 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1237 (ObjNum 1140 snap 383 seq_num 1140) dirty exists 2024-04-12T11:27:12.881 INFO:tasks.rados.rados.0.smithi029.stdout:3033: left oid 23 (ObjNum 1140 snap 383 seq_num 1140) 2024-04-12T11:27:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:12 smithi087 ceph-mon[89014]: pgmap v876: 89 pgs: 89 active+clean; 261 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-04-12T11:27:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:12 smithi087 ceph-mon[89014]: osdmap e827: 8 total, 8 up, 8 in 2024-04-12T11:27:12.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:12 smithi087 ceph-mon[89014]: osdmap e828: 8 total, 8 up, 8 in 2024-04-12T11:27:13.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:12 smithi029 ceph-mon[93650]: pgmap v876: 89 pgs: 89 active+clean; 261 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-04-12T11:27:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:12 smithi029 ceph-mon[93650]: osdmap e827: 8 total, 8 up, 8 in 2024-04-12T11:27:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:12 smithi029 ceph-mon[93650]: osdmap e828: 8 total, 8 up, 8 in 2024-04-12T11:27:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:12 smithi029 ceph-mon[97787]: pgmap v876: 89 pgs: 89 active+clean; 261 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-04-12T11:27:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:12 smithi029 ceph-mon[97787]: osdmap e827: 8 total, 8 up, 8 in 2024-04-12T11:27:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:12 smithi029 ceph-mon[97787]: osdmap e828: 8 total, 8 up, 8 in 2024-04-12T11:27:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:3032: done (3 left) 2024-04-12T11:27:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:3033: done (2 left) 2024-04-12T11:27:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:3034: done (1 left) 2024-04-12T11:27:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:3035: done (0 left) 2024-04-12T11:27:13.708 INFO:tasks.rados.rados.0.smithi029.stdout:3036: snap_create 2024-04-12T11:27:14.710 INFO:tasks.rados.rados.0.smithi029.stdout:3036: done (0 left) 2024-04-12T11:27:14.710 INFO:tasks.rados.rados.0.smithi029.stdout:3037: read oid 22 snap -1 2024-04-12T11:27:14.710 INFO:tasks.rados.rados.0.smithi029.stdout:3037: expect (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:27:14.710 INFO:tasks.rados.rados.0.smithi029.stdout:3038: snap_create 2024-04-12T11:27:14.726 INFO:tasks.rados.rados.0.smithi029.stdout:3037: expect (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:27:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:14 smithi087 ceph-mon[89014]: pgmap v879: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 267 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 995 KiB/s wr, 12 op/s 2024-04-12T11:27:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:14 smithi087 ceph-mon[89014]: osdmap e829: 8 total, 8 up, 8 in 2024-04-12T11:27:15.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:14 smithi029 ceph-mon[93650]: pgmap v879: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 267 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 995 KiB/s wr, 12 op/s 2024-04-12T11:27:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:14 smithi029 ceph-mon[93650]: osdmap e829: 8 total, 8 up, 8 in 2024-04-12T11:27:15.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:27:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:27:14] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:27:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:14 smithi029 ceph-mon[97787]: pgmap v879: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 267 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 995 KiB/s wr, 12 op/s 2024-04-12T11:27:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:14 smithi029 ceph-mon[97787]: osdmap e829: 8 total, 8 up, 8 in 2024-04-12T11:27:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3037: done (1 left) 2024-04-12T11:27:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3038: done (0 left) 2024-04-12T11:27:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3039: read oid 29 snap -1 2024-04-12T11:27:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3039: expect (ObjNum 1120 snap 374 seq_num 1120) 2024-04-12T11:27:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3040: snap_create 2024-04-12T11:27:15.726 INFO:tasks.rados.rados.0.smithi029.stdout:3039: expect (ObjNum 1120 snap 374 seq_num 1120) 2024-04-12T11:27:16.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:15 smithi029 ceph-mon[93650]: osdmap e830: 8 total, 8 up, 8 in 2024-04-12T11:27:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:15 smithi029 ceph-mon[97787]: osdmap e830: 8 total, 8 up, 8 in 2024-04-12T11:27:16.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:15 smithi087 ceph-mon[89014]: osdmap e830: 8 total, 8 up, 8 in 2024-04-12T11:27:16.721 INFO:tasks.rados.rados.0.smithi029.stdout:3039: done (1 left) 2024-04-12T11:27:16.721 INFO:tasks.rados.rados.0.smithi029.stdout:3040: done (0 left) 2024-04-12T11:27:16.721 INFO:tasks.rados.rados.0.smithi029.stdout:3041: write oid 19 current snap is 387 2024-04-12T11:27:16.721 INFO:tasks.rados.rados.0.smithi029.stdout:3041: seq_num 1141 ranges {776787=512059,1718425=766397,3000223=176564} 2024-04-12T11:27:16.727 INFO:tasks.rados.rados.0.smithi029.stdout:3041: writing smithi02979277-19 from 776787 to 1288846 tid 1 2024-04-12T11:27:16.735 INFO:tasks.rados.rados.0.smithi029.stdout:3041: writing smithi02979277-19 from 1718425 to 2484822 tid 2 2024-04-12T11:27:16.739 INFO:tasks.rados.rados.0.smithi029.stdout:3041: writing smithi02979277-19 from 3000223 to 3176787 tid 3 2024-04-12T11:27:16.740 INFO:tasks.rados.rados.0.smithi029.stdout:3042: delete oid 35 current snap is 387 2024-04-12T11:27:16.740 INFO:tasks.rados.rados.0.smithi029.stdout:3041: finishing write tid 1 to smithi02979277-19 2024-04-12T11:27:16.742 INFO:tasks.rados.rados.0.smithi029.stdout:3041: finishing write tid 2 to smithi02979277-19 2024-04-12T11:27:16.743 INFO:tasks.rados.rados.0.smithi029.stdout:3041: finishing write tid 3 to smithi02979277-19 2024-04-12T11:27:16.743 INFO:tasks.rados.rados.0.smithi029.stdout:3041: finishing write tid 4 to smithi02979277-19 2024-04-12T11:27:16.743 INFO:tasks.rados.rados.0.smithi029.stdout:3041: finishing write tid 5 to smithi02979277-19 2024-04-12T11:27:16.743 INFO:tasks.rados.rados.0.smithi029.stdout:3041: oid 19 updating version 0 to 1371 2024-04-12T11:27:16.743 INFO:tasks.rados.rados.0.smithi029.stdout:3041: oid 19 version 1371 is already newer than 1370 2024-04-12T11:27:16.743 INFO:tasks.rados.rados.0.smithi029.stdout:3041: oid 19 updating version 1371 to 1372 2024-04-12T11:27:16.743 INFO:tasks.rados.rados.0.smithi029.stdout:3041: oid 19 version 1372 is already newer than 1369 2024-04-12T11:27:16.743 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1372 (ObjNum 1141 snap 387 seq_num 1141) dirty exists 2024-04-12T11:27:16.743 INFO:tasks.rados.rados.0.smithi029.stdout:3041: left oid 19 (ObjNum 1141 snap 387 seq_num 1141) 2024-04-12T11:27:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3041: done (1 left) 2024-04-12T11:27:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3042: done (0 left) 2024-04-12T11:27:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3043: write oid 25 current snap is 387 2024-04-12T11:27:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3043: seq_num 1143 ranges {410401=662557,1719036=770066,2938716=271685} 2024-04-12T11:27:16.751 INFO:tasks.rados.rados.0.smithi029.stdout:3043: writing smithi02979277-25 from 410401 to 1072958 tid 1 2024-04-12T11:27:16.759 INFO:tasks.rados.rados.0.smithi029.stdout:3043: writing smithi02979277-25 from 1719036 to 2489102 tid 2 2024-04-12T11:27:16.763 INFO:tasks.rados.rados.0.smithi029.stdout:3043: writing smithi02979277-25 from 2938716 to 3210401 tid 3 2024-04-12T11:27:16.763 INFO:tasks.rados.rados.0.smithi029.stdout:3044: write oid 38 current snap is 387 2024-04-12T11:27:16.763 INFO:tasks.rados.rados.0.smithi029.stdout:3043: finishing write tid 1 to smithi02979277-25 2024-04-12T11:27:16.763 INFO:tasks.rados.rados.0.smithi029.stdout:3044: seq_num 1144 ranges {427208=737806,1777518=513724,2966139=261069} 2024-04-12T11:27:16.769 INFO:tasks.rados.rados.0.smithi029.stdout:3044: writing smithi02979277-38 from 427208 to 1165014 tid 1 2024-04-12T11:27:16.776 INFO:tasks.rados.rados.0.smithi029.stdout:3044: writing smithi02979277-38 from 1777518 to 2291242 tid 2 2024-04-12T11:27:16.781 INFO:tasks.rados.rados.0.smithi029.stdout:3044: writing smithi02979277-38 from 2966139 to 3227208 tid 3 2024-04-12T11:27:16.781 INFO:tasks.rados.rados.0.smithi029.stdout:3045: snap_create 2024-04-12T11:27:16.781 INFO:tasks.rados.rados.0.smithi029.stdout:3043: finishing write tid 2 to smithi02979277-25 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3043: finishing write tid 3 to smithi02979277-25 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3043: finishing write tid 4 to smithi02979277-25 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3043: finishing write tid 5 to smithi02979277-25 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3043: oid 25 updating version 0 to 1501 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3043: oid 25 version 1501 is already newer than 1499 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3043: oid 25 updating version 1501 to 1502 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3043: oid 25 version 1502 is already newer than 1500 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 1502 (ObjNum 1143 snap 387 seq_num 1143) dirty exists 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3043: left oid 25 (ObjNum 1143 snap 387 seq_num 1143) 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3044: finishing write tid 1 to smithi02979277-38 2024-04-12T11:27:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3044: finishing write tid 2 to smithi02979277-38 2024-04-12T11:27:16.784 INFO:tasks.rados.rados.0.smithi029.stdout:3044: finishing write tid 3 to smithi02979277-38 2024-04-12T11:27:16.784 INFO:tasks.rados.rados.0.smithi029.stdout:3044: finishing write tid 4 to smithi02979277-38 2024-04-12T11:27:16.784 INFO:tasks.rados.rados.0.smithi029.stdout:3044: finishing write tid 5 to smithi02979277-38 2024-04-12T11:27:16.784 INFO:tasks.rados.rados.0.smithi029.stdout:3044: oid 38 updating version 0 to 916 2024-04-12T11:27:16.784 INFO:tasks.rados.rados.0.smithi029.stdout:3044: oid 38 updating version 916 to 917 2024-04-12T11:27:16.784 INFO:tasks.rados.rados.0.smithi029.stdout:3044: oid 38 version 917 is already newer than 914 2024-04-12T11:27:16.784 INFO:tasks.rados.rados.0.smithi029.stdout:3044: oid 38 version 917 is already newer than 915 2024-04-12T11:27:16.784 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 917 (ObjNum 1144 snap 387 seq_num 1144) dirty exists 2024-04-12T11:27:16.784 INFO:tasks.rados.rados.0.smithi029.stdout:3044: left oid 38 (ObjNum 1144 snap 387 seq_num 1144) 2024-04-12T11:27:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:16 smithi029 ceph-mon[93650]: pgmap v882: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 271 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-04-12T11:27:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:16 smithi029 ceph-mon[93650]: osdmap e831: 8 total, 8 up, 8 in 2024-04-12T11:27:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:16 smithi029 ceph-mon[97787]: pgmap v882: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 271 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-04-12T11:27:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:16 smithi029 ceph-mon[97787]: osdmap e831: 8 total, 8 up, 8 in 2024-04-12T11:27:17.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:16 smithi087 ceph-mon[89014]: pgmap v882: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 271 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-04-12T11:27:17.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:16 smithi087 ceph-mon[89014]: osdmap e831: 8 total, 8 up, 8 in 2024-04-12T11:27:17.653 INFO:tasks.rados.rados.0.smithi029.stdout:3043: done (2 left) 2024-04-12T11:27:17.654 INFO:tasks.rados.rados.0.smithi029.stdout:3044: done (1 left) 2024-04-12T11:27:17.654 INFO:tasks.rados.rados.0.smithi029.stdout:3045: done (0 left) 2024-04-12T11:27:17.654 INFO:tasks.rados.rados.0.smithi029.stdout:3046: snap_create 2024-04-12T11:27:18.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:17 smithi029 ceph-mon[93650]: osdmap e832: 8 total, 8 up, 8 in 2024-04-12T11:27:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:17 smithi029 ceph-mon[93650]: osdmap e833: 8 total, 8 up, 8 in 2024-04-12T11:27:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:17 smithi029 ceph-mon[97787]: osdmap e832: 8 total, 8 up, 8 in 2024-04-12T11:27:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:17 smithi029 ceph-mon[97787]: osdmap e833: 8 total, 8 up, 8 in 2024-04-12T11:27:18.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:17 smithi087 ceph-mon[89014]: osdmap e832: 8 total, 8 up, 8 in 2024-04-12T11:27:18.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:17 smithi087 ceph-mon[89014]: osdmap e833: 8 total, 8 up, 8 in 2024-04-12T11:27:18.655 INFO:tasks.rados.rados.0.smithi029.stdout:3046: done (0 left) 2024-04-12T11:27:18.655 INFO:tasks.rados.rados.0.smithi029.stdout:3047: snap_create 2024-04-12T11:27:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:18 smithi029 ceph-mon[97787]: pgmap v885: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 279 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-04-12T11:27:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:18 smithi029 ceph-mon[97787]: osdmap e834: 8 total, 8 up, 8 in 2024-04-12T11:27:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:18 smithi029 ceph-mon[93650]: pgmap v885: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 279 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-04-12T11:27:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:18 smithi029 ceph-mon[93650]: osdmap e834: 8 total, 8 up, 8 in 2024-04-12T11:27:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:18 smithi087 ceph-mon[89014]: pgmap v885: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 279 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-04-12T11:27:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:18 smithi087 ceph-mon[89014]: osdmap e834: 8 total, 8 up, 8 in 2024-04-12T11:27:19.656 INFO:tasks.rados.rados.0.smithi029.stdout:3047: done (0 left) 2024-04-12T11:27:19.656 INFO:tasks.rados.rados.0.smithi029.stdout:3048: delete oid 36 current snap is 390 2024-04-12T11:27:19.658 INFO:tasks.rados.rados.0.smithi029.stdout:3048: done (0 left) 2024-04-12T11:27:19.658 INFO:tasks.rados.rados.0.smithi029.stdout:3049: read oid 20 snap -1 2024-04-12T11:27:19.658 INFO:tasks.rados.rados.0.smithi029.stdout:3049: expect (ObjNum 1138 snap 383 seq_num 1138) 2024-04-12T11:27:19.658 INFO:tasks.rados.rados.0.smithi029.stdout:3050: snap_remove snap 383 2024-04-12T11:27:20.661 INFO:tasks.rados.rados.0.smithi029.stdout:3050: done (1 left) 2024-04-12T11:27:20.661 INFO:tasks.rados.rados.0.smithi029.stdout:3051: snap_remove snap 384 2024-04-12T11:27:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:20 smithi087 ceph-mon[89014]: pgmap v888: 89 pgs: 89 active+clean; 281 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-04-12T11:27:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:20 smithi087 ceph-mon[89014]: osdmap e835: 8 total, 8 up, 8 in 2024-04-12T11:27:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:20 smithi029 ceph-mon[97787]: pgmap v888: 89 pgs: 89 active+clean; 281 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-04-12T11:27:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:20 smithi029 ceph-mon[97787]: osdmap e835: 8 total, 8 up, 8 in 2024-04-12T11:27:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:20 smithi029 ceph-mon[93650]: pgmap v888: 89 pgs: 89 active+clean; 281 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-04-12T11:27:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:20 smithi029 ceph-mon[93650]: osdmap e835: 8 total, 8 up, 8 in 2024-04-12T11:27:21.668 INFO:tasks.rados.rados.0.smithi029.stdout:3051: done (1 left) 2024-04-12T11:27:21.668 INFO:tasks.rados.rados.0.smithi029.stdout:3052: read oid 48 snap -1 2024-04-12T11:27:21.668 INFO:tasks.rados.rados.0.smithi029.stdout:3052: expect (ObjNum 1107 snap 370 seq_num 1107) 2024-04-12T11:27:21.668 INFO:tasks.rados.rados.0.smithi029.stdout:3053: write oid 11 current snap is 390 2024-04-12T11:27:21.668 INFO:tasks.rados.rados.0.smithi029.stdout:3053: seq_num 1146 ranges {460822=488304,1378129=517393,2504632=405962} 2024-04-12T11:27:21.673 INFO:tasks.rados.rados.0.smithi029.stdout:3053: writing smithi02979277-11 from 460822 to 949126 tid 1 2024-04-12T11:27:21.678 INFO:tasks.rados.rados.0.smithi029.stdout:3053: writing smithi02979277-11 from 1378129 to 1895522 tid 2 2024-04-12T11:27:21.684 INFO:tasks.rados.rados.0.smithi029.stdout:3053: writing smithi02979277-11 from 2504632 to 2910594 tid 3 2024-04-12T11:27:21.685 INFO:tasks.rados.rados.0.smithi029.stdout:3054: read oid 50 snap -1 2024-04-12T11:27:21.685 INFO:tasks.rados.rados.0.smithi029.stdout:3054: expect deleted 2024-04-12T11:27:21.685 INFO:tasks.rados.rados.0.smithi029.stdout:3055: delete oid 29 current snap is 390 2024-04-12T11:27:21.685 INFO:tasks.rados.rados.0.smithi029.stdout:3049: expect (ObjNum 1138 snap 383 seq_num 1138) 2024-04-12T11:27:21.899 INFO:tasks.rados.rados.0.smithi029.stdout:3052: expect (ObjNum 1107 snap 370 seq_num 1107) 2024-04-12T11:27:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:21 smithi087 ceph-mon[89014]: osdmap e836: 8 total, 8 up, 8 in 2024-04-12T11:27:21.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:22.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:21 smithi029 ceph-mon[97787]: osdmap e836: 8 total, 8 up, 8 in 2024-04-12T11:27:22.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:21 smithi029 ceph-mon[93650]: osdmap e836: 8 total, 8 up, 8 in 2024-04-12T11:27:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:22.095 INFO:tasks.rados.rados.0.smithi029.stdout:3053: finishing write tid 1 to smithi02979277-11 2024-04-12T11:27:22.095 INFO:tasks.rados.rados.0.smithi029.stdout:3053: finishing write tid 2 to smithi02979277-11 2024-04-12T11:27:22.095 INFO:tasks.rados.rados.0.smithi029.stdout:3049: done (4 left) 2024-04-12T11:27:22.095 INFO:tasks.rados.rados.0.smithi029.stdout:3052: done (3 left) 2024-04-12T11:27:22.095 INFO:tasks.rados.rados.0.smithi029.stdout:3055: done (2 left) 2024-04-12T11:27:22.095 INFO:tasks.rados.rados.0.smithi029.stdout:3056: read oid 31 snap -1 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:3056: expect (ObjNum 1136 snap 381 seq_num 1136) 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:3057: snap_create 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:3053: finishing write tid 3 to smithi02979277-11 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:3053: finishing write tid 4 to smithi02979277-11 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:3053: finishing write tid 5 to smithi02979277-11 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:3053: oid 11 updating version 0 to 1874 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:3053: oid 11 updating version 1874 to 1877 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:3053: oid 11 version 1877 is already newer than 1875 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:3053: oid 11 version 1877 is already newer than 1876 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 1877 (ObjNum 1146 snap 390 seq_num 1146) dirty exists 2024-04-12T11:27:22.096 INFO:tasks.rados.rados.0.smithi029.stdout:3053: left oid 11 (ObjNum 1146 snap 390 seq_num 1146) 2024-04-12T11:27:22.097 INFO:tasks.rados.rados.0.smithi029.stdout:3056: expect (ObjNum 1136 snap 381 seq_num 1136) 2024-04-12T11:27:22.654 INFO:tasks.rados.rados.0.smithi029.stdout:3053: done (3 left) 2024-04-12T11:27:22.655 INFO:tasks.rados.rados.0.smithi029.stdout:3054: done (2 left) 2024-04-12T11:27:22.655 INFO:tasks.rados.rados.0.smithi029.stdout:3056: done (1 left) 2024-04-12T11:27:22.655 INFO:tasks.rados.rados.0.smithi029.stdout:3057: done (0 left) 2024-04-12T11:27:22.655 INFO:tasks.rados.rados.0.smithi029.stdout:3058: rollback oid 39 current snap is 391 2024-04-12T11:27:22.655 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 368 2024-04-12T11:27:22.655 INFO:tasks.rados.rados.0.smithi029.stdout:3059: delete oid 40 current snap is 391 2024-04-12T11:27:22.660 INFO:tasks.rados.rados.0.smithi029.stdout:3058: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:27:22.660 INFO:tasks.rados.rados.0.smithi029.stdout:3058: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:27:22.661 INFO:tasks.rados.rados.0.smithi029.stdout:3058: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:27:22.661 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 409 (ObjNum 1088 snap 365 seq_num 1088) dirty exists 2024-04-12T11:27:22.662 INFO:tasks.rados.rados.0.smithi029.stdout:3058: done (1 left) 2024-04-12T11:27:22.662 INFO:tasks.rados.rados.0.smithi029.stdout:3059: done (0 left) 2024-04-12T11:27:22.662 INFO:tasks.rados.rados.0.smithi029.stdout:3060: write oid 14 current snap is 391 2024-04-12T11:27:22.662 INFO:tasks.rados.rados.0.smithi029.stdout:3060: seq_num 1149 ranges {511243=430404,1669928=664720,3103254=207989} 2024-04-12T11:27:22.664 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:27:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:27:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:27:22.667 INFO:tasks.rados.rados.0.smithi029.stdout:3060: writing smithi02979277-14 from 511243 to 941647 tid 1 2024-04-12T11:27:22.675 INFO:tasks.rados.rados.0.smithi029.stdout:3060: writing smithi02979277-14 from 1669928 to 2334648 tid 2 2024-04-12T11:27:22.681 INFO:tasks.rados.rados.0.smithi029.stdout:3060: writing smithi02979277-14 from 3103254 to 3311243 tid 3 2024-04-12T11:27:22.681 INFO:tasks.rados.rados.0.smithi029.stdout:3060: finishing write tid 1 to smithi02979277-14 2024-04-12T11:27:22.681 INFO:tasks.rados.rados.0.smithi029.stdout:3061: read oid 25 snap -1 2024-04-12T11:27:22.681 INFO:tasks.rados.rados.0.smithi029.stdout:3061: expect (ObjNum 1143 snap 387 seq_num 1143) 2024-04-12T11:27:22.682 INFO:tasks.rados.rados.0.smithi029.stdout:3062: read oid 8 snap -1 2024-04-12T11:27:22.682 INFO:tasks.rados.rados.0.smithi029.stdout:3062: expect deleted 2024-04-12T11:27:22.682 INFO:tasks.rados.rados.0.smithi029.stdout:3063: write oid 6 current snap is 391 2024-04-12T11:27:22.682 INFO:tasks.rados.rados.0.smithi029.stdout:3060: finishing write tid 2 to smithi02979277-14 2024-04-12T11:27:22.682 INFO:tasks.rados.rados.0.smithi029.stdout:3063: seq_num 1150 ranges {528050=505653,1612057=524731,2614324=567403} 2024-04-12T11:27:22.688 INFO:tasks.rados.rados.0.smithi029.stdout:3063: writing smithi02979277-6 from 528050 to 1033703 tid 1 2024-04-12T11:27:22.694 INFO:tasks.rados.rados.0.smithi029.stdout:3063: writing smithi02979277-6 from 1612057 to 2136788 tid 2 2024-04-12T11:27:22.700 INFO:tasks.rados.rados.0.smithi029.stdout:3063: writing smithi02979277-6 from 2614324 to 3181727 tid 3 2024-04-12T11:27:22.700 INFO:tasks.rados.rados.0.smithi029.stdout:3064: write oid 49 current snap is 391 2024-04-12T11:27:22.700 INFO:tasks.rados.rados.0.smithi029.stdout:3064: seq_num 1151 ranges {544857=580902,1670539=668389,3041747=303110} 2024-04-12T11:27:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:3064: writing smithi02979277-49 from 544857 to 1125759 tid 1 2024-04-12T11:27:22.713 INFO:tasks.rados.rados.0.smithi029.stdout:3064: writing smithi02979277-49 from 1670539 to 2338928 tid 2 2024-04-12T11:27:22.718 INFO:tasks.rados.rados.0.smithi029.stdout:3064: writing smithi02979277-49 from 3041747 to 3344857 tid 3 2024-04-12T11:27:22.718 INFO:tasks.rados.rados.0.smithi029.stdout:3065: snap_remove snap 387 2024-04-12T11:27:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:22 smithi087 ceph-mon[89014]: pgmap v891: 89 pgs: 89 active+clean; 281 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 369 KiB/s wr, 4 op/s 2024-04-12T11:27:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:22 smithi087 ceph-mon[89014]: osdmap e837: 8 total, 8 up, 8 in 2024-04-12T11:27:22.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:22 smithi087 ceph-mon[89014]: osdmap e838: 8 total, 8 up, 8 in 2024-04-12T11:27:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:22 smithi029 ceph-mon[97787]: pgmap v891: 89 pgs: 89 active+clean; 281 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 369 KiB/s wr, 4 op/s 2024-04-12T11:27:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:22 smithi029 ceph-mon[97787]: osdmap e837: 8 total, 8 up, 8 in 2024-04-12T11:27:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:22 smithi029 ceph-mon[97787]: osdmap e838: 8 total, 8 up, 8 in 2024-04-12T11:27:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:22 smithi029 ceph-mon[93650]: pgmap v891: 89 pgs: 89 active+clean; 281 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 369 KiB/s wr, 4 op/s 2024-04-12T11:27:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:22 smithi029 ceph-mon[93650]: osdmap e837: 8 total, 8 up, 8 in 2024-04-12T11:27:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:22 smithi029 ceph-mon[93650]: osdmap e838: 8 total, 8 up, 8 in 2024-04-12T11:27:23.672 INFO:tasks.rados.rados.0.smithi029.stdout:3065: done (5 left) 2024-04-12T11:27:23.672 INFO:tasks.rados.rados.0.smithi029.stdout:3066: snap_create 2024-04-12T11:27:23.672 INFO:tasks.rados.rados.0.smithi029.stdout:3060: finishing write tid 3 to smithi02979277-14 2024-04-12T11:27:23.672 INFO:tasks.rados.rados.0.smithi029.stdout:3060: finishing write tid 4 to smithi02979277-14 2024-04-12T11:27:23.673 INFO:tasks.rados.rados.0.smithi029.stdout:3060: finishing write tid 5 to smithi02979277-14 2024-04-12T11:27:23.673 INFO:tasks.rados.rados.0.smithi029.stdout:3060: oid 14 updating version 0 to 1376 2024-04-12T11:27:23.673 INFO:tasks.rados.rados.0.smithi029.stdout:3060: oid 14 version 1376 is already newer than 1375 2024-04-12T11:27:23.673 INFO:tasks.rados.rados.0.smithi029.stdout:3060: oid 14 updating version 1376 to 1377 2024-04-12T11:27:23.673 INFO:tasks.rados.rados.0.smithi029.stdout:3060: oid 14 version 1377 is already newer than 1374 2024-04-12T11:27:23.673 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 1377 (ObjNum 1149 snap 391 seq_num 1149) dirty exists 2024-04-12T11:27:23.673 INFO:tasks.rados.rados.0.smithi029.stdout:3060: left oid 14 (ObjNum 1149 snap 391 seq_num 1149) 2024-04-12T11:27:23.673 INFO:tasks.rados.rados.0.smithi029.stdout:3061: expect (ObjNum 1143 snap 387 seq_num 1143) 2024-04-12T11:27:23.779 INFO:tasks.rados.rados.0.smithi029.stdout:3063: finishing write tid 1 to smithi02979277-6 2024-04-12T11:27:23.779 INFO:tasks.rados.rados.0.smithi029.stdout:3063: finishing write tid 2 to smithi02979277-6 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3063: finishing write tid 3 to smithi02979277-6 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3063: finishing write tid 4 to smithi02979277-6 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3063: finishing write tid 5 to smithi02979277-6 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3063: oid 6 updating version 0 to 758 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3063: oid 6 updating version 758 to 760 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3063: oid 6 version 760 is already newer than 759 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3063: oid 6 version 760 is already newer than 757 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 760 (ObjNum 1150 snap 391 seq_num 1150) dirty exists 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3063: left oid 6 (ObjNum 1150 snap 391 seq_num 1150) 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3064: finishing write tid 1 to smithi02979277-49 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3064: finishing write tid 2 to smithi02979277-49 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3064: finishing write tid 3 to smithi02979277-49 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3064: finishing write tid 4 to smithi02979277-49 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3064: finishing write tid 5 to smithi02979277-49 2024-04-12T11:27:23.780 INFO:tasks.rados.rados.0.smithi029.stdout:3064: oid 49 updating version 0 to 1060 2024-04-12T11:27:23.781 INFO:tasks.rados.rados.0.smithi029.stdout:3064: oid 49 version 1060 is already newer than 1059 2024-04-12T11:27:23.781 INFO:tasks.rados.rados.0.smithi029.stdout:3064: oid 49 version 1060 is already newer than 1058 2024-04-12T11:27:23.781 INFO:tasks.rados.rados.0.smithi029.stdout:3064: oid 49 updating version 1060 to 1061 2024-04-12T11:27:23.781 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 1061 (ObjNum 1151 snap 391 seq_num 1151) dirty exists 2024-04-12T11:27:23.781 INFO:tasks.rados.rados.0.smithi029.stdout:3064: left oid 49 (ObjNum 1151 snap 391 seq_num 1151) 2024-04-12T11:27:24.674 INFO:tasks.rados.rados.0.smithi029.stdout:3060: done (5 left) 2024-04-12T11:27:24.675 INFO:tasks.rados.rados.0.smithi029.stdout:3061: done (4 left) 2024-04-12T11:27:24.675 INFO:tasks.rados.rados.0.smithi029.stdout:3062: done (3 left) 2024-04-12T11:27:24.675 INFO:tasks.rados.rados.0.smithi029.stdout:3063: done (2 left) 2024-04-12T11:27:24.675 INFO:tasks.rados.rados.0.smithi029.stdout:3064: done (1 left) 2024-04-12T11:27:24.675 INFO:tasks.rados.rados.0.smithi029.stdout:3066: done (0 left) 2024-04-12T11:27:24.675 INFO:tasks.rados.rados.0.smithi029.stdout:3067: write oid 7 current snap is 392 2024-04-12T11:27:24.675 INFO:tasks.rados.rados.0.smithi029.stdout:3067: seq_num 1152 ranges {561664=656151,1729021=528400,2669170=692494} 2024-04-12T11:27:24.681 INFO:tasks.rados.rados.0.smithi029.stdout:3067: writing smithi02979277-7 from 561664 to 1217815 tid 1 2024-04-12T11:27:24.687 INFO:tasks.rados.rados.0.smithi029.stdout:3067: writing smithi02979277-7 from 1729021 to 2257421 tid 2 2024-04-12T11:27:24.694 INFO:tasks.rados.rados.0.smithi029.stdout:3067: writing smithi02979277-7 from 2669170 to 3361664 tid 3 2024-04-12T11:27:24.694 INFO:tasks.rados.rados.0.smithi029.stdout:3068: read oid 21 snap -1 2024-04-12T11:27:24.694 INFO:tasks.rados.rados.0.smithi029.stdout:3068: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:27:24.694 INFO:tasks.rados.rados.0.smithi029.stdout:3069: rollback oid 17 current snap is 392 2024-04-12T11:27:24.694 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 371 2024-04-12T11:27:24.694 INFO:tasks.rados.rados.0.smithi029.stdout:3067: finishing write tid 1 to smithi02979277-7 2024-04-12T11:27:24.695 INFO:tasks.rados.rados.0.smithi029.stdout:3067: finishing write tid 2 to smithi02979277-7 2024-04-12T11:27:24.695 INFO:tasks.rados.rados.0.smithi029.stdout:3070: read oid 44 snap -1 2024-04-12T11:27:24.695 INFO:tasks.rados.rados.0.smithi029.stdout:3070: expect (ObjNum 1121 snap 374 seq_num 1121) 2024-04-12T11:27:24.695 INFO:tasks.rados.rados.0.smithi029.stdout:3071: snap_remove snap 382 2024-04-12T11:27:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:24 smithi087 ceph-mon[89014]: pgmap v894: 89 pgs: 89 active+clean; 281 MiB data, 875 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:27:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:24 smithi087 ceph-mon[89014]: osdmap e839: 8 total, 8 up, 8 in 2024-04-12T11:27:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:24 smithi029 ceph-mon[93650]: pgmap v894: 89 pgs: 89 active+clean; 281 MiB data, 875 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:27:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:24 smithi029 ceph-mon[93650]: osdmap e839: 8 total, 8 up, 8 in 2024-04-12T11:27:25.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:27:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:27:24] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:27:25.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:24 smithi029 ceph-mon[97787]: pgmap v894: 89 pgs: 89 active+clean; 281 MiB data, 875 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:27:25.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:24 smithi029 ceph-mon[97787]: osdmap e839: 8 total, 8 up, 8 in 2024-04-12T11:27:25.687 INFO:tasks.rados.rados.0.smithi029.stdout:3071: done (4 left) 2024-04-12T11:27:25.687 INFO:tasks.rados.rados.0.smithi029.stdout:3072: read oid 9 snap -1 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3072: expect (ObjNum 1132 snap 379 seq_num 1132) 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3073: rollback oid 6 current snap is 392 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 6 to 389 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3074: snap_create 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3067: finishing write tid 3 to smithi02979277-7 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3067: finishing write tid 4 to smithi02979277-7 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3067: finishing write tid 5 to smithi02979277-7 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3067: oid 7 updating version 0 to 1443 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3067: oid 7 updating version 1443 to 1445 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3067: oid 7 version 1445 is already newer than 1444 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3067: oid 7 version 1445 is already newer than 1442 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1445 (ObjNum 1152 snap 392 seq_num 1152) dirty exists 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3067: left oid 7 (ObjNum 1152 snap 392 seq_num 1152) 2024-04-12T11:27:25.688 INFO:tasks.rados.rados.0.smithi029.stdout:3069: finishing rollback tid 0 to smithi02979277-17 2024-04-12T11:27:25.689 INFO:tasks.rados.rados.0.smithi029.stdout:3069: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:27:25.689 INFO:tasks.rados.rados.0.smithi029.stdout:3069: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:27:25.689 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1506 (ObjNum 1019 snap 352 seq_num 1019) dirty exists 2024-04-12T11:27:25.689 INFO:tasks.rados.rados.0.smithi029.stdout:3068: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:27:25.849 INFO:tasks.rados.rados.0.smithi029.stdout:3070: expect (ObjNum 1121 snap 374 seq_num 1121) 2024-04-12T11:27:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:25 smithi087 ceph-mon[89014]: osdmap e840: 8 total, 8 up, 8 in 2024-04-12T11:27:25.978 INFO:tasks.rados.rados.0.smithi029.stdout:3073: finishing rollback tid 0 to smithi02979277-6 2024-04-12T11:27:25.979 INFO:tasks.rados.rados.0.smithi029.stdout:3073: finishing rollback tid 1 to smithi02979277-6 2024-04-12T11:27:25.979 INFO:tasks.rados.rados.0.smithi029.stdout:3073: finishing rollback tid 2 to smithi02979277-6 2024-04-12T11:27:25.979 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 768 (ObjNum 872 snap 307 seq_num 872) dirty exists 2024-04-12T11:27:25.979 INFO:tasks.rados.rados.0.smithi029.stdout:3072: expect (ObjNum 1132 snap 379 seq_num 1132) 2024-04-12T11:27:26.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:25 smithi029 ceph-mon[93650]: osdmap e840: 8 total, 8 up, 8 in 2024-04-12T11:27:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:25 smithi029 ceph-mon[97787]: osdmap e840: 8 total, 8 up, 8 in 2024-04-12T11:27:26.689 INFO:tasks.rados.rados.0.smithi029.stdout:3067: done (6 left) 2024-04-12T11:27:26.689 INFO:tasks.rados.rados.0.smithi029.stdout:3068: done (5 left) 2024-04-12T11:27:26.689 INFO:tasks.rados.rados.0.smithi029.stdout:3069: done (4 left) 2024-04-12T11:27:26.690 INFO:tasks.rados.rados.0.smithi029.stdout:3070: done (3 left) 2024-04-12T11:27:26.690 INFO:tasks.rados.rados.0.smithi029.stdout:3072: done (2 left) 2024-04-12T11:27:26.690 INFO:tasks.rados.rados.0.smithi029.stdout:3073: done (1 left) 2024-04-12T11:27:26.690 INFO:tasks.rados.rados.0.smithi029.stdout:3074: done (0 left) 2024-04-12T11:27:26.690 INFO:tasks.rados.rados.0.smithi029.stdout:3075: delete oid 11 current snap is 393 2024-04-12T11:27:26.695 INFO:tasks.rados.rados.0.smithi029.stdout:3075: done (0 left) 2024-04-12T11:27:26.695 INFO:tasks.rados.rados.0.smithi029.stdout:3076: snap_remove snap 371 2024-04-12T11:27:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:26 smithi087 ceph-mon[89014]: pgmap v897: 89 pgs: 89 active+clean; 284 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 735 KiB/s wr, 9 op/s 2024-04-12T11:27:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:26 smithi087 ceph-mon[89014]: osdmap e841: 8 total, 8 up, 8 in 2024-04-12T11:27:27.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:26 smithi029 ceph-mon[93650]: pgmap v897: 89 pgs: 89 active+clean; 284 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 735 KiB/s wr, 9 op/s 2024-04-12T11:27:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:26 smithi029 ceph-mon[93650]: osdmap e841: 8 total, 8 up, 8 in 2024-04-12T11:27:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:26 smithi029 ceph-mon[97787]: pgmap v897: 89 pgs: 89 active+clean; 284 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 735 KiB/s wr, 9 op/s 2024-04-12T11:27:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:26 smithi029 ceph-mon[97787]: osdmap e841: 8 total, 8 up, 8 in 2024-04-12T11:27:27.656 INFO:tasks.rados.rados.0.smithi029.stdout:3076: done (0 left) 2024-04-12T11:27:27.657 INFO:tasks.rados.rados.0.smithi029.stdout:3077: rollback oid 30 current snap is 393 2024-04-12T11:27:27.657 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 357 2024-04-12T11:27:27.657 INFO:tasks.rados.rados.0.smithi029.stdout:3078: delete oid 35 current snap is 393 2024-04-12T11:27:27.668 INFO:tasks.rados.rados.0.smithi029.stdout:3077: finishing rollback tid 0 to smithi02979277-30 2024-04-12T11:27:27.668 INFO:tasks.rados.rados.0.smithi029.stdout:3078: done (1 left) 2024-04-12T11:27:27.668 INFO:tasks.rados.rados.0.smithi029.stdout:3079: snap_create 2024-04-12T11:27:27.668 INFO:tasks.rados.rados.0.smithi029.stdout:3077: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:27:27.668 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 954 (ObjNum 129 snap 0 seq_num 1800734496) dirty dne 2024-04-12T11:27:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:27 smithi087 ceph-mon[89014]: osdmap e842: 8 total, 8 up, 8 in 2024-04-12T11:27:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:27 smithi087 ceph-mon[89014]: osdmap e843: 8 total, 8 up, 8 in 2024-04-12T11:27:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:27 smithi029 ceph-mon[93650]: osdmap e842: 8 total, 8 up, 8 in 2024-04-12T11:27:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:27 smithi029 ceph-mon[93650]: osdmap e843: 8 total, 8 up, 8 in 2024-04-12T11:27:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:27 smithi029 ceph-mon[97787]: osdmap e842: 8 total, 8 up, 8 in 2024-04-12T11:27:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:27 smithi029 ceph-mon[97787]: osdmap e843: 8 total, 8 up, 8 in 2024-04-12T11:27:28.658 INFO:tasks.rados.rados.0.smithi029.stdout:3077: done (1 left) 2024-04-12T11:27:28.658 INFO:tasks.rados.rados.0.smithi029.stdout:3079: done (0 left) 2024-04-12T11:27:28.658 INFO:tasks.rados.rados.0.smithi029.stdout:3080: read oid 34 snap -1 2024-04-12T11:27:28.658 INFO:tasks.rados.rados.0.smithi029.stdout:3080: expect deleted 2024-04-12T11:27:28.658 INFO:tasks.rados.rados.0.smithi029.stdout:3081: write oid 10 current snap is 394 2024-04-12T11:27:28.658 INFO:tasks.rados.rados.0.smithi029.stdout:3081: seq_num 1155 ranges {612085=481898,1788114=675727,3035086=376999} 2024-04-12T11:27:28.664 INFO:tasks.rados.rados.0.smithi029.stdout:3081: writing smithi02979277-10 from 612085 to 1093983 tid 1 2024-04-12T11:27:28.671 INFO:tasks.rados.rados.0.smithi029.stdout:3081: writing smithi02979277-10 from 1788114 to 2463841 tid 2 2024-04-12T11:27:28.677 INFO:tasks.rados.rados.0.smithi029.stdout:3081: writing smithi02979277-10 from 3035086 to 3412085 tid 3 2024-04-12T11:27:28.677 INFO:tasks.rados.rados.0.smithi029.stdout:3082: rollback oid 26 current snap is 394 2024-04-12T11:27:28.677 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 366 2024-04-12T11:27:28.677 INFO:tasks.rados.rados.0.smithi029.stdout:3081: finishing write tid 1 to smithi02979277-10 2024-04-12T11:27:28.677 INFO:tasks.rados.rados.0.smithi029.stdout:3081: finishing write tid 2 to smithi02979277-10 2024-04-12T11:27:28.677 INFO:tasks.rados.rados.0.smithi029.stdout:3080: done (2 left) 2024-04-12T11:27:28.677 INFO:tasks.rados.rados.0.smithi029.stdout:3083: read oid 50 snap -1 2024-04-12T11:27:28.678 INFO:tasks.rados.rados.0.smithi029.stdout:3083: expect deleted 2024-04-12T11:27:28.678 INFO:tasks.rados.rados.0.smithi029.stdout:3084: write oid 25 current snap is 394 2024-04-12T11:27:28.678 INFO:tasks.rados.rados.0.smithi029.stdout:3084: seq_num 1156 ranges {628892=673500,1962949=535738,3178862=250030} 2024-04-12T11:27:28.684 INFO:tasks.rados.rados.0.smithi029.stdout:3084: writing smithi02979277-25 from 628892 to 1302392 tid 1 2024-04-12T11:27:28.691 INFO:tasks.rados.rados.0.smithi029.stdout:3084: writing smithi02979277-25 from 1962949 to 2498687 tid 2 2024-04-12T11:27:28.696 INFO:tasks.rados.rados.0.smithi029.stdout:3084: writing smithi02979277-25 from 3178862 to 3428892 tid 3 2024-04-12T11:27:28.696 INFO:tasks.rados.rados.0.smithi029.stdout:3085: read oid 43 snap -1 2024-04-12T11:27:28.696 INFO:tasks.rados.rados.0.smithi029.stdout:3085: expect (ObjNum 1089 snap 365 seq_num 1089) 2024-04-12T11:27:28.696 INFO:tasks.rados.rados.0.smithi029.stdout:3086: write oid 9 current snap is 394 2024-04-12T11:27:28.697 INFO:tasks.rados.rados.0.smithi029.stdout:3086: seq_num 1157 ranges {645699=748749,2021431=679396,3206285=239414} 2024-04-12T11:27:28.704 INFO:tasks.rados.rados.0.smithi029.stdout:3086: writing smithi02979277-9 from 645699 to 1394448 tid 1 2024-04-12T11:27:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3086: writing smithi02979277-9 from 2021431 to 2700827 tid 2 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3086: writing smithi02979277-9 from 3206285 to 3445699 tid 3 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3087: read oid 17 snap 361 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3087: expect (ObjNum 1019 snap 352 seq_num 1019) 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3088: rollback oid 49 current snap is 394 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 49 to 362 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3082: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3081: finishing write tid 3 to smithi02979277-10 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3081: finishing write tid 4 to smithi02979277-10 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3081: finishing write tid 5 to smithi02979277-10 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3081: oid 10 updating version 0 to 1134 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3081: oid 10 version 1134 is already newer than 1132 2024-04-12T11:27:28.717 INFO:tasks.rados.rados.0.smithi029.stdout:3081: oid 10 version 1134 is already newer than 1133 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3081: oid 10 version 1134 is already newer than 1131 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 1134 (ObjNum 1155 snap 394 seq_num 1155) dirty exists 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3081: left oid 10 (ObjNum 1155 snap 394 seq_num 1155) 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3082: finishing rollback tid 2 to smithi02979277-26 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 1896 (ObjNum 1075 snap 364 seq_num 1075) dirty exists 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3084: finishing write tid 1 to smithi02979277-25 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3081: done (7 left) 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3082: done (6 left) 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3083: done (5 left) 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3089: read oid 10 snap 381 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3089: expect (ObjNum 975 snap 340 seq_num 975) 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3090: write oid 50 current snap is 394 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3084: finishing write tid 2 to smithi02979277-25 2024-04-12T11:27:28.718 INFO:tasks.rados.rados.0.smithi029.stdout:3090: seq_num 1158 ranges {662506=423998,1563560=539407,2717355=490285} 2024-04-12T11:27:28.722 INFO:tasks.rados.rados.0.smithi029.stdout:3090: writing smithi02979277-50 from 662506 to 1086504 tid 1 2024-04-12T11:27:28.727 INFO:tasks.rados.rados.0.smithi029.stdout:3090: writing smithi02979277-50 from 1563560 to 2102967 tid 2 2024-04-12T11:27:28.734 INFO:tasks.rados.rados.0.smithi029.stdout:3090: writing smithi02979277-50 from 2717355 to 3207640 tid 3 2024-04-12T11:27:28.734 INFO:tasks.rados.rados.0.smithi029.stdout:3091: rollback oid 4 current snap is 394 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 4 to 362 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:3084: finishing write tid 3 to smithi02979277-25 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:3084: finishing write tid 4 to smithi02979277-25 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:3084: finishing write tid 5 to smithi02979277-25 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:3084: oid 25 updating version 0 to 1517 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:3084: oid 25 version 1517 is already newer than 1514 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:3084: oid 25 version 1517 is already newer than 1515 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:3084: oid 25 version 1517 is already newer than 1516 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 1517 (ObjNum 1156 snap 394 seq_num 1156) dirty exists 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:3084: left oid 25 (ObjNum 1156 snap 394 seq_num 1156) 2024-04-12T11:27:28.735 INFO:tasks.rados.rados.0.smithi029.stdout:3085: expect (ObjNum 1089 snap 365 seq_num 1089) 2024-04-12T11:27:28.814 INFO:tasks.rados.rados.0.smithi029.stdout:3086: finishing write tid 1 to smithi02979277-9 2024-04-12T11:27:28.814 INFO:tasks.rados.rados.0.smithi029.stdout:3086: finishing write tid 2 to smithi02979277-9 2024-04-12T11:27:28.814 INFO:tasks.rados.rados.0.smithi029.stdout:3084: done (7 left) 2024-04-12T11:27:28.814 INFO:tasks.rados.rados.0.smithi029.stdout:3085: done (6 left) 2024-04-12T11:27:28.814 INFO:tasks.rados.rados.0.smithi029.stdout:3092: snap_remove snap 369 2024-04-12T11:27:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:28 smithi087 ceph-mon[89014]: pgmap v900: 89 pgs: 89 active+clean; 285 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-04-12T11:27:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:28 smithi087 ceph-mon[89014]: osdmap e844: 8 total, 8 up, 8 in 2024-04-12T11:27:29.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:28 smithi029 ceph-mon[93650]: pgmap v900: 89 pgs: 89 active+clean; 285 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-04-12T11:27:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:28 smithi029 ceph-mon[93650]: osdmap e844: 8 total, 8 up, 8 in 2024-04-12T11:27:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:28 smithi029 ceph-mon[97787]: pgmap v900: 89 pgs: 89 active+clean; 285 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-04-12T11:27:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:28 smithi029 ceph-mon[97787]: osdmap e844: 8 total, 8 up, 8 in 2024-04-12T11:27:29.702 INFO:tasks.rados.rados.0.smithi029.stdout:3092: done (6 left) 2024-04-12T11:27:29.703 INFO:tasks.rados.rados.0.smithi029.stdout:3093: write oid 29 current snap is 394 2024-04-12T11:27:29.703 INFO:tasks.rados.rados.0.smithi029.stdout:3093: seq_num 1159 ranges {679313=499247,1622042=683065,2744778=701557} 2024-04-12T11:27:29.709 INFO:tasks.rados.rados.0.smithi029.stdout:3093: writing smithi02979277-29 from 679313 to 1178560 tid 1 2024-04-12T11:27:29.715 INFO:tasks.rados.rados.0.smithi029.stdout:3093: writing smithi02979277-29 from 1622042 to 2305107 tid 2 2024-04-12T11:27:29.721 INFO:tasks.rados.rados.0.smithi029.stdout:3093: writing smithi02979277-29 from 2744778 to 3446335 tid 3 2024-04-12T11:27:29.722 INFO:tasks.rados.rados.0.smithi029.stdout:3094: write oid 28 current snap is 394 2024-04-12T11:27:29.722 INFO:tasks.rados.rados.0.smithi029.stdout:3094: seq_num 1160 ranges {696120=574496,1680524=543076,2888554=512829} 2024-04-12T11:27:29.728 INFO:tasks.rados.rados.0.smithi029.stdout:3094: writing smithi02979277-28 from 696120 to 1270616 tid 1 2024-04-12T11:27:29.734 INFO:tasks.rados.rados.0.smithi029.stdout:3094: writing smithi02979277-28 from 1680524 to 2223600 tid 2 2024-04-12T11:27:29.740 INFO:tasks.rados.rados.0.smithi029.stdout:3094: writing smithi02979277-28 from 2888554 to 3401383 tid 3 2024-04-12T11:27:29.741 INFO:tasks.rados.rados.0.smithi029.stdout:3095: read oid 30 snap -1 2024-04-12T11:27:29.741 INFO:tasks.rados.rados.0.smithi029.stdout:3095: expect deleted 2024-04-12T11:27:29.741 INFO:tasks.rados.rados.0.smithi029.stdout:3096: rollback oid 42 current snap is 394 2024-04-12T11:27:29.741 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 393 2024-04-12T11:27:29.741 INFO:tasks.rados.rados.0.smithi029.stdout:3089: expect (ObjNum 975 snap 340 seq_num 975) 2024-04-12T11:27:29.822 INFO:tasks.rados.rados.0.smithi029.stdout:3086: finishing write tid 3 to smithi02979277-9 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3086: finishing write tid 4 to smithi02979277-9 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3086: finishing write tid 5 to smithi02979277-9 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3086: oid 9 updating version 0 to 1074 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3086: oid 9 version 1074 is already newer than 1073 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3086: oid 9 version 1074 is already newer than 1071 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3086: oid 9 version 1074 is already newer than 1072 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 1074 (ObjNum 1157 snap 394 seq_num 1157) dirty exists 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3086: left oid 9 (ObjNum 1157 snap 394 seq_num 1157) 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3088: finishing rollback tid 0 to smithi02979277-49 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3086: done (9 left) 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3089: done (8 left) 2024-04-12T11:27:29.823 INFO:tasks.rados.rados.0.smithi029.stdout:3097: write oid 7 current snap is 394 2024-04-12T11:27:29.824 INFO:tasks.rados.rados.0.smithi029.stdout:3088: finishing rollback tid 1 to smithi02979277-49 2024-04-12T11:27:29.824 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 1077 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-04-12T11:27:29.824 INFO:tasks.rados.rados.0.smithi029.stdout:3087: expect (ObjNum 1019 snap 352 seq_num 1019) 2024-04-12T11:27:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:3090: finishing write tid 1 to smithi02979277-50 2024-04-12T11:27:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:3090: finishing write tid 2 to smithi02979277-50 2024-04-12T11:27:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:3091: finishing rollback tid 0 to smithi02979277-4 2024-04-12T11:27:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:3090: finishing write tid 3 to smithi02979277-50 2024-04-12T11:27:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:3090: finishing write tid 4 to smithi02979277-50 2024-04-12T11:27:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:3091: finishing rollback tid 1 to smithi02979277-4 2024-04-12T11:27:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1386 (ObjNum 65 snap 0 seq_num 1777349616) dirty dne 2024-04-12T11:27:29.927 INFO:tasks.rados.rados.0.smithi029.stdout:3097: seq_num 1161 ranges {712927=649745,2139006=686734} 2024-04-12T11:27:29.934 INFO:tasks.rados.rados.0.smithi029.stdout:3097: writing smithi02979277-7 from 712927 to 1362672 tid 1 2024-04-12T11:27:29.942 INFO:tasks.rados.rados.0.smithi029.stdout:3097: writing smithi02979277-7 from 2139006 to 2825740 tid 2 2024-04-12T11:27:29.942 INFO:tasks.rados.rados.0.smithi029.stdout:3087: done (8 left) 2024-04-12T11:27:29.942 INFO:tasks.rados.rados.0.smithi029.stdout:3088: done (7 left) 2024-04-12T11:27:29.942 INFO:tasks.rados.rados.0.smithi029.stdout:3091: done (6 left) 2024-04-12T11:27:29.942 INFO:tasks.rados.rados.0.smithi029.stdout:3098: write oid 10 current snap is 394 2024-04-12T11:27:29.943 INFO:tasks.rados.rados.0.smithi029.stdout:3098: seq_num 1162 ranges {729734=724994,2197488=430392,3227047=302687} 2024-04-12T11:27:29.950 INFO:tasks.rados.rados.0.smithi029.stdout:3098: writing smithi02979277-10 from 729734 to 1454728 tid 1 2024-04-12T11:27:29.956 INFO:tasks.rados.rados.0.smithi029.stdout:3098: writing smithi02979277-10 from 2197488 to 2627880 tid 2 2024-04-12T11:27:29.963 INFO:tasks.rados.rados.0.smithi029.stdout:3098: writing smithi02979277-10 from 3227047 to 3529734 tid 3 2024-04-12T11:27:29.963 INFO:tasks.rados.rados.0.smithi029.stdout:3099: read oid 22 snap -1 2024-04-12T11:27:29.963 INFO:tasks.rados.rados.0.smithi029.stdout:3099: expect (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:27:29.963 INFO:tasks.rados.rados.0.smithi029.stdout:3100: rollback oid 41 current snap is 394 2024-04-12T11:27:29.963 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 379 2024-04-12T11:27:29.963 INFO:tasks.rados.rados.0.smithi029.stdout:3090: finishing write tid 5 to smithi02979277-50 2024-04-12T11:27:29.963 INFO:tasks.rados.rados.0.smithi029.stdout:3090: oid 50 updating version 0 to 1247 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3090: oid 50 updating version 1247 to 1249 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3090: oid 50 version 1249 is already newer than 1248 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3090: oid 50 version 1249 is already newer than 1246 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1249 (ObjNum 1158 snap 394 seq_num 1158) dirty exists 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3090: left oid 50 (ObjNum 1158 snap 394 seq_num 1158) 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3093: finishing write tid 1 to smithi02979277-29 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3093: finishing write tid 2 to smithi02979277-29 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3090: done (8 left) 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3101: rollback oid 24 current snap is 394 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 24 to 366 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3093: finishing write tid 3 to smithi02979277-29 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3093: finishing write tid 4 to smithi02979277-29 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3093: finishing write tid 5 to smithi02979277-29 2024-04-12T11:27:29.964 INFO:tasks.rados.rados.0.smithi029.stdout:3093: oid 29 updating version 0 to 1391 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3093: oid 29 updating version 1391 to 1392 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3093: oid 29 updating version 1392 to 1393 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3093: oid 29 updating version 1393 to 1394 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1394 (ObjNum 1159 snap 394 seq_num 1159) dirty exists 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3093: left oid 29 (ObjNum 1159 snap 394 seq_num 1159) 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3094: finishing write tid 1 to smithi02979277-28 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3093: done (8 left) 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3102: snap_create 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3094: finishing write tid 2 to smithi02979277-28 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3096: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3094: finishing write tid 3 to smithi02979277-28 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3096: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3094: finishing write tid 4 to smithi02979277-28 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:3096: finishing rollback tid 2 to smithi02979277-42 2024-04-12T11:27:29.965 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 367 (ObjNum 1129 snap 379 seq_num 1129) dirty exists 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3094: finishing write tid 5 to smithi02979277-28 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3094: oid 28 updating version 0 to 1146 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3094: oid 28 version 1146 is already newer than 1145 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3094: oid 28 updating version 1146 to 1147 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3094: oid 28 version 1147 is already newer than 1144 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 1147 (ObjNum 1160 snap 394 seq_num 1160) dirty exists 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3094: left oid 28 (ObjNum 1160 snap 394 seq_num 1160) 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3097: finishing write tid 1 to smithi02979277-7 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3097: finishing write tid 2 to smithi02979277-7 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3097: finishing write tid 3 to smithi02979277-7 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3097: finishing write tid 4 to smithi02979277-7 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3097: oid 7 updating version 0 to 1453 2024-04-12T11:27:29.966 INFO:tasks.rados.rados.0.smithi029.stdout:3097: oid 7 version 1453 is already newer than 1451 2024-04-12T11:27:29.967 INFO:tasks.rados.rados.0.smithi029.stdout:3097: oid 7 version 1453 is already newer than 1452 2024-04-12T11:27:29.967 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1453 (ObjNum 1161 snap 394 seq_num 1161) dirty exists 2024-04-12T11:27:29.967 INFO:tasks.rados.rados.0.smithi029.stdout:3097: left oid 7 (ObjNum 1161 snap 394 seq_num 1161) 2024-04-12T11:27:29.967 INFO:tasks.rados.rados.0.smithi029.stdout:3098: finishing write tid 1 to smithi02979277-10 2024-04-12T11:27:29.967 INFO:tasks.rados.rados.0.smithi029.stdout:3098: finishing write tid 2 to smithi02979277-10 2024-04-12T11:27:29.968 INFO:tasks.rados.rados.0.smithi029.stdout:3100: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:27:29.968 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1421 (ObjNum 49 snap 0 seq_num 3425826268) dirty dne 2024-04-12T11:27:29.968 INFO:tasks.rados.rados.0.smithi029.stdout:3098: finishing write tid 3 to smithi02979277-10 2024-04-12T11:27:29.968 INFO:tasks.rados.rados.0.smithi029.stdout:3098: finishing write tid 4 to smithi02979277-10 2024-04-12T11:27:29.968 INFO:tasks.rados.rados.0.smithi029.stdout:3101: finishing rollback tid 1 to smithi02979277-24 2024-04-12T11:27:29.968 INFO:tasks.rados.rados.0.smithi029.stdout:3101: finishing rollback tid 2 to smithi02979277-24 2024-04-12T11:27:29.968 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1904 (ObjNum 1051 snap 357 seq_num 1051) dirty exists 2024-04-12T11:27:29.968 INFO:tasks.rados.rados.0.smithi029.stdout:3098: finishing write tid 5 to smithi02979277-10 2024-04-12T11:27:29.968 INFO:tasks.rados.rados.0.smithi029.stdout:3098: oid 10 updating version 0 to 1149 2024-04-12T11:27:29.969 INFO:tasks.rados.rados.0.smithi029.stdout:3098: oid 10 version 1149 is already newer than 1148 2024-04-12T11:27:29.969 INFO:tasks.rados.rados.0.smithi029.stdout:3098: oid 10 updating version 1149 to 1151 2024-04-12T11:27:29.969 INFO:tasks.rados.rados.0.smithi029.stdout:3098: oid 10 version 1151 is already newer than 1150 2024-04-12T11:27:29.969 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 1151 (ObjNum 1162 snap 394 seq_num 1162) dirty exists 2024-04-12T11:27:29.969 INFO:tasks.rados.rados.0.smithi029.stdout:3098: left oid 10 (ObjNum 1162 snap 394 seq_num 1162) 2024-04-12T11:27:29.969 INFO:tasks.rados.rados.0.smithi029.stdout:3099: expect (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:27:30.702 INFO:tasks.rados.rados.0.smithi029.stdout:3094: done (8 left) 2024-04-12T11:27:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:3095: done (7 left) 2024-04-12T11:27:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:3096: done (6 left) 2024-04-12T11:27:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:3097: done (5 left) 2024-04-12T11:27:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:3098: done (4 left) 2024-04-12T11:27:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:3099: done (3 left) 2024-04-12T11:27:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:3100: done (2 left) 2024-04-12T11:27:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:3101: done (1 left) 2024-04-12T11:27:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:3102: done (0 left) 2024-04-12T11:27:30.703 INFO:tasks.rados.rados.0.smithi029.stdout:3103: snap_remove snap 361 2024-04-12T11:27:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:30 smithi087 ceph-mon[89014]: pgmap v903: 89 pgs: 89 active+clean; 290 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.1 MiB/s wr, 13 op/s 2024-04-12T11:27:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:30 smithi087 ceph-mon[89014]: osdmap e845: 8 total, 8 up, 8 in 2024-04-12T11:27:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:30 smithi029 ceph-mon[93650]: pgmap v903: 89 pgs: 89 active+clean; 290 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.1 MiB/s wr, 13 op/s 2024-04-12T11:27:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:30 smithi029 ceph-mon[93650]: osdmap e845: 8 total, 8 up, 8 in 2024-04-12T11:27:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:30 smithi029 ceph-mon[97787]: pgmap v903: 89 pgs: 89 active+clean; 290 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.1 MiB/s wr, 13 op/s 2024-04-12T11:27:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:30 smithi029 ceph-mon[97787]: osdmap e845: 8 total, 8 up, 8 in 2024-04-12T11:27:31.713 INFO:tasks.rados.rados.0.smithi029.stdout:3103: done (0 left) 2024-04-12T11:27:31.713 INFO:tasks.rados.rados.0.smithi029.stdout:3104: read oid 35 snap -1 2024-04-12T11:27:31.713 INFO:tasks.rados.rados.0.smithi029.stdout:3104: expect deleted 2024-04-12T11:27:31.713 INFO:tasks.rados.rados.0.smithi029.stdout:3105: read oid 15 snap 379 2024-04-12T11:27:31.713 INFO:tasks.rados.rados.0.smithi029.stdout:3105: expect deleted 2024-04-12T11:27:31.714 INFO:tasks.rados.rados.0.smithi029.stdout:3106: rollback oid 13 current snap is 395 2024-04-12T11:27:31.714 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 13 to 386 2024-04-12T11:27:31.714 INFO:tasks.rados.rados.0.smithi029.stdout:3107: rollback oid 37 current snap is 395 2024-04-12T11:27:31.714 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 37 to 357 2024-04-12T11:27:31.714 INFO:tasks.rados.rados.0.smithi029.stdout:3108: read oid 46 snap -1 2024-04-12T11:27:31.714 INFO:tasks.rados.rados.0.smithi029.stdout:3108: expect (ObjNum 1131 snap 379 seq_num 1131) 2024-04-12T11:27:31.714 INFO:tasks.rados.rados.0.smithi029.stdout:3109: rollback oid 8 current snap is 395 2024-04-12T11:27:31.714 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 368 2024-04-12T11:27:31.715 INFO:tasks.rados.rados.0.smithi029.stdout:3110: write oid 12 current snap is 395 2024-04-12T11:27:31.715 INFO:tasks.rados.rados.0.smithi029.stdout:3110: seq_num 1163 ranges {746541=400243,1739617=690403,3138117=408424} 2024-04-12T11:27:31.720 INFO:tasks.rados.rados.0.smithi029.stdout:3110: writing smithi02979277-12 from 746541 to 1146784 tid 1 2024-04-12T11:27:31.728 INFO:tasks.rados.rados.0.smithi029.stdout:3110: writing smithi02979277-12 from 1739617 to 2430020 tid 2 2024-04-12T11:27:31.735 INFO:tasks.rados.rados.0.smithi029.stdout:3110: writing smithi02979277-12 from 3138117 to 3546541 tid 3 2024-04-12T11:27:31.735 INFO:tasks.rados.rados.0.smithi029.stdout:3111: delete oid 3 current snap is 395 2024-04-12T11:27:31.735 INFO:tasks.rados.rados.0.smithi029.stdout:3108: expect (ObjNum 1131 snap 379 seq_num 1131) 2024-04-12T11:27:31.871 INFO:tasks.rados.rados.0.smithi029.stdout:3107: finishing rollback tid 0 to smithi02979277-37 2024-04-12T11:27:31.872 INFO:tasks.rados.rados.0.smithi029.stdout:3107: finishing rollback tid 1 to smithi02979277-37 2024-04-12T11:27:31.872 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 1084 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:27:31.872 INFO:tasks.rados.rados.0.smithi029.stdout:3106: finishing rollback tid 0 to smithi02979277-13 2024-04-12T11:27:31.872 INFO:tasks.rados.rados.0.smithi029.stdout:3104: done (7 left) 2024-04-12T11:27:31.872 INFO:tasks.rados.rados.0.smithi029.stdout:3105: done (6 left) 2024-04-12T11:27:31.872 INFO:tasks.rados.rados.0.smithi029.stdout:3107: done (5 left) 2024-04-12T11:27:31.872 INFO:tasks.rados.rados.0.smithi029.stdout:3108: done (4 left) 2024-04-12T11:27:31.872 INFO:tasks.rados.rados.0.smithi029.stdout:3111: done (3 left) 2024-04-12T11:27:31.872 INFO:tasks.rados.rados.0.smithi029.stdout:3112: snap_remove snap 366 2024-04-12T11:27:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:31 smithi087 ceph-mon[89014]: osdmap e846: 8 total, 8 up, 8 in 2024-04-12T11:27:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:31 smithi029 ceph-mon[93650]: osdmap e846: 8 total, 8 up, 8 in 2024-04-12T11:27:32.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:31 smithi029 ceph-mon[97787]: osdmap e846: 8 total, 8 up, 8 in 2024-04-12T11:27:32.658 INFO:tasks.rados.rados.0.smithi029.stdout:3112: done (3 left) 2024-04-12T11:27:32.658 INFO:tasks.rados.rados.0.smithi029.stdout:3113: read oid 45 snap 357 2024-04-12T11:27:32.658 INFO:tasks.rados.rados.0.smithi029.stdout:3113: expect (ObjNum 962 snap 330 seq_num 962) 2024-04-12T11:27:32.658 INFO:tasks.rados.rados.0.smithi029.stdout:3114: write oid 40 current snap is 395 2024-04-12T11:27:32.658 INFO:tasks.rados.rados.0.smithi029.stdout:3114: seq_num 1165 ranges {780155=667094,1972934=694072,3309316=270839} 2024-04-12T11:27:32.666 INFO:tasks.rados.rados.0.smithi029.stdout:3114: writing smithi02979277-40 from 780155 to 1447249 tid 1 2024-04-12T11:27:32.674 INFO:tasks.rados.rados.0.smithi029.stdout:3114: writing smithi02979277-40 from 1972934 to 2667006 tid 2 2024-04-12T11:27:32.679 INFO:tasks.rados.rados.0.smithi029.stdout:3114: writing smithi02979277-40 from 3309316 to 3580155 tid 3 2024-04-12T11:27:32.679 INFO:tasks.rados.rados.0.smithi029.stdout:3115: rollback oid 28 current snap is 395 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 28 to 380 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3106: finishing rollback tid 1 to smithi02979277-13 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3106: finishing rollback tid 2 to smithi02979277-13 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1527 (ObjNum 1086 snap 365 seq_num 1086) dirty exists 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3109: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 326 (ObjNum 129 snap 0 seq_num 1801009104) dirty dne 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3110: finishing write tid 1 to smithi02979277-12 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3110: finishing write tid 2 to smithi02979277-12 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3110: finishing write tid 3 to smithi02979277-12 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3110: finishing write tid 4 to smithi02979277-12 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3110: finishing write tid 5 to smithi02979277-12 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3110: oid 12 updating version 0 to 1459 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3110: oid 12 updating version 1459 to 1463 2024-04-12T11:27:32.680 INFO:tasks.rados.rados.0.smithi029.stdout:3110: oid 12 version 1463 is already newer than 1462 2024-04-12T11:27:32.681 INFO:tasks.rados.rados.0.smithi029.stdout:3110: oid 12 version 1463 is already newer than 1458 2024-04-12T11:27:32.681 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1463 (ObjNum 1163 snap 395 seq_num 1163) dirty exists 2024-04-12T11:27:32.681 INFO:tasks.rados.rados.0.smithi029.stdout:3110: left oid 12 (ObjNum 1163 snap 395 seq_num 1163) 2024-04-12T11:27:32.681 INFO:tasks.rados.rados.0.smithi029.stdout:3113: expect (ObjNum 962 snap 330 seq_num 962) 2024-04-12T11:27:32.709 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:27:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:27:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3114: finishing write tid 1 to smithi02979277-40 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3114: finishing write tid 2 to smithi02979277-40 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3106: done (5 left) 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3109: done (4 left) 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3110: done (3 left) 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3113: done (2 left) 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3116: snap_create 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3114: finishing write tid 3 to smithi02979277-40 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3115: finishing rollback tid 0 to smithi02979277-28 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3114: finishing write tid 4 to smithi02979277-40 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3115: finishing rollback tid 1 to smithi02979277-28 2024-04-12T11:27:32.754 INFO:tasks.rados.rados.0.smithi029.stdout:3115: finishing rollback tid 2 to smithi02979277-28 2024-04-12T11:27:32.755 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 1167 (ObjNum 1108 snap 370 seq_num 1108) dirty exists 2024-04-12T11:27:32.755 INFO:tasks.rados.rados.0.smithi029.stdout:3114: finishing write tid 5 to smithi02979277-40 2024-04-12T11:27:32.755 INFO:tasks.rados.rados.0.smithi029.stdout:3114: oid 40 updating version 0 to 1919 2024-04-12T11:27:32.755 INFO:tasks.rados.rados.0.smithi029.stdout:3114: oid 40 updating version 1919 to 1929 2024-04-12T11:27:32.755 INFO:tasks.rados.rados.0.smithi029.stdout:3114: oid 40 version 1929 is already newer than 1924 2024-04-12T11:27:32.755 INFO:tasks.rados.rados.0.smithi029.stdout:3114: oid 40 updating version 1929 to 1930 2024-04-12T11:27:32.755 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 1930 (ObjNum 1165 snap 395 seq_num 1165) dirty exists 2024-04-12T11:27:32.755 INFO:tasks.rados.rados.0.smithi029.stdout:3114: left oid 40 (ObjNum 1165 snap 395 seq_num 1165) 2024-04-12T11:27:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:32 smithi087 ceph-mon[89014]: pgmap v906: 89 pgs: 89 active+clean; 291 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 838 KiB/s wr, 6 op/s 2024-04-12T11:27:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:32 smithi087 ceph-mon[89014]: osdmap e847: 8 total, 8 up, 8 in 2024-04-12T11:27:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:32 smithi087 ceph-mon[89014]: osdmap e848: 8 total, 8 up, 8 in 2024-04-12T11:27:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:32 smithi029 ceph-mon[93650]: pgmap v906: 89 pgs: 89 active+clean; 291 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 838 KiB/s wr, 6 op/s 2024-04-12T11:27:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:32 smithi029 ceph-mon[93650]: osdmap e847: 8 total, 8 up, 8 in 2024-04-12T11:27:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:32 smithi029 ceph-mon[93650]: osdmap e848: 8 total, 8 up, 8 in 2024-04-12T11:27:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:32 smithi029 ceph-mon[97787]: pgmap v906: 89 pgs: 89 active+clean; 291 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 838 KiB/s wr, 6 op/s 2024-04-12T11:27:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:32 smithi029 ceph-mon[97787]: osdmap e847: 8 total, 8 up, 8 in 2024-04-12T11:27:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:32 smithi029 ceph-mon[97787]: osdmap e848: 8 total, 8 up, 8 in 2024-04-12T11:27:33.718 INFO:tasks.rados.rados.0.smithi029.stdout:3114: done (2 left) 2024-04-12T11:27:33.718 INFO:tasks.rados.rados.0.smithi029.stdout:3115: done (1 left) 2024-04-12T11:27:33.718 INFO:tasks.rados.rados.0.smithi029.stdout:3116: done (0 left) 2024-04-12T11:27:33.718 INFO:tasks.rados.rados.0.smithi029.stdout:3117: read oid 8 snap -1 2024-04-12T11:27:33.718 INFO:tasks.rados.rados.0.smithi029.stdout:3117: expect deleted 2024-04-12T11:27:33.718 INFO:tasks.rados.rados.0.smithi029.stdout:3118: write oid 31 current snap is 396 2024-04-12T11:27:33.718 INFO:tasks.rados.rados.0.smithi029.stdout:3118: seq_num 1166 ranges {796962=742343,2031416=437730,2936739=413167} 2024-04-12T11:27:33.726 INFO:tasks.rados.rados.0.smithi029.stdout:3118: writing smithi02979277-31 from 796962 to 1539305 tid 1 2024-04-12T11:27:33.731 INFO:tasks.rados.rados.0.smithi029.stdout:3118: writing smithi02979277-31 from 2031416 to 2469146 tid 2 2024-04-12T11:27:33.736 INFO:tasks.rados.rados.0.smithi029.stdout:3118: writing smithi02979277-31 from 2936739 to 3349906 tid 3 2024-04-12T11:27:33.736 INFO:tasks.rados.rados.0.smithi029.stdout:3119: snap_create 2024-04-12T11:27:33.736 INFO:tasks.rados.rados.0.smithi029.stdout:3118: finishing write tid 1 to smithi02979277-31 2024-04-12T11:27:33.736 INFO:tasks.rados.rados.0.smithi029.stdout:3118: finishing write tid 2 to smithi02979277-31 2024-04-12T11:27:33.739 INFO:tasks.rados.rados.0.smithi029.stdout:3118: finishing write tid 3 to smithi02979277-31 2024-04-12T11:27:33.739 INFO:tasks.rados.rados.0.smithi029.stdout:3118: finishing write tid 4 to smithi02979277-31 2024-04-12T11:27:33.739 INFO:tasks.rados.rados.0.smithi029.stdout:3118: finishing write tid 5 to smithi02979277-31 2024-04-12T11:27:33.739 INFO:tasks.rados.rados.0.smithi029.stdout:3118: oid 31 updating version 0 to 1171 2024-04-12T11:27:33.739 INFO:tasks.rados.rados.0.smithi029.stdout:3118: oid 31 version 1171 is already newer than 1170 2024-04-12T11:27:33.739 INFO:tasks.rados.rados.0.smithi029.stdout:3118: oid 31 updating version 1171 to 1172 2024-04-12T11:27:33.739 INFO:tasks.rados.rados.0.smithi029.stdout:3118: oid 31 version 1172 is already newer than 1169 2024-04-12T11:27:33.739 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1172 (ObjNum 1166 snap 396 seq_num 1166) dirty exists 2024-04-12T11:27:33.740 INFO:tasks.rados.rados.0.smithi029.stdout:3118: left oid 31 (ObjNum 1166 snap 396 seq_num 1166) 2024-04-12T11:27:34.725 INFO:tasks.rados.rados.0.smithi029.stdout:3117: done (2 left) 2024-04-12T11:27:34.725 INFO:tasks.rados.rados.0.smithi029.stdout:3118: done (1 left) 2024-04-12T11:27:34.725 INFO:tasks.rados.rados.0.smithi029.stdout:3119: done (0 left) 2024-04-12T11:27:34.725 INFO:tasks.rados.rados.0.smithi029.stdout:3120: snap_remove snap 391 2024-04-12T11:27:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:34 smithi087 ceph-mon[89014]: pgmap v909: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 301 MiB data, 922 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-04-12T11:27:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:34 smithi087 ceph-mon[89014]: osdmap e849: 8 total, 8 up, 8 in 2024-04-12T11:27:35.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:34 smithi029 ceph-mon[93650]: pgmap v909: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 301 MiB data, 922 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-04-12T11:27:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:34 smithi029 ceph-mon[93650]: osdmap e849: 8 total, 8 up, 8 in 2024-04-12T11:27:35.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:34 smithi029 ceph-mon[97787]: pgmap v909: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 301 MiB data, 922 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-04-12T11:27:35.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:34 smithi029 ceph-mon[97787]: osdmap e849: 8 total, 8 up, 8 in 2024-04-12T11:27:35.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:27:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:27:34] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:27:35.732 INFO:tasks.rados.rados.0.smithi029.stdout:3120: done (0 left) 2024-04-12T11:27:35.732 INFO:tasks.rados.rados.0.smithi029.stdout:3121: read oid 35 snap -1 2024-04-12T11:27:35.732 INFO:tasks.rados.rados.0.smithi029.stdout:3121: expect deleted 2024-04-12T11:27:35.732 INFO:tasks.rados.rados.0.smithi029.stdout:3122: snap_remove snap 386 2024-04-12T11:27:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:35 smithi029 ceph-mon[97787]: osdmap e850: 8 total, 8 up, 8 in 2024-04-12T11:27:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:35 smithi029 ceph-mon[93650]: osdmap e850: 8 total, 8 up, 8 in 2024-04-12T11:27:36.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:35 smithi087 ceph-mon[89014]: osdmap e850: 8 total, 8 up, 8 in 2024-04-12T11:27:36.733 INFO:tasks.rados.rados.0.smithi029.stdout:3122: done (1 left) 2024-04-12T11:27:36.733 INFO:tasks.rados.rados.0.smithi029.stdout:3123: write oid 42 current snap is 397 2024-04-12T11:27:36.733 INFO:tasks.rados.rados.0.smithi029.stdout:3123: seq_num 1167 ranges {413769=417592,1573545=697741,2847809=624439} 2024-04-12T11:27:36.738 INFO:tasks.rados.rados.0.smithi029.stdout:3123: writing smithi02979277-42 from 413769 to 831361 tid 1 2024-04-12T11:27:36.747 INFO:tasks.rados.rados.0.smithi029.stdout:3123: writing smithi02979277-42 from 1573545 to 2271286 tid 2 2024-04-12T11:27:36.754 INFO:tasks.rados.rados.0.smithi029.stdout:3123: writing smithi02979277-42 from 2847809 to 3472248 tid 3 2024-04-12T11:27:36.754 INFO:tasks.rados.rados.0.smithi029.stdout:3124: read oid 37 snap -1 2024-04-12T11:27:36.754 INFO:tasks.rados.rados.0.smithi029.stdout:3124: expect deleted 2024-04-12T11:27:36.754 INFO:tasks.rados.rados.0.smithi029.stdout:3125: read oid 27 snap -1 2024-04-12T11:27:36.754 INFO:tasks.rados.rados.0.smithi029.stdout:3125: expect (ObjNum 1135 snap 381 seq_num 1135) 2024-04-12T11:27:36.755 INFO:tasks.rados.rados.0.smithi029.stdout:3126: write oid 24 current snap is 397 2024-04-12T11:27:36.755 INFO:tasks.rados.rados.0.smithi029.stdout:3126: seq_num 1168 ranges {430576=492841,1632027=441399,2475232=435711,3365397=265179} 2024-04-12T11:27:36.760 INFO:tasks.rados.rados.0.smithi029.stdout:3126: writing smithi02979277-24 from 430576 to 923417 tid 1 2024-04-12T11:27:36.766 INFO:tasks.rados.rados.0.smithi029.stdout:3126: writing smithi02979277-24 from 1632027 to 2073426 tid 2 2024-04-12T11:27:36.771 INFO:tasks.rados.rados.0.smithi029.stdout:3126: writing smithi02979277-24 from 2475232 to 2910943 tid 3 2024-04-12T11:27:36.776 INFO:tasks.rados.rados.0.smithi029.stdout:3126: writing smithi02979277-24 from 3365397 to 3630576 tid 4 2024-04-12T11:27:36.776 INFO:tasks.rados.rados.0.smithi029.stdout:3127: read oid 13 snap 357 2024-04-12T11:27:36.776 INFO:tasks.rados.rados.0.smithi029.stdout:3127: expect (ObjNum 1044 snap 356 seq_num 1044) 2024-04-12T11:27:36.776 INFO:tasks.rados.rados.0.smithi029.stdout:3128: rollback oid 38 current snap is 397 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 38 to 393 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3123: finishing write tid 1 to smithi02979277-42 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3123: finishing write tid 2 to smithi02979277-42 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3121: done (6 left) 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3129: read oid 28 snap 389 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3129: expect (ObjNum 1108 snap 370 seq_num 1108) 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3130: rollback oid 30 current snap is 397 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 376 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3123: finishing write tid 3 to smithi02979277-42 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3123: finishing write tid 4 to smithi02979277-42 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3123: finishing write tid 5 to smithi02979277-42 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3123: oid 42 updating version 0 to 376 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3123: oid 42 version 376 is already newer than 375 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3123: oid 42 updating version 376 to 377 2024-04-12T11:27:36.777 INFO:tasks.rados.rados.0.smithi029.stdout:3123: oid 42 updating version 377 to 378 2024-04-12T11:27:36.778 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 378 (ObjNum 1167 snap 397 seq_num 1167) dirty exists 2024-04-12T11:27:36.778 INFO:tasks.rados.rados.0.smithi029.stdout:3123: left oid 42 (ObjNum 1167 snap 397 seq_num 1167) 2024-04-12T11:27:36.778 INFO:tasks.rados.rados.0.smithi029.stdout:3125: expect (ObjNum 1135 snap 381 seq_num 1135) 2024-04-12T11:27:36.880 INFO:tasks.rados.rados.0.smithi029.stdout:3126: finishing write tid 1 to smithi02979277-24 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3126: finishing write tid 2 to smithi02979277-24 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3126: finishing write tid 3 to smithi02979277-24 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3130: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3130: finishing rollback tid 2 to smithi02979277-30 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 966 (ObjNum 1016 snap 350 seq_num 1016) dirty exists 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3126: finishing write tid 4 to smithi02979277-24 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3123: done (7 left) 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3124: done (6 left) 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3125: done (5 left) 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3130: done (4 left) 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3131: read oid 1 snap -1 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3131: expect (ObjNum 1000 snap 345 seq_num 1000) 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3132: delete oid 9 current snap is 397 2024-04-12T11:27:36.881 INFO:tasks.rados.rados.0.smithi029.stdout:3128: finishing rollback tid 0 to smithi02979277-38 2024-04-12T11:27:36.882 INFO:tasks.rados.rados.0.smithi029.stdout:3129: expect (ObjNum 1108 snap 370 seq_num 1108) 2024-04-12T11:27:37.004 INFO:tasks.rados.rados.0.smithi029.stdout:3128: finishing rollback tid 1 to smithi02979277-38 2024-04-12T11:27:37.004 INFO:tasks.rados.rados.0.smithi029.stdout:3126: finishing write tid 5 to smithi02979277-24 2024-04-12T11:27:37.004 INFO:tasks.rados.rados.0.smithi029.stdout:3128: finishing rollback tid 2 to smithi02979277-38 2024-04-12T11:27:37.004 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 951 (ObjNum 1144 snap 387 seq_num 1144) dirty exists 2024-04-12T11:27:37.004 INFO:tasks.rados.rados.0.smithi029.stdout:3128: done (5 left) 2024-04-12T11:27:37.004 INFO:tasks.rados.rados.0.smithi029.stdout:3129: done (4 left) 2024-04-12T11:27:37.004 INFO:tasks.rados.rados.0.smithi029.stdout:3132: done (3 left) 2024-04-12T11:27:37.005 INFO:tasks.rados.rados.0.smithi029.stdout:3133: snap_remove snap 390 2024-04-12T11:27:37.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:36 smithi029 ceph-mon[93650]: pgmap v912: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 318 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-04-12T11:27:37.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:36 smithi029 ceph-mon[93650]: osdmap e851: 8 total, 8 up, 8 in 2024-04-12T11:27:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:36 smithi029 ceph-mon[97787]: pgmap v912: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 318 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-04-12T11:27:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:36 smithi029 ceph-mon[97787]: osdmap e851: 8 total, 8 up, 8 in 2024-04-12T11:27:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:36 smithi087 ceph-mon[89014]: pgmap v912: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 318 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-04-12T11:27:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:36 smithi087 ceph-mon[89014]: osdmap e851: 8 total, 8 up, 8 in 2024-04-12T11:27:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:37.659 INFO:tasks.rados.rados.0.smithi029.stdout:3133: done (3 left) 2024-04-12T11:27:37.659 INFO:tasks.rados.rados.0.smithi029.stdout:3134: snap_remove snap 381 2024-04-12T11:27:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:37 smithi029 ceph-mon[93650]: osdmap e852: 8 total, 8 up, 8 in 2024-04-12T11:27:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:37 smithi029 ceph-mon[93650]: osdmap e853: 8 total, 8 up, 8 in 2024-04-12T11:27:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:37 smithi029 ceph-mon[97787]: osdmap e852: 8 total, 8 up, 8 in 2024-04-12T11:27:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:37 smithi029 ceph-mon[97787]: osdmap e853: 8 total, 8 up, 8 in 2024-04-12T11:27:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:37 smithi087 ceph-mon[89014]: osdmap e852: 8 total, 8 up, 8 in 2024-04-12T11:27:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:37 smithi087 ceph-mon[89014]: osdmap e853: 8 total, 8 up, 8 in 2024-04-12T11:27:38.662 INFO:tasks.rados.rados.0.smithi029.stdout:3134: done (3 left) 2024-04-12T11:27:38.662 INFO:tasks.rados.rados.0.smithi029.stdout:3135: write oid 10 current snap is 397 2024-04-12T11:27:38.662 INFO:tasks.rados.rados.0.smithi029.stdout:3135: seq_num 1170 ranges {464190=643339,1748991=445068,2930078=574608} 2024-04-12T11:27:38.668 INFO:tasks.rados.rados.0.smithi029.stdout:3135: writing smithi02979277-10 from 464190 to 1107529 tid 1 2024-04-12T11:27:38.675 INFO:tasks.rados.rados.0.smithi029.stdout:3135: writing smithi02979277-10 from 1748991 to 2194059 tid 2 2024-04-12T11:27:38.682 INFO:tasks.rados.rados.0.smithi029.stdout:3135: writing smithi02979277-10 from 2930078 to 3504686 tid 3 2024-04-12T11:27:38.682 INFO:tasks.rados.rados.0.smithi029.stdout:3136: delete oid 9 current snap is 397 2024-04-12T11:27:38.683 INFO:tasks.rados.rados.0.smithi029.stdout:3126: finishing write tid 6 to smithi02979277-24 2024-04-12T11:27:38.683 INFO:tasks.rados.rados.0.smithi029.stdout:3126: oid 24 updating version 0 to 1946 2024-04-12T11:27:38.683 INFO:tasks.rados.rados.0.smithi029.stdout:3126: oid 24 version 1946 is already newer than 1943 2024-04-12T11:27:38.683 INFO:tasks.rados.rados.0.smithi029.stdout:3126: oid 24 version 1946 is already newer than 1944 2024-04-12T11:27:38.683 INFO:tasks.rados.rados.0.smithi029.stdout:3126: oid 24 version 1946 is already newer than 1942 2024-04-12T11:27:38.683 INFO:tasks.rados.rados.0.smithi029.stdout:3126: oid 24 version 1946 is already newer than 1945 2024-04-12T11:27:38.683 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 1946 (ObjNum 1168 snap 397 seq_num 1168) dirty exists 2024-04-12T11:27:38.683 INFO:tasks.rados.rados.0.smithi029.stdout:3126: left oid 24 (ObjNum 1168 snap 397 seq_num 1168) 2024-04-12T11:27:38.683 INFO:tasks.rados.rados.0.smithi029.stdout:3127: expect (ObjNum 1044 snap 356 seq_num 1044) 2024-04-12T11:27:38.847 INFO:tasks.rados.rados.0.smithi029.stdout:3131: expect (ObjNum 1000 snap 345 seq_num 1000) 2024-04-12T11:27:38.978 INFO:tasks.rados.rados.0.smithi029.stdout:3135: finishing write tid 1 to smithi02979277-10 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3135: finishing write tid 2 to smithi02979277-10 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3135: finishing write tid 3 to smithi02979277-10 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3135: finishing write tid 4 to smithi02979277-10 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3135: finishing write tid 5 to smithi02979277-10 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3135: oid 10 updating version 0 to 1201 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3135: oid 10 version 1201 is already newer than 1199 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3135: oid 10 version 1201 is already newer than 1200 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3135: oid 10 version 1201 is already newer than 1198 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 1201 (ObjNum 1170 snap 397 seq_num 1170) dirty exists 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3135: left oid 10 (ObjNum 1170 snap 397 seq_num 1170) 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3126: done (4 left) 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3127: done (3 left) 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3131: done (2 left) 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3135: done (1 left) 2024-04-12T11:27:38.979 INFO:tasks.rados.rados.0.smithi029.stdout:3136: done (0 left) 2024-04-12T11:27:38.980 INFO:tasks.rados.rados.0.smithi029.stdout:3137: delete oid 32 current snap is 397 2024-04-12T11:27:38.981 INFO:tasks.rados.rados.0.smithi029.stdout:3137: done (0 left) 2024-04-12T11:27:38.981 INFO:tasks.rados.rados.0.smithi029.stdout:3138: snap_create 2024-04-12T11:27:39.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:38 smithi029 ceph-mon[93650]: pgmap v915: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 315 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-04-12T11:27:39.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:38 smithi029 ceph-mon[93650]: osdmap e854: 8 total, 8 up, 8 in 2024-04-12T11:27:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:38 smithi029 ceph-mon[97787]: pgmap v915: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 315 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-04-12T11:27:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:38 smithi029 ceph-mon[97787]: osdmap e854: 8 total, 8 up, 8 in 2024-04-12T11:27:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:38 smithi087 ceph-mon[89014]: pgmap v915: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 315 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-04-12T11:27:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:38 smithi087 ceph-mon[89014]: osdmap e854: 8 total, 8 up, 8 in 2024-04-12T11:27:39.751 INFO:tasks.rados.rados.0.smithi029.stdout:3138: done (0 left) 2024-04-12T11:27:39.752 INFO:tasks.rados.rados.0.smithi029.stdout:3139: rollback oid 4 current snap is 398 2024-04-12T11:27:39.752 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 4 to 396 2024-04-12T11:27:39.752 INFO:tasks.rados.rados.0.smithi029.stdout:3140: snap_create 2024-04-12T11:27:39.757 INFO:tasks.rados.rados.0.smithi029.stdout:3139: finishing rollback tid 1 to smithi02979277-4 2024-04-12T11:27:39.757 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1386 (ObjNum 49 snap 0 seq_num 1777349616) dirty dne 2024-04-12T11:27:40.754 INFO:tasks.rados.rados.0.smithi029.stdout:3139: done (1 left) 2024-04-12T11:27:40.754 INFO:tasks.rados.rados.0.smithi029.stdout:3140: done (0 left) 2024-04-12T11:27:40.754 INFO:tasks.rados.rados.0.smithi029.stdout:3141: rollback oid 18 current snap is 399 2024-04-12T11:27:40.754 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 18 to 385 2024-04-12T11:27:40.754 INFO:tasks.rados.rados.0.smithi029.stdout:3142: read oid 29 snap 397 2024-04-12T11:27:40.754 INFO:tasks.rados.rados.0.smithi029.stdout:3142: expect (ObjNum 1159 snap 394 seq_num 1159) 2024-04-12T11:27:40.754 INFO:tasks.rados.rados.0.smithi029.stdout:3143: snap_create 2024-04-12T11:27:40.762 INFO:tasks.rados.rados.0.smithi029.stdout:3141: finishing rollback tid 0 to smithi02979277-18 2024-04-12T11:27:40.762 INFO:tasks.rados.rados.0.smithi029.stdout:3141: finishing rollback tid 1 to smithi02979277-18 2024-04-12T11:27:40.762 INFO:tasks.rados.rados.0.smithi029.stdout:3141: finishing rollback tid 2 to smithi02979277-18 2024-04-12T11:27:40.763 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1563 (ObjNum 1127 snap 379 seq_num 1127) dirty exists 2024-04-12T11:27:40.765 INFO:tasks.rados.rados.0.smithi029.stdout:3142: expect (ObjNum 1159 snap 394 seq_num 1159) 2024-04-12T11:27:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:40 smithi029 ceph-mon[93650]: pgmap v918: 89 pgs: 89 active+clean; 321 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 789 KiB/s wr, 13 op/s 2024-04-12T11:27:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:40 smithi029 ceph-mon[93650]: osdmap e855: 8 total, 8 up, 8 in 2024-04-12T11:27:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:40 smithi029 ceph-mon[97787]: pgmap v918: 89 pgs: 89 active+clean; 321 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 789 KiB/s wr, 13 op/s 2024-04-12T11:27:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:40 smithi029 ceph-mon[97787]: osdmap e855: 8 total, 8 up, 8 in 2024-04-12T11:27:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:40 smithi087 ceph-mon[89014]: pgmap v918: 89 pgs: 89 active+clean; 321 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 789 KiB/s wr, 13 op/s 2024-04-12T11:27:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:40 smithi087 ceph-mon[89014]: osdmap e855: 8 total, 8 up, 8 in 2024-04-12T11:27:41.764 INFO:tasks.rados.rados.0.smithi029.stdout:3141: done (2 left) 2024-04-12T11:27:41.764 INFO:tasks.rados.rados.0.smithi029.stdout:3142: done (1 left) 2024-04-12T11:27:41.764 INFO:tasks.rados.rados.0.smithi029.stdout:3143: done (0 left) 2024-04-12T11:27:41.764 INFO:tasks.rados.rados.0.smithi029.stdout:3144: read oid 26 snap -1 2024-04-12T11:27:41.764 INFO:tasks.rados.rados.0.smithi029.stdout:3144: expect (ObjNum 1075 snap 364 seq_num 1075) 2024-04-12T11:27:41.765 INFO:tasks.rados.rados.0.smithi029.stdout:3145: snap_remove snap 394 2024-04-12T11:27:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:41 smithi029 ceph-mon[93650]: osdmap e856: 8 total, 8 up, 8 in 2024-04-12T11:27:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:41 smithi029 ceph-mon[97787]: osdmap e856: 8 total, 8 up, 8 in 2024-04-12T11:27:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:41 smithi087 ceph-mon[89014]: osdmap e856: 8 total, 8 up, 8 in 2024-04-12T11:27:42.660 INFO:tasks.rados.rados.0.smithi029.stdout:3145: done (1 left) 2024-04-12T11:27:42.660 INFO:tasks.rados.rados.0.smithi029.stdout:3146: write oid 47 current snap is 400 2024-04-12T11:27:42.660 INFO:tasks.rados.rados.0.smithi029.stdout:3146: seq_num 1173 ranges {514611=585439,1524437=592395,2895994=408424} 2024-04-12T11:27:42.667 INFO:tasks.rados.rados.0.smithi029.stdout:3146: writing smithi02979277-47 from 514611 to 1100050 tid 1 2024-04-12T11:27:42.674 INFO:tasks.rados.rados.0.smithi029.stdout:3146: writing smithi02979277-47 from 1524437 to 2116832 tid 2 2024-04-12T11:27:42.681 INFO:tasks.rados.rados.0.smithi029.stdout:3146: writing smithi02979277-47 from 2895994 to 3304418 tid 3 2024-04-12T11:27:42.681 INFO:tasks.rados.rados.0.smithi029.stdout:3147: snap_remove snap 393 2024-04-12T11:27:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:27:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:27:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:27:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:42 smithi029 ceph-mon[93650]: pgmap v921: 89 pgs: 89 active+clean; 321 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-04-12T11:27:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:42 smithi029 ceph-mon[93650]: osdmap e857: 8 total, 8 up, 8 in 2024-04-12T11:27:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:42 smithi029 ceph-mon[93650]: osdmap e858: 8 total, 8 up, 8 in 2024-04-12T11:27:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:42 smithi029 ceph-mon[97787]: pgmap v921: 89 pgs: 89 active+clean; 321 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-04-12T11:27:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:42 smithi029 ceph-mon[97787]: osdmap e857: 8 total, 8 up, 8 in 2024-04-12T11:27:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:42 smithi029 ceph-mon[97787]: osdmap e858: 8 total, 8 up, 8 in 2024-04-12T11:27:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:42 smithi087 ceph-mon[89014]: pgmap v921: 89 pgs: 89 active+clean; 321 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-04-12T11:27:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:42 smithi087 ceph-mon[89014]: osdmap e857: 8 total, 8 up, 8 in 2024-04-12T11:27:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:42 smithi087 ceph-mon[89014]: osdmap e858: 8 total, 8 up, 8 in 2024-04-12T11:27:43.663 INFO:tasks.rados.rados.0.smithi029.stdout:3144: expect (ObjNum 1075 snap 364 seq_num 1075) 2024-04-12T11:27:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3146: finishing write tid 1 to smithi02979277-47 2024-04-12T11:27:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3146: finishing write tid 2 to smithi02979277-47 2024-04-12T11:27:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3146: finishing write tid 3 to smithi02979277-47 2024-04-12T11:27:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3146: finishing write tid 4 to smithi02979277-47 2024-04-12T11:27:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3144: done (2 left) 2024-04-12T11:27:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3147: done (1 left) 2024-04-12T11:27:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3148: snap_remove snap 397 2024-04-12T11:27:44.767 INFO:tasks.rados.rados.0.smithi029.stdout:3148: done (1 left) 2024-04-12T11:27:44.767 INFO:tasks.rados.rados.0.smithi029.stdout:3149: delete oid 3 current snap is 400 2024-04-12T11:27:44.767 INFO:tasks.rados.rados.0.smithi029.stdout:3146: finishing write tid 5 to smithi02979277-47 2024-04-12T11:27:44.767 INFO:tasks.rados.rados.0.smithi029.stdout:3146: oid 47 updating version 0 to 1474 2024-04-12T11:27:44.767 INFO:tasks.rados.rados.0.smithi029.stdout:3146: oid 47 version 1474 is already newer than 1471 2024-04-12T11:27:44.767 INFO:tasks.rados.rados.0.smithi029.stdout:3146: oid 47 version 1474 is already newer than 1472 2024-04-12T11:27:44.767 INFO:tasks.rados.rados.0.smithi029.stdout:3146: oid 47 version 1474 is already newer than 1473 2024-04-12T11:27:44.768 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1474 (ObjNum 1173 snap 400 seq_num 1173) dirty exists 2024-04-12T11:27:44.768 INFO:tasks.rados.rados.0.smithi029.stdout:3146: left oid 47 (ObjNum 1173 snap 400 seq_num 1173) 2024-04-12T11:27:44.773 INFO:tasks.rados.rados.0.smithi029.stdout:3146: done (1 left) 2024-04-12T11:27:44.773 INFO:tasks.rados.rados.0.smithi029.stdout:3149: done (0 left) 2024-04-12T11:27:44.773 INFO:tasks.rados.rados.0.smithi029.stdout:3150: rollback oid 49 current snap is 400 2024-04-12T11:27:44.773 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 49 to 374 2024-04-12T11:27:44.773 INFO:tasks.rados.rados.0.smithi029.stdout:3151: rollback oid 18 current snap is 400 2024-04-12T11:27:44.773 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 18 to 379 2024-04-12T11:27:44.773 INFO:tasks.rados.rados.0.smithi029.stdout:3152: rollback oid 41 current snap is 400 2024-04-12T11:27:44.773 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 380 2024-04-12T11:27:44.773 INFO:tasks.rados.rados.0.smithi029.stdout:3153: read oid 30 snap -1 2024-04-12T11:27:44.774 INFO:tasks.rados.rados.0.smithi029.stdout:3153: expect (ObjNum 1016 snap 350 seq_num 1016) 2024-04-12T11:27:44.774 INFO:tasks.rados.rados.0.smithi029.stdout:3154: read oid 25 snap -1 2024-04-12T11:27:44.774 INFO:tasks.rados.rados.0.smithi029.stdout:3154: expect (ObjNum 1156 snap 394 seq_num 1156) 2024-04-12T11:27:44.774 INFO:tasks.rados.rados.0.smithi029.stdout:3155: write oid 37 current snap is 400 2024-04-12T11:27:44.774 INFO:tasks.rados.rados.0.smithi029.stdout:3155: seq_num 1175 ranges {548225=735937,1925048=596064,2950840=547321} 2024-04-12T11:27:44.781 INFO:tasks.rados.rados.0.smithi029.stdout:3155: writing smithi02979277-37 from 548225 to 1284162 tid 1 2024-04-12T11:27:44.789 INFO:tasks.rados.rados.0.smithi029.stdout:3155: writing smithi02979277-37 from 1925048 to 2521112 tid 2 2024-04-12T11:27:44.795 INFO:tasks.rados.rados.0.smithi029.stdout:3155: writing smithi02979277-37 from 2950840 to 3498161 tid 3 2024-04-12T11:27:44.795 INFO:tasks.rados.rados.0.smithi029.stdout:3156: write oid 23 current snap is 400 2024-04-12T11:27:44.795 INFO:tasks.rados.rados.0.smithi029.stdout:3152: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:27:44.795 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1421 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-04-12T11:27:44.795 INFO:tasks.rados.rados.0.smithi029.stdout:3156: seq_num 1176 ranges {565032=411186,1583530=456075,2578263=758593} 2024-04-12T11:27:44.800 INFO:tasks.rados.rados.0.smithi029.stdout:3156: writing smithi02979277-23 from 565032 to 976218 tid 1 2024-04-12T11:27:44.806 INFO:tasks.rados.rados.0.smithi029.stdout:3156: writing smithi02979277-23 from 1583530 to 2039605 tid 2 2024-04-12T11:27:44.814 INFO:tasks.rados.rados.0.smithi029.stdout:3156: writing smithi02979277-23 from 2578263 to 3336856 tid 3 2024-04-12T11:27:44.814 INFO:tasks.rados.rados.0.smithi029.stdout:3152: done (6 left) 2024-04-12T11:27:44.814 INFO:tasks.rados.rados.0.smithi029.stdout:3157: write oid 36 current snap is 400 2024-04-12T11:27:44.814 INFO:tasks.rados.rados.0.smithi029.stdout:3150: finishing rollback tid 1 to smithi02979277-49 2024-04-12T11:27:44.814 INFO:tasks.rados.rados.0.smithi029.stdout:3150: finishing rollback tid 2 to smithi02979277-49 2024-04-12T11:27:44.814 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 1125 (ObjNum 1097 snap 368 seq_num 1097) dirty exists 2024-04-12T11:27:44.814 INFO:tasks.rados.rados.0.smithi029.stdout:3153: expect (ObjNum 1016 snap 350 seq_num 1016) 2024-04-12T11:27:44.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:44 smithi029 ceph-mon[97787]: pgmap v924: 89 pgs: 89 active+clean; 321 MiB data, 922 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 425 KiB/s wr, 1 op/s 2024-04-12T11:27:44.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:44 smithi029 ceph-mon[97787]: osdmap e859: 8 total, 8 up, 8 in 2024-04-12T11:27:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:44 smithi029 ceph-mon[93650]: pgmap v924: 89 pgs: 89 active+clean; 321 MiB data, 922 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 425 KiB/s wr, 1 op/s 2024-04-12T11:27:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:44 smithi029 ceph-mon[93650]: osdmap e859: 8 total, 8 up, 8 in 2024-04-12T11:27:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:44 smithi087 ceph-mon[89014]: pgmap v924: 89 pgs: 89 active+clean; 321 MiB data, 922 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 425 KiB/s wr, 1 op/s 2024-04-12T11:27:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:44 smithi087 ceph-mon[89014]: osdmap e859: 8 total, 8 up, 8 in 2024-04-12T11:27:44.993 INFO:tasks.rados.rados.0.smithi029.stdout:3151: finishing rollback tid 0 to smithi02979277-18 2024-04-12T11:27:44.993 INFO:tasks.rados.rados.0.smithi029.stdout:3151: finishing rollback tid 1 to smithi02979277-18 2024-04-12T11:27:44.993 INFO:tasks.rados.rados.0.smithi029.stdout:3151: finishing rollback tid 2 to smithi02979277-18 2024-04-12T11:27:44.993 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1579 (ObjNum 1127 snap 379 seq_num 1127) dirty exists 2024-04-12T11:27:44.993 INFO:tasks.rados.rados.0.smithi029.stdout:3155: finishing write tid 1 to smithi02979277-37 2024-04-12T11:27:44.993 INFO:tasks.rados.rados.0.smithi029.stdout:3157: seq_num 1177 ranges {581839=486435,1642012=599733,3005686=569865} 2024-04-12T11:27:44.999 INFO:tasks.rados.rados.0.smithi029.stdout:3157: writing smithi02979277-36 from 581839 to 1068274 tid 1 2024-04-12T11:27:45.005 INFO:tasks.rados.rados.0.smithi029.stdout:3157: writing smithi02979277-36 from 1642012 to 2241745 tid 2 2024-04-12T11:27:45.013 INFO:tasks.rados.rados.0.smithi029.stdout:3157: writing smithi02979277-36 from 3005686 to 3575551 tid 3 2024-04-12T11:27:45.013 INFO:tasks.rados.rados.0.smithi029.stdout:3150: done (6 left) 2024-04-12T11:27:45.013 INFO:tasks.rados.rados.0.smithi029.stdout:3151: done (5 left) 2024-04-12T11:27:45.013 INFO:tasks.rados.rados.0.smithi029.stdout:3153: done (4 left) 2024-04-12T11:27:45.013 INFO:tasks.rados.rados.0.smithi029.stdout:3158: read oid 19 snap 389 2024-04-12T11:27:45.013 INFO:tasks.rados.rados.0.smithi029.stdout:3158: expect (ObjNum 1141 snap 387 seq_num 1141) 2024-04-12T11:27:45.013 INFO:tasks.rados.rados.0.smithi029.stdout:3159: delete oid 1 current snap is 400 2024-04-12T11:27:45.013 INFO:tasks.rados.rados.0.smithi029.stdout:3154: expect (ObjNum 1156 snap 394 seq_num 1156) 2024-04-12T11:27:45.171 INFO:tasks.rados.rados.0.smithi029.stdout:3155: finishing write tid 2 to smithi02979277-37 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3155: finishing write tid 3 to smithi02979277-37 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3155: finishing write tid 4 to smithi02979277-37 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3155: finishing write tid 5 to smithi02979277-37 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3155: oid 37 updating version 0 to 1126 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3155: oid 37 updating version 1126 to 1129 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3155: oid 37 version 1129 is already newer than 1127 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3155: oid 37 version 1129 is already newer than 1128 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 1129 (ObjNum 1175 snap 400 seq_num 1175) dirty exists 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3155: left oid 37 (ObjNum 1175 snap 400 seq_num 1175) 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3156: finishing write tid 1 to smithi02979277-23 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3156: finishing write tid 2 to smithi02979277-23 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3156: finishing write tid 3 to smithi02979277-23 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3156: finishing write tid 4 to smithi02979277-23 2024-04-12T11:27:45.172 INFO:tasks.rados.rados.0.smithi029.stdout:3156: finishing write tid 5 to smithi02979277-23 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3156: oid 23 updating version 0 to 1276 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3156: oid 23 updating version 1276 to 1278 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3156: oid 23 version 1278 is already newer than 1277 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3156: oid 23 version 1278 is already newer than 1275 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1278 (ObjNum 1176 snap 400 seq_num 1176) dirty exists 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3156: left oid 23 (ObjNum 1176 snap 400 seq_num 1176) 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3157: finishing write tid 1 to smithi02979277-36 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3157: finishing write tid 2 to smithi02979277-36 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3154: done (5 left) 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3155: done (4 left) 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3156: done (3 left) 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3159: done (2 left) 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3160: write oid 50 current snap is 400 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3157: finishing write tid 3 to smithi02979277-36 2024-04-12T11:27:45.173 INFO:tasks.rados.rados.0.smithi029.stdout:3157: finishing write tid 4 to smithi02979277-36 2024-04-12T11:27:45.174 INFO:tasks.rados.rados.0.smithi029.stdout:3157: finishing write tid 5 to smithi02979277-36 2024-04-12T11:27:45.174 INFO:tasks.rados.rados.0.smithi029.stdout:3157: oid 36 updating version 0 to 787 2024-04-12T11:27:45.174 INFO:tasks.rados.rados.0.smithi029.stdout:3157: oid 36 updating version 787 to 788 2024-04-12T11:27:45.174 INFO:tasks.rados.rados.0.smithi029.stdout:3157: oid 36 version 788 is already newer than 785 2024-04-12T11:27:45.174 INFO:tasks.rados.rados.0.smithi029.stdout:3157: oid 36 version 788 is already newer than 786 2024-04-12T11:27:45.174 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 788 (ObjNum 1177 snap 400 seq_num 1177) dirty exists 2024-04-12T11:27:45.174 INFO:tasks.rados.rados.0.smithi029.stdout:3157: left oid 36 (ObjNum 1177 snap 400 seq_num 1177) 2024-04-12T11:27:45.174 INFO:tasks.rados.rados.0.smithi029.stdout:3158: expect (ObjNum 1141 snap 387 seq_num 1141) 2024-04-12T11:27:45.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:27:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:27:44] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:27:45.277 INFO:tasks.rados.rados.0.smithi029.stdout:3160: seq_num 1179 ranges {615453=753286,2158976=603402,3460532=354921} 2024-04-12T11:27:45.284 INFO:tasks.rados.rados.0.smithi029.stdout:3160: writing smithi02979277-50 from 615453 to 1368739 tid 1 2024-04-12T11:27:45.292 INFO:tasks.rados.rados.0.smithi029.stdout:3160: writing smithi02979277-50 from 2158976 to 2762378 tid 2 2024-04-12T11:27:45.298 INFO:tasks.rados.rados.0.smithi029.stdout:3160: writing smithi02979277-50 from 3460532 to 3815453 tid 3 2024-04-12T11:27:45.298 INFO:tasks.rados.rados.0.smithi029.stdout:3157: done (2 left) 2024-04-12T11:27:45.298 INFO:tasks.rados.rados.0.smithi029.stdout:3158: done (1 left) 2024-04-12T11:27:45.298 INFO:tasks.rados.rados.0.smithi029.stdout:3161: read oid 34 snap 398 2024-04-12T11:27:45.298 INFO:tasks.rados.rados.0.smithi029.stdout:3161: expect deleted 2024-04-12T11:27:45.298 INFO:tasks.rados.rados.0.smithi029.stdout:3162: delete oid 37 current snap is 400 2024-04-12T11:27:45.298 INFO:tasks.rados.rados.0.smithi029.stdout:3160: finishing write tid 1 to smithi02979277-50 2024-04-12T11:27:45.298 INFO:tasks.rados.rados.0.smithi029.stdout:3160: finishing write tid 2 to smithi02979277-50 2024-04-12T11:27:45.300 INFO:tasks.rados.rados.0.smithi029.stdout:3161: done (2 left) 2024-04-12T11:27:45.300 INFO:tasks.rados.rados.0.smithi029.stdout:3162: done (1 left) 2024-04-12T11:27:45.300 INFO:tasks.rados.rados.0.smithi029.stdout:3163: read oid 3 snap -1 2024-04-12T11:27:45.300 INFO:tasks.rados.rados.0.smithi029.stdout:3163: expect deleted 2024-04-12T11:27:45.300 INFO:tasks.rados.rados.0.smithi029.stdout:3164: write oid 19 current snap is 400 2024-04-12T11:27:45.300 INFO:tasks.rados.rados.0.smithi029.stdout:3164: seq_num 1181 ranges {649067=503784,1875940=607071,3115378=731306} 2024-04-12T11:27:45.306 INFO:tasks.rados.rados.0.smithi029.stdout:3164: writing smithi02979277-19 from 649067 to 1152851 tid 1 2024-04-12T11:27:45.313 INFO:tasks.rados.rados.0.smithi029.stdout:3164: writing smithi02979277-19 from 1875940 to 2483011 tid 2 2024-04-12T11:27:45.321 INFO:tasks.rados.rados.0.smithi029.stdout:3164: writing smithi02979277-19 from 3115378 to 3846684 tid 3 2024-04-12T11:27:45.321 INFO:tasks.rados.rados.0.smithi029.stdout:3165: write oid 2 current snap is 400 2024-04-12T11:27:45.321 INFO:tasks.rados.rados.0.smithi029.stdout:3165: seq_num 1182 ranges {665874=579033,1934422=467082,3142801=542578} 2024-04-12T11:27:45.328 INFO:tasks.rados.rados.0.smithi029.stdout:3165: writing smithi02979277-2 from 665874 to 1244907 tid 1 2024-04-12T11:27:45.334 INFO:tasks.rados.rados.0.smithi029.stdout:3165: writing smithi02979277-2 from 1934422 to 2401504 tid 2 2024-04-12T11:27:45.341 INFO:tasks.rados.rados.0.smithi029.stdout:3165: writing smithi02979277-2 from 3142801 to 3685379 tid 3 2024-04-12T11:27:45.342 INFO:tasks.rados.rados.0.smithi029.stdout:3166: read oid 35 snap -1 2024-04-12T11:27:45.342 INFO:tasks.rados.rados.0.smithi029.stdout:3166: expect deleted 2024-04-12T11:27:45.342 INFO:tasks.rados.rados.0.smithi029.stdout:3167: write oid 9 current snap is 400 2024-04-12T11:27:45.342 INFO:tasks.rados.rados.0.smithi029.stdout:3167: seq_num 1183 ranges {682681=654282,1876551=610740,3053871=470203} 2024-04-12T11:27:45.349 INFO:tasks.rados.rados.0.smithi029.stdout:3167: writing smithi02979277-9 from 682681 to 1336963 tid 1 2024-04-12T11:27:45.355 INFO:tasks.rados.rados.0.smithi029.stdout:3167: writing smithi02979277-9 from 1876551 to 2487291 tid 2 2024-04-12T11:27:45.362 INFO:tasks.rados.rados.0.smithi029.stdout:3167: writing smithi02979277-9 from 3053871 to 3524074 tid 3 2024-04-12T11:27:45.362 INFO:tasks.rados.rados.0.smithi029.stdout:3168: read oid 32 snap 396 2024-04-12T11:27:45.362 INFO:tasks.rados.rados.0.smithi029.stdout:3168: expect (ObjNum 1043 snap 356 seq_num 1043) 2024-04-12T11:27:45.362 INFO:tasks.rados.rados.0.smithi029.stdout:3169: snap_create 2024-04-12T11:27:45.362 INFO:tasks.rados.rados.0.smithi029.stdout:3160: finishing write tid 3 to smithi02979277-50 2024-04-12T11:27:45.362 INFO:tasks.rados.rados.0.smithi029.stdout:3160: finishing write tid 4 to smithi02979277-50 2024-04-12T11:27:45.362 INFO:tasks.rados.rados.0.smithi029.stdout:3160: finishing write tid 5 to smithi02979277-50 2024-04-12T11:27:45.362 INFO:tasks.rados.rados.0.smithi029.stdout:3160: oid 50 updating version 0 to 1281 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3160: oid 50 updating version 1281 to 1283 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3160: oid 50 version 1283 is already newer than 1282 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3160: oid 50 version 1283 is already newer than 1280 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1283 (ObjNum 1179 snap 400 seq_num 1179) dirty exists 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3160: left oid 50 (ObjNum 1179 snap 400 seq_num 1179) 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3164: finishing write tid 1 to smithi02979277-19 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3164: finishing write tid 2 to smithi02979277-19 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3164: finishing write tid 3 to smithi02979277-19 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3164: finishing write tid 4 to smithi02979277-19 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3164: finishing write tid 5 to smithi02979277-19 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3164: oid 19 updating version 0 to 1413 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3164: oid 19 updating version 1413 to 1416 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3164: oid 19 version 1416 is already newer than 1414 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3164: oid 19 version 1416 is already newer than 1415 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1416 (ObjNum 1181 snap 400 seq_num 1181) dirty exists 2024-04-12T11:27:45.363 INFO:tasks.rados.rados.0.smithi029.stdout:3164: left oid 19 (ObjNum 1181 snap 400 seq_num 1181) 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3165: finishing write tid 1 to smithi02979277-2 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3165: finishing write tid 2 to smithi02979277-2 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3165: finishing write tid 3 to smithi02979277-2 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3165: finishing write tid 4 to smithi02979277-2 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3165: finishing write tid 5 to smithi02979277-2 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3165: oid 2 updating version 0 to 340 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3165: oid 2 updating version 340 to 343 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3165: oid 2 version 343 is already newer than 341 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3165: oid 2 version 343 is already newer than 342 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 343 (ObjNum 1182 snap 400 seq_num 1182) dirty exists 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3165: left oid 2 (ObjNum 1182 snap 400 seq_num 1182) 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3167: finishing write tid 1 to smithi02979277-9 2024-04-12T11:27:45.364 INFO:tasks.rados.rados.0.smithi029.stdout:3167: finishing write tid 2 to smithi02979277-9 2024-04-12T11:27:45.365 INFO:tasks.rados.rados.0.smithi029.stdout:3167: finishing write tid 3 to smithi02979277-9 2024-04-12T11:27:45.365 INFO:tasks.rados.rados.0.smithi029.stdout:3167: finishing write tid 4 to smithi02979277-9 2024-04-12T11:27:45.365 INFO:tasks.rados.rados.0.smithi029.stdout:3167: finishing write tid 5 to smithi02979277-9 2024-04-12T11:27:45.365 INFO:tasks.rados.rados.0.smithi029.stdout:3167: oid 9 updating version 0 to 1134 2024-04-12T11:27:45.365 INFO:tasks.rados.rados.0.smithi029.stdout:3167: oid 9 version 1134 is already newer than 1132 2024-04-12T11:27:45.365 INFO:tasks.rados.rados.0.smithi029.stdout:3167: oid 9 version 1134 is already newer than 1131 2024-04-12T11:27:45.365 INFO:tasks.rados.rados.0.smithi029.stdout:3167: oid 9 version 1134 is already newer than 1133 2024-04-12T11:27:45.366 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 1134 (ObjNum 1183 snap 400 seq_num 1183) dirty exists 2024-04-12T11:27:45.366 INFO:tasks.rados.rados.0.smithi029.stdout:3167: left oid 9 (ObjNum 1183 snap 400 seq_num 1183) 2024-04-12T11:27:45.366 INFO:tasks.rados.rados.0.smithi029.stdout:3168: expect (ObjNum 1043 snap 356 seq_num 1043) 2024-04-12T11:27:45.771 INFO:tasks.rados.rados.0.smithi029.stdout:3160: done (7 left) 2024-04-12T11:27:45.772 INFO:tasks.rados.rados.0.smithi029.stdout:3163: done (6 left) 2024-04-12T11:27:45.772 INFO:tasks.rados.rados.0.smithi029.stdout:3164: done (5 left) 2024-04-12T11:27:45.772 INFO:tasks.rados.rados.0.smithi029.stdout:3165: done (4 left) 2024-04-12T11:27:45.772 INFO:tasks.rados.rados.0.smithi029.stdout:3166: done (3 left) 2024-04-12T11:27:45.772 INFO:tasks.rados.rados.0.smithi029.stdout:3167: done (2 left) 2024-04-12T11:27:45.772 INFO:tasks.rados.rados.0.smithi029.stdout:3168: done (1 left) 2024-04-12T11:27:45.772 INFO:tasks.rados.rados.0.smithi029.stdout:3169: done (0 left) 2024-04-12T11:27:45.772 INFO:tasks.rados.rados.0.smithi029.stdout:3170: write oid 1 current snap is 401 2024-04-12T11:27:45.772 INFO:tasks.rados.rados.0.smithi029.stdout:3170: seq_num 1184 ranges {699488=729531,1935033=470751,3081294=681475} 2024-04-12T11:27:45.779 INFO:tasks.rados.rados.0.smithi029.stdout:3170: writing smithi02979277-1 from 699488 to 1429019 tid 1 2024-04-12T11:27:45.784 INFO:tasks.rados.rados.0.smithi029.stdout:3170: writing smithi02979277-1 from 1935033 to 2405784 tid 2 2024-04-12T11:27:45.792 INFO:tasks.rados.rados.0.smithi029.stdout:3170: writing smithi02979277-1 from 3081294 to 3762769 tid 3 2024-04-12T11:27:45.792 INFO:tasks.rados.rados.0.smithi029.stdout:3171: write oid 38 current snap is 401 2024-04-12T11:27:45.792 INFO:tasks.rados.rados.0.smithi029.stdout:3171: seq_num 1185 ranges {716295=404780,1593515=614409,2708717=492747,3640519=275776} 2024-04-12T11:27:45.798 INFO:tasks.rados.rados.0.smithi029.stdout:3171: writing smithi02979277-38 from 716295 to 1121075 tid 1 2024-04-12T11:27:45.805 INFO:tasks.rados.rados.0.smithi029.stdout:3171: writing smithi02979277-38 from 1593515 to 2207924 tid 2 2024-04-12T11:27:45.810 INFO:tasks.rados.rados.0.smithi029.stdout:3171: writing smithi02979277-38 from 2708717 to 3201464 tid 3 2024-04-12T11:27:45.814 INFO:tasks.rados.rados.0.smithi029.stdout:3171: writing smithi02979277-38 from 3640519 to 3916295 tid 4 2024-04-12T11:27:45.815 INFO:tasks.rados.rados.0.smithi029.stdout:3172: read oid 48 snap -1 2024-04-12T11:27:45.815 INFO:tasks.rados.rados.0.smithi029.stdout:3172: expect (ObjNum 1107 snap 370 seq_num 1107) 2024-04-12T11:27:45.815 INFO:tasks.rados.rados.0.smithi029.stdout:3173: write oid 13 current snap is 401 2024-04-12T11:27:45.815 INFO:tasks.rados.rados.0.smithi029.stdout:3170: finishing write tid 1 to smithi02979277-1 2024-04-12T11:27:45.815 INFO:tasks.rados.rados.0.smithi029.stdout:3173: seq_num 1186 ranges {733102=596382,1768350=758067,3136140=704019} 2024-04-12T11:27:45.822 INFO:tasks.rados.rados.0.smithi029.stdout:3173: writing smithi02979277-13 from 733102 to 1329484 tid 1 2024-04-12T11:27:45.829 INFO:tasks.rados.rados.0.smithi029.stdout:3173: writing smithi02979277-13 from 1768350 to 2526417 tid 2 2024-04-12T11:27:45.836 INFO:tasks.rados.rados.0.smithi029.stdout:3173: writing smithi02979277-13 from 3136140 to 3840159 tid 3 2024-04-12T11:27:45.836 INFO:tasks.rados.rados.0.smithi029.stdout:3174: read oid 33 snap -1 2024-04-12T11:27:45.836 INFO:tasks.rados.rados.0.smithi029.stdout:3174: expect deleted 2024-04-12T11:27:45.836 INFO:tasks.rados.rados.0.smithi029.stdout:3175: snap_create 2024-04-12T11:27:45.836 INFO:tasks.rados.rados.0.smithi029.stdout:3170: finishing write tid 2 to smithi02979277-1 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3170: finishing write tid 3 to smithi02979277-1 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3170: finishing write tid 4 to smithi02979277-1 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3170: finishing write tid 5 to smithi02979277-1 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3170: oid 1 updating version 0 to 1472 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3170: oid 1 updating version 1472 to 1474 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3170: oid 1 version 1474 is already newer than 1473 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3170: oid 1 version 1474 is already newer than 1471 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1474 (ObjNum 1184 snap 401 seq_num 1184) dirty exists 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3170: left oid 1 (ObjNum 1184 snap 401 seq_num 1184) 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3171: finishing write tid 1 to smithi02979277-38 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3171: finishing write tid 2 to smithi02979277-38 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3171: finishing write tid 3 to smithi02979277-38 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3171: finishing write tid 4 to smithi02979277-38 2024-04-12T11:27:45.837 INFO:tasks.rados.rados.0.smithi029.stdout:3171: finishing write tid 5 to smithi02979277-38 2024-04-12T11:27:45.838 INFO:tasks.rados.rados.0.smithi029.stdout:3171: finishing write tid 6 to smithi02979277-38 2024-04-12T11:27:45.838 INFO:tasks.rados.rados.0.smithi029.stdout:3171: oid 38 updating version 0 to 965 2024-04-12T11:27:45.838 INFO:tasks.rados.rados.0.smithi029.stdout:3171: oid 38 updating version 965 to 967 2024-04-12T11:27:45.838 INFO:tasks.rados.rados.0.smithi029.stdout:3171: oid 38 version 967 is already newer than 964 2024-04-12T11:27:45.838 INFO:tasks.rados.rados.0.smithi029.stdout:3171: oid 38 version 967 is already newer than 966 2024-04-12T11:27:45.838 INFO:tasks.rados.rados.0.smithi029.stdout:3171: oid 38 version 967 is already newer than 963 2024-04-12T11:27:45.838 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 967 (ObjNum 1185 snap 401 seq_num 1185) dirty exists 2024-04-12T11:27:45.838 INFO:tasks.rados.rados.0.smithi029.stdout:3171: left oid 38 (ObjNum 1185 snap 401 seq_num 1185) 2024-04-12T11:27:45.838 INFO:tasks.rados.rados.0.smithi029.stdout:3172: expect (ObjNum 1107 snap 370 seq_num 1107) 2024-04-12T11:27:46.031 INFO:tasks.rados.rados.0.smithi029.stdout:3173: finishing write tid 1 to smithi02979277-13 2024-04-12T11:27:46.031 INFO:tasks.rados.rados.0.smithi029.stdout:3173: finishing write tid 2 to smithi02979277-13 2024-04-12T11:27:46.031 INFO:tasks.rados.rados.0.smithi029.stdout:3173: finishing write tid 3 to smithi02979277-13 2024-04-12T11:27:46.031 INFO:tasks.rados.rados.0.smithi029.stdout:3173: finishing write tid 4 to smithi02979277-13 2024-04-12T11:27:46.031 INFO:tasks.rados.rados.0.smithi029.stdout:3173: finishing write tid 5 to smithi02979277-13 2024-04-12T11:27:46.031 INFO:tasks.rados.rados.0.smithi029.stdout:3173: oid 13 updating version 0 to 1581 2024-04-12T11:27:46.032 INFO:tasks.rados.rados.0.smithi029.stdout:3173: oid 13 updating version 1581 to 1582 2024-04-12T11:27:46.032 INFO:tasks.rados.rados.0.smithi029.stdout:3173: oid 13 updating version 1582 to 1584 2024-04-12T11:27:46.032 INFO:tasks.rados.rados.0.smithi029.stdout:3173: oid 13 version 1584 is already newer than 1583 2024-04-12T11:27:46.032 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1584 (ObjNum 1186 snap 401 seq_num 1186) dirty exists 2024-04-12T11:27:46.032 INFO:tasks.rados.rados.0.smithi029.stdout:3173: left oid 13 (ObjNum 1186 snap 401 seq_num 1186) 2024-04-12T11:27:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:45 smithi087 ceph-mon[89014]: osdmap e860: 8 total, 8 up, 8 in 2024-04-12T11:27:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:45 smithi029 ceph-mon[97787]: osdmap e860: 8 total, 8 up, 8 in 2024-04-12T11:27:46.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:45 smithi029 ceph-mon[93650]: osdmap e860: 8 total, 8 up, 8 in 2024-04-12T11:27:46.777 INFO:tasks.rados.rados.0.smithi029.stdout:3170: done (5 left) 2024-04-12T11:27:46.777 INFO:tasks.rados.rados.0.smithi029.stdout:3171: done (4 left) 2024-04-12T11:27:46.777 INFO:tasks.rados.rados.0.smithi029.stdout:3172: done (3 left) 2024-04-12T11:27:46.777 INFO:tasks.rados.rados.0.smithi029.stdout:3173: done (2 left) 2024-04-12T11:27:46.778 INFO:tasks.rados.rados.0.smithi029.stdout:3174: done (1 left) 2024-04-12T11:27:46.778 INFO:tasks.rados.rados.0.smithi029.stdout:3175: done (0 left) 2024-04-12T11:27:46.778 INFO:tasks.rados.rados.0.smithi029.stdout:3176: write oid 19 current snap is 402 2024-04-12T11:27:46.778 INFO:tasks.rados.rados.0.smithi029.stdout:3176: seq_num 1187 ranges {749909=671631,2110479=618078,3163563=515291} 2024-04-12T11:27:46.785 INFO:tasks.rados.rados.0.smithi029.stdout:3176: writing smithi02979277-19 from 749909 to 1421540 tid 1 2024-04-12T11:27:46.793 INFO:tasks.rados.rados.0.smithi029.stdout:3176: writing smithi02979277-19 from 2110479 to 2728557 tid 2 2024-04-12T11:27:46.798 INFO:tasks.rados.rados.0.smithi029.stdout:3176: writing smithi02979277-19 from 3163563 to 3678854 tid 3 2024-04-12T11:27:46.798 INFO:tasks.rados.rados.0.smithi029.stdout:3177: rollback oid 2 current snap is 402 2024-04-12T11:27:46.799 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 2 to 401 2024-04-12T11:27:46.799 INFO:tasks.rados.rados.0.smithi029.stdout:3176: finishing write tid 1 to smithi02979277-19 2024-04-12T11:27:46.799 INFO:tasks.rados.rados.0.smithi029.stdout:3176: finishing write tid 2 to smithi02979277-19 2024-04-12T11:27:46.799 INFO:tasks.rados.rados.0.smithi029.stdout:3178: delete oid 15 current snap is 402 2024-04-12T11:27:46.801 INFO:tasks.rados.rados.0.smithi029.stdout:3178: done (2 left) 2024-04-12T11:27:46.801 INFO:tasks.rados.rados.0.smithi029.stdout:3179: delete oid 8 current snap is 402 2024-04-12T11:27:46.801 INFO:tasks.rados.rados.0.smithi029.stdout:3177: finishing rollback tid 0 to smithi02979277-2 2024-04-12T11:27:46.802 INFO:tasks.rados.rados.0.smithi029.stdout:3177: finishing rollback tid 1 to smithi02979277-2 2024-04-12T11:27:46.803 INFO:tasks.rados.rados.0.smithi029.stdout:3179: done (2 left) 2024-04-12T11:27:46.803 INFO:tasks.rados.rados.0.smithi029.stdout:3180: write oid 25 current snap is 402 2024-04-12T11:27:46.803 INFO:tasks.rados.rados.0.smithi029.stdout:3180: seq_num 1190 ranges {400330=497378,1485925=514405} 2024-04-12T11:27:46.807 INFO:tasks.rados.rados.0.smithi029.stdout:3180: writing smithi02979277-25 from 400330 to 897708 tid 1 2024-04-12T11:27:46.814 INFO:tasks.rados.rados.0.smithi029.stdout:3180: writing smithi02979277-25 from 1485925 to 2000330 tid 2 2024-04-12T11:27:46.814 INFO:tasks.rados.rados.0.smithi029.stdout:3181: rollback oid 32 current snap is 402 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 32 to 385 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3177: finishing rollback tid 2 to smithi02979277-2 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 347 (ObjNum 1182 snap 400 seq_num 1182) dirty exists 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3176: finishing write tid 3 to smithi02979277-19 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3176: finishing write tid 4 to smithi02979277-19 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3176: finishing write tid 5 to smithi02979277-19 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3176: oid 19 updating version 0 to 1420 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3176: oid 19 updating version 1420 to 1421 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3176: oid 19 version 1421 is already newer than 1419 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3176: oid 19 version 1421 is already newer than 1418 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1421 (ObjNum 1187 snap 402 seq_num 1187) dirty exists 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3176: left oid 19 (ObjNum 1187 snap 402 seq_num 1187) 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3180: finishing write tid 1 to smithi02979277-25 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3176: done (3 left) 2024-04-12T11:27:46.815 INFO:tasks.rados.rados.0.smithi029.stdout:3177: done (2 left) 2024-04-12T11:27:46.816 INFO:tasks.rados.rados.0.smithi029.stdout:3182: rollback oid 38 current snap is 402 2024-04-12T11:27:46.816 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 38 to 401 2024-04-12T11:27:46.816 INFO:tasks.rados.rados.0.smithi029.stdout:3183: snap_remove snap 368 2024-04-12T11:27:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:46 smithi087 ceph-mon[89014]: pgmap v927: 89 pgs: 89 active+clean; 321 MiB data, 926 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 406 KiB/s wr, 6 op/s 2024-04-12T11:27:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:46 smithi087 ceph-mon[89014]: osdmap e861: 8 total, 8 up, 8 in 2024-04-12T11:27:47.217 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:27:46 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=sqlstore.transactions t=2024-04-12T11:27:46.770452968Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-12T11:27:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:46 smithi029 ceph-mon[93650]: pgmap v927: 89 pgs: 89 active+clean; 321 MiB data, 926 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 406 KiB/s wr, 6 op/s 2024-04-12T11:27:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:46 smithi029 ceph-mon[93650]: osdmap e861: 8 total, 8 up, 8 in 2024-04-12T11:27:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:46 smithi029 ceph-mon[97787]: pgmap v927: 89 pgs: 89 active+clean; 321 MiB data, 926 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 406 KiB/s wr, 6 op/s 2024-04-12T11:27:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:46 smithi029 ceph-mon[97787]: osdmap e861: 8 total, 8 up, 8 in 2024-04-12T11:27:47.663 INFO:tasks.rados.rados.0.smithi029.stdout:3183: done (3 left) 2024-04-12T11:27:47.663 INFO:tasks.rados.rados.0.smithi029.stdout:3184: write oid 44 current snap is 402 2024-04-12T11:27:47.664 INFO:tasks.rados.rados.0.smithi029.stdout:3184: seq_num 1191 ranges {417137=572627,1428054=589083} 2024-04-12T11:27:47.669 INFO:tasks.rados.rados.0.smithi029.stdout:3184: writing smithi02979277-44 from 417137 to 989764 tid 1 2024-04-12T11:27:47.676 INFO:tasks.rados.rados.0.smithi029.stdout:3184: writing smithi02979277-44 from 1428054 to 2017137 tid 2 2024-04-12T11:27:47.676 INFO:tasks.rados.rados.0.smithi029.stdout:3185: snap_remove snap 399 2024-04-12T11:27:48.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:47 smithi087 ceph-mon[89014]: osdmap e862: 8 total, 8 up, 8 in 2024-04-12T11:27:48.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:47 smithi087 ceph-mon[89014]: osdmap e863: 8 total, 8 up, 8 in 2024-04-12T11:27:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:47 smithi029 ceph-mon[93650]: osdmap e862: 8 total, 8 up, 8 in 2024-04-12T11:27:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:47 smithi029 ceph-mon[93650]: osdmap e863: 8 total, 8 up, 8 in 2024-04-12T11:27:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:47 smithi029 ceph-mon[97787]: osdmap e862: 8 total, 8 up, 8 in 2024-04-12T11:27:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:47 smithi029 ceph-mon[97787]: osdmap e863: 8 total, 8 up, 8 in 2024-04-12T11:27:48.664 INFO:tasks.rados.rados.0.smithi029.stdout:3185: done (4 left) 2024-04-12T11:27:48.664 INFO:tasks.rados.rados.0.smithi029.stdout:3186: snap_create 2024-04-12T11:27:48.664 INFO:tasks.rados.rados.0.smithi029.stdout:3181: finishing rollback tid 1 to smithi02979277-32 2024-04-12T11:27:48.664 INFO:tasks.rados.rados.0.smithi029.stdout:3182: finishing rollback tid 0 to smithi02979277-38 2024-04-12T11:27:48.664 INFO:tasks.rados.rados.0.smithi029.stdout:3181: finishing rollback tid 2 to smithi02979277-32 2024-04-12T11:27:48.664 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 1414 (ObjNum 1043 snap 356 seq_num 1043) dirty exists 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3182: finishing rollback tid 1 to smithi02979277-38 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3182: finishing rollback tid 2 to smithi02979277-38 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 971 (ObjNum 1185 snap 401 seq_num 1185) dirty exists 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3180: finishing write tid 2 to smithi02979277-25 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3180: finishing write tid 3 to smithi02979277-25 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3180: finishing write tid 4 to smithi02979277-25 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3180: oid 25 updating version 0 to 1587 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3180: oid 25 version 1587 is already newer than 1586 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3180: oid 25 updating version 1587 to 1588 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 1588 (ObjNum 1190 snap 402 seq_num 1190) dirty exists 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3180: left oid 25 (ObjNum 1190 snap 402 seq_num 1190) 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3184: finishing write tid 1 to smithi02979277-44 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3184: finishing write tid 2 to smithi02979277-44 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3184: finishing write tid 3 to smithi02979277-44 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3184: finishing write tid 4 to smithi02979277-44 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3184: oid 44 updating version 0 to 1486 2024-04-12T11:27:48.665 INFO:tasks.rados.rados.0.smithi029.stdout:3184: oid 44 version 1486 is already newer than 1480 2024-04-12T11:27:48.666 INFO:tasks.rados.rados.0.smithi029.stdout:3184: oid 44 version 1486 is already newer than 1485 2024-04-12T11:27:48.666 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1486 (ObjNum 1191 snap 402 seq_num 1191) dirty exists 2024-04-12T11:27:48.666 INFO:tasks.rados.rados.0.smithi029.stdout:3184: left oid 44 (ObjNum 1191 snap 402 seq_num 1191) 2024-04-12T11:27:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:48 smithi087 ceph-mon[89014]: pgmap v930: 89 pgs: 89 active+clean; 336 MiB data, 950 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.2 MiB/s wr, 21 op/s 2024-04-12T11:27:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:48 smithi087 ceph-mon[89014]: osdmap e864: 8 total, 8 up, 8 in 2024-04-12T11:27:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:48 smithi029 ceph-mon[97787]: pgmap v930: 89 pgs: 89 active+clean; 336 MiB data, 950 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.2 MiB/s wr, 21 op/s 2024-04-12T11:27:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:48 smithi029 ceph-mon[97787]: osdmap e864: 8 total, 8 up, 8 in 2024-04-12T11:27:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:48 smithi029 ceph-mon[93650]: pgmap v930: 89 pgs: 89 active+clean; 336 MiB data, 950 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.2 MiB/s wr, 21 op/s 2024-04-12T11:27:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:48 smithi029 ceph-mon[93650]: osdmap e864: 8 total, 8 up, 8 in 2024-04-12T11:27:49.665 INFO:tasks.rados.rados.0.smithi029.stdout:3180: done (4 left) 2024-04-12T11:27:49.665 INFO:tasks.rados.rados.0.smithi029.stdout:3181: done (3 left) 2024-04-12T11:27:49.665 INFO:tasks.rados.rados.0.smithi029.stdout:3182: done (2 left) 2024-04-12T11:27:49.665 INFO:tasks.rados.rados.0.smithi029.stdout:3184: done (1 left) 2024-04-12T11:27:49.665 INFO:tasks.rados.rados.0.smithi029.stdout:3186: done (0 left) 2024-04-12T11:27:49.665 INFO:tasks.rados.rados.0.smithi029.stdout:3187: write oid 45 current snap is 403 2024-04-12T11:27:49.666 INFO:tasks.rados.rados.0.smithi029.stdout:3187: seq_num 1192 ranges {433944=647876,1486536=547408} 2024-04-12T11:27:49.670 INFO:tasks.rados.rados.0.smithi029.stdout:3187: writing smithi02979277-45 from 433944 to 1081820 tid 1 2024-04-12T11:27:49.677 INFO:tasks.rados.rados.0.smithi029.stdout:3187: writing smithi02979277-45 from 1486536 to 2033944 tid 2 2024-04-12T11:27:49.677 INFO:tasks.rados.rados.0.smithi029.stdout:3188: write oid 14 current snap is 403 2024-04-12T11:27:49.677 INFO:tasks.rados.rados.0.smithi029.stdout:3188: seq_num 1193 ranges {450751=723125,1945018=105733} 2024-04-12T11:27:49.683 INFO:tasks.rados.rados.0.smithi029.stdout:3188: writing smithi02979277-14 from 450751 to 1173876 tid 1 2024-04-12T11:27:49.689 INFO:tasks.rados.rados.0.smithi029.stdout:3188: writing smithi02979277-14 from 1945018 to 2050751 tid 2 2024-04-12T11:27:49.689 INFO:tasks.rados.rados.0.smithi029.stdout:3189: snap_create 2024-04-12T11:27:49.689 INFO:tasks.rados.rados.0.smithi029.stdout:3187: finishing write tid 1 to smithi02979277-45 2024-04-12T11:27:49.689 INFO:tasks.rados.rados.0.smithi029.stdout:3187: finishing write tid 2 to smithi02979277-45 2024-04-12T11:27:49.689 INFO:tasks.rados.rados.0.smithi029.stdout:3187: finishing write tid 3 to smithi02979277-45 2024-04-12T11:27:49.689 INFO:tasks.rados.rados.0.smithi029.stdout:3187: finishing write tid 4 to smithi02979277-45 2024-04-12T11:27:49.689 INFO:tasks.rados.rados.0.smithi029.stdout:3187: oid 45 updating version 0 to 1488 2024-04-12T11:27:49.690 INFO:tasks.rados.rados.0.smithi029.stdout:3187: oid 45 updating version 1488 to 1489 2024-04-12T11:27:49.690 INFO:tasks.rados.rados.0.smithi029.stdout:3187: oid 45 updating version 1489 to 1490 2024-04-12T11:27:49.690 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1490 (ObjNum 1192 snap 403 seq_num 1192) dirty exists 2024-04-12T11:27:49.690 INFO:tasks.rados.rados.0.smithi029.stdout:3187: left oid 45 (ObjNum 1192 snap 403 seq_num 1192) 2024-04-12T11:27:49.691 INFO:tasks.rados.rados.0.smithi029.stdout:3188: finishing write tid 1 to smithi02979277-14 2024-04-12T11:27:49.692 INFO:tasks.rados.rados.0.smithi029.stdout:3188: finishing write tid 2 to smithi02979277-14 2024-04-12T11:27:49.692 INFO:tasks.rados.rados.0.smithi029.stdout:3188: finishing write tid 3 to smithi02979277-14 2024-04-12T11:27:49.692 INFO:tasks.rados.rados.0.smithi029.stdout:3188: finishing write tid 4 to smithi02979277-14 2024-04-12T11:27:49.692 INFO:tasks.rados.rados.0.smithi029.stdout:3188: oid 14 updating version 0 to 1427 2024-04-12T11:27:49.692 INFO:tasks.rados.rados.0.smithi029.stdout:3188: oid 14 updating version 1427 to 1429 2024-04-12T11:27:49.692 INFO:tasks.rados.rados.0.smithi029.stdout:3188: oid 14 version 1429 is already newer than 1428 2024-04-12T11:27:49.692 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 1429 (ObjNum 1193 snap 403 seq_num 1193) dirty exists 2024-04-12T11:27:49.692 INFO:tasks.rados.rados.0.smithi029.stdout:3188: left oid 14 (ObjNum 1193 snap 403 seq_num 1193) 2024-04-12T11:27:50.670 INFO:tasks.rados.rados.0.smithi029.stdout:3187: done (2 left) 2024-04-12T11:27:50.671 INFO:tasks.rados.rados.0.smithi029.stdout:3188: done (1 left) 2024-04-12T11:27:50.671 INFO:tasks.rados.rados.0.smithi029.stdout:3189: done (0 left) 2024-04-12T11:27:50.671 INFO:tasks.rados.rados.0.smithi029.stdout:3190: write oid 31 current snap is 404 2024-04-12T11:27:50.671 INFO:tasks.rados.rados.0.smithi029.stdout:3190: seq_num 1194 ranges {467558=514727,1719853=347705} 2024-04-12T11:27:50.676 INFO:tasks.rados.rados.0.smithi029.stdout:3190: writing smithi02979277-31 from 467558 to 982285 tid 1 2024-04-12T11:27:50.682 INFO:tasks.rados.rados.0.smithi029.stdout:3190: writing smithi02979277-31 from 1719853 to 2067558 tid 2 2024-04-12T11:27:50.682 INFO:tasks.rados.rados.0.smithi029.stdout:3191: delete oid 2 current snap is 404 2024-04-12T11:27:50.682 INFO:tasks.rados.rados.0.smithi029.stdout:3190: finishing write tid 1 to smithi02979277-31 2024-04-12T11:27:50.686 INFO:tasks.rados.rados.0.smithi029.stdout:3190: finishing write tid 2 to smithi02979277-31 2024-04-12T11:27:50.686 INFO:tasks.rados.rados.0.smithi029.stdout:3190: finishing write tid 3 to smithi02979277-31 2024-04-12T11:27:50.686 INFO:tasks.rados.rados.0.smithi029.stdout:3190: finishing write tid 4 to smithi02979277-31 2024-04-12T11:27:50.686 INFO:tasks.rados.rados.0.smithi029.stdout:3190: oid 31 updating version 0 to 1225 2024-04-12T11:27:50.686 INFO:tasks.rados.rados.0.smithi029.stdout:3190: oid 31 version 1225 is already newer than 1223 2024-04-12T11:27:50.686 INFO:tasks.rados.rados.0.smithi029.stdout:3190: oid 31 version 1225 is already newer than 1224 2024-04-12T11:27:50.686 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1225 (ObjNum 1194 snap 404 seq_num 1194) dirty exists 2024-04-12T11:27:50.686 INFO:tasks.rados.rados.0.smithi029.stdout:3190: left oid 31 (ObjNum 1194 snap 404 seq_num 1194) 2024-04-12T11:27:50.686 INFO:tasks.rados.rados.0.smithi029.stdout:3190: done (1 left) 2024-04-12T11:27:50.687 INFO:tasks.rados.rados.0.smithi029.stdout:3191: done (0 left) 2024-04-12T11:27:50.687 INFO:tasks.rados.rados.0.smithi029.stdout:3192: snap_create 2024-04-12T11:27:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:50 smithi087 ceph-mon[89014]: pgmap v933: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 345 MiB data, 967 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 4.9 MiB/s wr, 32 op/s 2024-04-12T11:27:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:50 smithi087 ceph-mon[89014]: osdmap e865: 8 total, 8 up, 8 in 2024-04-12T11:27:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:50 smithi029 ceph-mon[93650]: pgmap v933: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 345 MiB data, 967 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 4.9 MiB/s wr, 32 op/s 2024-04-12T11:27:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:50 smithi029 ceph-mon[93650]: osdmap e865: 8 total, 8 up, 8 in 2024-04-12T11:27:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:50 smithi029 ceph-mon[97787]: pgmap v933: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 345 MiB data, 967 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 4.9 MiB/s wr, 32 op/s 2024-04-12T11:27:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:50 smithi029 ceph-mon[97787]: osdmap e865: 8 total, 8 up, 8 in 2024-04-12T11:27:51.676 INFO:tasks.rados.rados.0.smithi029.stdout:3192: done (0 left) 2024-04-12T11:27:51.676 INFO:tasks.rados.rados.0.smithi029.stdout:3193: write oid 13 current snap is 405 2024-04-12T11:27:51.676 INFO:tasks.rados.rados.0.smithi029.stdout:3193: seq_num 1196 ranges {501172=665225,1720464=380708} 2024-04-12T11:27:51.682 INFO:tasks.rados.rados.0.smithi029.stdout:3193: writing smithi02979277-13 from 501172 to 1166397 tid 1 2024-04-12T11:27:51.688 INFO:tasks.rados.rados.0.smithi029.stdout:3193: writing smithi02979277-13 from 1720464 to 2101172 tid 2 2024-04-12T11:27:51.688 INFO:tasks.rados.rados.0.smithi029.stdout:3194: snap_create 2024-04-12T11:27:51.693 INFO:tasks.rados.rados.0.smithi029.stdout:3193: finishing write tid 1 to smithi02979277-13 2024-04-12T11:27:51.694 INFO:tasks.rados.rados.0.smithi029.stdout:3193: finishing write tid 2 to smithi02979277-13 2024-04-12T11:27:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:3193: finishing write tid 3 to smithi02979277-13 2024-04-12T11:27:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:3193: finishing write tid 4 to smithi02979277-13 2024-04-12T11:27:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:3193: oid 13 updating version 0 to 1604 2024-04-12T11:27:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:3193: oid 13 version 1604 is already newer than 1602 2024-04-12T11:27:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:3193: oid 13 version 1604 is already newer than 1603 2024-04-12T11:27:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1604 (ObjNum 1196 snap 405 seq_num 1196) dirty exists 2024-04-12T11:27:51.695 INFO:tasks.rados.rados.0.smithi029.stdout:3193: left oid 13 (ObjNum 1196 snap 405 seq_num 1196) 2024-04-12T11:27:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:51 smithi087 ceph-mon[89014]: osdmap e866: 8 total, 8 up, 8 in 2024-04-12T11:27:51.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:51 smithi029 ceph-mon[97787]: osdmap e866: 8 total, 8 up, 8 in 2024-04-12T11:27:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:51 smithi029 ceph-mon[93650]: osdmap e866: 8 total, 8 up, 8 in 2024-04-12T11:27:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:27:52.663 INFO:tasks.rados.rados.0.smithi029.stdout:3193: done (1 left) 2024-04-12T11:27:52.663 INFO:tasks.rados.rados.0.smithi029.stdout:3194: done (0 left) 2024-04-12T11:27:52.663 INFO:tasks.rados.rados.0.smithi029.stdout:3195: write oid 38 current snap is 406 2024-04-12T11:27:52.663 INFO:tasks.rados.rados.0.smithi029.stdout:3195: seq_num 1197 ranges {517979=740474,1778946=339033} 2024-04-12T11:27:52.670 INFO:tasks.rados.rados.0.smithi029.stdout:3195: writing smithi02979277-38 from 517979 to 1258453 tid 1 2024-04-12T11:27:52.673 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:27:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:27:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:27:52.675 INFO:tasks.rados.rados.0.smithi029.stdout:3195: writing smithi02979277-38 from 1778946 to 2117979 tid 2 2024-04-12T11:27:52.675 INFO:tasks.rados.rados.0.smithi029.stdout:3196: write oid 18 current snap is 406 2024-04-12T11:27:52.675 INFO:tasks.rados.rados.0.smithi029.stdout:3196: seq_num 1198 ranges {534786=415723,1437428=697358} 2024-04-12T11:27:52.681 INFO:tasks.rados.rados.0.smithi029.stdout:3196: writing smithi02979277-18 from 534786 to 950509 tid 1 2024-04-12T11:27:52.688 INFO:tasks.rados.rados.0.smithi029.stdout:3196: writing smithi02979277-18 from 1437428 to 2134786 tid 2 2024-04-12T11:27:52.688 INFO:tasks.rados.rados.0.smithi029.stdout:3197: rollback oid 40 current snap is 406 2024-04-12T11:27:52.688 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 40 to 380 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3195: finishing write tid 1 to smithi02979277-38 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3195: finishing write tid 2 to smithi02979277-38 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3195: finishing write tid 3 to smithi02979277-38 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3195: finishing write tid 4 to smithi02979277-38 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3195: oid 38 updating version 0 to 981 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3195: oid 38 updating version 981 to 983 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3195: oid 38 version 983 is already newer than 982 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 983 (ObjNum 1197 snap 406 seq_num 1197) dirty exists 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3195: left oid 38 (ObjNum 1197 snap 406 seq_num 1197) 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3196: finishing write tid 1 to smithi02979277-18 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3195: done (2 left) 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3198: read oid 7 snap 400 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3198: expect (ObjNum 1161 snap 394 seq_num 1161) 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3199: read oid 10 snap -1 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3199: expect (ObjNum 1170 snap 397 seq_num 1170) 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:3200: rollback oid 15 current snap is 406 2024-04-12T11:27:52.689 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 15 to 374 2024-04-12T11:27:52.690 INFO:tasks.rados.rados.0.smithi029.stdout:3201: write oid 1 current snap is 406 2024-04-12T11:27:52.690 INFO:tasks.rados.rados.0.smithi029.stdout:3201: seq_num 1199 ranges {551593=490972,1779557=372036} 2024-04-12T11:27:52.694 INFO:tasks.rados.rados.0.smithi029.stdout:3201: writing smithi02979277-1 from 551593 to 1042565 tid 1 2024-04-12T11:27:52.701 INFO:tasks.rados.rados.0.smithi029.stdout:3201: writing smithi02979277-1 from 1779557 to 2151593 tid 2 2024-04-12T11:27:52.701 INFO:tasks.rados.rados.0.smithi029.stdout:3202: delete oid 19 current snap is 406 2024-04-12T11:27:52.701 INFO:tasks.rados.rados.0.smithi029.stdout:3196: finishing write tid 2 to smithi02979277-18 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:3197: finishing rollback tid 0 to smithi02979277-40 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:3197: finishing rollback tid 1 to smithi02979277-40 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:3197: finishing rollback tid 2 to smithi02979277-40 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 1977 (ObjNum 1012 snap 348 seq_num 1012) dirty exists 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:3196: finishing write tid 3 to smithi02979277-18 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:3196: finishing write tid 4 to smithi02979277-18 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:3196: oid 18 updating version 0 to 1606 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:3196: oid 18 updating version 1606 to 1607 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:3196: oid 18 updating version 1607 to 1608 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1608 (ObjNum 1198 snap 406 seq_num 1198) dirty exists 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:3196: left oid 18 (ObjNum 1198 snap 406 seq_num 1198) 2024-04-12T11:27:52.702 INFO:tasks.rados.rados.0.smithi029.stdout:3199: expect (ObjNum 1170 snap 397 seq_num 1170) 2024-04-12T11:27:52.949 INFO:tasks.rados.rados.0.smithi029.stdout:3200: finishing rollback tid 1 to smithi02979277-15 2024-04-12T11:27:52.949 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 0 (ObjNum 33 snap 0 seq_num 1801242144) dirty dne 2024-04-12T11:27:52.949 INFO:tasks.rados.rados.0.smithi029.stdout:3198: expect (ObjNum 1161 snap 394 seq_num 1161) 2024-04-12T11:27:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:52 smithi087 ceph-mon[89014]: pgmap v936: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 347 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:27:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:52 smithi087 ceph-mon[89014]: osdmap e867: 8 total, 8 up, 8 in 2024-04-12T11:27:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:52 smithi087 ceph-mon[89014]: osdmap e868: 8 total, 8 up, 8 in 2024-04-12T11:27:53.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:52 smithi029 ceph-mon[93650]: pgmap v936: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 347 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:27:53.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:52 smithi029 ceph-mon[93650]: osdmap e867: 8 total, 8 up, 8 in 2024-04-12T11:27:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:52 smithi029 ceph-mon[93650]: osdmap e868: 8 total, 8 up, 8 in 2024-04-12T11:27:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:52 smithi029 ceph-mon[97787]: pgmap v936: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 347 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:27:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:52 smithi029 ceph-mon[97787]: osdmap e867: 8 total, 8 up, 8 in 2024-04-12T11:27:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:52 smithi029 ceph-mon[97787]: osdmap e868: 8 total, 8 up, 8 in 2024-04-12T11:27:53.110 INFO:tasks.rados.rados.0.smithi029.stdout:3201: finishing write tid 1 to smithi02979277-1 2024-04-12T11:27:53.110 INFO:tasks.rados.rados.0.smithi029.stdout:3201: finishing write tid 2 to smithi02979277-1 2024-04-12T11:27:53.110 INFO:tasks.rados.rados.0.smithi029.stdout:3201: finishing write tid 3 to smithi02979277-1 2024-04-12T11:27:53.110 INFO:tasks.rados.rados.0.smithi029.stdout:3201: finishing write tid 4 to smithi02979277-1 2024-04-12T11:27:53.110 INFO:tasks.rados.rados.0.smithi029.stdout:3201: oid 1 updating version 0 to 1493 2024-04-12T11:27:53.110 INFO:tasks.rados.rados.0.smithi029.stdout:3201: oid 1 updating version 1493 to 1494 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:3201: oid 1 version 1494 is already newer than 1492 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1494 (ObjNum 1199 snap 406 seq_num 1199) dirty exists 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:3201: left oid 1 (ObjNum 1199 snap 406 seq_num 1199) 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:3196: done (6 left) 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:3197: done (5 left) 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:3198: done (4 left) 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:3199: done (3 left) 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:3200: done (2 left) 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:3201: done (1 left) 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:3202: done (0 left) 2024-04-12T11:27:53.111 INFO:tasks.rados.rados.0.smithi029.stdout:3203: snap_remove snap 376 2024-04-12T11:27:53.682 INFO:tasks.rados.rados.0.smithi029.stdout:3203: done (0 left) 2024-04-12T11:27:53.683 INFO:tasks.rados.rados.0.smithi029.stdout:3204: rollback oid 8 current snap is 406 2024-04-12T11:27:53.683 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 357 2024-04-12T11:27:53.683 INFO:tasks.rados.rados.0.smithi029.stdout:3205: read oid 20 snap -1 2024-04-12T11:27:53.683 INFO:tasks.rados.rados.0.smithi029.stdout:3205: expect (ObjNum 1138 snap 383 seq_num 1138) 2024-04-12T11:27:53.683 INFO:tasks.rados.rados.0.smithi029.stdout:3206: delete oid 9 current snap is 406 2024-04-12T11:27:53.688 INFO:tasks.rados.rados.0.smithi029.stdout:3204: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:27:53.688 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 326 (ObjNum 49 snap 0 seq_num 1777349616) dirty dne 2024-04-12T11:27:53.689 INFO:tasks.rados.rados.0.smithi029.stdout:3205: expect (ObjNum 1138 snap 383 seq_num 1138) 2024-04-12T11:27:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:3204: done (2 left) 2024-04-12T11:27:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:3205: done (1 left) 2024-04-12T11:27:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:3206: done (0 left) 2024-04-12T11:27:53.905 INFO:tasks.rados.rados.0.smithi029.stdout:3207: snap_remove snap 405 2024-04-12T11:27:54.683 INFO:tasks.rados.rados.0.smithi029.stdout:3207: done (0 left) 2024-04-12T11:27:54.683 INFO:tasks.rados.rados.0.smithi029.stdout:3208: read oid 21 snap -1 2024-04-12T11:27:54.683 INFO:tasks.rados.rados.0.smithi029.stdout:3208: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:27:54.683 INFO:tasks.rados.rados.0.smithi029.stdout:3209: snap_remove snap 374 2024-04-12T11:27:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:54 smithi087 ceph-mon[89014]: pgmap v939: 89 pgs: 89 active+clean; 345 MiB data, 981 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-04-12T11:27:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:54 smithi087 ceph-mon[89014]: osdmap e869: 8 total, 8 up, 8 in 2024-04-12T11:27:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:27:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:54 smithi029 ceph-mon[93650]: pgmap v939: 89 pgs: 89 active+clean; 345 MiB data, 981 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-04-12T11:27:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:54 smithi029 ceph-mon[93650]: osdmap e869: 8 total, 8 up, 8 in 2024-04-12T11:27:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:27:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:54 smithi029 ceph-mon[97787]: pgmap v939: 89 pgs: 89 active+clean; 345 MiB data, 981 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-04-12T11:27:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:54 smithi029 ceph-mon[97787]: osdmap e869: 8 total, 8 up, 8 in 2024-04-12T11:27:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:27:55.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:27:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:27:54] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.43.0" 2024-04-12T11:27:55.691 INFO:tasks.rados.rados.0.smithi029.stdout:3209: done (1 left) 2024-04-12T11:27:55.692 INFO:tasks.rados.rados.0.smithi029.stdout:3210: write oid 13 current snap is 406 2024-04-12T11:27:55.692 INFO:tasks.rados.rados.0.smithi029.stdout:3210: seq_num 1202 ranges {602014=433072,1671356=530658} 2024-04-12T11:27:55.697 INFO:tasks.rados.rados.0.smithi029.stdout:3210: writing smithi02979277-13 from 602014 to 1035086 tid 1 2024-04-12T11:27:55.706 INFO:tasks.rados.rados.0.smithi029.stdout:3210: writing smithi02979277-13 from 1671356 to 2202014 tid 2 2024-04-12T11:27:55.706 INFO:tasks.rados.rados.0.smithi029.stdout:3211: delete oid 31 current snap is 406 2024-04-12T11:27:55.706 INFO:tasks.rados.rados.0.smithi029.stdout:3208: expect (ObjNum 941 snap 325 seq_num 941) 2024-04-12T11:27:55.805 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:55 smithi029 ceph-mon[93650]: osdmap e870: 8 total, 8 up, 8 in 2024-04-12T11:27:55.805 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.805 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.806 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.806 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:55 smithi029 ceph-mon[97787]: osdmap e870: 8 total, 8 up, 8 in 2024-04-12T11:27:55.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.870 INFO:tasks.rados.rados.0.smithi029.stdout:3210: finishing write tid 1 to smithi02979277-13 2024-04-12T11:27:55.870 INFO:tasks.rados.rados.0.smithi029.stdout:3210: finishing write tid 2 to smithi02979277-13 2024-04-12T11:27:55.870 INFO:tasks.rados.rados.0.smithi029.stdout:3210: finishing write tid 3 to smithi02979277-13 2024-04-12T11:27:55.870 INFO:tasks.rados.rados.0.smithi029.stdout:3210: finishing write tid 4 to smithi02979277-13 2024-04-12T11:27:55.870 INFO:tasks.rados.rados.0.smithi029.stdout:3210: oid 13 updating version 0 to 1621 2024-04-12T11:27:55.870 INFO:tasks.rados.rados.0.smithi029.stdout:3210: oid 13 version 1621 is already newer than 1620 2024-04-12T11:27:55.870 INFO:tasks.rados.rados.0.smithi029.stdout:3210: oid 13 version 1621 is already newer than 1615 2024-04-12T11:27:55.870 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1621 (ObjNum 1202 snap 406 seq_num 1202) dirty exists 2024-04-12T11:27:55.870 INFO:tasks.rados.rados.0.smithi029.stdout:3210: left oid 13 (ObjNum 1202 snap 406 seq_num 1202) 2024-04-12T11:27:55.871 INFO:tasks.rados.rados.0.smithi029.stdout:3208: done (2 left) 2024-04-12T11:27:55.871 INFO:tasks.rados.rados.0.smithi029.stdout:3210: done (1 left) 2024-04-12T11:27:55.871 INFO:tasks.rados.rados.0.smithi029.stdout:3211: done (0 left) 2024-04-12T11:27:55.871 INFO:tasks.rados.rados.0.smithi029.stdout:3212: write oid 9 current snap is 406 2024-04-12T11:27:55.871 INFO:tasks.rados.rados.0.smithi029.stdout:3212: seq_num 1204 ranges {635628=583570,1671967=563661} 2024-04-12T11:27:55.876 INFO:tasks.rados.rados.0.smithi029.stdout:3212: writing smithi02979277-9 from 635628 to 1219198 tid 1 2024-04-12T11:27:55.882 INFO:tasks.rados.rados.0.smithi029.stdout:3212: writing smithi02979277-9 from 1671967 to 2235628 tid 2 2024-04-12T11:27:55.882 INFO:tasks.rados.rados.0.smithi029.stdout:3213: rollback oid 19 current snap is 406 2024-04-12T11:27:55.882 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 19 to 402 2024-04-12T11:27:55.882 INFO:tasks.rados.rados.0.smithi029.stdout:3212: finishing write tid 1 to smithi02979277-9 2024-04-12T11:27:55.882 INFO:tasks.rados.rados.0.smithi029.stdout:3214: read oid 7 snap -1 2024-04-12T11:27:55.882 INFO:tasks.rados.rados.0.smithi029.stdout:3214: expect (ObjNum 1161 snap 394 seq_num 1161) 2024-04-12T11:27:55.882 INFO:tasks.rados.rados.0.smithi029.stdout:3215: write oid 31 current snap is 406 2024-04-12T11:27:55.882 INFO:tasks.rados.rados.0.smithi029.stdout:3215: seq_num 1205 ranges {652435=658819,1730449=521986} 2024-04-12T11:27:55.889 INFO:tasks.rados.rados.0.smithi029.stdout:3215: writing smithi02979277-31 from 652435 to 1311254 tid 1 2024-04-12T11:27:55.895 INFO:tasks.rados.rados.0.smithi029.stdout:3215: writing smithi02979277-31 from 1730449 to 2252435 tid 2 2024-04-12T11:27:55.895 INFO:tasks.rados.rados.0.smithi029.stdout:3216: read oid 40 snap -1 2024-04-12T11:27:55.895 INFO:tasks.rados.rados.0.smithi029.stdout:3216: expect (ObjNum 1012 snap 348 seq_num 1012) 2024-04-12T11:27:55.895 INFO:tasks.rados.rados.0.smithi029.stdout:3217: snap_remove snap 385 2024-04-12T11:27:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:55 smithi087 ceph-mon[89014]: osdmap e870: 8 total, 8 up, 8 in 2024-04-12T11:27:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:55.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:27:56.779 INFO:tasks.rados.rados.0.smithi029.stdout:3217: done (5 left) 2024-04-12T11:27:56.780 INFO:tasks.rados.rados.0.smithi029.stdout:3218: delete oid 3 current snap is 406 2024-04-12T11:27:56.780 INFO:tasks.rados.rados.0.smithi029.stdout:3212: finishing write tid 2 to smithi02979277-9 2024-04-12T11:27:56.780 INFO:tasks.rados.rados.0.smithi029.stdout:3212: finishing write tid 3 to smithi02979277-9 2024-04-12T11:27:56.780 INFO:tasks.rados.rados.0.smithi029.stdout:3213: finishing rollback tid 1 to smithi02979277-19 2024-04-12T11:27:56.780 INFO:tasks.rados.rados.0.smithi029.stdout:3213: finishing rollback tid 2 to smithi02979277-19 2024-04-12T11:27:56.780 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1443 (ObjNum 1187 snap 402 seq_num 1187) dirty exists 2024-04-12T11:27:56.780 INFO:tasks.rados.rados.0.smithi029.stdout:3214: expect (ObjNum 1161 snap 394 seq_num 1161) 2024-04-12T11:27:56.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:56 smithi029 ceph-mon[93650]: pgmap v942: 89 pgs: 89 active+clean; 350 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-04-12T11:27:56.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:56 smithi029 ceph-mon[93650]: osdmap e871: 8 total, 8 up, 8 in 2024-04-12T11:27:56.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:56 smithi029 ceph-mon[97787]: pgmap v942: 89 pgs: 89 active+clean; 350 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-04-12T11:27:56.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:56 smithi029 ceph-mon[97787]: osdmap e871: 8 total, 8 up, 8 in 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3212: finishing write tid 4 to smithi02979277-9 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3212: oid 9 updating version 0 to 1152 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3212: oid 9 version 1152 is already newer than 1151 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3212: oid 9 updating version 1152 to 1153 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 1153 (ObjNum 1204 snap 406 seq_num 1204) dirty exists 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3212: left oid 9 (ObjNum 1204 snap 406 seq_num 1204) 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3215: finishing write tid 1 to smithi02979277-31 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3212: done (5 left) 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3213: done (4 left) 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3214: done (3 left) 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3218: done (2 left) 2024-04-12T11:27:56.945 INFO:tasks.rados.rados.0.smithi029.stdout:3219: write oid 23 current snap is 406 2024-04-12T11:27:56.946 INFO:tasks.rados.rados.0.smithi029.stdout:3215: finishing write tid 2 to smithi02979277-31 2024-04-12T11:27:56.946 INFO:tasks.rados.rados.0.smithi029.stdout:3215: finishing write tid 3 to smithi02979277-31 2024-04-12T11:27:56.946 INFO:tasks.rados.rados.0.smithi029.stdout:3215: finishing write tid 4 to smithi02979277-31 2024-04-12T11:27:56.946 INFO:tasks.rados.rados.0.smithi029.stdout:3215: oid 31 updating version 0 to 1241 2024-04-12T11:27:56.946 INFO:tasks.rados.rados.0.smithi029.stdout:3215: oid 31 updating version 1241 to 1242 2024-04-12T11:27:56.946 INFO:tasks.rados.rados.0.smithi029.stdout:3215: oid 31 version 1242 is already newer than 1240 2024-04-12T11:27:56.946 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1242 (ObjNum 1205 snap 406 seq_num 1205) dirty exists 2024-04-12T11:27:56.946 INFO:tasks.rados.rados.0.smithi029.stdout:3215: left oid 31 (ObjNum 1205 snap 406 seq_num 1205) 2024-04-12T11:27:56.946 INFO:tasks.rados.rados.0.smithi029.stdout:3216: expect (ObjNum 1012 snap 348 seq_num 1012) 2024-04-12T11:27:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:56 smithi087 ceph-mon[89014]: pgmap v942: 89 pgs: 89 active+clean; 350 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-04-12T11:27:56.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:56 smithi087 ceph-mon[89014]: osdmap e871: 8 total, 8 up, 8 in 2024-04-12T11:27:57.165 INFO:tasks.rados.rados.0.smithi029.stdout:3219: seq_num 1207 ranges {686049=409317,1847413=438636} 2024-04-12T11:27:57.171 INFO:tasks.rados.rados.0.smithi029.stdout:3219: writing smithi02979277-23 from 686049 to 1095366 tid 1 2024-04-12T11:27:57.179 INFO:tasks.rados.rados.0.smithi029.stdout:3219: writing smithi02979277-23 from 1847413 to 2286049 tid 2 2024-04-12T11:27:57.179 INFO:tasks.rados.rados.0.smithi029.stdout:3215: done (2 left) 2024-04-12T11:27:57.179 INFO:tasks.rados.rados.0.smithi029.stdout:3216: done (1 left) 2024-04-12T11:27:57.179 INFO:tasks.rados.rados.0.smithi029.stdout:3220: snap_create 2024-04-12T11:27:57.179 INFO:tasks.rados.rados.0.smithi029.stdout:3219: finishing write tid 1 to smithi02979277-23 2024-04-12T11:27:57.182 INFO:tasks.rados.rados.0.smithi029.stdout:3219: finishing write tid 2 to smithi02979277-23 2024-04-12T11:27:57.183 INFO:tasks.rados.rados.0.smithi029.stdout:3219: finishing write tid 3 to smithi02979277-23 2024-04-12T11:27:57.183 INFO:tasks.rados.rados.0.smithi029.stdout:3219: finishing write tid 4 to smithi02979277-23 2024-04-12T11:27:57.183 INFO:tasks.rados.rados.0.smithi029.stdout:3219: oid 23 updating version 0 to 1305 2024-04-12T11:27:57.183 INFO:tasks.rados.rados.0.smithi029.stdout:3219: oid 23 updating version 1305 to 1307 2024-04-12T11:27:57.183 INFO:tasks.rados.rados.0.smithi029.stdout:3219: oid 23 version 1307 is already newer than 1306 2024-04-12T11:27:57.183 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1307 (ObjNum 1207 snap 406 seq_num 1207) dirty exists 2024-04-12T11:27:57.183 INFO:tasks.rados.rados.0.smithi029.stdout:3219: left oid 23 (ObjNum 1207 snap 406 seq_num 1207) 2024-04-12T11:27:57.664 INFO:tasks.rados.rados.0.smithi029.stdout:3219: done (1 left) 2024-04-12T11:27:57.664 INFO:tasks.rados.rados.0.smithi029.stdout:3220: done (0 left) 2024-04-12T11:27:57.665 INFO:tasks.rados.rados.0.smithi029.stdout:3221: rollback oid 11 current snap is 407 2024-04-12T11:27:57.665 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 11 to 357 2024-04-12T11:27:57.665 INFO:tasks.rados.rados.0.smithi029.stdout:3222: read oid 22 snap 388 2024-04-12T11:27:57.665 INFO:tasks.rados.rados.0.smithi029.stdout:3222: expect (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:27:57.665 INFO:tasks.rados.rados.0.smithi029.stdout:3223: snap_remove snap 379 2024-04-12T11:27:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:57 smithi087 ceph-mon[89014]: osdmap e872: 8 total, 8 up, 8 in 2024-04-12T11:27:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:57 smithi087 ceph-mon[89014]: osdmap e873: 8 total, 8 up, 8 in 2024-04-12T11:27:58.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:57 smithi029 ceph-mon[97787]: osdmap e872: 8 total, 8 up, 8 in 2024-04-12T11:27:58.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:57 smithi029 ceph-mon[97787]: osdmap e873: 8 total, 8 up, 8 in 2024-04-12T11:27:58.239 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:57 smithi029 ceph-mon[93650]: osdmap e872: 8 total, 8 up, 8 in 2024-04-12T11:27:58.239 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:57 smithi029 ceph-mon[93650]: osdmap e873: 8 total, 8 up, 8 in 2024-04-12T11:27:58.666 INFO:tasks.rados.rados.0.smithi029.stdout:3223: done (2 left) 2024-04-12T11:27:58.666 INFO:tasks.rados.rados.0.smithi029.stdout:3224: snap_remove snap 373 2024-04-12T11:27:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:58 smithi087 ceph-mon[89014]: pgmap v945: 89 pgs: 89 active+clean; 356 MiB data, 996 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-04-12T11:27:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:58 smithi087 ceph-mon[89014]: osdmap e874: 8 total, 8 up, 8 in 2024-04-12T11:27:59.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:58 smithi029 ceph-mon[97787]: pgmap v945: 89 pgs: 89 active+clean; 356 MiB data, 996 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-04-12T11:27:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:58 smithi029 ceph-mon[97787]: osdmap e874: 8 total, 8 up, 8 in 2024-04-12T11:27:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:58 smithi029 ceph-mon[93650]: pgmap v945: 89 pgs: 89 active+clean; 356 MiB data, 996 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-04-12T11:27:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:58 smithi029 ceph-mon[93650]: osdmap e874: 8 total, 8 up, 8 in 2024-04-12T11:27:59.667 INFO:tasks.rados.rados.0.smithi029.stdout:3224: done (2 left) 2024-04-12T11:27:59.668 INFO:tasks.rados.rados.0.smithi029.stdout:3225: write oid 10 current snap is 407 2024-04-12T11:27:59.668 INFO:tasks.rados.rados.0.smithi029.stdout:3225: seq_num 1208 ranges {702856=484566,1789542=513314} 2024-04-12T11:27:59.674 INFO:tasks.rados.rados.0.smithi029.stdout:3225: writing smithi02979277-10 from 702856 to 1187422 tid 1 2024-04-12T11:27:59.681 INFO:tasks.rados.rados.0.smithi029.stdout:3225: writing smithi02979277-10 from 1789542 to 2302856 tid 2 2024-04-12T11:27:59.681 INFO:tasks.rados.rados.0.smithi029.stdout:3226: snap_create 2024-04-12T11:27:59.681 INFO:tasks.rados.rados.0.smithi029.stdout:3221: finishing rollback tid 1 to smithi02979277-11 2024-04-12T11:27:59.681 INFO:tasks.rados.rados.0.smithi029.stdout:3221: finishing rollback tid 2 to smithi02979277-11 2024-04-12T11:27:59.681 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 2000 (ObjNum 1048 snap 357 seq_num 1048) dirty exists 2024-04-12T11:27:59.681 INFO:tasks.rados.rados.0.smithi029.stdout:3222: expect (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:27:59.841 INFO:tasks.rados.rados.0.smithi029.stdout:3225: finishing write tid 1 to smithi02979277-10 2024-04-12T11:27:59.841 INFO:tasks.rados.rados.0.smithi029.stdout:3225: finishing write tid 2 to smithi02979277-10 2024-04-12T11:27:59.841 INFO:tasks.rados.rados.0.smithi029.stdout:3225: finishing write tid 3 to smithi02979277-10 2024-04-12T11:27:59.841 INFO:tasks.rados.rados.0.smithi029.stdout:3225: finishing write tid 4 to smithi02979277-10 2024-04-12T11:27:59.841 INFO:tasks.rados.rados.0.smithi029.stdout:3225: oid 10 updating version 0 to 1263 2024-04-12T11:27:59.841 INFO:tasks.rados.rados.0.smithi029.stdout:3225: oid 10 version 1263 is already newer than 1256 2024-04-12T11:27:59.841 INFO:tasks.rados.rados.0.smithi029.stdout:3225: oid 10 updating version 1263 to 1264 2024-04-12T11:27:59.841 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 1264 (ObjNum 1208 snap 407 seq_num 1208) dirty exists 2024-04-12T11:27:59.841 INFO:tasks.rados.rados.0.smithi029.stdout:3225: left oid 10 (ObjNum 1208 snap 407 seq_num 1208) 2024-04-12T11:28:00.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:28:00.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:28:00.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:27:59 smithi087 ceph-mon[89014]: osdmap e875: 8 total, 8 up, 8 in 2024-04-12T11:28:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:28:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:28:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[93650]: osdmap e875: 8 total, 8 up, 8 in 2024-04-12T11:28:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:28:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:28:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:28:00.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:27:59 smithi029 ceph-mon[97787]: osdmap e875: 8 total, 8 up, 8 in 2024-04-12T11:28:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:3221: done (3 left) 2024-04-12T11:28:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:3222: done (2 left) 2024-04-12T11:28:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:3225: done (1 left) 2024-04-12T11:28:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:3226: done (0 left) 2024-04-12T11:28:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:3227: write oid 2 current snap is 408 2024-04-12T11:28:00.670 INFO:tasks.rados.rados.0.smithi029.stdout:3227: seq_num 1209 ranges {719663=676168,1964377=355286} 2024-04-12T11:28:00.677 INFO:tasks.rados.rados.0.smithi029.stdout:3227: writing smithi02979277-2 from 719663 to 1395831 tid 1 2024-04-12T11:28:00.683 INFO:tasks.rados.rados.0.smithi029.stdout:3227: writing smithi02979277-2 from 1964377 to 2319663 tid 2 2024-04-12T11:28:00.683 INFO:tasks.rados.rados.0.smithi029.stdout:3228: read oid 41 snap 400 2024-04-12T11:28:00.683 INFO:tasks.rados.rados.0.smithi029.stdout:3228: expect deleted 2024-04-12T11:28:00.683 INFO:tasks.rados.rados.0.smithi029.stdout:3229: read oid 12 snap -1 2024-04-12T11:28:00.683 INFO:tasks.rados.rados.0.smithi029.stdout:3229: expect (ObjNum 1163 snap 395 seq_num 1163) 2024-04-12T11:28:00.683 INFO:tasks.rados.rados.0.smithi029.stdout:3230: snap_remove snap 389 2024-04-12T11:28:01.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:01 smithi029 ceph-mon[93650]: pgmap v948: 89 pgs: 89 active+clean; 361 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.5 MiB/s wr, 19 op/s 2024-04-12T11:28:01.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:01 smithi029 ceph-mon[93650]: osdmap e876: 8 total, 8 up, 8 in 2024-04-12T11:28:01.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:01 smithi029 ceph-mon[97787]: pgmap v948: 89 pgs: 89 active+clean; 361 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.5 MiB/s wr, 19 op/s 2024-04-12T11:28:01.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:01 smithi029 ceph-mon[97787]: osdmap e876: 8 total, 8 up, 8 in 2024-04-12T11:28:01.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:01 smithi087 ceph-mon[89014]: pgmap v948: 89 pgs: 89 active+clean; 361 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.5 MiB/s wr, 19 op/s 2024-04-12T11:28:01.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:01 smithi087 ceph-mon[89014]: osdmap e876: 8 total, 8 up, 8 in 2024-04-12T11:28:01.671 INFO:tasks.rados.rados.0.smithi029.stdout:3230: done (3 left) 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:3231: write oid 23 current snap is 408 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:3227: finishing write tid 1 to smithi02979277-2 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:3227: finishing write tid 2 to smithi02979277-2 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:3227: finishing write tid 3 to smithi02979277-2 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:3227: finishing write tid 4 to smithi02979277-2 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:3227: oid 2 updating version 0 to 356 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:3227: oid 2 updating version 356 to 357 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:3227: oid 2 updating version 357 to 358 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 358 (ObjNum 1209 snap 408 seq_num 1209) dirty exists 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:3227: left oid 2 (ObjNum 1209 snap 408 seq_num 1209) 2024-04-12T11:28:01.672 INFO:tasks.rados.rados.0.smithi029.stdout:3229: expect (ObjNum 1163 snap 395 seq_num 1163) 2024-04-12T11:28:01.795 INFO:tasks.rados.rados.0.smithi029.stdout:3231: seq_num 1210 ranges {736470=751417,2022859=313611} 2024-04-12T11:28:01.803 INFO:tasks.rados.rados.0.smithi029.stdout:3231: writing smithi02979277-23 from 736470 to 1487887 tid 1 2024-04-12T11:28:01.808 INFO:tasks.rados.rados.0.smithi029.stdout:3231: writing smithi02979277-23 from 2022859 to 2336470 tid 2 2024-04-12T11:28:01.808 INFO:tasks.rados.rados.0.smithi029.stdout:3227: done (3 left) 2024-04-12T11:28:01.808 INFO:tasks.rados.rados.0.smithi029.stdout:3228: done (2 left) 2024-04-12T11:28:01.808 INFO:tasks.rados.rados.0.smithi029.stdout:3229: done (1 left) 2024-04-12T11:28:01.808 INFO:tasks.rados.rados.0.smithi029.stdout:3232: delete oid 22 current snap is 408 2024-04-12T11:28:01.808 INFO:tasks.rados.rados.0.smithi029.stdout:3231: finishing write tid 1 to smithi02979277-23 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3231: finishing write tid 2 to smithi02979277-23 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3231: finishing write tid 3 to smithi02979277-23 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3231: finishing write tid 4 to smithi02979277-23 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3231: oid 23 updating version 0 to 1319 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3231: oid 23 updating version 1319 to 1321 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3231: oid 23 version 1321 is already newer than 1320 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1321 (ObjNum 1210 snap 408 seq_num 1210) dirty exists 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3231: left oid 23 (ObjNum 1210 snap 408 seq_num 1210) 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3231: done (1 left) 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3232: done (0 left) 2024-04-12T11:28:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3233: snap_remove snap 392 2024-04-12T11:28:02.670 INFO:tasks.rados.rados.0.smithi029.stdout:3233: done (0 left) 2024-04-12T11:28:02.671 INFO:tasks.rados.rados.0.smithi029.stdout:3234: snap_create 2024-04-12T11:28:02.676 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:28:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:28:02.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:28:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:02 smithi087 ceph-mon[89014]: pgmap v951: 89 pgs: 89 active+clean; 361 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 569 KiB/s wr, 11 op/s 2024-04-12T11:28:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:02 smithi087 ceph-mon[89014]: osdmap e877: 8 total, 8 up, 8 in 2024-04-12T11:28:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:02 smithi087 ceph-mon[89014]: osdmap e878: 8 total, 8 up, 8 in 2024-04-12T11:28:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:02 smithi029 ceph-mon[93650]: pgmap v951: 89 pgs: 89 active+clean; 361 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 569 KiB/s wr, 11 op/s 2024-04-12T11:28:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:02 smithi029 ceph-mon[93650]: osdmap e877: 8 total, 8 up, 8 in 2024-04-12T11:28:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:02 smithi029 ceph-mon[93650]: osdmap e878: 8 total, 8 up, 8 in 2024-04-12T11:28:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:02 smithi029 ceph-mon[97787]: pgmap v951: 89 pgs: 89 active+clean; 361 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 569 KiB/s wr, 11 op/s 2024-04-12T11:28:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:02 smithi029 ceph-mon[97787]: osdmap e877: 8 total, 8 up, 8 in 2024-04-12T11:28:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:02 smithi029 ceph-mon[97787]: osdmap e878: 8 total, 8 up, 8 in 2024-04-12T11:28:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:3234: done (0 left) 2024-04-12T11:28:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:3235: write oid 30 current snap is 409 2024-04-12T11:28:03.668 INFO:tasks.rados.rados.0.smithi029.stdout:3235: seq_num 1212 ranges {770084=501915,2023470=346614} 2024-04-12T11:28:03.675 INFO:tasks.rados.rados.0.smithi029.stdout:3235: writing smithi02979277-30 from 770084 to 1271999 tid 1 2024-04-12T11:28:03.682 INFO:tasks.rados.rados.0.smithi029.stdout:3235: writing smithi02979277-30 from 2023470 to 2370084 tid 2 2024-04-12T11:28:03.682 INFO:tasks.rados.rados.0.smithi029.stdout:3236: read oid 44 snap 388 2024-04-12T11:28:03.682 INFO:tasks.rados.rados.0.smithi029.stdout:3236: expect (ObjNum 1121 snap 374 seq_num 1121) 2024-04-12T11:28:03.682 INFO:tasks.rados.rados.0.smithi029.stdout:3237: write oid 24 current snap is 409 2024-04-12T11:28:03.682 INFO:tasks.rados.rados.0.smithi029.stdout:3235: finishing write tid 1 to smithi02979277-30 2024-04-12T11:28:03.682 INFO:tasks.rados.rados.0.smithi029.stdout:3237: seq_num 1213 ranges {786891=577164,2081952=304939} 2024-04-12T11:28:03.689 INFO:tasks.rados.rados.0.smithi029.stdout:3237: writing smithi02979277-24 from 786891 to 1364055 tid 1 2024-04-12T11:28:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:3237: writing smithi02979277-24 from 2081952 to 2386891 tid 2 2024-04-12T11:28:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:3238: read oid 36 snap -1 2024-04-12T11:28:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:3238: expect (ObjNum 1177 snap 400 seq_num 1177) 2024-04-12T11:28:03.695 INFO:tasks.rados.rados.0.smithi029.stdout:3239: delete oid 16 current snap is 409 2024-04-12T11:28:03.696 INFO:tasks.rados.rados.0.smithi029.stdout:3235: finishing write tid 2 to smithi02979277-30 2024-04-12T11:28:03.696 INFO:tasks.rados.rados.0.smithi029.stdout:3235: finishing write tid 3 to smithi02979277-30 2024-04-12T11:28:03.696 INFO:tasks.rados.rados.0.smithi029.stdout:3235: finishing write tid 4 to smithi02979277-30 2024-04-12T11:28:03.696 INFO:tasks.rados.rados.0.smithi029.stdout:3235: oid 30 updating version 0 to 991 2024-04-12T11:28:03.696 INFO:tasks.rados.rados.0.smithi029.stdout:3235: oid 30 version 991 is already newer than 990 2024-04-12T11:28:03.696 INFO:tasks.rados.rados.0.smithi029.stdout:3235: oid 30 updating version 991 to 992 2024-04-12T11:28:03.696 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 992 (ObjNum 1212 snap 409 seq_num 1212) dirty exists 2024-04-12T11:28:03.696 INFO:tasks.rados.rados.0.smithi029.stdout:3235: left oid 30 (ObjNum 1212 snap 409 seq_num 1212) 2024-04-12T11:28:03.696 INFO:tasks.rados.rados.0.smithi029.stdout:3237: finishing write tid 1 to smithi02979277-24 2024-04-12T11:28:03.696 INFO:tasks.rados.rados.0.smithi029.stdout:3236: expect (ObjNum 1121 snap 374 seq_num 1121) 2024-04-12T11:28:03.825 INFO:tasks.rados.rados.0.smithi029.stdout:3237: finishing write tid 2 to smithi02979277-24 2024-04-12T11:28:03.825 INFO:tasks.rados.rados.0.smithi029.stdout:3237: finishing write tid 3 to smithi02979277-24 2024-04-12T11:28:03.825 INFO:tasks.rados.rados.0.smithi029.stdout:3237: finishing write tid 4 to smithi02979277-24 2024-04-12T11:28:03.825 INFO:tasks.rados.rados.0.smithi029.stdout:3237: oid 24 updating version 0 to 2022 2024-04-12T11:28:03.825 INFO:tasks.rados.rados.0.smithi029.stdout:3237: oid 24 updating version 2022 to 2024 2024-04-12T11:28:03.825 INFO:tasks.rados.rados.0.smithi029.stdout:3237: oid 24 version 2024 is already newer than 2023 2024-04-12T11:28:03.825 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 2024 (ObjNum 1213 snap 409 seq_num 1213) dirty exists 2024-04-12T11:28:03.825 INFO:tasks.rados.rados.0.smithi029.stdout:3237: left oid 24 (ObjNum 1213 snap 409 seq_num 1213) 2024-04-12T11:28:03.825 INFO:tasks.rados.rados.0.smithi029.stdout:3238: expect (ObjNum 1177 snap 400 seq_num 1177) 2024-04-12T11:28:03.951 INFO:tasks.rados.rados.0.smithi029.stdout:3235: done (4 left) 2024-04-12T11:28:03.951 INFO:tasks.rados.rados.0.smithi029.stdout:3236: done (3 left) 2024-04-12T11:28:03.951 INFO:tasks.rados.rados.0.smithi029.stdout:3237: done (2 left) 2024-04-12T11:28:03.952 INFO:tasks.rados.rados.0.smithi029.stdout:3238: done (1 left) 2024-04-12T11:28:03.952 INFO:tasks.rados.rados.0.smithi029.stdout:3239: done (0 left) 2024-04-12T11:28:03.952 INFO:tasks.rados.rados.0.smithi029.stdout:3240: snap_remove snap 362 2024-04-12T11:28:04.683 INFO:tasks.rados.rados.0.smithi029.stdout:3240: done (0 left) 2024-04-12T11:28:04.683 INFO:tasks.rados.rados.0.smithi029.stdout:3241: write oid 44 current snap is 409 2024-04-12T11:28:04.683 INFO:tasks.rados.rados.0.smithi029.stdout:3241: seq_num 1215 ranges {420505=727662,1798916=553091} 2024-04-12T11:28:04.689 INFO:tasks.rados.rados.0.smithi029.stdout:3241: writing smithi02979277-44 from 420505 to 1148167 tid 1 2024-04-12T11:28:04.696 INFO:tasks.rados.rados.0.smithi029.stdout:3241: writing smithi02979277-44 from 1798916 to 2352007 tid 2 2024-04-12T11:28:04.696 INFO:tasks.rados.rados.0.smithi029.stdout:3242: write oid 17 current snap is 409 2024-04-12T11:28:04.696 INFO:tasks.rados.rados.0.smithi029.stdout:3241: finishing write tid 1 to smithi02979277-44 2024-04-12T11:28:04.696 INFO:tasks.rados.rados.0.smithi029.stdout:3242: seq_num 1216 ranges {437312=402911,1341045=696749} 2024-04-12T11:28:04.700 INFO:tasks.rados.rados.0.smithi029.stdout:3242: writing smithi02979277-17 from 437312 to 840223 tid 1 2024-04-12T11:28:04.707 INFO:tasks.rados.rados.0.smithi029.stdout:3242: writing smithi02979277-17 from 1341045 to 2037794 tid 2 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3243: read oid 28 snap -1 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3243: expect (ObjNum 1108 snap 370 seq_num 1108) 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3244: write oid 21 current snap is 409 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3241: finishing write tid 2 to smithi02979277-44 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3241: finishing write tid 3 to smithi02979277-44 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3241: finishing write tid 4 to smithi02979277-44 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3241: oid 44 updating version 0 to 1544 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3241: oid 44 updating version 1544 to 1546 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3241: oid 44 version 1546 is already newer than 1545 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1546 (ObjNum 1215 snap 409 seq_num 1215) dirty exists 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3241: left oid 44 (ObjNum 1215 snap 409 seq_num 1215) 2024-04-12T11:28:04.708 INFO:tasks.rados.rados.0.smithi029.stdout:3244: seq_num 1217 ranges {454119=594513,1515880=556760} 2024-04-12T11:28:04.714 INFO:tasks.rados.rados.0.smithi029.stdout:3244: writing smithi02979277-21 from 454119 to 1048632 tid 1 2024-04-12T11:28:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3244: writing smithi02979277-21 from 1515880 to 2072640 tid 2 2024-04-12T11:28:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3241: done (3 left) 2024-04-12T11:28:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3245: write oid 32 current snap is 409 2024-04-12T11:28:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3242: finishing write tid 1 to smithi02979277-17 2024-04-12T11:28:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3242: finishing write tid 2 to smithi02979277-17 2024-04-12T11:28:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3242: finishing write tid 3 to smithi02979277-17 2024-04-12T11:28:04.721 INFO:tasks.rados.rados.0.smithi029.stdout:3243: expect (ObjNum 1108 snap 370 seq_num 1108) 2024-04-12T11:28:04.845 INFO:tasks.rados.rados.0.smithi029.stdout:3242: finishing write tid 4 to smithi02979277-17 2024-04-12T11:28:04.845 INFO:tasks.rados.rados.0.smithi029.stdout:3242: oid 17 updating version 0 to 1652 2024-04-12T11:28:04.845 INFO:tasks.rados.rados.0.smithi029.stdout:3242: oid 17 version 1652 is already newer than 1651 2024-04-12T11:28:04.845 INFO:tasks.rados.rados.0.smithi029.stdout:3242: oid 17 updating version 1652 to 1653 2024-04-12T11:28:04.845 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1653 (ObjNum 1216 snap 409 seq_num 1216) dirty exists 2024-04-12T11:28:04.845 INFO:tasks.rados.rados.0.smithi029.stdout:3242: left oid 17 (ObjNum 1216 snap 409 seq_num 1216) 2024-04-12T11:28:04.846 INFO:tasks.rados.rados.0.smithi029.stdout:3244: finishing write tid 1 to smithi02979277-21 2024-04-12T11:28:04.846 INFO:tasks.rados.rados.0.smithi029.stdout:3245: seq_num 1218 ranges {470926=669762,1574362=700418} 2024-04-12T11:28:04.851 INFO:tasks.rados.rados.0.smithi029.stdout:3245: writing smithi02979277-32 from 470926 to 1140688 tid 1 2024-04-12T11:28:04.858 INFO:tasks.rados.rados.0.smithi029.stdout:3245: writing smithi02979277-32 from 1574362 to 2274780 tid 2 2024-04-12T11:28:04.858 INFO:tasks.rados.rados.0.smithi029.stdout:3242: done (3 left) 2024-04-12T11:28:04.858 INFO:tasks.rados.rados.0.smithi029.stdout:3243: done (2 left) 2024-04-12T11:28:04.858 INFO:tasks.rados.rados.0.smithi029.stdout:3246: delete oid 22 current snap is 409 2024-04-12T11:28:04.858 INFO:tasks.rados.rados.0.smithi029.stdout:3244: finishing write tid 2 to smithi02979277-21 2024-04-12T11:28:04.858 INFO:tasks.rados.rados.0.smithi029.stdout:3244: finishing write tid 3 to smithi02979277-21 2024-04-12T11:28:04.858 INFO:tasks.rados.rados.0.smithi029.stdout:3244: finishing write tid 4 to smithi02979277-21 2024-04-12T11:28:04.858 INFO:tasks.rados.rados.0.smithi029.stdout:3244: oid 21 updating version 0 to 1549 2024-04-12T11:28:04.859 INFO:tasks.rados.rados.0.smithi029.stdout:3244: oid 21 updating version 1549 to 1550 2024-04-12T11:28:04.859 INFO:tasks.rados.rados.0.smithi029.stdout:3244: oid 21 version 1550 is already newer than 1548 2024-04-12T11:28:04.859 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1550 (ObjNum 1217 snap 409 seq_num 1217) dirty exists 2024-04-12T11:28:04.859 INFO:tasks.rados.rados.0.smithi029.stdout:3244: left oid 21 (ObjNum 1217 snap 409 seq_num 1217) 2024-04-12T11:28:04.859 INFO:tasks.rados.rados.0.smithi029.stdout:3245: finishing write tid 1 to smithi02979277-32 2024-04-12T11:28:04.860 INFO:tasks.rados.rados.0.smithi029.stdout:3244: done (2 left) 2024-04-12T11:28:04.861 INFO:tasks.rados.rados.0.smithi029.stdout:3246: done (1 left) 2024-04-12T11:28:04.861 INFO:tasks.rados.rados.0.smithi029.stdout:3247: write oid 45 current snap is 409 2024-04-12T11:28:04.861 INFO:tasks.rados.rados.0.smithi029.stdout:3247: seq_num 1220 ranges {504540=420260,1574973=704087} 2024-04-12T11:28:04.866 INFO:tasks.rados.rados.0.smithi029.stdout:3247: writing smithi02979277-45 from 504540 to 924800 tid 1 2024-04-12T11:28:04.873 INFO:tasks.rados.rados.0.smithi029.stdout:3247: writing smithi02979277-45 from 1574973 to 2279060 tid 2 2024-04-12T11:28:04.873 INFO:tasks.rados.rados.0.smithi029.stdout:3248: snap_remove snap 403 2024-04-12T11:28:04.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:04 smithi029 ceph-mon[93650]: pgmap v954: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 356 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 260 KiB/s wr, 5 op/s 2024-04-12T11:28:04.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:04 smithi029 ceph-mon[93650]: osdmap e879: 8 total, 8 up, 8 in 2024-04-12T11:28:04.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:04 smithi029 ceph-mon[97787]: pgmap v954: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 356 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 260 KiB/s wr, 5 op/s 2024-04-12T11:28:04.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:04 smithi029 ceph-mon[97787]: osdmap e879: 8 total, 8 up, 8 in 2024-04-12T11:28:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:04 smithi087 ceph-mon[89014]: pgmap v954: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 356 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 260 KiB/s wr, 5 op/s 2024-04-12T11:28:04.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:04 smithi087 ceph-mon[89014]: osdmap e879: 8 total, 8 up, 8 in 2024-04-12T11:28:05.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:28:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:28:04] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.43.0" 2024-04-12T11:28:05.689 INFO:tasks.rados.rados.0.smithi029.stdout:3245: finishing write tid 2 to smithi02979277-32 2024-04-12T11:28:05.689 INFO:tasks.rados.rados.0.smithi029.stdout:3245: finishing write tid 3 to smithi02979277-32 2024-04-12T11:28:05.689 INFO:tasks.rados.rados.0.smithi029.stdout:3245: finishing write tid 4 to smithi02979277-32 2024-04-12T11:28:05.689 INFO:tasks.rados.rados.0.smithi029.stdout:3245: oid 32 updating version 0 to 1449 2024-04-12T11:28:05.689 INFO:tasks.rados.rados.0.smithi029.stdout:3245: oid 32 updating version 1449 to 1450 2024-04-12T11:28:05.689 INFO:tasks.rados.rados.0.smithi029.stdout:3245: oid 32 version 1450 is already newer than 1448 2024-04-12T11:28:05.689 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 1450 (ObjNum 1218 snap 409 seq_num 1218) dirty exists 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3245: left oid 32 (ObjNum 1218 snap 409 seq_num 1218) 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3247: finishing write tid 1 to smithi02979277-45 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3247: finishing write tid 2 to smithi02979277-45 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3247: finishing write tid 3 to smithi02979277-45 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3247: finishing write tid 4 to smithi02979277-45 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3247: oid 45 updating version 0 to 1533 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3247: oid 45 updating version 1533 to 1534 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3247: oid 45 version 1534 is already newer than 1532 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1534 (ObjNum 1220 snap 409 seq_num 1220) dirty exists 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3247: left oid 45 (ObjNum 1220 snap 409 seq_num 1220) 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3245: done (2 left) 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3247: done (1 left) 2024-04-12T11:28:05.690 INFO:tasks.rados.rados.0.smithi029.stdout:3248: done (0 left) 2024-04-12T11:28:05.691 INFO:tasks.rados.rados.0.smithi029.stdout:3249: write oid 13 current snap is 409 2024-04-12T11:28:05.691 INFO:tasks.rados.rados.0.smithi029.stdout:3249: seq_num 1221 ranges {521347=495509,1633455=564098} 2024-04-12T11:28:05.694 INFO:tasks.rados.rados.0.smithi029.stdout:3249: writing smithi02979277-13 from 521347 to 1016856 tid 1 2024-04-12T11:28:05.702 INFO:tasks.rados.rados.0.smithi029.stdout:3249: writing smithi02979277-13 from 1633455 to 2197553 tid 2 2024-04-12T11:28:05.702 INFO:tasks.rados.rados.0.smithi029.stdout:3250: snap_remove snap 407 2024-04-12T11:28:05.702 INFO:tasks.rados.rados.0.smithi029.stdout:3249: finishing write tid 1 to smithi02979277-13 2024-04-12T11:28:05.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:05 smithi087 ceph-mon[89014]: osdmap e880: 8 total, 8 up, 8 in 2024-04-12T11:28:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:05 smithi029 ceph-mon[93650]: osdmap e880: 8 total, 8 up, 8 in 2024-04-12T11:28:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:05 smithi029 ceph-mon[97787]: osdmap e880: 8 total, 8 up, 8 in 2024-04-12T11:28:06.696 INFO:tasks.rados.rados.0.smithi029.stdout:3249: finishing write tid 2 to smithi02979277-13 2024-04-12T11:28:06.696 INFO:tasks.rados.rados.0.smithi029.stdout:3249: finishing write tid 3 to smithi02979277-13 2024-04-12T11:28:06.696 INFO:tasks.rados.rados.0.smithi029.stdout:3249: finishing write tid 4 to smithi02979277-13 2024-04-12T11:28:06.696 INFO:tasks.rados.rados.0.smithi029.stdout:3249: oid 13 updating version 0 to 1666 2024-04-12T11:28:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:3249: oid 13 updating version 1666 to 1667 2024-04-12T11:28:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:3249: oid 13 version 1667 is already newer than 1665 2024-04-12T11:28:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1667 (ObjNum 1221 snap 409 seq_num 1221) dirty exists 2024-04-12T11:28:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:3249: left oid 13 (ObjNum 1221 snap 409 seq_num 1221) 2024-04-12T11:28:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:3249: done (1 left) 2024-04-12T11:28:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:3250: done (0 left) 2024-04-12T11:28:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:3251: read oid 45 snap -1 2024-04-12T11:28:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:3251: expect (ObjNum 1220 snap 409 seq_num 1220) 2024-04-12T11:28:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:3252: write oid 12 current snap is 409 2024-04-12T11:28:06.697 INFO:tasks.rados.rados.0.smithi029.stdout:3252: seq_num 1222 ranges {538154=570758,1691937=707756} 2024-04-12T11:28:06.703 INFO:tasks.rados.rados.0.smithi029.stdout:3252: writing smithi02979277-12 from 538154 to 1108912 tid 1 2024-04-12T11:28:06.711 INFO:tasks.rados.rados.0.smithi029.stdout:3252: writing smithi02979277-12 from 1691937 to 2399693 tid 2 2024-04-12T11:28:06.711 INFO:tasks.rados.rados.0.smithi029.stdout:3253: rollback oid 7 current snap is 409 2024-04-12T11:28:06.711 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 357 2024-04-12T11:28:06.711 INFO:tasks.rados.rados.0.smithi029.stdout:3251: expect (ObjNum 1220 snap 409 seq_num 1220) 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3252: finishing write tid 1 to smithi02979277-12 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3252: finishing write tid 2 to smithi02979277-12 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3252: finishing write tid 3 to smithi02979277-12 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3252: finishing write tid 4 to smithi02979277-12 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3252: oid 12 updating version 0 to 1541 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3252: oid 12 updating version 1541 to 1542 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3252: oid 12 version 1542 is already newer than 1540 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1542 (ObjNum 1222 snap 409 seq_num 1222) dirty exists 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3252: left oid 12 (ObjNum 1222 snap 409 seq_num 1222) 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3253: finishing rollback tid 0 to smithi02979277-7 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3251: done (2 left) 2024-04-12T11:28:06.847 INFO:tasks.rados.rados.0.smithi029.stdout:3252: done (1 left) 2024-04-12T11:28:06.848 INFO:tasks.rados.rados.0.smithi029.stdout:3254: read oid 41 snap -1 2024-04-12T11:28:06.848 INFO:tasks.rados.rados.0.smithi029.stdout:3254: expect deleted 2024-04-12T11:28:06.848 INFO:tasks.rados.rados.0.smithi029.stdout:3255: snap_create 2024-04-12T11:28:06.848 INFO:tasks.rados.rados.0.smithi029.stdout:3253: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:28:06.848 INFO:tasks.rados.rados.0.smithi029.stdout:3253: finishing rollback tid 2 to smithi02979277-7 2024-04-12T11:28:06.848 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1546 (ObjNum 959 snap 329 seq_num 959) dirty exists 2024-04-12T11:28:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:06 smithi087 ceph-mon[89014]: pgmap v957: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 355 MiB data, 1005 MiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 756 KiB/s wr, 10 op/s 2024-04-12T11:28:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:06 smithi087 ceph-mon[89014]: osdmap e881: 8 total, 8 up, 8 in 2024-04-12T11:28:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:07.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:06 smithi029 ceph-mon[93650]: pgmap v957: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 355 MiB data, 1005 MiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 756 KiB/s wr, 10 op/s 2024-04-12T11:28:07.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:06 smithi029 ceph-mon[93650]: osdmap e881: 8 total, 8 up, 8 in 2024-04-12T11:28:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:06 smithi029 ceph-mon[97787]: pgmap v957: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 355 MiB data, 1005 MiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 756 KiB/s wr, 10 op/s 2024-04-12T11:28:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:06 smithi029 ceph-mon[97787]: osdmap e881: 8 total, 8 up, 8 in 2024-04-12T11:28:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:07.669 INFO:tasks.rados.rados.0.smithi029.stdout:3253: done (2 left) 2024-04-12T11:28:07.669 INFO:tasks.rados.rados.0.smithi029.stdout:3254: done (1 left) 2024-04-12T11:28:07.669 INFO:tasks.rados.rados.0.smithi029.stdout:3255: done (0 left) 2024-04-12T11:28:07.669 INFO:tasks.rados.rados.0.smithi029.stdout:3256: snap_create 2024-04-12T11:28:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:07 smithi087 ceph-mon[89014]: osdmap e882: 8 total, 8 up, 8 in 2024-04-12T11:28:07.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:07 smithi087 ceph-mon[89014]: osdmap e883: 8 total, 8 up, 8 in 2024-04-12T11:28:08.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:07 smithi029 ceph-mon[93650]: osdmap e882: 8 total, 8 up, 8 in 2024-04-12T11:28:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:07 smithi029 ceph-mon[93650]: osdmap e883: 8 total, 8 up, 8 in 2024-04-12T11:28:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:07 smithi029 ceph-mon[97787]: osdmap e882: 8 total, 8 up, 8 in 2024-04-12T11:28:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:07 smithi029 ceph-mon[97787]: osdmap e883: 8 total, 8 up, 8 in 2024-04-12T11:28:08.669 INFO:tasks.rados.rados.0.smithi029.stdout:3256: done (0 left) 2024-04-12T11:28:08.669 INFO:tasks.rados.rados.0.smithi029.stdout:3257: read oid 12 snap -1 2024-04-12T11:28:08.669 INFO:tasks.rados.rados.0.smithi029.stdout:3257: expect (ObjNum 1222 snap 409 seq_num 1222) 2024-04-12T11:28:08.670 INFO:tasks.rados.rados.0.smithi029.stdout:3258: write oid 27 current snap is 411 2024-04-12T11:28:08.670 INFO:tasks.rados.rados.0.smithi029.stdout:3258: seq_num 1223 ranges {554961=646007,1750419=451414} 2024-04-12T11:28:08.676 INFO:tasks.rados.rados.0.smithi029.stdout:3258: writing smithi02979277-27 from 554961 to 1200968 tid 1 2024-04-12T11:28:08.681 INFO:tasks.rados.rados.0.smithi029.stdout:3258: writing smithi02979277-27 from 1750419 to 2201833 tid 2 2024-04-12T11:28:08.681 INFO:tasks.rados.rados.0.smithi029.stdout:3259: rollback oid 3 current snap is 411 2024-04-12T11:28:08.681 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 395 2024-04-12T11:28:08.682 INFO:tasks.rados.rados.0.smithi029.stdout:3257: expect (ObjNum 1222 snap 409 seq_num 1222) 2024-04-12T11:28:08.805 INFO:tasks.rados.rados.0.smithi029.stdout:3258: finishing write tid 1 to smithi02979277-27 2024-04-12T11:28:08.805 INFO:tasks.rados.rados.0.smithi029.stdout:3258: finishing write tid 2 to smithi02979277-27 2024-04-12T11:28:08.805 INFO:tasks.rados.rados.0.smithi029.stdout:3258: finishing write tid 3 to smithi02979277-27 2024-04-12T11:28:08.805 INFO:tasks.rados.rados.0.smithi029.stdout:3258: finishing write tid 4 to smithi02979277-27 2024-04-12T11:28:08.805 INFO:tasks.rados.rados.0.smithi029.stdout:3258: oid 27 updating version 0 to 1000 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:3258: oid 27 updating version 1000 to 1001 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:3258: oid 27 updating version 1001 to 1002 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 1002 (ObjNum 1223 snap 411 seq_num 1223) dirty exists 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:3258: left oid 27 (ObjNum 1223 snap 411 seq_num 1223) 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:3259: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1914 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:3257: done (2 left) 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:3258: done (1 left) 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:3259: done (0 left) 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:3260: rollback oid 39 current snap is 411 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 401 2024-04-12T11:28:08.806 INFO:tasks.rados.rados.0.smithi029.stdout:3261: snap_remove snap 402 2024-04-12T11:28:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:08 smithi087 ceph-mon[89014]: pgmap v960: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 353 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-04-12T11:28:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:08 smithi087 ceph-mon[89014]: osdmap e884: 8 total, 8 up, 8 in 2024-04-12T11:28:09.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:08 smithi029 ceph-mon[97787]: pgmap v960: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 353 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-04-12T11:28:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:08 smithi029 ceph-mon[97787]: osdmap e884: 8 total, 8 up, 8 in 2024-04-12T11:28:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:08 smithi029 ceph-mon[93650]: pgmap v960: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 353 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-04-12T11:28:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:08 smithi029 ceph-mon[93650]: osdmap e884: 8 total, 8 up, 8 in 2024-04-12T11:28:09.708 INFO:tasks.rados.rados.0.smithi029.stdout:3260: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:28:09.708 INFO:tasks.rados.rados.0.smithi029.stdout:3260: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:28:09.708 INFO:tasks.rados.rados.0.smithi029.stdout:3260: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:28:09.708 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 447 (ObjNum 1088 snap 365 seq_num 1088) dirty exists 2024-04-12T11:28:09.708 INFO:tasks.rados.rados.0.smithi029.stdout:3260: done (1 left) 2024-04-12T11:28:09.708 INFO:tasks.rados.rados.0.smithi029.stdout:3261: done (0 left) 2024-04-12T11:28:09.708 INFO:tasks.rados.rados.0.smithi029.stdout:3262: read oid 37 snap -1 2024-04-12T11:28:09.708 INFO:tasks.rados.rados.0.smithi029.stdout:3262: expect deleted 2024-04-12T11:28:09.708 INFO:tasks.rados.rados.0.smithi029.stdout:3263: read oid 42 snap -1 2024-04-12T11:28:09.708 INFO:tasks.rados.rados.0.smithi029.stdout:3263: expect (ObjNum 1167 snap 397 seq_num 1167) 2024-04-12T11:28:09.709 INFO:tasks.rados.rados.0.smithi029.stdout:3264: rollback oid 26 current snap is 411 2024-04-12T11:28:09.709 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 341 2024-04-12T11:28:09.709 INFO:tasks.rados.rados.0.smithi029.stdout:3265: read oid 46 snap 406 2024-04-12T11:28:09.709 INFO:tasks.rados.rados.0.smithi029.stdout:3265: expect (ObjNum 1131 snap 379 seq_num 1131) 2024-04-12T11:28:09.709 INFO:tasks.rados.rados.0.smithi029.stdout:3266: read oid 29 snap -1 2024-04-12T11:28:09.709 INFO:tasks.rados.rados.0.smithi029.stdout:3266: expect (ObjNum 1159 snap 394 seq_num 1159) 2024-04-12T11:28:09.709 INFO:tasks.rados.rados.0.smithi029.stdout:3267: snap_create 2024-04-12T11:28:09.720 INFO:tasks.rados.rados.0.smithi029.stdout:3265: expect (ObjNum 1131 snap 379 seq_num 1131) 2024-04-12T11:28:09.857 INFO:tasks.rados.rados.0.smithi029.stdout:3264: finishing rollback tid 0 to smithi02979277-26 2024-04-12T11:28:09.857 INFO:tasks.rados.rados.0.smithi029.stdout:3266: expect (ObjNum 1159 snap 394 seq_num 1159) 2024-04-12T11:28:09.969 INFO:tasks.rados.rados.0.smithi029.stdout:3264: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:28:09.969 INFO:tasks.rados.rados.0.smithi029.stdout:3264: finishing rollback tid 2 to smithi02979277-26 2024-04-12T11:28:09.969 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 2047 (ObjNum 938 snap 325 seq_num 938) dirty exists 2024-04-12T11:28:09.969 INFO:tasks.rados.rados.0.smithi029.stdout:3263: expect (ObjNum 1167 snap 397 seq_num 1167) 2024-04-12T11:28:10.708 INFO:tasks.rados.rados.0.smithi029.stdout:3262: done (5 left) 2024-04-12T11:28:10.708 INFO:tasks.rados.rados.0.smithi029.stdout:3263: done (4 left) 2024-04-12T11:28:10.708 INFO:tasks.rados.rados.0.smithi029.stdout:3264: done (3 left) 2024-04-12T11:28:10.708 INFO:tasks.rados.rados.0.smithi029.stdout:3265: done (2 left) 2024-04-12T11:28:10.708 INFO:tasks.rados.rados.0.smithi029.stdout:3266: done (1 left) 2024-04-12T11:28:10.709 INFO:tasks.rados.rados.0.smithi029.stdout:3267: done (0 left) 2024-04-12T11:28:10.709 INFO:tasks.rados.rados.0.smithi029.stdout:3268: delete oid 47 current snap is 412 2024-04-12T11:28:10.714 INFO:tasks.rados.rados.0.smithi029.stdout:3268: done (0 left) 2024-04-12T11:28:10.714 INFO:tasks.rados.rados.0.smithi029.stdout:3269: rollback oid 17 current snap is 412 2024-04-12T11:28:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 409 2024-04-12T11:28:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:3270: read oid 18 snap -1 2024-04-12T11:28:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:3270: expect (ObjNum 1198 snap 406 seq_num 1198) 2024-04-12T11:28:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:3271: read oid 19 snap -1 2024-04-12T11:28:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:3271: expect (ObjNum 1187 snap 402 seq_num 1187) 2024-04-12T11:28:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:3272: read oid 23 snap -1 2024-04-12T11:28:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:3272: expect (ObjNum 1210 snap 408 seq_num 1210) 2024-04-12T11:28:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:3273: rollback oid 40 current snap is 412 2024-04-12T11:28:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 40 to 398 2024-04-12T11:28:10.715 INFO:tasks.rados.rados.0.smithi029.stdout:3274: snap_create 2024-04-12T11:28:10.722 INFO:tasks.rados.rados.0.smithi029.stdout:3273: finishing rollback tid 0 to smithi02979277-40 2024-04-12T11:28:10.725 INFO:tasks.rados.rados.0.smithi029.stdout:3273: finishing rollback tid 1 to smithi02979277-40 2024-04-12T11:28:10.725 INFO:tasks.rados.rados.0.smithi029.stdout:3273: finishing rollback tid 2 to smithi02979277-40 2024-04-12T11:28:10.725 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 2055 (ObjNum 1165 snap 395 seq_num 1165) dirty exists 2024-04-12T11:28:10.733 INFO:tasks.rados.rados.0.smithi029.stdout:3271: expect (ObjNum 1187 snap 402 seq_num 1187) 2024-04-12T11:28:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:10 smithi087 ceph-mon[89014]: pgmap v963: 89 pgs: 89 active+clean; 341 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:28:10.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:10 smithi087 ceph-mon[89014]: osdmap e885: 8 total, 8 up, 8 in 2024-04-12T11:28:10.979 INFO:tasks.rados.rados.0.smithi029.stdout:3272: expect (ObjNum 1210 snap 408 seq_num 1210) 2024-04-12T11:28:11.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:10 smithi029 ceph-mon[97787]: pgmap v963: 89 pgs: 89 active+clean; 341 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:28:11.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:10 smithi029 ceph-mon[97787]: osdmap e885: 8 total, 8 up, 8 in 2024-04-12T11:28:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:10 smithi029 ceph-mon[93650]: pgmap v963: 89 pgs: 89 active+clean; 341 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-04-12T11:28:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:10 smithi029 ceph-mon[93650]: osdmap e885: 8 total, 8 up, 8 in 2024-04-12T11:28:11.136 INFO:tasks.rados.rados.0.smithi029.stdout:3269: finishing rollback tid 0 to smithi02979277-17 2024-04-12T11:28:11.136 INFO:tasks.rados.rados.0.smithi029.stdout:3269: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:28:11.136 INFO:tasks.rados.rados.0.smithi029.stdout:3269: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:28:11.136 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1681 (ObjNum 1216 snap 409 seq_num 1216) dirty exists 2024-04-12T11:28:11.136 INFO:tasks.rados.rados.0.smithi029.stdout:3270: expect (ObjNum 1198 snap 406 seq_num 1198) 2024-04-12T11:28:11.720 INFO:tasks.rados.rados.0.smithi029.stdout:3269: done (5 left) 2024-04-12T11:28:11.720 INFO:tasks.rados.rados.0.smithi029.stdout:3270: done (4 left) 2024-04-12T11:28:11.720 INFO:tasks.rados.rados.0.smithi029.stdout:3271: done (3 left) 2024-04-12T11:28:11.720 INFO:tasks.rados.rados.0.smithi029.stdout:3272: done (2 left) 2024-04-12T11:28:11.720 INFO:tasks.rados.rados.0.smithi029.stdout:3273: done (1 left) 2024-04-12T11:28:11.720 INFO:tasks.rados.rados.0.smithi029.stdout:3274: done (0 left) 2024-04-12T11:28:11.720 INFO:tasks.rados.rados.0.smithi029.stdout:3275: write oid 17 current snap is 413 2024-04-12T11:28:11.720 INFO:tasks.rados.rados.0.smithi029.stdout:3275: seq_num 1225 ranges {588575=512858,1867383=455083} 2024-04-12T11:28:11.727 INFO:tasks.rados.rados.0.smithi029.stdout:3275: writing smithi02979277-17 from 588575 to 1101433 tid 1 2024-04-12T11:28:11.735 INFO:tasks.rados.rados.0.smithi029.stdout:3275: writing smithi02979277-17 from 1867383 to 2322466 tid 2 2024-04-12T11:28:11.735 INFO:tasks.rados.rados.0.smithi029.stdout:3276: write oid 21 current snap is 413 2024-04-12T11:28:11.735 INFO:tasks.rados.rados.0.smithi029.stdout:3276: seq_num 1226 ranges {605382=588107,1925865=679517} 2024-04-12T11:28:11.741 INFO:tasks.rados.rados.0.smithi029.stdout:3276: writing smithi02979277-21 from 605382 to 1193489 tid 1 2024-04-12T11:28:11.750 INFO:tasks.rados.rados.0.smithi029.stdout:3276: writing smithi02979277-21 from 1925865 to 2605382 tid 2 2024-04-12T11:28:11.750 INFO:tasks.rados.rados.0.smithi029.stdout:3277: write oid 47 current snap is 413 2024-04-12T11:28:11.750 INFO:tasks.rados.rados.0.smithi029.stdout:3277: seq_num 1227 ranges {622189=663356,1984347=458752} 2024-04-12T11:28:11.756 INFO:tasks.rados.rados.0.smithi029.stdout:3277: writing smithi02979277-47 from 622189 to 1285545 tid 1 2024-04-12T11:28:11.763 INFO:tasks.rados.rados.0.smithi029.stdout:3277: writing smithi02979277-47 from 1984347 to 2443099 tid 2 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3278: delete oid 5 current snap is 413 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3275: finishing write tid 1 to smithi02979277-17 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3275: finishing write tid 2 to smithi02979277-17 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3275: finishing write tid 3 to smithi02979277-17 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3275: finishing write tid 4 to smithi02979277-17 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3275: oid 17 updating version 0 to 1684 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3275: oid 17 updating version 1684 to 1685 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3275: oid 17 version 1685 is already newer than 1683 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1685 (ObjNum 1225 snap 413 seq_num 1225) dirty exists 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3275: left oid 17 (ObjNum 1225 snap 413 seq_num 1225) 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3276: finishing write tid 1 to smithi02979277-21 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3276: finishing write tid 2 to smithi02979277-21 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3276: finishing write tid 3 to smithi02979277-21 2024-04-12T11:28:11.764 INFO:tasks.rados.rados.0.smithi029.stdout:3276: finishing write tid 4 to smithi02979277-21 2024-04-12T11:28:11.765 INFO:tasks.rados.rados.0.smithi029.stdout:3276: oid 21 updating version 0 to 1568 2024-04-12T11:28:11.765 INFO:tasks.rados.rados.0.smithi029.stdout:3276: oid 21 updating version 1568 to 1570 2024-04-12T11:28:11.765 INFO:tasks.rados.rados.0.smithi029.stdout:3276: oid 21 version 1570 is already newer than 1569 2024-04-12T11:28:11.765 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1570 (ObjNum 1226 snap 413 seq_num 1226) dirty exists 2024-04-12T11:28:11.765 INFO:tasks.rados.rados.0.smithi029.stdout:3276: left oid 21 (ObjNum 1226 snap 413 seq_num 1226) 2024-04-12T11:28:11.765 INFO:tasks.rados.rados.0.smithi029.stdout:3277: finishing write tid 1 to smithi02979277-47 2024-04-12T11:28:11.765 INFO:tasks.rados.rados.0.smithi029.stdout:3275: done (3 left) 2024-04-12T11:28:11.765 INFO:tasks.rados.rados.0.smithi029.stdout:3276: done (2 left) 2024-04-12T11:28:11.765 INFO:tasks.rados.rados.0.smithi029.stdout:3278: done (1 left) 2024-04-12T11:28:11.766 INFO:tasks.rados.rados.0.smithi029.stdout:3279: delete oid 31 current snap is 413 2024-04-12T11:28:11.766 INFO:tasks.rados.rados.0.smithi029.stdout:3277: finishing write tid 2 to smithi02979277-47 2024-04-12T11:28:11.766 INFO:tasks.rados.rados.0.smithi029.stdout:3277: finishing write tid 3 to smithi02979277-47 2024-04-12T11:28:11.767 INFO:tasks.rados.rados.0.smithi029.stdout:3277: finishing write tid 4 to smithi02979277-47 2024-04-12T11:28:11.767 INFO:tasks.rados.rados.0.smithi029.stdout:3277: oid 47 updating version 0 to 1555 2024-04-12T11:28:11.767 INFO:tasks.rados.rados.0.smithi029.stdout:3277: oid 47 updating version 1555 to 1556 2024-04-12T11:28:11.767 INFO:tasks.rados.rados.0.smithi029.stdout:3277: oid 47 updating version 1556 to 1557 2024-04-12T11:28:11.767 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1557 (ObjNum 1227 snap 413 seq_num 1227) dirty exists 2024-04-12T11:28:11.767 INFO:tasks.rados.rados.0.smithi029.stdout:3277: left oid 47 (ObjNum 1227 snap 413 seq_num 1227) 2024-04-12T11:28:11.768 INFO:tasks.rados.rados.0.smithi029.stdout:3277: done (1 left) 2024-04-12T11:28:11.768 INFO:tasks.rados.rados.0.smithi029.stdout:3279: done (0 left) 2024-04-12T11:28:11.768 INFO:tasks.rados.rados.0.smithi029.stdout:3280: snap_create 2024-04-12T11:28:12.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:11 smithi029 ceph-mon[93650]: osdmap e886: 8 total, 8 up, 8 in 2024-04-12T11:28:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:11 smithi029 ceph-mon[97787]: osdmap e886: 8 total, 8 up, 8 in 2024-04-12T11:28:12.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:11 smithi087 ceph-mon[89014]: osdmap e886: 8 total, 8 up, 8 in 2024-04-12T11:28:12.670 INFO:tasks.rados.rados.0.smithi029.stdout:3280: done (0 left) 2024-04-12T11:28:12.670 INFO:tasks.rados.rados.0.smithi029.stdout:3281: snap_create 2024-04-12T11:28:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:28:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:28:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:28:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:12 smithi029 ceph-mon[93650]: pgmap v966: 89 pgs: 89 active+clean; 341 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-04-12T11:28:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:12 smithi029 ceph-mon[93650]: osdmap e887: 8 total, 8 up, 8 in 2024-04-12T11:28:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:12 smithi029 ceph-mon[93650]: osdmap e888: 8 total, 8 up, 8 in 2024-04-12T11:28:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:12 smithi029 ceph-mon[97787]: pgmap v966: 89 pgs: 89 active+clean; 341 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-04-12T11:28:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:12 smithi029 ceph-mon[97787]: osdmap e887: 8 total, 8 up, 8 in 2024-04-12T11:28:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:12 smithi029 ceph-mon[97787]: osdmap e888: 8 total, 8 up, 8 in 2024-04-12T11:28:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:12 smithi087 ceph-mon[89014]: pgmap v966: 89 pgs: 89 active+clean; 341 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-04-12T11:28:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:12 smithi087 ceph-mon[89014]: osdmap e887: 8 total, 8 up, 8 in 2024-04-12T11:28:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:12 smithi087 ceph-mon[89014]: osdmap e888: 8 total, 8 up, 8 in 2024-04-12T11:28:13.672 INFO:tasks.rados.rados.0.smithi029.stdout:3281: done (0 left) 2024-04-12T11:28:13.672 INFO:tasks.rados.rados.0.smithi029.stdout:3282: read oid 26 snap -1 2024-04-12T11:28:13.672 INFO:tasks.rados.rados.0.smithi029.stdout:3282: expect (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:28:13.672 INFO:tasks.rados.rados.0.smithi029.stdout:3283: snap_remove snap 413 2024-04-12T11:28:14.673 INFO:tasks.rados.rados.0.smithi029.stdout:3283: done (1 left) 2024-04-12T11:28:14.674 INFO:tasks.rados.rados.0.smithi029.stdout:3284: write oid 35 current snap is 415 2024-04-12T11:28:14.674 INFO:tasks.rados.rados.0.smithi029.stdout:3284: seq_num 1230 ranges {672610=489103,1643440=722432} 2024-04-12T11:28:14.680 INFO:tasks.rados.rados.0.smithi029.stdout:3284: writing smithi02979277-35 from 672610 to 1161713 tid 1 2024-04-12T11:28:14.687 INFO:tasks.rados.rados.0.smithi029.stdout:3284: writing smithi02979277-35 from 1643440 to 2365872 tid 2 2024-04-12T11:28:14.687 INFO:tasks.rados.rados.0.smithi029.stdout:3285: rollback oid 46 current snap is 415 2024-04-12T11:28:14.687 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 46 to 388 2024-04-12T11:28:14.687 INFO:tasks.rados.rados.0.smithi029.stdout:3282: expect (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:28:14.905 INFO:tasks.rados.rados.0.smithi029.stdout:3284: finishing write tid 1 to smithi02979277-35 2024-04-12T11:28:14.905 INFO:tasks.rados.rados.0.smithi029.stdout:3285: finishing rollback tid 0 to smithi02979277-46 2024-04-12T11:28:14.905 INFO:tasks.rados.rados.0.smithi029.stdout:3285: finishing rollback tid 1 to smithi02979277-46 2024-04-12T11:28:14.905 INFO:tasks.rados.rados.0.smithi029.stdout:3282: done (2 left) 2024-04-12T11:28:14.905 INFO:tasks.rados.rados.0.smithi029.stdout:3286: rollback oid 22 current snap is 415 2024-04-12T11:28:14.905 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 22 to 396 2024-04-12T11:28:14.905 INFO:tasks.rados.rados.0.smithi029.stdout:3285: finishing rollback tid 2 to smithi02979277-46 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 1025 (ObjNum 1131 snap 379 seq_num 1131) dirty exists 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:3284: finishing write tid 2 to smithi02979277-35 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:3284: finishing write tid 3 to smithi02979277-35 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:3284: finishing write tid 4 to smithi02979277-35 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:3284: oid 35 updating version 0 to 2056 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:3284: oid 35 updating version 2056 to 2058 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:3284: oid 35 version 2058 is already newer than 2057 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 2058 (ObjNum 1230 snap 415 seq_num 1230) dirty exists 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:3284: left oid 35 (ObjNum 1230 snap 415 seq_num 1230) 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:3284: done (2 left) 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:3285: done (1 left) 2024-04-12T11:28:14.906 INFO:tasks.rados.rados.0.smithi029.stdout:3287: delete oid 8 current snap is 415 2024-04-12T11:28:14.907 INFO:tasks.rados.rados.0.smithi029.stdout:3287: done (1 left) 2024-04-12T11:28:14.907 INFO:tasks.rados.rados.0.smithi029.stdout:3288: snap_remove snap 380 2024-04-12T11:28:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:14 smithi029 ceph-mon[93650]: pgmap v969: 89 pgs: 89 active+clean; 347 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.1 MiB/s wr, 14 op/s 2024-04-12T11:28:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:14 smithi029 ceph-mon[93650]: osdmap e889: 8 total, 8 up, 8 in 2024-04-12T11:28:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:14 smithi029 ceph-mon[97787]: pgmap v969: 89 pgs: 89 active+clean; 347 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.1 MiB/s wr, 14 op/s 2024-04-12T11:28:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:14 smithi029 ceph-mon[97787]: osdmap e889: 8 total, 8 up, 8 in 2024-04-12T11:28:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:14 smithi087 ceph-mon[89014]: pgmap v969: 89 pgs: 89 active+clean; 347 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.1 MiB/s wr, 14 op/s 2024-04-12T11:28:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:14 smithi087 ceph-mon[89014]: osdmap e889: 8 total, 8 up, 8 in 2024-04-12T11:28:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:28:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:28:14] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.43.0" 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:3286: finishing rollback tid 1 to smithi02979277-22 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:3286: finishing rollback tid 2 to smithi02979277-22 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 1348 (ObjNum 1114 snap 371 seq_num 1114) dirty exists 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:3286: done (1 left) 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:3288: done (0 left) 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:3289: rollback oid 24 current snap is 415 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 24 to 401 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:3290: rollback oid 3 current snap is 415 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 398 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:3291: rollback oid 20 current snap is 415 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 20 to 341 2024-04-12T11:28:15.686 INFO:tasks.rados.rados.0.smithi029.stdout:3292: rollback oid 43 current snap is 415 2024-04-12T11:28:15.687 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 396 2024-04-12T11:28:15.687 INFO:tasks.rados.rados.0.smithi029.stdout:3293: snap_remove snap 412 2024-04-12T11:28:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:15 smithi087 ceph-mon[89014]: osdmap e890: 8 total, 8 up, 8 in 2024-04-12T11:28:16.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:15 smithi029 ceph-mon[93650]: osdmap e890: 8 total, 8 up, 8 in 2024-04-12T11:28:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:15 smithi029 ceph-mon[97787]: osdmap e890: 8 total, 8 up, 8 in 2024-04-12T11:28:16.686 INFO:tasks.rados.rados.0.smithi029.stdout:3293: done (4 left) 2024-04-12T11:28:16.686 INFO:tasks.rados.rados.0.smithi029.stdout:3294: delete oid 21 current snap is 415 2024-04-12T11:28:16.686 INFO:tasks.rados.rados.0.smithi029.stdout:3292: finishing rollback tid 0 to smithi02979277-43 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:3289: finishing rollback tid 0 to smithi02979277-24 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:3291: finishing rollback tid 0 to smithi02979277-20 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:3291: finishing rollback tid 1 to smithi02979277-20 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 1028 (ObjNum 129 snap 0 seq_num 1801293920) dirty dne 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:3292: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:3292: finishing rollback tid 2 to smithi02979277-43 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1460 (ObjNum 1089 snap 365 seq_num 1089) dirty exists 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:3289: finishing rollback tid 1 to smithi02979277-24 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:3289: finishing rollback tid 2 to smithi02979277-24 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 2062 (ObjNum 1168 snap 397 seq_num 1168) dirty exists 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:3290: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:28:16.687 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 1914 (ObjNum 129 snap 0 seq_num 492) dirty dne 2024-04-12T11:28:16.697 INFO:tasks.rados.rados.0.smithi029.stdout:3289: done (4 left) 2024-04-12T11:28:16.698 INFO:tasks.rados.rados.0.smithi029.stdout:3290: done (3 left) 2024-04-12T11:28:16.698 INFO:tasks.rados.rados.0.smithi029.stdout:3291: done (2 left) 2024-04-12T11:28:16.698 INFO:tasks.rados.rados.0.smithi029.stdout:3292: done (1 left) 2024-04-12T11:28:16.698 INFO:tasks.rados.rados.0.smithi029.stdout:3294: done (0 left) 2024-04-12T11:28:16.698 INFO:tasks.rados.rados.0.smithi029.stdout:3295: snap_remove snap 396 2024-04-12T11:28:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:16 smithi087 ceph-mon[89014]: pgmap v972: 89 pgs: 89 active+clean; 354 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 1.1 MiB/s wr, 23 op/s 2024-04-12T11:28:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:16 smithi087 ceph-mon[89014]: osdmap e891: 8 total, 8 up, 8 in 2024-04-12T11:28:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:16 smithi029 ceph-mon[93650]: pgmap v972: 89 pgs: 89 active+clean; 354 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 1.1 MiB/s wr, 23 op/s 2024-04-12T11:28:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:16 smithi029 ceph-mon[93650]: osdmap e891: 8 total, 8 up, 8 in 2024-04-12T11:28:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:16 smithi029 ceph-mon[97787]: pgmap v972: 89 pgs: 89 active+clean; 354 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 1.1 MiB/s wr, 23 op/s 2024-04-12T11:28:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:16 smithi029 ceph-mon[97787]: osdmap e891: 8 total, 8 up, 8 in 2024-04-12T11:28:17.672 INFO:tasks.rados.rados.0.smithi029.stdout:3295: done (0 left) 2024-04-12T11:28:17.672 INFO:tasks.rados.rados.0.smithi029.stdout:3296: rollback oid 4 current snap is 415 2024-04-12T11:28:17.672 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 4 to 404 2024-04-12T11:28:17.672 INFO:tasks.rados.rados.0.smithi029.stdout:3297: rollback oid 39 current snap is 415 2024-04-12T11:28:17.672 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 404 2024-04-12T11:28:17.672 INFO:tasks.rados.rados.0.smithi029.stdout:3298: write oid 29 current snap is 415 2024-04-12T11:28:17.672 INFO:tasks.rados.rados.0.smithi029.stdout:3298: seq_num 1233 ranges {723031=431203,1818886=469759} 2024-04-12T11:28:17.679 INFO:tasks.rados.rados.0.smithi029.stdout:3298: writing smithi02979277-29 from 723031 to 1154234 tid 1 2024-04-12T11:28:17.686 INFO:tasks.rados.rados.0.smithi029.stdout:3298: writing smithi02979277-29 from 1818886 to 2288645 tid 2 2024-04-12T11:28:17.686 INFO:tasks.rados.rados.0.smithi029.stdout:3299: delete oid 45 current snap is 415 2024-04-12T11:28:17.687 INFO:tasks.rados.rados.0.smithi029.stdout:3296: finishing rollback tid 1 to smithi02979277-4 2024-04-12T11:28:17.687 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1386 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:28:17.687 INFO:tasks.rados.rados.0.smithi029.stdout:3297: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:28:17.687 INFO:tasks.rados.rados.0.smithi029.stdout:3297: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:28:17.688 INFO:tasks.rados.rados.0.smithi029.stdout:3297: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:28:17.688 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 455 (ObjNum 1088 snap 365 seq_num 1088) dirty exists 2024-04-12T11:28:17.694 INFO:tasks.rados.rados.0.smithi029.stdout:3298: finishing write tid 1 to smithi02979277-29 2024-04-12T11:28:17.695 INFO:tasks.rados.rados.0.smithi029.stdout:3296: done (3 left) 2024-04-12T11:28:17.695 INFO:tasks.rados.rados.0.smithi029.stdout:3297: done (2 left) 2024-04-12T11:28:17.695 INFO:tasks.rados.rados.0.smithi029.stdout:3299: done (1 left) 2024-04-12T11:28:17.696 INFO:tasks.rados.rados.0.smithi029.stdout:3300: write oid 27 current snap is 415 2024-04-12T11:28:17.696 INFO:tasks.rados.rados.0.smithi029.stdout:3300: seq_num 1235 ranges {756645=581701,1935850=473428} 2024-04-12T11:28:17.703 INFO:tasks.rados.rados.0.smithi029.stdout:3300: writing smithi02979277-27 from 756645 to 1338346 tid 1 2024-04-12T11:28:17.709 INFO:tasks.rados.rados.0.smithi029.stdout:3300: writing smithi02979277-27 from 1935850 to 2409278 tid 2 2024-04-12T11:28:17.709 INFO:tasks.rados.rados.0.smithi029.stdout:3301: delete oid 7 current snap is 415 2024-04-12T11:28:17.709 INFO:tasks.rados.rados.0.smithi029.stdout:3298: finishing write tid 2 to smithi02979277-29 2024-04-12T11:28:17.709 INFO:tasks.rados.rados.0.smithi029.stdout:3298: finishing write tid 3 to smithi02979277-29 2024-04-12T11:28:17.709 INFO:tasks.rados.rados.0.smithi029.stdout:3298: finishing write tid 4 to smithi02979277-29 2024-04-12T11:28:17.709 INFO:tasks.rados.rados.0.smithi029.stdout:3298: oid 29 updating version 0 to 1469 2024-04-12T11:28:17.710 INFO:tasks.rados.rados.0.smithi029.stdout:3298: oid 29 updating version 1469 to 1470 2024-04-12T11:28:17.710 INFO:tasks.rados.rados.0.smithi029.stdout:3298: oid 29 version 1470 is already newer than 1468 2024-04-12T11:28:17.710 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1470 (ObjNum 1233 snap 415 seq_num 1233) dirty exists 2024-04-12T11:28:17.710 INFO:tasks.rados.rados.0.smithi029.stdout:3298: left oid 29 (ObjNum 1233 snap 415 seq_num 1233) 2024-04-12T11:28:17.710 INFO:tasks.rados.rados.0.smithi029.stdout:3300: finishing write tid 1 to smithi02979277-27 2024-04-12T11:28:17.711 INFO:tasks.rados.rados.0.smithi029.stdout:3298: done (2 left) 2024-04-12T11:28:17.711 INFO:tasks.rados.rados.0.smithi029.stdout:3301: done (1 left) 2024-04-12T11:28:17.711 INFO:tasks.rados.rados.0.smithi029.stdout:3302: snap_remove snap 401 2024-04-12T11:28:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:17 smithi087 ceph-mon[89014]: osdmap e892: 8 total, 8 up, 8 in 2024-04-12T11:28:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:17 smithi087 ceph-mon[89014]: osdmap e893: 8 total, 8 up, 8 in 2024-04-12T11:28:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:17 smithi029 ceph-mon[93650]: osdmap e892: 8 total, 8 up, 8 in 2024-04-12T11:28:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:17 smithi029 ceph-mon[93650]: osdmap e893: 8 total, 8 up, 8 in 2024-04-12T11:28:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:17 smithi029 ceph-mon[97787]: osdmap e892: 8 total, 8 up, 8 in 2024-04-12T11:28:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:17 smithi029 ceph-mon[97787]: osdmap e893: 8 total, 8 up, 8 in 2024-04-12T11:28:18.708 INFO:tasks.rados.rados.0.smithi029.stdout:3302: done (1 left) 2024-04-12T11:28:18.708 INFO:tasks.rados.rados.0.smithi029.stdout:3303: snap_remove snap 341 2024-04-12T11:28:18.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:18 smithi087 ceph-mon[89014]: pgmap v975: 89 pgs: 89 active+clean; 352 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:28:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:18 smithi029 ceph-mon[93650]: pgmap v975: 89 pgs: 89 active+clean; 352 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:28:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:18 smithi029 ceph-mon[97787]: pgmap v975: 89 pgs: 89 active+clean; 352 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 0 B/s wr, 9 op/s 2024-04-12T11:28:19.715 INFO:tasks.rados.rados.0.smithi029.stdout:3303: done (1 left) 2024-04-12T11:28:19.716 INFO:tasks.rados.rados.0.smithi029.stdout:3304: snap_create 2024-04-12T11:28:19.716 INFO:tasks.rados.rados.0.smithi029.stdout:3300: finishing write tid 2 to smithi02979277-27 2024-04-12T11:28:19.716 INFO:tasks.rados.rados.0.smithi029.stdout:3300: finishing write tid 3 to smithi02979277-27 2024-04-12T11:28:19.716 INFO:tasks.rados.rados.0.smithi029.stdout:3300: finishing write tid 4 to smithi02979277-27 2024-04-12T11:28:19.716 INFO:tasks.rados.rados.0.smithi029.stdout:3300: oid 27 updating version 0 to 1013 2024-04-12T11:28:19.716 INFO:tasks.rados.rados.0.smithi029.stdout:3300: oid 27 updating version 1013 to 1014 2024-04-12T11:28:19.716 INFO:tasks.rados.rados.0.smithi029.stdout:3300: oid 27 version 1014 is already newer than 1012 2024-04-12T11:28:19.716 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 1014 (ObjNum 1235 snap 415 seq_num 1235) dirty exists 2024-04-12T11:28:19.716 INFO:tasks.rados.rados.0.smithi029.stdout:3300: left oid 27 (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:28:19.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:19 smithi087 ceph-mon[89014]: osdmap e894: 8 total, 8 up, 8 in 2024-04-12T11:28:20.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:19 smithi029 ceph-mon[93650]: osdmap e894: 8 total, 8 up, 8 in 2024-04-12T11:28:20.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:19 smithi029 ceph-mon[97787]: osdmap e894: 8 total, 8 up, 8 in 2024-04-12T11:28:20.717 INFO:tasks.rados.rados.0.smithi029.stdout:3300: done (1 left) 2024-04-12T11:28:20.718 INFO:tasks.rados.rados.0.smithi029.stdout:3304: done (0 left) 2024-04-12T11:28:20.718 INFO:tasks.rados.rados.0.smithi029.stdout:3305: snap_create 2024-04-12T11:28:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:20 smithi087 ceph-mon[89014]: pgmap v978: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 355 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 516 KiB/s wr, 7 op/s 2024-04-12T11:28:20.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:20 smithi087 ceph-mon[89014]: osdmap e895: 8 total, 8 up, 8 in 2024-04-12T11:28:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:20 smithi029 ceph-mon[93650]: pgmap v978: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 355 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 516 KiB/s wr, 7 op/s 2024-04-12T11:28:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:20 smithi029 ceph-mon[93650]: osdmap e895: 8 total, 8 up, 8 in 2024-04-12T11:28:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:20 smithi029 ceph-mon[97787]: pgmap v978: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 355 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 516 KiB/s wr, 7 op/s 2024-04-12T11:28:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:20 smithi029 ceph-mon[97787]: osdmap e895: 8 total, 8 up, 8 in 2024-04-12T11:28:21.725 INFO:tasks.rados.rados.0.smithi029.stdout:3305: done (0 left) 2024-04-12T11:28:21.725 INFO:tasks.rados.rados.0.smithi029.stdout:3306: rollback oid 7 current snap is 417 2024-04-12T11:28:21.725 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 410 2024-04-12T11:28:21.725 INFO:tasks.rados.rados.0.smithi029.stdout:3307: read oid 21 snap -1 2024-04-12T11:28:21.725 INFO:tasks.rados.rados.0.smithi029.stdout:3307: expect deleted 2024-04-12T11:28:21.725 INFO:tasks.rados.rados.0.smithi029.stdout:3308: read oid 27 snap -1 2024-04-12T11:28:21.725 INFO:tasks.rados.rados.0.smithi029.stdout:3308: expect (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:28:21.725 INFO:tasks.rados.rados.0.smithi029.stdout:3309: write oid 15 current snap is 417 2024-04-12T11:28:21.725 INFO:tasks.rados.rados.0.smithi029.stdout:3309: seq_num 1237 ranges {790259=732199,1936461=477097} 2024-04-12T11:28:21.733 INFO:tasks.rados.rados.0.smithi029.stdout:3309: writing smithi02979277-15 from 790259 to 1522458 tid 1 2024-04-12T11:28:21.738 INFO:tasks.rados.rados.0.smithi029.stdout:3309: writing smithi02979277-15 from 1936461 to 2413558 tid 2 2024-04-12T11:28:21.738 INFO:tasks.rados.rados.0.smithi029.stdout:3310: snap_create 2024-04-12T11:28:21.738 INFO:tasks.rados.rados.0.smithi029.stdout:3308: expect (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:28:21.909 INFO:tasks.rados.rados.0.smithi029.stdout:3306: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:28:21.909 INFO:tasks.rados.rados.0.smithi029.stdout:3306: finishing rollback tid 2 to smithi02979277-7 2024-04-12T11:28:21.909 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1587 (ObjNum 959 snap 329 seq_num 959) dirty exists 2024-04-12T11:28:21.909 INFO:tasks.rados.rados.0.smithi029.stdout:3309: finishing write tid 1 to smithi02979277-15 2024-04-12T11:28:21.909 INFO:tasks.rados.rados.0.smithi029.stdout:3309: finishing write tid 2 to smithi02979277-15 2024-04-12T11:28:21.909 INFO:tasks.rados.rados.0.smithi029.stdout:3309: finishing write tid 3 to smithi02979277-15 2024-04-12T11:28:21.909 INFO:tasks.rados.rados.0.smithi029.stdout:3309: finishing write tid 4 to smithi02979277-15 2024-04-12T11:28:21.910 INFO:tasks.rados.rados.0.smithi029.stdout:3309: oid 15 updating version 0 to 1025 2024-04-12T11:28:21.910 INFO:tasks.rados.rados.0.smithi029.stdout:3309: oid 15 version 1025 is already newer than 1024 2024-04-12T11:28:21.910 INFO:tasks.rados.rados.0.smithi029.stdout:3309: oid 15 version 1025 is already newer than 1023 2024-04-12T11:28:21.910 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 1025 (ObjNum 1237 snap 417 seq_num 1237) dirty exists 2024-04-12T11:28:21.910 INFO:tasks.rados.rados.0.smithi029.stdout:3309: left oid 15 (ObjNum 1237 snap 417 seq_num 1237) 2024-04-12T11:28:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:21 smithi029 ceph-mon[97787]: osdmap e896: 8 total, 8 up, 8 in 2024-04-12T11:28:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:21 smithi029 ceph-mon[93650]: osdmap e896: 8 total, 8 up, 8 in 2024-04-12T11:28:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:22.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:21 smithi087 ceph-mon[89014]: osdmap e896: 8 total, 8 up, 8 in 2024-04-12T11:28:22.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:22.673 INFO:tasks.rados.rados.0.smithi029.stdout:3306: done (4 left) 2024-04-12T11:28:22.673 INFO:tasks.rados.rados.0.smithi029.stdout:3307: done (3 left) 2024-04-12T11:28:22.673 INFO:tasks.rados.rados.0.smithi029.stdout:3308: done (2 left) 2024-04-12T11:28:22.673 INFO:tasks.rados.rados.0.smithi029.stdout:3309: done (1 left) 2024-04-12T11:28:22.673 INFO:tasks.rados.rados.0.smithi029.stdout:3310: done (0 left) 2024-04-12T11:28:22.673 INFO:tasks.rados.rados.0.smithi029.stdout:3311: write oid 20 current snap is 418 2024-04-12T11:28:22.674 INFO:tasks.rados.rados.0.smithi029.stdout:3311: seq_num 1238 ranges {407066=407448,1594943=620755} 2024-04-12T11:28:22.677 INFO:tasks.rados.rados.0.smithi029.stdout:3311: writing smithi02979277-20 from 407066 to 814514 tid 1 2024-04-12T11:28:22.685 INFO:tasks.rados.rados.0.smithi029.stdout:3311: writing smithi02979277-20 from 1594943 to 2215698 tid 2 2024-04-12T11:28:22.685 INFO:tasks.rados.rados.0.smithi029.stdout:3312: read oid 32 snap -1 2024-04-12T11:28:22.686 INFO:tasks.rados.rados.0.smithi029.stdout:3312: expect (ObjNum 1218 snap 409 seq_num 1218) 2024-04-12T11:28:22.686 INFO:tasks.rados.rados.0.smithi029.stdout:3313: snap_create 2024-04-12T11:28:22.686 INFO:tasks.rados.rados.0.smithi029.stdout:3311: finishing write tid 1 to smithi02979277-20 2024-04-12T11:28:22.690 INFO:tasks.rados.rados.0.smithi029.stdout:3312: expect (ObjNum 1218 snap 409 seq_num 1218) 2024-04-12T11:28:22.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:28:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:28:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:28:22.862 INFO:tasks.rados.rados.0.smithi029.stdout:3311: finishing write tid 2 to smithi02979277-20 2024-04-12T11:28:22.862 INFO:tasks.rados.rados.0.smithi029.stdout:3311: finishing write tid 3 to smithi02979277-20 2024-04-12T11:28:22.862 INFO:tasks.rados.rados.0.smithi029.stdout:3311: finishing write tid 4 to smithi02979277-20 2024-04-12T11:28:22.863 INFO:tasks.rados.rados.0.smithi029.stdout:3311: oid 20 updating version 0 to 1054 2024-04-12T11:28:22.863 INFO:tasks.rados.rados.0.smithi029.stdout:3311: oid 20 updating version 1054 to 1055 2024-04-12T11:28:22.863 INFO:tasks.rados.rados.0.smithi029.stdout:3311: oid 20 version 1055 is already newer than 1053 2024-04-12T11:28:22.863 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 1055 (ObjNum 1238 snap 418 seq_num 1238) dirty exists 2024-04-12T11:28:22.863 INFO:tasks.rados.rados.0.smithi029.stdout:3311: left oid 20 (ObjNum 1238 snap 418 seq_num 1238) 2024-04-12T11:28:22.988 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:22 smithi087 ceph-mon[89014]: pgmap v981: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 355 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 516 KiB/s wr, 6 op/s 2024-04-12T11:28:22.988 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:22 smithi087 ceph-mon[89014]: osdmap e897: 8 total, 8 up, 8 in 2024-04-12T11:28:22.988 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:22 smithi087 ceph-mon[89014]: osdmap e898: 8 total, 8 up, 8 in 2024-04-12T11:28:23.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:22 smithi029 ceph-mon[93650]: pgmap v981: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 355 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 516 KiB/s wr, 6 op/s 2024-04-12T11:28:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:22 smithi029 ceph-mon[93650]: osdmap e897: 8 total, 8 up, 8 in 2024-04-12T11:28:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:22 smithi029 ceph-mon[93650]: osdmap e898: 8 total, 8 up, 8 in 2024-04-12T11:28:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:22 smithi029 ceph-mon[97787]: pgmap v981: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 355 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 516 KiB/s wr, 6 op/s 2024-04-12T11:28:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:22 smithi029 ceph-mon[97787]: osdmap e897: 8 total, 8 up, 8 in 2024-04-12T11:28:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:22 smithi029 ceph-mon[97787]: osdmap e898: 8 total, 8 up, 8 in 2024-04-12T11:28:23.674 INFO:tasks.rados.rados.0.smithi029.stdout:3311: done (2 left) 2024-04-12T11:28:23.674 INFO:tasks.rados.rados.0.smithi029.stdout:3312: done (1 left) 2024-04-12T11:28:23.675 INFO:tasks.rados.rados.0.smithi029.stdout:3313: done (0 left) 2024-04-12T11:28:23.675 INFO:tasks.rados.rados.0.smithi029.stdout:3314: rollback oid 49 current snap is 419 2024-04-12T11:28:23.675 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 49 to 418 2024-04-12T11:28:23.675 INFO:tasks.rados.rados.0.smithi029.stdout:3315: delete oid 35 current snap is 419 2024-04-12T11:28:23.680 INFO:tasks.rados.rados.0.smithi029.stdout:3315: done (1 left) 2024-04-12T11:28:23.680 INFO:tasks.rados.rados.0.smithi029.stdout:3316: rollback oid 15 current snap is 419 2024-04-12T11:28:23.680 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 15 to 410 2024-04-12T11:28:23.680 INFO:tasks.rados.rados.0.smithi029.stdout:3317: snap_create 2024-04-12T11:28:23.681 INFO:tasks.rados.rados.0.smithi029.stdout:3314: finishing rollback tid 0 to smithi02979277-49 2024-04-12T11:28:23.681 INFO:tasks.rados.rados.0.smithi029.stdout:3314: finishing rollback tid 1 to smithi02979277-49 2024-04-12T11:28:23.681 INFO:tasks.rados.rados.0.smithi029.stdout:3314: finishing rollback tid 2 to smithi02979277-49 2024-04-12T11:28:23.681 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 1201 (ObjNum 1097 snap 368 seq_num 1097) dirty exists 2024-04-12T11:28:23.683 INFO:tasks.rados.rados.0.smithi029.stdout:3316: finishing rollback tid 0 to smithi02979277-15 2024-04-12T11:28:23.683 INFO:tasks.rados.rados.0.smithi029.stdout:3316: finishing rollback tid 1 to smithi02979277-15 2024-04-12T11:28:23.683 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 1028 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:28:24.679 INFO:tasks.rados.rados.0.smithi029.stdout:3314: done (2 left) 2024-04-12T11:28:24.679 INFO:tasks.rados.rados.0.smithi029.stdout:3316: done (1 left) 2024-04-12T11:28:24.679 INFO:tasks.rados.rados.0.smithi029.stdout:3317: done (0 left) 2024-04-12T11:28:24.679 INFO:tasks.rados.rados.0.smithi029.stdout:3318: snap_create 2024-04-12T11:28:24.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:24 smithi029 ceph-mon[97787]: pgmap v984: 89 pgs: 89 active+clean; 330 MiB data, 983 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 553 KiB/s wr, 5 op/s 2024-04-12T11:28:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:24 smithi029 ceph-mon[97787]: osdmap e899: 8 total, 8 up, 8 in 2024-04-12T11:28:24.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:24 smithi029 ceph-mon[93650]: pgmap v984: 89 pgs: 89 active+clean; 330 MiB data, 983 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 553 KiB/s wr, 5 op/s 2024-04-12T11:28:24.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:24 smithi029 ceph-mon[93650]: osdmap e899: 8 total, 8 up, 8 in 2024-04-12T11:28:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:24 smithi087 ceph-mon[89014]: pgmap v984: 89 pgs: 89 active+clean; 330 MiB data, 983 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 553 KiB/s wr, 5 op/s 2024-04-12T11:28:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:24 smithi087 ceph-mon[89014]: osdmap e899: 8 total, 8 up, 8 in 2024-04-12T11:28:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:28:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:28:24] "GET /metrics HTTP/1.1" 200 36369 "" "Prometheus/2.43.0" 2024-04-12T11:28:25.684 INFO:tasks.rados.rados.0.smithi029.stdout:3318: done (0 left) 2024-04-12T11:28:25.684 INFO:tasks.rados.rados.0.smithi029.stdout:3319: snap_remove snap 415 2024-04-12T11:28:25.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:25 smithi087 ceph-mon[89014]: osdmap e900: 8 total, 8 up, 8 in 2024-04-12T11:28:26.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:25 smithi029 ceph-mon[97787]: osdmap e900: 8 total, 8 up, 8 in 2024-04-12T11:28:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:25 smithi029 ceph-mon[93650]: osdmap e900: 8 total, 8 up, 8 in 2024-04-12T11:28:26.685 INFO:tasks.rados.rados.0.smithi029.stdout:3319: done (0 left) 2024-04-12T11:28:26.685 INFO:tasks.rados.rados.0.smithi029.stdout:3320: snap_remove snap 406 2024-04-12T11:28:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:26 smithi087 ceph-mon[89014]: pgmap v987: 89 pgs: 89 active+clean; 312 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 805 KiB/s wr, 8 op/s 2024-04-12T11:28:26.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:26 smithi087 ceph-mon[89014]: osdmap e901: 8 total, 8 up, 8 in 2024-04-12T11:28:27.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:26 smithi029 ceph-mon[93650]: pgmap v987: 89 pgs: 89 active+clean; 312 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 805 KiB/s wr, 8 op/s 2024-04-12T11:28:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:26 smithi029 ceph-mon[93650]: osdmap e901: 8 total, 8 up, 8 in 2024-04-12T11:28:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:26 smithi029 ceph-mon[97787]: pgmap v987: 89 pgs: 89 active+clean; 312 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 805 KiB/s wr, 8 op/s 2024-04-12T11:28:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:26 smithi029 ceph-mon[97787]: osdmap e901: 8 total, 8 up, 8 in 2024-04-12T11:28:27.674 INFO:tasks.rados.rados.0.smithi029.stdout:3320: done (0 left) 2024-04-12T11:28:27.674 INFO:tasks.rados.rados.0.smithi029.stdout:3321: rollback oid 50 current snap is 421 2024-04-12T11:28:27.674 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 50 to 420 2024-04-12T11:28:27.674 INFO:tasks.rados.rados.0.smithi029.stdout:3322: rollback oid 12 current snap is 421 2024-04-12T11:28:27.675 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 357 2024-04-12T11:28:27.675 INFO:tasks.rados.rados.0.smithi029.stdout:3323: delete oid 33 current snap is 421 2024-04-12T11:28:27.687 INFO:tasks.rados.rados.0.smithi029.stdout:3321: finishing rollback tid 0 to smithi02979277-50 2024-04-12T11:28:27.687 INFO:tasks.rados.rados.0.smithi029.stdout:3321: finishing rollback tid 1 to smithi02979277-50 2024-04-12T11:28:27.687 INFO:tasks.rados.rados.0.smithi029.stdout:3321: finishing rollback tid 2 to smithi02979277-50 2024-04-12T11:28:27.687 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1372 (ObjNum 1179 snap 400 seq_num 1179) dirty exists 2024-04-12T11:28:27.688 INFO:tasks.rados.rados.0.smithi029.stdout:3321: done (2 left) 2024-04-12T11:28:27.689 INFO:tasks.rados.rados.0.smithi029.stdout:3323: done (1 left) 2024-04-12T11:28:27.689 INFO:tasks.rados.rados.0.smithi029.stdout:3324: snap_create 2024-04-12T11:28:27.691 INFO:tasks.rados.rados.0.smithi029.stdout:3322: finishing rollback tid 0 to smithi02979277-12 2024-04-12T11:28:27.691 INFO:tasks.rados.rados.0.smithi029.stdout:3322: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:28:27.691 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1590 (ObjNum 305 snap 0 seq_num 862606956) dirty dne 2024-04-12T11:28:27.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:27 smithi087 ceph-mon[89014]: osdmap e902: 8 total, 8 up, 8 in 2024-04-12T11:28:27.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:27 smithi087 ceph-mon[89014]: osdmap e903: 8 total, 8 up, 8 in 2024-04-12T11:28:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:27 smithi029 ceph-mon[93650]: osdmap e902: 8 total, 8 up, 8 in 2024-04-12T11:28:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:27 smithi029 ceph-mon[93650]: osdmap e903: 8 total, 8 up, 8 in 2024-04-12T11:28:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:27 smithi029 ceph-mon[97787]: osdmap e902: 8 total, 8 up, 8 in 2024-04-12T11:28:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:27 smithi029 ceph-mon[97787]: osdmap e903: 8 total, 8 up, 8 in 2024-04-12T11:28:28.676 INFO:tasks.rados.rados.0.smithi029.stdout:3322: done (1 left) 2024-04-12T11:28:28.677 INFO:tasks.rados.rados.0.smithi029.stdout:3324: done (0 left) 2024-04-12T11:28:28.677 INFO:tasks.rados.rados.0.smithi029.stdout:3325: snap_create 2024-04-12T11:28:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:28 smithi087 ceph-mon[89014]: pgmap v990: 89 pgs: 89 active+clean; 312 MiB data, 959 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 251 KiB/s wr, 3 op/s 2024-04-12T11:28:28.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:28 smithi087 ceph-mon[89014]: osdmap e904: 8 total, 8 up, 8 in 2024-04-12T11:28:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:28 smithi029 ceph-mon[93650]: pgmap v990: 89 pgs: 89 active+clean; 312 MiB data, 959 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 251 KiB/s wr, 3 op/s 2024-04-12T11:28:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:28 smithi029 ceph-mon[93650]: osdmap e904: 8 total, 8 up, 8 in 2024-04-12T11:28:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:28 smithi029 ceph-mon[97787]: pgmap v990: 89 pgs: 89 active+clean; 312 MiB data, 959 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 251 KiB/s wr, 3 op/s 2024-04-12T11:28:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:28 smithi029 ceph-mon[97787]: osdmap e904: 8 total, 8 up, 8 in 2024-04-12T11:28:29.677 INFO:tasks.rados.rados.0.smithi029.stdout:3325: done (0 left) 2024-04-12T11:28:29.677 INFO:tasks.rados.rados.0.smithi029.stdout:3326: snap_remove snap 422 2024-04-12T11:28:30.680 INFO:tasks.rados.rados.0.smithi029.stdout:3326: done (0 left) 2024-04-12T11:28:30.681 INFO:tasks.rados.rados.0.smithi029.stdout:3327: snap_remove snap 400 2024-04-12T11:28:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:30 smithi087 ceph-mon[89014]: pgmap v993: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 312 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-04-12T11:28:30.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:30 smithi087 ceph-mon[89014]: osdmap e905: 8 total, 8 up, 8 in 2024-04-12T11:28:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:30 smithi029 ceph-mon[97787]: pgmap v993: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 312 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-04-12T11:28:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:30 smithi029 ceph-mon[97787]: osdmap e905: 8 total, 8 up, 8 in 2024-04-12T11:28:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:30 smithi029 ceph-mon[93650]: pgmap v993: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 312 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-04-12T11:28:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:30 smithi029 ceph-mon[93650]: osdmap e905: 8 total, 8 up, 8 in 2024-04-12T11:28:31.690 INFO:tasks.rados.rados.0.smithi029.stdout:3327: done (0 left) 2024-04-12T11:28:31.690 INFO:tasks.rados.rados.0.smithi029.stdout:3328: snap_create 2024-04-12T11:28:31.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:31 smithi087 ceph-mon[89014]: osdmap e906: 8 total, 8 up, 8 in 2024-04-12T11:28:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:31 smithi029 ceph-mon[93650]: osdmap e906: 8 total, 8 up, 8 in 2024-04-12T11:28:32.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:31 smithi029 ceph-mon[97787]: osdmap e906: 8 total, 8 up, 8 in 2024-04-12T11:28:32.676 INFO:tasks.rados.rados.0.smithi029.stdout:3328: done (0 left) 2024-04-12T11:28:32.676 INFO:tasks.rados.rados.0.smithi029.stdout:3329: snap_remove snap 411 2024-04-12T11:28:32.688 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:28:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:28:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:28:32.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:32 smithi087 ceph-mon[89014]: pgmap v996: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 312 MiB data, 959 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-04-12T11:28:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:32 smithi087 ceph-mon[89014]: osdmap e907: 8 total, 8 up, 8 in 2024-04-12T11:28:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:32 smithi087 ceph-mon[89014]: osdmap e908: 8 total, 8 up, 8 in 2024-04-12T11:28:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:32 smithi029 ceph-mon[93650]: pgmap v996: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 312 MiB data, 959 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-04-12T11:28:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:32 smithi029 ceph-mon[93650]: osdmap e907: 8 total, 8 up, 8 in 2024-04-12T11:28:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:32 smithi029 ceph-mon[93650]: osdmap e908: 8 total, 8 up, 8 in 2024-04-12T11:28:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:32 smithi029 ceph-mon[97787]: pgmap v996: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 312 MiB data, 959 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-04-12T11:28:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:32 smithi029 ceph-mon[97787]: osdmap e907: 8 total, 8 up, 8 in 2024-04-12T11:28:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:32 smithi029 ceph-mon[97787]: osdmap e908: 8 total, 8 up, 8 in 2024-04-12T11:28:33.679 INFO:tasks.rados.rados.0.smithi029.stdout:3329: done (0 left) 2024-04-12T11:28:33.679 INFO:tasks.rados.rados.0.smithi029.stdout:3330: read oid 42 snap -1 2024-04-12T11:28:33.679 INFO:tasks.rados.rados.0.smithi029.stdout:3330: expect (ObjNum 1167 snap 397 seq_num 1167) 2024-04-12T11:28:33.679 INFO:tasks.rados.rados.0.smithi029.stdout:3331: rollback oid 23 current snap is 424 2024-04-12T11:28:33.679 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 23 to 416 2024-04-12T11:28:33.679 INFO:tasks.rados.rados.0.smithi029.stdout:3332: delete oid 41 current snap is 424 2024-04-12T11:28:33.687 INFO:tasks.rados.rados.0.smithi029.stdout:3330: expect (ObjNum 1167 snap 397 seq_num 1167) 2024-04-12T11:28:33.855 INFO:tasks.rados.rados.0.smithi029.stdout:3331: finishing rollback tid 0 to smithi02979277-23 2024-04-12T11:28:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:3331: finishing rollback tid 1 to smithi02979277-23 2024-04-12T11:28:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:3331: finishing rollback tid 2 to smithi02979277-23 2024-04-12T11:28:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1387 (ObjNum 1210 snap 408 seq_num 1210) dirty exists 2024-04-12T11:28:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:3330: done (2 left) 2024-04-12T11:28:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:3331: done (1 left) 2024-04-12T11:28:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:3332: done (0 left) 2024-04-12T11:28:33.856 INFO:tasks.rados.rados.0.smithi029.stdout:3333: delete oid 46 current snap is 424 2024-04-12T11:28:33.858 INFO:tasks.rados.rados.0.smithi029.stdout:3333: done (0 left) 2024-04-12T11:28:33.858 INFO:tasks.rados.rados.0.smithi029.stdout:3334: read oid 31 snap 410 2024-04-12T11:28:33.858 INFO:tasks.rados.rados.0.smithi029.stdout:3334: expect (ObjNum 1205 snap 406 seq_num 1205) 2024-04-12T11:28:33.858 INFO:tasks.rados.rados.0.smithi029.stdout:3335: delete oid 8 current snap is 424 2024-04-12T11:28:33.859 INFO:tasks.rados.rados.0.smithi029.stdout:3335: done (1 left) 2024-04-12T11:28:33.859 INFO:tasks.rados.rados.0.smithi029.stdout:3336: write oid 40 current snap is 424 2024-04-12T11:28:33.859 INFO:tasks.rados.rados.0.smithi029.stdout:3336: seq_num 1244 ranges {507908=575295,1829482=631762} 2024-04-12T11:28:33.865 INFO:tasks.rados.rados.0.smithi029.stdout:3336: writing smithi02979277-40 from 507908 to 1083203 tid 1 2024-04-12T11:28:33.873 INFO:tasks.rados.rados.0.smithi029.stdout:3336: writing smithi02979277-40 from 1829482 to 2461244 tid 2 2024-04-12T11:28:33.873 INFO:tasks.rados.rados.0.smithi029.stdout:3337: snap_create 2024-04-12T11:28:33.873 INFO:tasks.rados.rados.0.smithi029.stdout:3334: expect (ObjNum 1205 snap 406 seq_num 1205) 2024-04-12T11:28:33.947 INFO:tasks.rados.rados.0.smithi029.stdout:3336: finishing write tid 1 to smithi02979277-40 2024-04-12T11:28:33.947 INFO:tasks.rados.rados.0.smithi029.stdout:3336: finishing write tid 2 to smithi02979277-40 2024-04-12T11:28:33.947 INFO:tasks.rados.rados.0.smithi029.stdout:3336: finishing write tid 3 to smithi02979277-40 2024-04-12T11:28:33.947 INFO:tasks.rados.rados.0.smithi029.stdout:3336: finishing write tid 4 to smithi02979277-40 2024-04-12T11:28:33.947 INFO:tasks.rados.rados.0.smithi029.stdout:3336: oid 40 updating version 0 to 2111 2024-04-12T11:28:33.948 INFO:tasks.rados.rados.0.smithi029.stdout:3336: oid 40 updating version 2111 to 2113 2024-04-12T11:28:33.948 INFO:tasks.rados.rados.0.smithi029.stdout:3336: oid 40 version 2113 is already newer than 2112 2024-04-12T11:28:33.948 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 2113 (ObjNum 1244 snap 424 seq_num 1244) dirty exists 2024-04-12T11:28:33.948 INFO:tasks.rados.rados.0.smithi029.stdout:3336: left oid 40 (ObjNum 1244 snap 424 seq_num 1244) 2024-04-12T11:28:34.695 INFO:tasks.rados.rados.0.smithi029.stdout:3334: done (2 left) 2024-04-12T11:28:34.695 INFO:tasks.rados.rados.0.smithi029.stdout:3336: done (1 left) 2024-04-12T11:28:34.696 INFO:tasks.rados.rados.0.smithi029.stdout:3337: done (0 left) 2024-04-12T11:28:34.696 INFO:tasks.rados.rados.0.smithi029.stdout:3338: read oid 33 snap 418 2024-04-12T11:28:34.696 INFO:tasks.rados.rados.0.smithi029.stdout:3338: expect deleted 2024-04-12T11:28:34.696 INFO:tasks.rados.rados.0.smithi029.stdout:3339: read oid 22 snap -1 2024-04-12T11:28:34.696 INFO:tasks.rados.rados.0.smithi029.stdout:3339: expect (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:28:34.696 INFO:tasks.rados.rados.0.smithi029.stdout:3340: write oid 8 current snap is 425 2024-04-12T11:28:34.696 INFO:tasks.rados.rados.0.smithi029.stdout:3340: seq_num 1245 ranges {524715=650544,1887964=491773} 2024-04-12T11:28:34.701 INFO:tasks.rados.rados.0.smithi029.stdout:3340: writing smithi02979277-8 from 524715 to 1175259 tid 1 2024-04-12T11:28:34.708 INFO:tasks.rados.rados.0.smithi029.stdout:3340: writing smithi02979277-8 from 1887964 to 2379737 tid 2 2024-04-12T11:28:34.708 INFO:tasks.rados.rados.0.smithi029.stdout:3341: rollback oid 42 current snap is 425 2024-04-12T11:28:34.708 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 42 to 395 2024-04-12T11:28:34.709 INFO:tasks.rados.rados.0.smithi029.stdout:3339: expect (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:28:34.869 INFO:tasks.rados.rados.0.smithi029.stdout:3340: finishing write tid 1 to smithi02979277-8 2024-04-12T11:28:34.869 INFO:tasks.rados.rados.0.smithi029.stdout:3340: finishing write tid 2 to smithi02979277-8 2024-04-12T11:28:34.869 INFO:tasks.rados.rados.0.smithi029.stdout:3340: finishing write tid 3 to smithi02979277-8 2024-04-12T11:28:34.869 INFO:tasks.rados.rados.0.smithi029.stdout:3340: finishing write tid 4 to smithi02979277-8 2024-04-12T11:28:34.869 INFO:tasks.rados.rados.0.smithi029.stdout:3340: oid 8 updating version 0 to 341 2024-04-12T11:28:34.869 INFO:tasks.rados.rados.0.smithi029.stdout:3340: oid 8 updating version 341 to 342 2024-04-12T11:28:34.869 INFO:tasks.rados.rados.0.smithi029.stdout:3340: oid 8 updating version 342 to 343 2024-04-12T11:28:34.869 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 343 (ObjNum 1245 snap 425 seq_num 1245) dirty exists 2024-04-12T11:28:34.869 INFO:tasks.rados.rados.0.smithi029.stdout:3340: left oid 8 (ObjNum 1245 snap 425 seq_num 1245) 2024-04-12T11:28:34.869 INFO:tasks.rados.rados.0.smithi029.stdout:3341: finishing rollback tid 0 to smithi02979277-42 2024-04-12T11:28:34.870 INFO:tasks.rados.rados.0.smithi029.stdout:3341: finishing rollback tid 1 to smithi02979277-42 2024-04-12T11:28:34.870 INFO:tasks.rados.rados.0.smithi029.stdout:3338: done (3 left) 2024-04-12T11:28:34.870 INFO:tasks.rados.rados.0.smithi029.stdout:3339: done (2 left) 2024-04-12T11:28:34.870 INFO:tasks.rados.rados.0.smithi029.stdout:3340: done (1 left) 2024-04-12T11:28:34.870 INFO:tasks.rados.rados.0.smithi029.stdout:3342: read oid 25 snap -1 2024-04-12T11:28:34.870 INFO:tasks.rados.rados.0.smithi029.stdout:3342: expect (ObjNum 1190 snap 402 seq_num 1190) 2024-04-12T11:28:34.870 INFO:tasks.rados.rados.0.smithi029.stdout:3343: snap_remove snap 410 2024-04-12T11:28:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:34 smithi087 ceph-mon[89014]: pgmap v999: 89 pgs: 89 active+clean; 310 MiB data, 956 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:28:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:34 smithi087 ceph-mon[89014]: osdmap e909: 8 total, 8 up, 8 in 2024-04-12T11:28:35.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:34 smithi029 ceph-mon[93650]: pgmap v999: 89 pgs: 89 active+clean; 310 MiB data, 956 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:28:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:34 smithi029 ceph-mon[93650]: osdmap e909: 8 total, 8 up, 8 in 2024-04-12T11:28:35.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:34 smithi029 ceph-mon[97787]: pgmap v999: 89 pgs: 89 active+clean; 310 MiB data, 956 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-12T11:28:35.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:34 smithi029 ceph-mon[97787]: osdmap e909: 8 total, 8 up, 8 in 2024-04-12T11:28:35.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:28:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:28:34] "GET /metrics HTTP/1.1" 200 36369 "" "Prometheus/2.43.0" 2024-04-12T11:28:35.704 INFO:tasks.rados.rados.0.smithi029.stdout:3343: done (2 left) 2024-04-12T11:28:35.704 INFO:tasks.rados.rados.0.smithi029.stdout:3344: rollback oid 47 current snap is 425 2024-04-12T11:28:35.704 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 416 2024-04-12T11:28:35.704 INFO:tasks.rados.rados.0.smithi029.stdout:3345: read oid 22 snap -1 2024-04-12T11:28:35.704 INFO:tasks.rados.rados.0.smithi029.stdout:3345: expect (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:28:35.704 INFO:tasks.rados.rados.0.smithi029.stdout:3346: rollback oid 23 current snap is 425 2024-04-12T11:28:35.704 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 23 to 408 2024-04-12T11:28:35.704 INFO:tasks.rados.rados.0.smithi029.stdout:3347: read oid 8 snap -1 2024-04-12T11:28:35.704 INFO:tasks.rados.rados.0.smithi029.stdout:3347: expect (ObjNum 1245 snap 425 seq_num 1245) 2024-04-12T11:28:35.705 INFO:tasks.rados.rados.0.smithi029.stdout:3348: read oid 3 snap -1 2024-04-12T11:28:35.705 INFO:tasks.rados.rados.0.smithi029.stdout:3348: expect deleted 2024-04-12T11:28:35.705 INFO:tasks.rados.rados.0.smithi029.stdout:3349: delete oid 14 current snap is 425 2024-04-12T11:28:35.705 INFO:tasks.rados.rados.0.smithi029.stdout:3341: finishing rollback tid 2 to smithi02979277-42 2024-04-12T11:28:35.705 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 404 (ObjNum 1129 snap 379 seq_num 1129) dirty exists 2024-04-12T11:28:35.705 INFO:tasks.rados.rados.0.smithi029.stdout:3342: expect (ObjNum 1190 snap 402 seq_num 1190) 2024-04-12T11:28:35.792 INFO:tasks.rados.rados.0.smithi029.stdout:3347: expect (ObjNum 1245 snap 425 seq_num 1245) 2024-04-12T11:28:35.891 INFO:tasks.rados.rados.0.smithi029.stdout:3345: expect (ObjNum 1114 snap 371 seq_num 1114) 2024-04-12T11:28:35.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:35 smithi087 ceph-mon[89014]: osdmap e910: 8 total, 8 up, 8 in 2024-04-12T11:28:36.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:35 smithi029 ceph-mon[97787]: osdmap e910: 8 total, 8 up, 8 in 2024-04-12T11:28:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:35 smithi029 ceph-mon[93650]: osdmap e910: 8 total, 8 up, 8 in 2024-04-12T11:28:36.049 INFO:tasks.rados.rados.0.smithi029.stdout:3346: finishing rollback tid 0 to smithi02979277-23 2024-04-12T11:28:36.049 INFO:tasks.rados.rados.0.smithi029.stdout:3346: finishing rollback tid 1 to smithi02979277-23 2024-04-12T11:28:36.049 INFO:tasks.rados.rados.0.smithi029.stdout:3346: finishing rollback tid 2 to smithi02979277-23 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1393 (ObjNum 1210 snap 408 seq_num 1210) dirty exists 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3344: finishing rollback tid 0 to smithi02979277-47 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3344: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3344: finishing rollback tid 2 to smithi02979277-47 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1618 (ObjNum 1227 snap 413 seq_num 1227) dirty exists 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3341: done (7 left) 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3342: done (6 left) 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3344: done (5 left) 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3345: done (4 left) 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3346: done (3 left) 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3347: done (2 left) 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3348: done (1 left) 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3349: done (0 left) 2024-04-12T11:28:36.050 INFO:tasks.rados.rados.0.smithi029.stdout:3350: delete oid 47 current snap is 425 2024-04-12T11:28:36.051 INFO:tasks.rados.rados.0.smithi029.stdout:3350: done (0 left) 2024-04-12T11:28:36.051 INFO:tasks.rados.rados.0.smithi029.stdout:3351: snap_remove snap 420 2024-04-12T11:28:36.713 INFO:tasks.rados.rados.0.smithi029.stdout:3351: done (0 left) 2024-04-12T11:28:36.713 INFO:tasks.rados.rados.0.smithi029.stdout:3352: snap_remove snap 414 2024-04-12T11:28:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:36 smithi087 ceph-mon[89014]: pgmap v1002: 89 pgs: 89 active+clean; 308 MiB data, 951 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 0 B/s wr, 2 op/s 2024-04-12T11:28:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:36 smithi087 ceph-mon[89014]: osdmap e911: 8 total, 8 up, 8 in 2024-04-12T11:28:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:36 smithi029 ceph-mon[93650]: pgmap v1002: 89 pgs: 89 active+clean; 308 MiB data, 951 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 0 B/s wr, 2 op/s 2024-04-12T11:28:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:36 smithi029 ceph-mon[93650]: osdmap e911: 8 total, 8 up, 8 in 2024-04-12T11:28:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:36 smithi029 ceph-mon[97787]: pgmap v1002: 89 pgs: 89 active+clean; 308 MiB data, 951 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 0 B/s wr, 2 op/s 2024-04-12T11:28:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:36 smithi029 ceph-mon[97787]: osdmap e911: 8 total, 8 up, 8 in 2024-04-12T11:28:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:37.678 INFO:tasks.rados.rados.0.smithi029.stdout:3352: done (0 left) 2024-04-12T11:28:37.679 INFO:tasks.rados.rados.0.smithi029.stdout:3353: delete oid 3 current snap is 425 2024-04-12T11:28:37.689 INFO:tasks.rados.rados.0.smithi029.stdout:3353: done (0 left) 2024-04-12T11:28:37.689 INFO:tasks.rados.rados.0.smithi029.stdout:3354: write oid 22 current snap is 425 2024-04-12T11:28:37.690 INFO:tasks.rados.rados.0.smithi029.stdout:3354: seq_num 1249 ranges {591943=667893,1721892=782758} 2024-04-12T11:28:37.696 INFO:tasks.rados.rados.0.smithi029.stdout:3354: writing smithi02979277-22 from 591943 to 1259836 tid 1 2024-04-12T11:28:37.703 INFO:tasks.rados.rados.0.smithi029.stdout:3354: writing smithi02979277-22 from 1721892 to 2504650 tid 2 2024-04-12T11:28:37.703 INFO:tasks.rados.rados.0.smithi029.stdout:3355: write oid 45 current snap is 425 2024-04-12T11:28:37.703 INFO:tasks.rados.rados.0.smithi029.stdout:3355: seq_num 1250 ranges {608750=743142,1780374=642769} 2024-04-12T11:28:37.711 INFO:tasks.rados.rados.0.smithi029.stdout:3355: writing smithi02979277-45 from 608750 to 1351892 tid 1 2024-04-12T11:28:37.718 INFO:tasks.rados.rados.0.smithi029.stdout:3355: writing smithi02979277-45 from 1780374 to 2423143 tid 2 2024-04-12T11:28:37.718 INFO:tasks.rados.rados.0.smithi029.stdout:3356: write oid 10 current snap is 425 2024-04-12T11:28:37.718 INFO:tasks.rados.rados.0.smithi029.stdout:3354: finishing write tid 1 to smithi02979277-22 2024-04-12T11:28:37.718 INFO:tasks.rados.rados.0.smithi029.stdout:3354: finishing write tid 2 to smithi02979277-22 2024-04-12T11:28:37.718 INFO:tasks.rados.rados.0.smithi029.stdout:3354: finishing write tid 3 to smithi02979277-22 2024-04-12T11:28:37.719 INFO:tasks.rados.rados.0.smithi029.stdout:3354: finishing write tid 4 to smithi02979277-22 2024-04-12T11:28:37.719 INFO:tasks.rados.rados.0.smithi029.stdout:3354: oid 22 updating version 0 to 1407 2024-04-12T11:28:37.719 INFO:tasks.rados.rados.0.smithi029.stdout:3354: oid 22 updating version 1407 to 1409 2024-04-12T11:28:37.719 INFO:tasks.rados.rados.0.smithi029.stdout:3354: oid 22 version 1409 is already newer than 1408 2024-04-12T11:28:37.719 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 1409 (ObjNum 1249 snap 425 seq_num 1249) dirty exists 2024-04-12T11:28:37.719 INFO:tasks.rados.rados.0.smithi029.stdout:3354: left oid 22 (ObjNum 1249 snap 425 seq_num 1249) 2024-04-12T11:28:37.719 INFO:tasks.rados.rados.0.smithi029.stdout:3355: finishing write tid 1 to smithi02979277-45 2024-04-12T11:28:37.719 INFO:tasks.rados.rados.0.smithi029.stdout:3356: seq_num 1251 ranges {625557=418391,1838856=786427} 2024-04-12T11:28:37.723 INFO:tasks.rados.rados.0.smithi029.stdout:3356: writing smithi02979277-10 from 625557 to 1043948 tid 1 2024-04-12T11:28:37.732 INFO:tasks.rados.rados.0.smithi029.stdout:3356: writing smithi02979277-10 from 1838856 to 2625283 tid 2 2024-04-12T11:28:37.732 INFO:tasks.rados.rados.0.smithi029.stdout:3354: done (2 left) 2024-04-12T11:28:37.733 INFO:tasks.rados.rados.0.smithi029.stdout:3357: write oid 21 current snap is 425 2024-04-12T11:28:37.733 INFO:tasks.rados.rados.0.smithi029.stdout:3355: finishing write tid 2 to smithi02979277-45 2024-04-12T11:28:37.733 INFO:tasks.rados.rados.0.smithi029.stdout:3355: finishing write tid 3 to smithi02979277-45 2024-04-12T11:28:37.733 INFO:tasks.rados.rados.0.smithi029.stdout:3357: seq_num 1252 ranges {642364=493640,1897338=646438} 2024-04-12T11:28:37.738 INFO:tasks.rados.rados.0.smithi029.stdout:3357: writing smithi02979277-21 from 642364 to 1136004 tid 1 2024-04-12T11:28:37.747 INFO:tasks.rados.rados.0.smithi029.stdout:3357: writing smithi02979277-21 from 1897338 to 2543776 tid 2 2024-04-12T11:28:37.747 INFO:tasks.rados.rados.0.smithi029.stdout:3358: read oid 35 snap 421 2024-04-12T11:28:37.747 INFO:tasks.rados.rados.0.smithi029.stdout:3358: expect deleted 2024-04-12T11:28:37.747 INFO:tasks.rados.rados.0.smithi029.stdout:3359: delete oid 29 current snap is 425 2024-04-12T11:28:37.747 INFO:tasks.rados.rados.0.smithi029.stdout:3355: finishing write tid 4 to smithi02979277-45 2024-04-12T11:28:37.747 INFO:tasks.rados.rados.0.smithi029.stdout:3355: oid 45 updating version 0 to 1641 2024-04-12T11:28:37.747 INFO:tasks.rados.rados.0.smithi029.stdout:3355: oid 45 version 1641 is already newer than 1640 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3355: oid 45 updating version 1641 to 1642 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1642 (ObjNum 1250 snap 425 seq_num 1250) dirty exists 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3355: left oid 45 (ObjNum 1250 snap 425 seq_num 1250) 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3356: finishing write tid 1 to smithi02979277-10 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3356: finishing write tid 2 to smithi02979277-10 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3356: finishing write tid 3 to smithi02979277-10 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3356: finishing write tid 4 to smithi02979277-10 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3356: oid 10 updating version 0 to 1329 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3356: oid 10 updating version 1329 to 1330 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3356: oid 10 version 1330 is already newer than 1328 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 10 v 1330 (ObjNum 1251 snap 425 seq_num 1251) dirty exists 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3356: left oid 10 (ObjNum 1251 snap 425 seq_num 1251) 2024-04-12T11:28:37.748 INFO:tasks.rados.rados.0.smithi029.stdout:3357: finishing write tid 1 to smithi02979277-21 2024-04-12T11:28:37.749 INFO:tasks.rados.rados.0.smithi029.stdout:3355: done (4 left) 2024-04-12T11:28:37.749 INFO:tasks.rados.rados.0.smithi029.stdout:3356: done (3 left) 2024-04-12T11:28:37.749 INFO:tasks.rados.rados.0.smithi029.stdout:3358: done (2 left) 2024-04-12T11:28:37.749 INFO:tasks.rados.rados.0.smithi029.stdout:3359: done (1 left) 2024-04-12T11:28:37.749 INFO:tasks.rados.rados.0.smithi029.stdout:3360: read oid 50 snap 418 2024-04-12T11:28:37.750 INFO:tasks.rados.rados.0.smithi029.stdout:3360: expect (ObjNum 1179 snap 400 seq_num 1179) 2024-04-12T11:28:37.750 INFO:tasks.rados.rados.0.smithi029.stdout:3361: write oid 16 current snap is 425 2024-04-12T11:28:37.750 INFO:tasks.rados.rados.0.smithi029.stdout:3361: seq_num 1254 ranges {675978=644138,1897949=650107} 2024-04-12T11:28:37.756 INFO:tasks.rados.rados.0.smithi029.stdout:3361: writing smithi02979277-16 from 675978 to 1320116 tid 1 2024-04-12T11:28:37.763 INFO:tasks.rados.rados.0.smithi029.stdout:3361: writing smithi02979277-16 from 1897949 to 2548056 tid 2 2024-04-12T11:28:37.763 INFO:tasks.rados.rados.0.smithi029.stdout:3362: snap_create 2024-04-12T11:28:37.763 INFO:tasks.rados.rados.0.smithi029.stdout:3357: finishing write tid 2 to smithi02979277-21 2024-04-12T11:28:37.763 INFO:tasks.rados.rados.0.smithi029.stdout:3357: finishing write tid 3 to smithi02979277-21 2024-04-12T11:28:37.763 INFO:tasks.rados.rados.0.smithi029.stdout:3357: finishing write tid 4 to smithi02979277-21 2024-04-12T11:28:37.763 INFO:tasks.rados.rados.0.smithi029.stdout:3357: oid 21 updating version 0 to 1616 2024-04-12T11:28:37.763 INFO:tasks.rados.rados.0.smithi029.stdout:3357: oid 21 version 1616 is already newer than 1614 2024-04-12T11:28:37.763 INFO:tasks.rados.rados.0.smithi029.stdout:3357: oid 21 version 1616 is already newer than 1615 2024-04-12T11:28:37.763 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1616 (ObjNum 1252 snap 425 seq_num 1252) dirty exists 2024-04-12T11:28:37.764 INFO:tasks.rados.rados.0.smithi029.stdout:3357: left oid 21 (ObjNum 1252 snap 425 seq_num 1252) 2024-04-12T11:28:37.764 INFO:tasks.rados.rados.0.smithi029.stdout:3360: expect (ObjNum 1179 snap 400 seq_num 1179) 2024-04-12T11:28:37.930 INFO:tasks.rados.rados.0.smithi029.stdout:3361: finishing write tid 1 to smithi02979277-16 2024-04-12T11:28:37.930 INFO:tasks.rados.rados.0.smithi029.stdout:3361: finishing write tid 2 to smithi02979277-16 2024-04-12T11:28:37.930 INFO:tasks.rados.rados.0.smithi029.stdout:3361: finishing write tid 3 to smithi02979277-16 2024-04-12T11:28:37.930 INFO:tasks.rados.rados.0.smithi029.stdout:3361: finishing write tid 4 to smithi02979277-16 2024-04-12T11:28:37.930 INFO:tasks.rados.rados.0.smithi029.stdout:3361: oid 16 updating version 0 to 1528 2024-04-12T11:28:37.930 INFO:tasks.rados.rados.0.smithi029.stdout:3361: oid 16 version 1528 is already newer than 1527 2024-04-12T11:28:37.931 INFO:tasks.rados.rados.0.smithi029.stdout:3361: oid 16 version 1528 is already newer than 1526 2024-04-12T11:28:37.931 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1528 (ObjNum 1254 snap 425 seq_num 1254) dirty exists 2024-04-12T11:28:37.931 INFO:tasks.rados.rados.0.smithi029.stdout:3361: left oid 16 (ObjNum 1254 snap 425 seq_num 1254) 2024-04-12T11:28:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:37 smithi087 ceph-mon[89014]: osdmap e912: 8 total, 8 up, 8 in 2024-04-12T11:28:37.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:37 smithi087 ceph-mon[89014]: osdmap e913: 8 total, 8 up, 8 in 2024-04-12T11:28:38.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:37 smithi029 ceph-mon[97787]: osdmap e912: 8 total, 8 up, 8 in 2024-04-12T11:28:38.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:37 smithi029 ceph-mon[97787]: osdmap e913: 8 total, 8 up, 8 in 2024-04-12T11:28:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:37 smithi029 ceph-mon[93650]: osdmap e912: 8 total, 8 up, 8 in 2024-04-12T11:28:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:37 smithi029 ceph-mon[93650]: osdmap e913: 8 total, 8 up, 8 in 2024-04-12T11:28:38.724 INFO:tasks.rados.rados.0.smithi029.stdout:3357: done (3 left) 2024-04-12T11:28:38.724 INFO:tasks.rados.rados.0.smithi029.stdout:3360: done (2 left) 2024-04-12T11:28:38.724 INFO:tasks.rados.rados.0.smithi029.stdout:3361: done (1 left) 2024-04-12T11:28:38.724 INFO:tasks.rados.rados.0.smithi029.stdout:3362: done (0 left) 2024-04-12T11:28:38.724 INFO:tasks.rados.rados.0.smithi029.stdout:3363: snap_remove snap 419 2024-04-12T11:28:39.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:38 smithi029 ceph-mon[93650]: pgmap v1005: 89 pgs: 89 active+clean; 311 MiB data, 955 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 279 KiB/s wr, 12 op/s 2024-04-12T11:28:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:38 smithi029 ceph-mon[97787]: pgmap v1005: 89 pgs: 89 active+clean; 311 MiB data, 955 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 279 KiB/s wr, 12 op/s 2024-04-12T11:28:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:38 smithi087 ceph-mon[89014]: pgmap v1005: 89 pgs: 89 active+clean; 311 MiB data, 955 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 279 KiB/s wr, 12 op/s 2024-04-12T11:28:39.730 INFO:tasks.rados.rados.0.smithi029.stdout:3363: done (0 left) 2024-04-12T11:28:39.730 INFO:tasks.rados.rados.0.smithi029.stdout:3364: read oid 7 snap 423 2024-04-12T11:28:39.730 INFO:tasks.rados.rados.0.smithi029.stdout:3364: expect (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:28:39.730 INFO:tasks.rados.rados.0.smithi029.stdout:3365: snap_create 2024-04-12T11:28:39.737 INFO:tasks.rados.rados.0.smithi029.stdout:3364: expect (ObjNum 959 snap 329 seq_num 959) 2024-04-12T11:28:40.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:39 smithi029 ceph-mon[93650]: osdmap e914: 8 total, 8 up, 8 in 2024-04-12T11:28:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:39 smithi029 ceph-mon[97787]: osdmap e914: 8 total, 8 up, 8 in 2024-04-12T11:28:40.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:39 smithi087 ceph-mon[89014]: osdmap e914: 8 total, 8 up, 8 in 2024-04-12T11:28:40.732 INFO:tasks.rados.rados.0.smithi029.stdout:3364: done (1 left) 2024-04-12T11:28:40.732 INFO:tasks.rados.rados.0.smithi029.stdout:3365: done (0 left) 2024-04-12T11:28:40.732 INFO:tasks.rados.rados.0.smithi029.stdout:3366: read oid 18 snap -1 2024-04-12T11:28:40.732 INFO:tasks.rados.rados.0.smithi029.stdout:3366: expect (ObjNum 1198 snap 406 seq_num 1198) 2024-04-12T11:28:40.732 INFO:tasks.rados.rados.0.smithi029.stdout:3367: read oid 27 snap -1 2024-04-12T11:28:40.732 INFO:tasks.rados.rados.0.smithi029.stdout:3367: expect (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:28:40.732 INFO:tasks.rados.rados.0.smithi029.stdout:3368: read oid 38 snap -1 2024-04-12T11:28:40.732 INFO:tasks.rados.rados.0.smithi029.stdout:3368: expect (ObjNum 1197 snap 406 seq_num 1197) 2024-04-12T11:28:40.733 INFO:tasks.rados.rados.0.smithi029.stdout:3369: read oid 25 snap 425 2024-04-12T11:28:40.733 INFO:tasks.rados.rados.0.smithi029.stdout:3369: expect (ObjNum 1190 snap 402 seq_num 1190) 2024-04-12T11:28:40.733 INFO:tasks.rados.rados.0.smithi029.stdout:3370: read oid 49 snap 416 2024-04-12T11:28:40.733 INFO:tasks.rados.rados.0.smithi029.stdout:3370: expect (ObjNum 1097 snap 368 seq_num 1097) 2024-04-12T11:28:40.733 INFO:tasks.rados.rados.0.smithi029.stdout:3371: snap_remove snap 423 2024-04-12T11:28:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:40 smithi029 ceph-mon[93650]: pgmap v1008: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 313 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 868 KiB/s wr, 17 op/s 2024-04-12T11:28:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:40 smithi029 ceph-mon[93650]: osdmap e915: 8 total, 8 up, 8 in 2024-04-12T11:28:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:40 smithi029 ceph-mon[97787]: pgmap v1008: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 313 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 868 KiB/s wr, 17 op/s 2024-04-12T11:28:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:40 smithi029 ceph-mon[97787]: osdmap e915: 8 total, 8 up, 8 in 2024-04-12T11:28:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:40 smithi087 ceph-mon[89014]: pgmap v1008: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 313 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 868 KiB/s wr, 17 op/s 2024-04-12T11:28:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:40 smithi087 ceph-mon[89014]: osdmap e915: 8 total, 8 up, 8 in 2024-04-12T11:28:41.743 INFO:tasks.rados.rados.0.smithi029.stdout:3371: done (5 left) 2024-04-12T11:28:41.743 INFO:tasks.rados.rados.0.smithi029.stdout:3372: rollback oid 20 current snap is 427 2024-04-12T11:28:41.744 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 20 to 409 2024-04-12T11:28:41.744 INFO:tasks.rados.rados.0.smithi029.stdout:3373: rollback oid 31 current snap is 427 2024-04-12T11:28:41.744 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 31 to 424 2024-04-12T11:28:41.744 INFO:tasks.rados.rados.0.smithi029.stdout:3374: snap_remove snap 388 2024-04-12T11:28:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:41 smithi029 ceph-mon[93650]: osdmap e916: 8 total, 8 up, 8 in 2024-04-12T11:28:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:41 smithi029 ceph-mon[97787]: osdmap e916: 8 total, 8 up, 8 in 2024-04-12T11:28:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:41 smithi087 ceph-mon[89014]: osdmap e916: 8 total, 8 up, 8 in 2024-04-12T11:28:42.679 INFO:tasks.rados.rados.0.smithi029.stdout:3374: done (7 left) 2024-04-12T11:28:42.679 INFO:tasks.rados.rados.0.smithi029.stdout:3375: snap_remove snap 395 2024-04-12T11:28:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:28:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:28:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:28:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:42 smithi029 ceph-mon[93650]: pgmap v1011: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 314 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 589 KiB/s wr, 6 op/s 2024-04-12T11:28:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:42 smithi029 ceph-mon[93650]: osdmap e917: 8 total, 8 up, 8 in 2024-04-12T11:28:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:42 smithi029 ceph-mon[93650]: osdmap e918: 8 total, 8 up, 8 in 2024-04-12T11:28:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:42 smithi029 ceph-mon[97787]: pgmap v1011: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 314 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 589 KiB/s wr, 6 op/s 2024-04-12T11:28:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:42 smithi029 ceph-mon[97787]: osdmap e917: 8 total, 8 up, 8 in 2024-04-12T11:28:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:42 smithi029 ceph-mon[97787]: osdmap e918: 8 total, 8 up, 8 in 2024-04-12T11:28:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:42 smithi087 ceph-mon[89014]: pgmap v1011: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 314 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 589 KiB/s wr, 6 op/s 2024-04-12T11:28:43.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:42 smithi087 ceph-mon[89014]: osdmap e917: 8 total, 8 up, 8 in 2024-04-12T11:28:43.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:42 smithi087 ceph-mon[89014]: osdmap e918: 8 total, 8 up, 8 in 2024-04-12T11:28:43.683 INFO:tasks.rados.rados.0.smithi029.stdout:3375: done (7 left) 2024-04-12T11:28:43.683 INFO:tasks.rados.rados.0.smithi029.stdout:3376: read oid 46 snap -1 2024-04-12T11:28:43.683 INFO:tasks.rados.rados.0.smithi029.stdout:3376: expect deleted 2024-04-12T11:28:43.683 INFO:tasks.rados.rados.0.smithi029.stdout:3377: write oid 41 current snap is 427 2024-04-12T11:28:43.683 INFO:tasks.rados.rados.0.smithi029.stdout:3377: seq_num 1255 ranges {692785=435740,1672784=793765,2897503=195282} 2024-04-12T11:28:43.689 INFO:tasks.rados.rados.0.smithi029.stdout:3377: writing smithi02979277-41 from 692785 to 1128525 tid 1 2024-04-12T11:28:43.698 INFO:tasks.rados.rados.0.smithi029.stdout:3377: writing smithi02979277-41 from 1672784 to 2466549 tid 2 2024-04-12T11:28:43.702 INFO:tasks.rados.rados.0.smithi029.stdout:3377: writing smithi02979277-41 from 2897503 to 3092785 tid 3 2024-04-12T11:28:43.702 INFO:tasks.rados.rados.0.smithi029.stdout:3378: rollback oid 2 current snap is 427 2024-04-12T11:28:43.702 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 2 to 426 2024-04-12T11:28:43.702 INFO:tasks.rados.rados.0.smithi029.stdout:3379: rollback oid 45 current snap is 427 2024-04-12T11:28:43.702 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 45 to 418 2024-04-12T11:28:43.702 INFO:tasks.rados.rados.0.smithi029.stdout:3380: snap_remove snap 408 2024-04-12T11:28:44.689 INFO:tasks.rados.rados.0.smithi029.stdout:3380: done (11 left) 2024-04-12T11:28:44.689 INFO:tasks.rados.rados.0.smithi029.stdout:3381: snap_remove snap 417 2024-04-12T11:28:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:44 smithi087 ceph-mon[89014]: pgmap v1014: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 324 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-04-12T11:28:44.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:44 smithi087 ceph-mon[89014]: osdmap e919: 8 total, 8 up, 8 in 2024-04-12T11:28:45.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:44 smithi029 ceph-mon[93650]: pgmap v1014: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 324 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-04-12T11:28:45.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:44 smithi029 ceph-mon[93650]: osdmap e919: 8 total, 8 up, 8 in 2024-04-12T11:28:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:44 smithi029 ceph-mon[97787]: pgmap v1014: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 324 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-04-12T11:28:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:44 smithi029 ceph-mon[97787]: osdmap e919: 8 total, 8 up, 8 in 2024-04-12T11:28:45.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:28:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:28:44] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:28:45.694 INFO:tasks.rados.rados.0.smithi029.stdout:3381: done (11 left) 2024-04-12T11:28:45.694 INFO:tasks.rados.rados.0.smithi029.stdout:3382: write oid 14 current snap is 427 2024-04-12T11:28:45.694 INFO:tasks.rados.rados.0.smithi029.stdout:3382: seq_num 1256 ranges {709592=510989,1731266=653776,3041279=68313} 2024-04-12T11:28:45.700 INFO:tasks.rados.rados.0.smithi029.stdout:3382: writing smithi02979277-14 from 709592 to 1220581 tid 1 2024-04-12T11:28:45.706 INFO:tasks.rados.rados.0.smithi029.stdout:3382: writing smithi02979277-14 from 1731266 to 2385042 tid 2 2024-04-12T11:28:45.711 INFO:tasks.rados.rados.0.smithi029.stdout:3382: writing smithi02979277-14 from 3041279 to 3109592 tid 3 2024-04-12T11:28:45.711 INFO:tasks.rados.rados.0.smithi029.stdout:3383: delete oid 10 current snap is 427 2024-04-12T11:28:45.711 INFO:tasks.rados.rados.0.smithi029.stdout:3367: expect (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:28:45.882 INFO:tasks.rados.rados.0.smithi029.stdout:3370: expect (ObjNum 1097 snap 368 seq_num 1097) 2024-04-12T11:28:45.964 INFO:tasks.rados.rados.0.smithi029.stdout:3368: expect (ObjNum 1197 snap 406 seq_num 1197) 2024-04-12T11:28:45.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:45 smithi087 ceph-mon[89014]: osdmap e920: 8 total, 8 up, 8 in 2024-04-12T11:28:46.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:45 smithi029 ceph-mon[93650]: osdmap e920: 8 total, 8 up, 8 in 2024-04-12T11:28:46.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:45 smithi029 ceph-mon[97787]: osdmap e920: 8 total, 8 up, 8 in 2024-04-12T11:28:46.088 INFO:tasks.rados.rados.0.smithi029.stdout:3366: expect (ObjNum 1198 snap 406 seq_num 1198) 2024-04-12T11:28:46.236 INFO:tasks.rados.rados.0.smithi029.stdout:3369: expect (ObjNum 1190 snap 402 seq_num 1190) 2024-04-12T11:28:46.318 INFO:tasks.rados.rados.0.smithi029.stdout:3373: finishing rollback tid 1 to smithi02979277-31 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1292 (ObjNum 129 snap 0 seq_num 1801258912) dirty dne 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3372: finishing rollback tid 0 to smithi02979277-20 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3372: finishing rollback tid 1 to smithi02979277-20 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3372: finishing rollback tid 2 to smithi02979277-20 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 1089 (ObjNum 1138 snap 383 seq_num 1138) dirty exists 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3377: finishing write tid 1 to smithi02979277-41 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3377: finishing write tid 2 to smithi02979277-41 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3378: finishing rollback tid 0 to smithi02979277-2 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3378: finishing rollback tid 1 to smithi02979277-2 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3378: finishing rollback tid 2 to smithi02979277-2 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 384 (ObjNum 1209 snap 408 seq_num 1209) dirty exists 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3377: finishing write tid 3 to smithi02979277-41 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3377: finishing write tid 4 to smithi02979277-41 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3379: finishing rollback tid 0 to smithi02979277-45 2024-04-12T11:28:46.319 INFO:tasks.rados.rados.0.smithi029.stdout:3377: finishing write tid 5 to smithi02979277-41 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3377: oid 41 updating version 0 to 1631 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3377: oid 41 version 1631 is already newer than 1627 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3377: oid 41 version 1631 is already newer than 1630 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3377: oid 41 updating version 1631 to 1632 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1632 (ObjNum 1255 snap 427 seq_num 1255) dirty exists 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3377: left oid 41 (ObjNum 1255 snap 427 seq_num 1255) 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3379: finishing rollback tid 1 to smithi02979277-45 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1663 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3382: finishing write tid 1 to smithi02979277-14 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3382: finishing write tid 2 to smithi02979277-14 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3382: finishing write tid 3 to smithi02979277-14 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3382: finishing write tid 4 to smithi02979277-14 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3366: done (12 left) 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3367: done (11 left) 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3368: done (10 left) 2024-04-12T11:28:46.320 INFO:tasks.rados.rados.0.smithi029.stdout:3369: done (9 left) 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3370: done (8 left) 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3372: done (7 left) 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3373: done (6 left) 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3376: done (5 left) 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3377: done (4 left) 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3378: done (3 left) 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3379: done (2 left) 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3383: done (1 left) 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3384: snap_create 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3382: finishing write tid 5 to smithi02979277-14 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3382: oid 14 updating version 0 to 1553 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3382: oid 14 updating version 1553 to 1554 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3382: oid 14 updating version 1554 to 1556 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3382: oid 14 version 1556 is already newer than 1555 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 1556 (ObjNum 1256 snap 427 seq_num 1256) dirty exists 2024-04-12T11:28:46.321 INFO:tasks.rados.rados.0.smithi029.stdout:3382: left oid 14 (ObjNum 1256 snap 427 seq_num 1256) 2024-04-12T11:28:46.693 INFO:tasks.rados.rados.0.smithi029.stdout:3382: done (1 left) 2024-04-12T11:28:46.693 INFO:tasks.rados.rados.0.smithi029.stdout:3384: done (0 left) 2024-04-12T11:28:46.693 INFO:tasks.rados.rados.0.smithi029.stdout:3385: write oid 4 current snap is 428 2024-04-12T11:28:46.693 INFO:tasks.rados.rados.0.smithi029.stdout:3385: seq_num 1258 ranges {743206=661487,2131877=541092} 2024-04-12T11:28:46.701 INFO:tasks.rados.rados.0.smithi029.stdout:3385: writing smithi02979277-4 from 743206 to 1404693 tid 1 2024-04-12T11:28:46.709 INFO:tasks.rados.rados.0.smithi029.stdout:3385: writing smithi02979277-4 from 2131877 to 2672969 tid 2 2024-04-12T11:28:46.709 INFO:tasks.rados.rados.0.smithi029.stdout:3386: read oid 29 snap -1 2024-04-12T11:28:46.709 INFO:tasks.rados.rados.0.smithi029.stdout:3386: expect deleted 2024-04-12T11:28:46.709 INFO:tasks.rados.rados.0.smithi029.stdout:3387: write oid 19 current snap is 428 2024-04-12T11:28:46.709 INFO:tasks.rados.rados.0.smithi029.stdout:3385: finishing write tid 1 to smithi02979277-4 2024-04-12T11:28:46.709 INFO:tasks.rados.rados.0.smithi029.stdout:3387: seq_num 1259 ranges {760013=736736,2190359=401103} 2024-04-12T11:28:46.717 INFO:tasks.rados.rados.0.smithi029.stdout:3387: writing smithi02979277-19 from 760013 to 1496749 tid 1 2024-04-12T11:28:46.724 INFO:tasks.rados.rados.0.smithi029.stdout:3387: writing smithi02979277-19 from 2190359 to 2591462 tid 2 2024-04-12T11:28:46.724 INFO:tasks.rados.rados.0.smithi029.stdout:3388: read oid 49 snap -1 2024-04-12T11:28:46.724 INFO:tasks.rados.rados.0.smithi029.stdout:3388: expect (ObjNum 1097 snap 368 seq_num 1097) 2024-04-12T11:28:46.724 INFO:tasks.rados.rados.0.smithi029.stdout:3389: rollback oid 30 current snap is 428 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 418 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3385: finishing write tid 2 to smithi02979277-4 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3385: finishing write tid 3 to smithi02979277-4 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3385: finishing write tid 4 to smithi02979277-4 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3385: oid 4 updating version 0 to 1558 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3385: oid 4 updating version 1558 to 1559 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3385: oid 4 version 1559 is already newer than 1557 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1559 (ObjNum 1258 snap 428 seq_num 1258) dirty exists 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3385: left oid 4 (ObjNum 1258 snap 428 seq_num 1258) 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3387: finishing write tid 1 to smithi02979277-19 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3385: done (4 left) 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3386: done (3 left) 2024-04-12T11:28:46.725 INFO:tasks.rados.rados.0.smithi029.stdout:3390: snap_create 2024-04-12T11:28:46.726 INFO:tasks.rados.rados.0.smithi029.stdout:3388: expect (ObjNum 1097 snap 368 seq_num 1097) 2024-04-12T11:28:46.809 INFO:tasks.rados.rados.0.smithi029.stdout:3389: finishing rollback tid 0 to smithi02979277-30 2024-04-12T11:28:46.809 INFO:tasks.rados.rados.0.smithi029.stdout:3389: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:28:46.809 INFO:tasks.rados.rados.0.smithi029.stdout:3389: finishing rollback tid 2 to smithi02979277-30 2024-04-12T11:28:46.809 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 1058 (ObjNum 1212 snap 409 seq_num 1212) dirty exists 2024-04-12T11:28:46.809 INFO:tasks.rados.rados.0.smithi029.stdout:3387: finishing write tid 2 to smithi02979277-19 2024-04-12T11:28:46.809 INFO:tasks.rados.rados.0.smithi029.stdout:3387: finishing write tid 3 to smithi02979277-19 2024-04-12T11:28:46.809 INFO:tasks.rados.rados.0.smithi029.stdout:3387: finishing write tid 4 to smithi02979277-19 2024-04-12T11:28:46.809 INFO:tasks.rados.rados.0.smithi029.stdout:3387: oid 19 updating version 0 to 1562 2024-04-12T11:28:46.810 INFO:tasks.rados.rados.0.smithi029.stdout:3387: oid 19 version 1562 is already newer than 1561 2024-04-12T11:28:46.810 INFO:tasks.rados.rados.0.smithi029.stdout:3387: oid 19 updating version 1562 to 1563 2024-04-12T11:28:46.810 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1563 (ObjNum 1259 snap 428 seq_num 1259) dirty exists 2024-04-12T11:28:46.810 INFO:tasks.rados.rados.0.smithi029.stdout:3387: left oid 19 (ObjNum 1259 snap 428 seq_num 1259) 2024-04-12T11:28:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:46 smithi087 ceph-mon[89014]: pgmap v1017: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 317 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 19 op/s 2024-04-12T11:28:46.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:46 smithi087 ceph-mon[89014]: osdmap e921: 8 total, 8 up, 8 in 2024-04-12T11:28:47.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:46 smithi029 ceph-mon[93650]: pgmap v1017: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 317 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 19 op/s 2024-04-12T11:28:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:46 smithi029 ceph-mon[93650]: osdmap e921: 8 total, 8 up, 8 in 2024-04-12T11:28:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:46 smithi029 ceph-mon[97787]: pgmap v1017: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 317 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 19 op/s 2024-04-12T11:28:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:46 smithi029 ceph-mon[97787]: osdmap e921: 8 total, 8 up, 8 in 2024-04-12T11:28:47.681 INFO:tasks.rados.rados.0.smithi029.stdout:3387: done (3 left) 2024-04-12T11:28:47.681 INFO:tasks.rados.rados.0.smithi029.stdout:3388: done (2 left) 2024-04-12T11:28:47.681 INFO:tasks.rados.rados.0.smithi029.stdout:3389: done (1 left) 2024-04-12T11:28:47.681 INFO:tasks.rados.rados.0.smithi029.stdout:3390: done (0 left) 2024-04-12T11:28:47.681 INFO:tasks.rados.rados.0.smithi029.stdout:3391: read oid 3 snap 424 2024-04-12T11:28:47.681 INFO:tasks.rados.rados.0.smithi029.stdout:3391: expect deleted 2024-04-12T11:28:47.681 INFO:tasks.rados.rados.0.smithi029.stdout:3392: snap_create 2024-04-12T11:28:47.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:47 smithi087 ceph-mon[89014]: osdmap e922: 8 total, 8 up, 8 in 2024-04-12T11:28:47.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:47 smithi087 ceph-mon[89014]: osdmap e923: 8 total, 8 up, 8 in 2024-04-12T11:28:48.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:47 smithi029 ceph-mon[93650]: osdmap e922: 8 total, 8 up, 8 in 2024-04-12T11:28:48.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:47 smithi029 ceph-mon[93650]: osdmap e923: 8 total, 8 up, 8 in 2024-04-12T11:28:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:47 smithi029 ceph-mon[97787]: osdmap e922: 8 total, 8 up, 8 in 2024-04-12T11:28:48.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:47 smithi029 ceph-mon[97787]: osdmap e923: 8 total, 8 up, 8 in 2024-04-12T11:28:48.683 INFO:tasks.rados.rados.0.smithi029.stdout:3391: done (1 left) 2024-04-12T11:28:48.683 INFO:tasks.rados.rados.0.smithi029.stdout:3392: done (0 left) 2024-04-12T11:28:48.683 INFO:tasks.rados.rados.0.smithi029.stdout:3393: read oid 2 snap -1 2024-04-12T11:28:48.683 INFO:tasks.rados.rados.0.smithi029.stdout:3393: expect (ObjNum 1209 snap 408 seq_num 1209) 2024-04-12T11:28:48.683 INFO:tasks.rados.rados.0.smithi029.stdout:3394: delete oid 48 current snap is 430 2024-04-12T11:28:48.686 INFO:tasks.rados.rados.0.smithi029.stdout:3394: done (1 left) 2024-04-12T11:28:48.686 INFO:tasks.rados.rados.0.smithi029.stdout:3395: delete oid 12 current snap is 430 2024-04-12T11:28:48.688 INFO:tasks.rados.rados.0.smithi029.stdout:3393: expect (ObjNum 1209 snap 408 seq_num 1209) 2024-04-12T11:28:48.768 INFO:tasks.rados.rados.0.smithi029.stdout:3393: done (1 left) 2024-04-12T11:28:48.768 INFO:tasks.rados.rados.0.smithi029.stdout:3395: done (0 left) 2024-04-12T11:28:48.768 INFO:tasks.rados.rados.0.smithi029.stdout:3396: rollback oid 39 current snap is 430 2024-04-12T11:28:48.768 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 404 2024-04-12T11:28:48.768 INFO:tasks.rados.rados.0.smithi029.stdout:3397: write oid 5 current snap is 430 2024-04-12T11:28:48.768 INFO:tasks.rados.rados.0.smithi029.stdout:3397: seq_num 1262 ranges {410434=678836,1565805=548430,2573111=637323} 2024-04-12T11:28:48.774 INFO:tasks.rados.rados.0.smithi029.stdout:3397: writing smithi02979277-5 from 410434 to 1089270 tid 1 2024-04-12T11:28:48.780 INFO:tasks.rados.rados.0.smithi029.stdout:3397: writing smithi02979277-5 from 1565805 to 2114235 tid 2 2024-04-12T11:28:48.786 INFO:tasks.rados.rados.0.smithi029.stdout:3397: writing smithi02979277-5 from 2573111 to 3210434 tid 3 2024-04-12T11:28:48.786 INFO:tasks.rados.rados.0.smithi029.stdout:3398: snap_create 2024-04-12T11:28:48.786 INFO:tasks.rados.rados.0.smithi029.stdout:3396: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:28:48.786 INFO:tasks.rados.rados.0.smithi029.stdout:3396: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:28:48.786 INFO:tasks.rados.rados.0.smithi029.stdout:3396: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:28:48.786 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 481 (ObjNum 1088 snap 365 seq_num 1088) dirty exists 2024-04-12T11:28:48.787 INFO:tasks.rados.rados.0.smithi029.stdout:3397: finishing write tid 1 to smithi02979277-5 2024-04-12T11:28:48.787 INFO:tasks.rados.rados.0.smithi029.stdout:3397: finishing write tid 2 to smithi02979277-5 2024-04-12T11:28:48.790 INFO:tasks.rados.rados.0.smithi029.stdout:3397: finishing write tid 3 to smithi02979277-5 2024-04-12T11:28:48.790 INFO:tasks.rados.rados.0.smithi029.stdout:3397: finishing write tid 4 to smithi02979277-5 2024-04-12T11:28:48.790 INFO:tasks.rados.rados.0.smithi029.stdout:3397: finishing write tid 5 to smithi02979277-5 2024-04-12T11:28:48.790 INFO:tasks.rados.rados.0.smithi029.stdout:3397: oid 5 updating version 0 to 1565 2024-04-12T11:28:48.790 INFO:tasks.rados.rados.0.smithi029.stdout:3397: oid 5 updating version 1565 to 1567 2024-04-12T11:28:48.790 INFO:tasks.rados.rados.0.smithi029.stdout:3397: oid 5 version 1567 is already newer than 1566 2024-04-12T11:28:48.791 INFO:tasks.rados.rados.0.smithi029.stdout:3397: oid 5 version 1567 is already newer than 1564 2024-04-12T11:28:48.791 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1567 (ObjNum 1262 snap 430 seq_num 1262) dirty exists 2024-04-12T11:28:48.791 INFO:tasks.rados.rados.0.smithi029.stdout:3397: left oid 5 (ObjNum 1262 snap 430 seq_num 1262) 2024-04-12T11:28:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:48 smithi087 ceph-mon[89014]: pgmap v1020: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 322 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-04-12T11:28:48.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:48 smithi087 ceph-mon[89014]: osdmap e924: 8 total, 8 up, 8 in 2024-04-12T11:28:49.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:48 smithi029 ceph-mon[93650]: pgmap v1020: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 322 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-04-12T11:28:49.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:48 smithi029 ceph-mon[93650]: osdmap e924: 8 total, 8 up, 8 in 2024-04-12T11:28:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:48 smithi029 ceph-mon[97787]: pgmap v1020: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 322 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-04-12T11:28:49.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:48 smithi029 ceph-mon[97787]: osdmap e924: 8 total, 8 up, 8 in 2024-04-12T11:28:49.708 INFO:tasks.rados.rados.0.smithi029.stdout:3396: done (2 left) 2024-04-12T11:28:49.708 INFO:tasks.rados.rados.0.smithi029.stdout:3397: done (1 left) 2024-04-12T11:28:49.708 INFO:tasks.rados.rados.0.smithi029.stdout:3398: done (0 left) 2024-04-12T11:28:49.708 INFO:tasks.rados.rados.0.smithi029.stdout:3399: read oid 46 snap -1 2024-04-12T11:28:49.708 INFO:tasks.rados.rados.0.smithi029.stdout:3399: expect deleted 2024-04-12T11:28:49.709 INFO:tasks.rados.rados.0.smithi029.stdout:3400: snap_create 2024-04-12T11:28:50.712 INFO:tasks.rados.rados.0.smithi029.stdout:3399: done (1 left) 2024-04-12T11:28:50.712 INFO:tasks.rados.rados.0.smithi029.stdout:3400: done (0 left) 2024-04-12T11:28:50.712 INFO:tasks.rados.rados.0.smithi029.stdout:3401: rollback oid 32 current snap is 432 2024-04-12T11:28:50.712 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 32 to 431 2024-04-12T11:28:50.712 INFO:tasks.rados.rados.0.smithi029.stdout:3402: delete oid 6 current snap is 432 2024-04-12T11:28:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:3402: done (1 left) 2024-04-12T11:28:50.715 INFO:tasks.rados.rados.0.smithi029.stdout:3403: snap_create 2024-04-12T11:28:50.717 INFO:tasks.rados.rados.0.smithi029.stdout:3401: finishing rollback tid 0 to smithi02979277-32 2024-04-12T11:28:50.717 INFO:tasks.rados.rados.0.smithi029.stdout:3401: finishing rollback tid 1 to smithi02979277-32 2024-04-12T11:28:50.717 INFO:tasks.rados.rados.0.smithi029.stdout:3401: finishing rollback tid 2 to smithi02979277-32 2024-04-12T11:28:50.717 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 1536 (ObjNum 1218 snap 409 seq_num 1218) dirty exists 2024-04-12T11:28:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:50 smithi087 ceph-mon[89014]: pgmap v1023: 89 pgs: 89 active+clean; 304 MiB data, 944 MiB used, 714 GiB / 715 GiB avail; 750 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-04-12T11:28:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:50 smithi087 ceph-mon[89014]: osdmap e925: 8 total, 8 up, 8 in 2024-04-12T11:28:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:50 smithi029 ceph-mon[97787]: pgmap v1023: 89 pgs: 89 active+clean; 304 MiB data, 944 MiB used, 714 GiB / 715 GiB avail; 750 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-04-12T11:28:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:50 smithi029 ceph-mon[97787]: osdmap e925: 8 total, 8 up, 8 in 2024-04-12T11:28:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:50 smithi029 ceph-mon[93650]: pgmap v1023: 89 pgs: 89 active+clean; 304 MiB data, 944 MiB used, 714 GiB / 715 GiB avail; 750 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-04-12T11:28:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:50 smithi029 ceph-mon[93650]: osdmap e925: 8 total, 8 up, 8 in 2024-04-12T11:28:51.723 INFO:tasks.rados.rados.0.smithi029.stdout:3401: done (1 left) 2024-04-12T11:28:51.724 INFO:tasks.rados.rados.0.smithi029.stdout:3403: done (0 left) 2024-04-12T11:28:51.724 INFO:tasks.rados.rados.0.smithi029.stdout:3404: read oid 2 snap -1 2024-04-12T11:28:51.724 INFO:tasks.rados.rados.0.smithi029.stdout:3404: expect (ObjNum 1209 snap 408 seq_num 1209) 2024-04-12T11:28:51.724 INFO:tasks.rados.rados.0.smithi029.stdout:3405: read oid 23 snap 418 2024-04-12T11:28:51.724 INFO:tasks.rados.rados.0.smithi029.stdout:3405: expect (ObjNum 1210 snap 408 seq_num 1210) 2024-04-12T11:28:51.724 INFO:tasks.rados.rados.0.smithi029.stdout:3406: delete oid 47 current snap is 433 2024-04-12T11:28:51.731 INFO:tasks.rados.rados.0.smithi029.stdout:3404: expect (ObjNum 1209 snap 408 seq_num 1209) 2024-04-12T11:28:51.812 INFO:tasks.rados.rados.0.smithi029.stdout:3405: expect (ObjNum 1210 snap 408 seq_num 1210) 2024-04-12T11:28:51.969 INFO:tasks.rados.rados.0.smithi029.stdout:3404: done (2 left) 2024-04-12T11:28:51.969 INFO:tasks.rados.rados.0.smithi029.stdout:3405: done (1 left) 2024-04-12T11:28:51.969 INFO:tasks.rados.rados.0.smithi029.stdout:3406: done (0 left) 2024-04-12T11:28:51.969 INFO:tasks.rados.rados.0.smithi029.stdout:3407: read oid 32 snap 431 2024-04-12T11:28:51.970 INFO:tasks.rados.rados.0.smithi029.stdout:3407: expect (ObjNum 1218 snap 409 seq_num 1218) 2024-04-12T11:28:51.970 INFO:tasks.rados.rados.0.smithi029.stdout:3408: snap_remove snap 421 2024-04-12T11:28:52.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:51 smithi029 ceph-mon[93650]: osdmap e926: 8 total, 8 up, 8 in 2024-04-12T11:28:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:51 smithi029 ceph-mon[97787]: osdmap e926: 8 total, 8 up, 8 in 2024-04-12T11:28:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:51 smithi087 ceph-mon[89014]: osdmap e926: 8 total, 8 up, 8 in 2024-04-12T11:28:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:28:52.683 INFO:tasks.rados.rados.0.smithi029.stdout:3408: done (1 left) 2024-04-12T11:28:52.683 INFO:tasks.rados.rados.0.smithi029.stdout:3409: snap_remove snap 416 2024-04-12T11:28:52.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:28:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:28:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:28:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:52 smithi029 ceph-mon[93650]: pgmap v1026: 89 pgs: 89 active+clean; 304 MiB data, 944 MiB used, 714 GiB / 715 GiB avail; 749 KiB/s rd, 0 B/s wr, 3 op/s 2024-04-12T11:28:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:52 smithi029 ceph-mon[93650]: osdmap e927: 8 total, 8 up, 8 in 2024-04-12T11:28:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:52 smithi029 ceph-mon[93650]: osdmap e928: 8 total, 8 up, 8 in 2024-04-12T11:28:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:52 smithi029 ceph-mon[97787]: pgmap v1026: 89 pgs: 89 active+clean; 304 MiB data, 944 MiB used, 714 GiB / 715 GiB avail; 749 KiB/s rd, 0 B/s wr, 3 op/s 2024-04-12T11:28:53.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:52 smithi029 ceph-mon[97787]: osdmap e927: 8 total, 8 up, 8 in 2024-04-12T11:28:53.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:52 smithi029 ceph-mon[97787]: osdmap e928: 8 total, 8 up, 8 in 2024-04-12T11:28:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:52 smithi087 ceph-mon[89014]: pgmap v1026: 89 pgs: 89 active+clean; 304 MiB data, 944 MiB used, 714 GiB / 715 GiB avail; 749 KiB/s rd, 0 B/s wr, 3 op/s 2024-04-12T11:28:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:52 smithi087 ceph-mon[89014]: osdmap e927: 8 total, 8 up, 8 in 2024-04-12T11:28:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:52 smithi087 ceph-mon[89014]: osdmap e928: 8 total, 8 up, 8 in 2024-04-12T11:28:53.685 INFO:tasks.rados.rados.0.smithi029.stdout:3407: expect (ObjNum 1218 snap 409 seq_num 1218) 2024-04-12T11:28:53.858 INFO:tasks.rados.rados.0.smithi029.stdout:3407: done (1 left) 2024-04-12T11:28:53.858 INFO:tasks.rados.rados.0.smithi029.stdout:3409: done (0 left) 2024-04-12T11:28:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:3410: read oid 32 snap -1 2024-04-12T11:28:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:3410: expect (ObjNum 1218 snap 409 seq_num 1218) 2024-04-12T11:28:53.859 INFO:tasks.rados.rados.0.smithi029.stdout:3411: snap_create 2024-04-12T11:28:53.864 INFO:tasks.rados.rados.0.smithi029.stdout:3410: expect (ObjNum 1218 snap 409 seq_num 1218) 2024-04-12T11:28:54.726 INFO:tasks.rados.rados.0.smithi029.stdout:3410: done (1 left) 2024-04-12T11:28:54.726 INFO:tasks.rados.rados.0.smithi029.stdout:3411: done (0 left) 2024-04-12T11:28:54.726 INFO:tasks.rados.rados.0.smithi029.stdout:3412: read oid 6 snap -1 2024-04-12T11:28:54.726 INFO:tasks.rados.rados.0.smithi029.stdout:3412: expect deleted 2024-04-12T11:28:54.726 INFO:tasks.rados.rados.0.smithi029.stdout:3413: delete oid 45 current snap is 434 2024-04-12T11:28:54.731 INFO:tasks.rados.rados.0.smithi029.stdout:3412: done (1 left) 2024-04-12T11:28:54.731 INFO:tasks.rados.rados.0.smithi029.stdout:3413: done (0 left) 2024-04-12T11:28:54.731 INFO:tasks.rados.rados.0.smithi029.stdout:3414: read oid 44 snap -1 2024-04-12T11:28:54.731 INFO:tasks.rados.rados.0.smithi029.stdout:3414: expect (ObjNum 1215 snap 409 seq_num 1215) 2024-04-12T11:28:54.731 INFO:tasks.rados.rados.0.smithi029.stdout:3415: snap_create 2024-04-12T11:28:54.739 INFO:tasks.rados.rados.0.smithi029.stdout:3414: expect (ObjNum 1215 snap 409 seq_num 1215) 2024-04-12T11:28:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:54 smithi087 ceph-mon[89014]: pgmap v1029: 89 pgs: 89 active+clean; 307 MiB data, 950 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 455 KiB/s wr, 3 op/s 2024-04-12T11:28:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:54 smithi087 ceph-mon[89014]: osdmap e929: 8 total, 8 up, 8 in 2024-04-12T11:28:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:54 smithi029 ceph-mon[97787]: pgmap v1029: 89 pgs: 89 active+clean; 307 MiB data, 950 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 455 KiB/s wr, 3 op/s 2024-04-12T11:28:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:54 smithi029 ceph-mon[97787]: osdmap e929: 8 total, 8 up, 8 in 2024-04-12T11:28:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:54 smithi029 ceph-mon[93650]: pgmap v1029: 89 pgs: 89 active+clean; 307 MiB data, 950 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 455 KiB/s wr, 3 op/s 2024-04-12T11:28:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:54 smithi029 ceph-mon[93650]: osdmap e929: 8 total, 8 up, 8 in 2024-04-12T11:28:55.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:28:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:28:54] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:28:55.731 INFO:tasks.rados.rados.0.smithi029.stdout:3414: done (1 left) 2024-04-12T11:28:55.732 INFO:tasks.rados.rados.0.smithi029.stdout:3415: done (0 left) 2024-04-12T11:28:55.732 INFO:tasks.rados.rados.0.smithi029.stdout:3416: delete oid 8 current snap is 435 2024-04-12T11:28:55.736 INFO:tasks.rados.rados.0.smithi029.stdout:3416: done (0 left) 2024-04-12T11:28:55.736 INFO:tasks.rados.rados.0.smithi029.stdout:3417: delete oid 35 current snap is 435 2024-04-12T11:28:55.743 INFO:tasks.rados.rados.0.smithi029.stdout:3417: done (0 left) 2024-04-12T11:28:55.743 INFO:tasks.rados.rados.0.smithi029.stdout:3418: write oid 45 current snap is 435 2024-04-12T11:28:55.743 INFO:tasks.rados.rados.0.smithi029.stdout:3418: seq_num 1268 ranges {511276=730330,1800344=559437,3021296=289980} 2024-04-12T11:28:55.749 INFO:tasks.rados.rados.0.smithi029.stdout:3418: writing smithi02979277-45 from 511276 to 1241606 tid 1 2024-04-12T11:28:55.756 INFO:tasks.rados.rados.0.smithi029.stdout:3418: writing smithi02979277-45 from 1800344 to 2359781 tid 2 2024-04-12T11:28:55.761 INFO:tasks.rados.rados.0.smithi029.stdout:3418: writing smithi02979277-45 from 3021296 to 3311276 tid 3 2024-04-12T11:28:55.761 INFO:tasks.rados.rados.0.smithi029.stdout:3419: snap_create 2024-04-12T11:28:55.761 INFO:tasks.rados.rados.0.smithi029.stdout:3418: finishing write tid 1 to smithi02979277-45 2024-04-12T11:28:55.761 INFO:tasks.rados.rados.0.smithi029.stdout:3418: finishing write tid 2 to smithi02979277-45 2024-04-12T11:28:55.764 INFO:tasks.rados.rados.0.smithi029.stdout:3418: finishing write tid 3 to smithi02979277-45 2024-04-12T11:28:55.764 INFO:tasks.rados.rados.0.smithi029.stdout:3418: finishing write tid 4 to smithi02979277-45 2024-04-12T11:28:55.765 INFO:tasks.rados.rados.0.smithi029.stdout:3418: finishing write tid 5 to smithi02979277-45 2024-04-12T11:28:55.765 INFO:tasks.rados.rados.0.smithi029.stdout:3418: oid 45 updating version 0 to 1688 2024-04-12T11:28:55.765 INFO:tasks.rados.rados.0.smithi029.stdout:3418: oid 45 version 1688 is already newer than 1686 2024-04-12T11:28:55.765 INFO:tasks.rados.rados.0.smithi029.stdout:3418: oid 45 version 1688 is already newer than 1685 2024-04-12T11:28:55.765 INFO:tasks.rados.rados.0.smithi029.stdout:3418: oid 45 version 1688 is already newer than 1687 2024-04-12T11:28:55.765 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1688 (ObjNum 1268 snap 435 seq_num 1268) dirty exists 2024-04-12T11:28:55.765 INFO:tasks.rados.rados.0.smithi029.stdout:3418: left oid 45 (ObjNum 1268 snap 435 seq_num 1268) 2024-04-12T11:28:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:55 smithi029 ceph-mon[93650]: osdmap e930: 8 total, 8 up, 8 in 2024-04-12T11:28:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:55 smithi029 ceph-mon[97787]: osdmap e930: 8 total, 8 up, 8 in 2024-04-12T11:28:56.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:55 smithi087 ceph-mon[89014]: osdmap e930: 8 total, 8 up, 8 in 2024-04-12T11:28:56.736 INFO:tasks.rados.rados.0.smithi029.stdout:3418: done (1 left) 2024-04-12T11:28:56.737 INFO:tasks.rados.rados.0.smithi029.stdout:3419: done (0 left) 2024-04-12T11:28:56.737 INFO:tasks.rados.rados.0.smithi029.stdout:3420: write oid 37 current snap is 436 2024-04-12T11:28:56.737 INFO:tasks.rados.rados.0.smithi029.stdout:3420: seq_num 1269 ranges {528083=405579,1342473=419448,2248719=733949} 2024-04-12T11:28:56.742 INFO:tasks.rados.rados.0.smithi029.stdout:3420: writing smithi02979277-37 from 528083 to 933662 tid 1 2024-04-12T11:28:56.746 INFO:tasks.rados.rados.0.smithi029.stdout:3420: writing smithi02979277-37 from 1342473 to 1761921 tid 2 2024-04-12T11:28:56.754 INFO:tasks.rados.rados.0.smithi029.stdout:3420: writing smithi02979277-37 from 2248719 to 2982668 tid 3 2024-04-12T11:28:56.754 INFO:tasks.rados.rados.0.smithi029.stdout:3421: write oid 25 current snap is 436 2024-04-12T11:28:56.754 INFO:tasks.rados.rados.0.smithi029.stdout:3420: finishing write tid 1 to smithi02979277-37 2024-04-12T11:28:56.754 INFO:tasks.rados.rados.0.smithi029.stdout:3420: finishing write tid 2 to smithi02979277-37 2024-04-12T11:28:56.754 INFO:tasks.rados.rados.0.smithi029.stdout:3421: seq_num 1270 ranges {544890=597181,1917308=563106,3076142=268748} 2024-04-12T11:28:56.760 INFO:tasks.rados.rados.0.smithi029.stdout:3421: writing smithi02979277-25 from 544890 to 1142071 tid 1 2024-04-12T11:28:56.767 INFO:tasks.rados.rados.0.smithi029.stdout:3421: writing smithi02979277-25 from 1917308 to 2480414 tid 2 2024-04-12T11:28:56.772 INFO:tasks.rados.rados.0.smithi029.stdout:3421: writing smithi02979277-25 from 3076142 to 3344890 tid 3 2024-04-12T11:28:56.772 INFO:tasks.rados.rados.0.smithi029.stdout:3422: rollback oid 18 current snap is 436 2024-04-12T11:28:56.772 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 18 to 430 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3420: finishing write tid 3 to smithi02979277-37 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3423: read oid 21 snap -1 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3423: expect (ObjNum 1252 snap 425 seq_num 1252) 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3424: write oid 16 current snap is 436 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3420: finishing write tid 4 to smithi02979277-37 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3420: finishing write tid 5 to smithi02979277-37 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3420: oid 37 updating version 0 to 1232 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3420: oid 37 version 1232 is already newer than 1231 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3420: oid 37 version 1232 is already newer than 1230 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3420: oid 37 updating version 1232 to 1233 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 1233 (ObjNum 1269 snap 436 seq_num 1269) dirty exists 2024-04-12T11:28:56.773 INFO:tasks.rados.rados.0.smithi029.stdout:3420: left oid 37 (ObjNum 1269 snap 436 seq_num 1269) 2024-04-12T11:28:56.774 INFO:tasks.rados.rados.0.smithi029.stdout:3421: finishing write tid 1 to smithi02979277-25 2024-04-12T11:28:56.774 INFO:tasks.rados.rados.0.smithi029.stdout:3421: finishing write tid 2 to smithi02979277-25 2024-04-12T11:28:56.774 INFO:tasks.rados.rados.0.smithi029.stdout:3424: seq_num 1271 ranges {561697=672430,1975790=706764,3103565=258132} 2024-04-12T11:28:56.779 INFO:tasks.rados.rados.0.smithi029.stdout:3424: writing smithi02979277-16 from 561697 to 1234127 tid 1 2024-04-12T11:28:56.787 INFO:tasks.rados.rados.0.smithi029.stdout:3424: writing smithi02979277-16 from 1975790 to 2682554 tid 2 2024-04-12T11:28:56.791 INFO:tasks.rados.rados.0.smithi029.stdout:3424: writing smithi02979277-16 from 3103565 to 3361697 tid 3 2024-04-12T11:28:56.791 INFO:tasks.rados.rados.0.smithi029.stdout:3420: done (4 left) 2024-04-12T11:28:56.791 INFO:tasks.rados.rados.0.smithi029.stdout:3425: write oid 5 current snap is 436 2024-04-12T11:28:56.791 INFO:tasks.rados.rados.0.smithi029.stdout:3425: seq_num 1272 ranges {578504=747679,2034272=566775,3130988=247516} 2024-04-12T11:28:56.798 INFO:tasks.rados.rados.0.smithi029.stdout:3425: writing smithi02979277-5 from 578504 to 1326183 tid 1 2024-04-12T11:28:56.805 INFO:tasks.rados.rados.0.smithi029.stdout:3425: writing smithi02979277-5 from 2034272 to 2601047 tid 2 2024-04-12T11:28:56.810 INFO:tasks.rados.rados.0.smithi029.stdout:3425: writing smithi02979277-5 from 3130988 to 3378504 tid 3 2024-04-12T11:28:56.810 INFO:tasks.rados.rados.0.smithi029.stdout:3426: rollback oid 38 current snap is 436 2024-04-12T11:28:56.810 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 38 to 357 2024-04-12T11:28:56.810 INFO:tasks.rados.rados.0.smithi029.stdout:3421: finishing write tid 3 to smithi02979277-25 2024-04-12T11:28:56.810 INFO:tasks.rados.rados.0.smithi029.stdout:3421: finishing write tid 4 to smithi02979277-25 2024-04-12T11:28:56.810 INFO:tasks.rados.rados.0.smithi029.stdout:3421: finishing write tid 5 to smithi02979277-25 2024-04-12T11:28:56.810 INFO:tasks.rados.rados.0.smithi029.stdout:3421: oid 25 updating version 0 to 1756 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:3421: oid 25 version 1756 is already newer than 1753 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:3421: oid 25 version 1756 is already newer than 1755 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:3421: oid 25 version 1756 is already newer than 1754 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 1756 (ObjNum 1270 snap 436 seq_num 1270) dirty exists 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:3421: left oid 25 (ObjNum 1270 snap 436 seq_num 1270) 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:3421: done (5 left) 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:3427: read oid 43 snap -1 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:3427: expect (ObjNum 1089 snap 365 seq_num 1089) 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:3428: delete oid 46 current snap is 436 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:3422: finishing rollback tid 0 to smithi02979277-18 2024-04-12T11:28:56.811 INFO:tasks.rados.rados.0.smithi029.stdout:3423: expect (ObjNum 1252 snap 425 seq_num 1252) 2024-04-12T11:28:56.915 INFO:tasks.rados.rados.0.smithi029.stdout:3422: finishing rollback tid 1 to smithi02979277-18 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:3422: finishing rollback tid 2 to smithi02979277-18 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1760 (ObjNum 1198 snap 406 seq_num 1198) dirty exists 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:3424: finishing write tid 1 to smithi02979277-16 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:3424: finishing write tid 2 to smithi02979277-16 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:3422: done (6 left) 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:3423: done (5 left) 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:3428: done (4 left) 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:3429: write oid 50 current snap is 436 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:3424: finishing write tid 3 to smithi02979277-16 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:3424: finishing write tid 4 to smithi02979277-16 2024-04-12T11:28:56.916 INFO:tasks.rados.rados.0.smithi029.stdout:3429: seq_num 1274 ranges {612118=498177,1634883=570444,2669481=706662} 2024-04-12T11:28:56.921 INFO:tasks.rados.rados.0.smithi029.stdout:3429: writing smithi02979277-50 from 612118 to 1110295 tid 1 2024-04-12T11:28:56.927 INFO:tasks.rados.rados.0.smithi029.stdout:3429: writing smithi02979277-50 from 1634883 to 2205327 tid 2 2024-04-12T11:28:56.934 INFO:tasks.rados.rados.0.smithi029.stdout:3429: writing smithi02979277-50 from 2669481 to 3376143 tid 3 2024-04-12T11:28:56.934 INFO:tasks.rados.rados.0.smithi029.stdout:3430: read oid 45 snap 357 2024-04-12T11:28:56.934 INFO:tasks.rados.rados.0.smithi029.stdout:3430: expect (ObjNum 962 snap 330 seq_num 962) 2024-04-12T11:28:56.934 INFO:tasks.rados.rados.0.smithi029.stdout:3431: write oid 47 current snap is 436 2024-04-12T11:28:56.934 INFO:tasks.rados.rados.0.smithi029.stdout:3424: finishing write tid 5 to smithi02979277-16 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:3424: oid 16 updating version 0 to 1577 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:3424: oid 16 updating version 1577 to 1579 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:3424: oid 16 version 1579 is already newer than 1578 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:3424: oid 16 updating version 1579 to 1580 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1580 (ObjNum 1271 snap 436 seq_num 1271) dirty exists 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:3424: left oid 16 (ObjNum 1271 snap 436 seq_num 1271) 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:3425: finishing write tid 1 to smithi02979277-5 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:3425: finishing write tid 2 to smithi02979277-5 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:3425: finishing write tid 3 to smithi02979277-5 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:3425: finishing write tid 4 to smithi02979277-5 2024-04-12T11:28:56.935 INFO:tasks.rados.rados.0.smithi029.stdout:3431: seq_num 1275 ranges {628925=573426,1693365=714102,3096904=332021} 2024-04-12T11:28:56.940 INFO:tasks.rados.rados.0.smithi029.stdout:3431: writing smithi02979277-47 from 628925 to 1202351 tid 1 2024-04-12T11:28:56.947 INFO:tasks.rados.rados.0.smithi029.stdout:3431: writing smithi02979277-47 from 1693365 to 2407467 tid 2 2024-04-12T11:28:56.953 INFO:tasks.rados.rados.0.smithi029.stdout:3431: writing smithi02979277-47 from 3096904 to 3428925 tid 3 2024-04-12T11:28:56.953 INFO:tasks.rados.rados.0.smithi029.stdout:3424: done (6 left) 2024-04-12T11:28:56.953 INFO:tasks.rados.rados.0.smithi029.stdout:3432: snap_remove snap 427 2024-04-12T11:28:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:56 smithi029 ceph-mon[93650]: pgmap v1032: 89 pgs: 89 active+clean; 307 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 455 KiB/s wr, 11 op/s 2024-04-12T11:28:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:56 smithi029 ceph-mon[93650]: osdmap e931: 8 total, 8 up, 8 in 2024-04-12T11:28:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:56 smithi029 ceph-mon[97787]: pgmap v1032: 89 pgs: 89 active+clean; 307 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 455 KiB/s wr, 11 op/s 2024-04-12T11:28:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:56 smithi029 ceph-mon[97787]: osdmap e931: 8 total, 8 up, 8 in 2024-04-12T11:28:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:56 smithi087 ceph-mon[89014]: pgmap v1032: 89 pgs: 89 active+clean; 307 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 455 KiB/s wr, 11 op/s 2024-04-12T11:28:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:56 smithi087 ceph-mon[89014]: osdmap e931: 8 total, 8 up, 8 in 2024-04-12T11:28:57.743 INFO:tasks.rados.rados.0.smithi029.stdout:3432: done (6 left) 2024-04-12T11:28:57.743 INFO:tasks.rados.rados.0.smithi029.stdout:3433: write oid 4 current snap is 436 2024-04-12T11:28:57.743 INFO:tasks.rados.rados.0.smithi029.stdout:3433: seq_num 1276 ranges {645732=648675,1751847=574113,3124327=321405} 2024-04-12T11:28:57.750 INFO:tasks.rados.rados.0.smithi029.stdout:3433: writing smithi02979277-4 from 645732 to 1294407 tid 1 2024-04-12T11:28:57.756 INFO:tasks.rados.rados.0.smithi029.stdout:3433: writing smithi02979277-4 from 1751847 to 2325960 tid 2 2024-04-12T11:28:57.762 INFO:tasks.rados.rados.0.smithi029.stdout:3433: writing smithi02979277-4 from 3124327 to 3445732 tid 3 2024-04-12T11:28:57.762 INFO:tasks.rados.rados.0.smithi029.stdout:3434: snap_create 2024-04-12T11:28:57.762 INFO:tasks.rados.rados.0.smithi029.stdout:3425: finishing write tid 5 to smithi02979277-5 2024-04-12T11:28:57.762 INFO:tasks.rados.rados.0.smithi029.stdout:3425: oid 5 updating version 0 to 1582 2024-04-12T11:28:57.763 INFO:tasks.rados.rados.0.smithi029.stdout:3425: oid 5 updating version 1582 to 1584 2024-04-12T11:28:57.763 INFO:tasks.rados.rados.0.smithi029.stdout:3425: oid 5 version 1584 is already newer than 1583 2024-04-12T11:28:57.763 INFO:tasks.rados.rados.0.smithi029.stdout:3425: oid 5 updating version 1584 to 1585 2024-04-12T11:28:57.763 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1585 (ObjNum 1272 snap 436 seq_num 1272) dirty exists 2024-04-12T11:28:57.763 INFO:tasks.rados.rados.0.smithi029.stdout:3425: left oid 5 (ObjNum 1272 snap 436 seq_num 1272) 2024-04-12T11:28:57.763 INFO:tasks.rados.rados.0.smithi029.stdout:3427: expect (ObjNum 1089 snap 365 seq_num 1089) 2024-04-12T11:28:57.842 INFO:tasks.rados.rados.0.smithi029.stdout:3426: finishing rollback tid 0 to smithi02979277-38 2024-04-12T11:28:57.842 INFO:tasks.rados.rados.0.smithi029.stdout:3426: finishing rollback tid 1 to smithi02979277-38 2024-04-12T11:28:57.842 INFO:tasks.rados.rados.0.smithi029.stdout:3426: finishing rollback tid 2 to smithi02979277-38 2024-04-12T11:28:57.842 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 1117 (ObjNum 995 snap 345 seq_num 995) dirty exists 2024-04-12T11:28:57.842 INFO:tasks.rados.rados.0.smithi029.stdout:3429: finishing write tid 1 to smithi02979277-50 2024-04-12T11:28:57.842 INFO:tasks.rados.rados.0.smithi029.stdout:3429: finishing write tid 2 to smithi02979277-50 2024-04-12T11:28:57.842 INFO:tasks.rados.rados.0.smithi029.stdout:3430: expect (ObjNum 962 snap 330 seq_num 962) 2024-04-12T11:28:57.915 INFO:tasks.rados.rados.0.smithi029.stdout:3429: finishing write tid 3 to smithi02979277-50 2024-04-12T11:28:57.916 INFO:tasks.rados.rados.0.smithi029.stdout:3429: finishing write tid 4 to smithi02979277-50 2024-04-12T11:28:57.916 INFO:tasks.rados.rados.0.smithi029.stdout:3429: finishing write tid 5 to smithi02979277-50 2024-04-12T11:28:57.916 INFO:tasks.rados.rados.0.smithi029.stdout:3429: oid 50 updating version 0 to 1452 2024-04-12T11:28:57.916 INFO:tasks.rados.rados.0.smithi029.stdout:3429: oid 50 updating version 1452 to 1453 2024-04-12T11:28:57.916 INFO:tasks.rados.rados.0.smithi029.stdout:3429: oid 50 updating version 1453 to 1454 2024-04-12T11:28:57.916 INFO:tasks.rados.rados.0.smithi029.stdout:3429: oid 50 version 1454 is already newer than 1451 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1454 (ObjNum 1274 snap 436 seq_num 1274) dirty exists 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3429: left oid 50 (ObjNum 1274 snap 436 seq_num 1274) 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3431: finishing write tid 1 to smithi02979277-47 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3431: finishing write tid 2 to smithi02979277-47 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3431: finishing write tid 3 to smithi02979277-47 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3431: finishing write tid 4 to smithi02979277-47 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3431: finishing write tid 5 to smithi02979277-47 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3431: oid 47 updating version 0 to 1689 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3431: oid 47 updating version 1689 to 1690 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3431: oid 47 updating version 1690 to 1692 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3431: oid 47 version 1692 is already newer than 1691 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1692 (ObjNum 1275 snap 436 seq_num 1275) dirty exists 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3431: left oid 47 (ObjNum 1275 snap 436 seq_num 1275) 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3433: finishing write tid 1 to smithi02979277-4 2024-04-12T11:28:57.917 INFO:tasks.rados.rados.0.smithi029.stdout:3433: finishing write tid 2 to smithi02979277-4 2024-04-12T11:28:57.918 INFO:tasks.rados.rados.0.smithi029.stdout:3433: finishing write tid 3 to smithi02979277-4 2024-04-12T11:28:57.918 INFO:tasks.rados.rados.0.smithi029.stdout:3433: finishing write tid 4 to smithi02979277-4 2024-04-12T11:28:57.918 INFO:tasks.rados.rados.0.smithi029.stdout:3433: finishing write tid 5 to smithi02979277-4 2024-04-12T11:28:57.918 INFO:tasks.rados.rados.0.smithi029.stdout:3433: oid 4 updating version 0 to 1591 2024-04-12T11:28:57.918 INFO:tasks.rados.rados.0.smithi029.stdout:3433: oid 4 updating version 1591 to 1594 2024-04-12T11:28:57.918 INFO:tasks.rados.rados.0.smithi029.stdout:3433: oid 4 version 1594 is already newer than 1592 2024-04-12T11:28:57.918 INFO:tasks.rados.rados.0.smithi029.stdout:3433: oid 4 version 1594 is already newer than 1593 2024-04-12T11:28:57.918 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1594 (ObjNum 1276 snap 436 seq_num 1276) dirty exists 2024-04-12T11:28:57.918 INFO:tasks.rados.rados.0.smithi029.stdout:3433: left oid 4 (ObjNum 1276 snap 436 seq_num 1276) 2024-04-12T11:28:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:57 smithi029 ceph-mon[97787]: osdmap e932: 8 total, 8 up, 8 in 2024-04-12T11:28:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:57 smithi029 ceph-mon[93650]: osdmap e932: 8 total, 8 up, 8 in 2024-04-12T11:28:58.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:57 smithi087 ceph-mon[89014]: osdmap e932: 8 total, 8 up, 8 in 2024-04-12T11:28:58.748 INFO:tasks.rados.rados.0.smithi029.stdout:3425: done (7 left) 2024-04-12T11:28:58.749 INFO:tasks.rados.rados.0.smithi029.stdout:3426: done (6 left) 2024-04-12T11:28:58.749 INFO:tasks.rados.rados.0.smithi029.stdout:3427: done (5 left) 2024-04-12T11:28:58.749 INFO:tasks.rados.rados.0.smithi029.stdout:3429: done (4 left) 2024-04-12T11:28:58.749 INFO:tasks.rados.rados.0.smithi029.stdout:3430: done (3 left) 2024-04-12T11:28:58.749 INFO:tasks.rados.rados.0.smithi029.stdout:3431: done (2 left) 2024-04-12T11:28:58.749 INFO:tasks.rados.rados.0.smithi029.stdout:3433: done (1 left) 2024-04-12T11:28:58.749 INFO:tasks.rados.rados.0.smithi029.stdout:3434: done (0 left) 2024-04-12T11:28:58.749 INFO:tasks.rados.rados.0.smithi029.stdout:3435: write oid 16 current snap is 437 2024-04-12T11:28:58.749 INFO:tasks.rados.rados.0.smithi029.stdout:3435: seq_num 1277 ranges {662539=723924,2093976=717771,3435397=27142} 2024-04-12T11:28:58.756 INFO:tasks.rados.rados.0.smithi029.stdout:3435: writing smithi02979277-16 from 662539 to 1386463 tid 1 2024-04-12T11:28:58.764 INFO:tasks.rados.rados.0.smithi029.stdout:3435: writing smithi02979277-16 from 2093976 to 2811747 tid 2 2024-04-12T11:28:58.768 INFO:tasks.rados.rados.0.smithi029.stdout:3435: writing smithi02979277-16 from 3435397 to 3462539 tid 3 2024-04-12T11:28:58.768 INFO:tasks.rados.rados.0.smithi029.stdout:3436: write oid 24 current snap is 437 2024-04-12T11:28:58.768 INFO:tasks.rados.rados.0.smithi029.stdout:3435: finishing write tid 1 to smithi02979277-16 2024-04-12T11:28:58.768 INFO:tasks.rados.rados.0.smithi029.stdout:3436: seq_num 1278 ranges {679346=515526,1868811=577782,3179173=300173} 2024-04-12T11:28:58.775 INFO:tasks.rados.rados.0.smithi029.stdout:3436: writing smithi02979277-24 from 679346 to 1194872 tid 1 2024-04-12T11:28:58.782 INFO:tasks.rados.rados.0.smithi029.stdout:3436: writing smithi02979277-24 from 1868811 to 2446593 tid 2 2024-04-12T11:28:58.788 INFO:tasks.rados.rados.0.smithi029.stdout:3436: writing smithi02979277-24 from 3179173 to 3479346 tid 3 2024-04-12T11:28:58.788 INFO:tasks.rados.rados.0.smithi029.stdout:3437: rollback oid 35 current snap is 437 2024-04-12T11:28:58.788 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 35 to 429 2024-04-12T11:28:58.788 INFO:tasks.rados.rados.0.smithi029.stdout:3435: finishing write tid 2 to smithi02979277-16 2024-04-12T11:28:58.788 INFO:tasks.rados.rados.0.smithi029.stdout:3435: finishing write tid 3 to smithi02979277-16 2024-04-12T11:28:58.788 INFO:tasks.rados.rados.0.smithi029.stdout:3435: finishing write tid 4 to smithi02979277-16 2024-04-12T11:28:58.788 INFO:tasks.rados.rados.0.smithi029.stdout:3435: finishing write tid 5 to smithi02979277-16 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:3435: oid 16 updating version 0 to 1598 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:3435: oid 16 updating version 1598 to 1599 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:3435: oid 16 version 1599 is already newer than 1596 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:3435: oid 16 version 1599 is already newer than 1597 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1599 (ObjNum 1277 snap 437 seq_num 1277) dirty exists 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:3435: left oid 16 (ObjNum 1277 snap 437 seq_num 1277) 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:3436: finishing write tid 1 to smithi02979277-24 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:3436: finishing write tid 2 to smithi02979277-24 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:3435: done (2 left) 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:3438: write oid 45 current snap is 437 2024-04-12T11:28:58.789 INFO:tasks.rados.rados.0.smithi029.stdout:3438: seq_num 1279 ranges {696153=590775,1927293=721440,3206596=289557} 2024-04-12T11:28:58.795 INFO:tasks.rados.rados.0.smithi029.stdout:3438: writing smithi02979277-45 from 696153 to 1286928 tid 1 2024-04-12T11:28:58.802 INFO:tasks.rados.rados.0.smithi029.stdout:3438: writing smithi02979277-45 from 1927293 to 2648733 tid 2 2024-04-12T11:28:58.807 INFO:tasks.rados.rados.0.smithi029.stdout:3438: writing smithi02979277-45 from 3206596 to 3496153 tid 3 2024-04-12T11:28:58.807 INFO:tasks.rados.rados.0.smithi029.stdout:3439: snap_create 2024-04-12T11:28:58.807 INFO:tasks.rados.rados.0.smithi029.stdout:3436: finishing write tid 3 to smithi02979277-24 2024-04-12T11:28:58.807 INFO:tasks.rados.rados.0.smithi029.stdout:3436: finishing write tid 4 to smithi02979277-24 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:3436: finishing write tid 5 to smithi02979277-24 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:3436: oid 24 updating version 0 to 2163 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:3436: oid 24 updating version 2163 to 2164 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:3436: oid 24 version 2164 is already newer than 2162 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:3436: oid 24 version 2164 is already newer than 2161 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 2164 (ObjNum 1278 snap 437 seq_num 1278) dirty exists 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:3436: left oid 24 (ObjNum 1278 snap 437 seq_num 1278) 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:3437: finishing rollback tid 1 to smithi02979277-35 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 2091 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:3438: finishing write tid 1 to smithi02979277-45 2024-04-12T11:28:58.808 INFO:tasks.rados.rados.0.smithi029.stdout:3438: finishing write tid 2 to smithi02979277-45 2024-04-12T11:28:58.811 INFO:tasks.rados.rados.0.smithi029.stdout:3438: finishing write tid 3 to smithi02979277-45 2024-04-12T11:28:58.811 INFO:tasks.rados.rados.0.smithi029.stdout:3438: finishing write tid 4 to smithi02979277-45 2024-04-12T11:28:58.811 INFO:tasks.rados.rados.0.smithi029.stdout:3438: finishing write tid 5 to smithi02979277-45 2024-04-12T11:28:58.811 INFO:tasks.rados.rados.0.smithi029.stdout:3438: oid 45 updating version 0 to 1695 2024-04-12T11:28:58.811 INFO:tasks.rados.rados.0.smithi029.stdout:3438: oid 45 updating version 1695 to 1697 2024-04-12T11:28:58.811 INFO:tasks.rados.rados.0.smithi029.stdout:3438: oid 45 version 1697 is already newer than 1694 2024-04-12T11:28:58.811 INFO:tasks.rados.rados.0.smithi029.stdout:3438: oid 45 version 1697 is already newer than 1696 2024-04-12T11:28:58.811 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1697 (ObjNum 1279 snap 437 seq_num 1279) dirty exists 2024-04-12T11:28:58.811 INFO:tasks.rados.rados.0.smithi029.stdout:3438: left oid 45 (ObjNum 1279 snap 437 seq_num 1279) 2024-04-12T11:28:59.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:58 smithi029 ceph-mon[93650]: pgmap v1035: 89 pgs: 89 active+clean; 314 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-04-12T11:28:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:58 smithi029 ceph-mon[93650]: osdmap e933: 8 total, 8 up, 8 in 2024-04-12T11:28:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:58 smithi029 ceph-mon[97787]: pgmap v1035: 89 pgs: 89 active+clean; 314 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-04-12T11:28:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:58 smithi029 ceph-mon[97787]: osdmap e933: 8 total, 8 up, 8 in 2024-04-12T11:28:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:58 smithi087 ceph-mon[89014]: pgmap v1035: 89 pgs: 89 active+clean; 314 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-04-12T11:28:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:58 smithi087 ceph-mon[89014]: osdmap e933: 8 total, 8 up, 8 in 2024-04-12T11:28:59.753 INFO:tasks.rados.rados.0.smithi029.stdout:3436: done (3 left) 2024-04-12T11:28:59.753 INFO:tasks.rados.rados.0.smithi029.stdout:3437: done (2 left) 2024-04-12T11:28:59.753 INFO:tasks.rados.rados.0.smithi029.stdout:3438: done (1 left) 2024-04-12T11:28:59.753 INFO:tasks.rados.rados.0.smithi029.stdout:3439: done (0 left) 2024-04-12T11:28:59.753 INFO:tasks.rados.rados.0.smithi029.stdout:3440: snap_remove snap 357 2024-04-12T11:29:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:59 smithi029 ceph-mon[97787]: osdmap e934: 8 total, 8 up, 8 in 2024-04-12T11:29:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:28:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:29:00.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:59 smithi029 ceph-mon[93650]: osdmap e934: 8 total, 8 up, 8 in 2024-04-12T11:29:00.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:28:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:29:00.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:59 smithi087 ceph-mon[89014]: osdmap e934: 8 total, 8 up, 8 in 2024-04-12T11:29:00.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:28:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:29:00.756 INFO:tasks.rados.rados.0.smithi029.stdout:3440: done (0 left) 2024-04-12T11:29:00.756 INFO:tasks.rados.rados.0.smithi029.stdout:3441: read oid 31 snap 429 2024-04-12T11:29:00.756 INFO:tasks.rados.rados.0.smithi029.stdout:3441: expect deleted 2024-04-12T11:29:00.756 INFO:tasks.rados.rados.0.smithi029.stdout:3442: write oid 25 current snap is 438 2024-04-12T11:29:00.756 INFO:tasks.rados.rados.0.smithi029.stdout:3442: seq_num 1280 ranges {712960=666024,1985775=581451,3234019=278941} 2024-04-12T11:29:00.763 INFO:tasks.rados.rados.0.smithi029.stdout:3442: writing smithi02979277-25 from 712960 to 1378984 tid 1 2024-04-12T11:29:00.771 INFO:tasks.rados.rados.0.smithi029.stdout:3442: writing smithi02979277-25 from 1985775 to 2567226 tid 2 2024-04-12T11:29:00.776 INFO:tasks.rados.rados.0.smithi029.stdout:3442: writing smithi02979277-25 from 3234019 to 3512960 tid 3 2024-04-12T11:29:00.776 INFO:tasks.rados.rados.0.smithi029.stdout:3443: snap_remove snap 431 2024-04-12T11:29:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:00 smithi029 ceph-mon[97787]: pgmap v1038: 89 pgs: 89 active+clean; 326 MiB data, 973 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.7 MiB/s wr, 22 op/s 2024-04-12T11:29:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:00 smithi029 ceph-mon[97787]: osdmap e935: 8 total, 8 up, 8 in 2024-04-12T11:29:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:00 smithi029 ceph-mon[93650]: pgmap v1038: 89 pgs: 89 active+clean; 326 MiB data, 973 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.7 MiB/s wr, 22 op/s 2024-04-12T11:29:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:00 smithi029 ceph-mon[93650]: osdmap e935: 8 total, 8 up, 8 in 2024-04-12T11:29:01.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:00 smithi087 ceph-mon[89014]: pgmap v1038: 89 pgs: 89 active+clean; 326 MiB data, 973 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.7 MiB/s wr, 22 op/s 2024-04-12T11:29:01.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:00 smithi087 ceph-mon[89014]: osdmap e935: 8 total, 8 up, 8 in 2024-04-12T11:29:01.763 INFO:tasks.rados.rados.0.smithi029.stdout:3443: done (2 left) 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3444: read oid 21 snap -1 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3444: expect (ObjNum 1252 snap 425 seq_num 1252) 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3445: snap_create 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3442: finishing write tid 1 to smithi02979277-25 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3442: finishing write tid 2 to smithi02979277-25 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3442: finishing write tid 3 to smithi02979277-25 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3442: finishing write tid 4 to smithi02979277-25 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3442: finishing write tid 5 to smithi02979277-25 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3442: oid 25 updating version 0 to 1766 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3442: oid 25 updating version 1766 to 1777 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3442: oid 25 version 1777 is already newer than 1771 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3442: oid 25 version 1777 is already newer than 1776 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 1777 (ObjNum 1280 snap 438 seq_num 1280) dirty exists 2024-04-12T11:29:01.764 INFO:tasks.rados.rados.0.smithi029.stdout:3442: left oid 25 (ObjNum 1280 snap 438 seq_num 1280) 2024-04-12T11:29:01.776 INFO:tasks.rados.rados.0.smithi029.stdout:3444: expect (ObjNum 1252 snap 425 seq_num 1252) 2024-04-12T11:29:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:01 smithi029 ceph-mon[97787]: osdmap e936: 8 total, 8 up, 8 in 2024-04-12T11:29:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:01 smithi029 ceph-mon[93650]: osdmap e936: 8 total, 8 up, 8 in 2024-04-12T11:29:02.068 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:01 smithi087 ceph-mon[89014]: osdmap e936: 8 total, 8 up, 8 in 2024-04-12T11:29:02.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:29:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:29:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:29:02.699 INFO:tasks.rados.rados.0.smithi029.stdout:3441: done (3 left) 2024-04-12T11:29:02.700 INFO:tasks.rados.rados.0.smithi029.stdout:3442: done (2 left) 2024-04-12T11:29:02.700 INFO:tasks.rados.rados.0.smithi029.stdout:3444: done (1 left) 2024-04-12T11:29:02.700 INFO:tasks.rados.rados.0.smithi029.stdout:3445: done (0 left) 2024-04-12T11:29:02.700 INFO:tasks.rados.rados.0.smithi029.stdout:3446: snap_remove snap 426 2024-04-12T11:29:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:02 smithi087 ceph-mon[89014]: pgmap v1041: 89 pgs: 89 active+clean; 326 MiB data, 976 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:29:02.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:02 smithi087 ceph-mon[89014]: osdmap e937: 8 total, 8 up, 8 in 2024-04-12T11:29:02.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:02 smithi087 ceph-mon[89014]: osdmap e938: 8 total, 8 up, 8 in 2024-04-12T11:29:03.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:02 smithi029 ceph-mon[97787]: pgmap v1041: 89 pgs: 89 active+clean; 326 MiB data, 976 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:29:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:02 smithi029 ceph-mon[97787]: osdmap e937: 8 total, 8 up, 8 in 2024-04-12T11:29:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:02 smithi029 ceph-mon[97787]: osdmap e938: 8 total, 8 up, 8 in 2024-04-12T11:29:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:02 smithi029 ceph-mon[93650]: pgmap v1041: 89 pgs: 89 active+clean; 326 MiB data, 976 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-04-12T11:29:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:02 smithi029 ceph-mon[93650]: osdmap e937: 8 total, 8 up, 8 in 2024-04-12T11:29:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:02 smithi029 ceph-mon[93650]: osdmap e938: 8 total, 8 up, 8 in 2024-04-12T11:29:03.702 INFO:tasks.rados.rados.0.smithi029.stdout:3446: done (0 left) 2024-04-12T11:29:03.702 INFO:tasks.rados.rados.0.smithi029.stdout:3447: rollback oid 11 current snap is 439 2024-04-12T11:29:03.702 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 11 to 428 2024-04-12T11:29:03.702 INFO:tasks.rados.rados.0.smithi029.stdout:3448: snap_remove snap 398 2024-04-12T11:29:04.702 INFO:tasks.rados.rados.0.smithi029.stdout:3448: done (1 left) 2024-04-12T11:29:04.702 INFO:tasks.rados.rados.0.smithi029.stdout:3449: rollback oid 47 current snap is 439 2024-04-12T11:29:04.702 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 433 2024-04-12T11:29:04.702 INFO:tasks.rados.rados.0.smithi029.stdout:3447: finishing rollback tid 0 to smithi02979277-11 2024-04-12T11:29:04.702 INFO:tasks.rados.rados.0.smithi029.stdout:3447: finishing rollback tid 1 to smithi02979277-11 2024-04-12T11:29:04.703 INFO:tasks.rados.rados.0.smithi029.stdout:3447: finishing rollback tid 2 to smithi02979277-11 2024-04-12T11:29:04.703 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 2181 (ObjNum 1048 snap 357 seq_num 1048) dirty exists 2024-04-12T11:29:04.703 INFO:tasks.rados.rados.0.smithi029.stdout:3447: done (1 left) 2024-04-12T11:29:04.703 INFO:tasks.rados.rados.0.smithi029.stdout:3450: delete oid 2 current snap is 439 2024-04-12T11:29:04.712 INFO:tasks.rados.rados.0.smithi029.stdout:3450: done (1 left) 2024-04-12T11:29:04.712 INFO:tasks.rados.rados.0.smithi029.stdout:3451: snap_create 2024-04-12T11:29:04.715 INFO:tasks.rados.rados.0.smithi029.stdout:3449: finishing rollback tid 0 to smithi02979277-47 2024-04-12T11:29:04.716 INFO:tasks.rados.rados.0.smithi029.stdout:3449: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:29:04.716 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1708 (ObjNum 33 snap 0 seq_num 1801223040) dirty dne 2024-04-12T11:29:04.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:04 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:04 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:04 smithi087 ceph-mon[89014]: pgmap v1044: 89 pgs: 89 active+clean; 296 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-04-12T11:29:04.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:04 smithi087 ceph-mon[89014]: osdmap e939: 8 total, 8 up, 8 in 2024-04-12T11:29:04.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:04 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:04 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[93650]: pgmap v1044: 89 pgs: 89 active+clean; 296 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-04-12T11:29:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[93650]: osdmap e939: 8 total, 8 up, 8 in 2024-04-12T11:29:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[97787]: pgmap v1044: 89 pgs: 89 active+clean; 296 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2024-04-12T11:29:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[97787]: osdmap e939: 8 total, 8 up, 8 in 2024-04-12T11:29:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:29:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:29:04] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:29:05.704 INFO:tasks.rados.rados.0.smithi029.stdout:3449: done (1 left) 2024-04-12T11:29:05.704 INFO:tasks.rados.rados.0.smithi029.stdout:3451: done (0 left) 2024-04-12T11:29:05.704 INFO:tasks.rados.rados.0.smithi029.stdout:3452: snap_remove snap 425 2024-04-12T11:29:05.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:05 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:29:05.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:05 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:29:05.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:05 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:05.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:05 smithi087 ceph-mon[89014]: osdmap e940: 8 total, 8 up, 8 in 2024-04-12T11:29:05.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:29:05.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:29:05.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:05.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:05 smithi029 ceph-mon[93650]: osdmap e940: 8 total, 8 up, 8 in 2024-04-12T11:29:05.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:29:05.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:29:05.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:29:05.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:05 smithi029 ceph-mon[97787]: osdmap e940: 8 total, 8 up, 8 in 2024-04-12T11:29:06.706 INFO:tasks.rados.rados.0.smithi029.stdout:3452: done (0 left) 2024-04-12T11:29:06.707 INFO:tasks.rados.rados.0.smithi029.stdout:3453: rollback oid 36 current snap is 440 2024-04-12T11:29:06.707 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 36 to 433 2024-04-12T11:29:06.707 INFO:tasks.rados.rados.0.smithi029.stdout:3454: snap_create 2024-04-12T11:29:06.713 INFO:tasks.rados.rados.0.smithi029.stdout:3453: finishing rollback tid 0 to smithi02979277-36 2024-04-12T11:29:06.714 INFO:tasks.rados.rados.0.smithi029.stdout:3453: finishing rollback tid 1 to smithi02979277-36 2024-04-12T11:29:06.714 INFO:tasks.rados.rados.0.smithi029.stdout:3453: finishing rollback tid 2 to smithi02979277-36 2024-04-12T11:29:06.714 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 834 (ObjNum 1177 snap 400 seq_num 1177) dirty exists 2024-04-12T11:29:06.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:06 smithi087 ceph-mon[89014]: pgmap v1047: 89 pgs: 89 active+clean; 258 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-04-12T11:29:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:06 smithi087 ceph-mon[89014]: osdmap e941: 8 total, 8 up, 8 in 2024-04-12T11:29:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:07.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:06 smithi029 ceph-mon[93650]: pgmap v1047: 89 pgs: 89 active+clean; 258 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-04-12T11:29:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:06 smithi029 ceph-mon[93650]: osdmap e941: 8 total, 8 up, 8 in 2024-04-12T11:29:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:06 smithi029 ceph-mon[97787]: pgmap v1047: 89 pgs: 89 active+clean; 258 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-04-12T11:29:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:06 smithi029 ceph-mon[97787]: osdmap e941: 8 total, 8 up, 8 in 2024-04-12T11:29:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:07.702 INFO:tasks.rados.rados.0.smithi029.stdout:3453: done (1 left) 2024-04-12T11:29:07.702 INFO:tasks.rados.rados.0.smithi029.stdout:3454: done (0 left) 2024-04-12T11:29:07.702 INFO:tasks.rados.rados.0.smithi029.stdout:3455: rollback oid 33 current snap is 441 2024-04-12T11:29:07.702 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 409 2024-04-12T11:29:07.702 INFO:tasks.rados.rados.0.smithi029.stdout:3456: write oid 20 current snap is 441 2024-04-12T11:29:07.702 INFO:tasks.rados.rados.0.smithi029.stdout:3456: seq_num 1282 ranges {746574=416522,1586386=468767,2772512=629544} 2024-04-12T11:29:07.708 INFO:tasks.rados.rados.0.smithi029.stdout:3456: writing smithi02979277-20 from 746574 to 1163096 tid 1 2024-04-12T11:29:07.713 INFO:tasks.rados.rados.0.smithi029.stdout:3456: writing smithi02979277-20 from 1586386 to 2055153 tid 2 2024-04-12T11:29:07.722 INFO:tasks.rados.rados.0.smithi029.stdout:3456: writing smithi02979277-20 from 2772512 to 3402056 tid 3 2024-04-12T11:29:07.722 INFO:tasks.rados.rados.0.smithi029.stdout:3457: write oid 18 current snap is 441 2024-04-12T11:29:07.722 INFO:tasks.rados.rados.0.smithi029.stdout:3455: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:29:07.722 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:29:07.722 INFO:tasks.rados.rados.0.smithi029.stdout:3456: finishing write tid 1 to smithi02979277-20 2024-04-12T11:29:07.722 INFO:tasks.rados.rados.0.smithi029.stdout:3457: seq_num 1283 ranges {763381=491771,2044868=728778,3199935=363446} 2024-04-12T11:29:07.728 INFO:tasks.rados.rados.0.smithi029.stdout:3457: writing smithi02979277-18 from 763381 to 1255152 tid 1 2024-04-12T11:29:07.736 INFO:tasks.rados.rados.0.smithi029.stdout:3457: writing smithi02979277-18 from 2044868 to 2773646 tid 2 2024-04-12T11:29:07.741 INFO:tasks.rados.rados.0.smithi029.stdout:3457: writing smithi02979277-18 from 3199935 to 3563381 tid 3 2024-04-12T11:29:07.741 INFO:tasks.rados.rados.0.smithi029.stdout:3455: done (2 left) 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3458: snap_create 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3456: finishing write tid 2 to smithi02979277-20 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3456: finishing write tid 3 to smithi02979277-20 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3456: finishing write tid 4 to smithi02979277-20 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3456: finishing write tid 5 to smithi02979277-20 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3456: oid 20 updating version 0 to 1143 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3456: oid 20 updating version 1143 to 1144 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3456: oid 20 version 1144 is already newer than 1142 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3456: oid 20 updating version 1144 to 1145 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 1145 (ObjNum 1282 snap 441 seq_num 1282) dirty exists 2024-04-12T11:29:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3456: left oid 20 (ObjNum 1282 snap 441 seq_num 1282) 2024-04-12T11:29:07.743 INFO:tasks.rados.rados.0.smithi029.stdout:3457: finishing write tid 1 to smithi02979277-18 2024-04-12T11:29:07.743 INFO:tasks.rados.rados.0.smithi029.stdout:3457: finishing write tid 2 to smithi02979277-18 2024-04-12T11:29:07.745 INFO:tasks.rados.rados.0.smithi029.stdout:3457: finishing write tid 3 to smithi02979277-18 2024-04-12T11:29:07.745 INFO:tasks.rados.rados.0.smithi029.stdout:3457: finishing write tid 4 to smithi02979277-18 2024-04-12T11:29:07.745 INFO:tasks.rados.rados.0.smithi029.stdout:3457: finishing write tid 5 to smithi02979277-18 2024-04-12T11:29:07.746 INFO:tasks.rados.rados.0.smithi029.stdout:3457: oid 18 updating version 0 to 1802 2024-04-12T11:29:07.746 INFO:tasks.rados.rados.0.smithi029.stdout:3457: oid 18 version 1802 is already newer than 1799 2024-04-12T11:29:07.746 INFO:tasks.rados.rados.0.smithi029.stdout:3457: oid 18 version 1802 is already newer than 1800 2024-04-12T11:29:07.746 INFO:tasks.rados.rados.0.smithi029.stdout:3457: oid 18 version 1802 is already newer than 1801 2024-04-12T11:29:07.746 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1802 (ObjNum 1283 snap 441 seq_num 1283) dirty exists 2024-04-12T11:29:07.746 INFO:tasks.rados.rados.0.smithi029.stdout:3457: left oid 18 (ObjNum 1283 snap 441 seq_num 1283) 2024-04-12T11:29:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:07 smithi087 ceph-mon[89014]: osdmap e942: 8 total, 8 up, 8 in 2024-04-12T11:29:07.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:07 smithi087 ceph-mon[89014]: osdmap e943: 8 total, 8 up, 8 in 2024-04-12T11:29:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:07 smithi029 ceph-mon[93650]: osdmap e942: 8 total, 8 up, 8 in 2024-04-12T11:29:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:07 smithi029 ceph-mon[93650]: osdmap e943: 8 total, 8 up, 8 in 2024-04-12T11:29:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:07 smithi029 ceph-mon[97787]: osdmap e942: 8 total, 8 up, 8 in 2024-04-12T11:29:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:07 smithi029 ceph-mon[97787]: osdmap e943: 8 total, 8 up, 8 in 2024-04-12T11:29:08.723 INFO:tasks.rados.rados.0.smithi029.stdout:3456: done (2 left) 2024-04-12T11:29:08.723 INFO:tasks.rados.rados.0.smithi029.stdout:3457: done (1 left) 2024-04-12T11:29:08.723 INFO:tasks.rados.rados.0.smithi029.stdout:3458: done (0 left) 2024-04-12T11:29:08.723 INFO:tasks.rados.rados.0.smithi029.stdout:3459: write oid 49 current snap is 442 2024-04-12T11:29:08.723 INFO:tasks.rados.rados.0.smithi029.stdout:3459: seq_num 1284 ranges {780188=567020,2103350=472436,3227358=352830} 2024-04-12T11:29:08.730 INFO:tasks.rados.rados.0.smithi029.stdout:3459: writing smithi02979277-49 from 780188 to 1347208 tid 1 2024-04-12T11:29:08.737 INFO:tasks.rados.rados.0.smithi029.stdout:3459: writing smithi02979277-49 from 2103350 to 2575786 tid 2 2024-04-12T11:29:08.743 INFO:tasks.rados.rados.0.smithi029.stdout:3459: writing smithi02979277-49 from 3227358 to 3580188 tid 3 2024-04-12T11:29:08.743 INFO:tasks.rados.rados.0.smithi029.stdout:3460: write oid 11 current snap is 442 2024-04-12T11:29:08.743 INFO:tasks.rados.rados.0.smithi029.stdout:3460: seq_num 1285 ranges {796995=758622,2161832=732447} 2024-04-12T11:29:08.751 INFO:tasks.rados.rados.0.smithi029.stdout:3460: writing smithi02979277-11 from 796995 to 1555617 tid 1 2024-04-12T11:29:08.758 INFO:tasks.rados.rados.0.smithi029.stdout:3460: writing smithi02979277-11 from 2161832 to 2894279 tid 2 2024-04-12T11:29:08.759 INFO:tasks.rados.rados.0.smithi029.stdout:3461: write oid 7 current snap is 442 2024-04-12T11:29:08.759 INFO:tasks.rados.rados.0.smithi029.stdout:3461: seq_num 1286 ranges {413802=433871,1420314=476105,2482204=790985} 2024-04-12T11:29:08.763 INFO:tasks.rados.rados.0.smithi029.stdout:3461: writing smithi02979277-7 from 413802 to 847673 tid 1 2024-04-12T11:29:08.769 INFO:tasks.rados.rados.0.smithi029.stdout:3461: writing smithi02979277-7 from 1420314 to 1896419 tid 2 2024-04-12T11:29:08.777 INFO:tasks.rados.rados.0.smithi029.stdout:3461: writing smithi02979277-7 from 2482204 to 3273189 tid 3 2024-04-12T11:29:08.777 INFO:tasks.rados.rados.0.smithi029.stdout:3462: delete oid 28 current snap is 442 2024-04-12T11:29:08.777 INFO:tasks.rados.rados.0.smithi029.stdout:3459: finishing write tid 1 to smithi02979277-49 2024-04-12T11:29:08.777 INFO:tasks.rados.rados.0.smithi029.stdout:3459: finishing write tid 2 to smithi02979277-49 2024-04-12T11:29:08.777 INFO:tasks.rados.rados.0.smithi029.stdout:3459: finishing write tid 3 to smithi02979277-49 2024-04-12T11:29:08.777 INFO:tasks.rados.rados.0.smithi029.stdout:3459: finishing write tid 4 to smithi02979277-49 2024-04-12T11:29:08.777 INFO:tasks.rados.rados.0.smithi029.stdout:3459: finishing write tid 5 to smithi02979277-49 2024-04-12T11:29:08.777 INFO:tasks.rados.rados.0.smithi029.stdout:3459: oid 49 updating version 0 to 1236 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3459: oid 49 version 1236 is already newer than 1235 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3459: oid 49 updating version 1236 to 1238 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3459: oid 49 version 1238 is already newer than 1237 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 1238 (ObjNum 1284 snap 442 seq_num 1284) dirty exists 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3459: left oid 49 (ObjNum 1284 snap 442 seq_num 1284) 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3460: finishing write tid 1 to smithi02979277-11 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3460: finishing write tid 2 to smithi02979277-11 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3460: finishing write tid 3 to smithi02979277-11 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3460: finishing write tid 4 to smithi02979277-11 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3460: oid 11 updating version 0 to 2196 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3460: oid 11 updating version 2196 to 2197 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:3460: oid 11 version 2197 is already newer than 2195 2024-04-12T11:29:08.778 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 2197 (ObjNum 1285 snap 442 seq_num 1285) dirty exists 2024-04-12T11:29:08.779 INFO:tasks.rados.rados.0.smithi029.stdout:3460: left oid 11 (ObjNum 1285 snap 442 seq_num 1285) 2024-04-12T11:29:08.779 INFO:tasks.rados.rados.0.smithi029.stdout:3461: finishing write tid 1 to smithi02979277-7 2024-04-12T11:29:08.779 INFO:tasks.rados.rados.0.smithi029.stdout:3461: finishing write tid 2 to smithi02979277-7 2024-04-12T11:29:08.782 INFO:tasks.rados.rados.0.smithi029.stdout:3461: finishing write tid 3 to smithi02979277-7 2024-04-12T11:29:08.782 INFO:tasks.rados.rados.0.smithi029.stdout:3459: done (3 left) 2024-04-12T11:29:08.782 INFO:tasks.rados.rados.0.smithi029.stdout:3460: done (2 left) 2024-04-12T11:29:08.782 INFO:tasks.rados.rados.0.smithi029.stdout:3462: done (1 left) 2024-04-12T11:29:08.782 INFO:tasks.rados.rados.0.smithi029.stdout:3463: snap_create 2024-04-12T11:29:08.782 INFO:tasks.rados.rados.0.smithi029.stdout:3461: finishing write tid 4 to smithi02979277-7 2024-04-12T11:29:08.782 INFO:tasks.rados.rados.0.smithi029.stdout:3461: finishing write tid 5 to smithi02979277-7 2024-04-12T11:29:08.782 INFO:tasks.rados.rados.0.smithi029.stdout:3461: oid 7 updating version 0 to 1720 2024-04-12T11:29:08.782 INFO:tasks.rados.rados.0.smithi029.stdout:3461: oid 7 updating version 1720 to 1723 2024-04-12T11:29:08.782 INFO:tasks.rados.rados.0.smithi029.stdout:3461: oid 7 version 1723 is already newer than 1722 2024-04-12T11:29:08.783 INFO:tasks.rados.rados.0.smithi029.stdout:3461: oid 7 version 1723 is already newer than 1721 2024-04-12T11:29:08.783 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1723 (ObjNum 1286 snap 442 seq_num 1286) dirty exists 2024-04-12T11:29:08.783 INFO:tasks.rados.rados.0.smithi029.stdout:3461: left oid 7 (ObjNum 1286 snap 442 seq_num 1286) 2024-04-12T11:29:08.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:08 smithi087 ceph-mon[89014]: pgmap v1050: 89 pgs: 89 active+clean; 243 MiB data, 874 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 341 KiB/s wr, 1 op/s 2024-04-12T11:29:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:08 smithi029 ceph-mon[93650]: pgmap v1050: 89 pgs: 89 active+clean; 243 MiB data, 874 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 341 KiB/s wr, 1 op/s 2024-04-12T11:29:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:08 smithi029 ceph-mon[97787]: pgmap v1050: 89 pgs: 89 active+clean; 243 MiB data, 874 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 341 KiB/s wr, 1 op/s 2024-04-12T11:29:09.730 INFO:tasks.rados.rados.0.smithi029.stdout:3461: done (1 left) 2024-04-12T11:29:09.730 INFO:tasks.rados.rados.0.smithi029.stdout:3463: done (0 left) 2024-04-12T11:29:09.730 INFO:tasks.rados.rados.0.smithi029.stdout:3464: snap_remove snap 437 2024-04-12T11:29:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:09 smithi029 ceph-mon[93650]: osdmap e944: 8 total, 8 up, 8 in 2024-04-12T11:29:10.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:09 smithi029 ceph-mon[97787]: osdmap e944: 8 total, 8 up, 8 in 2024-04-12T11:29:10.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:09 smithi087 ceph-mon[89014]: osdmap e944: 8 total, 8 up, 8 in 2024-04-12T11:29:10.734 INFO:tasks.rados.rados.0.smithi029.stdout:3464: done (0 left) 2024-04-12T11:29:10.734 INFO:tasks.rados.rados.0.smithi029.stdout:3465: snap_remove snap 441 2024-04-12T11:29:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:10 smithi029 ceph-mon[93650]: pgmap v1053: 89 pgs: 89 active+clean; 231 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 370 KiB/s wr, 2 op/s 2024-04-12T11:29:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:10 smithi029 ceph-mon[93650]: osdmap e945: 8 total, 8 up, 8 in 2024-04-12T11:29:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:10 smithi029 ceph-mon[97787]: pgmap v1053: 89 pgs: 89 active+clean; 231 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 370 KiB/s wr, 2 op/s 2024-04-12T11:29:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:10 smithi029 ceph-mon[97787]: osdmap e945: 8 total, 8 up, 8 in 2024-04-12T11:29:11.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:10 smithi087 ceph-mon[89014]: pgmap v1053: 89 pgs: 89 active+clean; 231 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 370 KiB/s wr, 2 op/s 2024-04-12T11:29:11.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:10 smithi087 ceph-mon[89014]: osdmap e945: 8 total, 8 up, 8 in 2024-04-12T11:29:11.732 INFO:tasks.rados.rados.0.smithi029.stdout:3465: done (0 left) 2024-04-12T11:29:11.733 INFO:tasks.rados.rados.0.smithi029.stdout:3466: delete oid 7 current snap is 443 2024-04-12T11:29:11.737 INFO:tasks.rados.rados.0.smithi029.stdout:3466: done (0 left) 2024-04-12T11:29:11.738 INFO:tasks.rados.rados.0.smithi029.stdout:3467: write oid 7 current snap is 443 2024-04-12T11:29:11.738 INFO:tasks.rados.rados.0.smithi029.stdout:3467: seq_num 1289 ranges {464223=659618,1879407=739785,3248120=416103} 2024-04-12T11:29:11.743 INFO:tasks.rados.rados.0.smithi029.stdout:3467: writing smithi02979277-7 from 464223 to 1123841 tid 1 2024-04-12T11:29:11.754 INFO:tasks.rados.rados.0.smithi029.stdout:3467: writing smithi02979277-7 from 1879407 to 2619192 tid 2 2024-04-12T11:29:11.757 INFO:tasks.rados.rados.0.smithi029.stdout:3467: writing smithi02979277-7 from 3248120 to 3664223 tid 3 2024-04-12T11:29:11.758 INFO:tasks.rados.rados.0.smithi029.stdout:3468: snap_remove snap 404 2024-04-12T11:29:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:11 smithi029 ceph-mon[93650]: osdmap e946: 8 total, 8 up, 8 in 2024-04-12T11:29:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:11 smithi029 ceph-mon[93650]: osdmap e947: 8 total, 8 up, 8 in 2024-04-12T11:29:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:11 smithi029 ceph-mon[97787]: osdmap e946: 8 total, 8 up, 8 in 2024-04-12T11:29:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:11 smithi029 ceph-mon[97787]: osdmap e947: 8 total, 8 up, 8 in 2024-04-12T11:29:12.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:11 smithi087 ceph-mon[89014]: osdmap e946: 8 total, 8 up, 8 in 2024-04-12T11:29:12.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:11 smithi087 ceph-mon[89014]: osdmap e947: 8 total, 8 up, 8 in 2024-04-12T11:29:12.706 INFO:tasks.rados.rados.0.smithi029.stdout:3468: done (1 left) 2024-04-12T11:29:12.706 INFO:tasks.rados.rados.0.smithi029.stdout:3469: write oid 5 current snap is 443 2024-04-12T11:29:12.706 INFO:tasks.rados.rados.0.smithi029.stdout:3467: finishing write tid 1 to smithi02979277-7 2024-04-12T11:29:12.706 INFO:tasks.rados.rados.0.smithi029.stdout:3467: finishing write tid 2 to smithi02979277-7 2024-04-12T11:29:12.706 INFO:tasks.rados.rados.0.smithi029.stdout:3467: finishing write tid 3 to smithi02979277-7 2024-04-12T11:29:12.706 INFO:tasks.rados.rados.0.smithi029.stdout:3467: finishing write tid 4 to smithi02979277-7 2024-04-12T11:29:12.706 INFO:tasks.rados.rados.0.smithi029.stdout:3467: finishing write tid 5 to smithi02979277-7 2024-04-12T11:29:12.706 INFO:tasks.rados.rados.0.smithi029.stdout:3467: oid 7 updating version 0 to 1735 2024-04-12T11:29:12.706 INFO:tasks.rados.rados.0.smithi029.stdout:3467: oid 7 version 1735 is already newer than 1732 2024-04-12T11:29:12.706 INFO:tasks.rados.rados.0.smithi029.stdout:3467: oid 7 version 1735 is already newer than 1733 2024-04-12T11:29:12.707 INFO:tasks.rados.rados.0.smithi029.stdout:3467: oid 7 version 1735 is already newer than 1734 2024-04-12T11:29:12.707 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1735 (ObjNum 1289 snap 443 seq_num 1289) dirty exists 2024-04-12T11:29:12.707 INFO:tasks.rados.rados.0.smithi029.stdout:3467: left oid 7 (ObjNum 1289 snap 443 seq_num 1289) 2024-04-12T11:29:12.707 INFO:tasks.rados.rados.0.smithi029.stdout:3469: seq_num 1290 ranges {481030=734867,1937889=483443,2875543=552426} 2024-04-12T11:29:12.713 INFO:tasks.rados.rados.0.smithi029.stdout:3469: writing smithi02979277-5 from 481030 to 1215897 tid 1 2024-04-12T11:29:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:29:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:29:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:29:12.721 INFO:tasks.rados.rados.0.smithi029.stdout:3469: writing smithi02979277-5 from 1937889 to 2421332 tid 2 2024-04-12T11:29:12.727 INFO:tasks.rados.rados.0.smithi029.stdout:3469: writing smithi02979277-5 from 2875543 to 3427969 tid 3 2024-04-12T11:29:12.727 INFO:tasks.rados.rados.0.smithi029.stdout:3467: done (1 left) 2024-04-12T11:29:12.727 INFO:tasks.rados.rados.0.smithi029.stdout:3470: snap_remove snap 409 2024-04-12T11:29:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:12 smithi029 ceph-mon[97787]: pgmap v1056: 89 pgs: 89 active+clean; 231 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 370 KiB/s wr, 3 op/s 2024-04-12T11:29:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:12 smithi029 ceph-mon[97787]: osdmap e948: 8 total, 8 up, 8 in 2024-04-12T11:29:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:12 smithi029 ceph-mon[93650]: pgmap v1056: 89 pgs: 89 active+clean; 231 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 370 KiB/s wr, 3 op/s 2024-04-12T11:29:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:12 smithi029 ceph-mon[93650]: osdmap e948: 8 total, 8 up, 8 in 2024-04-12T11:29:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:12 smithi087 ceph-mon[89014]: pgmap v1056: 89 pgs: 89 active+clean; 231 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 370 KiB/s wr, 3 op/s 2024-04-12T11:29:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:12 smithi087 ceph-mon[89014]: osdmap e948: 8 total, 8 up, 8 in 2024-04-12T11:29:13.705 INFO:tasks.rados.rados.0.smithi029.stdout:3469: finishing write tid 1 to smithi02979277-5 2024-04-12T11:29:13.705 INFO:tasks.rados.rados.0.smithi029.stdout:3469: finishing write tid 2 to smithi02979277-5 2024-04-12T11:29:13.705 INFO:tasks.rados.rados.0.smithi029.stdout:3469: finishing write tid 3 to smithi02979277-5 2024-04-12T11:29:13.705 INFO:tasks.rados.rados.0.smithi029.stdout:3469: finishing write tid 4 to smithi02979277-5 2024-04-12T11:29:13.705 INFO:tasks.rados.rados.0.smithi029.stdout:3469: finishing write tid 5 to smithi02979277-5 2024-04-12T11:29:13.705 INFO:tasks.rados.rados.0.smithi029.stdout:3469: oid 5 updating version 0 to 1631 2024-04-12T11:29:13.705 INFO:tasks.rados.rados.0.smithi029.stdout:3469: oid 5 updating version 1631 to 1634 2024-04-12T11:29:13.705 INFO:tasks.rados.rados.0.smithi029.stdout:3469: oid 5 updating version 1634 to 1636 2024-04-12T11:29:13.705 INFO:tasks.rados.rados.0.smithi029.stdout:3469: oid 5 version 1636 is already newer than 1635 2024-04-12T11:29:13.705 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1636 (ObjNum 1290 snap 443 seq_num 1290) dirty exists 2024-04-12T11:29:13.706 INFO:tasks.rados.rados.0.smithi029.stdout:3469: left oid 5 (ObjNum 1290 snap 443 seq_num 1290) 2024-04-12T11:29:13.706 INFO:tasks.rados.rados.0.smithi029.stdout:3469: done (1 left) 2024-04-12T11:29:13.706 INFO:tasks.rados.rados.0.smithi029.stdout:3470: done (0 left) 2024-04-12T11:29:13.706 INFO:tasks.rados.rados.0.smithi029.stdout:3471: snap_create 2024-04-12T11:29:14.713 INFO:tasks.rados.rados.0.smithi029.stdout:3471: done (0 left) 2024-04-12T11:29:14.713 INFO:tasks.rados.rados.0.smithi029.stdout:3472: read oid 25 snap -1 2024-04-12T11:29:14.713 INFO:tasks.rados.rados.0.smithi029.stdout:3472: expect (ObjNum 1280 snap 438 seq_num 1280) 2024-04-12T11:29:14.713 INFO:tasks.rados.rados.0.smithi029.stdout:3473: read oid 23 snap -1 2024-04-12T11:29:14.713 INFO:tasks.rados.rados.0.smithi029.stdout:3473: expect (ObjNum 1210 snap 408 seq_num 1210) 2024-04-12T11:29:14.714 INFO:tasks.rados.rados.0.smithi029.stdout:3474: delete oid 45 current snap is 444 2024-04-12T11:29:14.719 INFO:tasks.rados.rados.0.smithi029.stdout:3474: done (2 left) 2024-04-12T11:29:14.719 INFO:tasks.rados.rados.0.smithi029.stdout:3475: write oid 43 current snap is 444 2024-04-12T11:29:14.719 INFO:tasks.rados.rados.0.smithi029.stdout:3475: seq_num 1292 ranges {514644=485365,1654853=487112,2930389=574970} 2024-04-12T11:29:14.724 INFO:tasks.rados.rados.0.smithi029.stdout:3475: writing smithi02979277-43 from 514644 to 1000009 tid 1 2024-04-12T11:29:14.731 INFO:tasks.rados.rados.0.smithi029.stdout:3475: writing smithi02979277-43 from 1654853 to 2141965 tid 2 2024-04-12T11:29:14.739 INFO:tasks.rados.rados.0.smithi029.stdout:3475: writing smithi02979277-43 from 2930389 to 3505359 tid 3 2024-04-12T11:29:14.739 INFO:tasks.rados.rados.0.smithi029.stdout:3476: read oid 33 snap -1 2024-04-12T11:29:14.739 INFO:tasks.rados.rados.0.smithi029.stdout:3476: expect deleted 2024-04-12T11:29:14.739 INFO:tasks.rados.rados.0.smithi029.stdout:3477: write oid 36 current snap is 444 2024-04-12T11:29:14.739 INFO:tasks.rados.rados.0.smithi029.stdout:3473: expect (ObjNum 1210 snap 408 seq_num 1210) 2024-04-12T11:29:14.896 INFO:tasks.rados.rados.0.smithi029.stdout:3472: expect (ObjNum 1280 snap 438 seq_num 1280) 2024-04-12T11:29:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:14 smithi087 ceph-mon[89014]: pgmap v1059: 89 pgs: 89 active+clean; 241 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 6 op/s 2024-04-12T11:29:14.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:14 smithi087 ceph-mon[89014]: osdmap e949: 8 total, 8 up, 8 in 2024-04-12T11:29:15.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:14 smithi029 ceph-mon[93650]: pgmap v1059: 89 pgs: 89 active+clean; 241 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 6 op/s 2024-04-12T11:29:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:14 smithi029 ceph-mon[93650]: osdmap e949: 8 total, 8 up, 8 in 2024-04-12T11:29:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:14 smithi029 ceph-mon[97787]: pgmap v1059: 89 pgs: 89 active+clean; 241 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 6 op/s 2024-04-12T11:29:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:14 smithi029 ceph-mon[97787]: osdmap e949: 8 total, 8 up, 8 in 2024-04-12T11:29:15.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:29:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:29:14] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-12T11:29:15.096 INFO:tasks.rados.rados.0.smithi029.stdout:3475: finishing write tid 1 to smithi02979277-43 2024-04-12T11:29:15.097 INFO:tasks.rados.rados.0.smithi029.stdout:3475: finishing write tid 2 to smithi02979277-43 2024-04-12T11:29:15.097 INFO:tasks.rados.rados.0.smithi029.stdout:3477: seq_num 1293 ranges {531451=676967,1713335=747123,2957812=502595} 2024-04-12T11:29:15.103 INFO:tasks.rados.rados.0.smithi029.stdout:3477: writing smithi02979277-36 from 531451 to 1208418 tid 1 2024-04-12T11:29:15.109 INFO:tasks.rados.rados.0.smithi029.stdout:3477: writing smithi02979277-36 from 1713335 to 2460458 tid 2 2024-04-12T11:29:15.116 INFO:tasks.rados.rados.0.smithi029.stdout:3477: writing smithi02979277-36 from 2957812 to 3460407 tid 3 2024-04-12T11:29:15.116 INFO:tasks.rados.rados.0.smithi029.stdout:3472: done (4 left) 2024-04-12T11:29:15.116 INFO:tasks.rados.rados.0.smithi029.stdout:3473: done (3 left) 2024-04-12T11:29:15.116 INFO:tasks.rados.rados.0.smithi029.stdout:3478: write oid 1 current snap is 444 2024-04-12T11:29:15.116 INFO:tasks.rados.rados.0.smithi029.stdout:3475: finishing write tid 3 to smithi02979277-43 2024-04-12T11:29:15.116 INFO:tasks.rados.rados.0.smithi029.stdout:3475: finishing write tid 4 to smithi02979277-43 2024-04-12T11:29:15.116 INFO:tasks.rados.rados.0.smithi029.stdout:3475: finishing write tid 5 to smithi02979277-43 2024-04-12T11:29:15.116 INFO:tasks.rados.rados.0.smithi029.stdout:3475: oid 43 updating version 0 to 1579 2024-04-12T11:29:15.117 INFO:tasks.rados.rados.0.smithi029.stdout:3475: oid 43 version 1579 is already newer than 1578 2024-04-12T11:29:15.117 INFO:tasks.rados.rados.0.smithi029.stdout:3475: oid 43 version 1579 is already newer than 1576 2024-04-12T11:29:15.117 INFO:tasks.rados.rados.0.smithi029.stdout:3475: oid 43 version 1579 is already newer than 1577 2024-04-12T11:29:15.117 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1579 (ObjNum 1292 snap 444 seq_num 1292) dirty exists 2024-04-12T11:29:15.117 INFO:tasks.rados.rados.0.smithi029.stdout:3475: left oid 43 (ObjNum 1292 snap 444 seq_num 1292) 2024-04-12T11:29:15.117 INFO:tasks.rados.rados.0.smithi029.stdout:3477: finishing write tid 1 to smithi02979277-36 2024-04-12T11:29:15.117 INFO:tasks.rados.rados.0.smithi029.stdout:3477: finishing write tid 2 to smithi02979277-36 2024-04-12T11:29:15.117 INFO:tasks.rados.rados.0.smithi029.stdout:3478: seq_num 1294 ranges {548258=752216,1771817=490781,2985235=713867} 2024-04-12T11:29:15.122 INFO:tasks.rados.rados.0.smithi029.stdout:3478: writing smithi02979277-1 from 548258 to 1300474 tid 1 2024-04-12T11:29:15.128 INFO:tasks.rados.rados.0.smithi029.stdout:3478: writing smithi02979277-1 from 1771817 to 2262598 tid 2 2024-04-12T11:29:15.136 INFO:tasks.rados.rados.0.smithi029.stdout:3478: writing smithi02979277-1 from 2985235 to 3699102 tid 3 2024-04-12T11:29:15.136 INFO:tasks.rados.rados.0.smithi029.stdout:3475: done (3 left) 2024-04-12T11:29:15.136 INFO:tasks.rados.rados.0.smithi029.stdout:3476: done (2 left) 2024-04-12T11:29:15.136 INFO:tasks.rados.rados.0.smithi029.stdout:3479: write oid 26 current snap is 444 2024-04-12T11:29:15.136 INFO:tasks.rados.rados.0.smithi029.stdout:3477: finishing write tid 3 to smithi02979277-36 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:3477: finishing write tid 4 to smithi02979277-36 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:3477: finishing write tid 5 to smithi02979277-36 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:3477: oid 36 updating version 0 to 846 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:3477: oid 36 updating version 846 to 847 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:3477: oid 36 updating version 847 to 848 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:3477: oid 36 updating version 848 to 849 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 849 (ObjNum 1293 snap 444 seq_num 1293) dirty exists 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:3477: left oid 36 (ObjNum 1293 snap 444 seq_num 1293) 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:3478: finishing write tid 1 to smithi02979277-1 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:3478: finishing write tid 2 to smithi02979277-1 2024-04-12T11:29:15.137 INFO:tasks.rados.rados.0.smithi029.stdout:3479: seq_num 1295 ranges {565065=427465,1430299=634439,2496305=525139,3472104=292961} 2024-04-12T11:29:15.141 INFO:tasks.rados.rados.0.smithi029.stdout:3479: writing smithi02979277-26 from 565065 to 992530 tid 1 2024-04-12T11:29:15.147 INFO:tasks.rados.rados.0.smithi029.stdout:3479: writing smithi02979277-26 from 1430299 to 2064738 tid 2 2024-04-12T11:29:15.152 INFO:tasks.rados.rados.0.smithi029.stdout:3479: writing smithi02979277-26 from 2496305 to 3021444 tid 3 2024-04-12T11:29:15.156 INFO:tasks.rados.rados.0.smithi029.stdout:3479: writing smithi02979277-26 from 3472104 to 3765065 tid 4 2024-04-12T11:29:15.156 INFO:tasks.rados.rados.0.smithi029.stdout:3477: done (2 left) 2024-04-12T11:29:15.156 INFO:tasks.rados.rados.0.smithi029.stdout:3480: snap_remove snap 430 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3480: done (2 left) 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3481: read oid 34 snap -1 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3481: expect deleted 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3482: read oid 33 snap -1 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3482: expect deleted 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3483: read oid 39 snap -1 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3483: expect (ObjNum 1088 snap 365 seq_num 1088) 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3484: rollback oid 17 current snap is 444 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 434 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3485: read oid 49 snap -1 2024-04-12T11:29:15.720 INFO:tasks.rados.rados.0.smithi029.stdout:3485: expect (ObjNum 1284 snap 442 seq_num 1284) 2024-04-12T11:29:15.721 INFO:tasks.rados.rados.0.smithi029.stdout:3486: write oid 12 current snap is 444 2024-04-12T11:29:15.721 INFO:tasks.rados.rados.0.smithi029.stdout:3486: seq_num 1296 ranges {581872=502714,1488781=494450,2640081=736411} 2024-04-12T11:29:15.726 INFO:tasks.rados.rados.0.smithi029.stdout:3486: writing smithi02979277-12 from 581872 to 1084586 tid 1 2024-04-12T11:29:15.732 INFO:tasks.rados.rados.0.smithi029.stdout:3486: writing smithi02979277-12 from 1488781 to 1983231 tid 2 2024-04-12T11:29:15.741 INFO:tasks.rados.rados.0.smithi029.stdout:3486: writing smithi02979277-12 from 2640081 to 3376492 tid 3 2024-04-12T11:29:15.741 INFO:tasks.rados.rados.0.smithi029.stdout:3487: delete oid 10 current snap is 444 2024-04-12T11:29:15.741 INFO:tasks.rados.rados.0.smithi029.stdout:3478: finishing write tid 3 to smithi02979277-1 2024-04-12T11:29:15.741 INFO:tasks.rados.rados.0.smithi029.stdout:3478: finishing write tid 4 to smithi02979277-1 2024-04-12T11:29:15.741 INFO:tasks.rados.rados.0.smithi029.stdout:3478: finishing write tid 5 to smithi02979277-1 2024-04-12T11:29:15.741 INFO:tasks.rados.rados.0.smithi029.stdout:3478: oid 1 updating version 0 to 1662 2024-04-12T11:29:15.741 INFO:tasks.rados.rados.0.smithi029.stdout:3478: oid 1 version 1662 is already newer than 1660 2024-04-12T11:29:15.741 INFO:tasks.rados.rados.0.smithi029.stdout:3478: oid 1 version 1662 is already newer than 1661 2024-04-12T11:29:15.741 INFO:tasks.rados.rados.0.smithi029.stdout:3478: oid 1 updating version 1662 to 1663 2024-04-12T11:29:15.741 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1663 (ObjNum 1294 snap 444 seq_num 1294) dirty exists 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3478: left oid 1 (ObjNum 1294 snap 444 seq_num 1294) 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: finishing write tid 1 to smithi02979277-26 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: finishing write tid 2 to smithi02979277-26 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: finishing write tid 3 to smithi02979277-26 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: finishing write tid 4 to smithi02979277-26 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: finishing write tid 5 to smithi02979277-26 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: finishing write tid 6 to smithi02979277-26 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: oid 26 updating version 0 to 2218 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: oid 26 updating version 2218 to 2221 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: oid 26 version 2221 is already newer than 2217 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: oid 26 version 2221 is already newer than 2220 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: oid 26 version 2221 is already newer than 2219 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 2221 (ObjNum 1295 snap 444 seq_num 1295) dirty exists 2024-04-12T11:29:15.742 INFO:tasks.rados.rados.0.smithi029.stdout:3479: left oid 26 (ObjNum 1295 snap 444 seq_num 1295) 2024-04-12T11:29:15.743 INFO:tasks.rados.rados.0.smithi029.stdout:3485: expect (ObjNum 1284 snap 442 seq_num 1284) 2024-04-12T11:29:15.894 INFO:tasks.rados.rados.0.smithi029.stdout:3483: expect (ObjNum 1088 snap 365 seq_num 1088) 2024-04-12T11:29:15.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:15 smithi087 ceph-mon[89014]: osdmap e950: 8 total, 8 up, 8 in 2024-04-12T11:29:16.017 INFO:tasks.rados.rados.0.smithi029.stdout:3484: finishing rollback tid 0 to smithi02979277-17 2024-04-12T11:29:16.017 INFO:tasks.rados.rados.0.smithi029.stdout:3484: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3484: finishing rollback tid 2 to smithi02979277-17 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1824 (ObjNum 1225 snap 413 seq_num 1225) dirty exists 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3486: finishing write tid 1 to smithi02979277-12 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3486: finishing write tid 2 to smithi02979277-12 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3478: done (8 left) 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3479: done (7 left) 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3481: done (6 left) 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3482: done (5 left) 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3483: done (4 left) 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3484: done (3 left) 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3485: done (2 left) 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3487: done (1 left) 2024-04-12T11:29:16.018 INFO:tasks.rados.rados.0.smithi029.stdout:3488: read oid 8 snap -1 2024-04-12T11:29:16.019 INFO:tasks.rados.rados.0.smithi029.stdout:3488: expect deleted 2024-04-12T11:29:16.019 INFO:tasks.rados.rados.0.smithi029.stdout:3489: write oid 41 current snap is 444 2024-04-12T11:29:16.019 INFO:tasks.rados.rados.0.smithi029.stdout:3489: seq_num 1298 ranges {615486=653212,1889392=498119,2978574=475308} 2024-04-12T11:29:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:15 smithi029 ceph-mon[93650]: osdmap e950: 8 total, 8 up, 8 in 2024-04-12T11:29:16.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:15 smithi029 ceph-mon[97787]: osdmap e950: 8 total, 8 up, 8 in 2024-04-12T11:29:16.024 INFO:tasks.rados.rados.0.smithi029.stdout:3489: writing smithi02979277-41 from 615486 to 1268698 tid 1 2024-04-12T11:29:16.030 INFO:tasks.rados.rados.0.smithi029.stdout:3489: writing smithi02979277-41 from 1889392 to 2387511 tid 2 2024-04-12T11:29:16.036 INFO:tasks.rados.rados.0.smithi029.stdout:3489: writing smithi02979277-41 from 2978574 to 3453882 tid 3 2024-04-12T11:29:16.036 INFO:tasks.rados.rados.0.smithi029.stdout:3490: write oid 38 current snap is 444 2024-04-12T11:29:16.036 INFO:tasks.rados.rados.0.smithi029.stdout:3490: seq_num 1299 ranges {632293=728461,1947874=641777,3005997=686580} 2024-04-12T11:29:16.043 INFO:tasks.rados.rados.0.smithi029.stdout:3490: writing smithi02979277-38 from 632293 to 1360754 tid 1 2024-04-12T11:29:16.050 INFO:tasks.rados.rados.0.smithi029.stdout:3490: writing smithi02979277-38 from 1947874 to 2589651 tid 2 2024-04-12T11:29:16.056 INFO:tasks.rados.rados.0.smithi029.stdout:3490: writing smithi02979277-38 from 3005997 to 3692577 tid 3 2024-04-12T11:29:16.056 INFO:tasks.rados.rados.0.smithi029.stdout:3491: write oid 15 current snap is 444 2024-04-12T11:29:16.056 INFO:tasks.rados.rados.0.smithi029.stdout:3491: seq_num 1300 ranges {649100=403710,1606356=501788,2633420=497852,3636005=213095} 2024-04-12T11:29:16.062 INFO:tasks.rados.rados.0.smithi029.stdout:3491: writing smithi02979277-15 from 649100 to 1052810 tid 1 2024-04-12T11:29:16.068 INFO:tasks.rados.rados.0.smithi029.stdout:3491: writing smithi02979277-15 from 1606356 to 2108144 tid 2 2024-04-12T11:29:16.073 INFO:tasks.rados.rados.0.smithi029.stdout:3491: writing smithi02979277-15 from 2633420 to 3131272 tid 3 2024-04-12T11:29:16.078 INFO:tasks.rados.rados.0.smithi029.stdout:3491: writing smithi02979277-15 from 3636005 to 3849100 tid 4 2024-04-12T11:29:16.078 INFO:tasks.rados.rados.0.smithi029.stdout:3492: read oid 2 snap -1 2024-04-12T11:29:16.078 INFO:tasks.rados.rados.0.smithi029.stdout:3492: expect deleted 2024-04-12T11:29:16.078 INFO:tasks.rados.rados.0.smithi029.stdout:3493: snap_create 2024-04-12T11:29:16.078 INFO:tasks.rados.rados.0.smithi029.stdout:3486: finishing write tid 3 to smithi02979277-12 2024-04-12T11:29:16.078 INFO:tasks.rados.rados.0.smithi029.stdout:3486: finishing write tid 4 to smithi02979277-12 2024-04-12T11:29:16.078 INFO:tasks.rados.rados.0.smithi029.stdout:3486: finishing write tid 5 to smithi02979277-12 2024-04-12T11:29:16.078 INFO:tasks.rados.rados.0.smithi029.stdout:3486: oid 12 updating version 0 to 1757 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3486: oid 12 updating version 1757 to 1759 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3486: oid 12 version 1759 is already newer than 1756 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3486: oid 12 version 1759 is already newer than 1758 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1759 (ObjNum 1296 snap 444 seq_num 1296) dirty exists 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3486: left oid 12 (ObjNum 1296 snap 444 seq_num 1296) 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3489: finishing write tid 1 to smithi02979277-41 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3489: finishing write tid 2 to smithi02979277-41 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3489: finishing write tid 3 to smithi02979277-41 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3489: finishing write tid 4 to smithi02979277-41 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3489: finishing write tid 5 to smithi02979277-41 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3489: oid 41 updating version 0 to 1670 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3489: oid 41 version 1670 is already newer than 1669 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3489: oid 41 version 1670 is already newer than 1667 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:3489: oid 41 version 1670 is already newer than 1668 2024-04-12T11:29:16.079 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1670 (ObjNum 1298 snap 444 seq_num 1298) dirty exists 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3489: left oid 41 (ObjNum 1298 snap 444 seq_num 1298) 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3490: finishing write tid 1 to smithi02979277-38 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3490: finishing write tid 2 to smithi02979277-38 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3490: finishing write tid 3 to smithi02979277-38 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3490: finishing write tid 4 to smithi02979277-38 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3490: finishing write tid 5 to smithi02979277-38 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3490: oid 38 updating version 0 to 1168 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3490: oid 38 version 1168 is already newer than 1167 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3490: oid 38 version 1168 is already newer than 1165 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3490: oid 38 version 1168 is already newer than 1166 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 38 v 1168 (ObjNum 1299 snap 444 seq_num 1299) dirty exists 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3490: left oid 38 (ObjNum 1299 snap 444 seq_num 1299) 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3491: finishing write tid 1 to smithi02979277-15 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3491: finishing write tid 2 to smithi02979277-15 2024-04-12T11:29:16.080 INFO:tasks.rados.rados.0.smithi029.stdout:3491: finishing write tid 3 to smithi02979277-15 2024-04-12T11:29:16.081 INFO:tasks.rados.rados.0.smithi029.stdout:3491: finishing write tid 4 to smithi02979277-15 2024-04-12T11:29:16.081 INFO:tasks.rados.rados.0.smithi029.stdout:3491: finishing write tid 5 to smithi02979277-15 2024-04-12T11:29:16.081 INFO:tasks.rados.rados.0.smithi029.stdout:3491: finishing write tid 6 to smithi02979277-15 2024-04-12T11:29:16.081 INFO:tasks.rados.rados.0.smithi029.stdout:3491: oid 15 updating version 0 to 1083 2024-04-12T11:29:16.081 INFO:tasks.rados.rados.0.smithi029.stdout:3491: oid 15 updating version 1083 to 1084 2024-04-12T11:29:16.082 INFO:tasks.rados.rados.0.smithi029.stdout:3491: oid 15 updating version 1084 to 1086 2024-04-12T11:29:16.082 INFO:tasks.rados.rados.0.smithi029.stdout:3491: oid 15 version 1086 is already newer than 1085 2024-04-12T11:29:16.082 INFO:tasks.rados.rados.0.smithi029.stdout:3491: oid 15 updating version 1086 to 1087 2024-04-12T11:29:16.082 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 1087 (ObjNum 1300 snap 444 seq_num 1300) dirty exists 2024-04-12T11:29:16.082 INFO:tasks.rados.rados.0.smithi029.stdout:3491: left oid 15 (ObjNum 1300 snap 444 seq_num 1300) 2024-04-12T11:29:16.724 INFO:tasks.rados.rados.0.smithi029.stdout:3486: done (6 left) 2024-04-12T11:29:16.724 INFO:tasks.rados.rados.0.smithi029.stdout:3488: done (5 left) 2024-04-12T11:29:16.724 INFO:tasks.rados.rados.0.smithi029.stdout:3489: done (4 left) 2024-04-12T11:29:16.724 INFO:tasks.rados.rados.0.smithi029.stdout:3490: done (3 left) 2024-04-12T11:29:16.724 INFO:tasks.rados.rados.0.smithi029.stdout:3491: done (2 left) 2024-04-12T11:29:16.724 INFO:tasks.rados.rados.0.smithi029.stdout:3492: done (1 left) 2024-04-12T11:29:16.724 INFO:tasks.rados.rados.0.smithi029.stdout:3493: done (0 left) 2024-04-12T11:29:16.724 INFO:tasks.rados.rados.0.smithi029.stdout:3494: write oid 42 current snap is 445 2024-04-12T11:29:16.724 INFO:tasks.rados.rados.0.smithi029.stdout:3494: seq_num 1301 ranges {665907=595312,1781191=645446,3177196=688711} 2024-04-12T11:29:16.730 INFO:tasks.rados.rados.0.smithi029.stdout:3494: writing smithi02979277-42 from 665907 to 1261219 tid 1 2024-04-12T11:29:16.737 INFO:tasks.rados.rados.0.smithi029.stdout:3494: writing smithi02979277-42 from 1781191 to 2426637 tid 2 2024-04-12T11:29:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3494: writing smithi02979277-42 from 3177196 to 3865907 tid 3 2024-04-12T11:29:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3495: read oid 26 snap -1 2024-04-12T11:29:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3495: expect (ObjNum 1295 snap 444 seq_num 1295) 2024-04-12T11:29:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3496: write oid 2 current snap is 445 2024-04-12T11:29:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3494: finishing write tid 1 to smithi02979277-42 2024-04-12T11:29:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3494: finishing write tid 2 to smithi02979277-42 2024-04-12T11:29:16.745 INFO:tasks.rados.rados.0.smithi029.stdout:3496: seq_num 1302 ranges {682714=670561,2123320=505457,3088266=636749} 2024-04-12T11:29:16.752 INFO:tasks.rados.rados.0.smithi029.stdout:3496: writing smithi02979277-2 from 682714 to 1353275 tid 1 2024-04-12T11:29:16.759 INFO:tasks.rados.rados.0.smithi029.stdout:3496: writing smithi02979277-2 from 2123320 to 2628777 tid 2 2024-04-12T11:29:16.766 INFO:tasks.rados.rados.0.smithi029.stdout:3496: writing smithi02979277-2 from 3088266 to 3725015 tid 3 2024-04-12T11:29:16.766 INFO:tasks.rados.rados.0.smithi029.stdout:3497: write oid 36 current snap is 445 2024-04-12T11:29:16.766 INFO:tasks.rados.rados.0.smithi029.stdout:3494: finishing write tid 3 to smithi02979277-42 2024-04-12T11:29:16.766 INFO:tasks.rados.rados.0.smithi029.stdout:3494: finishing write tid 4 to smithi02979277-42 2024-04-12T11:29:16.766 INFO:tasks.rados.rados.0.smithi029.stdout:3497: seq_num 1303 ranges {699521=745810,2181802=649115,3515689=383832} 2024-04-12T11:29:16.774 INFO:tasks.rados.rados.0.smithi029.stdout:3497: writing smithi02979277-36 from 699521 to 1445331 tid 1 2024-04-12T11:29:16.782 INFO:tasks.rados.rados.0.smithi029.stdout:3497: writing smithi02979277-36 from 2181802 to 2830917 tid 2 2024-04-12T11:29:16.788 INFO:tasks.rados.rados.0.smithi029.stdout:3497: writing smithi02979277-36 from 3515689 to 3899521 tid 3 2024-04-12T11:29:16.788 INFO:tasks.rados.rados.0.smithi029.stdout:3498: read oid 14 snap -1 2024-04-12T11:29:16.788 INFO:tasks.rados.rados.0.smithi029.stdout:3498: expect (ObjNum 1256 snap 427 seq_num 1256) 2024-04-12T11:29:16.789 INFO:tasks.rados.rados.0.smithi029.stdout:3499: rollback oid 19 current snap is 445 2024-04-12T11:29:16.789 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 19 to 443 2024-04-12T11:29:16.789 INFO:tasks.rados.rados.0.smithi029.stdout:3494: finishing write tid 5 to smithi02979277-42 2024-04-12T11:29:16.789 INFO:tasks.rados.rados.0.smithi029.stdout:3494: oid 42 updating version 0 to 436 2024-04-12T11:29:16.789 INFO:tasks.rados.rados.0.smithi029.stdout:3494: oid 42 updating version 436 to 437 2024-04-12T11:29:16.789 INFO:tasks.rados.rados.0.smithi029.stdout:3494: oid 42 version 437 is already newer than 435 2024-04-12T11:29:16.789 INFO:tasks.rados.rados.0.smithi029.stdout:3494: oid 42 version 437 is already newer than 434 2024-04-12T11:29:16.789 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 437 (ObjNum 1301 snap 445 seq_num 1301) dirty exists 2024-04-12T11:29:16.789 INFO:tasks.rados.rados.0.smithi029.stdout:3494: left oid 42 (ObjNum 1301 snap 445 seq_num 1301) 2024-04-12T11:29:16.789 INFO:tasks.rados.rados.0.smithi029.stdout:3495: expect (ObjNum 1295 snap 444 seq_num 1295) 2024-04-12T11:29:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:16 smithi087 ceph-mon[89014]: pgmap v1062: 89 pgs: 89 active+clean; 227 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-04-12T11:29:16.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:16 smithi087 ceph-mon[89014]: osdmap e951: 8 total, 8 up, 8 in 2024-04-12T11:29:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:16 smithi029 ceph-mon[97787]: pgmap v1062: 89 pgs: 89 active+clean; 227 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-04-12T11:29:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:16 smithi029 ceph-mon[97787]: osdmap e951: 8 total, 8 up, 8 in 2024-04-12T11:29:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:16 smithi029 ceph-mon[93650]: pgmap v1062: 89 pgs: 89 active+clean; 227 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.9 MiB/s wr, 7 op/s 2024-04-12T11:29:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:16 smithi029 ceph-mon[93650]: osdmap e951: 8 total, 8 up, 8 in 2024-04-12T11:29:17.050 INFO:tasks.rados.rados.0.smithi029.stdout:3496: finishing write tid 1 to smithi02979277-2 2024-04-12T11:29:17.050 INFO:tasks.rados.rados.0.smithi029.stdout:3496: finishing write tid 2 to smithi02979277-2 2024-04-12T11:29:17.050 INFO:tasks.rados.rados.0.smithi029.stdout:3496: finishing write tid 3 to smithi02979277-2 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3496: finishing write tid 4 to smithi02979277-2 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3496: finishing write tid 5 to smithi02979277-2 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3496: oid 2 updating version 0 to 410 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3496: oid 2 updating version 410 to 412 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3496: oid 2 version 412 is already newer than 411 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3496: oid 2 version 412 is already newer than 409 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 412 (ObjNum 1302 snap 445 seq_num 1302) dirty exists 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3496: left oid 2 (ObjNum 1302 snap 445 seq_num 1302) 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3497: finishing write tid 1 to smithi02979277-36 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3494: done (5 left) 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3495: done (4 left) 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3496: done (3 left) 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3500: read oid 43 snap -1 2024-04-12T11:29:17.051 INFO:tasks.rados.rados.0.smithi029.stdout:3500: expect (ObjNum 1292 snap 444 seq_num 1292) 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3501: write oid 41 current snap is 445 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3497: finishing write tid 2 to smithi02979277-36 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3497: finishing write tid 3 to smithi02979277-36 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3497: finishing write tid 4 to smithi02979277-36 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3497: finishing write tid 5 to smithi02979277-36 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3497: oid 36 updating version 0 to 857 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3497: oid 36 updating version 857 to 858 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3497: oid 36 version 858 is already newer than 856 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3497: oid 36 version 858 is already newer than 855 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 858 (ObjNum 1303 snap 445 seq_num 1303) dirty exists 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3497: left oid 36 (ObjNum 1303 snap 445 seq_num 1303) 2024-04-12T11:29:17.052 INFO:tasks.rados.rados.0.smithi029.stdout:3498: expect (ObjNum 1256 snap 427 seq_num 1256) 2024-04-12T11:29:17.156 INFO:tasks.rados.rados.0.smithi029.stdout:3499: finishing rollback tid 0 to smithi02979277-19 2024-04-12T11:29:17.156 INFO:tasks.rados.rados.0.smithi029.stdout:3499: finishing rollback tid 1 to smithi02979277-19 2024-04-12T11:29:17.156 INFO:tasks.rados.rados.0.smithi029.stdout:3499: finishing rollback tid 2 to smithi02979277-19 2024-04-12T11:29:17.156 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1650 (ObjNum 1259 snap 428 seq_num 1259) dirty exists 2024-04-12T11:29:17.156 INFO:tasks.rados.rados.0.smithi029.stdout:3501: seq_num 1304 ranges {716328=421059,1840284=509126,3143112=659293} 2024-04-12T11:29:17.162 INFO:tasks.rados.rados.0.smithi029.stdout:3501: writing smithi02979277-41 from 716328 to 1137387 tid 1 2024-04-12T11:29:17.169 INFO:tasks.rados.rados.0.smithi029.stdout:3501: writing smithi02979277-41 from 1840284 to 2349410 tid 2 2024-04-12T11:29:17.177 INFO:tasks.rados.rados.0.smithi029.stdout:3501: writing smithi02979277-41 from 3143112 to 3802405 tid 3 2024-04-12T11:29:17.177 INFO:tasks.rados.rados.0.smithi029.stdout:3497: done (4 left) 2024-04-12T11:29:17.177 INFO:tasks.rados.rados.0.smithi029.stdout:3498: done (3 left) 2024-04-12T11:29:17.177 INFO:tasks.rados.rados.0.smithi029.stdout:3499: done (2 left) 2024-04-12T11:29:17.177 INFO:tasks.rados.rados.0.smithi029.stdout:3502: read oid 7 snap -1 2024-04-12T11:29:17.178 INFO:tasks.rados.rados.0.smithi029.stdout:3502: expect (ObjNum 1289 snap 443 seq_num 1289) 2024-04-12T11:29:17.178 INFO:tasks.rados.rados.0.smithi029.stdout:3503: snap_remove snap 444 2024-04-12T11:29:17.704 INFO:tasks.rados.rados.0.smithi029.stdout:3503: done (3 left) 2024-04-12T11:29:17.704 INFO:tasks.rados.rados.0.smithi029.stdout:3504: write oid 8 current snap is 445 2024-04-12T11:29:17.704 INFO:tasks.rados.rados.0.smithi029.stdout:3504: seq_num 1305 ranges {733135=496308,1898766=652784,3170535=470565} 2024-04-12T11:29:17.711 INFO:tasks.rados.rados.0.smithi029.stdout:3504: writing smithi02979277-8 from 733135 to 1229443 tid 1 2024-04-12T11:29:17.718 INFO:tasks.rados.rados.0.smithi029.stdout:3504: writing smithi02979277-8 from 1898766 to 2551550 tid 2 2024-04-12T11:29:17.725 INFO:tasks.rados.rados.0.smithi029.stdout:3504: writing smithi02979277-8 from 3170535 to 3641100 tid 3 2024-04-12T11:29:17.725 INFO:tasks.rados.rados.0.smithi029.stdout:3505: write oid 46 current snap is 445 2024-04-12T11:29:17.725 INFO:tasks.rados.rados.0.smithi029.stdout:3505: seq_num 1306 ranges {749942=571557,1840895=796442,3365252=584690} 2024-04-12T11:29:17.732 INFO:tasks.rados.rados.0.smithi029.stdout:3505: writing smithi02979277-46 from 749942 to 1321499 tid 1 2024-04-12T11:29:17.740 INFO:tasks.rados.rados.0.smithi029.stdout:3505: writing smithi02979277-46 from 1840895 to 2637337 tid 2 2024-04-12T11:29:17.747 INFO:tasks.rados.rados.0.smithi029.stdout:3505: writing smithi02979277-46 from 3365252 to 3949942 tid 3 2024-04-12T11:29:17.747 INFO:tasks.rados.rados.0.smithi029.stdout:3506: write oid 23 current snap is 445 2024-04-12T11:29:17.747 INFO:tasks.rados.rados.0.smithi029.stdout:3506: seq_num 1307 ranges {766749=646806,1899377=656453,3109028=609462} 2024-04-12T11:29:17.755 INFO:tasks.rados.rados.0.smithi029.stdout:3506: writing smithi02979277-23 from 766749 to 1413555 tid 1 2024-04-12T11:29:17.761 INFO:tasks.rados.rados.0.smithi029.stdout:3506: writing smithi02979277-23 from 1899377 to 2555830 tid 2 2024-04-12T11:29:17.767 INFO:tasks.rados.rados.0.smithi029.stdout:3506: writing smithi02979277-23 from 3109028 to 3718490 tid 3 2024-04-12T11:29:17.767 INFO:tasks.rados.rados.0.smithi029.stdout:3507: write oid 13 current snap is 445 2024-04-12T11:29:17.767 INFO:tasks.rados.rados.0.smithi029.stdout:3500: expect (ObjNum 1292 snap 444 seq_num 1292) 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:3501: finishing write tid 1 to smithi02979277-41 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:3501: finishing write tid 2 to smithi02979277-41 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:3501: finishing write tid 3 to smithi02979277-41 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:3501: finishing write tid 4 to smithi02979277-41 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:3501: finishing write tid 5 to smithi02979277-41 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:3501: oid 41 updating version 0 to 1674 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:3501: oid 41 version 1674 is already newer than 1672 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:3501: oid 41 version 1674 is already newer than 1673 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:3501: oid 41 updating version 1674 to 1675 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1675 (ObjNum 1304 snap 445 seq_num 1304) dirty exists 2024-04-12T11:29:17.945 INFO:tasks.rados.rados.0.smithi029.stdout:3501: left oid 41 (ObjNum 1304 snap 445 seq_num 1304) 2024-04-12T11:29:17.946 INFO:tasks.rados.rados.0.smithi029.stdout:3502: expect (ObjNum 1289 snap 443 seq_num 1289) 2024-04-12T11:29:18.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:17 smithi029 ceph-mon[93650]: osdmap e952: 8 total, 8 up, 8 in 2024-04-12T11:29:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:17 smithi029 ceph-mon[93650]: osdmap e953: 8 total, 8 up, 8 in 2024-04-12T11:29:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:17 smithi029 ceph-mon[97787]: osdmap e952: 8 total, 8 up, 8 in 2024-04-12T11:29:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:17 smithi029 ceph-mon[97787]: osdmap e953: 8 total, 8 up, 8 in 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3504: finishing write tid 1 to smithi02979277-8 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3504: finishing write tid 2 to smithi02979277-8 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3504: finishing write tid 3 to smithi02979277-8 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3504: finishing write tid 4 to smithi02979277-8 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3504: finishing write tid 5 to smithi02979277-8 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3504: oid 8 updating version 0 to 356 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3504: oid 8 updating version 356 to 359 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3504: oid 8 version 359 is already newer than 357 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3504: oid 8 version 359 is already newer than 358 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 359 (ObjNum 1305 snap 445 seq_num 1305) dirty exists 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3504: left oid 8 (ObjNum 1305 snap 445 seq_num 1305) 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3505: finishing write tid 1 to smithi02979277-46 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3505: finishing write tid 2 to smithi02979277-46 2024-04-12T11:29:18.065 INFO:tasks.rados.rados.0.smithi029.stdout:3507: seq_num 1308 ranges {783556=438408,1674212=400111,2736451=420734,3882270=101286} 2024-04-12T11:29:18.071 INFO:tasks.rados.rados.0.smithi029.stdout:3507: writing smithi02979277-13 from 783556 to 1221964 tid 1 2024-04-12T11:29:18.076 INFO:tasks.rados.rados.0.smithi029.stdout:3507: writing smithi02979277-13 from 1674212 to 2074323 tid 2 2024-04-12T11:29:18.082 INFO:tasks.rados.rados.0.smithi029.stdout:3507: writing smithi02979277-13 from 2736451 to 3157185 tid 3 2024-04-12T11:29:18.089 INFO:tasks.rados.rados.0.smithi029.stdout:3507: writing smithi02979277-13 from 3882270 to 3983556 tid 4 2024-04-12T11:29:18.089 INFO:tasks.rados.rados.0.smithi029.stdout:3500: done (6 left) 2024-04-12T11:29:18.089 INFO:tasks.rados.rados.0.smithi029.stdout:3501: done (5 left) 2024-04-12T11:29:18.089 INFO:tasks.rados.rados.0.smithi029.stdout:3502: done (4 left) 2024-04-12T11:29:18.089 INFO:tasks.rados.rados.0.smithi029.stdout:3504: done (3 left) 2024-04-12T11:29:18.089 INFO:tasks.rados.rados.0.smithi029.stdout:3508: rollback oid 24 current snap is 445 2024-04-12T11:29:18.089 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 24 to 442 2024-04-12T11:29:18.090 INFO:tasks.rados.rados.0.smithi029.stdout:3505: finishing write tid 3 to smithi02979277-46 2024-04-12T11:29:18.090 INFO:tasks.rados.rados.0.smithi029.stdout:3505: finishing write tid 4 to smithi02979277-46 2024-04-12T11:29:18.090 INFO:tasks.rados.rados.0.smithi029.stdout:3505: finishing write tid 5 to smithi02979277-46 2024-04-12T11:29:18.090 INFO:tasks.rados.rados.0.smithi029.stdout:3505: oid 46 updating version 0 to 1171 2024-04-12T11:29:18.090 INFO:tasks.rados.rados.0.smithi029.stdout:3505: oid 46 version 1171 is already newer than 1170 2024-04-12T11:29:18.090 INFO:tasks.rados.rados.0.smithi029.stdout:3505: oid 46 updating version 1171 to 1172 2024-04-12T11:29:18.090 INFO:tasks.rados.rados.0.smithi029.stdout:3505: oid 46 version 1172 is already newer than 1169 2024-04-12T11:29:18.090 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 1172 (ObjNum 1306 snap 445 seq_num 1306) dirty exists 2024-04-12T11:29:18.090 INFO:tasks.rados.rados.0.smithi029.stdout:3505: left oid 46 (ObjNum 1306 snap 445 seq_num 1306) 2024-04-12T11:29:18.090 INFO:tasks.rados.rados.0.smithi029.stdout:3506: finishing write tid 1 to smithi02979277-23 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3506: finishing write tid 2 to smithi02979277-23 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3505: done (3 left) 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3509: write oid 50 current snap is 445 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3506: finishing write tid 3 to smithi02979277-23 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3506: finishing write tid 4 to smithi02979277-23 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3506: finishing write tid 5 to smithi02979277-23 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3506: oid 23 updating version 0 to 1488 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3506: oid 23 updating version 1488 to 1489 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3506: oid 23 updating version 1489 to 1490 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3506: oid 23 updating version 1490 to 1491 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1491 (ObjNum 1307 snap 445 seq_num 1307) dirty exists 2024-04-12T11:29:18.091 INFO:tasks.rados.rados.0.smithi029.stdout:3506: left oid 23 (ObjNum 1307 snap 445 seq_num 1307) 2024-04-12T11:29:18.092 INFO:tasks.rados.rados.0.smithi029.stdout:3507: finishing write tid 1 to smithi02979277-13 2024-04-12T11:29:18.092 INFO:tasks.rados.rados.0.smithi029.stdout:3507: finishing write tid 2 to smithi02979277-13 2024-04-12T11:29:18.092 INFO:tasks.rados.rados.0.smithi029.stdout:3507: finishing write tid 3 to smithi02979277-13 2024-04-12T11:29:18.092 INFO:tasks.rados.rados.0.smithi029.stdout:3509: seq_num 1309 ranges {400363=513657,1332694=660122} 2024-04-12T11:29:18.092 INFO:tasks.rados.rados.0.smithi029.stdout:3509: writing smithi02979277-50 from 400363 to 914020 tid 1 2024-04-12T11:29:18.098 INFO:tasks.rados.rados.0.smithi029.stdout:3509: writing smithi02979277-50 from 1332694 to 1992816 tid 2 2024-04-12T11:29:18.098 INFO:tasks.rados.rados.0.smithi029.stdout:3506: done (3 left) 2024-04-12T11:29:18.098 INFO:tasks.rados.rados.0.smithi029.stdout:3510: rollback oid 27 current snap is 445 2024-04-12T11:29:18.098 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 432 2024-04-12T11:29:18.098 INFO:tasks.rados.rados.0.smithi029.stdout:3507: finishing write tid 4 to smithi02979277-13 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3508: finishing rollback tid 0 to smithi02979277-24 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3507: finishing write tid 5 to smithi02979277-13 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3508: finishing rollback tid 1 to smithi02979277-24 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3507: finishing write tid 6 to smithi02979277-13 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3507: oid 13 updating version 0 to 1834 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3507: oid 13 version 1834 is already newer than 1832 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3507: oid 13 version 1834 is already newer than 1830 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3507: oid 13 version 1834 is already newer than 1831 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3507: oid 13 version 1834 is already newer than 1833 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1834 (ObjNum 1308 snap 445 seq_num 1308) dirty exists 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3507: left oid 13 (ObjNum 1308 snap 445 seq_num 1308) 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3508: finishing rollback tid 2 to smithi02979277-24 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 2231 (ObjNum 1278 snap 437 seq_num 1278) dirty exists 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3509: finishing write tid 1 to smithi02979277-50 2024-04-12T11:29:18.099 INFO:tasks.rados.rados.0.smithi029.stdout:3507: done (3 left) 2024-04-12T11:29:18.100 INFO:tasks.rados.rados.0.smithi029.stdout:3508: done (2 left) 2024-04-12T11:29:18.100 INFO:tasks.rados.rados.0.smithi029.stdout:3511: read oid 11 snap -1 2024-04-12T11:29:18.100 INFO:tasks.rados.rados.0.smithi029.stdout:3511: expect (ObjNum 1285 snap 442 seq_num 1285) 2024-04-12T11:29:18.100 INFO:tasks.rados.rados.0.smithi029.stdout:3512: snap_create 2024-04-12T11:29:18.101 INFO:tasks.rados.rados.0.smithi029.stdout:3510: finishing rollback tid 0 to smithi02979277-27 2024-04-12T11:29:18.101 INFO:tasks.rados.rados.0.smithi029.stdout:3510: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:29:18.101 INFO:tasks.rados.rados.0.smithi029.stdout:3510: finishing rollback tid 2 to smithi02979277-27 2024-04-12T11:29:18.101 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 1091 (ObjNum 1235 snap 415 seq_num 1235) dirty exists 2024-04-12T11:29:18.103 INFO:tasks.rados.rados.0.smithi029.stdout:3511: expect (ObjNum 1285 snap 442 seq_num 1285) 2024-04-12T11:29:18.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:17 smithi087 ceph-mon[89014]: osdmap e952: 8 total, 8 up, 8 in 2024-04-12T11:29:18.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:17 smithi087 ceph-mon[89014]: osdmap e953: 8 total, 8 up, 8 in 2024-04-12T11:29:18.368 INFO:tasks.rados.rados.0.smithi029.stdout:3509: finishing write tid 2 to smithi02979277-50 2024-04-12T11:29:18.368 INFO:tasks.rados.rados.0.smithi029.stdout:3509: finishing write tid 3 to smithi02979277-50 2024-04-12T11:29:18.368 INFO:tasks.rados.rados.0.smithi029.stdout:3509: finishing write tid 4 to smithi02979277-50 2024-04-12T11:29:18.368 INFO:tasks.rados.rados.0.smithi029.stdout:3509: oid 50 updating version 0 to 1494 2024-04-12T11:29:18.368 INFO:tasks.rados.rados.0.smithi029.stdout:3509: oid 50 version 1494 is already newer than 1493 2024-04-12T11:29:18.368 INFO:tasks.rados.rados.0.smithi029.stdout:3509: oid 50 updating version 1494 to 1495 2024-04-12T11:29:18.368 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1495 (ObjNum 1309 snap 445 seq_num 1309) dirty exists 2024-04-12T11:29:18.368 INFO:tasks.rados.rados.0.smithi029.stdout:3509: left oid 50 (ObjNum 1309 snap 445 seq_num 1309) 2024-04-12T11:29:18.729 INFO:tasks.rados.rados.0.smithi029.stdout:3509: done (3 left) 2024-04-12T11:29:18.730 INFO:tasks.rados.rados.0.smithi029.stdout:3510: done (2 left) 2024-04-12T11:29:18.730 INFO:tasks.rados.rados.0.smithi029.stdout:3511: done (1 left) 2024-04-12T11:29:18.730 INFO:tasks.rados.rados.0.smithi029.stdout:3512: done (0 left) 2024-04-12T11:29:18.730 INFO:tasks.rados.rados.0.smithi029.stdout:3513: rollback oid 36 current snap is 446 2024-04-12T11:29:18.730 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 36 to 429 2024-04-12T11:29:18.730 INFO:tasks.rados.rados.0.smithi029.stdout:3514: rollback oid 29 current snap is 446 2024-04-12T11:29:18.730 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 29 to 440 2024-04-12T11:29:18.730 INFO:tasks.rados.rados.0.smithi029.stdout:3515: snap_create 2024-04-12T11:29:18.733 INFO:tasks.rados.rados.0.smithi029.stdout:3513: finishing rollback tid 0 to smithi02979277-36 2024-04-12T11:29:18.733 INFO:tasks.rados.rados.0.smithi029.stdout:3513: finishing rollback tid 1 to smithi02979277-36 2024-04-12T11:29:18.734 INFO:tasks.rados.rados.0.smithi029.stdout:3513: finishing rollback tid 2 to smithi02979277-36 2024-04-12T11:29:18.734 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 864 (ObjNum 1177 snap 400 seq_num 1177) dirty exists 2024-04-12T11:29:18.734 INFO:tasks.rados.rados.0.smithi029.stdout:3514: finishing rollback tid 1 to smithi02979277-29 2024-04-12T11:29:18.734 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1508 (ObjNum 129 snap 0 seq_num 1800811680) dirty dne 2024-04-12T11:29:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:18 smithi029 ceph-mon[93650]: pgmap v1065: 89 pgs: 89 active+clean; 208 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-04-12T11:29:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:18 smithi029 ceph-mon[97787]: pgmap v1065: 89 pgs: 89 active+clean; 208 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-04-12T11:29:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:18 smithi087 ceph-mon[89014]: pgmap v1065: 89 pgs: 89 active+clean; 208 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-04-12T11:29:19.732 INFO:tasks.rados.rados.0.smithi029.stdout:3513: done (2 left) 2024-04-12T11:29:19.732 INFO:tasks.rados.rados.0.smithi029.stdout:3514: done (1 left) 2024-04-12T11:29:19.732 INFO:tasks.rados.rados.0.smithi029.stdout:3515: done (0 left) 2024-04-12T11:29:19.732 INFO:tasks.rados.rados.0.smithi029.stdout:3516: snap_create 2024-04-12T11:29:20.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:19 smithi029 ceph-mon[97787]: osdmap e954: 8 total, 8 up, 8 in 2024-04-12T11:29:20.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:19 smithi029 ceph-mon[93650]: osdmap e954: 8 total, 8 up, 8 in 2024-04-12T11:29:20.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:19 smithi087 ceph-mon[89014]: osdmap e954: 8 total, 8 up, 8 in 2024-04-12T11:29:20.733 INFO:tasks.rados.rados.0.smithi029.stdout:3516: done (0 left) 2024-04-12T11:29:20.734 INFO:tasks.rados.rados.0.smithi029.stdout:3517: delete oid 2 current snap is 448 2024-04-12T11:29:20.738 INFO:tasks.rados.rados.0.smithi029.stdout:3517: done (0 left) 2024-04-12T11:29:20.738 INFO:tasks.rados.rados.0.smithi029.stdout:3518: rollback oid 20 current snap is 448 2024-04-12T11:29:20.738 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 20 to 442 2024-04-12T11:29:20.738 INFO:tasks.rados.rados.0.smithi029.stdout:3519: write oid 39 current snap is 448 2024-04-12T11:29:20.738 INFO:tasks.rados.rados.0.smithi029.stdout:3519: seq_num 1311 ranges {433977=664155,1733305=300672} 2024-04-12T11:29:20.744 INFO:tasks.rados.rados.0.smithi029.stdout:3519: writing smithi02979277-39 from 433977 to 1098132 tid 1 2024-04-12T11:29:20.749 INFO:tasks.rados.rados.0.smithi029.stdout:3519: writing smithi02979277-39 from 1733305 to 2033977 tid 2 2024-04-12T11:29:20.750 INFO:tasks.rados.rados.0.smithi029.stdout:3520: snap_remove snap 447 2024-04-12T11:29:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:20 smithi029 ceph-mon[93650]: pgmap v1068: 89 pgs: 89 active+clean; 195 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 4.7 MiB/s wr, 37 op/s 2024-04-12T11:29:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:20 smithi029 ceph-mon[93650]: osdmap e955: 8 total, 8 up, 8 in 2024-04-12T11:29:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:20 smithi029 ceph-mon[93650]: osdmap e956: 8 total, 8 up, 8 in 2024-04-12T11:29:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:20 smithi029 ceph-mon[97787]: pgmap v1068: 89 pgs: 89 active+clean; 195 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 4.7 MiB/s wr, 37 op/s 2024-04-12T11:29:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:20 smithi029 ceph-mon[97787]: osdmap e955: 8 total, 8 up, 8 in 2024-04-12T11:29:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:20 smithi029 ceph-mon[97787]: osdmap e956: 8 total, 8 up, 8 in 2024-04-12T11:29:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:20 smithi087 ceph-mon[89014]: pgmap v1068: 89 pgs: 89 active+clean; 195 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 4.7 MiB/s wr, 37 op/s 2024-04-12T11:29:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:20 smithi087 ceph-mon[89014]: osdmap e955: 8 total, 8 up, 8 in 2024-04-12T11:29:21.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:20 smithi087 ceph-mon[89014]: osdmap e956: 8 total, 8 up, 8 in 2024-04-12T11:29:21.747 INFO:tasks.rados.rados.0.smithi029.stdout:3520: done (2 left) 2024-04-12T11:29:21.747 INFO:tasks.rados.rados.0.smithi029.stdout:3521: write oid 7 current snap is 448 2024-04-12T11:29:21.747 INFO:tasks.rados.rados.0.smithi029.stdout:3518: finishing rollback tid 0 to smithi02979277-20 2024-04-12T11:29:21.747 INFO:tasks.rados.rados.0.smithi029.stdout:3521: seq_num 1312 ranges {450784=739404,1791787=258997} 2024-04-12T11:29:21.753 INFO:tasks.rados.rados.0.smithi029.stdout:3521: writing smithi02979277-7 from 450784 to 1190188 tid 1 2024-04-12T11:29:21.759 INFO:tasks.rados.rados.0.smithi029.stdout:3521: writing smithi02979277-7 from 1791787 to 2050784 tid 2 2024-04-12T11:29:21.759 INFO:tasks.rados.rados.0.smithi029.stdout:3522: rollback oid 26 current snap is 448 2024-04-12T11:29:21.759 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 433 2024-04-12T11:29:21.759 INFO:tasks.rados.rados.0.smithi029.stdout:3518: finishing rollback tid 1 to smithi02979277-20 2024-04-12T11:29:21.759 INFO:tasks.rados.rados.0.smithi029.stdout:3518: finishing rollback tid 2 to smithi02979277-20 2024-04-12T11:29:21.759 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 1176 (ObjNum 1282 snap 441 seq_num 1282) dirty exists 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3519: finishing write tid 1 to smithi02979277-39 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3519: finishing write tid 2 to smithi02979277-39 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3519: finishing write tid 3 to smithi02979277-39 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3519: finishing write tid 4 to smithi02979277-39 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3519: oid 39 updating version 0 to 503 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3519: oid 39 version 503 is already newer than 502 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3519: oid 39 version 503 is already newer than 501 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 503 (ObjNum 1311 snap 448 seq_num 1311) dirty exists 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3519: left oid 39 (ObjNum 1311 snap 448 seq_num 1311) 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3518: done (3 left) 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3519: done (2 left) 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3523: snap_create 2024-04-12T11:29:21.760 INFO:tasks.rados.rados.0.smithi029.stdout:3521: finishing write tid 1 to smithi02979277-7 2024-04-12T11:29:21.763 INFO:tasks.rados.rados.0.smithi029.stdout:3521: finishing write tid 2 to smithi02979277-7 2024-04-12T11:29:21.763 INFO:tasks.rados.rados.0.smithi029.stdout:3521: finishing write tid 3 to smithi02979277-7 2024-04-12T11:29:21.764 INFO:tasks.rados.rados.0.smithi029.stdout:3521: finishing write tid 4 to smithi02979277-7 2024-04-12T11:29:21.764 INFO:tasks.rados.rados.0.smithi029.stdout:3521: oid 7 updating version 0 to 1762 2024-04-12T11:29:21.764 INFO:tasks.rados.rados.0.smithi029.stdout:3521: oid 7 version 1762 is already newer than 1761 2024-04-12T11:29:21.764 INFO:tasks.rados.rados.0.smithi029.stdout:3521: oid 7 updating version 1762 to 1763 2024-04-12T11:29:21.764 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1763 (ObjNum 1312 snap 448 seq_num 1312) dirty exists 2024-04-12T11:29:21.764 INFO:tasks.rados.rados.0.smithi029.stdout:3521: left oid 7 (ObjNum 1312 snap 448 seq_num 1312) 2024-04-12T11:29:21.768 INFO:tasks.rados.rados.0.smithi029.stdout:3522: finishing rollback tid 0 to smithi02979277-26 2024-04-12T11:29:21.769 INFO:tasks.rados.rados.0.smithi029.stdout:3522: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:29:21.769 INFO:tasks.rados.rados.0.smithi029.stdout:3522: finishing rollback tid 2 to smithi02979277-26 2024-04-12T11:29:21.769 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 2235 (ObjNum 938 snap 325 seq_num 938) dirty exists 2024-04-12T11:29:22.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:22.118 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:22.381 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:29:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:29:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:29:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:3521: done (2 left) 2024-04-12T11:29:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:3522: done (1 left) 2024-04-12T11:29:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:3523: done (0 left) 2024-04-12T11:29:22.706 INFO:tasks.rados.rados.0.smithi029.stdout:3524: delete oid 23 current snap is 449 2024-04-12T11:29:22.710 INFO:tasks.rados.rados.0.smithi029.stdout:3524: done (0 left) 2024-04-12T11:29:22.711 INFO:tasks.rados.rados.0.smithi029.stdout:3525: read oid 34 snap -1 2024-04-12T11:29:22.711 INFO:tasks.rados.rados.0.smithi029.stdout:3525: expect deleted 2024-04-12T11:29:22.711 INFO:tasks.rados.rados.0.smithi029.stdout:3526: snap_remove snap 439 2024-04-12T11:29:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:22 smithi029 ceph-mon[93650]: pgmap v1071: 89 pgs: 89 active+clean; 203 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 4.4 MiB/s wr, 29 op/s 2024-04-12T11:29:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:22 smithi029 ceph-mon[93650]: osdmap e957: 8 total, 8 up, 8 in 2024-04-12T11:29:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:22 smithi029 ceph-mon[93650]: osdmap e958: 8 total, 8 up, 8 in 2024-04-12T11:29:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:22 smithi029 ceph-mon[97787]: pgmap v1071: 89 pgs: 89 active+clean; 203 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 4.4 MiB/s wr, 29 op/s 2024-04-12T11:29:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:22 smithi029 ceph-mon[97787]: osdmap e957: 8 total, 8 up, 8 in 2024-04-12T11:29:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:22 smithi029 ceph-mon[97787]: osdmap e958: 8 total, 8 up, 8 in 2024-04-12T11:29:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:22 smithi087 ceph-mon[89014]: pgmap v1071: 89 pgs: 89 active+clean; 203 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 4.4 MiB/s wr, 29 op/s 2024-04-12T11:29:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:22 smithi087 ceph-mon[89014]: osdmap e957: 8 total, 8 up, 8 in 2024-04-12T11:29:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:22 smithi087 ceph-mon[89014]: osdmap e958: 8 total, 8 up, 8 in 2024-04-12T11:29:23.708 INFO:tasks.rados.rados.0.smithi029.stdout:3526: done (1 left) 2024-04-12T11:29:23.708 INFO:tasks.rados.rados.0.smithi029.stdout:3527: write oid 22 current snap is 449 2024-04-12T11:29:23.708 INFO:tasks.rados.rados.0.smithi029.stdout:3527: seq_num 1314 ranges {484398=489902,1392398=411118} 2024-04-12T11:29:23.714 INFO:tasks.rados.rados.0.smithi029.stdout:3527: writing smithi02979277-22 from 484398 to 974300 tid 1 2024-04-12T11:29:23.720 INFO:tasks.rados.rados.0.smithi029.stdout:3527: writing smithi02979277-22 from 1392398 to 1803516 tid 2 2024-04-12T11:29:23.720 INFO:tasks.rados.rados.0.smithi029.stdout:3528: delete oid 7 current snap is 449 2024-04-12T11:29:23.720 INFO:tasks.rados.rados.0.smithi029.stdout:3527: finishing write tid 1 to smithi02979277-22 2024-04-12T11:29:23.725 INFO:tasks.rados.rados.0.smithi029.stdout:3525: done (2 left) 2024-04-12T11:29:23.725 INFO:tasks.rados.rados.0.smithi029.stdout:3528: done (1 left) 2024-04-12T11:29:23.725 INFO:tasks.rados.rados.0.smithi029.stdout:3529: read oid 35 snap -1 2024-04-12T11:29:23.725 INFO:tasks.rados.rados.0.smithi029.stdout:3529: expect deleted 2024-04-12T11:29:23.725 INFO:tasks.rados.rados.0.smithi029.stdout:3530: write oid 31 current snap is 449 2024-04-12T11:29:23.725 INFO:tasks.rados.rados.0.smithi029.stdout:3530: seq_num 1316 ranges {518012=756753,2025715=92297} 2024-04-12T11:29:23.731 INFO:tasks.rados.rados.0.smithi029.stdout:3530: writing smithi02979277-31 from 518012 to 1274765 tid 1 2024-04-12T11:29:23.736 INFO:tasks.rados.rados.0.smithi029.stdout:3530: writing smithi02979277-31 from 2025715 to 2118012 tid 2 2024-04-12T11:29:23.736 INFO:tasks.rados.rados.0.smithi029.stdout:3531: delete oid 2 current snap is 449 2024-04-12T11:29:23.737 INFO:tasks.rados.rados.0.smithi029.stdout:3527: finishing write tid 2 to smithi02979277-22 2024-04-12T11:29:23.737 INFO:tasks.rados.rados.0.smithi029.stdout:3527: finishing write tid 3 to smithi02979277-22 2024-04-12T11:29:23.737 INFO:tasks.rados.rados.0.smithi029.stdout:3527: finishing write tid 4 to smithi02979277-22 2024-04-12T11:29:23.737 INFO:tasks.rados.rados.0.smithi029.stdout:3527: oid 22 updating version 0 to 1504 2024-04-12T11:29:23.737 INFO:tasks.rados.rados.0.smithi029.stdout:3527: oid 22 version 1504 is already newer than 1503 2024-04-12T11:29:23.737 INFO:tasks.rados.rados.0.smithi029.stdout:3527: oid 22 updating version 1504 to 1505 2024-04-12T11:29:23.737 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 1505 (ObjNum 1314 snap 449 seq_num 1314) dirty exists 2024-04-12T11:29:23.737 INFO:tasks.rados.rados.0.smithi029.stdout:3527: left oid 22 (ObjNum 1314 snap 449 seq_num 1314) 2024-04-12T11:29:23.737 INFO:tasks.rados.rados.0.smithi029.stdout:3530: finishing write tid 1 to smithi02979277-31 2024-04-12T11:29:23.738 INFO:tasks.rados.rados.0.smithi029.stdout:3527: done (3 left) 2024-04-12T11:29:23.738 INFO:tasks.rados.rados.0.smithi029.stdout:3529: done (2 left) 2024-04-12T11:29:23.738 INFO:tasks.rados.rados.0.smithi029.stdout:3531: done (1 left) 2024-04-12T11:29:23.738 INFO:tasks.rados.rados.0.smithi029.stdout:3532: write oid 42 current snap is 449 2024-04-12T11:29:23.738 INFO:tasks.rados.rados.0.smithi029.stdout:3532: seq_num 1318 ranges {551626=507251,1626326=418456} 2024-04-12T11:29:23.744 INFO:tasks.rados.rados.0.smithi029.stdout:3532: writing smithi02979277-42 from 551626 to 1058877 tid 1 2024-04-12T11:29:23.750 INFO:tasks.rados.rados.0.smithi029.stdout:3532: writing smithi02979277-42 from 1626326 to 2044782 tid 2 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3533: read oid 26 snap 424 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3533: expect (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3534: rollback oid 5 current snap is 449 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 5 to 442 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3530: finishing write tid 2 to smithi02979277-31 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3530: finishing write tid 3 to smithi02979277-31 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3530: finishing write tid 4 to smithi02979277-31 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3530: oid 31 updating version 0 to 1414 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3530: oid 31 version 1414 is already newer than 1413 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3530: oid 31 updating version 1414 to 1415 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1415 (ObjNum 1316 snap 449 seq_num 1316) dirty exists 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3530: left oid 31 (ObjNum 1316 snap 449 seq_num 1316) 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3532: finishing write tid 1 to smithi02979277-42 2024-04-12T11:29:23.751 INFO:tasks.rados.rados.0.smithi029.stdout:3530: done (3 left) 2024-04-12T11:29:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:3535: read oid 47 snap 429 2024-04-12T11:29:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:3535: expect deleted 2024-04-12T11:29:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:3536: rollback oid 12 current snap is 449 2024-04-12T11:29:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 12 to 446 2024-04-12T11:29:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:3537: read oid 11 snap -1 2024-04-12T11:29:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:3537: expect (ObjNum 1285 snap 442 seq_num 1285) 2024-04-12T11:29:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:3538: rollback oid 29 current snap is 449 2024-04-12T11:29:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 29 to 428 2024-04-12T11:29:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:3539: write oid 1 current snap is 449 2024-04-12T11:29:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:3539: seq_num 1319 ranges {568433=582500,1684808=483625} 2024-04-12T11:29:23.757 INFO:tasks.rados.rados.0.smithi029.stdout:3539: writing smithi02979277-1 from 568433 to 1150933 tid 1 2024-04-12T11:29:23.763 INFO:tasks.rados.rados.0.smithi029.stdout:3539: writing smithi02979277-1 from 1684808 to 2168433 tid 2 2024-04-12T11:29:23.763 INFO:tasks.rados.rados.0.smithi029.stdout:3540: snap_remove snap 434 2024-04-12T11:29:24.750 INFO:tasks.rados.rados.0.smithi029.stdout:3540: done (8 left) 2024-04-12T11:29:24.750 INFO:tasks.rados.rados.0.smithi029.stdout:3541: read oid 41 snap -1 2024-04-12T11:29:24.750 INFO:tasks.rados.rados.0.smithi029.stdout:3541: expect (ObjNum 1304 snap 445 seq_num 1304) 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3542: snap_create 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3532: finishing write tid 2 to smithi02979277-42 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3532: finishing write tid 3 to smithi02979277-42 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3532: finishing write tid 4 to smithi02979277-42 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3532: oid 42 updating version 0 to 443 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3532: oid 42 updating version 443 to 445 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3532: oid 42 version 445 is already newer than 444 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 445 (ObjNum 1318 snap 449 seq_num 1318) dirty exists 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3532: left oid 42 (ObjNum 1318 snap 449 seq_num 1318) 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3536: finishing rollback tid 0 to smithi02979277-12 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3534: finishing rollback tid 0 to smithi02979277-5 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3534: finishing rollback tid 1 to smithi02979277-5 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:3534: finishing rollback tid 2 to smithi02979277-5 2024-04-12T11:29:24.751 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1660 (ObjNum 1272 snap 436 seq_num 1272) dirty exists 2024-04-12T11:29:24.752 INFO:tasks.rados.rados.0.smithi029.stdout:3538: finishing rollback tid 1 to smithi02979277-29 2024-04-12T11:29:24.752 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1508 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:29:24.752 INFO:tasks.rados.rados.0.smithi029.stdout:3533: expect (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:29:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:24 smithi087 ceph-mon[89014]: pgmap v1074: 89 pgs: 89 active+clean; 212 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.0 MiB/s wr, 12 op/s 2024-04-12T11:29:24.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:24 smithi087 ceph-mon[89014]: osdmap e959: 8 total, 8 up, 8 in 2024-04-12T11:29:24.974 INFO:tasks.rados.rados.0.smithi029.stdout:3536: finishing rollback tid 1 to smithi02979277-12 2024-04-12T11:29:24.974 INFO:tasks.rados.rados.0.smithi029.stdout:3536: finishing rollback tid 2 to smithi02979277-12 2024-04-12T11:29:24.974 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1773 (ObjNum 1296 snap 444 seq_num 1296) dirty exists 2024-04-12T11:29:24.974 INFO:tasks.rados.rados.0.smithi029.stdout:3537: expect (ObjNum 1285 snap 442 seq_num 1285) 2024-04-12T11:29:25.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:24 smithi029 ceph-mon[97787]: pgmap v1074: 89 pgs: 89 active+clean; 212 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.0 MiB/s wr, 12 op/s 2024-04-12T11:29:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:24 smithi029 ceph-mon[97787]: osdmap e959: 8 total, 8 up, 8 in 2024-04-12T11:29:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:24 smithi029 ceph-mon[93650]: pgmap v1074: 89 pgs: 89 active+clean; 212 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 3.0 MiB/s wr, 12 op/s 2024-04-12T11:29:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:24 smithi029 ceph-mon[93650]: osdmap e959: 8 total, 8 up, 8 in 2024-04-12T11:29:25.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:29:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:29:24] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:29:25.239 INFO:tasks.rados.rados.0.smithi029.stdout:3539: finishing write tid 1 to smithi02979277-1 2024-04-12T11:29:25.239 INFO:tasks.rados.rados.0.smithi029.stdout:3539: finishing write tid 2 to smithi02979277-1 2024-04-12T11:29:25.239 INFO:tasks.rados.rados.0.smithi029.stdout:3539: finishing write tid 3 to smithi02979277-1 2024-04-12T11:29:25.239 INFO:tasks.rados.rados.0.smithi029.stdout:3539: finishing write tid 4 to smithi02979277-1 2024-04-12T11:29:25.239 INFO:tasks.rados.rados.0.smithi029.stdout:3539: oid 1 updating version 0 to 1685 2024-04-12T11:29:25.239 INFO:tasks.rados.rados.0.smithi029.stdout:3539: oid 1 version 1685 is already newer than 1684 2024-04-12T11:29:25.239 INFO:tasks.rados.rados.0.smithi029.stdout:3539: oid 1 version 1685 is already newer than 1683 2024-04-12T11:29:25.239 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1685 (ObjNum 1319 snap 449 seq_num 1319) dirty exists 2024-04-12T11:29:25.239 INFO:tasks.rados.rados.0.smithi029.stdout:3539: left oid 1 (ObjNum 1319 snap 449 seq_num 1319) 2024-04-12T11:29:25.240 INFO:tasks.rados.rados.0.smithi029.stdout:3541: expect (ObjNum 1304 snap 445 seq_num 1304) 2024-04-12T11:29:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:3532: done (9 left) 2024-04-12T11:29:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:3533: done (8 left) 2024-04-12T11:29:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:3534: done (7 left) 2024-04-12T11:29:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:3535: done (6 left) 2024-04-12T11:29:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:3536: done (5 left) 2024-04-12T11:29:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:3537: done (4 left) 2024-04-12T11:29:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:3538: done (3 left) 2024-04-12T11:29:25.755 INFO:tasks.rados.rados.0.smithi029.stdout:3539: done (2 left) 2024-04-12T11:29:25.755 INFO:tasks.rados.rados.0.smithi029.stdout:3541: done (1 left) 2024-04-12T11:29:25.755 INFO:tasks.rados.rados.0.smithi029.stdout:3542: done (0 left) 2024-04-12T11:29:25.755 INFO:tasks.rados.rados.0.smithi029.stdout:3543: read oid 49 snap -1 2024-04-12T11:29:25.755 INFO:tasks.rados.rados.0.smithi029.stdout:3543: expect (ObjNum 1284 snap 442 seq_num 1284) 2024-04-12T11:29:25.755 INFO:tasks.rados.rados.0.smithi029.stdout:3544: snap_create 2024-04-12T11:29:25.759 INFO:tasks.rados.rados.0.smithi029.stdout:3543: expect (ObjNum 1284 snap 442 seq_num 1284) 2024-04-12T11:29:26.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:25 smithi029 ceph-mon[97787]: osdmap e960: 8 total, 8 up, 8 in 2024-04-12T11:29:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:25 smithi029 ceph-mon[93650]: osdmap e960: 8 total, 8 up, 8 in 2024-04-12T11:29:26.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:25 smithi087 ceph-mon[89014]: osdmap e960: 8 total, 8 up, 8 in 2024-04-12T11:29:26.756 INFO:tasks.rados.rados.0.smithi029.stdout:3543: done (1 left) 2024-04-12T11:29:26.756 INFO:tasks.rados.rados.0.smithi029.stdout:3544: done (0 left) 2024-04-12T11:29:26.756 INFO:tasks.rados.rados.0.smithi029.stdout:3545: write oid 33 current snap is 451 2024-04-12T11:29:26.756 INFO:tasks.rados.rados.0.smithi029.stdout:3545: seq_num 1320 ranges {585240=657749,1743290=422125} 2024-04-12T11:29:26.762 INFO:tasks.rados.rados.0.smithi029.stdout:3545: writing smithi02979277-33 from 585240 to 1242989 tid 1 2024-04-12T11:29:26.768 INFO:tasks.rados.rados.0.smithi029.stdout:3545: writing smithi02979277-33 from 1743290 to 2165415 tid 2 2024-04-12T11:29:26.768 INFO:tasks.rados.rados.0.smithi029.stdout:3546: write oid 11 current snap is 451 2024-04-12T11:29:26.768 INFO:tasks.rados.rados.0.smithi029.stdout:3546: seq_num 1321 ranges {602047=732998,1801772=400275} 2024-04-12T11:29:26.775 INFO:tasks.rados.rados.0.smithi029.stdout:3546: writing smithi02979277-11 from 602047 to 1335045 tid 1 2024-04-12T11:29:26.780 INFO:tasks.rados.rados.0.smithi029.stdout:3546: writing smithi02979277-11 from 1801772 to 2202047 tid 2 2024-04-12T11:29:26.780 INFO:tasks.rados.rados.0.smithi029.stdout:3547: rollback oid 15 current snap is 451 2024-04-12T11:29:26.780 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 15 to 446 2024-04-12T11:29:26.780 INFO:tasks.rados.rados.0.smithi029.stdout:3545: finishing write tid 1 to smithi02979277-33 2024-04-12T11:29:26.780 INFO:tasks.rados.rados.0.smithi029.stdout:3545: finishing write tid 2 to smithi02979277-33 2024-04-12T11:29:26.780 INFO:tasks.rados.rados.0.smithi029.stdout:3545: finishing write tid 3 to smithi02979277-33 2024-04-12T11:29:26.781 INFO:tasks.rados.rados.0.smithi029.stdout:3545: finishing write tid 4 to smithi02979277-33 2024-04-12T11:29:26.781 INFO:tasks.rados.rados.0.smithi029.stdout:3545: oid 33 updating version 0 to 1513 2024-04-12T11:29:26.781 INFO:tasks.rados.rados.0.smithi029.stdout:3545: oid 33 updating version 1513 to 1514 2024-04-12T11:29:26.781 INFO:tasks.rados.rados.0.smithi029.stdout:3545: oid 33 version 1514 is already newer than 1512 2024-04-12T11:29:26.781 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 1514 (ObjNum 1320 snap 451 seq_num 1320) dirty exists 2024-04-12T11:29:26.781 INFO:tasks.rados.rados.0.smithi029.stdout:3545: left oid 33 (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:29:26.781 INFO:tasks.rados.rados.0.smithi029.stdout:3545: done (2 left) 2024-04-12T11:29:26.781 INFO:tasks.rados.rados.0.smithi029.stdout:3548: write oid 13 current snap is 451 2024-04-12T11:29:26.781 INFO:tasks.rados.rados.0.smithi029.stdout:3548: seq_num 1322 ranges {618854=408247,1743901=425794} 2024-04-12T11:29:26.786 INFO:tasks.rados.rados.0.smithi029.stdout:3548: writing smithi02979277-13 from 618854 to 1027101 tid 1 2024-04-12T11:29:26.793 INFO:tasks.rados.rados.0.smithi029.stdout:3548: writing smithi02979277-13 from 1743901 to 2169695 tid 2 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3549: rollback oid 6 current snap is 451 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 6 to 448 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3546: finishing write tid 1 to smithi02979277-11 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3547: finishing rollback tid 0 to smithi02979277-15 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3546: finishing write tid 2 to smithi02979277-11 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3547: finishing rollback tid 1 to smithi02979277-15 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3546: finishing write tid 3 to smithi02979277-11 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3546: finishing write tid 4 to smithi02979277-11 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3546: oid 11 updating version 0 to 2251 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3546: oid 11 version 2251 is already newer than 2249 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3546: oid 11 version 2251 is already newer than 2250 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 2251 (ObjNum 1321 snap 451 seq_num 1321) dirty exists 2024-04-12T11:29:26.794 INFO:tasks.rados.rados.0.smithi029.stdout:3546: left oid 11 (ObjNum 1321 snap 451 seq_num 1321) 2024-04-12T11:29:26.795 INFO:tasks.rados.rados.0.smithi029.stdout:3546: done (3 left) 2024-04-12T11:29:26.795 INFO:tasks.rados.rados.0.smithi029.stdout:3550: delete oid 32 current snap is 451 2024-04-12T11:29:26.795 INFO:tasks.rados.rados.0.smithi029.stdout:3547: finishing rollback tid 2 to smithi02979277-15 2024-04-12T11:29:26.795 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 1099 (ObjNum 1300 snap 444 seq_num 1300) dirty exists 2024-04-12T11:29:26.795 INFO:tasks.rados.rados.0.smithi029.stdout:3548: finishing write tid 1 to smithi02979277-13 2024-04-12T11:29:26.795 INFO:tasks.rados.rados.0.smithi029.stdout:3549: finishing rollback tid 1 to smithi02979277-6 2024-04-12T11:29:26.795 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 814 (ObjNum 33 snap 0 seq_num 1777349880) dirty dne 2024-04-12T11:29:26.796 INFO:tasks.rados.rados.0.smithi029.stdout:3547: done (3 left) 2024-04-12T11:29:26.796 INFO:tasks.rados.rados.0.smithi029.stdout:3549: done (2 left) 2024-04-12T11:29:26.796 INFO:tasks.rados.rados.0.smithi029.stdout:3550: done (1 left) 2024-04-12T11:29:26.796 INFO:tasks.rados.rados.0.smithi029.stdout:3551: write oid 19 current snap is 451 2024-04-12T11:29:26.796 INFO:tasks.rados.rados.0.smithi029.stdout:3551: seq_num 1324 ranges {652468=675098,1977218=275250} 2024-04-12T11:29:26.802 INFO:tasks.rados.rados.0.smithi029.stdout:3551: writing smithi02979277-19 from 652468 to 1327566 tid 1 2024-04-12T11:29:26.807 INFO:tasks.rados.rados.0.smithi029.stdout:3551: writing smithi02979277-19 from 1977218 to 2252468 tid 2 2024-04-12T11:29:26.807 INFO:tasks.rados.rados.0.smithi029.stdout:3552: write oid 36 current snap is 451 2024-04-12T11:29:26.807 INFO:tasks.rados.rados.0.smithi029.stdout:3548: finishing write tid 2 to smithi02979277-13 2024-04-12T11:29:26.808 INFO:tasks.rados.rados.0.smithi029.stdout:3552: seq_num 1325 ranges {669275=750347,2035700=233575} 2024-04-12T11:29:26.815 INFO:tasks.rados.rados.0.smithi029.stdout:3552: writing smithi02979277-36 from 669275 to 1419622 tid 1 2024-04-12T11:29:26.820 INFO:tasks.rados.rados.0.smithi029.stdout:3552: writing smithi02979277-36 from 2035700 to 2269275 tid 2 2024-04-12T11:29:26.820 INFO:tasks.rados.rados.0.smithi029.stdout:3553: read oid 27 snap -1 2024-04-12T11:29:26.820 INFO:tasks.rados.rados.0.smithi029.stdout:3553: expect (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:29:26.820 INFO:tasks.rados.rados.0.smithi029.stdout:3554: snap_remove snap 448 2024-04-12T11:29:27.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:26 smithi029 ceph-mon[97787]: pgmap v1077: 89 pgs: 89 active+clean; 216 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-04-12T11:29:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:26 smithi029 ceph-mon[97787]: osdmap e961: 8 total, 8 up, 8 in 2024-04-12T11:29:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:26 smithi029 ceph-mon[93650]: pgmap v1077: 89 pgs: 89 active+clean; 216 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-04-12T11:29:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:26 smithi029 ceph-mon[93650]: osdmap e961: 8 total, 8 up, 8 in 2024-04-12T11:29:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:26 smithi087 ceph-mon[89014]: pgmap v1077: 89 pgs: 89 active+clean; 216 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-04-12T11:29:27.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:26 smithi087 ceph-mon[89014]: osdmap e961: 8 total, 8 up, 8 in 2024-04-12T11:29:27.707 INFO:tasks.rados.rados.0.smithi029.stdout:3554: done (4 left) 2024-04-12T11:29:27.708 INFO:tasks.rados.rados.0.smithi029.stdout:3555: snap_remove snap 440 2024-04-12T11:29:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:27 smithi029 ceph-mon[93650]: osdmap e962: 8 total, 8 up, 8 in 2024-04-12T11:29:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:27 smithi029 ceph-mon[93650]: osdmap e963: 8 total, 8 up, 8 in 2024-04-12T11:29:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:27 smithi029 ceph-mon[97787]: osdmap e962: 8 total, 8 up, 8 in 2024-04-12T11:29:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:27 smithi029 ceph-mon[97787]: osdmap e963: 8 total, 8 up, 8 in 2024-04-12T11:29:28.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:27 smithi087 ceph-mon[89014]: osdmap e962: 8 total, 8 up, 8 in 2024-04-12T11:29:28.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:27 smithi087 ceph-mon[89014]: osdmap e963: 8 total, 8 up, 8 in 2024-04-12T11:29:28.710 INFO:tasks.rados.rados.0.smithi029.stdout:3548: finishing write tid 3 to smithi02979277-13 2024-04-12T11:29:28.710 INFO:tasks.rados.rados.0.smithi029.stdout:3548: finishing write tid 4 to smithi02979277-13 2024-04-12T11:29:28.710 INFO:tasks.rados.rados.0.smithi029.stdout:3548: oid 13 updating version 0 to 1850 2024-04-12T11:29:28.710 INFO:tasks.rados.rados.0.smithi029.stdout:3548: oid 13 updating version 1850 to 1851 2024-04-12T11:29:28.710 INFO:tasks.rados.rados.0.smithi029.stdout:3548: oid 13 updating version 1851 to 1852 2024-04-12T11:29:28.710 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1852 (ObjNum 1322 snap 451 seq_num 1322) dirty exists 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3548: left oid 13 (ObjNum 1322 snap 451 seq_num 1322) 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3551: finishing write tid 1 to smithi02979277-19 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3551: finishing write tid 2 to smithi02979277-19 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3551: finishing write tid 3 to smithi02979277-19 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3551: finishing write tid 4 to smithi02979277-19 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3551: oid 19 updating version 0 to 1670 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3551: oid 19 updating version 1670 to 1672 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3551: oid 19 version 1672 is already newer than 1671 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1672 (ObjNum 1324 snap 451 seq_num 1324) dirty exists 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3551: left oid 19 (ObjNum 1324 snap 451 seq_num 1324) 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3552: finishing write tid 1 to smithi02979277-36 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3552: finishing write tid 2 to smithi02979277-36 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3552: finishing write tid 3 to smithi02979277-36 2024-04-12T11:29:28.711 INFO:tasks.rados.rados.0.smithi029.stdout:3552: finishing write tid 4 to smithi02979277-36 2024-04-12T11:29:28.712 INFO:tasks.rados.rados.0.smithi029.stdout:3552: oid 36 updating version 0 to 871 2024-04-12T11:29:28.712 INFO:tasks.rados.rados.0.smithi029.stdout:3552: oid 36 updating version 871 to 872 2024-04-12T11:29:28.712 INFO:tasks.rados.rados.0.smithi029.stdout:3552: oid 36 updating version 872 to 873 2024-04-12T11:29:28.712 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 873 (ObjNum 1325 snap 451 seq_num 1325) dirty exists 2024-04-12T11:29:28.712 INFO:tasks.rados.rados.0.smithi029.stdout:3552: left oid 36 (ObjNum 1325 snap 451 seq_num 1325) 2024-04-12T11:29:28.712 INFO:tasks.rados.rados.0.smithi029.stdout:3553: expect (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:29:28.883 INFO:tasks.rados.rados.0.smithi029.stdout:3548: done (4 left) 2024-04-12T11:29:28.883 INFO:tasks.rados.rados.0.smithi029.stdout:3551: done (3 left) 2024-04-12T11:29:28.883 INFO:tasks.rados.rados.0.smithi029.stdout:3552: done (2 left) 2024-04-12T11:29:28.883 INFO:tasks.rados.rados.0.smithi029.stdout:3553: done (1 left) 2024-04-12T11:29:28.883 INFO:tasks.rados.rados.0.smithi029.stdout:3555: done (0 left) 2024-04-12T11:29:28.883 INFO:tasks.rados.rados.0.smithi029.stdout:3556: write oid 46 current snap is 451 2024-04-12T11:29:28.883 INFO:tasks.rados.rados.0.smithi029.stdout:3556: seq_num 1326 ranges {686082=425596,1577829=433132} 2024-04-12T11:29:28.889 INFO:tasks.rados.rados.0.smithi029.stdout:3556: writing smithi02979277-46 from 686082 to 1111678 tid 1 2024-04-12T11:29:28.894 INFO:tasks.rados.rados.0.smithi029.stdout:3556: writing smithi02979277-46 from 1577829 to 2010961 tid 2 2024-04-12T11:29:28.894 INFO:tasks.rados.rados.0.smithi029.stdout:3557: write oid 3 current snap is 451 2024-04-12T11:29:28.894 INFO:tasks.rados.rados.0.smithi029.stdout:3557: seq_num 1327 ranges {702889=500845,1636311=576790} 2024-04-12T11:29:28.901 INFO:tasks.rados.rados.0.smithi029.stdout:3557: writing smithi02979277-3 from 702889 to 1203734 tid 1 2024-04-12T11:29:28.906 INFO:tasks.rados.rados.0.smithi029.stdout:3557: writing smithi02979277-3 from 1636311 to 2213101 tid 2 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:3558: delete oid 10 current snap is 451 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:3556: finishing write tid 1 to smithi02979277-46 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:3556: finishing write tid 2 to smithi02979277-46 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:3556: finishing write tid 3 to smithi02979277-46 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:3556: finishing write tid 4 to smithi02979277-46 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:3556: oid 46 updating version 0 to 1193 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:3556: oid 46 updating version 1193 to 1194 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:3556: oid 46 version 1194 is already newer than 1192 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 1194 (ObjNum 1326 snap 451 seq_num 1326) dirty exists 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:3556: left oid 46 (ObjNum 1326 snap 451 seq_num 1326) 2024-04-12T11:29:28.907 INFO:tasks.rados.rados.0.smithi029.stdout:3557: finishing write tid 1 to smithi02979277-3 2024-04-12T11:29:28.909 INFO:tasks.rados.rados.0.smithi029.stdout:3556: done (2 left) 2024-04-12T11:29:28.909 INFO:tasks.rados.rados.0.smithi029.stdout:3558: done (1 left) 2024-04-12T11:29:28.909 INFO:tasks.rados.rados.0.smithi029.stdout:3559: write oid 45 current snap is 451 2024-04-12T11:29:28.909 INFO:tasks.rados.rados.0.smithi029.stdout:3559: seq_num 1329 ranges {736503=651343,2153275=183228} 2024-04-12T11:29:28.916 INFO:tasks.rados.rados.0.smithi029.stdout:3559: writing smithi02979277-45 from 736503 to 1387846 tid 1 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:3559: writing smithi02979277-45 from 2153275 to 2336503 tid 2 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:3560: delete oid 5 current snap is 451 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:3557: finishing write tid 2 to smithi02979277-3 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:3557: finishing write tid 3 to smithi02979277-3 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:3557: finishing write tid 4 to smithi02979277-3 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:3557: oid 3 updating version 0 to 2262 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:3557: oid 3 version 2262 is already newer than 2261 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:3557: oid 3 version 2262 is already newer than 2260 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 2262 (ObjNum 1327 snap 451 seq_num 1327) dirty exists 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:3557: left oid 3 (ObjNum 1327 snap 451 seq_num 1327) 2024-04-12T11:29:28.922 INFO:tasks.rados.rados.0.smithi029.stdout:3559: finishing write tid 1 to smithi02979277-45 2024-04-12T11:29:28.924 INFO:tasks.rados.rados.0.smithi029.stdout:3557: done (2 left) 2024-04-12T11:29:28.924 INFO:tasks.rados.rados.0.smithi029.stdout:3560: done (1 left) 2024-04-12T11:29:28.924 INFO:tasks.rados.rados.0.smithi029.stdout:3561: snap_create 2024-04-12T11:29:28.925 INFO:tasks.rados.rados.0.smithi029.stdout:3559: finishing write tid 2 to smithi02979277-45 2024-04-12T11:29:28.925 INFO:tasks.rados.rados.0.smithi029.stdout:3559: finishing write tid 3 to smithi02979277-45 2024-04-12T11:29:28.925 INFO:tasks.rados.rados.0.smithi029.stdout:3559: finishing write tid 4 to smithi02979277-45 2024-04-12T11:29:28.925 INFO:tasks.rados.rados.0.smithi029.stdout:3559: oid 45 updating version 0 to 1786 2024-04-12T11:29:28.925 INFO:tasks.rados.rados.0.smithi029.stdout:3559: oid 45 version 1786 is already newer than 1784 2024-04-12T11:29:28.925 INFO:tasks.rados.rados.0.smithi029.stdout:3559: oid 45 version 1786 is already newer than 1785 2024-04-12T11:29:28.925 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1786 (ObjNum 1329 snap 451 seq_num 1329) dirty exists 2024-04-12T11:29:28.925 INFO:tasks.rados.rados.0.smithi029.stdout:3559: left oid 45 (ObjNum 1329 snap 451 seq_num 1329) 2024-04-12T11:29:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:28 smithi029 ceph-mon[93650]: pgmap v1080: 89 pgs: 89 active+clean; 223 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-04-12T11:29:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:28 smithi029 ceph-mon[93650]: osdmap e964: 8 total, 8 up, 8 in 2024-04-12T11:29:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:28 smithi029 ceph-mon[97787]: pgmap v1080: 89 pgs: 89 active+clean; 223 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-04-12T11:29:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:28 smithi029 ceph-mon[97787]: osdmap e964: 8 total, 8 up, 8 in 2024-04-12T11:29:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:28 smithi087 ceph-mon[89014]: pgmap v1080: 89 pgs: 89 active+clean; 223 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-04-12T11:29:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:28 smithi087 ceph-mon[89014]: osdmap e964: 8 total, 8 up, 8 in 2024-04-12T11:29:29.770 INFO:tasks.rados.rados.0.smithi029.stdout:3559: done (1 left) 2024-04-12T11:29:29.770 INFO:tasks.rados.rados.0.smithi029.stdout:3561: done (0 left) 2024-04-12T11:29:29.770 INFO:tasks.rados.rados.0.smithi029.stdout:3562: snap_create 2024-04-12T11:29:30.773 INFO:tasks.rados.rados.0.smithi029.stdout:3562: done (0 left) 2024-04-12T11:29:30.773 INFO:tasks.rados.rados.0.smithi029.stdout:3563: write oid 17 current snap is 453 2024-04-12T11:29:30.773 INFO:tasks.rados.rados.0.smithi029.stdout:3563: seq_num 1331 ranges {770117=518194,1870239=499878} 2024-04-12T11:29:30.779 INFO:tasks.rados.rados.0.smithi029.stdout:3563: writing smithi02979277-17 from 770117 to 1288311 tid 1 2024-04-12T11:29:30.786 INFO:tasks.rados.rados.0.smithi029.stdout:3563: writing smithi02979277-17 from 1870239 to 2370117 tid 2 2024-04-12T11:29:30.786 INFO:tasks.rados.rados.0.smithi029.stdout:3564: rollback oid 8 current snap is 453 2024-04-12T11:29:30.786 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 438 2024-04-12T11:29:30.786 INFO:tasks.rados.rados.0.smithi029.stdout:3565: write oid 34 current snap is 453 2024-04-12T11:29:30.786 INFO:tasks.rados.rados.0.smithi029.stdout:3565: seq_num 1332 ranges {786924=593443,1928721=458203} 2024-04-12T11:29:30.793 INFO:tasks.rados.rados.0.smithi029.stdout:3565: writing smithi02979277-34 from 786924 to 1380367 tid 1 2024-04-12T11:29:30.800 INFO:tasks.rados.rados.0.smithi029.stdout:3565: writing smithi02979277-34 from 1928721 to 2386924 tid 2 2024-04-12T11:29:30.800 INFO:tasks.rados.rados.0.smithi029.stdout:3566: delete oid 11 current snap is 453 2024-04-12T11:29:30.800 INFO:tasks.rados.rados.0.smithi029.stdout:3563: finishing write tid 1 to smithi02979277-17 2024-04-12T11:29:30.800 INFO:tasks.rados.rados.0.smithi029.stdout:3564: finishing rollback tid 0 to smithi02979277-8 2024-04-12T11:29:30.800 INFO:tasks.rados.rados.0.smithi029.stdout:3564: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:29:30.800 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 364 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:29:30.800 INFO:tasks.rados.rados.0.smithi029.stdout:3563: finishing write tid 2 to smithi02979277-17 2024-04-12T11:29:30.800 INFO:tasks.rados.rados.0.smithi029.stdout:3563: finishing write tid 3 to smithi02979277-17 2024-04-12T11:29:30.801 INFO:tasks.rados.rados.0.smithi029.stdout:3563: finishing write tid 4 to smithi02979277-17 2024-04-12T11:29:30.801 INFO:tasks.rados.rados.0.smithi029.stdout:3563: oid 17 updating version 0 to 1863 2024-04-12T11:29:30.801 INFO:tasks.rados.rados.0.smithi029.stdout:3563: oid 17 updating version 1863 to 1864 2024-04-12T11:29:30.801 INFO:tasks.rados.rados.0.smithi029.stdout:3563: oid 17 version 1864 is already newer than 1862 2024-04-12T11:29:30.801 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 1864 (ObjNum 1331 snap 453 seq_num 1331) dirty exists 2024-04-12T11:29:30.801 INFO:tasks.rados.rados.0.smithi029.stdout:3563: left oid 17 (ObjNum 1331 snap 453 seq_num 1331) 2024-04-12T11:29:30.801 INFO:tasks.rados.rados.0.smithi029.stdout:3565: finishing write tid 1 to smithi02979277-34 2024-04-12T11:29:30.803 INFO:tasks.rados.rados.0.smithi029.stdout:3563: done (3 left) 2024-04-12T11:29:30.803 INFO:tasks.rados.rados.0.smithi029.stdout:3564: done (2 left) 2024-04-12T11:29:30.803 INFO:tasks.rados.rados.0.smithi029.stdout:3566: done (1 left) 2024-04-12T11:29:30.803 INFO:tasks.rados.rados.0.smithi029.stdout:3567: read oid 27 snap -1 2024-04-12T11:29:30.803 INFO:tasks.rados.rados.0.smithi029.stdout:3567: expect (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:29:30.803 INFO:tasks.rados.rados.0.smithi029.stdout:3568: snap_remove snap 436 2024-04-12T11:29:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:30 smithi087 ceph-mon[89014]: pgmap v1083: 89 pgs: 89 active+clean; 226 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2024-04-12T11:29:31.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:30 smithi087 ceph-mon[89014]: osdmap e965: 8 total, 8 up, 8 in 2024-04-12T11:29:31.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:30 smithi029 ceph-mon[93650]: pgmap v1083: 89 pgs: 89 active+clean; 226 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2024-04-12T11:29:31.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:30 smithi029 ceph-mon[93650]: osdmap e965: 8 total, 8 up, 8 in 2024-04-12T11:29:31.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:30 smithi029 ceph-mon[97787]: pgmap v1083: 89 pgs: 89 active+clean; 226 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2024-04-12T11:29:31.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:30 smithi029 ceph-mon[97787]: osdmap e965: 8 total, 8 up, 8 in 2024-04-12T11:29:31.783 INFO:tasks.rados.rados.0.smithi029.stdout:3568: done (2 left) 2024-04-12T11:29:31.783 INFO:tasks.rados.rados.0.smithi029.stdout:3569: delete oid 30 current snap is 453 2024-04-12T11:29:31.784 INFO:tasks.rados.rados.0.smithi029.stdout:3565: finishing write tid 2 to smithi02979277-34 2024-04-12T11:29:31.784 INFO:tasks.rados.rados.0.smithi029.stdout:3565: finishing write tid 3 to smithi02979277-34 2024-04-12T11:29:31.784 INFO:tasks.rados.rados.0.smithi029.stdout:3565: finishing write tid 4 to smithi02979277-34 2024-04-12T11:29:31.784 INFO:tasks.rados.rados.0.smithi029.stdout:3565: oid 34 updating version 0 to 1599 2024-04-12T11:29:31.784 INFO:tasks.rados.rados.0.smithi029.stdout:3565: oid 34 version 1599 is already newer than 1598 2024-04-12T11:29:31.784 INFO:tasks.rados.rados.0.smithi029.stdout:3565: oid 34 updating version 1599 to 1600 2024-04-12T11:29:31.784 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1600 (ObjNum 1332 snap 453 seq_num 1332) dirty exists 2024-04-12T11:29:31.784 INFO:tasks.rados.rados.0.smithi029.stdout:3565: left oid 34 (ObjNum 1332 snap 453 seq_num 1332) 2024-04-12T11:29:31.784 INFO:tasks.rados.rados.0.smithi029.stdout:3567: expect (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:29:31.958 INFO:tasks.rados.rados.0.smithi029.stdout:3565: done (2 left) 2024-04-12T11:29:31.958 INFO:tasks.rados.rados.0.smithi029.stdout:3567: done (1 left) 2024-04-12T11:29:31.958 INFO:tasks.rados.rados.0.smithi029.stdout:3569: done (0 left) 2024-04-12T11:29:31.958 INFO:tasks.rados.rados.0.smithi029.stdout:3570: read oid 21 snap -1 2024-04-12T11:29:31.958 INFO:tasks.rados.rados.0.smithi029.stdout:3570: expect (ObjNum 1252 snap 425 seq_num 1252) 2024-04-12T11:29:31.958 INFO:tasks.rados.rados.0.smithi029.stdout:3571: delete oid 36 current snap is 453 2024-04-12T11:29:31.960 INFO:tasks.rados.rados.0.smithi029.stdout:3571: done (1 left) 2024-04-12T11:29:31.960 INFO:tasks.rados.rados.0.smithi029.stdout:3572: read oid 31 snap -1 2024-04-12T11:29:31.960 INFO:tasks.rados.rados.0.smithi029.stdout:3572: expect (ObjNum 1316 snap 449 seq_num 1316) 2024-04-12T11:29:31.960 INFO:tasks.rados.rados.0.smithi029.stdout:3573: write oid 44 current snap is 453 2024-04-12T11:29:31.960 INFO:tasks.rados.rados.0.smithi029.stdout:3573: seq_num 1336 ranges {454152=494439,1646296=735124} 2024-04-12T11:29:31.966 INFO:tasks.rados.rados.0.smithi029.stdout:3573: writing smithi02979277-44 from 454152 to 948591 tid 1 2024-04-12T11:29:31.974 INFO:tasks.rados.rados.0.smithi029.stdout:3573: writing smithi02979277-44 from 1646296 to 2381420 tid 2 2024-04-12T11:29:31.974 INFO:tasks.rados.rados.0.smithi029.stdout:3574: read oid 24 snap -1 2024-04-12T11:29:31.975 INFO:tasks.rados.rados.0.smithi029.stdout:3574: expect (ObjNum 1278 snap 437 seq_num 1278) 2024-04-12T11:29:31.975 INFO:tasks.rados.rados.0.smithi029.stdout:3575: read oid 17 snap -1 2024-04-12T11:29:31.975 INFO:tasks.rados.rados.0.smithi029.stdout:3575: expect (ObjNum 1331 snap 453 seq_num 1331) 2024-04-12T11:29:31.975 INFO:tasks.rados.rados.0.smithi029.stdout:3576: read oid 32 snap -1 2024-04-12T11:29:31.975 INFO:tasks.rados.rados.0.smithi029.stdout:3576: expect deleted 2024-04-12T11:29:31.975 INFO:tasks.rados.rados.0.smithi029.stdout:3577: read oid 9 snap -1 2024-04-12T11:29:31.975 INFO:tasks.rados.rados.0.smithi029.stdout:3577: expect (ObjNum 1204 snap 406 seq_num 1204) 2024-04-12T11:29:31.975 INFO:tasks.rados.rados.0.smithi029.stdout:3578: rollback oid 47 current snap is 453 2024-04-12T11:29:31.975 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 452 2024-04-12T11:29:31.975 INFO:tasks.rados.rados.0.smithi029.stdout:3572: expect (ObjNum 1316 snap 449 seq_num 1316) 2024-04-12T11:29:32.048 INFO:tasks.rados.rados.0.smithi029.stdout:3570: expect (ObjNum 1252 snap 425 seq_num 1252) 2024-04-12T11:29:32.152 INFO:tasks.rados.rados.0.smithi029.stdout:3573: finishing write tid 1 to smithi02979277-44 2024-04-12T11:29:32.152 INFO:tasks.rados.rados.0.smithi029.stdout:3578: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:29:32.152 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1708 (ObjNum 129 snap 0 seq_num 1801014544) dirty dne 2024-04-12T11:29:32.152 INFO:tasks.rados.rados.0.smithi029.stdout:3570: done (7 left) 2024-04-12T11:29:32.153 INFO:tasks.rados.rados.0.smithi029.stdout:3572: done (6 left) 2024-04-12T11:29:32.153 INFO:tasks.rados.rados.0.smithi029.stdout:3576: done (5 left) 2024-04-12T11:29:32.153 INFO:tasks.rados.rados.0.smithi029.stdout:3578: done (4 left) 2024-04-12T11:29:32.153 INFO:tasks.rados.rados.0.smithi029.stdout:3579: read oid 13 snap -1 2024-04-12T11:29:32.153 INFO:tasks.rados.rados.0.smithi029.stdout:3579: expect (ObjNum 1322 snap 451 seq_num 1322) 2024-04-12T11:29:32.153 INFO:tasks.rados.rados.0.smithi029.stdout:3580: snap_remove snap 429 2024-04-12T11:29:32.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:31 smithi087 ceph-mon[89014]: osdmap e966: 8 total, 8 up, 8 in 2024-04-12T11:29:32.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:31 smithi029 ceph-mon[97787]: osdmap e966: 8 total, 8 up, 8 in 2024-04-12T11:29:32.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:31 smithi029 ceph-mon[93650]: osdmap e966: 8 total, 8 up, 8 in 2024-04-12T11:29:32.709 INFO:tasks.rados.rados.0.smithi029.stdout:3580: done (5 left) 2024-04-12T11:29:32.709 INFO:tasks.rados.rados.0.smithi029.stdout:3581: write oid 48 current snap is 453 2024-04-12T11:29:32.709 INFO:tasks.rados.rados.0.smithi029.stdout:3581: seq_num 1337 ranges {470959=569688,1704778=595135} 2024-04-12T11:29:32.715 INFO:tasks.rados.rados.0.smithi029.stdout:3581: writing smithi02979277-48 from 470959 to 1040647 tid 1 2024-04-12T11:29:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:29:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:29:32.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:29:32.723 INFO:tasks.rados.rados.0.smithi029.stdout:3581: writing smithi02979277-48 from 1704778 to 2299913 tid 2 2024-04-12T11:29:32.723 INFO:tasks.rados.rados.0.smithi029.stdout:3582: rollback oid 47 current snap is 453 2024-04-12T11:29:32.723 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 47 to 445 2024-04-12T11:29:32.723 INFO:tasks.rados.rados.0.smithi029.stdout:3583: snap_remove snap 435 2024-04-12T11:29:33.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:32 smithi087 ceph-mon[89014]: pgmap v1086: 89 pgs: 89 active+clean; 227 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 725 KiB/s wr, 13 op/s 2024-04-12T11:29:33.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:32 smithi087 ceph-mon[89014]: osdmap e967: 8 total, 8 up, 8 in 2024-04-12T11:29:33.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:32 smithi087 ceph-mon[89014]: osdmap e968: 8 total, 8 up, 8 in 2024-04-12T11:29:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:32 smithi029 ceph-mon[97787]: pgmap v1086: 89 pgs: 89 active+clean; 227 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 725 KiB/s wr, 13 op/s 2024-04-12T11:29:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:32 smithi029 ceph-mon[97787]: osdmap e967: 8 total, 8 up, 8 in 2024-04-12T11:29:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:32 smithi029 ceph-mon[97787]: osdmap e968: 8 total, 8 up, 8 in 2024-04-12T11:29:33.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:32 smithi029 ceph-mon[93650]: pgmap v1086: 89 pgs: 89 active+clean; 227 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 725 KiB/s wr, 13 op/s 2024-04-12T11:29:33.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:32 smithi029 ceph-mon[93650]: osdmap e967: 8 total, 8 up, 8 in 2024-04-12T11:29:33.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:32 smithi029 ceph-mon[93650]: osdmap e968: 8 total, 8 up, 8 in 2024-04-12T11:29:33.712 INFO:tasks.rados.rados.0.smithi029.stdout:3583: done (7 left) 2024-04-12T11:29:33.712 INFO:tasks.rados.rados.0.smithi029.stdout:3584: write oid 1 current snap is 453 2024-04-12T11:29:33.712 INFO:tasks.rados.rados.0.smithi029.stdout:3584: seq_num 1338 ranges {487766=644937,1646907=738793} 2024-04-12T11:29:33.718 INFO:tasks.rados.rados.0.smithi029.stdout:3584: writing smithi02979277-1 from 487766 to 1132703 tid 1 2024-04-12T11:29:33.726 INFO:tasks.rados.rados.0.smithi029.stdout:3584: writing smithi02979277-1 from 1646907 to 2385700 tid 2 2024-04-12T11:29:33.726 INFO:tasks.rados.rados.0.smithi029.stdout:3585: snap_remove snap 452 2024-04-12T11:29:34.715 INFO:tasks.rados.rados.0.smithi029.stdout:3585: done (8 left) 2024-04-12T11:29:34.716 INFO:tasks.rados.rados.0.smithi029.stdout:3586: write oid 43 current snap is 453 2024-04-12T11:29:34.716 INFO:tasks.rados.rados.0.smithi029.stdout:3573: finishing write tid 2 to smithi02979277-44 2024-04-12T11:29:34.716 INFO:tasks.rados.rados.0.smithi029.stdout:3586: seq_num 1339 ranges {504573=436539,1421742=598804,2437505=67068} 2024-04-12T11:29:34.720 INFO:tasks.rados.rados.0.smithi029.stdout:3586: writing smithi02979277-43 from 504573 to 941112 tid 1 2024-04-12T11:29:34.728 INFO:tasks.rados.rados.0.smithi029.stdout:3586: writing smithi02979277-43 from 1421742 to 2020546 tid 2 2024-04-12T11:29:34.731 INFO:tasks.rados.rados.0.smithi029.stdout:3586: writing smithi02979277-43 from 2437505 to 2504573 tid 3 2024-04-12T11:29:34.731 INFO:tasks.rados.rados.0.smithi029.stdout:3587: read oid 27 snap -1 2024-04-12T11:29:34.731 INFO:tasks.rados.rados.0.smithi029.stdout:3587: expect (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:29:34.731 INFO:tasks.rados.rados.0.smithi029.stdout:3588: write oid 7 current snap is 453 2024-04-12T11:29:34.731 INFO:tasks.rados.rados.0.smithi029.stdout:3588: seq_num 1340 ranges {521380=511788,1480224=742462} 2024-04-12T11:29:34.736 INFO:tasks.rados.rados.0.smithi029.stdout:3588: writing smithi02979277-7 from 521380 to 1033168 tid 1 2024-04-12T11:29:34.743 INFO:tasks.rados.rados.0.smithi029.stdout:3588: writing smithi02979277-7 from 1480224 to 2222686 tid 2 2024-04-12T11:29:34.743 INFO:tasks.rados.rados.0.smithi029.stdout:3589: write oid 31 current snap is 453 2024-04-12T11:29:34.743 INFO:tasks.rados.rados.0.smithi029.stdout:3573: finishing write tid 3 to smithi02979277-44 2024-04-12T11:29:34.744 INFO:tasks.rados.rados.0.smithi029.stdout:3573: finishing write tid 4 to smithi02979277-44 2024-04-12T11:29:34.744 INFO:tasks.rados.rados.0.smithi029.stdout:3573: oid 44 updating version 0 to 1701 2024-04-12T11:29:34.744 INFO:tasks.rados.rados.0.smithi029.stdout:3573: oid 44 updating version 1701 to 1702 2024-04-12T11:29:34.744 INFO:tasks.rados.rados.0.smithi029.stdout:3573: oid 44 updating version 1702 to 1703 2024-04-12T11:29:34.744 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1703 (ObjNum 1336 snap 453 seq_num 1336) dirty exists 2024-04-12T11:29:34.744 INFO:tasks.rados.rados.0.smithi029.stdout:3573: left oid 44 (ObjNum 1336 snap 453 seq_num 1336) 2024-04-12T11:29:34.744 INFO:tasks.rados.rados.0.smithi029.stdout:3577: expect (ObjNum 1204 snap 406 seq_num 1204) 2024-04-12T11:29:34.816 INFO:tasks.rados.rados.0.smithi029.stdout:3574: expect (ObjNum 1278 snap 437 seq_num 1278) 2024-04-12T11:29:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:34 smithi087 ceph-mon[89014]: pgmap v1089: 89 pgs: 89 active+clean; 228 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 994 KiB/s wr, 16 op/s 2024-04-12T11:29:34.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:34 smithi087 ceph-mon[89014]: osdmap e969: 8 total, 8 up, 8 in 2024-04-12T11:29:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:34 smithi029 ceph-mon[93650]: pgmap v1089: 89 pgs: 89 active+clean; 228 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 994 KiB/s wr, 16 op/s 2024-04-12T11:29:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:34 smithi029 ceph-mon[93650]: osdmap e969: 8 total, 8 up, 8 in 2024-04-12T11:29:35.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:29:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:29:34] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:29:35.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:34 smithi029 ceph-mon[97787]: pgmap v1089: 89 pgs: 89 active+clean; 228 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 994 KiB/s wr, 16 op/s 2024-04-12T11:29:35.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:34 smithi029 ceph-mon[97787]: osdmap e969: 8 total, 8 up, 8 in 2024-04-12T11:29:35.067 INFO:tasks.rados.rados.0.smithi029.stdout:3575: expect (ObjNum 1331 snap 453 seq_num 1331) 2024-04-12T11:29:35.218 INFO:tasks.rados.rados.0.smithi029.stdout:3579: expect (ObjNum 1322 snap 451 seq_num 1322) 2024-04-12T11:29:35.417 INFO:tasks.rados.rados.0.smithi029.stdout:3581: finishing write tid 1 to smithi02979277-48 2024-04-12T11:29:35.417 INFO:tasks.rados.rados.0.smithi029.stdout:3582: finishing rollback tid 1 to smithi02979277-47 2024-04-12T11:29:35.417 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1708 (ObjNum 129 snap 0 seq_num 1801014544) dirty dne 2024-04-12T11:29:35.417 INFO:tasks.rados.rados.0.smithi029.stdout:3581: finishing write tid 2 to smithi02979277-48 2024-04-12T11:29:35.417 INFO:tasks.rados.rados.0.smithi029.stdout:3581: finishing write tid 3 to smithi02979277-48 2024-04-12T11:29:35.417 INFO:tasks.rados.rados.0.smithi029.stdout:3581: finishing write tid 4 to smithi02979277-48 2024-04-12T11:29:35.417 INFO:tasks.rados.rados.0.smithi029.stdout:3581: oid 48 updating version 0 to 1428 2024-04-12T11:29:35.417 INFO:tasks.rados.rados.0.smithi029.stdout:3581: oid 48 version 1428 is already newer than 1427 2024-04-12T11:29:35.418 INFO:tasks.rados.rados.0.smithi029.stdout:3581: oid 48 updating version 1428 to 1429 2024-04-12T11:29:35.418 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 1429 (ObjNum 1337 snap 453 seq_num 1337) dirty exists 2024-04-12T11:29:35.418 INFO:tasks.rados.rados.0.smithi029.stdout:3581: left oid 48 (ObjNum 1337 snap 453 seq_num 1337) 2024-04-12T11:29:35.418 INFO:tasks.rados.rados.0.smithi029.stdout:3584: finishing write tid 1 to smithi02979277-1 2024-04-12T11:29:35.418 INFO:tasks.rados.rados.0.smithi029.stdout:3584: finishing write tid 2 to smithi02979277-1 2024-04-12T11:29:35.418 INFO:tasks.rados.rados.0.smithi029.stdout:3584: finishing write tid 3 to smithi02979277-1 2024-04-12T11:29:35.418 INFO:tasks.rados.rados.0.smithi029.stdout:3589: seq_num 1341 ranges {538187=587037,1538706=602473} 2024-04-12T11:29:35.423 INFO:tasks.rados.rados.0.smithi029.stdout:3589: writing smithi02979277-31 from 538187 to 1125224 tid 1 2024-04-12T11:29:35.429 INFO:tasks.rados.rados.0.smithi029.stdout:3589: writing smithi02979277-31 from 1538706 to 2141179 tid 2 2024-04-12T11:29:35.429 INFO:tasks.rados.rados.0.smithi029.stdout:3573: done (11 left) 2024-04-12T11:29:35.429 INFO:tasks.rados.rados.0.smithi029.stdout:3574: done (10 left) 2024-04-12T11:29:35.429 INFO:tasks.rados.rados.0.smithi029.stdout:3575: done (9 left) 2024-04-12T11:29:35.429 INFO:tasks.rados.rados.0.smithi029.stdout:3577: done (8 left) 2024-04-12T11:29:35.429 INFO:tasks.rados.rados.0.smithi029.stdout:3579: done (7 left) 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:3581: done (6 left) 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:3582: done (5 left) 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:3590: delete oid 35 current snap is 453 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:3584: finishing write tid 4 to smithi02979277-1 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:3584: oid 1 updating version 0 to 1715 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:3584: oid 1 updating version 1715 to 1719 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:3584: oid 1 version 1719 is already newer than 1718 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1719 (ObjNum 1338 snap 453 seq_num 1338) dirty exists 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:3584: left oid 1 (ObjNum 1338 snap 453 seq_num 1338) 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:3586: finishing write tid 1 to smithi02979277-43 2024-04-12T11:29:35.430 INFO:tasks.rados.rados.0.smithi029.stdout:3587: expect (ObjNum 1235 snap 415 seq_num 1235) 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3586: finishing write tid 2 to smithi02979277-43 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3586: finishing write tid 3 to smithi02979277-43 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3586: finishing write tid 4 to smithi02979277-43 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3586: finishing write tid 5 to smithi02979277-43 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3586: oid 43 updating version 0 to 1614 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3586: oid 43 updating version 1614 to 1617 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3586: oid 43 version 1617 is already newer than 1616 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3586: oid 43 version 1617 is already newer than 1615 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1617 (ObjNum 1339 snap 453 seq_num 1339) dirty exists 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3586: left oid 43 (ObjNum 1339 snap 453 seq_num 1339) 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3588: finishing write tid 1 to smithi02979277-7 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3584: done (5 left) 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3586: done (4 left) 2024-04-12T11:29:35.600 INFO:tasks.rados.rados.0.smithi029.stdout:3587: done (3 left) 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3590: done (2 left) 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3591: snap_create 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3588: finishing write tid 2 to smithi02979277-7 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3588: finishing write tid 3 to smithi02979277-7 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3588: finishing write tid 4 to smithi02979277-7 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3588: oid 7 updating version 0 to 1803 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3588: oid 7 version 1803 is already newer than 1801 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3588: oid 7 version 1803 is already newer than 1802 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1803 (ObjNum 1340 snap 453 seq_num 1340) dirty exists 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3588: left oid 7 (ObjNum 1340 snap 453 seq_num 1340) 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3589: finishing write tid 1 to smithi02979277-31 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3589: finishing write tid 2 to smithi02979277-31 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3589: finishing write tid 3 to smithi02979277-31 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3589: finishing write tid 4 to smithi02979277-31 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3589: oid 31 updating version 0 to 1435 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3589: oid 31 version 1435 is already newer than 1433 2024-04-12T11:29:35.601 INFO:tasks.rados.rados.0.smithi029.stdout:3589: oid 31 version 1435 is already newer than 1434 2024-04-12T11:29:35.602 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1435 (ObjNum 1341 snap 453 seq_num 1341) dirty exists 2024-04-12T11:29:35.602 INFO:tasks.rados.rados.0.smithi029.stdout:3589: left oid 31 (ObjNum 1341 snap 453 seq_num 1341) 2024-04-12T11:29:35.719 INFO:tasks.rados.rados.0.smithi029.stdout:3588: done (2 left) 2024-04-12T11:29:35.720 INFO:tasks.rados.rados.0.smithi029.stdout:3589: done (1 left) 2024-04-12T11:29:35.720 INFO:tasks.rados.rados.0.smithi029.stdout:3591: done (0 left) 2024-04-12T11:29:35.720 INFO:tasks.rados.rados.0.smithi029.stdout:3592: read oid 9 snap -1 2024-04-12T11:29:35.720 INFO:tasks.rados.rados.0.smithi029.stdout:3592: expect (ObjNum 1204 snap 406 seq_num 1204) 2024-04-12T11:29:35.720 INFO:tasks.rados.rados.0.smithi029.stdout:3593: snap_create 2024-04-12T11:29:35.723 INFO:tasks.rados.rados.0.smithi029.stdout:3592: expect (ObjNum 1204 snap 406 seq_num 1204) 2024-04-12T11:29:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:35 smithi029 ceph-mon[93650]: osdmap e970: 8 total, 8 up, 8 in 2024-04-12T11:29:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:35 smithi029 ceph-mon[97787]: osdmap e970: 8 total, 8 up, 8 in 2024-04-12T11:29:36.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:35 smithi087 ceph-mon[89014]: osdmap e970: 8 total, 8 up, 8 in 2024-04-12T11:29:36.722 INFO:tasks.rados.rados.0.smithi029.stdout:3592: done (1 left) 2024-04-12T11:29:36.722 INFO:tasks.rados.rados.0.smithi029.stdout:3593: done (0 left) 2024-04-12T11:29:36.722 INFO:tasks.rados.rados.0.smithi029.stdout:3594: snap_create 2024-04-12T11:29:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:36 smithi029 ceph-mon[93650]: pgmap v1092: 89 pgs: 89 active+clean; 236 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.7 MiB/s wr, 25 op/s 2024-04-12T11:29:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:36 smithi029 ceph-mon[93650]: osdmap e971: 8 total, 8 up, 8 in 2024-04-12T11:29:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:36 smithi029 ceph-mon[97787]: pgmap v1092: 89 pgs: 89 active+clean; 236 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.7 MiB/s wr, 25 op/s 2024-04-12T11:29:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:36 smithi029 ceph-mon[97787]: osdmap e971: 8 total, 8 up, 8 in 2024-04-12T11:29:37.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:36 smithi087 ceph-mon[89014]: pgmap v1092: 89 pgs: 89 active+clean; 236 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.7 MiB/s wr, 25 op/s 2024-04-12T11:29:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:36 smithi087 ceph-mon[89014]: osdmap e971: 8 total, 8 up, 8 in 2024-04-12T11:29:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:37.710 INFO:tasks.rados.rados.0.smithi029.stdout:3594: done (0 left) 2024-04-12T11:29:37.711 INFO:tasks.rados.rados.0.smithi029.stdout:3595: snap_remove snap 451 2024-04-12T11:29:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:37 smithi029 ceph-mon[97787]: osdmap e972: 8 total, 8 up, 8 in 2024-04-12T11:29:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:37 smithi029 ceph-mon[97787]: osdmap e973: 8 total, 8 up, 8 in 2024-04-12T11:29:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:37 smithi029 ceph-mon[93650]: osdmap e972: 8 total, 8 up, 8 in 2024-04-12T11:29:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:37 smithi029 ceph-mon[93650]: osdmap e973: 8 total, 8 up, 8 in 2024-04-12T11:29:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:37 smithi087 ceph-mon[89014]: osdmap e972: 8 total, 8 up, 8 in 2024-04-12T11:29:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:37 smithi087 ceph-mon[89014]: osdmap e973: 8 total, 8 up, 8 in 2024-04-12T11:29:38.713 INFO:tasks.rados.rados.0.smithi029.stdout:3595: done (0 left) 2024-04-12T11:29:38.713 INFO:tasks.rados.rados.0.smithi029.stdout:3596: write oid 1 current snap is 456 2024-04-12T11:29:38.713 INFO:tasks.rados.rados.0.smithi029.stdout:3596: seq_num 1343 ranges {571801=737535,1939317=489789} 2024-04-12T11:29:38.721 INFO:tasks.rados.rados.0.smithi029.stdout:3596: writing smithi02979277-1 from 571801 to 1309336 tid 1 2024-04-12T11:29:38.727 INFO:tasks.rados.rados.0.smithi029.stdout:3596: writing smithi02979277-1 from 1939317 to 2429106 tid 2 2024-04-12T11:29:38.727 INFO:tasks.rados.rados.0.smithi029.stdout:3597: read oid 2 snap -1 2024-04-12T11:29:38.727 INFO:tasks.rados.rados.0.smithi029.stdout:3597: expect deleted 2024-04-12T11:29:38.728 INFO:tasks.rados.rados.0.smithi029.stdout:3598: write oid 13 current snap is 456 2024-04-12T11:29:38.728 INFO:tasks.rados.rados.0.smithi029.stdout:3596: finishing write tid 1 to smithi02979277-1 2024-04-12T11:29:38.728 INFO:tasks.rados.rados.0.smithi029.stdout:3598: seq_num 1344 ranges {588608=412784,1597799=749800} 2024-04-12T11:29:38.733 INFO:tasks.rados.rados.0.smithi029.stdout:3598: writing smithi02979277-13 from 588608 to 1001392 tid 1 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:3598: writing smithi02979277-13 from 1597799 to 2347599 tid 2 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:3599: delete oid 34 current snap is 456 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:3596: finishing write tid 2 to smithi02979277-1 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:3596: finishing write tid 3 to smithi02979277-1 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:3596: finishing write tid 4 to smithi02979277-1 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:3596: oid 1 updating version 0 to 1729 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:3596: oid 1 updating version 1729 to 1731 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:3596: oid 1 version 1731 is already newer than 1730 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1731 (ObjNum 1343 snap 456 seq_num 1343) dirty exists 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:3596: left oid 1 (ObjNum 1343 snap 456 seq_num 1343) 2024-04-12T11:29:38.741 INFO:tasks.rados.rados.0.smithi029.stdout:3598: finishing write tid 1 to smithi02979277-13 2024-04-12T11:29:38.743 INFO:tasks.rados.rados.0.smithi029.stdout:3596: done (3 left) 2024-04-12T11:29:38.743 INFO:tasks.rados.rados.0.smithi029.stdout:3597: done (2 left) 2024-04-12T11:29:38.743 INFO:tasks.rados.rados.0.smithi029.stdout:3599: done (1 left) 2024-04-12T11:29:38.743 INFO:tasks.rados.rados.0.smithi029.stdout:3600: snap_remove snap 449 2024-04-12T11:29:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:38 smithi029 ceph-mon[93650]: pgmap v1095: 89 pgs: 89 active+clean; 237 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:29:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:38 smithi029 ceph-mon[93650]: osdmap e974: 8 total, 8 up, 8 in 2024-04-12T11:29:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:38 smithi029 ceph-mon[97787]: pgmap v1095: 89 pgs: 89 active+clean; 237 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:29:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:38 smithi029 ceph-mon[97787]: osdmap e974: 8 total, 8 up, 8 in 2024-04-12T11:29:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:38 smithi087 ceph-mon[89014]: pgmap v1095: 89 pgs: 89 active+clean; 237 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:29:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:38 smithi087 ceph-mon[89014]: osdmap e974: 8 total, 8 up, 8 in 2024-04-12T11:29:39.737 INFO:tasks.rados.rados.0.smithi029.stdout:3600: done (1 left) 2024-04-12T11:29:39.737 INFO:tasks.rados.rados.0.smithi029.stdout:3601: rollback oid 41 current snap is 456 2024-04-12T11:29:39.737 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 445 2024-04-12T11:29:39.737 INFO:tasks.rados.rados.0.smithi029.stdout:3598: finishing write tid 2 to smithi02979277-13 2024-04-12T11:29:39.738 INFO:tasks.rados.rados.0.smithi029.stdout:3598: finishing write tid 3 to smithi02979277-13 2024-04-12T11:29:39.738 INFO:tasks.rados.rados.0.smithi029.stdout:3598: finishing write tid 4 to smithi02979277-13 2024-04-12T11:29:39.738 INFO:tasks.rados.rados.0.smithi029.stdout:3598: oid 13 updating version 0 to 1894 2024-04-12T11:29:39.738 INFO:tasks.rados.rados.0.smithi029.stdout:3598: oid 13 updating version 1894 to 1896 2024-04-12T11:29:39.738 INFO:tasks.rados.rados.0.smithi029.stdout:3598: oid 13 version 1896 is already newer than 1895 2024-04-12T11:29:39.738 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1896 (ObjNum 1344 snap 456 seq_num 1344) dirty exists 2024-04-12T11:29:39.738 INFO:tasks.rados.rados.0.smithi029.stdout:3598: left oid 13 (ObjNum 1344 snap 456 seq_num 1344) 2024-04-12T11:29:39.738 INFO:tasks.rados.rados.0.smithi029.stdout:3598: done (1 left) 2024-04-12T11:29:39.738 INFO:tasks.rados.rados.0.smithi029.stdout:3602: write oid 16 current snap is 456 2024-04-12T11:29:39.738 INFO:tasks.rados.rados.0.smithi029.stdout:3602: seq_num 1346 ranges {622222=679635,1831116=753469} 2024-04-12T11:29:39.745 INFO:tasks.rados.rados.0.smithi029.stdout:3602: writing smithi02979277-16 from 622222 to 1301857 tid 1 2024-04-12T11:29:39.753 INFO:tasks.rados.rados.0.smithi029.stdout:3602: writing smithi02979277-16 from 1831116 to 2584585 tid 2 2024-04-12T11:29:39.753 INFO:tasks.rados.rados.0.smithi029.stdout:3603: delete oid 22 current snap is 456 2024-04-12T11:29:39.753 INFO:tasks.rados.rados.0.smithi029.stdout:3601: finishing rollback tid 0 to smithi02979277-41 2024-04-12T11:29:39.753 INFO:tasks.rados.rados.0.smithi029.stdout:3601: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:29:39.753 INFO:tasks.rados.rados.0.smithi029.stdout:3601: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:29:39.753 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1735 (ObjNum 1304 snap 445 seq_num 1304) dirty exists 2024-04-12T11:29:39.753 INFO:tasks.rados.rados.0.smithi029.stdout:3602: finishing write tid 1 to smithi02979277-16 2024-04-12T11:29:39.758 INFO:tasks.rados.rados.0.smithi029.stdout:3602: finishing write tid 2 to smithi02979277-16 2024-04-12T11:29:39.759 INFO:tasks.rados.rados.0.smithi029.stdout:3601: done (2 left) 2024-04-12T11:29:39.759 INFO:tasks.rados.rados.0.smithi029.stdout:3603: done (1 left) 2024-04-12T11:29:39.759 INFO:tasks.rados.rados.0.smithi029.stdout:3604: read oid 19 snap 443 2024-04-12T11:29:39.759 INFO:tasks.rados.rados.0.smithi029.stdout:3604: expect (ObjNum 1259 snap 428 seq_num 1259) 2024-04-12T11:29:39.759 INFO:tasks.rados.rados.0.smithi029.stdout:3605: snap_remove snap 453 2024-04-12T11:29:40.743 INFO:tasks.rados.rados.0.smithi029.stdout:3602: finishing write tid 3 to smithi02979277-16 2024-04-12T11:29:40.743 INFO:tasks.rados.rados.0.smithi029.stdout:3605: done (2 left) 2024-04-12T11:29:40.743 INFO:tasks.rados.rados.0.smithi029.stdout:3606: read oid 31 snap -1 2024-04-12T11:29:40.743 INFO:tasks.rados.rados.0.smithi029.stdout:3606: expect (ObjNum 1341 snap 453 seq_num 1341) 2024-04-12T11:29:40.743 INFO:tasks.rados.rados.0.smithi029.stdout:3607: read oid 26 snap -1 2024-04-12T11:29:40.743 INFO:tasks.rados.rados.0.smithi029.stdout:3607: expect (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:29:40.744 INFO:tasks.rados.rados.0.smithi029.stdout:3602: finishing write tid 4 to smithi02979277-16 2024-04-12T11:29:40.744 INFO:tasks.rados.rados.0.smithi029.stdout:3602: oid 16 updating version 0 to 1710 2024-04-12T11:29:40.744 INFO:tasks.rados.rados.0.smithi029.stdout:3602: oid 16 version 1710 is already newer than 1708 2024-04-12T11:29:40.744 INFO:tasks.rados.rados.0.smithi029.stdout:3602: oid 16 version 1710 is already newer than 1709 2024-04-12T11:29:40.744 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1710 (ObjNum 1346 snap 456 seq_num 1346) dirty exists 2024-04-12T11:29:40.744 INFO:tasks.rados.rados.0.smithi029.stdout:3602: left oid 16 (ObjNum 1346 snap 456 seq_num 1346) 2024-04-12T11:29:40.744 INFO:tasks.rados.rados.0.smithi029.stdout:3604: expect (ObjNum 1259 snap 428 seq_num 1259) 2024-04-12T11:29:40.990 INFO:tasks.rados.rados.0.smithi029.stdout:3606: expect (ObjNum 1341 snap 453 seq_num 1341) 2024-04-12T11:29:41.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:40 smithi029 ceph-mon[97787]: pgmap v1098: 89 pgs: 89 active+clean; 241 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:29:41.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:40 smithi029 ceph-mon[97787]: osdmap e975: 8 total, 8 up, 8 in 2024-04-12T11:29:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:40 smithi029 ceph-mon[93650]: pgmap v1098: 89 pgs: 89 active+clean; 241 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:29:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:40 smithi029 ceph-mon[93650]: osdmap e975: 8 total, 8 up, 8 in 2024-04-12T11:29:41.107 INFO:tasks.rados.rados.0.smithi029.stdout:3607: expect (ObjNum 938 snap 325 seq_num 938) 2024-04-12T11:29:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:40 smithi087 ceph-mon[89014]: pgmap v1098: 89 pgs: 89 active+clean; 241 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-04-12T11:29:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:40 smithi087 ceph-mon[89014]: osdmap e975: 8 total, 8 up, 8 in 2024-04-12T11:29:41.323 INFO:tasks.rados.rados.0.smithi029.stdout:3602: done (3 left) 2024-04-12T11:29:41.324 INFO:tasks.rados.rados.0.smithi029.stdout:3604: done (2 left) 2024-04-12T11:29:41.324 INFO:tasks.rados.rados.0.smithi029.stdout:3606: done (1 left) 2024-04-12T11:29:41.324 INFO:tasks.rados.rados.0.smithi029.stdout:3607: done (0 left) 2024-04-12T11:29:41.324 INFO:tasks.rados.rados.0.smithi029.stdout:3608: write oid 40 current snap is 456 2024-04-12T11:29:41.324 INFO:tasks.rados.rados.0.smithi029.stdout:3608: seq_num 1348 ranges {655836=430133,1831727=757138} 2024-04-12T11:29:41.329 INFO:tasks.rados.rados.0.smithi029.stdout:3608: writing smithi02979277-40 from 655836 to 1085969 tid 1 2024-04-12T11:29:41.337 INFO:tasks.rados.rados.0.smithi029.stdout:3608: writing smithi02979277-40 from 1831727 to 2588865 tid 2 2024-04-12T11:29:41.338 INFO:tasks.rados.rados.0.smithi029.stdout:3609: write oid 4 current snap is 456 2024-04-12T11:29:41.338 INFO:tasks.rados.rados.0.smithi029.stdout:3608: finishing write tid 1 to smithi02979277-40 2024-04-12T11:29:41.338 INFO:tasks.rados.rados.0.smithi029.stdout:3609: seq_num 1349 ranges {672643=505382,1890209=500796} 2024-04-12T11:29:41.344 INFO:tasks.rados.rados.0.smithi029.stdout:3609: writing smithi02979277-4 from 672643 to 1178025 tid 1 2024-04-12T11:29:41.351 INFO:tasks.rados.rados.0.smithi029.stdout:3609: writing smithi02979277-4 from 1890209 to 2391005 tid 2 2024-04-12T11:29:41.351 INFO:tasks.rados.rados.0.smithi029.stdout:3610: delete oid 43 current snap is 456 2024-04-12T11:29:41.351 INFO:tasks.rados.rados.0.smithi029.stdout:3608: finishing write tid 2 to smithi02979277-40 2024-04-12T11:29:41.351 INFO:tasks.rados.rados.0.smithi029.stdout:3608: finishing write tid 3 to smithi02979277-40 2024-04-12T11:29:41.351 INFO:tasks.rados.rados.0.smithi029.stdout:3608: finishing write tid 4 to smithi02979277-40 2024-04-12T11:29:41.351 INFO:tasks.rados.rados.0.smithi029.stdout:3608: oid 40 updating version 0 to 2292 2024-04-12T11:29:41.351 INFO:tasks.rados.rados.0.smithi029.stdout:3608: oid 40 version 2292 is already newer than 2291 2024-04-12T11:29:41.352 INFO:tasks.rados.rados.0.smithi029.stdout:3608: oid 40 updating version 2292 to 2293 2024-04-12T11:29:41.352 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 2293 (ObjNum 1348 snap 456 seq_num 1348) dirty exists 2024-04-12T11:29:41.352 INFO:tasks.rados.rados.0.smithi029.stdout:3608: left oid 40 (ObjNum 1348 snap 456 seq_num 1348) 2024-04-12T11:29:41.352 INFO:tasks.rados.rados.0.smithi029.stdout:3609: finishing write tid 1 to smithi02979277-4 2024-04-12T11:29:41.353 INFO:tasks.rados.rados.0.smithi029.stdout:3608: done (2 left) 2024-04-12T11:29:41.353 INFO:tasks.rados.rados.0.smithi029.stdout:3610: done (1 left) 2024-04-12T11:29:41.353 INFO:tasks.rados.rados.0.smithi029.stdout:3611: read oid 10 snap -1 2024-04-12T11:29:41.354 INFO:tasks.rados.rados.0.smithi029.stdout:3611: expect deleted 2024-04-12T11:29:41.354 INFO:tasks.rados.rados.0.smithi029.stdout:3612: rollback oid 43 current snap is 456 2024-04-12T11:29:41.354 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 454 2024-04-12T11:29:41.354 INFO:tasks.rados.rados.0.smithi029.stdout:3613: snap_create 2024-04-12T11:29:41.355 INFO:tasks.rados.rados.0.smithi029.stdout:3609: finishing write tid 2 to smithi02979277-4 2024-04-12T11:29:41.355 INFO:tasks.rados.rados.0.smithi029.stdout:3609: finishing write tid 3 to smithi02979277-4 2024-04-12T11:29:41.355 INFO:tasks.rados.rados.0.smithi029.stdout:3609: finishing write tid 4 to smithi02979277-4 2024-04-12T11:29:41.355 INFO:tasks.rados.rados.0.smithi029.stdout:3609: oid 4 updating version 0 to 1714 2024-04-12T11:29:41.355 INFO:tasks.rados.rados.0.smithi029.stdout:3609: oid 4 updating version 1714 to 1716 2024-04-12T11:29:41.355 INFO:tasks.rados.rados.0.smithi029.stdout:3609: oid 4 version 1716 is already newer than 1715 2024-04-12T11:29:41.355 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1716 (ObjNum 1349 snap 456 seq_num 1349) dirty exists 2024-04-12T11:29:41.355 INFO:tasks.rados.rados.0.smithi029.stdout:3609: left oid 4 (ObjNum 1349 snap 456 seq_num 1349) 2024-04-12T11:29:41.356 INFO:tasks.rados.rados.0.smithi029.stdout:3612: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:29:41.356 INFO:tasks.rados.rados.0.smithi029.stdout:3612: finishing rollback tid 2 to smithi02979277-43 2024-04-12T11:29:41.356 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1631 (ObjNum 1339 snap 453 seq_num 1339) dirty exists 2024-04-12T11:29:41.747 INFO:tasks.rados.rados.0.smithi029.stdout:3609: done (3 left) 2024-04-12T11:29:41.747 INFO:tasks.rados.rados.0.smithi029.stdout:3611: done (2 left) 2024-04-12T11:29:41.747 INFO:tasks.rados.rados.0.smithi029.stdout:3612: done (1 left) 2024-04-12T11:29:41.748 INFO:tasks.rados.rados.0.smithi029.stdout:3613: done (0 left) 2024-04-12T11:29:41.748 INFO:tasks.rados.rados.0.smithi029.stdout:3614: read oid 2 snap -1 2024-04-12T11:29:41.748 INFO:tasks.rados.rados.0.smithi029.stdout:3614: expect deleted 2024-04-12T11:29:41.748 INFO:tasks.rados.rados.0.smithi029.stdout:3615: snap_remove snap 446 2024-04-12T11:29:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:41 smithi029 ceph-mon[93650]: osdmap e976: 8 total, 8 up, 8 in 2024-04-12T11:29:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:41 smithi029 ceph-mon[97787]: osdmap e976: 8 total, 8 up, 8 in 2024-04-12T11:29:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:41 smithi087 ceph-mon[89014]: osdmap e976: 8 total, 8 up, 8 in 2024-04-12T11:29:42.712 INFO:tasks.rados.rados.0.smithi029.stdout:3615: done (1 left) 2024-04-12T11:29:42.712 INFO:tasks.rados.rados.0.smithi029.stdout:3616: write oid 18 current snap is 457 2024-04-12T11:29:42.712 INFO:tasks.rados.rados.0.smithi029.stdout:3616: seq_num 1351 ranges {706257=655880,1890820=504465} 2024-04-12T11:29:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:29:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:29:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:29:42.720 INFO:tasks.rados.rados.0.smithi029.stdout:3616: writing smithi02979277-18 from 706257 to 1362137 tid 1 2024-04-12T11:29:42.727 INFO:tasks.rados.rados.0.smithi029.stdout:3616: writing smithi02979277-18 from 1890820 to 2395285 tid 2 2024-04-12T11:29:42.727 INFO:tasks.rados.rados.0.smithi029.stdout:3617: delete oid 32 current snap is 457 2024-04-12T11:29:42.731 INFO:tasks.rados.rados.0.smithi029.stdout:3614: done (2 left) 2024-04-12T11:29:42.731 INFO:tasks.rados.rados.0.smithi029.stdout:3617: done (1 left) 2024-04-12T11:29:42.732 INFO:tasks.rados.rados.0.smithi029.stdout:3618: read oid 1 snap -1 2024-04-12T11:29:42.732 INFO:tasks.rados.rados.0.smithi029.stdout:3618: expect (ObjNum 1343 snap 456 seq_num 1343) 2024-04-12T11:29:42.732 INFO:tasks.rados.rados.0.smithi029.stdout:3619: write oid 23 current snap is 457 2024-04-12T11:29:42.732 INFO:tasks.rados.rados.0.smithi029.stdout:3619: seq_num 1353 ranges {739871=406378,1607784=508134} 2024-04-12T11:29:42.738 INFO:tasks.rados.rados.0.smithi029.stdout:3619: writing smithi02979277-23 from 739871 to 1146249 tid 1 2024-04-12T11:29:42.743 INFO:tasks.rados.rados.0.smithi029.stdout:3619: writing smithi02979277-23 from 1607784 to 2115918 tid 2 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:3620: delete oid 27 current snap is 457 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:3616: finishing write tid 1 to smithi02979277-18 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:3616: finishing write tid 2 to smithi02979277-18 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:3616: finishing write tid 3 to smithi02979277-18 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:3616: finishing write tid 4 to smithi02979277-18 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:3616: oid 18 updating version 0 to 1909 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:3616: oid 18 updating version 1909 to 1910 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:3616: oid 18 version 1910 is already newer than 1908 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 1910 (ObjNum 1351 snap 457 seq_num 1351) dirty exists 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:3616: left oid 18 (ObjNum 1351 snap 457 seq_num 1351) 2024-04-12T11:29:42.744 INFO:tasks.rados.rados.0.smithi029.stdout:3618: expect (ObjNum 1343 snap 456 seq_num 1343) 2024-04-12T11:29:42.926 INFO:tasks.rados.rados.0.smithi029.stdout:3619: finishing write tid 1 to smithi02979277-23 2024-04-12T11:29:42.926 INFO:tasks.rados.rados.0.smithi029.stdout:3619: finishing write tid 2 to smithi02979277-23 2024-04-12T11:29:42.926 INFO:tasks.rados.rados.0.smithi029.stdout:3619: finishing write tid 3 to smithi02979277-23 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:3619: finishing write tid 4 to smithi02979277-23 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:3619: oid 23 updating version 0 to 1551 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:3619: oid 23 updating version 1551 to 1552 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:3619: oid 23 updating version 1552 to 1553 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1553 (ObjNum 1353 snap 457 seq_num 1353) dirty exists 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:3619: left oid 23 (ObjNum 1353 snap 457 seq_num 1353) 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:3616: done (3 left) 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:3618: done (2 left) 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:3619: done (1 left) 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:3620: done (0 left) 2024-04-12T11:29:42.927 INFO:tasks.rados.rados.0.smithi029.stdout:3621: delete oid 5 current snap is 457 2024-04-12T11:29:42.929 INFO:tasks.rados.rados.0.smithi029.stdout:3621: done (0 left) 2024-04-12T11:29:42.929 INFO:tasks.rados.rados.0.smithi029.stdout:3622: write oid 32 current snap is 457 2024-04-12T11:29:42.929 INFO:tasks.rados.rados.0.smithi029.stdout:3622: seq_num 1356 ranges {790292=748478,2183230=607062} 2024-04-12T11:29:42.937 INFO:tasks.rados.rados.0.smithi029.stdout:3622: writing smithi02979277-32 from 790292 to 1538770 tid 1 2024-04-12T11:29:42.945 INFO:tasks.rados.rados.0.smithi029.stdout:3622: writing smithi02979277-32 from 2183230 to 2790292 tid 2 2024-04-12T11:29:42.945 INFO:tasks.rados.rados.0.smithi029.stdout:3623: write oid 41 current snap is 457 2024-04-12T11:29:42.945 INFO:tasks.rados.rados.0.smithi029.stdout:3623: seq_num 1357 ranges {407099=423727,1441712=515472,2582060=225039} 2024-04-12T11:29:42.949 INFO:tasks.rados.rados.0.smithi029.stdout:3623: writing smithi02979277-41 from 407099 to 830826 tid 1 2024-04-12T11:29:42.955 INFO:tasks.rados.rados.0.smithi029.stdout:3623: writing smithi02979277-41 from 1441712 to 1957184 tid 2 2024-04-12T11:29:42.960 INFO:tasks.rados.rados.0.smithi029.stdout:3623: writing smithi02979277-41 from 2582060 to 2807099 tid 3 2024-04-12T11:29:42.960 INFO:tasks.rados.rados.0.smithi029.stdout:3624: snap_remove snap 432 2024-04-12T11:29:43.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:42 smithi029 ceph-mon[93650]: pgmap v1101: 89 pgs: 89 active+clean; 240 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 867 KiB/s wr, 7 op/s 2024-04-12T11:29:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:42 smithi029 ceph-mon[93650]: osdmap e977: 8 total, 8 up, 8 in 2024-04-12T11:29:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:42 smithi029 ceph-mon[93650]: osdmap e978: 8 total, 8 up, 8 in 2024-04-12T11:29:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:42 smithi029 ceph-mon[97787]: pgmap v1101: 89 pgs: 89 active+clean; 240 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 867 KiB/s wr, 7 op/s 2024-04-12T11:29:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:42 smithi029 ceph-mon[97787]: osdmap e977: 8 total, 8 up, 8 in 2024-04-12T11:29:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:42 smithi029 ceph-mon[97787]: osdmap e978: 8 total, 8 up, 8 in 2024-04-12T11:29:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:42 smithi087 ceph-mon[89014]: pgmap v1101: 89 pgs: 89 active+clean; 240 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 867 KiB/s wr, 7 op/s 2024-04-12T11:29:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:42 smithi087 ceph-mon[89014]: osdmap e977: 8 total, 8 up, 8 in 2024-04-12T11:29:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:42 smithi087 ceph-mon[89014]: osdmap e978: 8 total, 8 up, 8 in 2024-04-12T11:29:43.754 INFO:tasks.rados.rados.0.smithi029.stdout:3624: done (2 left) 2024-04-12T11:29:43.754 INFO:tasks.rados.rados.0.smithi029.stdout:3625: snap_create 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3622: finishing write tid 1 to smithi02979277-32 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3622: finishing write tid 2 to smithi02979277-32 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3622: finishing write tid 3 to smithi02979277-32 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3622: finishing write tid 4 to smithi02979277-32 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3622: oid 32 updating version 0 to 1639 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3622: oid 32 version 1639 is already newer than 1637 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3622: oid 32 version 1639 is already newer than 1638 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 1639 (ObjNum 1356 snap 457 seq_num 1356) dirty exists 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3622: left oid 32 (ObjNum 1356 snap 457 seq_num 1356) 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3623: finishing write tid 1 to smithi02979277-41 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3623: finishing write tid 2 to smithi02979277-41 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3623: finishing write tid 3 to smithi02979277-41 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3623: finishing write tid 4 to smithi02979277-41 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3623: finishing write tid 5 to smithi02979277-41 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3623: oid 41 updating version 0 to 1754 2024-04-12T11:29:43.755 INFO:tasks.rados.rados.0.smithi029.stdout:3623: oid 41 version 1754 is already newer than 1752 2024-04-12T11:29:43.756 INFO:tasks.rados.rados.0.smithi029.stdout:3623: oid 41 version 1754 is already newer than 1751 2024-04-12T11:29:43.756 INFO:tasks.rados.rados.0.smithi029.stdout:3623: oid 41 version 1754 is already newer than 1753 2024-04-12T11:29:43.756 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1754 (ObjNum 1357 snap 457 seq_num 1357) dirty exists 2024-04-12T11:29:43.756 INFO:tasks.rados.rados.0.smithi029.stdout:3623: left oid 41 (ObjNum 1357 snap 457 seq_num 1357) 2024-04-12T11:29:44.760 INFO:tasks.rados.rados.0.smithi029.stdout:3622: done (2 left) 2024-04-12T11:29:44.760 INFO:tasks.rados.rados.0.smithi029.stdout:3623: done (1 left) 2024-04-12T11:29:44.760 INFO:tasks.rados.rados.0.smithi029.stdout:3625: done (0 left) 2024-04-12T11:29:44.760 INFO:tasks.rados.rados.0.smithi029.stdout:3626: snap_create 2024-04-12T11:29:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:44 smithi029 ceph-mon[97787]: pgmap v1104: 89 pgs: 89 active+clean; 245 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-04-12T11:29:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:44 smithi029 ceph-mon[97787]: osdmap e979: 8 total, 8 up, 8 in 2024-04-12T11:29:45.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:44 smithi029 ceph-mon[93650]: pgmap v1104: 89 pgs: 89 active+clean; 245 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-04-12T11:29:45.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:44 smithi029 ceph-mon[93650]: osdmap e979: 8 total, 8 up, 8 in 2024-04-12T11:29:45.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:29:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:29:44] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2024-04-12T11:29:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:44 smithi087 ceph-mon[89014]: pgmap v1104: 89 pgs: 89 active+clean; 245 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-04-12T11:29:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:44 smithi087 ceph-mon[89014]: osdmap e979: 8 total, 8 up, 8 in 2024-04-12T11:29:45.766 INFO:tasks.rados.rados.0.smithi029.stdout:3626: done (0 left) 2024-04-12T11:29:45.767 INFO:tasks.rados.rados.0.smithi029.stdout:3627: write oid 9 current snap is 459 2024-04-12T11:29:45.767 INFO:tasks.rados.rados.0.smithi029.stdout:3627: seq_num 1358 ranges {423906=498976,1500194=659130,2609483=214423} 2024-04-12T11:29:45.771 INFO:tasks.rados.rados.0.smithi029.stdout:3627: writing smithi02979277-9 from 423906 to 922882 tid 1 2024-04-12T11:29:45.779 INFO:tasks.rados.rados.0.smithi029.stdout:3627: writing smithi02979277-9 from 1500194 to 2159324 tid 2 2024-04-12T11:29:45.783 INFO:tasks.rados.rados.0.smithi029.stdout:3627: writing smithi02979277-9 from 2609483 to 2823906 tid 3 2024-04-12T11:29:45.783 INFO:tasks.rados.rados.0.smithi029.stdout:3628: write oid 7 current snap is 459 2024-04-12T11:29:45.783 INFO:tasks.rados.rados.0.smithi029.stdout:3628: seq_num 1359 ranges {440713=574225,1442323=519141,2636906=203807} 2024-04-12T11:29:45.788 INFO:tasks.rados.rados.0.smithi029.stdout:3628: writing smithi02979277-7 from 440713 to 1014938 tid 1 2024-04-12T11:29:45.793 INFO:tasks.rados.rados.0.smithi029.stdout:3628: writing smithi02979277-7 from 1442323 to 1961464 tid 2 2024-04-12T11:29:45.799 INFO:tasks.rados.rados.0.smithi029.stdout:3628: writing smithi02979277-7 from 2636906 to 2840713 tid 3 2024-04-12T11:29:45.799 INFO:tasks.rados.rados.0.smithi029.stdout:3629: delete oid 18 current snap is 459 2024-04-12T11:29:45.799 INFO:tasks.rados.rados.0.smithi029.stdout:3627: finishing write tid 1 to smithi02979277-9 2024-04-12T11:29:45.799 INFO:tasks.rados.rados.0.smithi029.stdout:3627: finishing write tid 2 to smithi02979277-9 2024-04-12T11:29:45.799 INFO:tasks.rados.rados.0.smithi029.stdout:3627: finishing write tid 3 to smithi02979277-9 2024-04-12T11:29:45.799 INFO:tasks.rados.rados.0.smithi029.stdout:3627: finishing write tid 4 to smithi02979277-9 2024-04-12T11:29:45.799 INFO:tasks.rados.rados.0.smithi029.stdout:3627: finishing write tid 5 to smithi02979277-9 2024-04-12T11:29:45.799 INFO:tasks.rados.rados.0.smithi029.stdout:3627: oid 9 updating version 0 to 1267 2024-04-12T11:29:45.799 INFO:tasks.rados.rados.0.smithi029.stdout:3627: oid 9 updating version 1267 to 1269 2024-04-12T11:29:45.800 INFO:tasks.rados.rados.0.smithi029.stdout:3627: oid 9 version 1269 is already newer than 1268 2024-04-12T11:29:45.800 INFO:tasks.rados.rados.0.smithi029.stdout:3627: oid 9 version 1269 is already newer than 1266 2024-04-12T11:29:45.800 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 1269 (ObjNum 1358 snap 459 seq_num 1358) dirty exists 2024-04-12T11:29:45.800 INFO:tasks.rados.rados.0.smithi029.stdout:3627: left oid 9 (ObjNum 1358 snap 459 seq_num 1358) 2024-04-12T11:29:45.800 INFO:tasks.rados.rados.0.smithi029.stdout:3628: finishing write tid 1 to smithi02979277-7 2024-04-12T11:29:45.800 INFO:tasks.rados.rados.0.smithi029.stdout:3628: finishing write tid 2 to smithi02979277-7 2024-04-12T11:29:45.802 INFO:tasks.rados.rados.0.smithi029.stdout:3628: finishing write tid 3 to smithi02979277-7 2024-04-12T11:29:45.802 INFO:tasks.rados.rados.0.smithi029.stdout:3628: finishing write tid 4 to smithi02979277-7 2024-04-12T11:29:45.802 INFO:tasks.rados.rados.0.smithi029.stdout:3628: finishing write tid 5 to smithi02979277-7 2024-04-12T11:29:45.802 INFO:tasks.rados.rados.0.smithi029.stdout:3628: oid 7 updating version 0 to 1813 2024-04-12T11:29:45.803 INFO:tasks.rados.rados.0.smithi029.stdout:3628: oid 7 version 1813 is already newer than 1811 2024-04-12T11:29:45.803 INFO:tasks.rados.rados.0.smithi029.stdout:3628: oid 7 version 1813 is already newer than 1812 2024-04-12T11:29:45.803 INFO:tasks.rados.rados.0.smithi029.stdout:3628: oid 7 updating version 1813 to 1814 2024-04-12T11:29:45.803 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1814 (ObjNum 1359 snap 459 seq_num 1359) dirty exists 2024-04-12T11:29:45.803 INFO:tasks.rados.rados.0.smithi029.stdout:3628: left oid 7 (ObjNum 1359 snap 459 seq_num 1359) 2024-04-12T11:29:45.804 INFO:tasks.rados.rados.0.smithi029.stdout:3627: done (2 left) 2024-04-12T11:29:45.804 INFO:tasks.rados.rados.0.smithi029.stdout:3628: done (1 left) 2024-04-12T11:29:45.804 INFO:tasks.rados.rados.0.smithi029.stdout:3629: done (0 left) 2024-04-12T11:29:45.804 INFO:tasks.rados.rados.0.smithi029.stdout:3630: delete oid 35 current snap is 459 2024-04-12T11:29:45.807 INFO:tasks.rados.rados.0.smithi029.stdout:3630: done (0 left) 2024-04-12T11:29:45.807 INFO:tasks.rados.rados.0.smithi029.stdout:3631: snap_create 2024-04-12T11:29:46.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:45 smithi029 ceph-mon[93650]: osdmap e980: 8 total, 8 up, 8 in 2024-04-12T11:29:46.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:45 smithi029 ceph-mon[97787]: osdmap e980: 8 total, 8 up, 8 in 2024-04-12T11:29:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:45 smithi087 ceph-mon[89014]: osdmap e980: 8 total, 8 up, 8 in 2024-04-12T11:29:46.768 INFO:tasks.rados.rados.0.smithi029.stdout:3631: done (0 left) 2024-04-12T11:29:46.768 INFO:tasks.rados.rados.0.smithi029.stdout:3632: snap_remove snap 459 2024-04-12T11:29:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:46 smithi087 ceph-mon[89014]: pgmap v1107: 89 pgs: 89 active+clean; 250 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-04-12T11:29:47.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:46 smithi087 ceph-mon[89014]: osdmap e981: 8 total, 8 up, 8 in 2024-04-12T11:29:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:46 smithi029 ceph-mon[97787]: pgmap v1107: 89 pgs: 89 active+clean; 250 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-04-12T11:29:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:46 smithi029 ceph-mon[97787]: osdmap e981: 8 total, 8 up, 8 in 2024-04-12T11:29:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:46 smithi029 ceph-mon[93650]: pgmap v1107: 89 pgs: 89 active+clean; 250 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-04-12T11:29:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:46 smithi029 ceph-mon[93650]: osdmap e981: 8 total, 8 up, 8 in 2024-04-12T11:29:47.715 INFO:tasks.rados.rados.0.smithi029.stdout:3632: done (0 left) 2024-04-12T11:29:47.715 INFO:tasks.rados.rados.0.smithi029.stdout:3633: rollback oid 39 current snap is 460 2024-04-12T11:29:47.715 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 445 2024-04-12T11:29:47.715 INFO:tasks.rados.rados.0.smithi029.stdout:3634: delete oid 14 current snap is 460 2024-04-12T11:29:47.721 INFO:tasks.rados.rados.0.smithi029.stdout:3633: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:29:47.721 INFO:tasks.rados.rados.0.smithi029.stdout:3633: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:29:47.722 INFO:tasks.rados.rados.0.smithi029.stdout:3633: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:29:47.722 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 525 (ObjNum 1088 snap 365 seq_num 1088) dirty exists 2024-04-12T11:29:47.722 INFO:tasks.rados.rados.0.smithi029.stdout:3633: done (1 left) 2024-04-12T11:29:47.722 INFO:tasks.rados.rados.0.smithi029.stdout:3634: done (0 left) 2024-04-12T11:29:47.722 INFO:tasks.rados.rados.0.smithi029.stdout:3635: snap_create 2024-04-12T11:29:48.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:47 smithi087 ceph-mon[89014]: osdmap e982: 8 total, 8 up, 8 in 2024-04-12T11:29:48.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:47 smithi087 ceph-mon[89014]: osdmap e983: 8 total, 8 up, 8 in 2024-04-12T11:29:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:47 smithi029 ceph-mon[93650]: osdmap e982: 8 total, 8 up, 8 in 2024-04-12T11:29:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:47 smithi029 ceph-mon[93650]: osdmap e983: 8 total, 8 up, 8 in 2024-04-12T11:29:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:47 smithi029 ceph-mon[97787]: osdmap e982: 8 total, 8 up, 8 in 2024-04-12T11:29:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:47 smithi029 ceph-mon[97787]: osdmap e983: 8 total, 8 up, 8 in 2024-04-12T11:29:48.717 INFO:tasks.rados.rados.0.smithi029.stdout:3635: done (0 left) 2024-04-12T11:29:48.717 INFO:tasks.rados.rados.0.smithi029.stdout:3636: read oid 43 snap 456 2024-04-12T11:29:48.717 INFO:tasks.rados.rados.0.smithi029.stdout:3636: expect (ObjNum 1339 snap 453 seq_num 1339) 2024-04-12T11:29:48.717 INFO:tasks.rados.rados.0.smithi029.stdout:3637: read oid 9 snap -1 2024-04-12T11:29:48.717 INFO:tasks.rados.rados.0.smithi029.stdout:3637: expect (ObjNum 1358 snap 459 seq_num 1358) 2024-04-12T11:29:48.717 INFO:tasks.rados.rados.0.smithi029.stdout:3638: read oid 10 snap 433 2024-04-12T11:29:48.717 INFO:tasks.rados.rados.0.smithi029.stdout:3638: expect deleted 2024-04-12T11:29:48.717 INFO:tasks.rados.rados.0.smithi029.stdout:3639: snap_create 2024-04-12T11:29:48.720 INFO:tasks.rados.rados.0.smithi029.stdout:3637: expect (ObjNum 1358 snap 459 seq_num 1358) 2024-04-12T11:29:48.846 INFO:tasks.rados.rados.0.smithi029.stdout:3636: expect (ObjNum 1339 snap 453 seq_num 1339) 2024-04-12T11:29:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:48 smithi087 ceph-mon[89014]: pgmap v1110: 89 pgs: 89 active+clean; 255 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-04-12T11:29:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:48 smithi087 ceph-mon[89014]: osdmap e984: 8 total, 8 up, 8 in 2024-04-12T11:29:49.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:48 smithi029 ceph-mon[97787]: pgmap v1110: 89 pgs: 89 active+clean; 255 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-04-12T11:29:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:48 smithi029 ceph-mon[97787]: osdmap e984: 8 total, 8 up, 8 in 2024-04-12T11:29:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:48 smithi029 ceph-mon[93650]: pgmap v1110: 89 pgs: 89 active+clean; 255 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-04-12T11:29:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:48 smithi029 ceph-mon[93650]: osdmap e984: 8 total, 8 up, 8 in 2024-04-12T11:29:49.718 INFO:tasks.rados.rados.0.smithi029.stdout:3636: done (3 left) 2024-04-12T11:29:49.718 INFO:tasks.rados.rados.0.smithi029.stdout:3637: done (2 left) 2024-04-12T11:29:49.718 INFO:tasks.rados.rados.0.smithi029.stdout:3638: done (1 left) 2024-04-12T11:29:49.718 INFO:tasks.rados.rados.0.smithi029.stdout:3639: done (0 left) 2024-04-12T11:29:49.718 INFO:tasks.rados.rados.0.smithi029.stdout:3640: read oid 34 snap -1 2024-04-12T11:29:49.718 INFO:tasks.rados.rados.0.smithi029.stdout:3640: expect deleted 2024-04-12T11:29:49.718 INFO:tasks.rados.rados.0.smithi029.stdout:3641: delete oid 28 current snap is 462 2024-04-12T11:29:49.723 INFO:tasks.rados.rados.0.smithi029.stdout:3640: done (1 left) 2024-04-12T11:29:49.723 INFO:tasks.rados.rados.0.smithi029.stdout:3641: done (0 left) 2024-04-12T11:29:49.723 INFO:tasks.rados.rados.0.smithi029.stdout:3642: write oid 5 current snap is 462 2024-04-12T11:29:49.724 INFO:tasks.rados.rados.0.smithi029.stdout:3642: seq_num 1364 ranges {524748=666823,1734733=670137} 2024-04-12T11:29:49.729 INFO:tasks.rados.rados.0.smithi029.stdout:3642: writing smithi02979277-5 from 524748 to 1191571 tid 1 2024-04-12T11:29:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3642: writing smithi02979277-5 from 1734733 to 2404870 tid 2 2024-04-12T11:29:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3643: read oid 15 snap -1 2024-04-12T11:29:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3643: expect (ObjNum 1300 snap 444 seq_num 1300) 2024-04-12T11:29:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3644: snap_remove snap 457 2024-04-12T11:29:50.723 INFO:tasks.rados.rados.0.smithi029.stdout:3644: done (2 left) 2024-04-12T11:29:50.723 INFO:tasks.rados.rados.0.smithi029.stdout:3645: snap_create 2024-04-12T11:29:50.723 INFO:tasks.rados.rados.0.smithi029.stdout:3642: finishing write tid 1 to smithi02979277-5 2024-04-12T11:29:50.723 INFO:tasks.rados.rados.0.smithi029.stdout:3643: expect (ObjNum 1300 snap 444 seq_num 1300) 2024-04-12T11:29:50.855 INFO:tasks.rados.rados.0.smithi029.stdout:3642: finishing write tid 2 to smithi02979277-5 2024-04-12T11:29:50.855 INFO:tasks.rados.rados.0.smithi029.stdout:3642: finishing write tid 3 to smithi02979277-5 2024-04-12T11:29:50.855 INFO:tasks.rados.rados.0.smithi029.stdout:3642: finishing write tid 4 to smithi02979277-5 2024-04-12T11:29:50.855 INFO:tasks.rados.rados.0.smithi029.stdout:3642: oid 5 updating version 0 to 1736 2024-04-12T11:29:50.856 INFO:tasks.rados.rados.0.smithi029.stdout:3642: oid 5 updating version 1736 to 1737 2024-04-12T11:29:50.856 INFO:tasks.rados.rados.0.smithi029.stdout:3642: oid 5 updating version 1737 to 1738 2024-04-12T11:29:50.856 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1738 (ObjNum 1364 snap 462 seq_num 1364) dirty exists 2024-04-12T11:29:50.856 INFO:tasks.rados.rados.0.smithi029.stdout:3642: left oid 5 (ObjNum 1364 snap 462 seq_num 1364) 2024-04-12T11:29:50.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:50 smithi087 ceph-mon[89014]: pgmap v1113: 89 pgs: 89 active+clean; 259 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2024-04-12T11:29:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:50 smithi087 ceph-mon[89014]: osdmap e985: 8 total, 8 up, 8 in 2024-04-12T11:29:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:50 smithi029 ceph-mon[93650]: pgmap v1113: 89 pgs: 89 active+clean; 259 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2024-04-12T11:29:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:50 smithi029 ceph-mon[93650]: osdmap e985: 8 total, 8 up, 8 in 2024-04-12T11:29:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:50 smithi029 ceph-mon[97787]: pgmap v1113: 89 pgs: 89 active+clean; 259 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 10 op/s 2024-04-12T11:29:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:50 smithi029 ceph-mon[97787]: osdmap e985: 8 total, 8 up, 8 in 2024-04-12T11:29:51.728 INFO:tasks.rados.rados.0.smithi029.stdout:3642: done (2 left) 2024-04-12T11:29:51.728 INFO:tasks.rados.rados.0.smithi029.stdout:3643: done (1 left) 2024-04-12T11:29:51.729 INFO:tasks.rados.rados.0.smithi029.stdout:3645: done (0 left) 2024-04-12T11:29:51.729 INFO:tasks.rados.rados.0.smithi029.stdout:3646: delete oid 6 current snap is 463 2024-04-12T11:29:51.731 INFO:tasks.rados.rados.0.smithi029.stdout:3646: done (0 left) 2024-04-12T11:29:51.731 INFO:tasks.rados.rados.0.smithi029.stdout:3647: delete oid 32 current snap is 463 2024-04-12T11:29:51.735 INFO:tasks.rados.rados.0.smithi029.stdout:3647: done (0 left) 2024-04-12T11:29:51.735 INFO:tasks.rados.rados.0.smithi029.stdout:3648: read oid 33 snap -1 2024-04-12T11:29:51.735 INFO:tasks.rados.rados.0.smithi029.stdout:3648: expect (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:29:51.735 INFO:tasks.rados.rados.0.smithi029.stdout:3649: write oid 16 current snap is 463 2024-04-12T11:29:51.735 INFO:tasks.rados.rados.0.smithi029.stdout:3649: seq_num 1367 ranges {575169=492570,1793826=417464,2623584=351585} 2024-04-12T11:29:51.741 INFO:tasks.rados.rados.0.smithi029.stdout:3649: writing smithi02979277-16 from 575169 to 1067739 tid 1 2024-04-12T11:29:51.748 INFO:tasks.rados.rados.0.smithi029.stdout:3649: writing smithi02979277-16 from 1793826 to 2211290 tid 2 2024-04-12T11:29:51.752 INFO:tasks.rados.rados.0.smithi029.stdout:3649: writing smithi02979277-16 from 2623584 to 2975169 tid 3 2024-04-12T11:29:51.752 INFO:tasks.rados.rados.0.smithi029.stdout:3650: snap_remove snap 458 2024-04-12T11:29:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:51 smithi029 ceph-mon[93650]: osdmap e986: 8 total, 8 up, 8 in 2024-04-12T11:29:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:51 smithi029 ceph-mon[97787]: osdmap e986: 8 total, 8 up, 8 in 2024-04-12T11:29:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:51 smithi087 ceph-mon[89014]: osdmap e986: 8 total, 8 up, 8 in 2024-04-12T11:29:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:29:52.716 INFO:tasks.rados.rados.0.smithi029.stdout:3650: done (2 left) 2024-04-12T11:29:52.716 INFO:tasks.rados.rados.0.smithi029.stdout:3651: snap_remove snap 445 2024-04-12T11:29:52.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:29:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:29:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:29:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:52 smithi029 ceph-mon[93650]: pgmap v1116: 89 pgs: 89 active+clean; 259 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 652 KiB/s wr, 3 op/s 2024-04-12T11:29:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:52 smithi029 ceph-mon[93650]: osdmap e987: 8 total, 8 up, 8 in 2024-04-12T11:29:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:52 smithi029 ceph-mon[93650]: osdmap e988: 8 total, 8 up, 8 in 2024-04-12T11:29:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:52 smithi029 ceph-mon[97787]: pgmap v1116: 89 pgs: 89 active+clean; 259 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 652 KiB/s wr, 3 op/s 2024-04-12T11:29:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:52 smithi029 ceph-mon[97787]: osdmap e987: 8 total, 8 up, 8 in 2024-04-12T11:29:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:52 smithi029 ceph-mon[97787]: osdmap e988: 8 total, 8 up, 8 in 2024-04-12T11:29:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:52 smithi087 ceph-mon[89014]: pgmap v1116: 89 pgs: 89 active+clean; 259 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 652 KiB/s wr, 3 op/s 2024-04-12T11:29:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:52 smithi087 ceph-mon[89014]: osdmap e987: 8 total, 8 up, 8 in 2024-04-12T11:29:53.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:52 smithi087 ceph-mon[89014]: osdmap e988: 8 total, 8 up, 8 in 2024-04-12T11:29:53.718 INFO:tasks.rados.rados.0.smithi029.stdout:3651: done (2 left) 2024-04-12T11:29:53.718 INFO:tasks.rados.rados.0.smithi029.stdout:3652: delete oid 48 current snap is 463 2024-04-12T11:29:53.718 INFO:tasks.rados.rados.0.smithi029.stdout:3648: expect (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:29:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3649: finishing write tid 1 to smithi02979277-16 2024-04-12T11:29:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3649: finishing write tid 2 to smithi02979277-16 2024-04-12T11:29:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3648: done (2 left) 2024-04-12T11:29:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3652: done (1 left) 2024-04-12T11:29:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3653: snap_create 2024-04-12T11:29:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3649: finishing write tid 3 to smithi02979277-16 2024-04-12T11:29:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3649: finishing write tid 4 to smithi02979277-16 2024-04-12T11:29:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3649: finishing write tid 5 to smithi02979277-16 2024-04-12T11:29:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3649: oid 16 updating version 0 to 1744 2024-04-12T11:29:53.797 INFO:tasks.rados.rados.0.smithi029.stdout:3649: oid 16 version 1744 is already newer than 1743 2024-04-12T11:29:53.797 INFO:tasks.rados.rados.0.smithi029.stdout:3649: oid 16 version 1744 is already newer than 1742 2024-04-12T11:29:53.797 INFO:tasks.rados.rados.0.smithi029.stdout:3649: oid 16 updating version 1744 to 1745 2024-04-12T11:29:53.797 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1745 (ObjNum 1367 snap 463 seq_num 1367) dirty exists 2024-04-12T11:29:53.797 INFO:tasks.rados.rados.0.smithi029.stdout:3649: left oid 16 (ObjNum 1367 snap 463 seq_num 1367) 2024-04-12T11:29:54.733 INFO:tasks.rados.rados.0.smithi029.stdout:3649: done (1 left) 2024-04-12T11:29:54.733 INFO:tasks.rados.rados.0.smithi029.stdout:3653: done (0 left) 2024-04-12T11:29:54.734 INFO:tasks.rados.rados.0.smithi029.stdout:3654: read oid 47 snap -1 2024-04-12T11:29:54.734 INFO:tasks.rados.rados.0.smithi029.stdout:3654: expect deleted 2024-04-12T11:29:54.734 INFO:tasks.rados.rados.0.smithi029.stdout:3655: write oid 4 current snap is 464 2024-04-12T11:29:54.734 INFO:tasks.rados.rados.0.smithi029.stdout:3655: seq_num 1369 ranges {608783=759421,2027143=421133} 2024-04-12T11:29:54.741 INFO:tasks.rados.rados.0.smithi029.stdout:3655: writing smithi02979277-4 from 608783 to 1368204 tid 1 2024-04-12T11:29:54.747 INFO:tasks.rados.rados.0.smithi029.stdout:3655: writing smithi02979277-4 from 2027143 to 2448276 tid 2 2024-04-12T11:29:54.747 INFO:tasks.rados.rados.0.smithi029.stdout:3656: write oid 13 current snap is 464 2024-04-12T11:29:54.747 INFO:tasks.rados.rados.0.smithi029.stdout:3656: seq_num 1370 ranges {625590=434670,1685625=681144,2822206=203384} 2024-04-12T11:29:54.753 INFO:tasks.rados.rados.0.smithi029.stdout:3656: writing smithi02979277-13 from 625590 to 1060260 tid 1 2024-04-12T11:29:54.761 INFO:tasks.rados.rados.0.smithi029.stdout:3656: writing smithi02979277-13 from 1685625 to 2366769 tid 2 2024-04-12T11:29:54.765 INFO:tasks.rados.rados.0.smithi029.stdout:3656: writing smithi02979277-13 from 2822206 to 3025590 tid 3 2024-04-12T11:29:54.765 INFO:tasks.rados.rados.0.smithi029.stdout:3657: write oid 14 current snap is 464 2024-04-12T11:29:54.765 INFO:tasks.rados.rados.0.smithi029.stdout:3657: seq_num 1371 ranges {642397=509919,1627754=424802,2733276=309121} 2024-04-12T11:29:54.771 INFO:tasks.rados.rados.0.smithi029.stdout:3657: writing smithi02979277-14 from 642397 to 1152316 tid 1 2024-04-12T11:29:54.776 INFO:tasks.rados.rados.0.smithi029.stdout:3657: writing smithi02979277-14 from 1627754 to 2052556 tid 2 2024-04-12T11:29:54.782 INFO:tasks.rados.rados.0.smithi029.stdout:3657: writing smithi02979277-14 from 2733276 to 3042397 tid 3 2024-04-12T11:29:54.782 INFO:tasks.rados.rados.0.smithi029.stdout:3658: write oid 16 current snap is 464 2024-04-12T11:29:54.782 INFO:tasks.rados.rados.0.smithi029.stdout:3658: seq_num 1372 ranges {659204=585168,1686236=684813} 2024-04-12T11:29:54.789 INFO:tasks.rados.rados.0.smithi029.stdout:3658: writing smithi02979277-16 from 659204 to 1244372 tid 1 2024-04-12T11:29:54.796 INFO:tasks.rados.rados.0.smithi029.stdout:3658: writing smithi02979277-16 from 1686236 to 2371049 tid 2 2024-04-12T11:29:54.796 INFO:tasks.rados.rados.0.smithi029.stdout:3659: snap_create 2024-04-12T11:29:54.796 INFO:tasks.rados.rados.0.smithi029.stdout:3655: finishing write tid 1 to smithi02979277-4 2024-04-12T11:29:54.796 INFO:tasks.rados.rados.0.smithi029.stdout:3655: finishing write tid 2 to smithi02979277-4 2024-04-12T11:29:54.796 INFO:tasks.rados.rados.0.smithi029.stdout:3655: finishing write tid 3 to smithi02979277-4 2024-04-12T11:29:54.796 INFO:tasks.rados.rados.0.smithi029.stdout:3655: finishing write tid 4 to smithi02979277-4 2024-04-12T11:29:54.796 INFO:tasks.rados.rados.0.smithi029.stdout:3655: oid 4 updating version 0 to 1762 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3655: oid 4 updating version 1762 to 1763 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3655: oid 4 version 1763 is already newer than 1761 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1763 (ObjNum 1369 snap 464 seq_num 1369) dirty exists 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3655: left oid 4 (ObjNum 1369 snap 464 seq_num 1369) 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3656: finishing write tid 1 to smithi02979277-13 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3656: finishing write tid 2 to smithi02979277-13 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3656: finishing write tid 3 to smithi02979277-13 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3656: finishing write tid 4 to smithi02979277-13 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3656: finishing write tid 5 to smithi02979277-13 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3656: oid 13 updating version 0 to 1934 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3656: oid 13 version 1934 is already newer than 1933 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3656: oid 13 updating version 1934 to 1935 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3656: oid 13 version 1935 is already newer than 1932 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1935 (ObjNum 1370 snap 464 seq_num 1370) dirty exists 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3656: left oid 13 (ObjNum 1370 snap 464 seq_num 1370) 2024-04-12T11:29:54.797 INFO:tasks.rados.rados.0.smithi029.stdout:3657: finishing write tid 1 to smithi02979277-14 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:3657: finishing write tid 2 to smithi02979277-14 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:3657: finishing write tid 3 to smithi02979277-14 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:3657: finishing write tid 4 to smithi02979277-14 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:3657: finishing write tid 5 to smithi02979277-14 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:3657: oid 14 updating version 0 to 1764 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:3657: oid 14 updating version 1764 to 1766 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:3657: oid 14 version 1766 is already newer than 1765 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:3657: oid 14 updating version 1766 to 1767 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 1767 (ObjNum 1371 snap 464 seq_num 1371) dirty exists 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:3657: left oid 14 (ObjNum 1371 snap 464 seq_num 1371) 2024-04-12T11:29:54.798 INFO:tasks.rados.rados.0.smithi029.stdout:3658: finishing write tid 1 to smithi02979277-16 2024-04-12T11:29:54.800 INFO:tasks.rados.rados.0.smithi029.stdout:3658: finishing write tid 2 to smithi02979277-16 2024-04-12T11:29:54.800 INFO:tasks.rados.rados.0.smithi029.stdout:3658: finishing write tid 3 to smithi02979277-16 2024-04-12T11:29:54.800 INFO:tasks.rados.rados.0.smithi029.stdout:3658: finishing write tid 4 to smithi02979277-16 2024-04-12T11:29:54.800 INFO:tasks.rados.rados.0.smithi029.stdout:3658: oid 16 updating version 0 to 1771 2024-04-12T11:29:54.800 INFO:tasks.rados.rados.0.smithi029.stdout:3658: oid 16 version 1771 is already newer than 1770 2024-04-12T11:29:54.800 INFO:tasks.rados.rados.0.smithi029.stdout:3658: oid 16 version 1771 is already newer than 1769 2024-04-12T11:29:54.800 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1771 (ObjNum 1372 snap 464 seq_num 1372) dirty exists 2024-04-12T11:29:54.800 INFO:tasks.rados.rados.0.smithi029.stdout:3658: left oid 16 (ObjNum 1372 snap 464 seq_num 1372) 2024-04-12T11:29:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:54 smithi087 ceph-mon[89014]: pgmap v1119: 89 pgs: 89 active+clean; 263 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 635 KiB/s wr, 8 op/s 2024-04-12T11:29:54.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:54 smithi087 ceph-mon[89014]: osdmap e989: 8 total, 8 up, 8 in 2024-04-12T11:29:55.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:54 smithi029 ceph-mon[93650]: pgmap v1119: 89 pgs: 89 active+clean; 263 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 635 KiB/s wr, 8 op/s 2024-04-12T11:29:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:54 smithi029 ceph-mon[93650]: osdmap e989: 8 total, 8 up, 8 in 2024-04-12T11:29:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:54 smithi029 ceph-mon[97787]: pgmap v1119: 89 pgs: 89 active+clean; 263 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 635 KiB/s wr, 8 op/s 2024-04-12T11:29:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:54 smithi029 ceph-mon[97787]: osdmap e989: 8 total, 8 up, 8 in 2024-04-12T11:29:55.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:29:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:29:54] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-12T11:29:55.739 INFO:tasks.rados.rados.0.smithi029.stdout:3654: done (5 left) 2024-04-12T11:29:55.739 INFO:tasks.rados.rados.0.smithi029.stdout:3655: done (4 left) 2024-04-12T11:29:55.739 INFO:tasks.rados.rados.0.smithi029.stdout:3656: done (3 left) 2024-04-12T11:29:55.739 INFO:tasks.rados.rados.0.smithi029.stdout:3657: done (2 left) 2024-04-12T11:29:55.740 INFO:tasks.rados.rados.0.smithi029.stdout:3658: done (1 left) 2024-04-12T11:29:55.740 INFO:tasks.rados.rados.0.smithi029.stdout:3659: done (0 left) 2024-04-12T11:29:55.740 INFO:tasks.rados.rados.0.smithi029.stdout:3660: delete oid 14 current snap is 465 2024-04-12T11:29:55.748 INFO:tasks.rados.rados.0.smithi029.stdout:3660: done (0 left) 2024-04-12T11:29:55.748 INFO:tasks.rados.rados.0.smithi029.stdout:3661: rollback oid 7 current snap is 465 2024-04-12T11:29:55.748 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 7 to 433 2024-04-12T11:29:55.748 INFO:tasks.rados.rados.0.smithi029.stdout:3662: write oid 20 current snap is 465 2024-04-12T11:29:55.748 INFO:tasks.rados.rados.0.smithi029.stdout:3662: seq_num 1374 ranges {692818=735666,2203200=688482} 2024-04-12T11:29:55.756 INFO:tasks.rados.rados.0.smithi029.stdout:3662: writing smithi02979277-20 from 692818 to 1428484 tid 1 2024-04-12T11:29:55.764 INFO:tasks.rados.rados.0.smithi029.stdout:3662: writing smithi02979277-20 from 2203200 to 2891682 tid 2 2024-04-12T11:29:55.764 INFO:tasks.rados.rados.0.smithi029.stdout:3663: snap_remove snap 461 2024-04-12T11:29:56.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:55 smithi029 ceph-mon[93650]: osdmap e990: 8 total, 8 up, 8 in 2024-04-12T11:29:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:55 smithi029 ceph-mon[97787]: osdmap e990: 8 total, 8 up, 8 in 2024-04-12T11:29:56.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:55 smithi087 ceph-mon[89014]: osdmap e990: 8 total, 8 up, 8 in 2024-04-12T11:29:56.744 INFO:tasks.rados.rados.0.smithi029.stdout:3661: finishing rollback tid 0 to smithi02979277-7 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:3661: finishing rollback tid 1 to smithi02979277-7 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:3661: finishing rollback tid 2 to smithi02979277-7 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 7 v 1826 (ObjNum 959 snap 329 seq_num 959) dirty exists 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:3662: finishing write tid 1 to smithi02979277-20 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:3662: finishing write tid 2 to smithi02979277-20 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:3662: finishing write tid 3 to smithi02979277-20 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:3662: finishing write tid 4 to smithi02979277-20 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:3662: oid 20 updating version 0 to 1222 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:3662: oid 20 updating version 1222 to 1223 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:3662: oid 20 updating version 1223 to 1224 2024-04-12T11:29:56.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 1224 (ObjNum 1374 snap 465 seq_num 1374) dirty exists 2024-04-12T11:29:56.746 INFO:tasks.rados.rados.0.smithi029.stdout:3662: left oid 20 (ObjNum 1374 snap 465 seq_num 1374) 2024-04-12T11:29:56.746 INFO:tasks.rados.rados.0.smithi029.stdout:3661: done (2 left) 2024-04-12T11:29:56.746 INFO:tasks.rados.rados.0.smithi029.stdout:3662: done (1 left) 2024-04-12T11:29:56.746 INFO:tasks.rados.rados.0.smithi029.stdout:3663: done (0 left) 2024-04-12T11:29:56.746 INFO:tasks.rados.rados.0.smithi029.stdout:3664: write oid 47 current snap is 465 2024-04-12T11:29:56.746 INFO:tasks.rados.rados.0.smithi029.stdout:3664: seq_num 1375 ranges {709625=410915,1745329=432140,2726615=383010} 2024-04-12T11:29:56.751 INFO:tasks.rados.rados.0.smithi029.stdout:3664: writing smithi02979277-47 from 709625 to 1120540 tid 1 2024-04-12T11:29:56.757 INFO:tasks.rados.rados.0.smithi029.stdout:3664: writing smithi02979277-47 from 1745329 to 2177469 tid 2 2024-04-12T11:29:56.762 INFO:tasks.rados.rados.0.smithi029.stdout:3664: writing smithi02979277-47 from 2726615 to 3109625 tid 3 2024-04-12T11:29:56.762 INFO:tasks.rados.rados.0.smithi029.stdout:3665: delete oid 28 current snap is 465 2024-04-12T11:29:56.763 INFO:tasks.rados.rados.0.smithi029.stdout:3664: finishing write tid 1 to smithi02979277-47 2024-04-12T11:29:56.763 INFO:tasks.rados.rados.0.smithi029.stdout:3664: finishing write tid 2 to smithi02979277-47 2024-04-12T11:29:56.765 INFO:tasks.rados.rados.0.smithi029.stdout:3665: done (1 left) 2024-04-12T11:29:56.765 INFO:tasks.rados.rados.0.smithi029.stdout:3666: write oid 21 current snap is 465 2024-04-12T11:29:56.765 INFO:tasks.rados.rados.0.smithi029.stdout:3666: seq_num 1377 ranges {743239=677766,1978646=435809,2897814=245425} 2024-04-12T11:29:56.772 INFO:tasks.rados.rados.0.smithi029.stdout:3666: writing smithi02979277-21 from 743239 to 1421005 tid 1 2024-04-12T11:29:56.778 INFO:tasks.rados.rados.0.smithi029.stdout:3666: writing smithi02979277-21 from 1978646 to 2414455 tid 2 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3666: writing smithi02979277-21 from 2897814 to 3143239 tid 3 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3667: read oid 32 snap 462 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3667: expect (ObjNum 1356 snap 457 seq_num 1356) 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3668: rollback oid 43 current snap is 465 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 460 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3664: finishing write tid 3 to smithi02979277-47 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3664: finishing write tid 4 to smithi02979277-47 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3664: finishing write tid 5 to smithi02979277-47 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3664: oid 47 updating version 0 to 1832 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3664: oid 47 version 1832 is already newer than 1830 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3664: oid 47 version 1832 is already newer than 1829 2024-04-12T11:29:56.783 INFO:tasks.rados.rados.0.smithi029.stdout:3664: oid 47 version 1832 is already newer than 1831 2024-04-12T11:29:56.784 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1832 (ObjNum 1375 snap 465 seq_num 1375) dirty exists 2024-04-12T11:29:56.784 INFO:tasks.rados.rados.0.smithi029.stdout:3664: left oid 47 (ObjNum 1375 snap 465 seq_num 1375) 2024-04-12T11:29:56.784 INFO:tasks.rados.rados.0.smithi029.stdout:3666: finishing write tid 1 to smithi02979277-21 2024-04-12T11:29:56.784 INFO:tasks.rados.rados.0.smithi029.stdout:3666: finishing write tid 2 to smithi02979277-21 2024-04-12T11:29:56.784 INFO:tasks.rados.rados.0.smithi029.stdout:3664: done (3 left) 2024-04-12T11:29:56.784 INFO:tasks.rados.rados.0.smithi029.stdout:3669: write oid 47 current snap is 465 2024-04-12T11:29:56.784 INFO:tasks.rados.rados.0.smithi029.stdout:3669: seq_num 1378 ranges {760046=753015,2037128=695820} 2024-04-12T11:29:56.793 INFO:tasks.rados.rados.0.smithi029.stdout:3669: writing smithi02979277-47 from 760046 to 1513061 tid 1 2024-04-12T11:29:56.798 INFO:tasks.rados.rados.0.smithi029.stdout:3669: writing smithi02979277-47 from 2037128 to 2732948 tid 2 2024-04-12T11:29:56.799 INFO:tasks.rados.rados.0.smithi029.stdout:3670: read oid 41 snap -1 2024-04-12T11:29:56.799 INFO:tasks.rados.rados.0.smithi029.stdout:3670: expect (ObjNum 1357 snap 457 seq_num 1357) 2024-04-12T11:29:56.799 INFO:tasks.rados.rados.0.smithi029.stdout:3671: write oid 9 current snap is 465 2024-04-12T11:29:56.799 INFO:tasks.rados.rados.0.smithi029.stdout:3666: finishing write tid 3 to smithi02979277-21 2024-04-12T11:29:56.799 INFO:tasks.rados.rados.0.smithi029.stdout:3671: seq_num 1379 ranges {776853=428264,1979257=439478,2836307=340546} 2024-04-12T11:29:56.805 INFO:tasks.rados.rados.0.smithi029.stdout:3671: writing smithi02979277-9 from 776853 to 1205117 tid 1 2024-04-12T11:29:56.812 INFO:tasks.rados.rados.0.smithi029.stdout:3671: writing smithi02979277-9 from 1979257 to 2418735 tid 2 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3671: writing smithi02979277-9 from 2836307 to 3176853 tid 3 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3672: read oid 40 snap -1 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3672: expect (ObjNum 1348 snap 456 seq_num 1348) 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3673: snap_create 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3666: finishing write tid 4 to smithi02979277-21 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3666: finishing write tid 5 to smithi02979277-21 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3666: oid 21 updating version 0 to 1770 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3666: oid 21 version 1770 is already newer than 1768 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3666: oid 21 updating version 1770 to 1771 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3666: oid 21 version 1771 is already newer than 1769 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1771 (ObjNum 1377 snap 465 seq_num 1377) dirty exists 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3666: left oid 21 (ObjNum 1377 snap 465 seq_num 1377) 2024-04-12T11:29:56.817 INFO:tasks.rados.rados.0.smithi029.stdout:3667: expect (ObjNum 1356 snap 457 seq_num 1356) 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3668: finishing rollback tid 0 to smithi02979277-43 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3668: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3668: finishing rollback tid 2 to smithi02979277-43 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1657 (ObjNum 1339 snap 453 seq_num 1339) dirty exists 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3669: finishing write tid 1 to smithi02979277-47 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3669: finishing write tid 2 to smithi02979277-47 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3669: finishing write tid 3 to smithi02979277-47 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3669: finishing write tid 4 to smithi02979277-47 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3669: oid 47 updating version 0 to 1835 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3669: oid 47 version 1835 is already newer than 1833 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3669: oid 47 version 1835 is already newer than 1834 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1835 (ObjNum 1378 snap 465 seq_num 1378) dirty exists 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3669: left oid 47 (ObjNum 1378 snap 465 seq_num 1378) 2024-04-12T11:29:56.910 INFO:tasks.rados.rados.0.smithi029.stdout:3670: expect (ObjNum 1357 snap 457 seq_num 1357) 2024-04-12T11:29:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:56 smithi029 ceph-mon[97787]: pgmap v1122: 89 pgs: 89 active+clean; 258 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 635 KiB/s wr, 11 op/s 2024-04-12T11:29:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:56 smithi029 ceph-mon[97787]: osdmap e991: 8 total, 8 up, 8 in 2024-04-12T11:29:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:56 smithi029 ceph-mon[93650]: pgmap v1122: 89 pgs: 89 active+clean; 258 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 635 KiB/s wr, 11 op/s 2024-04-12T11:29:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:56 smithi029 ceph-mon[93650]: osdmap e991: 8 total, 8 up, 8 in 2024-04-12T11:29:57.109 INFO:tasks.rados.rados.0.smithi029.stdout:3671: finishing write tid 1 to smithi02979277-9 2024-04-12T11:29:57.109 INFO:tasks.rados.rados.0.smithi029.stdout:3671: finishing write tid 2 to smithi02979277-9 2024-04-12T11:29:57.109 INFO:tasks.rados.rados.0.smithi029.stdout:3671: finishing write tid 3 to smithi02979277-9 2024-04-12T11:29:57.110 INFO:tasks.rados.rados.0.smithi029.stdout:3671: finishing write tid 4 to smithi02979277-9 2024-04-12T11:29:57.110 INFO:tasks.rados.rados.0.smithi029.stdout:3671: finishing write tid 5 to smithi02979277-9 2024-04-12T11:29:57.110 INFO:tasks.rados.rados.0.smithi029.stdout:3671: oid 9 updating version 0 to 1278 2024-04-12T11:29:57.110 INFO:tasks.rados.rados.0.smithi029.stdout:3671: oid 9 version 1278 is already newer than 1277 2024-04-12T11:29:57.110 INFO:tasks.rados.rados.0.smithi029.stdout:3671: oid 9 updating version 1278 to 1280 2024-04-12T11:29:57.110 INFO:tasks.rados.rados.0.smithi029.stdout:3671: oid 9 version 1280 is already newer than 1279 2024-04-12T11:29:57.110 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 1280 (ObjNum 1379 snap 465 seq_num 1379) dirty exists 2024-04-12T11:29:57.110 INFO:tasks.rados.rados.0.smithi029.stdout:3671: left oid 9 (ObjNum 1379 snap 465 seq_num 1379) 2024-04-12T11:29:57.110 INFO:tasks.rados.rados.0.smithi029.stdout:3672: expect (ObjNum 1348 snap 456 seq_num 1348) 2024-04-12T11:29:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:56 smithi087 ceph-mon[89014]: pgmap v1122: 89 pgs: 89 active+clean; 258 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 635 KiB/s wr, 11 op/s 2024-04-12T11:29:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:56 smithi087 ceph-mon[89014]: osdmap e991: 8 total, 8 up, 8 in 2024-04-12T11:29:57.716 INFO:tasks.rados.rados.0.smithi029.stdout:3666: done (7 left) 2024-04-12T11:29:57.716 INFO:tasks.rados.rados.0.smithi029.stdout:3667: done (6 left) 2024-04-12T11:29:57.716 INFO:tasks.rados.rados.0.smithi029.stdout:3668: done (5 left) 2024-04-12T11:29:57.716 INFO:tasks.rados.rados.0.smithi029.stdout:3669: done (4 left) 2024-04-12T11:29:57.716 INFO:tasks.rados.rados.0.smithi029.stdout:3670: done (3 left) 2024-04-12T11:29:57.716 INFO:tasks.rados.rados.0.smithi029.stdout:3671: done (2 left) 2024-04-12T11:29:57.716 INFO:tasks.rados.rados.0.smithi029.stdout:3672: done (1 left) 2024-04-12T11:29:57.717 INFO:tasks.rados.rados.0.smithi029.stdout:3673: done (0 left) 2024-04-12T11:29:57.717 INFO:tasks.rados.rados.0.smithi029.stdout:3674: write oid 16 current snap is 466 2024-04-12T11:29:57.717 INFO:tasks.rados.rados.0.smithi029.stdout:3674: seq_num 1380 ranges {793660=503513,2037739=583136} 2024-04-12T11:29:57.723 INFO:tasks.rados.rados.0.smithi029.stdout:3674: writing smithi02979277-16 from 793660 to 1297173 tid 1 2024-04-12T11:29:57.731 INFO:tasks.rados.rados.0.smithi029.stdout:3674: writing smithi02979277-16 from 2037739 to 2620875 tid 2 2024-04-12T11:29:57.731 INFO:tasks.rados.rados.0.smithi029.stdout:3675: snap_remove snap 418 2024-04-12T11:29:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:57 smithi029 ceph-mon[93650]: osdmap e992: 8 total, 8 up, 8 in 2024-04-12T11:29:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:57 smithi029 ceph-mon[93650]: osdmap e993: 8 total, 8 up, 8 in 2024-04-12T11:29:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:57 smithi029 ceph-mon[97787]: osdmap e992: 8 total, 8 up, 8 in 2024-04-12T11:29:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:57 smithi029 ceph-mon[97787]: osdmap e993: 8 total, 8 up, 8 in 2024-04-12T11:29:58.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:57 smithi087 ceph-mon[89014]: osdmap e992: 8 total, 8 up, 8 in 2024-04-12T11:29:58.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:57 smithi087 ceph-mon[89014]: osdmap e993: 8 total, 8 up, 8 in 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:3675: done (1 left) 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:3676: delete oid 28 current snap is 466 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:3674: finishing write tid 1 to smithi02979277-16 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:3674: finishing write tid 2 to smithi02979277-16 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:3674: finishing write tid 3 to smithi02979277-16 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:3674: finishing write tid 4 to smithi02979277-16 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:3674: oid 16 updating version 0 to 1779 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:3674: oid 16 updating version 1779 to 1780 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:3674: oid 16 updating version 1780 to 1781 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1781 (ObjNum 1380 snap 466 seq_num 1380) dirty exists 2024-04-12T11:29:58.719 INFO:tasks.rados.rados.0.smithi029.stdout:3674: left oid 16 (ObjNum 1380 snap 466 seq_num 1380) 2024-04-12T11:29:58.729 INFO:tasks.rados.rados.0.smithi029.stdout:3674: done (1 left) 2024-04-12T11:29:58.730 INFO:tasks.rados.rados.0.smithi029.stdout:3676: done (0 left) 2024-04-12T11:29:58.730 INFO:tasks.rados.rados.0.smithi029.stdout:3677: snap_create 2024-04-12T11:29:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:58 smithi029 ceph-mon[97787]: pgmap v1125: 89 pgs: 89 active+clean; 261 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-04-12T11:29:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:29:58 smithi029 ceph-mon[97787]: osdmap e994: 8 total, 8 up, 8 in 2024-04-12T11:29:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:58 smithi029 ceph-mon[93650]: pgmap v1125: 89 pgs: 89 active+clean; 261 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-04-12T11:29:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:29:58 smithi029 ceph-mon[93650]: osdmap e994: 8 total, 8 up, 8 in 2024-04-12T11:29:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:58 smithi087 ceph-mon[89014]: pgmap v1125: 89 pgs: 89 active+clean; 261 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.5 MiB/s wr, 11 op/s 2024-04-12T11:29:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:29:58 smithi087 ceph-mon[89014]: osdmap e994: 8 total, 8 up, 8 in 2024-04-12T11:29:59.720 INFO:tasks.rados.rados.0.smithi029.stdout:3677: done (0 left) 2024-04-12T11:29:59.720 INFO:tasks.rados.rados.0.smithi029.stdout:3678: rollback oid 30 current snap is 467 2024-04-12T11:29:59.720 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 30 to 465 2024-04-12T11:29:59.720 INFO:tasks.rados.rados.0.smithi029.stdout:3679: snap_remove snap 460 2024-04-12T11:30:00.728 INFO:tasks.rados.rados.0.smithi029.stdout:3679: done (1 left) 2024-04-12T11:30:00.728 INFO:tasks.rados.rados.0.smithi029.stdout:3680: read oid 9 snap 428 2024-04-12T11:30:00.728 INFO:tasks.rados.rados.0.smithi029.stdout:3680: expect (ObjNum 1204 snap 406 seq_num 1204) 2024-04-12T11:30:00.728 INFO:tasks.rados.rados.0.smithi029.stdout:3681: rollback oid 3 current snap is 467 2024-04-12T11:30:00.728 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 438 2024-04-12T11:30:00.728 INFO:tasks.rados.rados.0.smithi029.stdout:3678: finishing rollback tid 1 to smithi02979277-30 2024-04-12T11:30:00.728 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 1105 (ObjNum 417 snap 0 seq_num 1801305312) dirty dne 2024-04-12T11:30:00.728 INFO:tasks.rados.rados.0.smithi029.stdout:3678: done (2 left) 2024-04-12T11:30:00.729 INFO:tasks.rados.rados.0.smithi029.stdout:3682: rollback oid 33 current snap is 467 2024-04-12T11:30:00.729 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 424 2024-04-12T11:30:00.729 INFO:tasks.rados.rados.0.smithi029.stdout:3683: snap_remove snap 433 2024-04-12T11:30:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:00 smithi029 ceph-mon[97787]: pgmap v1128: 89 pgs: 89 active+clean; 263 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.8 MiB/s wr, 21 op/s 2024-04-12T11:30:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:00 smithi029 ceph-mon[97787]: osdmap e995: 8 total, 8 up, 8 in 2024-04-12T11:30:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:00 smithi029 ceph-mon[97787]: overall HEALTH_OK 2024-04-12T11:30:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:00 smithi029 ceph-mon[93650]: pgmap v1128: 89 pgs: 89 active+clean; 263 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.8 MiB/s wr, 21 op/s 2024-04-12T11:30:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:00 smithi029 ceph-mon[93650]: osdmap e995: 8 total, 8 up, 8 in 2024-04-12T11:30:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:00 smithi029 ceph-mon[93650]: overall HEALTH_OK 2024-04-12T11:30:01.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:00 smithi087 ceph-mon[89014]: pgmap v1128: 89 pgs: 89 active+clean; 263 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 2.8 MiB/s wr, 21 op/s 2024-04-12T11:30:01.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:00 smithi087 ceph-mon[89014]: osdmap e995: 8 total, 8 up, 8 in 2024-04-12T11:30:01.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:00 smithi087 ceph-mon[89014]: overall HEALTH_OK 2024-04-12T11:30:01.734 INFO:tasks.rados.rados.0.smithi029.stdout:3683: done (3 left) 2024-04-12T11:30:01.735 INFO:tasks.rados.rados.0.smithi029.stdout:3684: rollback oid 48 current snap is 467 2024-04-12T11:30:01.735 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 48 to 450 2024-04-12T11:30:01.735 INFO:tasks.rados.rados.0.smithi029.stdout:3680: expect (ObjNum 1204 snap 406 seq_num 1204) 2024-04-12T11:30:01.813 INFO:tasks.rados.rados.0.smithi029.stdout:3681: finishing rollback tid 0 to smithi02979277-3 2024-04-12T11:30:01.813 INFO:tasks.rados.rados.0.smithi029.stdout:3681: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:30:01.813 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 2327 (ObjNum 129 snap 0 seq_num 1801180816) dirty dne 2024-04-12T11:30:01.813 INFO:tasks.rados.rados.0.smithi029.stdout:3682: finishing rollback tid 0 to smithi02979277-33 2024-04-12T11:30:01.813 INFO:tasks.rados.rados.0.smithi029.stdout:3682: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:30:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 1572 (ObjNum 209 snap 0 seq_num 1777352200) dirty dne 2024-04-12T11:30:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3680: done (3 left) 2024-04-12T11:30:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3681: done (2 left) 2024-04-12T11:30:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3682: done (1 left) 2024-04-12T11:30:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3685: read oid 14 snap 450 2024-04-12T11:30:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3685: expect (ObjNum 1256 snap 427 seq_num 1256) 2024-04-12T11:30:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3686: write oid 35 current snap is 467 2024-04-12T11:30:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3684: finishing rollback tid 1 to smithi02979277-48 2024-04-12T11:30:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 48 v 1444 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-04-12T11:30:01.814 INFO:tasks.rados.rados.0.smithi029.stdout:3686: seq_num 1382 ranges {427274=654011,1754703=586805,2918576=308698} 2024-04-12T11:30:01.818 INFO:tasks.rados.rados.0.smithi029.stdout:3686: writing smithi02979277-35 from 427274 to 1081285 tid 1 2024-04-12T11:30:01.826 INFO:tasks.rados.rados.0.smithi029.stdout:3686: writing smithi02979277-35 from 1754703 to 2341508 tid 2 2024-04-12T11:30:01.831 INFO:tasks.rados.rados.0.smithi029.stdout:3686: writing smithi02979277-35 from 2918576 to 3227274 tid 3 2024-04-12T11:30:01.831 INFO:tasks.rados.rados.0.smithi029.stdout:3684: done (2 left) 2024-04-12T11:30:01.831 INFO:tasks.rados.rados.0.smithi029.stdout:3687: write oid 41 current snap is 467 2024-04-12T11:30:01.831 INFO:tasks.rados.rados.0.smithi029.stdout:3687: seq_num 1383 ranges {444081=729260,1813185=446816,2945999=298082} 2024-04-12T11:30:01.837 INFO:tasks.rados.rados.0.smithi029.stdout:3687: writing smithi02979277-41 from 444081 to 1173341 tid 1 2024-04-12T11:30:01.844 INFO:tasks.rados.rados.0.smithi029.stdout:3687: writing smithi02979277-41 from 1813185 to 2260001 tid 2 2024-04-12T11:30:01.849 INFO:tasks.rados.rados.0.smithi029.stdout:3687: writing smithi02979277-41 from 2945999 to 3244081 tid 3 2024-04-12T11:30:01.849 INFO:tasks.rados.rados.0.smithi029.stdout:3688: write oid 49 current snap is 467 2024-04-12T11:30:01.850 INFO:tasks.rados.rados.0.smithi029.stdout:3688: seq_num 1384 ranges {460888=520862,1471667=590474,2573422=455407} 2024-04-12T11:30:01.855 INFO:tasks.rados.rados.0.smithi029.stdout:3688: writing smithi02979277-49 from 460888 to 981750 tid 1 2024-04-12T11:30:01.861 INFO:tasks.rados.rados.0.smithi029.stdout:3688: writing smithi02979277-49 from 1471667 to 2062141 tid 2 2024-04-12T11:30:01.867 INFO:tasks.rados.rados.0.smithi029.stdout:3688: writing smithi02979277-49 from 2573422 to 3028829 tid 3 2024-04-12T11:30:01.867 INFO:tasks.rados.rados.0.smithi029.stdout:3689: read oid 19 snap 462 2024-04-12T11:30:01.867 INFO:tasks.rados.rados.0.smithi029.stdout:3689: expect (ObjNum 1324 snap 451 seq_num 1324) 2024-04-12T11:30:01.867 INFO:tasks.rados.rados.0.smithi029.stdout:3690: snap_remove snap 428 2024-04-12T11:30:01.867 INFO:tasks.rados.rados.0.smithi029.stdout:3685: expect (ObjNum 1256 snap 427 seq_num 1256) 2024-04-12T11:30:01.975 INFO:tasks.rados.rados.0.smithi029.stdout:3686: finishing write tid 1 to smithi02979277-35 2024-04-12T11:30:01.975 INFO:tasks.rados.rados.0.smithi029.stdout:3686: finishing write tid 2 to smithi02979277-35 2024-04-12T11:30:01.975 INFO:tasks.rados.rados.0.smithi029.stdout:3686: finishing write tid 3 to smithi02979277-35 2024-04-12T11:30:01.976 INFO:tasks.rados.rados.0.smithi029.stdout:3686: finishing write tid 4 to smithi02979277-35 2024-04-12T11:30:01.976 INFO:tasks.rados.rados.0.smithi029.stdout:3686: finishing write tid 5 to smithi02979277-35 2024-04-12T11:30:01.976 INFO:tasks.rados.rados.0.smithi029.stdout:3686: oid 35 updating version 0 to 2339 2024-04-12T11:30:01.976 INFO:tasks.rados.rados.0.smithi029.stdout:3686: oid 35 version 2339 is already newer than 2337 2024-04-12T11:30:01.976 INFO:tasks.rados.rados.0.smithi029.stdout:3686: oid 35 version 2339 is already newer than 2338 2024-04-12T11:30:01.976 INFO:tasks.rados.rados.0.smithi029.stdout:3686: oid 35 version 2339 is already newer than 2336 2024-04-12T11:30:01.976 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 2339 (ObjNum 1382 snap 467 seq_num 1382) dirty exists 2024-04-12T11:30:01.976 INFO:tasks.rados.rados.0.smithi029.stdout:3686: left oid 35 (ObjNum 1382 snap 467 seq_num 1382) 2024-04-12T11:30:01.976 INFO:tasks.rados.rados.0.smithi029.stdout:3687: finishing write tid 1 to smithi02979277-41 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:3687: finishing write tid 2 to smithi02979277-41 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:3687: finishing write tid 3 to smithi02979277-41 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:3687: finishing write tid 4 to smithi02979277-41 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:3687: finishing write tid 5 to smithi02979277-41 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:3687: oid 41 updating version 0 to 1788 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:3687: oid 41 updating version 1788 to 1789 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:3687: oid 41 version 1789 is already newer than 1787 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:3687: oid 41 updating version 1789 to 1790 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1790 (ObjNum 1383 snap 467 seq_num 1383) dirty exists 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:3687: left oid 41 (ObjNum 1383 snap 467 seq_num 1383) 2024-04-12T11:30:01.977 INFO:tasks.rados.rados.0.smithi029.stdout:3688: finishing write tid 1 to smithi02979277-49 2024-04-12T11:30:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:01 smithi029 ceph-mon[93650]: osdmap e996: 8 total, 8 up, 8 in 2024-04-12T11:30:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:01 smithi029 ceph-mon[97787]: osdmap e996: 8 total, 8 up, 8 in 2024-04-12T11:30:02.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:01 smithi087 ceph-mon[89014]: osdmap e996: 8 total, 8 up, 8 in 2024-04-12T11:30:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:30:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:30:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:30:02.719 INFO:tasks.rados.rados.0.smithi029.stdout:3685: done (5 left) 2024-04-12T11:30:02.719 INFO:tasks.rados.rados.0.smithi029.stdout:3686: done (4 left) 2024-04-12T11:30:02.719 INFO:tasks.rados.rados.0.smithi029.stdout:3687: done (3 left) 2024-04-12T11:30:02.719 INFO:tasks.rados.rados.0.smithi029.stdout:3690: done (2 left) 2024-04-12T11:30:02.719 INFO:tasks.rados.rados.0.smithi029.stdout:3691: write oid 2 current snap is 467 2024-04-12T11:30:02.719 INFO:tasks.rados.rados.0.smithi029.stdout:3691: seq_num 1385 ranges {477695=596111,1530149=450485,2600845=666679} 2024-04-12T11:30:02.725 INFO:tasks.rados.rados.0.smithi029.stdout:3691: writing smithi02979277-2 from 477695 to 1073806 tid 1 2024-04-12T11:30:02.731 INFO:tasks.rados.rados.0.smithi029.stdout:3691: writing smithi02979277-2 from 1530149 to 1980634 tid 2 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3691: writing smithi02979277-2 from 2600845 to 3267524 tid 3 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3692: delete oid 14 current snap is 467 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3688: finishing write tid 2 to smithi02979277-49 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3688: finishing write tid 3 to smithi02979277-49 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3688: finishing write tid 4 to smithi02979277-49 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3688: finishing write tid 5 to smithi02979277-49 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3688: oid 49 updating version 0 to 1294 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3688: oid 49 version 1294 is already newer than 1292 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3688: oid 49 updating version 1294 to 1295 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3688: oid 49 version 1295 is already newer than 1293 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 49 v 1295 (ObjNum 1384 snap 467 seq_num 1384) dirty exists 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3688: left oid 49 (ObjNum 1384 snap 467 seq_num 1384) 2024-04-12T11:30:02.740 INFO:tasks.rados.rados.0.smithi029.stdout:3689: expect (ObjNum 1324 snap 451 seq_num 1324) 2024-04-12T11:30:02.899 INFO:tasks.rados.rados.0.smithi029.stdout:3691: finishing write tid 1 to smithi02979277-2 2024-04-12T11:30:02.899 INFO:tasks.rados.rados.0.smithi029.stdout:3691: finishing write tid 2 to smithi02979277-2 2024-04-12T11:30:02.899 INFO:tasks.rados.rados.0.smithi029.stdout:3691: finishing write tid 3 to smithi02979277-2 2024-04-12T11:30:02.899 INFO:tasks.rados.rados.0.smithi029.stdout:3691: finishing write tid 4 to smithi02979277-2 2024-04-12T11:30:02.899 INFO:tasks.rados.rados.0.smithi029.stdout:3691: finishing write tid 5 to smithi02979277-2 2024-04-12T11:30:02.900 INFO:tasks.rados.rados.0.smithi029.stdout:3691: oid 2 updating version 0 to 440 2024-04-12T11:30:02.900 INFO:tasks.rados.rados.0.smithi029.stdout:3691: oid 2 version 440 is already newer than 437 2024-04-12T11:30:02.900 INFO:tasks.rados.rados.0.smithi029.stdout:3691: oid 2 version 440 is already newer than 436 2024-04-12T11:30:02.900 INFO:tasks.rados.rados.0.smithi029.stdout:3691: oid 2 updating version 440 to 441 2024-04-12T11:30:02.900 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 441 (ObjNum 1385 snap 467 seq_num 1385) dirty exists 2024-04-12T11:30:02.900 INFO:tasks.rados.rados.0.smithi029.stdout:3691: left oid 2 (ObjNum 1385 snap 467 seq_num 1385) 2024-04-12T11:30:02.900 INFO:tasks.rados.rados.0.smithi029.stdout:3688: done (3 left) 2024-04-12T11:30:02.900 INFO:tasks.rados.rados.0.smithi029.stdout:3689: done (2 left) 2024-04-12T11:30:02.901 INFO:tasks.rados.rados.0.smithi029.stdout:3691: done (1 left) 2024-04-12T11:30:02.901 INFO:tasks.rados.rados.0.smithi029.stdout:3692: done (0 left) 2024-04-12T11:30:02.901 INFO:tasks.rados.rados.0.smithi029.stdout:3693: delete oid 41 current snap is 467 2024-04-12T11:30:02.902 INFO:tasks.rados.rados.0.smithi029.stdout:3693: done (0 left) 2024-04-12T11:30:02.902 INFO:tasks.rados.rados.0.smithi029.stdout:3694: read oid 39 snap -1 2024-04-12T11:30:02.902 INFO:tasks.rados.rados.0.smithi029.stdout:3694: expect (ObjNum 1088 snap 365 seq_num 1088) 2024-04-12T11:30:02.902 INFO:tasks.rados.rados.0.smithi029.stdout:3695: write oid 45 current snap is 467 2024-04-12T11:30:02.902 INFO:tasks.rados.rados.0.smithi029.stdout:3695: seq_num 1388 ranges {528116=421858,1589242=597812,2966761=361355} 2024-04-12T11:30:02.907 INFO:tasks.rados.rados.0.smithi029.stdout:3695: writing smithi02979277-45 from 528116 to 949974 tid 1 2024-04-12T11:30:02.914 INFO:tasks.rados.rados.0.smithi029.stdout:3695: writing smithi02979277-45 from 1589242 to 2187054 tid 2 2024-04-12T11:30:02.920 INFO:tasks.rados.rados.0.smithi029.stdout:3695: writing smithi02979277-45 from 2966761 to 3328116 tid 3 2024-04-12T11:30:02.920 INFO:tasks.rados.rados.0.smithi029.stdout:3696: delete oid 43 current snap is 467 2024-04-12T11:30:02.920 INFO:tasks.rados.rados.0.smithi029.stdout:3694: expect (ObjNum 1088 snap 365 seq_num 1088) 2024-04-12T11:30:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:02 smithi029 ceph-mon[93650]: pgmap v1131: 89 pgs: 89 active+clean; 263 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:30:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:02 smithi029 ceph-mon[93650]: osdmap e997: 8 total, 8 up, 8 in 2024-04-12T11:30:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:02 smithi029 ceph-mon[93650]: osdmap e998: 8 total, 8 up, 8 in 2024-04-12T11:30:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:02 smithi029 ceph-mon[97787]: pgmap v1131: 89 pgs: 89 active+clean; 263 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:30:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:02 smithi029 ceph-mon[97787]: osdmap e997: 8 total, 8 up, 8 in 2024-04-12T11:30:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:02 smithi029 ceph-mon[97787]: osdmap e998: 8 total, 8 up, 8 in 2024-04-12T11:30:03.044 INFO:tasks.rados.rados.0.smithi029.stdout:3695: finishing write tid 1 to smithi02979277-45 2024-04-12T11:30:03.044 INFO:tasks.rados.rados.0.smithi029.stdout:3695: finishing write tid 2 to smithi02979277-45 2024-04-12T11:30:03.044 INFO:tasks.rados.rados.0.smithi029.stdout:3695: finishing write tid 3 to smithi02979277-45 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3695: finishing write tid 4 to smithi02979277-45 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3695: finishing write tid 5 to smithi02979277-45 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3695: oid 45 updating version 0 to 1850 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3695: oid 45 updating version 1850 to 1852 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3695: oid 45 version 1852 is already newer than 1849 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3695: oid 45 version 1852 is already newer than 1851 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1852 (ObjNum 1388 snap 467 seq_num 1388) dirty exists 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3695: left oid 45 (ObjNum 1388 snap 467 seq_num 1388) 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3694: done (2 left) 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3695: done (1 left) 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3696: done (0 left) 2024-04-12T11:30:03.045 INFO:tasks.rados.rados.0.smithi029.stdout:3697: delete oid 15 current snap is 467 2024-04-12T11:30:03.046 INFO:tasks.rados.rados.0.smithi029.stdout:3697: done (0 left) 2024-04-12T11:30:03.046 INFO:tasks.rados.rados.0.smithi029.stdout:3698: delete oid 40 current snap is 467 2024-04-12T11:30:03.049 INFO:tasks.rados.rados.0.smithi029.stdout:3698: done (0 left) 2024-04-12T11:30:03.049 INFO:tasks.rados.rados.0.smithi029.stdout:3699: snap_remove snap 454 2024-04-12T11:30:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:02 smithi087 ceph-mon[89014]: pgmap v1131: 89 pgs: 89 active+clean; 263 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-04-12T11:30:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:02 smithi087 ceph-mon[89014]: osdmap e997: 8 total, 8 up, 8 in 2024-04-12T11:30:03.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:02 smithi087 ceph-mon[89014]: osdmap e998: 8 total, 8 up, 8 in 2024-04-12T11:30:03.741 INFO:tasks.rados.rados.0.smithi029.stdout:3699: done (0 left) 2024-04-12T11:30:03.742 INFO:tasks.rados.rados.0.smithi029.stdout:3700: snap_remove snap 464 2024-04-12T11:30:04.747 INFO:tasks.rados.rados.0.smithi029.stdout:3700: done (0 left) 2024-04-12T11:30:04.748 INFO:tasks.rados.rados.0.smithi029.stdout:3701: write oid 31 current snap is 467 2024-04-12T11:30:04.748 INFO:tasks.rados.rados.0.smithi029.stdout:3701: seq_num 1392 ranges {595344=439207,1823170=605150,3076453=318891} 2024-04-12T11:30:04.753 INFO:tasks.rados.rados.0.smithi029.stdout:3701: writing smithi02979277-31 from 595344 to 1034551 tid 1 2024-04-12T11:30:04.763 INFO:tasks.rados.rados.0.smithi029.stdout:3701: writing smithi02979277-31 from 1823170 to 2428320 tid 2 2024-04-12T11:30:04.769 INFO:tasks.rados.rados.0.smithi029.stdout:3701: writing smithi02979277-31 from 3076453 to 3395344 tid 3 2024-04-12T11:30:04.769 INFO:tasks.rados.rados.0.smithi029.stdout:3702: snap_create 2024-04-12T11:30:04.769 INFO:tasks.rados.rados.0.smithi029.stdout:3701: finishing write tid 1 to smithi02979277-31 2024-04-12T11:30:04.769 INFO:tasks.rados.rados.0.smithi029.stdout:3701: finishing write tid 2 to smithi02979277-31 2024-04-12T11:30:04.772 INFO:tasks.rados.rados.0.smithi029.stdout:3701: finishing write tid 3 to smithi02979277-31 2024-04-12T11:30:04.772 INFO:tasks.rados.rados.0.smithi029.stdout:3701: finishing write tid 4 to smithi02979277-31 2024-04-12T11:30:04.772 INFO:tasks.rados.rados.0.smithi029.stdout:3701: finishing write tid 5 to smithi02979277-31 2024-04-12T11:30:04.772 INFO:tasks.rados.rados.0.smithi029.stdout:3701: oid 31 updating version 0 to 1470 2024-04-12T11:30:04.772 INFO:tasks.rados.rados.0.smithi029.stdout:3701: oid 31 version 1470 is already newer than 1469 2024-04-12T11:30:04.772 INFO:tasks.rados.rados.0.smithi029.stdout:3701: oid 31 version 1470 is already newer than 1467 2024-04-12T11:30:04.772 INFO:tasks.rados.rados.0.smithi029.stdout:3701: oid 31 version 1470 is already newer than 1468 2024-04-12T11:30:04.772 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1470 (ObjNum 1392 snap 467 seq_num 1392) dirty exists 2024-04-12T11:30:04.772 INFO:tasks.rados.rados.0.smithi029.stdout:3701: left oid 31 (ObjNum 1392 snap 467 seq_num 1392) 2024-04-12T11:30:05.000 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:30:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:30:04] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-12T11:30:05.000 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:04 smithi029 ceph-mon[93650]: pgmap v1134: 89 pgs: 89 active+clean; 261 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 1009 KiB/s wr, 11 op/s 2024-04-12T11:30:05.000 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:04 smithi029 ceph-mon[93650]: osdmap e999: 8 total, 8 up, 8 in 2024-04-12T11:30:05.001 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:04 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:30:05.001 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:04 smithi029 ceph-mon[97787]: pgmap v1134: 89 pgs: 89 active+clean; 261 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 1009 KiB/s wr, 11 op/s 2024-04-12T11:30:05.001 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:04 smithi029 ceph-mon[97787]: osdmap e999: 8 total, 8 up, 8 in 2024-04-12T11:30:05.001 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:04 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:30:05.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:04 smithi087 ceph-mon[89014]: pgmap v1134: 89 pgs: 89 active+clean; 261 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 1009 KiB/s wr, 11 op/s 2024-04-12T11:30:05.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:04 smithi087 ceph-mon[89014]: osdmap e999: 8 total, 8 up, 8 in 2024-04-12T11:30:05.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:04 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:30:05.752 INFO:tasks.rados.rados.0.smithi029.stdout:3701: done (1 left) 2024-04-12T11:30:05.753 INFO:tasks.rados.rados.0.smithi029.stdout:3702: done (0 left) 2024-04-12T11:30:05.753 INFO:tasks.rados.rados.0.smithi029.stdout:3703: rollback oid 14 current snap is 468 2024-04-12T11:30:05.753 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 450 2024-04-12T11:30:05.753 INFO:tasks.rados.rados.0.smithi029.stdout:3704: write oid 3 current snap is 468 2024-04-12T11:30:05.753 INFO:tasks.rados.rados.0.smithi029.stdout:3704: seq_num 1393 ranges {612151=514456,1881652=748808,3103876=308275} 2024-04-12T11:30:05.759 INFO:tasks.rados.rados.0.smithi029.stdout:3704: writing smithi02979277-3 from 612151 to 1126607 tid 1 2024-04-12T11:30:05.767 INFO:tasks.rados.rados.0.smithi029.stdout:3704: writing smithi02979277-3 from 1881652 to 2630460 tid 2 2024-04-12T11:30:05.772 INFO:tasks.rados.rados.0.smithi029.stdout:3704: writing smithi02979277-3 from 3103876 to 3412151 tid 3 2024-04-12T11:30:05.772 INFO:tasks.rados.rados.0.smithi029.stdout:3705: delete oid 10 current snap is 468 2024-04-12T11:30:05.772 INFO:tasks.rados.rados.0.smithi029.stdout:3703: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:30:05.772 INFO:tasks.rados.rados.0.smithi029.stdout:3703: finishing rollback tid 2 to smithi02979277-14 2024-04-12T11:30:05.773 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 1820 (ObjNum 1256 snap 427 seq_num 1256) dirty exists 2024-04-12T11:30:05.773 INFO:tasks.rados.rados.0.smithi029.stdout:3704: finishing write tid 1 to smithi02979277-3 2024-04-12T11:30:05.773 INFO:tasks.rados.rados.0.smithi029.stdout:3704: finishing write tid 2 to smithi02979277-3 2024-04-12T11:30:05.776 INFO:tasks.rados.rados.0.smithi029.stdout:3704: finishing write tid 3 to smithi02979277-3 2024-04-12T11:30:05.776 INFO:tasks.rados.rados.0.smithi029.stdout:3703: done (2 left) 2024-04-12T11:30:05.776 INFO:tasks.rados.rados.0.smithi029.stdout:3705: done (1 left) 2024-04-12T11:30:05.777 INFO:tasks.rados.rados.0.smithi029.stdout:3706: snap_create 2024-04-12T11:30:05.777 INFO:tasks.rados.rados.0.smithi029.stdout:3704: finishing write tid 4 to smithi02979277-3 2024-04-12T11:30:05.777 INFO:tasks.rados.rados.0.smithi029.stdout:3704: finishing write tid 5 to smithi02979277-3 2024-04-12T11:30:05.777 INFO:tasks.rados.rados.0.smithi029.stdout:3704: oid 3 updating version 0 to 2360 2024-04-12T11:30:05.777 INFO:tasks.rados.rados.0.smithi029.stdout:3704: oid 3 updating version 2360 to 2361 2024-04-12T11:30:05.777 INFO:tasks.rados.rados.0.smithi029.stdout:3704: oid 3 version 2361 is already newer than 2358 2024-04-12T11:30:05.777 INFO:tasks.rados.rados.0.smithi029.stdout:3704: oid 3 version 2361 is already newer than 2359 2024-04-12T11:30:05.777 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 2361 (ObjNum 1393 snap 468 seq_num 1393) dirty exists 2024-04-12T11:30:05.777 INFO:tasks.rados.rados.0.smithi029.stdout:3704: left oid 3 (ObjNum 1393 snap 468 seq_num 1393) 2024-04-12T11:30:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:05 smithi029 ceph-mon[97787]: osdmap e1000: 8 total, 8 up, 8 in 2024-04-12T11:30:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:05 smithi029 ceph-mon[93650]: osdmap e1000: 8 total, 8 up, 8 in 2024-04-12T11:30:06.034 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:05 smithi087 ceph-mon[89014]: osdmap e1000: 8 total, 8 up, 8 in 2024-04-12T11:30:06.756 INFO:tasks.rados.rados.0.smithi029.stdout:3704: done (1 left) 2024-04-12T11:30:06.756 INFO:tasks.rados.rados.0.smithi029.stdout:3706: done (0 left) 2024-04-12T11:30:06.756 INFO:tasks.rados.rados.0.smithi029.stdout:3707: read oid 18 snap -1 2024-04-12T11:30:06.756 INFO:tasks.rados.rados.0.smithi029.stdout:3707: expect deleted 2024-04-12T11:30:06.756 INFO:tasks.rados.rados.0.smithi029.stdout:3708: rollback oid 32 current snap is 469 2024-04-12T11:30:06.756 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 32 to 442 2024-04-12T11:30:06.756 INFO:tasks.rados.rados.0.smithi029.stdout:3709: write oid 8 current snap is 469 2024-04-12T11:30:06.756 INFO:tasks.rados.rados.0.smithi029.stdout:3709: seq_num 1395 ranges {645765=664954,1998616=752477,3158722=287043} 2024-04-12T11:30:06.763 INFO:tasks.rados.rados.0.smithi029.stdout:3709: writing smithi02979277-8 from 645765 to 1310719 tid 1 2024-04-12T11:30:06.771 INFO:tasks.rados.rados.0.smithi029.stdout:3709: writing smithi02979277-8 from 1998616 to 2751093 tid 2 2024-04-12T11:30:06.775 INFO:tasks.rados.rados.0.smithi029.stdout:3709: writing smithi02979277-8 from 3158722 to 3445765 tid 3 2024-04-12T11:30:06.775 INFO:tasks.rados.rados.0.smithi029.stdout:3710: read oid 16 snap -1 2024-04-12T11:30:06.775 INFO:tasks.rados.rados.0.smithi029.stdout:3710: expect (ObjNum 1380 snap 466 seq_num 1380) 2024-04-12T11:30:06.775 INFO:tasks.rados.rados.0.smithi029.stdout:3711: delete oid 24 current snap is 469 2024-04-12T11:30:06.775 INFO:tasks.rados.rados.0.smithi029.stdout:3708: finishing rollback tid 1 to smithi02979277-32 2024-04-12T11:30:06.776 INFO:tasks.rados.rados.0.smithi029.stdout:3708: finishing rollback tid 2 to smithi02979277-32 2024-04-12T11:30:06.776 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 32 v 1685 (ObjNum 1218 snap 409 seq_num 1218) dirty exists 2024-04-12T11:30:06.776 INFO:tasks.rados.rados.0.smithi029.stdout:3709: finishing write tid 1 to smithi02979277-8 2024-04-12T11:30:06.776 INFO:tasks.rados.rados.0.smithi029.stdout:3709: finishing write tid 2 to smithi02979277-8 2024-04-12T11:30:06.780 INFO:tasks.rados.rados.0.smithi029.stdout:3707: done (4 left) 2024-04-12T11:30:06.780 INFO:tasks.rados.rados.0.smithi029.stdout:3708: done (3 left) 2024-04-12T11:30:06.780 INFO:tasks.rados.rados.0.smithi029.stdout:3711: done (2 left) 2024-04-12T11:30:06.780 INFO:tasks.rados.rados.0.smithi029.stdout:3712: rollback oid 13 current snap is 469 2024-04-12T11:30:06.780 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 13 to 456 2024-04-12T11:30:06.780 INFO:tasks.rados.rados.0.smithi029.stdout:3713: rollback oid 1 current snap is 469 2024-04-12T11:30:06.780 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 1 to 462 2024-04-12T11:30:06.780 INFO:tasks.rados.rados.0.smithi029.stdout:3714: write oid 45 current snap is 469 2024-04-12T11:30:06.780 INFO:tasks.rados.rados.0.smithi029.stdout:3714: seq_num 1397 ranges {679379=415452,1599227=756146,3097215=382164} 2024-04-12T11:30:06.786 INFO:tasks.rados.rados.0.smithi029.stdout:3714: writing smithi02979277-45 from 679379 to 1094831 tid 1 2024-04-12T11:30:06.793 INFO:tasks.rados.rados.0.smithi029.stdout:3714: writing smithi02979277-45 from 1599227 to 2355373 tid 2 2024-04-12T11:30:06.800 INFO:tasks.rados.rados.0.smithi029.stdout:3714: writing smithi02979277-45 from 3097215 to 3479379 tid 3 2024-04-12T11:30:06.800 INFO:tasks.rados.rados.0.smithi029.stdout:3715: read oid 46 snap -1 2024-04-12T11:30:06.800 INFO:tasks.rados.rados.0.smithi029.stdout:3715: expect (ObjNum 1326 snap 451 seq_num 1326) 2024-04-12T11:30:06.800 INFO:tasks.rados.rados.0.smithi029.stdout:3716: write oid 11 current snap is 469 2024-04-12T11:30:06.800 INFO:tasks.rados.rados.0.smithi029.stdout:3716: seq_num 1398 ranges {696186=490701,1657709=616157,2724638=562274} 2024-04-12T11:30:06.806 INFO:tasks.rados.rados.0.smithi029.stdout:3716: writing smithi02979277-11 from 696186 to 1186887 tid 1 2024-04-12T11:30:06.812 INFO:tasks.rados.rados.0.smithi029.stdout:3716: writing smithi02979277-11 from 1657709 to 2273866 tid 2 2024-04-12T11:30:06.818 INFO:tasks.rados.rados.0.smithi029.stdout:3716: writing smithi02979277-11 from 2724638 to 3286912 tid 3 2024-04-12T11:30:06.818 INFO:tasks.rados.rados.0.smithi029.stdout:3717: delete oid 35 current snap is 469 2024-04-12T11:30:06.818 INFO:tasks.rados.rados.0.smithi029.stdout:3712: finishing rollback tid 0 to smithi02979277-13 2024-04-12T11:30:06.819 INFO:tasks.rados.rados.0.smithi029.stdout:3712: finishing rollback tid 1 to smithi02979277-13 2024-04-12T11:30:06.819 INFO:tasks.rados.rados.0.smithi029.stdout:3713: finishing rollback tid 0 to smithi02979277-1 2024-04-12T11:30:06.819 INFO:tasks.rados.rados.0.smithi029.stdout:3713: finishing rollback tid 1 to smithi02979277-1 2024-04-12T11:30:06.819 INFO:tasks.rados.rados.0.smithi029.stdout:3712: finishing rollback tid 2 to smithi02979277-13 2024-04-12T11:30:06.819 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 1971 (ObjNum 1344 snap 456 seq_num 1344) dirty exists 2024-04-12T11:30:06.819 INFO:tasks.rados.rados.0.smithi029.stdout:3713: finishing rollback tid 2 to smithi02979277-1 2024-04-12T11:30:06.819 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1813 (ObjNum 1343 snap 456 seq_num 1343) dirty exists 2024-04-12T11:30:06.819 INFO:tasks.rados.rados.0.smithi029.stdout:3709: finishing write tid 3 to smithi02979277-8 2024-04-12T11:30:06.819 INFO:tasks.rados.rados.0.smithi029.stdout:3709: finishing write tid 4 to smithi02979277-8 2024-04-12T11:30:06.819 INFO:tasks.rados.rados.0.smithi029.stdout:3710: expect (ObjNum 1380 snap 466 seq_num 1380) 2024-04-12T11:30:06.955 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:06 smithi087 ceph-mon[89014]: pgmap v1137: 89 pgs: 89 active+clean; 260 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-04-12T11:30:06.955 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:06 smithi087 ceph-mon[89014]: osdmap e1001: 8 total, 8 up, 8 in 2024-04-12T11:30:06.955 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:06 smithi029 ceph-mon[93650]: pgmap v1137: 89 pgs: 89 active+clean; 260 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-04-12T11:30:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:06 smithi029 ceph-mon[93650]: osdmap e1001: 8 total, 8 up, 8 in 2024-04-12T11:30:07.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:06 smithi029 ceph-mon[97787]: pgmap v1137: 89 pgs: 89 active+clean; 260 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-04-12T11:30:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:06 smithi029 ceph-mon[97787]: osdmap e1001: 8 total, 8 up, 8 in 2024-04-12T11:30:07.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:07.112 INFO:tasks.rados.rados.0.smithi029.stdout:3709: finishing write tid 5 to smithi02979277-8 2024-04-12T11:30:07.112 INFO:tasks.rados.rados.0.smithi029.stdout:3709: oid 8 updating version 0 to 386 2024-04-12T11:30:07.112 INFO:tasks.rados.rados.0.smithi029.stdout:3709: oid 8 version 386 is already newer than 384 2024-04-12T11:30:07.112 INFO:tasks.rados.rados.0.smithi029.stdout:3709: oid 8 version 386 is already newer than 383 2024-04-12T11:30:07.112 INFO:tasks.rados.rados.0.smithi029.stdout:3709: oid 8 version 386 is already newer than 385 2024-04-12T11:30:07.112 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 386 (ObjNum 1395 snap 469 seq_num 1395) dirty exists 2024-04-12T11:30:07.112 INFO:tasks.rados.rados.0.smithi029.stdout:3709: left oid 8 (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:30:07.112 INFO:tasks.rados.rados.0.smithi029.stdout:3714: finishing write tid 1 to smithi02979277-45 2024-04-12T11:30:07.113 INFO:tasks.rados.rados.0.smithi029.stdout:3714: finishing write tid 2 to smithi02979277-45 2024-04-12T11:30:07.113 INFO:tasks.rados.rados.0.smithi029.stdout:3709: done (7 left) 2024-04-12T11:30:07.113 INFO:tasks.rados.rados.0.smithi029.stdout:3710: done (6 left) 2024-04-12T11:30:07.113 INFO:tasks.rados.rados.0.smithi029.stdout:3712: done (5 left) 2024-04-12T11:30:07.113 INFO:tasks.rados.rados.0.smithi029.stdout:3713: done (4 left) 2024-04-12T11:30:07.113 INFO:tasks.rados.rados.0.smithi029.stdout:3717: done (3 left) 2024-04-12T11:30:07.113 INFO:tasks.rados.rados.0.smithi029.stdout:3718: rollback oid 31 current snap is 469 2024-04-12T11:30:07.113 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 31 to 463 2024-04-12T11:30:07.113 INFO:tasks.rados.rados.0.smithi029.stdout:3715: expect (ObjNum 1326 snap 451 seq_num 1326) 2024-04-12T11:30:07.222 INFO:tasks.rados.rados.0.smithi029.stdout:3714: finishing write tid 3 to smithi02979277-45 2024-04-12T11:30:07.222 INFO:tasks.rados.rados.0.smithi029.stdout:3714: finishing write tid 4 to smithi02979277-45 2024-04-12T11:30:07.222 INFO:tasks.rados.rados.0.smithi029.stdout:3714: finishing write tid 5 to smithi02979277-45 2024-04-12T11:30:07.222 INFO:tasks.rados.rados.0.smithi029.stdout:3714: oid 45 updating version 0 to 1865 2024-04-12T11:30:07.222 INFO:tasks.rados.rados.0.smithi029.stdout:3714: oid 45 version 1865 is already newer than 1863 2024-04-12T11:30:07.222 INFO:tasks.rados.rados.0.smithi029.stdout:3714: oid 45 version 1865 is already newer than 1864 2024-04-12T11:30:07.222 INFO:tasks.rados.rados.0.smithi029.stdout:3714: oid 45 version 1865 is already newer than 1862 2024-04-12T11:30:07.222 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1865 (ObjNum 1397 snap 469 seq_num 1397) dirty exists 2024-04-12T11:30:07.222 INFO:tasks.rados.rados.0.smithi029.stdout:3714: left oid 45 (ObjNum 1397 snap 469 seq_num 1397) 2024-04-12T11:30:07.222 INFO:tasks.rados.rados.0.smithi029.stdout:3716: finishing write tid 1 to smithi02979277-11 2024-04-12T11:30:07.223 INFO:tasks.rados.rados.0.smithi029.stdout:3714: done (3 left) 2024-04-12T11:30:07.223 INFO:tasks.rados.rados.0.smithi029.stdout:3715: done (2 left) 2024-04-12T11:30:07.223 INFO:tasks.rados.rados.0.smithi029.stdout:3719: snap_remove snap 424 2024-04-12T11:30:07.721 INFO:tasks.rados.rados.0.smithi029.stdout:3719: done (2 left) 2024-04-12T11:30:07.721 INFO:tasks.rados.rados.0.smithi029.stdout:3720: read oid 46 snap -1 2024-04-12T11:30:07.721 INFO:tasks.rados.rados.0.smithi029.stdout:3720: expect (ObjNum 1326 snap 451 seq_num 1326) 2024-04-12T11:30:07.721 INFO:tasks.rados.rados.0.smithi029.stdout:3721: snap_remove snap 438 2024-04-12T11:30:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:07 smithi029 ceph-mon[97787]: osdmap e1002: 8 total, 8 up, 8 in 2024-04-12T11:30:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:07 smithi029 ceph-mon[97787]: osdmap e1003: 8 total, 8 up, 8 in 2024-04-12T11:30:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:07 smithi029 ceph-mon[93650]: osdmap e1002: 8 total, 8 up, 8 in 2024-04-12T11:30:08.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:07 smithi029 ceph-mon[93650]: osdmap e1003: 8 total, 8 up, 8 in 2024-04-12T11:30:08.094 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:07 smithi087 ceph-mon[89014]: osdmap e1002: 8 total, 8 up, 8 in 2024-04-12T11:30:08.094 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:07 smithi087 ceph-mon[89014]: osdmap e1003: 8 total, 8 up, 8 in 2024-04-12T11:30:08.720 INFO:tasks.rados.rados.0.smithi029.stdout:3721: done (3 left) 2024-04-12T11:30:08.720 INFO:tasks.rados.rados.0.smithi029.stdout:3722: delete oid 21 current snap is 469 2024-04-12T11:30:08.720 INFO:tasks.rados.rados.0.smithi029.stdout:3716: finishing write tid 2 to smithi02979277-11 2024-04-12T11:30:08.720 INFO:tasks.rados.rados.0.smithi029.stdout:3716: finishing write tid 3 to smithi02979277-11 2024-04-12T11:30:08.720 INFO:tasks.rados.rados.0.smithi029.stdout:3716: finishing write tid 4 to smithi02979277-11 2024-04-12T11:30:08.720 INFO:tasks.rados.rados.0.smithi029.stdout:3716: finishing write tid 5 to smithi02979277-11 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:3716: oid 11 updating version 0 to 2365 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:3716: oid 11 updating version 2365 to 2367 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:3716: oid 11 version 2367 is already newer than 2364 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:3716: oid 11 version 2367 is already newer than 2366 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 2367 (ObjNum 1398 snap 469 seq_num 1398) dirty exists 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:3716: left oid 11 (ObjNum 1398 snap 469 seq_num 1398) 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:3718: finishing rollback tid 0 to smithi02979277-31 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:3718: finishing rollback tid 1 to smithi02979277-31 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:3718: finishing rollback tid 2 to smithi02979277-31 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1475 (ObjNum 1341 snap 453 seq_num 1341) dirty exists 2024-04-12T11:30:08.721 INFO:tasks.rados.rados.0.smithi029.stdout:3720: expect (ObjNum 1326 snap 451 seq_num 1326) 2024-04-12T11:30:08.835 INFO:tasks.rados.rados.0.smithi029.stdout:3716: done (3 left) 2024-04-12T11:30:08.835 INFO:tasks.rados.rados.0.smithi029.stdout:3718: done (2 left) 2024-04-12T11:30:08.835 INFO:tasks.rados.rados.0.smithi029.stdout:3720: done (1 left) 2024-04-12T11:30:08.835 INFO:tasks.rados.rados.0.smithi029.stdout:3722: done (0 left) 2024-04-12T11:30:08.835 INFO:tasks.rados.rados.0.smithi029.stdout:3723: snap_create 2024-04-12T11:30:09.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:09 smithi087 ceph-mon[89014]: pgmap v1140: 89 pgs: 89 active+clean; 257 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-04-12T11:30:09.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:09 smithi087 ceph-mon[89014]: osdmap e1004: 8 total, 8 up, 8 in 2024-04-12T11:30:09.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[97787]: pgmap v1140: 89 pgs: 89 active+clean; 257 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-04-12T11:30:09.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[97787]: osdmap e1004: 8 total, 8 up, 8 in 2024-04-12T11:30:09.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[93650]: pgmap v1140: 89 pgs: 89 active+clean; 257 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-04-12T11:30:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:09 smithi029 ceph-mon[93650]: osdmap e1004: 8 total, 8 up, 8 in 2024-04-12T11:30:09.721 INFO:tasks.rados.rados.0.smithi029.stdout:3723: done (0 left) 2024-04-12T11:30:09.722 INFO:tasks.rados.rados.0.smithi029.stdout:3724: rollback oid 3 current snap is 470 2024-04-12T11:30:09.722 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 3 to 469 2024-04-12T11:30:09.722 INFO:tasks.rados.rados.0.smithi029.stdout:3725: rollback oid 16 current snap is 470 2024-04-12T11:30:09.722 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 16 to 467 2024-04-12T11:30:09.722 INFO:tasks.rados.rados.0.smithi029.stdout:3726: rollback oid 33 current snap is 470 2024-04-12T11:30:09.722 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 463 2024-04-12T11:30:09.722 INFO:tasks.rados.rados.0.smithi029.stdout:3727: write oid 27 current snap is 470 2024-04-12T11:30:09.722 INFO:tasks.rados.rados.0.smithi029.stdout:3727: seq_num 1401 ranges {746607=432801,1833155=763484,3206907=339700} 2024-04-12T11:30:09.728 INFO:tasks.rados.rados.0.smithi029.stdout:3727: writing smithi02979277-27 from 746607 to 1179408 tid 1 2024-04-12T11:30:09.736 INFO:tasks.rados.rados.0.smithi029.stdout:3727: writing smithi02979277-27 from 1833155 to 2596639 tid 2 2024-04-12T11:30:09.741 INFO:tasks.rados.rados.0.smithi029.stdout:3727: writing smithi02979277-27 from 3206907 to 3546607 tid 3 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:3728: snap_create 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:3726: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:3726: finishing rollback tid 2 to smithi02979277-33 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 1604 (ObjNum 1320 snap 451 seq_num 1320) dirty exists 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:3724: finishing rollback tid 0 to smithi02979277-3 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:3724: finishing rollback tid 1 to smithi02979277-3 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:3724: finishing rollback tid 2 to smithi02979277-3 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 2389 (ObjNum 1393 snap 468 seq_num 1393) dirty exists 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:3725: finishing rollback tid 0 to smithi02979277-16 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:3725: finishing rollback tid 1 to smithi02979277-16 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:3725: finishing rollback tid 2 to smithi02979277-16 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1838 (ObjNum 1380 snap 466 seq_num 1380) dirty exists 2024-04-12T11:30:09.742 INFO:tasks.rados.rados.0.smithi029.stdout:3727: finishing write tid 1 to smithi02979277-27 2024-04-12T11:30:09.743 INFO:tasks.rados.rados.0.smithi029.stdout:3727: finishing write tid 2 to smithi02979277-27 2024-04-12T11:30:09.744 INFO:tasks.rados.rados.0.smithi029.stdout:3727: finishing write tid 3 to smithi02979277-27 2024-04-12T11:30:09.744 INFO:tasks.rados.rados.0.smithi029.stdout:3727: finishing write tid 4 to smithi02979277-27 2024-04-12T11:30:09.744 INFO:tasks.rados.rados.0.smithi029.stdout:3727: finishing write tid 5 to smithi02979277-27 2024-04-12T11:30:09.744 INFO:tasks.rados.rados.0.smithi029.stdout:3727: oid 27 updating version 0 to 1172 2024-04-12T11:30:09.745 INFO:tasks.rados.rados.0.smithi029.stdout:3727: oid 27 updating version 1172 to 1173 2024-04-12T11:30:09.745 INFO:tasks.rados.rados.0.smithi029.stdout:3727: oid 27 version 1173 is already newer than 1171 2024-04-12T11:30:09.745 INFO:tasks.rados.rados.0.smithi029.stdout:3727: oid 27 updating version 1173 to 1174 2024-04-12T11:30:09.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 1174 (ObjNum 1401 snap 470 seq_num 1401) dirty exists 2024-04-12T11:30:09.745 INFO:tasks.rados.rados.0.smithi029.stdout:3727: left oid 27 (ObjNum 1401 snap 470 seq_num 1401) 2024-04-12T11:30:10.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:10 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:30:10.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:10 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:30:10.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:10 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:10.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:10 smithi087 ceph-mon[89014]: osdmap e1005: 8 total, 8 up, 8 in 2024-04-12T11:30:10.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:10 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:30:10.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:10 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:30:10.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:10 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:10.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:10 smithi029 ceph-mon[97787]: osdmap e1005: 8 total, 8 up, 8 in 2024-04-12T11:30:10.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:10 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:30:10.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:10 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:30:10.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:10 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:30:10.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:10 smithi029 ceph-mon[93650]: osdmap e1005: 8 total, 8 up, 8 in 2024-04-12T11:30:10.723 INFO:tasks.rados.rados.0.smithi029.stdout:3724: done (4 left) 2024-04-12T11:30:10.723 INFO:tasks.rados.rados.0.smithi029.stdout:3725: done (3 left) 2024-04-12T11:30:10.723 INFO:tasks.rados.rados.0.smithi029.stdout:3726: done (2 left) 2024-04-12T11:30:10.723 INFO:tasks.rados.rados.0.smithi029.stdout:3727: done (1 left) 2024-04-12T11:30:10.723 INFO:tasks.rados.rados.0.smithi029.stdout:3728: done (0 left) 2024-04-12T11:30:10.723 INFO:tasks.rados.rados.0.smithi029.stdout:3729: snap_create 2024-04-12T11:30:11.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:11 smithi087 ceph-mon[89014]: pgmap v1143: 89 pgs: 89 active+clean; 259 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-04-12T11:30:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:11 smithi087 ceph-mon[89014]: osdmap e1006: 8 total, 8 up, 8 in 2024-04-12T11:30:11.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:11 smithi029 ceph-mon[93650]: pgmap v1143: 89 pgs: 89 active+clean; 259 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-04-12T11:30:11.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:11 smithi029 ceph-mon[93650]: osdmap e1006: 8 total, 8 up, 8 in 2024-04-12T11:30:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:11 smithi029 ceph-mon[97787]: pgmap v1143: 89 pgs: 89 active+clean; 259 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-04-12T11:30:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:11 smithi029 ceph-mon[97787]: osdmap e1006: 8 total, 8 up, 8 in 2024-04-12T11:30:11.725 INFO:tasks.rados.rados.0.smithi029.stdout:3729: done (0 left) 2024-04-12T11:30:11.725 INFO:tasks.rados.rados.0.smithi029.stdout:3730: write oid 6 current snap is 472 2024-04-12T11:30:11.725 INFO:tasks.rados.rados.0.smithi029.stdout:3730: seq_num 1402 ranges {763414=508050,1891637=623495,3234330=329084} 2024-04-12T11:30:11.734 INFO:tasks.rados.rados.0.smithi029.stdout:3730: writing smithi02979277-6 from 763414 to 1271464 tid 1 2024-04-12T11:30:11.738 INFO:tasks.rados.rados.0.smithi029.stdout:3730: writing smithi02979277-6 from 1891637 to 2515132 tid 2 2024-04-12T11:30:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:3730: writing smithi02979277-6 from 3234330 to 3563414 tid 3 2024-04-12T11:30:11.744 INFO:tasks.rados.rados.0.smithi029.stdout:3731: read oid 23 snap -1 2024-04-12T11:30:11.745 INFO:tasks.rados.rados.0.smithi029.stdout:3731: expect (ObjNum 1353 snap 457 seq_num 1353) 2024-04-12T11:30:11.745 INFO:tasks.rados.rados.0.smithi029.stdout:3732: snap_remove snap 465 2024-04-12T11:30:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:30:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:30:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:30:12.722 INFO:tasks.rados.rados.0.smithi029.stdout:3730: finishing write tid 1 to smithi02979277-6 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:3730: finishing write tid 2 to smithi02979277-6 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:3730: finishing write tid 3 to smithi02979277-6 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:3730: finishing write tid 4 to smithi02979277-6 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:3730: finishing write tid 5 to smithi02979277-6 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:3730: oid 6 updating version 0 to 918 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:3730: oid 6 updating version 918 to 919 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:3730: oid 6 version 919 is already newer than 917 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:3730: oid 6 updating version 919 to 920 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 920 (ObjNum 1402 snap 472 seq_num 1402) dirty exists 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:3730: left oid 6 (ObjNum 1402 snap 472 seq_num 1402) 2024-04-12T11:30:12.723 INFO:tasks.rados.rados.0.smithi029.stdout:3731: expect (ObjNum 1353 snap 457 seq_num 1353) 2024-04-12T11:30:12.820 INFO:tasks.rados.rados.0.smithi029.stdout:3730: done (2 left) 2024-04-12T11:30:12.821 INFO:tasks.rados.rados.0.smithi029.stdout:3731: done (1 left) 2024-04-12T11:30:12.821 INFO:tasks.rados.rados.0.smithi029.stdout:3732: done (0 left) 2024-04-12T11:30:12.821 INFO:tasks.rados.rados.0.smithi029.stdout:3733: rollback oid 36 current snap is 472 2024-04-12T11:30:12.821 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 36 to 450 2024-04-12T11:30:12.821 INFO:tasks.rados.rados.0.smithi029.stdout:3734: snap_remove snap 467 2024-04-12T11:30:13.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:12 smithi029 ceph-mon[97787]: pgmap v1146: 89 pgs: 89 active+clean; 255 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-04-12T11:30:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:12 smithi029 ceph-mon[97787]: osdmap e1007: 8 total, 8 up, 8 in 2024-04-12T11:30:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:12 smithi029 ceph-mon[93650]: pgmap v1146: 89 pgs: 89 active+clean; 255 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-04-12T11:30:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:12 smithi029 ceph-mon[93650]: osdmap e1007: 8 total, 8 up, 8 in 2024-04-12T11:30:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:12 smithi087 ceph-mon[89014]: pgmap v1146: 89 pgs: 89 active+clean; 255 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-04-12T11:30:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:12 smithi087 ceph-mon[89014]: osdmap e1007: 8 total, 8 up, 8 in 2024-04-12T11:30:13.735 INFO:tasks.rados.rados.0.smithi029.stdout:3734: done (1 left) 2024-04-12T11:30:13.735 INFO:tasks.rados.rados.0.smithi029.stdout:3735: snap_remove snap 468 2024-04-12T11:30:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:13 smithi029 ceph-mon[93650]: osdmap e1008: 8 total, 8 up, 8 in 2024-04-12T11:30:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:13 smithi029 ceph-mon[97787]: osdmap e1008: 8 total, 8 up, 8 in 2024-04-12T11:30:14.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:13 smithi087 ceph-mon[89014]: osdmap e1008: 8 total, 8 up, 8 in 2024-04-12T11:30:14.736 INFO:tasks.rados.rados.0.smithi029.stdout:3733: finishing rollback tid 1 to smithi02979277-36 2024-04-12T11:30:14.736 INFO:tasks.rados.rados.0.smithi029.stdout:3733: finishing rollback tid 2 to smithi02979277-36 2024-04-12T11:30:14.736 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 922 (ObjNum 1177 snap 400 seq_num 1177) dirty exists 2024-04-12T11:30:14.736 INFO:tasks.rados.rados.0.smithi029.stdout:3733: done (1 left) 2024-04-12T11:30:14.736 INFO:tasks.rados.rados.0.smithi029.stdout:3735: done (0 left) 2024-04-12T11:30:14.736 INFO:tasks.rados.rados.0.smithi029.stdout:3736: snap_remove snap 450 2024-04-12T11:30:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:14 smithi029 ceph-mon[93650]: pgmap v1149: 89 pgs: 89 active+clean; 244 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 375 KiB/s wr, 5 op/s 2024-04-12T11:30:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:14 smithi029 ceph-mon[93650]: osdmap e1009: 8 total, 8 up, 8 in 2024-04-12T11:30:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:14 smithi029 ceph-mon[97787]: pgmap v1149: 89 pgs: 89 active+clean; 244 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 375 KiB/s wr, 5 op/s 2024-04-12T11:30:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:14 smithi029 ceph-mon[97787]: osdmap e1009: 8 total, 8 up, 8 in 2024-04-12T11:30:15.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:30:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:30:14] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.43.0" 2024-04-12T11:30:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:14 smithi087 ceph-mon[89014]: pgmap v1149: 89 pgs: 89 active+clean; 244 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 375 KiB/s wr, 5 op/s 2024-04-12T11:30:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:14 smithi087 ceph-mon[89014]: osdmap e1009: 8 total, 8 up, 8 in 2024-04-12T11:30:15.737 INFO:tasks.rados.rados.0.smithi029.stdout:3736: done (0 left) 2024-04-12T11:30:15.738 INFO:tasks.rados.rados.0.smithi029.stdout:3737: delete oid 41 current snap is 472 2024-04-12T11:30:15.746 INFO:tasks.rados.rados.0.smithi029.stdout:3737: done (0 left) 2024-04-12T11:30:15.746 INFO:tasks.rados.rados.0.smithi029.stdout:3738: write oid 36 current snap is 472 2024-04-12T11:30:15.746 INFO:tasks.rados.rados.0.smithi029.stdout:3738: seq_num 1404 ranges {797028=658548,1892248=510811,3172823=424205} 2024-04-12T11:30:15.754 INFO:tasks.rados.rados.0.smithi029.stdout:3738: writing smithi02979277-36 from 797028 to 1455576 tid 1 2024-04-12T11:30:15.759 INFO:tasks.rados.rados.0.smithi029.stdout:3738: writing smithi02979277-36 from 1892248 to 2403059 tid 2 2024-04-12T11:30:15.766 INFO:tasks.rados.rados.0.smithi029.stdout:3738: writing smithi02979277-36 from 3172823 to 3597028 tid 3 2024-04-12T11:30:15.766 INFO:tasks.rados.rados.0.smithi029.stdout:3739: write oid 9 current snap is 472 2024-04-12T11:30:15.766 INFO:tasks.rados.rados.0.smithi029.stdout:3738: finishing write tid 1 to smithi02979277-36 2024-04-12T11:30:15.766 INFO:tasks.rados.rados.0.smithi029.stdout:3738: finishing write tid 2 to smithi02979277-36 2024-04-12T11:30:15.766 INFO:tasks.rados.rados.0.smithi029.stdout:3739: seq_num 1405 ranges {413835=733797,1550730=770822,2800246=557531} 2024-04-12T11:30:15.773 INFO:tasks.rados.rados.0.smithi029.stdout:3739: writing smithi02979277-9 from 413835 to 1147632 tid 1 2024-04-12T11:30:15.779 INFO:tasks.rados.rados.0.smithi029.stdout:3739: writing smithi02979277-9 from 1550730 to 2321552 tid 2 2024-04-12T11:30:15.785 INFO:tasks.rados.rados.0.smithi029.stdout:3739: writing smithi02979277-9 from 2800246 to 3357777 tid 3 2024-04-12T11:30:15.785 INFO:tasks.rados.rados.0.smithi029.stdout:3740: write oid 3 current snap is 472 2024-04-12T11:30:15.785 INFO:tasks.rados.rados.0.smithi029.stdout:3738: finishing write tid 3 to smithi02979277-36 2024-04-12T11:30:15.786 INFO:tasks.rados.rados.0.smithi029.stdout:3740: seq_num 1406 ranges {430642=409046,1609212=514480,2827669=768803} 2024-04-12T11:30:15.789 INFO:tasks.rados.rados.0.smithi029.stdout:3740: writing smithi02979277-3 from 430642 to 839688 tid 1 2024-04-12T11:30:15.797 INFO:tasks.rados.rados.0.smithi029.stdout:3740: writing smithi02979277-3 from 1609212 to 2123692 tid 2 2024-04-12T11:30:15.805 INFO:tasks.rados.rados.0.smithi029.stdout:3740: writing smithi02979277-3 from 2827669 to 3596472 tid 3 2024-04-12T11:30:15.805 INFO:tasks.rados.rados.0.smithi029.stdout:3741: write oid 27 current snap is 472 2024-04-12T11:30:15.805 INFO:tasks.rados.rados.0.smithi029.stdout:3738: finishing write tid 4 to smithi02979277-36 2024-04-12T11:30:15.806 INFO:tasks.rados.rados.0.smithi029.stdout:3741: seq_num 1407 ranges {447449=600648,1784047=774491,2971445=676004} 2024-04-12T11:30:15.811 INFO:tasks.rados.rados.0.smithi029.stdout:3741: writing smithi02979277-27 from 447449 to 1048097 tid 1 2024-04-12T11:30:15.822 INFO:tasks.rados.rados.0.smithi029.stdout:3741: writing smithi02979277-27 from 1784047 to 2558538 tid 2 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:3741: writing smithi02979277-27 from 2971445 to 3647449 tid 3 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:3742: delete oid 4 current snap is 472 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:3738: finishing write tid 5 to smithi02979277-36 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:3738: oid 36 updating version 0 to 928 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:3738: oid 36 version 928 is already newer than 926 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:3738: oid 36 version 928 is already newer than 927 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:3738: oid 36 version 928 is already newer than 925 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 928 (ObjNum 1404 snap 472 seq_num 1404) dirty exists 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:3738: left oid 36 (ObjNum 1404 snap 472 seq_num 1404) 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:3739: finishing write tid 1 to smithi02979277-9 2024-04-12T11:30:15.826 INFO:tasks.rados.rados.0.smithi029.stdout:3739: finishing write tid 2 to smithi02979277-9 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3739: finishing write tid 3 to smithi02979277-9 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3739: finishing write tid 4 to smithi02979277-9 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3739: finishing write tid 5 to smithi02979277-9 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3739: oid 9 updating version 0 to 1325 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3739: oid 9 version 1325 is already newer than 1324 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3739: oid 9 version 1325 is already newer than 1323 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3739: oid 9 updating version 1325 to 1326 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 1326 (ObjNum 1405 snap 472 seq_num 1405) dirty exists 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3739: left oid 9 (ObjNum 1405 snap 472 seq_num 1405) 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3740: finishing write tid 1 to smithi02979277-3 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3740: finishing write tid 2 to smithi02979277-3 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3740: finishing write tid 3 to smithi02979277-3 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3740: finishing write tid 4 to smithi02979277-3 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3740: finishing write tid 5 to smithi02979277-3 2024-04-12T11:30:15.827 INFO:tasks.rados.rados.0.smithi029.stdout:3740: oid 3 updating version 0 to 2415 2024-04-12T11:30:15.828 INFO:tasks.rados.rados.0.smithi029.stdout:3740: oid 3 version 2415 is already newer than 2407 2024-04-12T11:30:15.828 INFO:tasks.rados.rados.0.smithi029.stdout:3740: oid 3 version 2415 is already newer than 2412 2024-04-12T11:30:15.828 INFO:tasks.rados.rados.0.smithi029.stdout:3740: oid 3 updating version 2415 to 2416 2024-04-12T11:30:15.828 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 2416 (ObjNum 1406 snap 472 seq_num 1406) dirty exists 2024-04-12T11:30:15.828 INFO:tasks.rados.rados.0.smithi029.stdout:3740: left oid 3 (ObjNum 1406 snap 472 seq_num 1406) 2024-04-12T11:30:15.828 INFO:tasks.rados.rados.0.smithi029.stdout:3741: finishing write tid 1 to smithi02979277-27 2024-04-12T11:30:15.828 INFO:tasks.rados.rados.0.smithi029.stdout:3741: finishing write tid 2 to smithi02979277-27 2024-04-12T11:30:15.828 INFO:tasks.rados.rados.0.smithi029.stdout:3738: done (4 left) 2024-04-12T11:30:15.828 INFO:tasks.rados.rados.0.smithi029.stdout:3739: done (3 left) 2024-04-12T11:30:15.828 INFO:tasks.rados.rados.0.smithi029.stdout:3740: done (2 left) 2024-04-12T11:30:15.829 INFO:tasks.rados.rados.0.smithi029.stdout:3742: done (1 left) 2024-04-12T11:30:15.829 INFO:tasks.rados.rados.0.smithi029.stdout:3743: snap_remove snap 456 2024-04-12T11:30:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:15 smithi029 ceph-mon[97787]: osdmap e1010: 8 total, 8 up, 8 in 2024-04-12T11:30:16.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:15 smithi029 ceph-mon[97787]: osdmap e1011: 8 total, 8 up, 8 in 2024-04-12T11:30:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:15 smithi029 ceph-mon[93650]: osdmap e1010: 8 total, 8 up, 8 in 2024-04-12T11:30:16.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:15 smithi029 ceph-mon[93650]: osdmap e1011: 8 total, 8 up, 8 in 2024-04-12T11:30:16.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:15 smithi087 ceph-mon[89014]: osdmap e1010: 8 total, 8 up, 8 in 2024-04-12T11:30:16.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:15 smithi087 ceph-mon[89014]: osdmap e1011: 8 total, 8 up, 8 in 2024-04-12T11:30:16.756 INFO:tasks.rados.rados.0.smithi029.stdout:3741: finishing write tid 3 to smithi02979277-27 2024-04-12T11:30:16.756 INFO:tasks.rados.rados.0.smithi029.stdout:3741: finishing write tid 4 to smithi02979277-27 2024-04-12T11:30:16.756 INFO:tasks.rados.rados.0.smithi029.stdout:3741: finishing write tid 5 to smithi02979277-27 2024-04-12T11:30:16.756 INFO:tasks.rados.rados.0.smithi029.stdout:3741: oid 27 updating version 0 to 1184 2024-04-12T11:30:16.756 INFO:tasks.rados.rados.0.smithi029.stdout:3741: oid 27 updating version 1184 to 1186 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:3741: oid 27 updating version 1186 to 1187 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:3741: oid 27 version 1187 is already newer than 1185 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 1187 (ObjNum 1407 snap 472 seq_num 1407) dirty exists 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:3741: left oid 27 (ObjNum 1407 snap 472 seq_num 1407) 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:3741: done (1 left) 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:3743: done (0 left) 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:3744: read oid 46 snap -1 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:3744: expect (ObjNum 1326 snap 451 seq_num 1326) 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:3745: read oid 21 snap -1 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:3745: expect deleted 2024-04-12T11:30:16.757 INFO:tasks.rados.rados.0.smithi029.stdout:3746: delete oid 22 current snap is 472 2024-04-12T11:30:16.763 INFO:tasks.rados.rados.0.smithi029.stdout:3744: expect (ObjNum 1326 snap 451 seq_num 1326) 2024-04-12T11:30:16.876 INFO:tasks.rados.rados.0.smithi029.stdout:3744: done (2 left) 2024-04-12T11:30:16.876 INFO:tasks.rados.rados.0.smithi029.stdout:3745: done (1 left) 2024-04-12T11:30:16.876 INFO:tasks.rados.rados.0.smithi029.stdout:3746: done (0 left) 2024-04-12T11:30:16.876 INFO:tasks.rados.rados.0.smithi029.stdout:3747: read oid 33 snap -1 2024-04-12T11:30:16.876 INFO:tasks.rados.rados.0.smithi029.stdout:3747: expect (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:30:16.876 INFO:tasks.rados.rados.0.smithi029.stdout:3748: write oid 11 current snap is 472 2024-04-12T11:30:16.876 INFO:tasks.rados.rados.0.smithi029.stdout:3748: seq_num 1410 ranges {497870=426395,1443140=521818,2537361=530244,3583943=113927} 2024-04-12T11:30:16.880 INFO:tasks.rados.rados.0.smithi029.stdout:3748: writing smithi02979277-11 from 497870 to 924265 tid 1 2024-04-12T11:30:16.886 INFO:tasks.rados.rados.0.smithi029.stdout:3748: writing smithi02979277-11 from 1443140 to 1964958 tid 2 2024-04-12T11:30:16.892 INFO:tasks.rados.rados.0.smithi029.stdout:3748: writing smithi02979277-11 from 2537361 to 3067605 tid 3 2024-04-12T11:30:16.896 INFO:tasks.rados.rados.0.smithi029.stdout:3748: writing smithi02979277-11 from 3583943 to 3697870 tid 4 2024-04-12T11:30:16.896 INFO:tasks.rados.rados.0.smithi029.stdout:3749: delete oid 34 current snap is 472 2024-04-12T11:30:16.896 INFO:tasks.rados.rados.0.smithi029.stdout:3747: expect (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:30:16.968 INFO:tasks.rados.rados.0.smithi029.stdout:3748: finishing write tid 1 to smithi02979277-11 2024-04-12T11:30:16.968 INFO:tasks.rados.rados.0.smithi029.stdout:3748: finishing write tid 2 to smithi02979277-11 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: finishing write tid 3 to smithi02979277-11 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: finishing write tid 4 to smithi02979277-11 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: finishing write tid 5 to smithi02979277-11 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: finishing write tid 6 to smithi02979277-11 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: oid 11 updating version 0 to 2427 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: oid 11 updating version 2427 to 2428 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: oid 11 version 2428 is already newer than 2424 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: oid 11 version 2428 is already newer than 2426 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: oid 11 version 2428 is already newer than 2425 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 2428 (ObjNum 1410 snap 472 seq_num 1410) dirty exists 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: left oid 11 (ObjNum 1410 snap 472 seq_num 1410) 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3747: done (2 left) 2024-04-12T11:30:16.969 INFO:tasks.rados.rados.0.smithi029.stdout:3748: done (1 left) 2024-04-12T11:30:16.970 INFO:tasks.rados.rados.0.smithi029.stdout:3749: done (0 left) 2024-04-12T11:30:16.970 INFO:tasks.rados.rados.0.smithi029.stdout:3750: read oid 2 snap -1 2024-04-12T11:30:16.970 INFO:tasks.rados.rados.0.smithi029.stdout:3750: expect (ObjNum 1385 snap 467 seq_num 1385) 2024-04-12T11:30:16.970 INFO:tasks.rados.rados.0.smithi029.stdout:3751: read oid 13 snap 442 2024-04-12T11:30:16.970 INFO:tasks.rados.rados.0.smithi029.stdout:3751: expect (ObjNum 1221 snap 409 seq_num 1221) 2024-04-12T11:30:16.970 INFO:tasks.rados.rados.0.smithi029.stdout:3752: snap_remove snap 463 2024-04-12T11:30:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:16 smithi029 ceph-mon[93650]: pgmap v1152: 89 pgs: 89 active+clean; 244 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 732 KiB/s wr, 7 op/s 2024-04-12T11:30:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:16 smithi029 ceph-mon[97787]: pgmap v1152: 89 pgs: 89 active+clean; 244 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 732 KiB/s wr, 7 op/s 2024-04-12T11:30:17.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:16 smithi087 ceph-mon[89014]: pgmap v1152: 89 pgs: 89 active+clean; 244 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 732 KiB/s wr, 7 op/s 2024-04-12T11:30:17.724 INFO:tasks.rados.rados.0.smithi029.stdout:3752: done (2 left) 2024-04-12T11:30:17.724 INFO:tasks.rados.rados.0.smithi029.stdout:3753: read oid 22 snap -1 2024-04-12T11:30:17.724 INFO:tasks.rados.rados.0.smithi029.stdout:3753: expect deleted 2024-04-12T11:30:17.724 INFO:tasks.rados.rados.0.smithi029.stdout:3754: write oid 17 current snap is 472 2024-04-12T11:30:17.724 INFO:tasks.rados.rados.0.smithi029.stdout:3754: seq_num 1412 ranges {531484=576893,1843751=525487,2875854=669141} 2024-04-12T11:30:17.731 INFO:tasks.rados.rados.0.smithi029.stdout:3754: writing smithi02979277-17 from 531484 to 1108377 tid 1 2024-04-12T11:30:17.739 INFO:tasks.rados.rados.0.smithi029.stdout:3754: writing smithi02979277-17 from 1843751 to 2369238 tid 2 2024-04-12T11:30:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3754: writing smithi02979277-17 from 2875854 to 3544995 tid 3 2024-04-12T11:30:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3755: snap_remove snap 470 2024-04-12T11:30:18.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:17 smithi029 ceph-mon[93650]: osdmap e1012: 8 total, 8 up, 8 in 2024-04-12T11:30:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:17 smithi029 ceph-mon[93650]: osdmap e1013: 8 total, 8 up, 8 in 2024-04-12T11:30:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:17 smithi029 ceph-mon[97787]: osdmap e1012: 8 total, 8 up, 8 in 2024-04-12T11:30:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:17 smithi029 ceph-mon[97787]: osdmap e1013: 8 total, 8 up, 8 in 2024-04-12T11:30:18.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:17 smithi087 ceph-mon[89014]: osdmap e1012: 8 total, 8 up, 8 in 2024-04-12T11:30:18.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:17 smithi087 ceph-mon[89014]: osdmap e1013: 8 total, 8 up, 8 in 2024-04-12T11:30:18.724 INFO:tasks.rados.rados.0.smithi029.stdout:3755: done (4 left) 2024-04-12T11:30:18.725 INFO:tasks.rados.rados.0.smithi029.stdout:3756: snap_create 2024-04-12T11:30:18.725 INFO:tasks.rados.rados.0.smithi029.stdout:3750: expect (ObjNum 1385 snap 467 seq_num 1385) 2024-04-12T11:30:18.833 INFO:tasks.rados.rados.0.smithi029.stdout:3751: expect (ObjNum 1221 snap 409 seq_num 1221) 2024-04-12T11:30:18.990 INFO:tasks.rados.rados.0.smithi029.stdout:3754: finishing write tid 1 to smithi02979277-17 2024-04-12T11:30:18.990 INFO:tasks.rados.rados.0.smithi029.stdout:3754: finishing write tid 2 to smithi02979277-17 2024-04-12T11:30:18.990 INFO:tasks.rados.rados.0.smithi029.stdout:3754: finishing write tid 3 to smithi02979277-17 2024-04-12T11:30:18.990 INFO:tasks.rados.rados.0.smithi029.stdout:3754: finishing write tid 4 to smithi02979277-17 2024-04-12T11:30:18.991 INFO:tasks.rados.rados.0.smithi029.stdout:3754: finishing write tid 5 to smithi02979277-17 2024-04-12T11:30:18.991 INFO:tasks.rados.rados.0.smithi029.stdout:3754: oid 17 updating version 0 to 2007 2024-04-12T11:30:18.991 INFO:tasks.rados.rados.0.smithi029.stdout:3754: oid 17 version 2007 is already newer than 2005 2024-04-12T11:30:18.991 INFO:tasks.rados.rados.0.smithi029.stdout:3754: oid 17 updating version 2007 to 2008 2024-04-12T11:30:18.991 INFO:tasks.rados.rados.0.smithi029.stdout:3754: oid 17 version 2008 is already newer than 2006 2024-04-12T11:30:18.991 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 2008 (ObjNum 1412 snap 472 seq_num 1412) dirty exists 2024-04-12T11:30:18.991 INFO:tasks.rados.rados.0.smithi029.stdout:3754: left oid 17 (ObjNum 1412 snap 472 seq_num 1412) 2024-04-12T11:30:19.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:18 smithi029 ceph-mon[93650]: pgmap v1155: 89 pgs: 89 active+clean; 235 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 357 KiB/s wr, 6 op/s 2024-04-12T11:30:19.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:18 smithi029 ceph-mon[93650]: osdmap e1014: 8 total, 8 up, 8 in 2024-04-12T11:30:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:18 smithi029 ceph-mon[97787]: pgmap v1155: 89 pgs: 89 active+clean; 235 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 357 KiB/s wr, 6 op/s 2024-04-12T11:30:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:18 smithi029 ceph-mon[97787]: osdmap e1014: 8 total, 8 up, 8 in 2024-04-12T11:30:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:18 smithi087 ceph-mon[89014]: pgmap v1155: 89 pgs: 89 active+clean; 235 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 357 KiB/s wr, 6 op/s 2024-04-12T11:30:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:18 smithi087 ceph-mon[89014]: osdmap e1014: 8 total, 8 up, 8 in 2024-04-12T11:30:19.726 INFO:tasks.rados.rados.0.smithi029.stdout:3750: done (4 left) 2024-04-12T11:30:19.727 INFO:tasks.rados.rados.0.smithi029.stdout:3751: done (3 left) 2024-04-12T11:30:19.727 INFO:tasks.rados.rados.0.smithi029.stdout:3753: done (2 left) 2024-04-12T11:30:19.727 INFO:tasks.rados.rados.0.smithi029.stdout:3754: done (1 left) 2024-04-12T11:30:19.727 INFO:tasks.rados.rados.0.smithi029.stdout:3756: done (0 left) 2024-04-12T11:30:19.727 INFO:tasks.rados.rados.0.smithi029.stdout:3757: read oid 2 snap -1 2024-04-12T11:30:19.727 INFO:tasks.rados.rados.0.smithi029.stdout:3757: expect (ObjNum 1385 snap 467 seq_num 1385) 2024-04-12T11:30:19.727 INFO:tasks.rados.rados.0.smithi029.stdout:3758: snap_create 2024-04-12T11:30:19.735 INFO:tasks.rados.rados.0.smithi029.stdout:3757: expect (ObjNum 1385 snap 467 seq_num 1385) 2024-04-12T11:30:20.731 INFO:tasks.rados.rados.0.smithi029.stdout:3757: done (1 left) 2024-04-12T11:30:20.731 INFO:tasks.rados.rados.0.smithi029.stdout:3758: done (0 left) 2024-04-12T11:30:20.731 INFO:tasks.rados.rados.0.smithi029.stdout:3759: snap_create 2024-04-12T11:30:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:20 smithi029 ceph-mon[93650]: pgmap v1158: 89 pgs: 89 active+clean; 231 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-04-12T11:30:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:20 smithi029 ceph-mon[93650]: osdmap e1015: 8 total, 8 up, 8 in 2024-04-12T11:30:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:20 smithi029 ceph-mon[97787]: pgmap v1158: 89 pgs: 89 active+clean; 231 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-04-12T11:30:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:20 smithi029 ceph-mon[97787]: osdmap e1015: 8 total, 8 up, 8 in 2024-04-12T11:30:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:20 smithi087 ceph-mon[89014]: pgmap v1158: 89 pgs: 89 active+clean; 231 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-04-12T11:30:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:20 smithi087 ceph-mon[89014]: osdmap e1015: 8 total, 8 up, 8 in 2024-04-12T11:30:21.737 INFO:tasks.rados.rados.0.smithi029.stdout:3759: done (0 left) 2024-04-12T11:30:21.737 INFO:tasks.rados.rados.0.smithi029.stdout:3760: snap_remove snap 443 2024-04-12T11:30:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:21 smithi029 ceph-mon[93650]: osdmap e1016: 8 total, 8 up, 8 in 2024-04-12T11:30:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:21 smithi029 ceph-mon[97787]: osdmap e1016: 8 total, 8 up, 8 in 2024-04-12T11:30:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:22.118 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:21 smithi087 ceph-mon[89014]: osdmap e1016: 8 total, 8 up, 8 in 2024-04-12T11:30:22.118 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:22.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:30:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:30:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:30:22.724 INFO:tasks.rados.rados.0.smithi029.stdout:3760: done (0 left) 2024-04-12T11:30:22.725 INFO:tasks.rados.rados.0.smithi029.stdout:3761: snap_remove snap 462 2024-04-12T11:30:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:22 smithi029 ceph-mon[93650]: pgmap v1161: 89 pgs: 89 active+clean; 228 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-04-12T11:30:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:22 smithi029 ceph-mon[93650]: osdmap e1017: 8 total, 8 up, 8 in 2024-04-12T11:30:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:22 smithi029 ceph-mon[93650]: osdmap e1018: 8 total, 8 up, 8 in 2024-04-12T11:30:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:22 smithi029 ceph-mon[97787]: pgmap v1161: 89 pgs: 89 active+clean; 228 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-04-12T11:30:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:22 smithi029 ceph-mon[97787]: osdmap e1017: 8 total, 8 up, 8 in 2024-04-12T11:30:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:22 smithi029 ceph-mon[97787]: osdmap e1018: 8 total, 8 up, 8 in 2024-04-12T11:30:23.132 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:22 smithi087 ceph-mon[89014]: pgmap v1161: 89 pgs: 89 active+clean; 228 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2024-04-12T11:30:23.132 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:22 smithi087 ceph-mon[89014]: osdmap e1017: 8 total, 8 up, 8 in 2024-04-12T11:30:23.132 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:22 smithi087 ceph-mon[89014]: osdmap e1018: 8 total, 8 up, 8 in 2024-04-12T11:30:23.727 INFO:tasks.rados.rados.0.smithi029.stdout:3761: done (0 left) 2024-04-12T11:30:23.727 INFO:tasks.rados.rados.0.smithi029.stdout:3762: write oid 39 current snap is 475 2024-04-12T11:30:23.727 INFO:tasks.rados.rados.0.smithi029.stdout:3762: seq_num 1413 ranges {548291=652142,1902233=785498,3303277=445014} 2024-04-12T11:30:23.734 INFO:tasks.rados.rados.0.smithi029.stdout:3762: writing smithi02979277-39 from 548291 to 1200433 tid 1 2024-04-12T11:30:23.743 INFO:tasks.rados.rados.0.smithi029.stdout:3762: writing smithi02979277-39 from 1902233 to 2687731 tid 2 2024-04-12T11:30:23.749 INFO:tasks.rados.rados.0.smithi029.stdout:3762: writing smithi02979277-39 from 3303277 to 3748291 tid 3 2024-04-12T11:30:23.749 INFO:tasks.rados.rados.0.smithi029.stdout:3763: write oid 30 current snap is 475 2024-04-12T11:30:23.749 INFO:tasks.rados.rados.0.smithi029.stdout:3762: finishing write tid 1 to smithi02979277-39 2024-04-12T11:30:23.749 INFO:tasks.rados.rados.0.smithi029.stdout:3763: seq_num 1414 ranges {565098=727391,1960715=529156,2930700=691685} 2024-04-12T11:30:23.756 INFO:tasks.rados.rados.0.smithi029.stdout:3763: writing smithi02979277-30 from 565098 to 1292489 tid 1 2024-04-12T11:30:23.763 INFO:tasks.rados.rados.0.smithi029.stdout:3763: writing smithi02979277-30 from 1960715 to 2489871 tid 2 2024-04-12T11:30:23.769 INFO:tasks.rados.rados.0.smithi029.stdout:3763: writing smithi02979277-30 from 2930700 to 3622385 tid 3 2024-04-12T11:30:23.769 INFO:tasks.rados.rados.0.smithi029.stdout:3764: write oid 1 current snap is 475 2024-04-12T11:30:23.769 INFO:tasks.rados.rados.0.smithi029.stdout:3764: seq_num 1415 ranges {581905=518993,1735550=672814,2958123=502957} 2024-04-12T11:30:23.775 INFO:tasks.rados.rados.0.smithi029.stdout:3764: writing smithi02979277-1 from 581905 to 1100898 tid 1 2024-04-12T11:30:23.782 INFO:tasks.rados.rados.0.smithi029.stdout:3764: writing smithi02979277-1 from 1735550 to 2408364 tid 2 2024-04-12T11:30:23.788 INFO:tasks.rados.rados.0.smithi029.stdout:3764: writing smithi02979277-1 from 2958123 to 3461080 tid 3 2024-04-12T11:30:23.788 INFO:tasks.rados.rados.0.smithi029.stdout:3765: snap_remove snap 472 2024-04-12T11:30:24.742 INFO:tasks.rados.rados.0.smithi029.stdout:3765: done (3 left) 2024-04-12T11:30:24.742 INFO:tasks.rados.rados.0.smithi029.stdout:3766: snap_create 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3762: finishing write tid 2 to smithi02979277-39 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3762: finishing write tid 3 to smithi02979277-39 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3762: finishing write tid 4 to smithi02979277-39 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3762: finishing write tid 5 to smithi02979277-39 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3762: oid 39 updating version 0 to 552 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3762: oid 39 version 552 is already newer than 551 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3762: oid 39 updating version 552 to 553 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3762: oid 39 updating version 553 to 554 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 554 (ObjNum 1413 snap 475 seq_num 1413) dirty exists 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3762: left oid 39 (ObjNum 1413 snap 475 seq_num 1413) 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3763: finishing write tid 1 to smithi02979277-30 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3763: finishing write tid 2 to smithi02979277-30 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3763: finishing write tid 3 to smithi02979277-30 2024-04-12T11:30:24.743 INFO:tasks.rados.rados.0.smithi029.stdout:3763: finishing write tid 4 to smithi02979277-30 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3763: finishing write tid 5 to smithi02979277-30 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3763: oid 30 updating version 0 to 1203 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3763: oid 30 version 1203 is already newer than 1202 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3763: oid 30 updating version 1203 to 1205 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3763: oid 30 version 1205 is already newer than 1204 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 1205 (ObjNum 1414 snap 475 seq_num 1414) dirty exists 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3763: left oid 30 (ObjNum 1414 snap 475 seq_num 1414) 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3764: finishing write tid 1 to smithi02979277-1 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3764: finishing write tid 2 to smithi02979277-1 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3764: finishing write tid 3 to smithi02979277-1 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3764: finishing write tid 4 to smithi02979277-1 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3764: finishing write tid 5 to smithi02979277-1 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3764: oid 1 updating version 0 to 1878 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3764: oid 1 updating version 1878 to 1880 2024-04-12T11:30:24.744 INFO:tasks.rados.rados.0.smithi029.stdout:3764: oid 1 version 1880 is already newer than 1879 2024-04-12T11:30:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:3764: oid 1 updating version 1880 to 1881 2024-04-12T11:30:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1881 (ObjNum 1415 snap 475 seq_num 1415) dirty exists 2024-04-12T11:30:24.745 INFO:tasks.rados.rados.0.smithi029.stdout:3764: left oid 1 (ObjNum 1415 snap 475 seq_num 1415) 2024-04-12T11:30:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:24 smithi029 ceph-mon[97787]: pgmap v1164: 89 pgs: 89 active+clean; 229 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 822 KiB/s wr, 2 op/s 2024-04-12T11:30:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:24 smithi029 ceph-mon[97787]: osdmap e1019: 8 total, 8 up, 8 in 2024-04-12T11:30:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:24 smithi029 ceph-mon[93650]: pgmap v1164: 89 pgs: 89 active+clean; 229 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 822 KiB/s wr, 2 op/s 2024-04-12T11:30:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:24 smithi029 ceph-mon[93650]: osdmap e1019: 8 total, 8 up, 8 in 2024-04-12T11:30:25.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:30:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:30:24] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-12T11:30:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:24 smithi087 ceph-mon[89014]: pgmap v1164: 89 pgs: 89 active+clean; 229 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 822 KiB/s wr, 2 op/s 2024-04-12T11:30:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:24 smithi087 ceph-mon[89014]: osdmap e1019: 8 total, 8 up, 8 in 2024-04-12T11:30:25.748 INFO:tasks.rados.rados.0.smithi029.stdout:3762: done (3 left) 2024-04-12T11:30:25.748 INFO:tasks.rados.rados.0.smithi029.stdout:3763: done (2 left) 2024-04-12T11:30:25.748 INFO:tasks.rados.rados.0.smithi029.stdout:3764: done (1 left) 2024-04-12T11:30:25.748 INFO:tasks.rados.rados.0.smithi029.stdout:3766: done (0 left) 2024-04-12T11:30:25.748 INFO:tasks.rados.rados.0.smithi029.stdout:3767: write oid 24 current snap is 476 2024-04-12T11:30:25.748 INFO:tasks.rados.rados.0.smithi029.stdout:3767: seq_num 1416 ranges {598712=594242,1677679=532825,2985546=430582} 2024-04-12T11:30:25.754 INFO:tasks.rados.rados.0.smithi029.stdout:3767: writing smithi02979277-24 from 598712 to 1192954 tid 1 2024-04-12T11:30:25.760 INFO:tasks.rados.rados.0.smithi029.stdout:3767: writing smithi02979277-24 from 1677679 to 2210504 tid 2 2024-04-12T11:30:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:3767: writing smithi02979277-24 from 2985546 to 3416128 tid 3 2024-04-12T11:30:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:3768: rollback oid 4 current snap is 476 2024-04-12T11:30:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 4 to 442 2024-04-12T11:30:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:3767: finishing write tid 1 to smithi02979277-24 2024-04-12T11:30:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:3767: finishing write tid 2 to smithi02979277-24 2024-04-12T11:30:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:3769: write oid 12 current snap is 476 2024-04-12T11:30:25.767 INFO:tasks.rados.rados.0.smithi029.stdout:3769: seq_num 1417 ranges {615519=669491,1736161=676483,2896616=641854} 2024-04-12T11:30:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3769: writing smithi02979277-12 from 615519 to 1285010 tid 1 2024-04-12T11:30:25.780 INFO:tasks.rados.rados.0.smithi029.stdout:3769: writing smithi02979277-12 from 1736161 to 2412644 tid 2 2024-04-12T11:30:25.786 INFO:tasks.rados.rados.0.smithi029.stdout:3769: writing smithi02979277-12 from 2896616 to 3538470 tid 3 2024-04-12T11:30:25.787 INFO:tasks.rados.rados.0.smithi029.stdout:3770: write oid 29 current snap is 476 2024-04-12T11:30:25.787 INFO:tasks.rados.rados.0.smithi029.stdout:3770: seq_num 1418 ranges {632326=744740,1794643=536494,3040392=453126} 2024-04-12T11:30:25.794 INFO:tasks.rados.rados.0.smithi029.stdout:3770: writing smithi02979277-29 from 632326 to 1377066 tid 1 2024-04-12T11:30:25.799 INFO:tasks.rados.rados.0.smithi029.stdout:3770: writing smithi02979277-29 from 1794643 to 2331137 tid 2 2024-04-12T11:30:25.806 INFO:tasks.rados.rados.0.smithi029.stdout:3770: writing smithi02979277-29 from 3040392 to 3493518 tid 3 2024-04-12T11:30:25.806 INFO:tasks.rados.rados.0.smithi029.stdout:3771: write oid 26 current snap is 476 2024-04-12T11:30:25.806 INFO:tasks.rados.rados.0.smithi029.stdout:3771: seq_num 1419 ranges {649133=419989,1853125=680152,2951462=664398} 2024-04-12T11:30:25.812 INFO:tasks.rados.rados.0.smithi029.stdout:3771: writing smithi02979277-26 from 649133 to 1069122 tid 1 2024-04-12T11:30:25.820 INFO:tasks.rados.rados.0.smithi029.stdout:3771: writing smithi02979277-26 from 1853125 to 2533277 tid 2 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3771: writing smithi02979277-26 from 2951462 to 3615860 tid 3 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3772: read oid 6 snap -1 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3772: expect (ObjNum 1402 snap 472 seq_num 1402) 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3773: write oid 34 current snap is 476 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3768: finishing rollback tid 1 to smithi02979277-4 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3768: finishing rollback tid 2 to smithi02979277-4 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 4 v 1884 (ObjNum 1276 snap 436 seq_num 1276) dirty exists 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3767: finishing write tid 3 to smithi02979277-24 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3767: finishing write tid 4 to smithi02979277-24 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3767: finishing write tid 5 to smithi02979277-24 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3767: oid 24 updating version 0 to 2453 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3767: oid 24 updating version 2453 to 2455 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3767: oid 24 updating version 2455 to 2456 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:3767: oid 24 version 2456 is already newer than 2454 2024-04-12T11:30:25.827 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 2456 (ObjNum 1416 snap 476 seq_num 1416) dirty exists 2024-04-12T11:30:25.828 INFO:tasks.rados.rados.0.smithi029.stdout:3767: left oid 24 (ObjNum 1416 snap 476 seq_num 1416) 2024-04-12T11:30:25.828 INFO:tasks.rados.rados.0.smithi029.stdout:3769: finishing write tid 1 to smithi02979277-12 2024-04-12T11:30:25.828 INFO:tasks.rados.rados.0.smithi029.stdout:3769: finishing write tid 2 to smithi02979277-12 2024-04-12T11:30:25.828 INFO:tasks.rados.rados.0.smithi029.stdout:3769: finishing write tid 3 to smithi02979277-12 2024-04-12T11:30:25.828 INFO:tasks.rados.rados.0.smithi029.stdout:3773: seq_num 1420 ranges {665940=495238,1795254=540163,2978885=475670} 2024-04-12T11:30:25.832 INFO:tasks.rados.rados.0.smithi029.stdout:3773: writing smithi02979277-34 from 665940 to 1161178 tid 1 2024-04-12T11:30:25.839 INFO:tasks.rados.rados.0.smithi029.stdout:3773: writing smithi02979277-34 from 1795254 to 2335417 tid 2 2024-04-12T11:30:25.845 INFO:tasks.rados.rados.0.smithi029.stdout:3773: writing smithi02979277-34 from 2978885 to 3454555 tid 3 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3767: done (6 left) 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3768: done (5 left) 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3774: snap_create 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3769: finishing write tid 4 to smithi02979277-12 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3769: finishing write tid 5 to smithi02979277-12 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3769: oid 12 updating version 0 to 1901 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3769: oid 12 version 1901 is already newer than 1900 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3769: oid 12 version 1901 is already newer than 1899 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3769: oid 12 updating version 1901 to 1902 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 12 v 1902 (ObjNum 1417 snap 476 seq_num 1417) dirty exists 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3769: left oid 12 (ObjNum 1417 snap 476 seq_num 1417) 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3770: finishing write tid 1 to smithi02979277-29 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3770: finishing write tid 2 to smithi02979277-29 2024-04-12T11:30:25.846 INFO:tasks.rados.rados.0.smithi029.stdout:3770: finishing write tid 3 to smithi02979277-29 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3770: finishing write tid 4 to smithi02979277-29 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3770: finishing write tid 5 to smithi02979277-29 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3770: oid 29 updating version 0 to 1715 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3770: oid 29 updating version 1715 to 1716 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3770: oid 29 updating version 1716 to 1718 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3770: oid 29 version 1718 is already newer than 1717 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1718 (ObjNum 1418 snap 476 seq_num 1418) dirty exists 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3770: left oid 29 (ObjNum 1418 snap 476 seq_num 1418) 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3771: finishing write tid 1 to smithi02979277-26 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3771: finishing write tid 2 to smithi02979277-26 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3771: finishing write tid 3 to smithi02979277-26 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3771: finishing write tid 4 to smithi02979277-26 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3771: finishing write tid 5 to smithi02979277-26 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3771: oid 26 updating version 0 to 2458 2024-04-12T11:30:25.847 INFO:tasks.rados.rados.0.smithi029.stdout:3771: oid 26 updating version 2458 to 2459 2024-04-12T11:30:25.848 INFO:tasks.rados.rados.0.smithi029.stdout:3771: oid 26 updating version 2459 to 2461 2024-04-12T11:30:25.848 INFO:tasks.rados.rados.0.smithi029.stdout:3771: oid 26 version 2461 is already newer than 2460 2024-04-12T11:30:25.848 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 2461 (ObjNum 1419 snap 476 seq_num 1419) dirty exists 2024-04-12T11:30:25.848 INFO:tasks.rados.rados.0.smithi029.stdout:3771: left oid 26 (ObjNum 1419 snap 476 seq_num 1419) 2024-04-12T11:30:25.848 INFO:tasks.rados.rados.0.smithi029.stdout:3772: expect (ObjNum 1402 snap 472 seq_num 1402) 2024-04-12T11:30:25.966 INFO:tasks.rados.rados.0.smithi029.stdout:3773: finishing write tid 1 to smithi02979277-34 2024-04-12T11:30:25.966 INFO:tasks.rados.rados.0.smithi029.stdout:3773: finishing write tid 2 to smithi02979277-34 2024-04-12T11:30:25.966 INFO:tasks.rados.rados.0.smithi029.stdout:3773: finishing write tid 3 to smithi02979277-34 2024-04-12T11:30:25.966 INFO:tasks.rados.rados.0.smithi029.stdout:3773: finishing write tid 4 to smithi02979277-34 2024-04-12T11:30:25.966 INFO:tasks.rados.rados.0.smithi029.stdout:3773: finishing write tid 5 to smithi02979277-34 2024-04-12T11:30:25.966 INFO:tasks.rados.rados.0.smithi029.stdout:3773: oid 34 updating version 0 to 1720 2024-04-12T11:30:25.966 INFO:tasks.rados.rados.0.smithi029.stdout:3773: oid 34 updating version 1720 to 1722 2024-04-12T11:30:25.966 INFO:tasks.rados.rados.0.smithi029.stdout:3773: oid 34 version 1722 is already newer than 1719 2024-04-12T11:30:25.967 INFO:tasks.rados.rados.0.smithi029.stdout:3773: oid 34 version 1722 is already newer than 1721 2024-04-12T11:30:25.967 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1722 (ObjNum 1420 snap 476 seq_num 1420) dirty exists 2024-04-12T11:30:25.967 INFO:tasks.rados.rados.0.smithi029.stdout:3773: left oid 34 (ObjNum 1420 snap 476 seq_num 1420) 2024-04-12T11:30:26.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:25 smithi029 ceph-mon[93650]: osdmap e1020: 8 total, 8 up, 8 in 2024-04-12T11:30:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:25 smithi029 ceph-mon[97787]: osdmap e1020: 8 total, 8 up, 8 in 2024-04-12T11:30:26.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:25 smithi087 ceph-mon[89014]: osdmap e1020: 8 total, 8 up, 8 in 2024-04-12T11:30:26.752 INFO:tasks.rados.rados.0.smithi029.stdout:3769: done (5 left) 2024-04-12T11:30:26.752 INFO:tasks.rados.rados.0.smithi029.stdout:3770: done (4 left) 2024-04-12T11:30:26.752 INFO:tasks.rados.rados.0.smithi029.stdout:3771: done (3 left) 2024-04-12T11:30:26.753 INFO:tasks.rados.rados.0.smithi029.stdout:3772: done (2 left) 2024-04-12T11:30:26.753 INFO:tasks.rados.rados.0.smithi029.stdout:3773: done (1 left) 2024-04-12T11:30:26.753 INFO:tasks.rados.rados.0.smithi029.stdout:3774: done (0 left) 2024-04-12T11:30:26.753 INFO:tasks.rados.rados.0.smithi029.stdout:3775: read oid 33 snap -1 2024-04-12T11:30:26.753 INFO:tasks.rados.rados.0.smithi029.stdout:3775: expect (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:30:26.753 INFO:tasks.rados.rados.0.smithi029.stdout:3776: snap_remove snap 475 2024-04-12T11:30:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:26 smithi029 ceph-mon[97787]: pgmap v1167: 89 pgs: 89 active+clean; 229 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 433 KiB/s wr, 3 op/s 2024-04-12T11:30:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:26 smithi029 ceph-mon[97787]: osdmap e1021: 8 total, 8 up, 8 in 2024-04-12T11:30:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:26 smithi029 ceph-mon[93650]: pgmap v1167: 89 pgs: 89 active+clean; 229 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 433 KiB/s wr, 3 op/s 2024-04-12T11:30:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:26 smithi029 ceph-mon[93650]: osdmap e1021: 8 total, 8 up, 8 in 2024-04-12T11:30:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:26 smithi087 ceph-mon[89014]: pgmap v1167: 89 pgs: 89 active+clean; 229 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 433 KiB/s wr, 3 op/s 2024-04-12T11:30:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:26 smithi087 ceph-mon[89014]: osdmap e1021: 8 total, 8 up, 8 in 2024-04-12T11:30:27.727 INFO:tasks.rados.rados.0.smithi029.stdout:3776: done (1 left) 2024-04-12T11:30:27.727 INFO:tasks.rados.rados.0.smithi029.stdout:3777: snap_remove snap 455 2024-04-12T11:30:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:27 smithi029 ceph-mon[97787]: osdmap e1022: 8 total, 8 up, 8 in 2024-04-12T11:30:28.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:27 smithi029 ceph-mon[97787]: osdmap e1023: 8 total, 8 up, 8 in 2024-04-12T11:30:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:27 smithi029 ceph-mon[93650]: osdmap e1022: 8 total, 8 up, 8 in 2024-04-12T11:30:28.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:27 smithi029 ceph-mon[93650]: osdmap e1023: 8 total, 8 up, 8 in 2024-04-12T11:30:28.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:27 smithi087 ceph-mon[89014]: osdmap e1022: 8 total, 8 up, 8 in 2024-04-12T11:30:28.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:27 smithi087 ceph-mon[89014]: osdmap e1023: 8 total, 8 up, 8 in 2024-04-12T11:30:28.729 INFO:tasks.rados.rados.0.smithi029.stdout:3777: done (1 left) 2024-04-12T11:30:28.729 INFO:tasks.rados.rados.0.smithi029.stdout:3778: read oid 4 snap -1 2024-04-12T11:30:28.729 INFO:tasks.rados.rados.0.smithi029.stdout:3778: expect (ObjNum 1276 snap 436 seq_num 1276) 2024-04-12T11:30:28.729 INFO:tasks.rados.rados.0.smithi029.stdout:3779: write oid 1 current snap is 477 2024-04-12T11:30:28.729 INFO:tasks.rados.rados.0.smithi029.stdout:3779: seq_num 1421 ranges {682747=570487,1853736=683821,3006308=403295} 2024-04-12T11:30:28.736 INFO:tasks.rados.rados.0.smithi029.stdout:3779: writing smithi02979277-1 from 682747 to 1253234 tid 1 2024-04-12T11:30:28.745 INFO:tasks.rados.rados.0.smithi029.stdout:3779: writing smithi02979277-1 from 1853736 to 2537557 tid 2 2024-04-12T11:30:28.751 INFO:tasks.rados.rados.0.smithi029.stdout:3779: writing smithi02979277-1 from 3006308 to 3409603 tid 3 2024-04-12T11:30:28.751 INFO:tasks.rados.rados.0.smithi029.stdout:3780: rollback oid 44 current snap is 477 2024-04-12T11:30:28.751 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 44 to 469 2024-04-12T11:30:28.751 INFO:tasks.rados.rados.0.smithi029.stdout:3775: expect (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:30:28.823 INFO:tasks.rados.rados.0.smithi029.stdout:3779: finishing write tid 1 to smithi02979277-1 2024-04-12T11:30:28.823 INFO:tasks.rados.rados.0.smithi029.stdout:3778: expect (ObjNum 1276 snap 436 seq_num 1276) 2024-04-12T11:30:28.976 INFO:tasks.rados.rados.0.smithi029.stdout:3779: finishing write tid 2 to smithi02979277-1 2024-04-12T11:30:28.976 INFO:tasks.rados.rados.0.smithi029.stdout:3779: finishing write tid 3 to smithi02979277-1 2024-04-12T11:30:28.976 INFO:tasks.rados.rados.0.smithi029.stdout:3779: finishing write tid 4 to smithi02979277-1 2024-04-12T11:30:28.976 INFO:tasks.rados.rados.0.smithi029.stdout:3779: finishing write tid 5 to smithi02979277-1 2024-04-12T11:30:28.976 INFO:tasks.rados.rados.0.smithi029.stdout:3779: oid 1 updating version 0 to 1891 2024-04-12T11:30:28.976 INFO:tasks.rados.rados.0.smithi029.stdout:3779: oid 1 updating version 1891 to 1892 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3779: oid 1 updating version 1892 to 1893 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3779: oid 1 updating version 1893 to 1894 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 1 v 1894 (ObjNum 1421 snap 477 seq_num 1421) dirty exists 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3779: left oid 1 (ObjNum 1421 snap 477 seq_num 1421) 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3780: finishing rollback tid 0 to smithi02979277-44 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3780: finishing rollback tid 1 to smithi02979277-44 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3780: finishing rollback tid 2 to smithi02979277-44 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1898 (ObjNum 1336 snap 453 seq_num 1336) dirty exists 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3775: done (3 left) 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3778: done (2 left) 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3779: done (1 left) 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3780: done (0 left) 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3781: write oid 11 current snap is 477 2024-04-12T11:30:28.977 INFO:tasks.rados.rados.0.smithi029.stdout:3781: seq_num 1422 ranges {699554=762089,2028571=543832,3150084=614567} 2024-04-12T11:30:28.984 INFO:tasks.rados.rados.0.smithi029.stdout:3781: writing smithi02979277-11 from 699554 to 1461643 tid 1 2024-04-12T11:30:28.990 INFO:tasks.rados.rados.0.smithi029.stdout:3781: writing smithi02979277-11 from 2028571 to 2572403 tid 2 2024-04-12T11:30:28.997 INFO:tasks.rados.rados.0.smithi029.stdout:3781: writing smithi02979277-11 from 3150084 to 3764651 tid 3 2024-04-12T11:30:28.997 INFO:tasks.rados.rados.0.smithi029.stdout:3782: write oid 15 current snap is 477 2024-04-12T11:30:28.997 INFO:tasks.rados.rados.0.smithi029.stdout:3781: finishing write tid 1 to smithi02979277-11 2024-04-12T11:30:28.997 INFO:tasks.rados.rados.0.smithi029.stdout:3781: finishing write tid 2 to smithi02979277-11 2024-04-12T11:30:28.997 INFO:tasks.rados.rados.0.smithi029.stdout:3782: seq_num 1423 ranges {716361=437338,1687053=687490,2777507=425839,3877756=38605} 2024-04-12T11:30:29.003 INFO:tasks.rados.rados.0.smithi029.stdout:3782: writing smithi02979277-15 from 716361 to 1153699 tid 1 2024-04-12T11:30:29.010 INFO:tasks.rados.rados.0.smithi029.stdout:3782: writing smithi02979277-15 from 1687053 to 2374543 tid 2 2024-04-12T11:30:29.014 INFO:tasks.rados.rados.0.smithi029.stdout:3782: writing smithi02979277-15 from 2777507 to 3203346 tid 3 2024-04-12T11:30:29.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:28 smithi029 ceph-mon[97787]: pgmap v1170: 89 pgs: 89 active+clean; 230 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 414 KiB/s wr, 6 op/s 2024-04-12T11:30:29.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:28 smithi029 ceph-mon[97787]: osdmap e1024: 8 total, 8 up, 8 in 2024-04-12T11:30:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:28 smithi029 ceph-mon[93650]: pgmap v1170: 89 pgs: 89 active+clean; 230 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 414 KiB/s wr, 6 op/s 2024-04-12T11:30:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:28 smithi029 ceph-mon[93650]: osdmap e1024: 8 total, 8 up, 8 in 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3782: writing smithi02979277-15 from 3877756 to 3916361 tid 4 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3783: snap_create 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3781: finishing write tid 3 to smithi02979277-11 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3781: finishing write tid 4 to smithi02979277-11 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3781: finishing write tid 5 to smithi02979277-11 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3781: oid 11 updating version 0 to 2476 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3781: oid 11 version 2476 is already newer than 2474 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3781: oid 11 version 2476 is already newer than 2475 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3781: oid 11 version 2476 is already newer than 2473 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 2476 (ObjNum 1422 snap 477 seq_num 1422) dirty exists 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3781: left oid 11 (ObjNum 1422 snap 477 seq_num 1422) 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3782: finishing write tid 1 to smithi02979277-15 2024-04-12T11:30:29.019 INFO:tasks.rados.rados.0.smithi029.stdout:3782: finishing write tid 2 to smithi02979277-15 2024-04-12T11:30:29.020 INFO:tasks.rados.rados.0.smithi029.stdout:3782: finishing write tid 3 to smithi02979277-15 2024-04-12T11:30:29.021 INFO:tasks.rados.rados.0.smithi029.stdout:3782: finishing write tid 4 to smithi02979277-15 2024-04-12T11:30:29.021 INFO:tasks.rados.rados.0.smithi029.stdout:3782: finishing write tid 5 to smithi02979277-15 2024-04-12T11:30:29.021 INFO:tasks.rados.rados.0.smithi029.stdout:3782: finishing write tid 6 to smithi02979277-15 2024-04-12T11:30:29.021 INFO:tasks.rados.rados.0.smithi029.stdout:3782: oid 15 updating version 0 to 1210 2024-04-12T11:30:29.021 INFO:tasks.rados.rados.0.smithi029.stdout:3782: oid 15 updating version 1210 to 1211 2024-04-12T11:30:29.021 INFO:tasks.rados.rados.0.smithi029.stdout:3782: oid 15 updating version 1211 to 1212 2024-04-12T11:30:29.021 INFO:tasks.rados.rados.0.smithi029.stdout:3782: oid 15 updating version 1212 to 1214 2024-04-12T11:30:29.021 INFO:tasks.rados.rados.0.smithi029.stdout:3782: oid 15 version 1214 is already newer than 1213 2024-04-12T11:30:29.021 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 1214 (ObjNum 1423 snap 477 seq_num 1423) dirty exists 2024-04-12T11:30:29.021 INFO:tasks.rados.rados.0.smithi029.stdout:3782: left oid 15 (ObjNum 1423 snap 477 seq_num 1423) 2024-04-12T11:30:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:28 smithi087 ceph-mon[89014]: pgmap v1170: 89 pgs: 89 active+clean; 230 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 414 KiB/s wr, 6 op/s 2024-04-12T11:30:29.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:28 smithi087 ceph-mon[89014]: osdmap e1024: 8 total, 8 up, 8 in 2024-04-12T11:30:29.764 INFO:tasks.rados.rados.0.smithi029.stdout:3781: done (2 left) 2024-04-12T11:30:29.764 INFO:tasks.rados.rados.0.smithi029.stdout:3782: done (1 left) 2024-04-12T11:30:29.764 INFO:tasks.rados.rados.0.smithi029.stdout:3783: done (0 left) 2024-04-12T11:30:29.765 INFO:tasks.rados.rados.0.smithi029.stdout:3784: read oid 30 snap 471 2024-04-12T11:30:29.765 INFO:tasks.rados.rados.0.smithi029.stdout:3784: expect deleted 2024-04-12T11:30:29.765 INFO:tasks.rados.rados.0.smithi029.stdout:3785: write oid 29 current snap is 478 2024-04-12T11:30:29.765 INFO:tasks.rados.rados.0.smithi029.stdout:3785: seq_num 1424 ranges {733168=512587,2029182=547501,3088577=637111} 2024-04-12T11:30:29.771 INFO:tasks.rados.rados.0.smithi029.stdout:3785: writing smithi02979277-29 from 733168 to 1245755 tid 1 2024-04-12T11:30:29.778 INFO:tasks.rados.rados.0.smithi029.stdout:3785: writing smithi02979277-29 from 2029182 to 2576683 tid 2 2024-04-12T11:30:29.785 INFO:tasks.rados.rados.0.smithi029.stdout:3785: writing smithi02979277-29 from 3088577 to 3725688 tid 3 2024-04-12T11:30:29.785 INFO:tasks.rados.rados.0.smithi029.stdout:3786: snap_remove snap 476 2024-04-12T11:30:30.771 INFO:tasks.rados.rados.0.smithi029.stdout:3786: done (2 left) 2024-04-12T11:30:30.771 INFO:tasks.rados.rados.0.smithi029.stdout:3787: write oid 11 current snap is 478 2024-04-12T11:30:30.771 INFO:tasks.rados.rados.0.smithi029.stdout:3787: seq_num 1425 ranges {749975=587836,2087664=691159,3516000=433975} 2024-04-12T11:30:30.778 INFO:tasks.rados.rados.0.smithi029.stdout:3787: writing smithi02979277-11 from 749975 to 1337811 tid 1 2024-04-12T11:30:30.786 INFO:tasks.rados.rados.0.smithi029.stdout:3787: writing smithi02979277-11 from 2087664 to 2778823 tid 2 2024-04-12T11:30:30.793 INFO:tasks.rados.rados.0.smithi029.stdout:3787: writing smithi02979277-11 from 3516000 to 3949975 tid 3 2024-04-12T11:30:30.793 INFO:tasks.rados.rados.0.smithi029.stdout:3788: write oid 35 current snap is 478 2024-04-12T11:30:30.793 INFO:tasks.rados.rados.0.smithi029.stdout:3788: seq_num 1426 ranges {766782=663085,2146146=551170,3143423=776008} 2024-04-12T11:30:30.800 INFO:tasks.rados.rados.0.smithi029.stdout:3788: writing smithi02979277-35 from 766782 to 1429867 tid 1 2024-04-12T11:30:30.807 INFO:tasks.rados.rados.0.smithi029.stdout:3788: writing smithi02979277-35 from 2146146 to 2697316 tid 2 2024-04-12T11:30:30.815 INFO:tasks.rados.rados.0.smithi029.stdout:3788: writing smithi02979277-35 from 3143423 to 3919431 tid 3 2024-04-12T11:30:30.815 INFO:tasks.rados.rados.0.smithi029.stdout:3789: write oid 25 current snap is 478 2024-04-12T11:30:30.815 INFO:tasks.rados.rados.0.smithi029.stdout:3785: finishing write tid 1 to smithi02979277-29 2024-04-12T11:30:30.815 INFO:tasks.rados.rados.0.smithi029.stdout:3785: finishing write tid 2 to smithi02979277-29 2024-04-12T11:30:30.815 INFO:tasks.rados.rados.0.smithi029.stdout:3789: seq_num 1427 ranges {783589=738334,2204628=694828,3570846=412743} 2024-04-12T11:30:30.823 INFO:tasks.rados.rados.0.smithi029.stdout:3789: writing smithi02979277-25 from 783589 to 1521923 tid 1 2024-04-12T11:30:30.830 INFO:tasks.rados.rados.0.smithi029.stdout:3789: writing smithi02979277-25 from 2204628 to 2899456 tid 2 2024-04-12T11:30:30.837 INFO:tasks.rados.rados.0.smithi029.stdout:3789: writing smithi02979277-25 from 3570846 to 3983589 tid 3 2024-04-12T11:30:30.837 INFO:tasks.rados.rados.0.smithi029.stdout:3784: done (4 left) 2024-04-12T11:30:30.837 INFO:tasks.rados.rados.0.smithi029.stdout:3790: rollback oid 39 current snap is 478 2024-04-12T11:30:30.837 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 474 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3785: finishing write tid 3 to smithi02979277-29 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3785: finishing write tid 4 to smithi02979277-29 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3785: finishing write tid 5 to smithi02979277-29 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3785: oid 29 updating version 0 to 1728 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3785: oid 29 updating version 1728 to 1730 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3785: oid 29 version 1730 is already newer than 1729 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3785: oid 29 updating version 1730 to 1731 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1731 (ObjNum 1424 snap 478 seq_num 1424) dirty exists 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3785: left oid 29 (ObjNum 1424 snap 478 seq_num 1424) 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3787: finishing write tid 1 to smithi02979277-11 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3787: finishing write tid 2 to smithi02979277-11 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3787: finishing write tid 3 to smithi02979277-11 2024-04-12T11:30:30.838 INFO:tasks.rados.rados.0.smithi029.stdout:3787: finishing write tid 4 to smithi02979277-11 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3787: finishing write tid 5 to smithi02979277-11 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3787: oid 11 updating version 0 to 2483 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3787: oid 11 version 2483 is already newer than 2481 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3787: oid 11 version 2483 is already newer than 2478 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3787: oid 11 version 2483 is already newer than 2482 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 11 v 2483 (ObjNum 1425 snap 478 seq_num 1425) dirty exists 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3787: left oid 11 (ObjNum 1425 snap 478 seq_num 1425) 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3788: finishing write tid 1 to smithi02979277-35 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3788: finishing write tid 2 to smithi02979277-35 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3788: finishing write tid 3 to smithi02979277-35 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3788: finishing write tid 4 to smithi02979277-35 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3788: finishing write tid 5 to smithi02979277-35 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3788: oid 35 updating version 0 to 2486 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3788: oid 35 updating version 2486 to 2487 2024-04-12T11:30:30.839 INFO:tasks.rados.rados.0.smithi029.stdout:3788: oid 35 version 2487 is already newer than 2484 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:3788: oid 35 version 2487 is already newer than 2485 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 35 v 2487 (ObjNum 1426 snap 478 seq_num 1426) dirty exists 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:3788: left oid 35 (ObjNum 1426 snap 478 seq_num 1426) 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:3789: finishing write tid 1 to smithi02979277-25 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:3789: finishing write tid 2 to smithi02979277-25 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:3785: done (4 left) 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:3787: done (3 left) 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:3788: done (2 left) 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:3791: delete oid 12 current snap is 478 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:3791: done (2 left) 2024-04-12T11:30:30.840 INFO:tasks.rados.rados.0.smithi029.stdout:3792: write oid 22 current snap is 478 2024-04-12T11:30:30.841 INFO:tasks.rados.rados.0.smithi029.stdout:3792: seq_num 1429 ranges {417203=488832,1405239=611964} 2024-04-12T11:30:30.845 INFO:tasks.rados.rados.0.smithi029.stdout:3792: writing smithi02979277-22 from 417203 to 906035 tid 1 2024-04-12T11:30:30.852 INFO:tasks.rados.rados.0.smithi029.stdout:3792: writing smithi02979277-22 from 1405239 to 2017203 tid 2 2024-04-12T11:30:30.852 INFO:tasks.rados.rados.0.smithi029.stdout:3793: rollback oid 50 current snap is 478 2024-04-12T11:30:30.852 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 50 to 477 2024-04-12T11:30:30.852 INFO:tasks.rados.rados.0.smithi029.stdout:3790: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:30:30.852 INFO:tasks.rados.rados.0.smithi029.stdout:3789: finishing write tid 3 to smithi02979277-25 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3789: finishing write tid 4 to smithi02979277-25 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3789: finishing write tid 5 to smithi02979277-25 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3789: oid 25 updating version 0 to 2031 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3789: oid 25 version 2031 is already newer than 2030 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3789: oid 25 version 2031 is already newer than 2029 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3789: oid 25 version 2031 is already newer than 2028 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 2031 (ObjNum 1427 snap 478 seq_num 1427) dirty exists 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3789: left oid 25 (ObjNum 1427 snap 478 seq_num 1427) 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3790: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3790: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 562 (ObjNum 1088 snap 365 seq_num 1088) dirty exists 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3789: done (3 left) 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3790: done (2 left) 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3794: read oid 11 snap 474 2024-04-12T11:30:30.853 INFO:tasks.rados.rados.0.smithi029.stdout:3794: expect (ObjNum 1410 snap 472 seq_num 1410) 2024-04-12T11:30:30.854 INFO:tasks.rados.rados.0.smithi029.stdout:3795: read oid 8 snap -1 2024-04-12T11:30:30.854 INFO:tasks.rados.rados.0.smithi029.stdout:3795: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:30:30.854 INFO:tasks.rados.rados.0.smithi029.stdout:3796: rollback oid 31 current snap is 478 2024-04-12T11:30:30.854 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 31 to 469 2024-04-12T11:30:30.854 INFO:tasks.rados.rados.0.smithi029.stdout:3792: finishing write tid 1 to smithi02979277-22 2024-04-12T11:30:30.854 INFO:tasks.rados.rados.0.smithi029.stdout:3797: read oid 9 snap 477 2024-04-12T11:30:30.854 INFO:tasks.rados.rados.0.smithi029.stdout:3797: expect (ObjNum 1405 snap 472 seq_num 1405) 2024-04-12T11:30:30.854 INFO:tasks.rados.rados.0.smithi029.stdout:3798: snap_create 2024-04-12T11:30:30.857 INFO:tasks.rados.rados.0.smithi029.stdout:3796: finishing rollback tid 0 to smithi02979277-31 2024-04-12T11:30:30.857 INFO:tasks.rados.rados.0.smithi029.stdout:3796: finishing rollback tid 1 to smithi02979277-31 2024-04-12T11:30:30.857 INFO:tasks.rados.rados.0.smithi029.stdout:3797: expect (ObjNum 1405 snap 472 seq_num 1405) 2024-04-12T11:30:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:30 smithi029 ceph-mon[93650]: pgmap v1173: 89 pgs: 89 active+clean; 242 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.4 MiB/s wr, 14 op/s 2024-04-12T11:30:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:30 smithi029 ceph-mon[93650]: osdmap e1025: 8 total, 8 up, 8 in 2024-04-12T11:30:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:30 smithi029 ceph-mon[97787]: pgmap v1173: 89 pgs: 89 active+clean; 242 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.4 MiB/s wr, 14 op/s 2024-04-12T11:30:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:30 smithi029 ceph-mon[97787]: osdmap e1025: 8 total, 8 up, 8 in 2024-04-12T11:30:31.107 INFO:tasks.rados.rados.0.smithi029.stdout:3796: finishing rollback tid 2 to smithi02979277-31 2024-04-12T11:30:31.107 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1509 (ObjNum 1341 snap 453 seq_num 1341) dirty exists 2024-04-12T11:30:31.107 INFO:tasks.rados.rados.0.smithi029.stdout:3794: expect (ObjNum 1410 snap 472 seq_num 1410) 2024-04-12T11:30:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:30 smithi087 ceph-mon[89014]: pgmap v1173: 89 pgs: 89 active+clean; 242 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.4 MiB/s wr, 14 op/s 2024-04-12T11:30:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:30 smithi087 ceph-mon[89014]: osdmap e1025: 8 total, 8 up, 8 in 2024-04-12T11:30:31.277 INFO:tasks.rados.rados.0.smithi029.stdout:3792: finishing write tid 2 to smithi02979277-22 2024-04-12T11:30:31.277 INFO:tasks.rados.rados.0.smithi029.stdout:3792: finishing write tid 3 to smithi02979277-22 2024-04-12T11:30:31.277 INFO:tasks.rados.rados.0.smithi029.stdout:3792: finishing write tid 4 to smithi02979277-22 2024-04-12T11:30:31.277 INFO:tasks.rados.rados.0.smithi029.stdout:3792: oid 22 updating version 0 to 1628 2024-04-12T11:30:31.278 INFO:tasks.rados.rados.0.smithi029.stdout:3792: oid 22 version 1628 is already newer than 1627 2024-04-12T11:30:31.278 INFO:tasks.rados.rados.0.smithi029.stdout:3792: oid 22 version 1628 is already newer than 1626 2024-04-12T11:30:31.278 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 22 v 1628 (ObjNum 1429 snap 478 seq_num 1429) dirty exists 2024-04-12T11:30:31.278 INFO:tasks.rados.rados.0.smithi029.stdout:3792: left oid 22 (ObjNum 1429 snap 478 seq_num 1429) 2024-04-12T11:30:31.278 INFO:tasks.rados.rados.0.smithi029.stdout:3793: finishing rollback tid 0 to smithi02979277-50 2024-04-12T11:30:31.278 INFO:tasks.rados.rados.0.smithi029.stdout:3793: finishing rollback tid 1 to smithi02979277-50 2024-04-12T11:30:31.278 INFO:tasks.rados.rados.0.smithi029.stdout:3793: finishing rollback tid 2 to smithi02979277-50 2024-04-12T11:30:31.278 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1632 (ObjNum 1309 snap 445 seq_num 1309) dirty exists 2024-04-12T11:30:31.278 INFO:tasks.rados.rados.0.smithi029.stdout:3795: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:30:31.775 INFO:tasks.rados.rados.0.smithi029.stdout:3792: done (6 left) 2024-04-12T11:30:31.776 INFO:tasks.rados.rados.0.smithi029.stdout:3793: done (5 left) 2024-04-12T11:30:31.776 INFO:tasks.rados.rados.0.smithi029.stdout:3794: done (4 left) 2024-04-12T11:30:31.776 INFO:tasks.rados.rados.0.smithi029.stdout:3795: done (3 left) 2024-04-12T11:30:31.776 INFO:tasks.rados.rados.0.smithi029.stdout:3796: done (2 left) 2024-04-12T11:30:31.776 INFO:tasks.rados.rados.0.smithi029.stdout:3797: done (1 left) 2024-04-12T11:30:31.776 INFO:tasks.rados.rados.0.smithi029.stdout:3798: done (0 left) 2024-04-12T11:30:31.776 INFO:tasks.rados.rados.0.smithi029.stdout:3799: snap_create 2024-04-12T11:30:32.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:31 smithi087 ceph-mon[89014]: osdmap e1026: 8 total, 8 up, 8 in 2024-04-12T11:30:32.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:31 smithi029 ceph-mon[93650]: osdmap e1026: 8 total, 8 up, 8 in 2024-04-12T11:30:32.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:31 smithi029 ceph-mon[97787]: osdmap e1026: 8 total, 8 up, 8 in 2024-04-12T11:30:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:30:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:30:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:30:32.728 INFO:tasks.rados.rados.0.smithi029.stdout:3799: done (0 left) 2024-04-12T11:30:32.728 INFO:tasks.rados.rados.0.smithi029.stdout:3800: delete oid 32 current snap is 480 2024-04-12T11:30:32.735 INFO:tasks.rados.rados.0.smithi029.stdout:3800: done (0 left) 2024-04-12T11:30:32.735 INFO:tasks.rados.rados.0.smithi029.stdout:3801: write oid 40 current snap is 480 2024-04-12T11:30:32.735 INFO:tasks.rados.rados.0.smithi029.stdout:3801: seq_num 1431 ranges {450817=755683,1638556=412261} 2024-04-12T11:30:32.742 INFO:tasks.rados.rados.0.smithi029.stdout:3801: writing smithi02979277-40 from 450817 to 1206500 tid 1 2024-04-12T11:30:32.746 INFO:tasks.rados.rados.0.smithi029.stdout:3801: writing smithi02979277-40 from 1638556 to 2050817 tid 2 2024-04-12T11:30:32.747 INFO:tasks.rados.rados.0.smithi029.stdout:3802: snap_create 2024-04-12T11:30:32.750 INFO:tasks.rados.rados.0.smithi029.stdout:3801: finishing write tid 1 to smithi02979277-40 2024-04-12T11:30:32.751 INFO:tasks.rados.rados.0.smithi029.stdout:3801: finishing write tid 2 to smithi02979277-40 2024-04-12T11:30:32.751 INFO:tasks.rados.rados.0.smithi029.stdout:3801: finishing write tid 3 to smithi02979277-40 2024-04-12T11:30:32.751 INFO:tasks.rados.rados.0.smithi029.stdout:3801: finishing write tid 4 to smithi02979277-40 2024-04-12T11:30:32.751 INFO:tasks.rados.rados.0.smithi029.stdout:3801: oid 40 updating version 0 to 2488 2024-04-12T11:30:32.751 INFO:tasks.rados.rados.0.smithi029.stdout:3801: oid 40 updating version 2488 to 2490 2024-04-12T11:30:32.751 INFO:tasks.rados.rados.0.smithi029.stdout:3801: oid 40 version 2490 is already newer than 2489 2024-04-12T11:30:32.751 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 2490 (ObjNum 1431 snap 480 seq_num 1431) dirty exists 2024-04-12T11:30:32.751 INFO:tasks.rados.rados.0.smithi029.stdout:3801: left oid 40 (ObjNum 1431 snap 480 seq_num 1431) 2024-04-12T11:30:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:32 smithi087 ceph-mon[89014]: pgmap v1176: 89 pgs: 89 active+clean; 242 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.0 MiB/s wr, 12 op/s 2024-04-12T11:30:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:32 smithi087 ceph-mon[89014]: osdmap e1027: 8 total, 8 up, 8 in 2024-04-12T11:30:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:32 smithi087 ceph-mon[89014]: osdmap e1028: 8 total, 8 up, 8 in 2024-04-12T11:30:33.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:32 smithi029 ceph-mon[93650]: pgmap v1176: 89 pgs: 89 active+clean; 242 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.0 MiB/s wr, 12 op/s 2024-04-12T11:30:33.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:32 smithi029 ceph-mon[93650]: osdmap e1027: 8 total, 8 up, 8 in 2024-04-12T11:30:33.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:32 smithi029 ceph-mon[93650]: osdmap e1028: 8 total, 8 up, 8 in 2024-04-12T11:30:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:32 smithi029 ceph-mon[97787]: pgmap v1176: 89 pgs: 89 active+clean; 242 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.0 MiB/s wr, 12 op/s 2024-04-12T11:30:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:32 smithi029 ceph-mon[97787]: osdmap e1027: 8 total, 8 up, 8 in 2024-04-12T11:30:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:32 smithi029 ceph-mon[97787]: osdmap e1028: 8 total, 8 up, 8 in 2024-04-12T11:30:33.730 INFO:tasks.rados.rados.0.smithi029.stdout:3801: done (1 left) 2024-04-12T11:30:33.730 INFO:tasks.rados.rados.0.smithi029.stdout:3802: done (0 left) 2024-04-12T11:30:33.730 INFO:tasks.rados.rados.0.smithi029.stdout:3803: delete oid 17 current snap is 481 2024-04-12T11:30:33.735 INFO:tasks.rados.rados.0.smithi029.stdout:3803: done (0 left) 2024-04-12T11:30:33.735 INFO:tasks.rados.rados.0.smithi029.stdout:3804: delete oid 17 current snap is 481 2024-04-12T11:30:33.738 INFO:tasks.rados.rados.0.smithi029.stdout:3804: done (0 left) 2024-04-12T11:30:33.738 INFO:tasks.rados.rados.0.smithi029.stdout:3805: read oid 32 snap -1 2024-04-12T11:30:33.738 INFO:tasks.rados.rados.0.smithi029.stdout:3805: expect deleted 2024-04-12T11:30:33.738 INFO:tasks.rados.rados.0.smithi029.stdout:3806: rollback oid 27 current snap is 481 2024-04-12T11:30:33.738 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 471 2024-04-12T11:30:33.738 INFO:tasks.rados.rados.0.smithi029.stdout:3807: write oid 28 current snap is 481 2024-04-12T11:30:33.738 INFO:tasks.rados.rados.0.smithi029.stdout:3807: seq_num 1434 ranges {501238=581430,1697649=403589} 2024-04-12T11:30:33.744 INFO:tasks.rados.rados.0.smithi029.stdout:3807: writing smithi02979277-28 from 501238 to 1082668 tid 1 2024-04-12T11:30:33.750 INFO:tasks.rados.rados.0.smithi029.stdout:3807: writing smithi02979277-28 from 1697649 to 2101238 tid 2 2024-04-12T11:30:33.750 INFO:tasks.rados.rados.0.smithi029.stdout:3808: read oid 38 snap -1 2024-04-12T11:30:33.750 INFO:tasks.rados.rados.0.smithi029.stdout:3808: expect (ObjNum 1299 snap 444 seq_num 1299) 2024-04-12T11:30:33.750 INFO:tasks.rados.rados.0.smithi029.stdout:3809: snap_remove snap 479 2024-04-12T11:30:34.736 INFO:tasks.rados.rados.0.smithi029.stdout:3809: done (4 left) 2024-04-12T11:30:34.736 INFO:tasks.rados.rados.0.smithi029.stdout:3810: read oid 34 snap -1 2024-04-12T11:30:34.736 INFO:tasks.rados.rados.0.smithi029.stdout:3810: expect (ObjNum 1420 snap 476 seq_num 1420) 2024-04-12T11:30:34.736 INFO:tasks.rados.rados.0.smithi029.stdout:3811: read oid 44 snap 471 2024-04-12T11:30:34.736 INFO:tasks.rados.rados.0.smithi029.stdout:3811: expect (ObjNum 1336 snap 453 seq_num 1336) 2024-04-12T11:30:34.736 INFO:tasks.rados.rados.0.smithi029.stdout:3812: write oid 17 current snap is 481 2024-04-12T11:30:34.736 INFO:tasks.rados.rados.0.smithi029.stdout:3812: seq_num 1435 ranges {518045=656679,1756131=361914} 2024-04-12T11:30:34.743 INFO:tasks.rados.rados.0.smithi029.stdout:3812: writing smithi02979277-17 from 518045 to 1174724 tid 1 2024-04-12T11:30:34.749 INFO:tasks.rados.rados.0.smithi029.stdout:3812: writing smithi02979277-17 from 1756131 to 2118045 tid 2 2024-04-12T11:30:34.750 INFO:tasks.rados.rados.0.smithi029.stdout:3813: write oid 41 current snap is 481 2024-04-12T11:30:34.750 INFO:tasks.rados.rados.0.smithi029.stdout:3813: seq_num 1436 ranges {534852=731928,1814613=320239} 2024-04-12T11:30:34.756 INFO:tasks.rados.rados.0.smithi029.stdout:3813: writing smithi02979277-41 from 534852 to 1266780 tid 1 2024-04-12T11:30:34.762 INFO:tasks.rados.rados.0.smithi029.stdout:3813: writing smithi02979277-41 from 1814613 to 2134852 tid 2 2024-04-12T11:30:34.762 INFO:tasks.rados.rados.0.smithi029.stdout:3814: delete oid 14 current snap is 481 2024-04-12T11:30:34.762 INFO:tasks.rados.rados.0.smithi029.stdout:3806: finishing rollback tid 0 to smithi02979277-27 2024-04-12T11:30:34.762 INFO:tasks.rados.rados.0.smithi029.stdout:3806: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:30:34.762 INFO:tasks.rados.rados.0.smithi029.stdout:3806: finishing rollback tid 2 to smithi02979277-27 2024-04-12T11:30:34.762 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 1218 (ObjNum 1401 snap 470 seq_num 1401) dirty exists 2024-04-12T11:30:34.762 INFO:tasks.rados.rados.0.smithi029.stdout:3807: finishing write tid 1 to smithi02979277-28 2024-04-12T11:30:34.762 INFO:tasks.rados.rados.0.smithi029.stdout:3807: finishing write tid 2 to smithi02979277-28 2024-04-12T11:30:34.763 INFO:tasks.rados.rados.0.smithi029.stdout:3807: finishing write tid 3 to smithi02979277-28 2024-04-12T11:30:34.763 INFO:tasks.rados.rados.0.smithi029.stdout:3807: finishing write tid 4 to smithi02979277-28 2024-04-12T11:30:34.763 INFO:tasks.rados.rados.0.smithi029.stdout:3807: oid 28 updating version 0 to 1512 2024-04-12T11:30:34.763 INFO:tasks.rados.rados.0.smithi029.stdout:3807: oid 28 version 1512 is already newer than 1510 2024-04-12T11:30:34.763 INFO:tasks.rados.rados.0.smithi029.stdout:3807: oid 28 version 1512 is already newer than 1511 2024-04-12T11:30:34.763 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 28 v 1512 (ObjNum 1434 snap 481 seq_num 1434) dirty exists 2024-04-12T11:30:34.763 INFO:tasks.rados.rados.0.smithi029.stdout:3807: left oid 28 (ObjNum 1434 snap 481 seq_num 1434) 2024-04-12T11:30:34.763 INFO:tasks.rados.rados.0.smithi029.stdout:3808: expect (ObjNum 1299 snap 444 seq_num 1299) 2024-04-12T11:30:34.997 INFO:tasks.rados.rados.0.smithi029.stdout:3810: expect (ObjNum 1420 snap 476 seq_num 1420) 2024-04-12T11:30:35.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:30:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:30:34] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-12T11:30:35.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:34 smithi029 ceph-mon[97787]: pgmap v1179: 89 pgs: 89 active+clean; 244 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:30:35.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:34 smithi029 ceph-mon[97787]: osdmap e1029: 8 total, 8 up, 8 in 2024-04-12T11:30:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:34 smithi029 ceph-mon[93650]: pgmap v1179: 89 pgs: 89 active+clean; 244 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:30:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:34 smithi029 ceph-mon[93650]: osdmap e1029: 8 total, 8 up, 8 in 2024-04-12T11:30:35.129 INFO:tasks.rados.rados.0.smithi029.stdout:3811: expect (ObjNum 1336 snap 453 seq_num 1336) 2024-04-12T11:30:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:34 smithi087 ceph-mon[89014]: pgmap v1179: 89 pgs: 89 active+clean; 244 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:30:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:34 smithi087 ceph-mon[89014]: osdmap e1029: 8 total, 8 up, 8 in 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3812: finishing write tid 1 to smithi02979277-17 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3812: finishing write tid 2 to smithi02979277-17 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3812: finishing write tid 3 to smithi02979277-17 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3812: finishing write tid 4 to smithi02979277-17 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3812: oid 17 updating version 0 to 2037 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3812: oid 17 version 2037 is already newer than 2036 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3812: oid 17 version 2037 is already newer than 2035 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 2037 (ObjNum 1435 snap 481 seq_num 1435) dirty exists 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3812: left oid 17 (ObjNum 1435 snap 481 seq_num 1435) 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3813: finishing write tid 1 to smithi02979277-41 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3805: done (8 left) 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3806: done (7 left) 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3807: done (6 left) 2024-04-12T11:30:35.301 INFO:tasks.rados.rados.0.smithi029.stdout:3808: done (5 left) 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3810: done (4 left) 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3811: done (3 left) 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3812: done (2 left) 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3814: done (1 left) 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3815: delete oid 17 current snap is 481 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3813: finishing write tid 2 to smithi02979277-41 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3813: finishing write tid 3 to smithi02979277-41 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3813: finishing write tid 4 to smithi02979277-41 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3813: oid 41 updating version 0 to 1904 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3813: oid 41 updating version 1904 to 1905 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3813: oid 41 version 1905 is already newer than 1903 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1905 (ObjNum 1436 snap 481 seq_num 1436) dirty exists 2024-04-12T11:30:35.302 INFO:tasks.rados.rados.0.smithi029.stdout:3813: left oid 41 (ObjNum 1436 snap 481 seq_num 1436) 2024-04-12T11:30:35.303 INFO:tasks.rados.rados.0.smithi029.stdout:3813: done (1 left) 2024-04-12T11:30:35.303 INFO:tasks.rados.rados.0.smithi029.stdout:3815: done (0 left) 2024-04-12T11:30:35.303 INFO:tasks.rados.rados.0.smithi029.stdout:3816: snap_create 2024-04-12T11:30:35.738 INFO:tasks.rados.rados.0.smithi029.stdout:3816: done (0 left) 2024-04-12T11:30:35.739 INFO:tasks.rados.rados.0.smithi029.stdout:3817: snap_remove snap 474 2024-04-12T11:30:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:35 smithi029 ceph-mon[97787]: osdmap e1030: 8 total, 8 up, 8 in 2024-04-12T11:30:36.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:35 smithi029 ceph-mon[93650]: osdmap e1030: 8 total, 8 up, 8 in 2024-04-12T11:30:36.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:35 smithi087 ceph-mon[89014]: osdmap e1030: 8 total, 8 up, 8 in 2024-04-12T11:30:36.742 INFO:tasks.rados.rados.0.smithi029.stdout:3817: done (0 left) 2024-04-12T11:30:36.742 INFO:tasks.rados.rados.0.smithi029.stdout:3818: rollback oid 34 current snap is 482 2024-04-12T11:30:36.743 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 34 to 473 2024-04-12T11:30:36.743 INFO:tasks.rados.rados.0.smithi029.stdout:3819: delete oid 9 current snap is 482 2024-04-12T11:30:36.746 INFO:tasks.rados.rados.0.smithi029.stdout:3819: done (1 left) 2024-04-12T11:30:36.746 INFO:tasks.rados.rados.0.smithi029.stdout:3820: write oid 27 current snap is 482 2024-04-12T11:30:36.747 INFO:tasks.rados.rados.0.smithi029.stdout:3820: seq_num 1440 ranges {602080=749277,2048541=153539} 2024-04-12T11:30:36.755 INFO:tasks.rados.rados.0.smithi029.stdout:3820: writing smithi02979277-27 from 602080 to 1351357 tid 1 2024-04-12T11:30:36.760 INFO:tasks.rados.rados.0.smithi029.stdout:3820: writing smithi02979277-27 from 2048541 to 2202080 tid 2 2024-04-12T11:30:36.760 INFO:tasks.rados.rados.0.smithi029.stdout:3821: delete oid 44 current snap is 482 2024-04-12T11:30:36.760 INFO:tasks.rados.rados.0.smithi029.stdout:3818: finishing rollback tid 0 to smithi02979277-34 2024-04-12T11:30:36.760 INFO:tasks.rados.rados.0.smithi029.stdout:3818: finishing rollback tid 1 to smithi02979277-34 2024-04-12T11:30:36.760 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1740 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-04-12T11:30:36.760 INFO:tasks.rados.rados.0.smithi029.stdout:3820: finishing write tid 1 to smithi02979277-27 2024-04-12T11:30:36.762 INFO:tasks.rados.rados.0.smithi029.stdout:3818: done (2 left) 2024-04-12T11:30:36.762 INFO:tasks.rados.rados.0.smithi029.stdout:3821: done (1 left) 2024-04-12T11:30:36.762 INFO:tasks.rados.rados.0.smithi029.stdout:3822: delete oid 3 current snap is 482 2024-04-12T11:30:36.762 INFO:tasks.rados.rados.0.smithi029.stdout:3820: finishing write tid 2 to smithi02979277-27 2024-04-12T11:30:36.763 INFO:tasks.rados.rados.0.smithi029.stdout:3820: finishing write tid 3 to smithi02979277-27 2024-04-12T11:30:36.763 INFO:tasks.rados.rados.0.smithi029.stdout:3820: finishing write tid 4 to smithi02979277-27 2024-04-12T11:30:36.763 INFO:tasks.rados.rados.0.smithi029.stdout:3820: oid 27 updating version 0 to 1224 2024-04-12T11:30:36.763 INFO:tasks.rados.rados.0.smithi029.stdout:3820: oid 27 updating version 1224 to 1225 2024-04-12T11:30:36.763 INFO:tasks.rados.rados.0.smithi029.stdout:3820: oid 27 updating version 1225 to 1226 2024-04-12T11:30:36.763 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 1226 (ObjNum 1440 snap 482 seq_num 1440) dirty exists 2024-04-12T11:30:36.763 INFO:tasks.rados.rados.0.smithi029.stdout:3820: left oid 27 (ObjNum 1440 snap 482 seq_num 1440) 2024-04-12T11:30:36.765 INFO:tasks.rados.rados.0.smithi029.stdout:3820: done (1 left) 2024-04-12T11:30:36.765 INFO:tasks.rados.rados.0.smithi029.stdout:3822: done (0 left) 2024-04-12T11:30:36.765 INFO:tasks.rados.rados.0.smithi029.stdout:3823: read oid 3 snap -1 2024-04-12T11:30:36.766 INFO:tasks.rados.rados.0.smithi029.stdout:3823: expect deleted 2024-04-12T11:30:36.766 INFO:tasks.rados.rados.0.smithi029.stdout:3824: read oid 8 snap 477 2024-04-12T11:30:36.766 INFO:tasks.rados.rados.0.smithi029.stdout:3824: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:30:36.766 INFO:tasks.rados.rados.0.smithi029.stdout:3825: read oid 33 snap -1 2024-04-12T11:30:36.766 INFO:tasks.rados.rados.0.smithi029.stdout:3825: expect (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:30:36.766 INFO:tasks.rados.rados.0.smithi029.stdout:3826: write oid 19 current snap is 482 2024-04-12T11:30:36.766 INFO:tasks.rados.rados.0.smithi029.stdout:3826: seq_num 1443 ranges {652501=575024,1707634=544867} 2024-04-12T11:30:36.772 INFO:tasks.rados.rados.0.smithi029.stdout:3826: writing smithi02979277-19 from 652501 to 1227525 tid 1 2024-04-12T11:30:36.778 INFO:tasks.rados.rados.0.smithi029.stdout:3826: writing smithi02979277-19 from 1707634 to 2252501 tid 2 2024-04-12T11:30:36.778 INFO:tasks.rados.rados.0.smithi029.stdout:3827: snap_create 2024-04-12T11:30:36.779 INFO:tasks.rados.rados.0.smithi029.stdout:3825: expect (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:30:36.852 INFO:tasks.rados.rados.0.smithi029.stdout:3824: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:30:36.964 INFO:tasks.rados.rados.0.smithi029.stdout:3826: finishing write tid 1 to smithi02979277-19 2024-04-12T11:30:36.965 INFO:tasks.rados.rados.0.smithi029.stdout:3826: finishing write tid 2 to smithi02979277-19 2024-04-12T11:30:36.965 INFO:tasks.rados.rados.0.smithi029.stdout:3826: finishing write tid 3 to smithi02979277-19 2024-04-12T11:30:36.965 INFO:tasks.rados.rados.0.smithi029.stdout:3826: finishing write tid 4 to smithi02979277-19 2024-04-12T11:30:36.965 INFO:tasks.rados.rados.0.smithi029.stdout:3826: oid 19 updating version 0 to 1897 2024-04-12T11:30:36.965 INFO:tasks.rados.rados.0.smithi029.stdout:3826: oid 19 version 1897 is already newer than 1896 2024-04-12T11:30:36.965 INFO:tasks.rados.rados.0.smithi029.stdout:3826: oid 19 updating version 1897 to 1898 2024-04-12T11:30:36.965 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 19 v 1898 (ObjNum 1443 snap 482 seq_num 1443) dirty exists 2024-04-12T11:30:36.965 INFO:tasks.rados.rados.0.smithi029.stdout:3826: left oid 19 (ObjNum 1443 snap 482 seq_num 1443) 2024-04-12T11:30:37.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:36 smithi029 ceph-mon[93650]: pgmap v1182: 89 pgs: 89 active+clean; 249 MiB data, 889 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 23 op/s 2024-04-12T11:30:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:36 smithi029 ceph-mon[93650]: osdmap e1031: 8 total, 8 up, 8 in 2024-04-12T11:30:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:36 smithi029 ceph-mon[97787]: pgmap v1182: 89 pgs: 89 active+clean; 249 MiB data, 889 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 23 op/s 2024-04-12T11:30:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:36 smithi029 ceph-mon[97787]: osdmap e1031: 8 total, 8 up, 8 in 2024-04-12T11:30:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:36 smithi087 ceph-mon[89014]: pgmap v1182: 89 pgs: 89 active+clean; 249 MiB data, 889 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 23 op/s 2024-04-12T11:30:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:36 smithi087 ceph-mon[89014]: osdmap e1031: 8 total, 8 up, 8 in 2024-04-12T11:30:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:37.729 INFO:tasks.rados.rados.0.smithi029.stdout:3823: done (4 left) 2024-04-12T11:30:37.730 INFO:tasks.rados.rados.0.smithi029.stdout:3824: done (3 left) 2024-04-12T11:30:37.730 INFO:tasks.rados.rados.0.smithi029.stdout:3825: done (2 left) 2024-04-12T11:30:37.730 INFO:tasks.rados.rados.0.smithi029.stdout:3826: done (1 left) 2024-04-12T11:30:37.730 INFO:tasks.rados.rados.0.smithi029.stdout:3827: done (0 left) 2024-04-12T11:30:37.730 INFO:tasks.rados.rados.0.smithi029.stdout:3828: write oid 25 current snap is 483 2024-04-12T11:30:37.730 INFO:tasks.rados.rados.0.smithi029.stdout:3828: seq_num 1444 ranges {669308=650273,1766116=467838} 2024-04-12T11:30:37.736 INFO:tasks.rados.rados.0.smithi029.stdout:3828: writing smithi02979277-25 from 669308 to 1319581 tid 1 2024-04-12T11:30:37.742 INFO:tasks.rados.rados.0.smithi029.stdout:3828: writing smithi02979277-25 from 1766116 to 2233954 tid 2 2024-04-12T11:30:37.742 INFO:tasks.rados.rados.0.smithi029.stdout:3829: read oid 43 snap -1 2024-04-12T11:30:37.742 INFO:tasks.rados.rados.0.smithi029.stdout:3829: expect deleted 2024-04-12T11:30:37.742 INFO:tasks.rados.rados.0.smithi029.stdout:3830: snap_remove snap 482 2024-04-12T11:30:38.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:37 smithi029 ceph-mon[93650]: osdmap e1032: 8 total, 8 up, 8 in 2024-04-12T11:30:38.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:37 smithi029 ceph-mon[93650]: osdmap e1033: 8 total, 8 up, 8 in 2024-04-12T11:30:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:37 smithi029 ceph-mon[97787]: osdmap e1032: 8 total, 8 up, 8 in 2024-04-12T11:30:38.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:37 smithi029 ceph-mon[97787]: osdmap e1033: 8 total, 8 up, 8 in 2024-04-12T11:30:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:37 smithi087 ceph-mon[89014]: osdmap e1032: 8 total, 8 up, 8 in 2024-04-12T11:30:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:37 smithi087 ceph-mon[89014]: osdmap e1033: 8 total, 8 up, 8 in 2024-04-12T11:30:38.731 INFO:tasks.rados.rados.0.smithi029.stdout:3830: done (2 left) 2024-04-12T11:30:38.731 INFO:tasks.rados.rados.0.smithi029.stdout:3831: read oid 15 snap -1 2024-04-12T11:30:38.731 INFO:tasks.rados.rados.0.smithi029.stdout:3831: expect (ObjNum 1423 snap 477 seq_num 1423) 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3832: read oid 11 snap -1 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3832: expect (ObjNum 1425 snap 478 seq_num 1425) 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3833: delete oid 1 current snap is 483 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3828: finishing write tid 1 to smithi02979277-25 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3828: finishing write tid 2 to smithi02979277-25 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3828: finishing write tid 3 to smithi02979277-25 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3828: finishing write tid 4 to smithi02979277-25 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3828: oid 25 updating version 0 to 2048 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3828: oid 25 version 2048 is already newer than 2047 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3828: oid 25 version 2048 is already newer than 2046 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 2048 (ObjNum 1444 snap 483 seq_num 1444) dirty exists 2024-04-12T11:30:38.732 INFO:tasks.rados.rados.0.smithi029.stdout:3828: left oid 25 (ObjNum 1444 snap 483 seq_num 1444) 2024-04-12T11:30:38.736 INFO:tasks.rados.rados.0.smithi029.stdout:3828: done (4 left) 2024-04-12T11:30:38.736 INFO:tasks.rados.rados.0.smithi029.stdout:3829: done (3 left) 2024-04-12T11:30:38.736 INFO:tasks.rados.rados.0.smithi029.stdout:3833: done (2 left) 2024-04-12T11:30:38.737 INFO:tasks.rados.rados.0.smithi029.stdout:3834: write oid 13 current snap is 483 2024-04-12T11:30:38.737 INFO:tasks.rados.rados.0.smithi029.stdout:3834: seq_num 1446 ranges {702922=517124,1883080=419842} 2024-04-12T11:30:38.744 INFO:tasks.rados.rados.0.smithi029.stdout:3834: writing smithi02979277-13 from 702922 to 1220046 tid 1 2024-04-12T11:30:38.750 INFO:tasks.rados.rados.0.smithi029.stdout:3834: writing smithi02979277-13 from 1883080 to 2302922 tid 2 2024-04-12T11:30:38.750 INFO:tasks.rados.rados.0.smithi029.stdout:3835: snap_remove snap 466 2024-04-12T11:30:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:38 smithi029 ceph-mon[97787]: pgmap v1185: 89 pgs: 89 active+clean; 252 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 22 op/s 2024-04-12T11:30:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:38 smithi029 ceph-mon[97787]: osdmap e1034: 8 total, 8 up, 8 in 2024-04-12T11:30:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:38 smithi029 ceph-mon[93650]: pgmap v1185: 89 pgs: 89 active+clean; 252 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 22 op/s 2024-04-12T11:30:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:38 smithi029 ceph-mon[93650]: osdmap e1034: 8 total, 8 up, 8 in 2024-04-12T11:30:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:38 smithi087 ceph-mon[89014]: pgmap v1185: 89 pgs: 89 active+clean; 252 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 22 op/s 2024-04-12T11:30:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:38 smithi087 ceph-mon[89014]: osdmap e1034: 8 total, 8 up, 8 in 2024-04-12T11:30:39.757 INFO:tasks.rados.rados.0.smithi029.stdout:3835: done (3 left) 2024-04-12T11:30:39.758 INFO:tasks.rados.rados.0.smithi029.stdout:3836: read oid 12 snap 473 2024-04-12T11:30:39.758 INFO:tasks.rados.rados.0.smithi029.stdout:3836: expect (ObjNum 1296 snap 444 seq_num 1296) 2024-04-12T11:30:39.758 INFO:tasks.rados.rados.0.smithi029.stdout:3837: snap_create 2024-04-12T11:30:39.758 INFO:tasks.rados.rados.0.smithi029.stdout:3832: expect (ObjNum 1425 snap 478 seq_num 1425) 2024-04-12T11:30:40.030 INFO:tasks.rados.rados.0.smithi029.stdout:3831: expect (ObjNum 1423 snap 477 seq_num 1423) 2024-04-12T11:30:40.163 INFO:tasks.rados.rados.0.smithi029.stdout:3834: finishing write tid 1 to smithi02979277-13 2024-04-12T11:30:40.163 INFO:tasks.rados.rados.0.smithi029.stdout:3834: finishing write tid 2 to smithi02979277-13 2024-04-12T11:30:40.163 INFO:tasks.rados.rados.0.smithi029.stdout:3834: finishing write tid 3 to smithi02979277-13 2024-04-12T11:30:40.163 INFO:tasks.rados.rados.0.smithi029.stdout:3834: finishing write tid 4 to smithi02979277-13 2024-04-12T11:30:40.163 INFO:tasks.rados.rados.0.smithi029.stdout:3834: oid 13 updating version 0 to 2054 2024-04-12T11:30:40.163 INFO:tasks.rados.rados.0.smithi029.stdout:3834: oid 13 version 2054 is already newer than 2050 2024-04-12T11:30:40.163 INFO:tasks.rados.rados.0.smithi029.stdout:3834: oid 13 version 2054 is already newer than 2053 2024-04-12T11:30:40.163 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 2054 (ObjNum 1446 snap 483 seq_num 1446) dirty exists 2024-04-12T11:30:40.163 INFO:tasks.rados.rados.0.smithi029.stdout:3834: left oid 13 (ObjNum 1446 snap 483 seq_num 1446) 2024-04-12T11:30:40.163 INFO:tasks.rados.rados.0.smithi029.stdout:3836: expect (ObjNum 1296 snap 444 seq_num 1296) 2024-04-12T11:30:40.759 INFO:tasks.rados.rados.0.smithi029.stdout:3831: done (4 left) 2024-04-12T11:30:40.759 INFO:tasks.rados.rados.0.smithi029.stdout:3832: done (3 left) 2024-04-12T11:30:40.759 INFO:tasks.rados.rados.0.smithi029.stdout:3834: done (2 left) 2024-04-12T11:30:40.759 INFO:tasks.rados.rados.0.smithi029.stdout:3836: done (1 left) 2024-04-12T11:30:40.760 INFO:tasks.rados.rados.0.smithi029.stdout:3837: done (0 left) 2024-04-12T11:30:40.760 INFO:tasks.rados.rados.0.smithi029.stdout:3838: read oid 18 snap -1 2024-04-12T11:30:40.760 INFO:tasks.rados.rados.0.smithi029.stdout:3838: expect deleted 2024-04-12T11:30:40.760 INFO:tasks.rados.rados.0.smithi029.stdout:3839: snap_create 2024-04-12T11:30:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:40 smithi029 ceph-mon[93650]: pgmap v1188: 89 pgs: 89 active+clean; 257 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-04-12T11:30:41.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:40 smithi029 ceph-mon[93650]: osdmap e1035: 8 total, 8 up, 8 in 2024-04-12T11:30:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:40 smithi029 ceph-mon[97787]: pgmap v1188: 89 pgs: 89 active+clean; 257 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-04-12T11:30:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:40 smithi029 ceph-mon[97787]: osdmap e1035: 8 total, 8 up, 8 in 2024-04-12T11:30:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:40 smithi087 ceph-mon[89014]: pgmap v1188: 89 pgs: 89 active+clean; 257 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-04-12T11:30:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:40 smithi087 ceph-mon[89014]: osdmap e1035: 8 total, 8 up, 8 in 2024-04-12T11:30:41.762 INFO:tasks.rados.rados.0.smithi029.stdout:3838: done (1 left) 2024-04-12T11:30:41.762 INFO:tasks.rados.rados.0.smithi029.stdout:3839: done (0 left) 2024-04-12T11:30:41.762 INFO:tasks.rados.rados.0.smithi029.stdout:3840: write oid 20 current snap is 485 2024-04-12T11:30:41.762 INFO:tasks.rados.rados.0.smithi029.stdout:3840: seq_num 1447 ranges {719729=592373,1941562=378167} 2024-04-12T11:30:41.769 INFO:tasks.rados.rados.0.smithi029.stdout:3840: writing smithi02979277-20 from 719729 to 1312102 tid 1 2024-04-12T11:30:41.774 INFO:tasks.rados.rados.0.smithi029.stdout:3840: writing smithi02979277-20 from 1941562 to 2319729 tid 2 2024-04-12T11:30:41.774 INFO:tasks.rados.rados.0.smithi029.stdout:3841: snap_create 2024-04-12T11:30:41.777 INFO:tasks.rados.rados.0.smithi029.stdout:3840: finishing write tid 1 to smithi02979277-20 2024-04-12T11:30:41.778 INFO:tasks.rados.rados.0.smithi029.stdout:3840: finishing write tid 2 to smithi02979277-20 2024-04-12T11:30:41.778 INFO:tasks.rados.rados.0.smithi029.stdout:3840: finishing write tid 3 to smithi02979277-20 2024-04-12T11:30:41.778 INFO:tasks.rados.rados.0.smithi029.stdout:3840: finishing write tid 4 to smithi02979277-20 2024-04-12T11:30:41.778 INFO:tasks.rados.rados.0.smithi029.stdout:3840: oid 20 updating version 0 to 1272 2024-04-12T11:30:41.778 INFO:tasks.rados.rados.0.smithi029.stdout:3840: oid 20 updating version 1272 to 1273 2024-04-12T11:30:41.778 INFO:tasks.rados.rados.0.smithi029.stdout:3840: oid 20 updating version 1273 to 1274 2024-04-12T11:30:41.778 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 1274 (ObjNum 1447 snap 485 seq_num 1447) dirty exists 2024-04-12T11:30:41.778 INFO:tasks.rados.rados.0.smithi029.stdout:3840: left oid 20 (ObjNum 1447 snap 485 seq_num 1447) 2024-04-12T11:30:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:41 smithi029 ceph-mon[97787]: osdmap e1036: 8 total, 8 up, 8 in 2024-04-12T11:30:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:41 smithi029 ceph-mon[93650]: osdmap e1036: 8 total, 8 up, 8 in 2024-04-12T11:30:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:41 smithi087 ceph-mon[89014]: osdmap e1036: 8 total, 8 up, 8 in 2024-04-12T11:30:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:30:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:30:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:30:42.731 INFO:tasks.rados.rados.0.smithi029.stdout:3840: done (1 left) 2024-04-12T11:30:42.731 INFO:tasks.rados.rados.0.smithi029.stdout:3841: done (0 left) 2024-04-12T11:30:42.731 INFO:tasks.rados.rados.0.smithi029.stdout:3842: write oid 43 current snap is 486 2024-04-12T11:30:42.732 INFO:tasks.rados.rados.0.smithi029.stdout:3842: seq_num 1448 ranges {736536=667622,2000044=336492} 2024-04-12T11:30:42.739 INFO:tasks.rados.rados.0.smithi029.stdout:3842: writing smithi02979277-43 from 736536 to 1404158 tid 1 2024-04-12T11:30:42.745 INFO:tasks.rados.rados.0.smithi029.stdout:3842: writing smithi02979277-43 from 2000044 to 2336536 tid 2 2024-04-12T11:30:42.745 INFO:tasks.rados.rados.0.smithi029.stdout:3843: rollback oid 16 current snap is 486 2024-04-12T11:30:42.745 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 16 to 471 2024-04-12T11:30:42.745 INFO:tasks.rados.rados.0.smithi029.stdout:3842: finishing write tid 1 to smithi02979277-43 2024-04-12T11:30:42.745 INFO:tasks.rados.rados.0.smithi029.stdout:3844: rollback oid 31 current snap is 486 2024-04-12T11:30:42.745 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 31 to 473 2024-04-12T11:30:42.745 INFO:tasks.rados.rados.0.smithi029.stdout:3845: read oid 4 snap -1 2024-04-12T11:30:42.745 INFO:tasks.rados.rados.0.smithi029.stdout:3845: expect (ObjNum 1276 snap 436 seq_num 1276) 2024-04-12T11:30:42.745 INFO:tasks.rados.rados.0.smithi029.stdout:3846: write oid 29 current snap is 486 2024-04-12T11:30:42.745 INFO:tasks.rados.rados.0.smithi029.stdout:3846: seq_num 1449 ranges {753343=742871,1942173=411170} 2024-04-12T11:30:42.753 INFO:tasks.rados.rados.0.smithi029.stdout:3846: writing smithi02979277-29 from 753343 to 1496214 tid 1 2024-04-12T11:30:42.758 INFO:tasks.rados.rados.0.smithi029.stdout:3846: writing smithi02979277-29 from 1942173 to 2353343 tid 2 2024-04-12T11:30:42.758 INFO:tasks.rados.rados.0.smithi029.stdout:3847: write oid 44 current snap is 486 2024-04-12T11:30:42.759 INFO:tasks.rados.rados.0.smithi029.stdout:3844: finishing rollback tid 0 to smithi02979277-31 2024-04-12T11:30:42.759 INFO:tasks.rados.rados.0.smithi029.stdout:3847: seq_num 1450 ranges {770150=418120,1600655=478845} 2024-04-12T11:30:42.765 INFO:tasks.rados.rados.0.smithi029.stdout:3847: writing smithi02979277-44 from 770150 to 1188270 tid 1 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3847: writing smithi02979277-44 from 1600655 to 2079500 tid 2 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3848: read oid 33 snap -1 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3848: expect (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3849: snap_create 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3842: finishing write tid 2 to smithi02979277-43 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3842: finishing write tid 3 to smithi02979277-43 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3842: finishing write tid 4 to smithi02979277-43 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3842: oid 43 updating version 0 to 1745 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3842: oid 43 updating version 1745 to 1747 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3842: oid 43 version 1747 is already newer than 1746 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1747 (ObjNum 1448 snap 486 seq_num 1448) dirty exists 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3842: left oid 43 (ObjNum 1448 snap 486 seq_num 1448) 2024-04-12T11:30:42.771 INFO:tasks.rados.rados.0.smithi029.stdout:3844: finishing rollback tid 1 to smithi02979277-31 2024-04-12T11:30:42.772 INFO:tasks.rados.rados.0.smithi029.stdout:3844: finishing rollback tid 2 to smithi02979277-31 2024-04-12T11:30:42.772 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 31 v 1520 (ObjNum 1341 snap 453 seq_num 1341) dirty exists 2024-04-12T11:30:42.772 INFO:tasks.rados.rados.0.smithi029.stdout:3845: expect (ObjNum 1276 snap 436 seq_num 1276) 2024-04-12T11:30:42.923 INFO:tasks.rados.rados.0.smithi029.stdout:3843: finishing rollback tid 0 to smithi02979277-16 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3843: finishing rollback tid 1 to smithi02979277-16 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3843: finishing rollback tid 2 to smithi02979277-16 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1908 (ObjNum 1380 snap 466 seq_num 1380) dirty exists 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3846: finishing write tid 1 to smithi02979277-29 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3846: finishing write tid 2 to smithi02979277-29 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3846: finishing write tid 3 to smithi02979277-29 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3846: finishing write tid 4 to smithi02979277-29 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3846: oid 29 updating version 0 to 1750 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3846: oid 29 updating version 1750 to 1751 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3846: oid 29 version 1751 is already newer than 1749 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1751 (ObjNum 1449 snap 486 seq_num 1449) dirty exists 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3846: left oid 29 (ObjNum 1449 snap 486 seq_num 1449) 2024-04-12T11:30:42.924 INFO:tasks.rados.rados.0.smithi029.stdout:3847: finishing write tid 1 to smithi02979277-44 2024-04-12T11:30:42.925 INFO:tasks.rados.rados.0.smithi029.stdout:3847: finishing write tid 2 to smithi02979277-44 2024-04-12T11:30:42.925 INFO:tasks.rados.rados.0.smithi029.stdout:3847: finishing write tid 3 to smithi02979277-44 2024-04-12T11:30:42.925 INFO:tasks.rados.rados.0.smithi029.stdout:3847: finishing write tid 4 to smithi02979277-44 2024-04-12T11:30:42.925 INFO:tasks.rados.rados.0.smithi029.stdout:3847: oid 44 updating version 0 to 1922 2024-04-12T11:30:42.925 INFO:tasks.rados.rados.0.smithi029.stdout:3847: oid 44 updating version 1922 to 1923 2024-04-12T11:30:42.925 INFO:tasks.rados.rados.0.smithi029.stdout:3847: oid 44 updating version 1923 to 1924 2024-04-12T11:30:42.925 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1924 (ObjNum 1450 snap 486 seq_num 1450) dirty exists 2024-04-12T11:30:42.925 INFO:tasks.rados.rados.0.smithi029.stdout:3847: left oid 44 (ObjNum 1450 snap 486 seq_num 1450) 2024-04-12T11:30:42.925 INFO:tasks.rados.rados.0.smithi029.stdout:3848: expect (ObjNum 1320 snap 451 seq_num 1320) 2024-04-12T11:30:43.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:42 smithi029 ceph-mon[93650]: pgmap v1191: 89 pgs: 89 active+clean; 257 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 461 KiB/s wr, 8 op/s 2024-04-12T11:30:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:42 smithi029 ceph-mon[93650]: osdmap e1037: 8 total, 8 up, 8 in 2024-04-12T11:30:43.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:42 smithi029 ceph-mon[93650]: osdmap e1038: 8 total, 8 up, 8 in 2024-04-12T11:30:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:42 smithi029 ceph-mon[97787]: pgmap v1191: 89 pgs: 89 active+clean; 257 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 461 KiB/s wr, 8 op/s 2024-04-12T11:30:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:42 smithi029 ceph-mon[97787]: osdmap e1037: 8 total, 8 up, 8 in 2024-04-12T11:30:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:42 smithi029 ceph-mon[97787]: osdmap e1038: 8 total, 8 up, 8 in 2024-04-12T11:30:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:42 smithi087 ceph-mon[89014]: pgmap v1191: 89 pgs: 89 active+clean; 257 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 461 KiB/s wr, 8 op/s 2024-04-12T11:30:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:42 smithi087 ceph-mon[89014]: osdmap e1037: 8 total, 8 up, 8 in 2024-04-12T11:30:43.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:42 smithi087 ceph-mon[89014]: osdmap e1038: 8 total, 8 up, 8 in 2024-04-12T11:30:43.775 INFO:tasks.rados.rados.0.smithi029.stdout:3842: done (7 left) 2024-04-12T11:30:43.775 INFO:tasks.rados.rados.0.smithi029.stdout:3843: done (6 left) 2024-04-12T11:30:43.775 INFO:tasks.rados.rados.0.smithi029.stdout:3844: done (5 left) 2024-04-12T11:30:43.776 INFO:tasks.rados.rados.0.smithi029.stdout:3845: done (4 left) 2024-04-12T11:30:43.776 INFO:tasks.rados.rados.0.smithi029.stdout:3846: done (3 left) 2024-04-12T11:30:43.776 INFO:tasks.rados.rados.0.smithi029.stdout:3847: done (2 left) 2024-04-12T11:30:43.776 INFO:tasks.rados.rados.0.smithi029.stdout:3848: done (1 left) 2024-04-12T11:30:43.776 INFO:tasks.rados.rados.0.smithi029.stdout:3849: done (0 left) 2024-04-12T11:30:43.776 INFO:tasks.rados.rados.0.smithi029.stdout:3850: write oid 44 current snap is 487 2024-04-12T11:30:43.777 INFO:tasks.rados.rados.0.smithi029.stdout:3850: seq_num 1451 ranges {786957=493369,2059137=327820} 2024-04-12T11:30:43.782 INFO:tasks.rados.rados.0.smithi029.stdout:3850: writing smithi02979277-44 from 786957 to 1280326 tid 1 2024-04-12T11:30:43.789 INFO:tasks.rados.rados.0.smithi029.stdout:3850: writing smithi02979277-44 from 2059137 to 2386957 tid 2 2024-04-12T11:30:43.789 INFO:tasks.rados.rados.0.smithi029.stdout:3851: read oid 10 snap 485 2024-04-12T11:30:43.789 INFO:tasks.rados.rados.0.smithi029.stdout:3851: expect deleted 2024-04-12T11:30:43.789 INFO:tasks.rados.rados.0.smithi029.stdout:3852: write oid 41 current snap is 487 2024-04-12T11:30:43.789 INFO:tasks.rados.rados.0.smithi029.stdout:3850: finishing write tid 1 to smithi02979277-44 2024-04-12T11:30:43.789 INFO:tasks.rados.rados.0.smithi029.stdout:3852: seq_num 1452 ranges {403764=568618,1717619=686145} 2024-04-12T11:30:43.794 INFO:tasks.rados.rados.0.smithi029.stdout:3852: writing smithi02979277-41 from 403764 to 972382 tid 1 2024-04-12T11:30:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3852: writing smithi02979277-41 from 1717619 to 2403764 tid 2 2024-04-12T11:30:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3853: delete oid 37 current snap is 487 2024-04-12T11:30:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3850: finishing write tid 2 to smithi02979277-44 2024-04-12T11:30:43.802 INFO:tasks.rados.rados.0.smithi029.stdout:3850: finishing write tid 3 to smithi02979277-44 2024-04-12T11:30:43.803 INFO:tasks.rados.rados.0.smithi029.stdout:3850: finishing write tid 4 to smithi02979277-44 2024-04-12T11:30:43.803 INFO:tasks.rados.rados.0.smithi029.stdout:3850: oid 44 updating version 0 to 1927 2024-04-12T11:30:43.803 INFO:tasks.rados.rados.0.smithi029.stdout:3850: oid 44 updating version 1927 to 1928 2024-04-12T11:30:43.803 INFO:tasks.rados.rados.0.smithi029.stdout:3850: oid 44 version 1928 is already newer than 1926 2024-04-12T11:30:43.803 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1928 (ObjNum 1451 snap 487 seq_num 1451) dirty exists 2024-04-12T11:30:43.803 INFO:tasks.rados.rados.0.smithi029.stdout:3850: left oid 44 (ObjNum 1451 snap 487 seq_num 1451) 2024-04-12T11:30:43.803 INFO:tasks.rados.rados.0.smithi029.stdout:3852: finishing write tid 1 to smithi02979277-41 2024-04-12T11:30:43.805 INFO:tasks.rados.rados.0.smithi029.stdout:3850: done (3 left) 2024-04-12T11:30:43.805 INFO:tasks.rados.rados.0.smithi029.stdout:3851: done (2 left) 2024-04-12T11:30:43.805 INFO:tasks.rados.rados.0.smithi029.stdout:3853: done (1 left) 2024-04-12T11:30:43.805 INFO:tasks.rados.rados.0.smithi029.stdout:3854: delete oid 21 current snap is 487 2024-04-12T11:30:43.807 INFO:tasks.rados.rados.0.smithi029.stdout:3852: finishing write tid 2 to smithi02979277-41 2024-04-12T11:30:43.807 INFO:tasks.rados.rados.0.smithi029.stdout:3852: finishing write tid 3 to smithi02979277-41 2024-04-12T11:30:43.807 INFO:tasks.rados.rados.0.smithi029.stdout:3852: finishing write tid 4 to smithi02979277-41 2024-04-12T11:30:43.807 INFO:tasks.rados.rados.0.smithi029.stdout:3852: oid 41 updating version 0 to 1932 2024-04-12T11:30:43.807 INFO:tasks.rados.rados.0.smithi029.stdout:3852: oid 41 version 1932 is already newer than 1930 2024-04-12T11:30:43.807 INFO:tasks.rados.rados.0.smithi029.stdout:3852: oid 41 version 1932 is already newer than 1931 2024-04-12T11:30:43.807 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 1932 (ObjNum 1452 snap 487 seq_num 1452) dirty exists 2024-04-12T11:30:43.807 INFO:tasks.rados.rados.0.smithi029.stdout:3852: left oid 41 (ObjNum 1452 snap 487 seq_num 1452) 2024-04-12T11:30:43.809 INFO:tasks.rados.rados.0.smithi029.stdout:3852: done (1 left) 2024-04-12T11:30:43.809 INFO:tasks.rados.rados.0.smithi029.stdout:3854: done (0 left) 2024-04-12T11:30:43.809 INFO:tasks.rados.rados.0.smithi029.stdout:3855: delete oid 47 current snap is 487 2024-04-12T11:30:43.812 INFO:tasks.rados.rados.0.smithi029.stdout:3855: done (0 left) 2024-04-12T11:30:43.812 INFO:tasks.rados.rados.0.smithi029.stdout:3856: snap_remove snap 478 2024-04-12T11:30:44.784 INFO:tasks.rados.rados.0.smithi029.stdout:3856: done (0 left) 2024-04-12T11:30:44.784 INFO:tasks.rados.rados.0.smithi029.stdout:3857: write oid 30 current snap is 487 2024-04-12T11:30:44.784 INFO:tasks.rados.rados.0.smithi029.stdout:3857: seq_num 1456 ranges {470992=585967,1551547=773499} 2024-04-12T11:30:44.790 INFO:tasks.rados.rados.0.smithi029.stdout:3857: writing smithi02979277-30 from 470992 to 1056959 tid 1 2024-04-12T11:30:44.797 INFO:tasks.rados.rados.0.smithi029.stdout:3857: writing smithi02979277-30 from 1551547 to 2325046 tid 2 2024-04-12T11:30:44.797 INFO:tasks.rados.rados.0.smithi029.stdout:3858: snap_remove snap 486 2024-04-12T11:30:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:44 smithi087 ceph-mon[89014]: pgmap v1194: 89 pgs: 89 active+clean; 247 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 502 KiB/s wr, 6 op/s 2024-04-12T11:30:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:44 smithi087 ceph-mon[89014]: osdmap e1039: 8 total, 8 up, 8 in 2024-04-12T11:30:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:44 smithi029 ceph-mon[93650]: pgmap v1194: 89 pgs: 89 active+clean; 247 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 502 KiB/s wr, 6 op/s 2024-04-12T11:30:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:44 smithi029 ceph-mon[93650]: osdmap e1039: 8 total, 8 up, 8 in 2024-04-12T11:30:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:44 smithi029 ceph-mon[97787]: pgmap v1194: 89 pgs: 89 active+clean; 247 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 502 KiB/s wr, 6 op/s 2024-04-12T11:30:45.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:44 smithi029 ceph-mon[97787]: osdmap e1039: 8 total, 8 up, 8 in 2024-04-12T11:30:45.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:30:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:30:44] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-12T11:30:45.789 INFO:tasks.rados.rados.0.smithi029.stdout:3858: done (1 left) 2024-04-12T11:30:45.789 INFO:tasks.rados.rados.0.smithi029.stdout:3859: rollback oid 43 current snap is 487 2024-04-12T11:30:45.789 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 471 2024-04-12T11:30:45.789 INFO:tasks.rados.rados.0.smithi029.stdout:3857: finishing write tid 1 to smithi02979277-30 2024-04-12T11:30:45.790 INFO:tasks.rados.rados.0.smithi029.stdout:3857: finishing write tid 2 to smithi02979277-30 2024-04-12T11:30:45.790 INFO:tasks.rados.rados.0.smithi029.stdout:3857: finishing write tid 3 to smithi02979277-30 2024-04-12T11:30:45.790 INFO:tasks.rados.rados.0.smithi029.stdout:3857: finishing write tid 4 to smithi02979277-30 2024-04-12T11:30:45.790 INFO:tasks.rados.rados.0.smithi029.stdout:3857: oid 30 updating version 0 to 1234 2024-04-12T11:30:45.790 INFO:tasks.rados.rados.0.smithi029.stdout:3857: oid 30 version 1234 is already newer than 1232 2024-04-12T11:30:45.790 INFO:tasks.rados.rados.0.smithi029.stdout:3857: oid 30 version 1234 is already newer than 1233 2024-04-12T11:30:45.790 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 1234 (ObjNum 1456 snap 487 seq_num 1456) dirty exists 2024-04-12T11:30:45.790 INFO:tasks.rados.rados.0.smithi029.stdout:3857: left oid 30 (ObjNum 1456 snap 487 seq_num 1456) 2024-04-12T11:30:45.790 INFO:tasks.rados.rados.0.smithi029.stdout:3857: done (1 left) 2024-04-12T11:30:45.790 INFO:tasks.rados.rados.0.smithi029.stdout:3860: snap_create 2024-04-12T11:30:45.795 INFO:tasks.rados.rados.0.smithi029.stdout:3859: finishing rollback tid 0 to smithi02979277-43 2024-04-12T11:30:45.795 INFO:tasks.rados.rados.0.smithi029.stdout:3859: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:30:45.795 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1759 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-04-12T11:30:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:45 smithi087 ceph-mon[89014]: osdmap e1040: 8 total, 8 up, 8 in 2024-04-12T11:30:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:45 smithi029 ceph-mon[97787]: osdmap e1040: 8 total, 8 up, 8 in 2024-04-12T11:30:46.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:45 smithi029 ceph-mon[93650]: osdmap e1040: 8 total, 8 up, 8 in 2024-04-12T11:30:46.790 INFO:tasks.rados.rados.0.smithi029.stdout:3859: done (1 left) 2024-04-12T11:30:46.790 INFO:tasks.rados.rados.0.smithi029.stdout:3860: done (0 left) 2024-04-12T11:30:46.790 INFO:tasks.rados.rados.0.smithi029.stdout:3861: snap_create 2024-04-12T11:30:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:46 smithi087 ceph-mon[89014]: pgmap v1197: 89 pgs: 89 active+clean; 244 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:30:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:46 smithi087 ceph-mon[89014]: osdmap e1041: 8 total, 8 up, 8 in 2024-04-12T11:30:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:46 smithi029 ceph-mon[97787]: pgmap v1197: 89 pgs: 89 active+clean; 244 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:30:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:46 smithi029 ceph-mon[97787]: osdmap e1041: 8 total, 8 up, 8 in 2024-04-12T11:30:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:46 smithi029 ceph-mon[93650]: pgmap v1197: 89 pgs: 89 active+clean; 244 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:30:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:46 smithi029 ceph-mon[93650]: osdmap e1041: 8 total, 8 up, 8 in 2024-04-12T11:30:47.732 INFO:tasks.rados.rados.0.smithi029.stdout:3861: done (0 left) 2024-04-12T11:30:47.733 INFO:tasks.rados.rados.0.smithi029.stdout:3862: write oid 3 current snap is 489 2024-04-12T11:30:47.733 INFO:tasks.rados.rados.0.smithi029.stdout:3862: seq_num 1457 ranges {487799=661216,1893676=594123} 2024-04-12T11:30:47.739 INFO:tasks.rados.rados.0.smithi029.stdout:3862: writing smithi02979277-3 from 487799 to 1149015 tid 1 2024-04-12T11:30:47.746 INFO:tasks.rados.rados.0.smithi029.stdout:3862: writing smithi02979277-3 from 1893676 to 2487799 tid 2 2024-04-12T11:30:47.746 INFO:tasks.rados.rados.0.smithi029.stdout:3863: snap_remove snap 483 2024-04-12T11:30:48.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:47 smithi087 ceph-mon[89014]: osdmap e1042: 8 total, 8 up, 8 in 2024-04-12T11:30:48.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:47 smithi087 ceph-mon[89014]: osdmap e1043: 8 total, 8 up, 8 in 2024-04-12T11:30:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:47 smithi029 ceph-mon[97787]: osdmap e1042: 8 total, 8 up, 8 in 2024-04-12T11:30:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:47 smithi029 ceph-mon[97787]: osdmap e1043: 8 total, 8 up, 8 in 2024-04-12T11:30:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:47 smithi029 ceph-mon[93650]: osdmap e1042: 8 total, 8 up, 8 in 2024-04-12T11:30:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:47 smithi029 ceph-mon[93650]: osdmap e1043: 8 total, 8 up, 8 in 2024-04-12T11:30:48.734 INFO:tasks.rados.rados.0.smithi029.stdout:3863: done (1 left) 2024-04-12T11:30:48.734 INFO:tasks.rados.rados.0.smithi029.stdout:3864: snap_remove snap 442 2024-04-12T11:30:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:48 smithi087 ceph-mon[89014]: pgmap v1200: 89 pgs: 89 active+clean; 248 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-04-12T11:30:49.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:48 smithi087 ceph-mon[89014]: osdmap e1044: 8 total, 8 up, 8 in 2024-04-12T11:30:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:48 smithi029 ceph-mon[93650]: pgmap v1200: 89 pgs: 89 active+clean; 248 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-04-12T11:30:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:48 smithi029 ceph-mon[93650]: osdmap e1044: 8 total, 8 up, 8 in 2024-04-12T11:30:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:48 smithi029 ceph-mon[97787]: pgmap v1200: 89 pgs: 89 active+clean; 248 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-04-12T11:30:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:48 smithi029 ceph-mon[97787]: osdmap e1044: 8 total, 8 up, 8 in 2024-04-12T11:30:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3864: done (1 left) 2024-04-12T11:30:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3865: delete oid 45 current snap is 489 2024-04-12T11:30:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3862: finishing write tid 1 to smithi02979277-3 2024-04-12T11:30:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3862: finishing write tid 2 to smithi02979277-3 2024-04-12T11:30:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3862: finishing write tid 3 to smithi02979277-3 2024-04-12T11:30:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3862: finishing write tid 4 to smithi02979277-3 2024-04-12T11:30:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3862: oid 3 updating version 0 to 2509 2024-04-12T11:30:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3862: oid 3 version 2509 is already newer than 2507 2024-04-12T11:30:49.737 INFO:tasks.rados.rados.0.smithi029.stdout:3862: oid 3 version 2509 is already newer than 2508 2024-04-12T11:30:49.738 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 2509 (ObjNum 1457 snap 489 seq_num 1457) dirty exists 2024-04-12T11:30:49.738 INFO:tasks.rados.rados.0.smithi029.stdout:3862: left oid 3 (ObjNum 1457 snap 489 seq_num 1457) 2024-04-12T11:30:49.754 INFO:tasks.rados.rados.0.smithi029.stdout:3862: done (1 left) 2024-04-12T11:30:49.754 INFO:tasks.rados.rados.0.smithi029.stdout:3865: done (0 left) 2024-04-12T11:30:49.754 INFO:tasks.rados.rados.0.smithi029.stdout:3866: rollback oid 39 current snap is 489 2024-04-12T11:30:49.754 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 39 to 471 2024-04-12T11:30:49.754 INFO:tasks.rados.rados.0.smithi029.stdout:3867: write oid 50 current snap is 489 2024-04-12T11:30:49.754 INFO:tasks.rados.rados.0.smithi029.stdout:3867: seq_num 1459 ranges {521413=411714,1610640=637179} 2024-04-12T11:30:49.759 INFO:tasks.rados.rados.0.smithi029.stdout:3867: writing smithi02979277-50 from 521413 to 933127 tid 1 2024-04-12T11:30:49.767 INFO:tasks.rados.rados.0.smithi029.stdout:3867: writing smithi02979277-50 from 1610640 to 2247819 tid 2 2024-04-12T11:30:49.767 INFO:tasks.rados.rados.0.smithi029.stdout:3868: write oid 37 current snap is 489 2024-04-12T11:30:49.767 INFO:tasks.rados.rados.0.smithi029.stdout:3866: finishing rollback tid 0 to smithi02979277-39 2024-04-12T11:30:49.767 INFO:tasks.rados.rados.0.smithi029.stdout:3866: finishing rollback tid 1 to smithi02979277-39 2024-04-12T11:30:49.767 INFO:tasks.rados.rados.0.smithi029.stdout:3866: finishing rollback tid 2 to smithi02979277-39 2024-04-12T11:30:49.768 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 39 v 572 (ObjNum 1088 snap 365 seq_num 1088) dirty exists 2024-04-12T11:30:49.768 INFO:tasks.rados.rados.0.smithi029.stdout:3867: finishing write tid 1 to smithi02979277-50 2024-04-12T11:30:49.768 INFO:tasks.rados.rados.0.smithi029.stdout:3868: seq_num 1460 ranges {538220=603316,1785475=752745} 2024-04-12T11:30:49.773 INFO:tasks.rados.rados.0.smithi029.stdout:3868: writing smithi02979277-37 from 538220 to 1141536 tid 1 2024-04-12T11:30:49.781 INFO:tasks.rados.rados.0.smithi029.stdout:3868: writing smithi02979277-37 from 1785475 to 2538220 tid 2 2024-04-12T11:30:49.781 INFO:tasks.rados.rados.0.smithi029.stdout:3866: done (2 left) 2024-04-12T11:30:49.781 INFO:tasks.rados.rados.0.smithi029.stdout:3869: snap_remove snap 481 2024-04-12T11:30:50.743 INFO:tasks.rados.rados.0.smithi029.stdout:3869: done (2 left) 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3870: snap_create 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3867: finishing write tid 2 to smithi02979277-50 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3867: finishing write tid 3 to smithi02979277-50 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3867: finishing write tid 4 to smithi02979277-50 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3867: oid 50 updating version 0 to 1648 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3867: oid 50 version 1648 is already newer than 1647 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3867: oid 50 version 1648 is already newer than 1646 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1648 (ObjNum 1459 snap 489 seq_num 1459) dirty exists 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3867: left oid 50 (ObjNum 1459 snap 489 seq_num 1459) 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3868: finishing write tid 1 to smithi02979277-37 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3868: finishing write tid 2 to smithi02979277-37 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3868: finishing write tid 3 to smithi02979277-37 2024-04-12T11:30:50.744 INFO:tasks.rados.rados.0.smithi029.stdout:3868: finishing write tid 4 to smithi02979277-37 2024-04-12T11:30:50.745 INFO:tasks.rados.rados.0.smithi029.stdout:3868: oid 37 updating version 0 to 1373 2024-04-12T11:30:50.745 INFO:tasks.rados.rados.0.smithi029.stdout:3868: oid 37 version 1373 is already newer than 1372 2024-04-12T11:30:50.745 INFO:tasks.rados.rados.0.smithi029.stdout:3868: oid 37 version 1373 is already newer than 1371 2024-04-12T11:30:50.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 37 v 1373 (ObjNum 1460 snap 489 seq_num 1460) dirty exists 2024-04-12T11:30:50.745 INFO:tasks.rados.rados.0.smithi029.stdout:3868: left oid 37 (ObjNum 1460 snap 489 seq_num 1460) 2024-04-12T11:30:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:50 smithi029 ceph-mon[93650]: pgmap v1203: 89 pgs: 89 active+clean; 249 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-04-12T11:30:51.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:50 smithi029 ceph-mon[93650]: osdmap e1045: 8 total, 8 up, 8 in 2024-04-12T11:30:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:50 smithi029 ceph-mon[97787]: pgmap v1203: 89 pgs: 89 active+clean; 249 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-04-12T11:30:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:50 smithi029 ceph-mon[97787]: osdmap e1045: 8 total, 8 up, 8 in 2024-04-12T11:30:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:50 smithi087 ceph-mon[89014]: pgmap v1203: 89 pgs: 89 active+clean; 249 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-04-12T11:30:51.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:50 smithi087 ceph-mon[89014]: osdmap e1045: 8 total, 8 up, 8 in 2024-04-12T11:30:51.748 INFO:tasks.rados.rados.0.smithi029.stdout:3867: done (2 left) 2024-04-12T11:30:51.748 INFO:tasks.rados.rados.0.smithi029.stdout:3868: done (1 left) 2024-04-12T11:30:51.748 INFO:tasks.rados.rados.0.smithi029.stdout:3870: done (0 left) 2024-04-12T11:30:51.748 INFO:tasks.rados.rados.0.smithi029.stdout:3871: snap_create 2024-04-12T11:30:52.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:51 smithi029 ceph-mon[93650]: osdmap e1046: 8 total, 8 up, 8 in 2024-04-12T11:30:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:51 smithi029 ceph-mon[97787]: osdmap e1046: 8 total, 8 up, 8 in 2024-04-12T11:30:52.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:52.118 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:51 smithi087 ceph-mon[89014]: osdmap e1046: 8 total, 8 up, 8 in 2024-04-12T11:30:52.118 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:30:52.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:30:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:30:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:30:52.734 INFO:tasks.rados.rados.0.smithi029.stdout:3871: done (0 left) 2024-04-12T11:30:52.734 INFO:tasks.rados.rados.0.smithi029.stdout:3872: snap_create 2024-04-12T11:30:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:52 smithi029 ceph-mon[93650]: pgmap v1206: 89 pgs: 89 active+clean; 249 MiB data, 898 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 639 KiB/s wr, 3 op/s 2024-04-12T11:30:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:52 smithi029 ceph-mon[93650]: osdmap e1047: 8 total, 8 up, 8 in 2024-04-12T11:30:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:52 smithi029 ceph-mon[93650]: osdmap e1048: 8 total, 8 up, 8 in 2024-04-12T11:30:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:52 smithi029 ceph-mon[97787]: pgmap v1206: 89 pgs: 89 active+clean; 249 MiB data, 898 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 639 KiB/s wr, 3 op/s 2024-04-12T11:30:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:52 smithi029 ceph-mon[97787]: osdmap e1047: 8 total, 8 up, 8 in 2024-04-12T11:30:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:52 smithi029 ceph-mon[97787]: osdmap e1048: 8 total, 8 up, 8 in 2024-04-12T11:30:53.137 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:52 smithi087 ceph-mon[89014]: pgmap v1206: 89 pgs: 89 active+clean; 249 MiB data, 898 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 639 KiB/s wr, 3 op/s 2024-04-12T11:30:53.137 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:52 smithi087 ceph-mon[89014]: osdmap e1047: 8 total, 8 up, 8 in 2024-04-12T11:30:53.137 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:52 smithi087 ceph-mon[89014]: osdmap e1048: 8 total, 8 up, 8 in 2024-04-12T11:30:53.736 INFO:tasks.rados.rados.0.smithi029.stdout:3872: done (0 left) 2024-04-12T11:30:53.736 INFO:tasks.rados.rados.0.smithi029.stdout:3873: write oid 5 current snap is 492 2024-04-12T11:30:53.736 INFO:tasks.rados.rados.0.smithi029.stdout:3873: seq_num 1461 ranges {555027=678565,1727604=640848} 2024-04-12T11:30:53.742 INFO:tasks.rados.rados.0.smithi029.stdout:3873: writing smithi02979277-5 from 555027 to 1233592 tid 1 2024-04-12T11:30:53.749 INFO:tasks.rados.rados.0.smithi029.stdout:3873: writing smithi02979277-5 from 1727604 to 2368452 tid 2 2024-04-12T11:30:53.750 INFO:tasks.rados.rados.0.smithi029.stdout:3874: write oid 16 current snap is 492 2024-04-12T11:30:53.750 INFO:tasks.rados.rados.0.smithi029.stdout:3874: seq_num 1462 ranges {571834=753814,1786086=784506} 2024-04-12T11:30:53.757 INFO:tasks.rados.rados.0.smithi029.stdout:3874: writing smithi02979277-16 from 571834 to 1325648 tid 1 2024-04-12T11:30:53.764 INFO:tasks.rados.rados.0.smithi029.stdout:3874: writing smithi02979277-16 from 1786086 to 2570592 tid 2 2024-04-12T11:30:53.764 INFO:tasks.rados.rados.0.smithi029.stdout:3875: write oid 46 current snap is 492 2024-04-12T11:30:53.764 INFO:tasks.rados.rados.0.smithi029.stdout:3873: finishing write tid 1 to smithi02979277-5 2024-04-12T11:30:53.764 INFO:tasks.rados.rados.0.smithi029.stdout:3873: finishing write tid 2 to smithi02979277-5 2024-04-12T11:30:53.765 INFO:tasks.rados.rados.0.smithi029.stdout:3873: finishing write tid 3 to smithi02979277-5 2024-04-12T11:30:53.765 INFO:tasks.rados.rados.0.smithi029.stdout:3875: seq_num 1463 ranges {588641=429063,1444568=644517,2492662=95979} 2024-04-12T11:30:53.770 INFO:tasks.rados.rados.0.smithi029.stdout:3875: writing smithi02979277-46 from 588641 to 1017704 tid 1 2024-04-12T11:30:53.776 INFO:tasks.rados.rados.0.smithi029.stdout:3875: writing smithi02979277-46 from 1444568 to 2089085 tid 2 2024-04-12T11:30:53.779 INFO:tasks.rados.rados.0.smithi029.stdout:3875: writing smithi02979277-46 from 2492662 to 2588641 tid 3 2024-04-12T11:30:53.779 INFO:tasks.rados.rados.0.smithi029.stdout:3876: read oid 38 snap -1 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3876: expect (ObjNum 1299 snap 444 seq_num 1299) 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3877: delete oid 48 current snap is 492 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3873: finishing write tid 4 to smithi02979277-5 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3873: oid 5 updating version 0 to 1933 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3873: oid 5 version 1933 is already newer than 1932 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3873: oid 5 updating version 1933 to 1934 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1934 (ObjNum 1461 snap 492 seq_num 1461) dirty exists 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3873: left oid 5 (ObjNum 1461 snap 492 seq_num 1461) 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3874: finishing write tid 1 to smithi02979277-16 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3874: finishing write tid 2 to smithi02979277-16 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3874: finishing write tid 3 to smithi02979277-16 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3874: finishing write tid 4 to smithi02979277-16 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3874: oid 16 updating version 0 to 1936 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3874: oid 16 updating version 1936 to 1938 2024-04-12T11:30:53.780 INFO:tasks.rados.rados.0.smithi029.stdout:3874: oid 16 version 1938 is already newer than 1937 2024-04-12T11:30:53.781 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 16 v 1938 (ObjNum 1462 snap 492 seq_num 1462) dirty exists 2024-04-12T11:30:53.781 INFO:tasks.rados.rados.0.smithi029.stdout:3874: left oid 16 (ObjNum 1462 snap 492 seq_num 1462) 2024-04-12T11:30:53.781 INFO:tasks.rados.rados.0.smithi029.stdout:3875: finishing write tid 1 to smithi02979277-46 2024-04-12T11:30:53.781 INFO:tasks.rados.rados.0.smithi029.stdout:3875: finishing write tid 2 to smithi02979277-46 2024-04-12T11:30:53.781 INFO:tasks.rados.rados.0.smithi029.stdout:3875: finishing write tid 3 to smithi02979277-46 2024-04-12T11:30:53.782 INFO:tasks.rados.rados.0.smithi029.stdout:3873: done (4 left) 2024-04-12T11:30:53.782 INFO:tasks.rados.rados.0.smithi029.stdout:3874: done (3 left) 2024-04-12T11:30:53.782 INFO:tasks.rados.rados.0.smithi029.stdout:3877: done (2 left) 2024-04-12T11:30:53.782 INFO:tasks.rados.rados.0.smithi029.stdout:3878: read oid 3 snap -1 2024-04-12T11:30:53.782 INFO:tasks.rados.rados.0.smithi029.stdout:3878: expect (ObjNum 1457 snap 489 seq_num 1457) 2024-04-12T11:30:53.782 INFO:tasks.rados.rados.0.smithi029.stdout:3879: write oid 15 current snap is 492 2024-04-12T11:30:53.782 INFO:tasks.rados.rados.0.smithi029.stdout:3879: seq_num 1465 ranges {622255=579561,1845179=531833} 2024-04-12T11:30:53.789 INFO:tasks.rados.rados.0.smithi029.stdout:3879: writing smithi02979277-15 from 622255 to 1201816 tid 1 2024-04-12T11:30:53.795 INFO:tasks.rados.rados.0.smithi029.stdout:3879: writing smithi02979277-15 from 1845179 to 2377012 tid 2 2024-04-12T11:30:53.795 INFO:tasks.rados.rados.0.smithi029.stdout:3880: delete oid 5 current snap is 492 2024-04-12T11:30:53.795 INFO:tasks.rados.rados.0.smithi029.stdout:3875: finishing write tid 4 to smithi02979277-46 2024-04-12T11:30:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3875: finishing write tid 5 to smithi02979277-46 2024-04-12T11:30:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3875: oid 46 updating version 0 to 1289 2024-04-12T11:30:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3875: oid 46 version 1289 is already newer than 1288 2024-04-12T11:30:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3875: oid 46 version 1289 is already newer than 1287 2024-04-12T11:30:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3875: oid 46 version 1289 is already newer than 1286 2024-04-12T11:30:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 46 v 1289 (ObjNum 1463 snap 492 seq_num 1463) dirty exists 2024-04-12T11:30:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3875: left oid 46 (ObjNum 1463 snap 492 seq_num 1463) 2024-04-12T11:30:53.796 INFO:tasks.rados.rados.0.smithi029.stdout:3876: expect (ObjNum 1299 snap 444 seq_num 1299) 2024-04-12T11:30:54.031 INFO:tasks.rados.rados.0.smithi029.stdout:3878: expect (ObjNum 1457 snap 489 seq_num 1457) 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3879: finishing write tid 1 to smithi02979277-15 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3879: finishing write tid 2 to smithi02979277-15 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3879: finishing write tid 3 to smithi02979277-15 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3879: finishing write tid 4 to smithi02979277-15 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3879: oid 15 updating version 0 to 1249 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3879: oid 15 version 1249 is already newer than 1248 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3879: oid 15 updating version 1249 to 1250 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 1250 (ObjNum 1465 snap 492 seq_num 1465) dirty exists 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3879: left oid 15 (ObjNum 1465 snap 492 seq_num 1465) 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3875: done (4 left) 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3876: done (3 left) 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3878: done (2 left) 2024-04-12T11:30:54.112 INFO:tasks.rados.rados.0.smithi029.stdout:3879: done (1 left) 2024-04-12T11:30:54.113 INFO:tasks.rados.rados.0.smithi029.stdout:3880: done (0 left) 2024-04-12T11:30:54.113 INFO:tasks.rados.rados.0.smithi029.stdout:3881: write oid 47 current snap is 492 2024-04-12T11:30:54.113 INFO:tasks.rados.rados.0.smithi029.stdout:3881: seq_num 1467 ranges {655869=730059,1962143=535502} 2024-04-12T11:30:54.119 INFO:tasks.rados.rados.0.smithi029.stdout:3881: writing smithi02979277-47 from 655869 to 1385928 tid 1 2024-04-12T11:30:54.125 INFO:tasks.rados.rados.0.smithi029.stdout:3881: writing smithi02979277-47 from 1962143 to 2497645 tid 2 2024-04-12T11:30:54.125 INFO:tasks.rados.rados.0.smithi029.stdout:3882: read oid 49 snap -1 2024-04-12T11:30:54.125 INFO:tasks.rados.rados.0.smithi029.stdout:3882: expect (ObjNum 1384 snap 467 seq_num 1384) 2024-04-12T11:30:54.125 INFO:tasks.rados.rados.0.smithi029.stdout:3883: write oid 23 current snap is 492 2024-04-12T11:30:54.126 INFO:tasks.rados.rados.0.smithi029.stdout:3881: finishing write tid 1 to smithi02979277-47 2024-04-12T11:30:54.126 INFO:tasks.rados.rados.0.smithi029.stdout:3883: seq_num 1468 ranges {672676=521661,1736978=795513} 2024-04-12T11:30:54.132 INFO:tasks.rados.rados.0.smithi029.stdout:3883: writing smithi02979277-23 from 672676 to 1194337 tid 1 2024-04-12T11:30:54.139 INFO:tasks.rados.rados.0.smithi029.stdout:3883: writing smithi02979277-23 from 1736978 to 2532491 tid 2 2024-04-12T11:30:54.139 INFO:tasks.rados.rados.0.smithi029.stdout:3884: write oid 44 current snap is 492 2024-04-12T11:30:54.139 INFO:tasks.rados.rados.0.smithi029.stdout:3884: seq_num 1469 ranges {689483=596910,2079107=539171} 2024-04-12T11:30:54.146 INFO:tasks.rados.rados.0.smithi029.stdout:3884: writing smithi02979277-44 from 689483 to 1286393 tid 1 2024-04-12T11:30:54.154 INFO:tasks.rados.rados.0.smithi029.stdout:3884: writing smithi02979277-44 from 2079107 to 2618278 tid 2 2024-04-12T11:30:54.154 INFO:tasks.rados.rados.0.smithi029.stdout:3885: snap_create 2024-04-12T11:30:54.154 INFO:tasks.rados.rados.0.smithi029.stdout:3882: expect (ObjNum 1384 snap 467 seq_num 1384) 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3881: finishing write tid 2 to smithi02979277-47 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3881: finishing write tid 3 to smithi02979277-47 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3881: finishing write tid 4 to smithi02979277-47 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3881: oid 47 updating version 0 to 1939 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3881: oid 47 version 1939 is already newer than 1938 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3881: oid 47 version 1939 is already newer than 1937 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1939 (ObjNum 1467 snap 492 seq_num 1467) dirty exists 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3881: left oid 47 (ObjNum 1467 snap 492 seq_num 1467) 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3883: finishing write tid 1 to smithi02979277-23 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3883: finishing write tid 2 to smithi02979277-23 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3883: finishing write tid 3 to smithi02979277-23 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3883: finishing write tid 4 to smithi02979277-23 2024-04-12T11:30:54.352 INFO:tasks.rados.rados.0.smithi029.stdout:3883: oid 23 updating version 0 to 1653 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3883: oid 23 updating version 1653 to 1654 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3883: oid 23 version 1654 is already newer than 1652 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1654 (ObjNum 1468 snap 492 seq_num 1468) dirty exists 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3883: left oid 23 (ObjNum 1468 snap 492 seq_num 1468) 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3884: finishing write tid 1 to smithi02979277-44 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3884: finishing write tid 2 to smithi02979277-44 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3884: finishing write tid 3 to smithi02979277-44 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3884: finishing write tid 4 to smithi02979277-44 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3884: oid 44 updating version 0 to 1961 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3884: oid 44 version 1961 is already newer than 1959 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3884: oid 44 version 1961 is already newer than 1960 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1961 (ObjNum 1469 snap 492 seq_num 1469) dirty exists 2024-04-12T11:30:54.353 INFO:tasks.rados.rados.0.smithi029.stdout:3884: left oid 44 (ObjNum 1469 snap 492 seq_num 1469) 2024-04-12T11:30:54.751 INFO:tasks.rados.rados.0.smithi029.stdout:3881: done (4 left) 2024-04-12T11:30:54.751 INFO:tasks.rados.rados.0.smithi029.stdout:3882: done (3 left) 2024-04-12T11:30:54.751 INFO:tasks.rados.rados.0.smithi029.stdout:3883: done (2 left) 2024-04-12T11:30:54.751 INFO:tasks.rados.rados.0.smithi029.stdout:3884: done (1 left) 2024-04-12T11:30:54.751 INFO:tasks.rados.rados.0.smithi029.stdout:3885: done (0 left) 2024-04-12T11:30:54.752 INFO:tasks.rados.rados.0.smithi029.stdout:3886: read oid 50 snap -1 2024-04-12T11:30:54.752 INFO:tasks.rados.rados.0.smithi029.stdout:3886: expect (ObjNum 1459 snap 489 seq_num 1459) 2024-04-12T11:30:54.752 INFO:tasks.rados.rados.0.smithi029.stdout:3887: write oid 13 current snap is 493 2024-04-12T11:30:54.752 INFO:tasks.rados.rados.0.smithi029.stdout:3887: seq_num 1470 ranges {706290=672159,2137589=568701} 2024-04-12T11:30:54.759 INFO:tasks.rados.rados.0.smithi029.stdout:3887: writing smithi02979277-13 from 706290 to 1378449 tid 1 2024-04-12T11:30:54.767 INFO:tasks.rados.rados.0.smithi029.stdout:3887: writing smithi02979277-13 from 2137589 to 2706290 tid 2 2024-04-12T11:30:54.767 INFO:tasks.rados.rados.0.smithi029.stdout:3888: read oid 30 snap 484 2024-04-12T11:30:54.767 INFO:tasks.rados.rados.0.smithi029.stdout:3888: expect (ObjNum 1414 snap 475 seq_num 1414) 2024-04-12T11:30:54.768 INFO:tasks.rados.rados.0.smithi029.stdout:3889: read oid 5 snap -1 2024-04-12T11:30:54.768 INFO:tasks.rados.rados.0.smithi029.stdout:3889: expect deleted 2024-04-12T11:30:54.768 INFO:tasks.rados.rados.0.smithi029.stdout:3890: delete oid 12 current snap is 493 2024-04-12T11:30:54.768 INFO:tasks.rados.rados.0.smithi029.stdout:3886: expect (ObjNum 1459 snap 489 seq_num 1459) 2024-04-12T11:30:54.921 INFO:tasks.rados.rados.0.smithi029.stdout:3887: finishing write tid 1 to smithi02979277-13 2024-04-12T11:30:54.921 INFO:tasks.rados.rados.0.smithi029.stdout:3886: done (4 left) 2024-04-12T11:30:54.921 INFO:tasks.rados.rados.0.smithi029.stdout:3890: done (3 left) 2024-04-12T11:30:54.921 INFO:tasks.rados.rados.0.smithi029.stdout:3891: snap_create 2024-04-12T11:30:54.921 INFO:tasks.rados.rados.0.smithi029.stdout:3887: finishing write tid 2 to smithi02979277-13 2024-04-12T11:30:54.921 INFO:tasks.rados.rados.0.smithi029.stdout:3887: finishing write tid 3 to smithi02979277-13 2024-04-12T11:30:54.921 INFO:tasks.rados.rados.0.smithi029.stdout:3887: finishing write tid 4 to smithi02979277-13 2024-04-12T11:30:54.921 INFO:tasks.rados.rados.0.smithi029.stdout:3887: oid 13 updating version 0 to 2080 2024-04-12T11:30:54.921 INFO:tasks.rados.rados.0.smithi029.stdout:3887: oid 13 updating version 2080 to 2082 2024-04-12T11:30:54.922 INFO:tasks.rados.rados.0.smithi029.stdout:3887: oid 13 version 2082 is already newer than 2081 2024-04-12T11:30:54.922 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 13 v 2082 (ObjNum 1470 snap 493 seq_num 1470) dirty exists 2024-04-12T11:30:54.922 INFO:tasks.rados.rados.0.smithi029.stdout:3887: left oid 13 (ObjNum 1470 snap 493 seq_num 1470) 2024-04-12T11:30:54.922 INFO:tasks.rados.rados.0.smithi029.stdout:3888: expect (ObjNum 1414 snap 475 seq_num 1414) 2024-04-12T11:30:55.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:54 smithi029 ceph-mon[97787]: pgmap v1209: 89 pgs: 89 active+clean; 202 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 588 KiB/s wr, 2 op/s 2024-04-12T11:30:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:54 smithi029 ceph-mon[97787]: osdmap e1049: 8 total, 8 up, 8 in 2024-04-12T11:30:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:54 smithi029 ceph-mon[93650]: pgmap v1209: 89 pgs: 89 active+clean; 202 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 588 KiB/s wr, 2 op/s 2024-04-12T11:30:55.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:54 smithi029 ceph-mon[93650]: osdmap e1049: 8 total, 8 up, 8 in 2024-04-12T11:30:55.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:30:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:30:54] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:30:55.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:54 smithi087 ceph-mon[89014]: pgmap v1209: 89 pgs: 89 active+clean; 202 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 588 KiB/s wr, 2 op/s 2024-04-12T11:30:55.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:54 smithi087 ceph-mon[89014]: osdmap e1049: 8 total, 8 up, 8 in 2024-04-12T11:30:55.757 INFO:tasks.rados.rados.0.smithi029.stdout:3887: done (3 left) 2024-04-12T11:30:55.757 INFO:tasks.rados.rados.0.smithi029.stdout:3888: done (2 left) 2024-04-12T11:30:55.757 INFO:tasks.rados.rados.0.smithi029.stdout:3889: done (1 left) 2024-04-12T11:30:55.757 INFO:tasks.rados.rados.0.smithi029.stdout:3891: done (0 left) 2024-04-12T11:30:55.757 INFO:tasks.rados.rados.0.smithi029.stdout:3892: read oid 3 snap -1 2024-04-12T11:30:55.757 INFO:tasks.rados.rados.0.smithi029.stdout:3892: expect (ObjNum 1457 snap 489 seq_num 1457) 2024-04-12T11:30:55.757 INFO:tasks.rados.rados.0.smithi029.stdout:3893: delete oid 34 current snap is 494 2024-04-12T11:30:55.762 INFO:tasks.rados.rados.0.smithi029.stdout:3892: expect (ObjNum 1457 snap 489 seq_num 1457) 2024-04-12T11:30:55.844 INFO:tasks.rados.rados.0.smithi029.stdout:3892: done (1 left) 2024-04-12T11:30:55.844 INFO:tasks.rados.rados.0.smithi029.stdout:3893: done (0 left) 2024-04-12T11:30:55.844 INFO:tasks.rados.rados.0.smithi029.stdout:3894: rollback oid 14 current snap is 494 2024-04-12T11:30:55.844 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 14 to 493 2024-04-12T11:30:55.844 INFO:tasks.rados.rados.0.smithi029.stdout:3895: write oid 26 current snap is 494 2024-04-12T11:30:55.844 INFO:tasks.rados.rados.0.smithi029.stdout:3895: seq_num 1473 ranges {756711=497906,1913035=546509} 2024-04-12T11:30:55.851 INFO:tasks.rados.rados.0.smithi029.stdout:3895: writing smithi02979277-26 from 756711 to 1254617 tid 1 2024-04-12T11:30:55.857 INFO:tasks.rados.rados.0.smithi029.stdout:3895: writing smithi02979277-26 from 1913035 to 2459544 tid 2 2024-04-12T11:30:55.858 INFO:tasks.rados.rados.0.smithi029.stdout:3896: read oid 38 snap -1 2024-04-12T11:30:55.858 INFO:tasks.rados.rados.0.smithi029.stdout:3896: expect (ObjNum 1299 snap 444 seq_num 1299) 2024-04-12T11:30:55.858 INFO:tasks.rados.rados.0.smithi029.stdout:3897: delete oid 21 current snap is 494 2024-04-12T11:30:55.858 INFO:tasks.rados.rados.0.smithi029.stdout:3894: finishing rollback tid 1 to smithi02979277-14 2024-04-12T11:30:55.858 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 14 v 1892 (ObjNum 129 snap 0 seq_num 1800999760) dirty dne 2024-04-12T11:30:55.858 INFO:tasks.rados.rados.0.smithi029.stdout:3895: finishing write tid 1 to smithi02979277-26 2024-04-12T11:30:55.861 INFO:tasks.rados.rados.0.smithi029.stdout:3896: expect (ObjNum 1299 snap 444 seq_num 1299) 2024-04-12T11:30:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:55 smithi029 ceph-mon[93650]: osdmap e1050: 8 total, 8 up, 8 in 2024-04-12T11:30:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:55 smithi029 ceph-mon[97787]: osdmap e1050: 8 total, 8 up, 8 in 2024-04-12T11:30:56.096 INFO:tasks.rados.rados.0.smithi029.stdout:3895: finishing write tid 2 to smithi02979277-26 2024-04-12T11:30:56.097 INFO:tasks.rados.rados.0.smithi029.stdout:3895: finishing write tid 3 to smithi02979277-26 2024-04-12T11:30:56.097 INFO:tasks.rados.rados.0.smithi029.stdout:3895: finishing write tid 4 to smithi02979277-26 2024-04-12T11:30:56.097 INFO:tasks.rados.rados.0.smithi029.stdout:3895: oid 26 updating version 0 to 2522 2024-04-12T11:30:56.097 INFO:tasks.rados.rados.0.smithi029.stdout:3895: oid 26 version 2522 is already newer than 2521 2024-04-12T11:30:56.097 INFO:tasks.rados.rados.0.smithi029.stdout:3895: oid 26 updating version 2522 to 2523 2024-04-12T11:30:56.097 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 2523 (ObjNum 1473 snap 494 seq_num 1473) dirty exists 2024-04-12T11:30:56.097 INFO:tasks.rados.rados.0.smithi029.stdout:3895: left oid 26 (ObjNum 1473 snap 494 seq_num 1473) 2024-04-12T11:30:56.097 INFO:tasks.rados.rados.0.smithi029.stdout:3894: done (3 left) 2024-04-12T11:30:56.097 INFO:tasks.rados.rados.0.smithi029.stdout:3895: done (2 left) 2024-04-12T11:30:56.097 INFO:tasks.rados.rados.0.smithi029.stdout:3896: done (1 left) 2024-04-12T11:30:56.098 INFO:tasks.rados.rados.0.smithi029.stdout:3897: done (0 left) 2024-04-12T11:30:56.098 INFO:tasks.rados.rados.0.smithi029.stdout:3898: write oid 42 current snap is 494 2024-04-12T11:30:56.098 INFO:tasks.rados.rados.0.smithi029.stdout:3898: seq_num 1475 ranges {790325=764757,2029999=550178} 2024-04-12T11:30:56.104 INFO:tasks.rados.rados.0.smithi029.stdout:3898: writing smithi02979277-42 from 790325 to 1555082 tid 1 2024-04-12T11:30:56.110 INFO:tasks.rados.rados.0.smithi029.stdout:3898: writing smithi02979277-42 from 2029999 to 2580177 tid 2 2024-04-12T11:30:56.111 INFO:tasks.rados.rados.0.smithi029.stdout:3899: delete oid 29 current snap is 494 2024-04-12T11:30:56.111 INFO:tasks.rados.rados.0.smithi029.stdout:3898: finishing write tid 1 to smithi02979277-42 2024-04-12T11:30:56.113 INFO:tasks.rados.rados.0.smithi029.stdout:3899: done (1 left) 2024-04-12T11:30:56.113 INFO:tasks.rados.rados.0.smithi029.stdout:3900: write oid 27 current snap is 494 2024-04-12T11:30:56.113 INFO:tasks.rados.rados.0.smithi029.stdout:3900: seq_num 1477 ranges {423939=515255,1346963=553847,2643878=180061} 2024-04-12T11:30:56.118 INFO:tasks.rados.rados.0.smithi029.stdout:3900: writing smithi02979277-27 from 423939 to 939194 tid 1 2024-04-12T11:30:56.123 INFO:tasks.rados.rados.0.smithi029.stdout:3900: writing smithi02979277-27 from 1346963 to 1900810 tid 2 2024-04-12T11:30:56.129 INFO:tasks.rados.rados.0.smithi029.stdout:3900: writing smithi02979277-27 from 2643878 to 2823939 tid 3 2024-04-12T11:30:56.129 INFO:tasks.rados.rados.0.smithi029.stdout:3901: read oid 38 snap -1 2024-04-12T11:30:56.129 INFO:tasks.rados.rados.0.smithi029.stdout:3901: expect (ObjNum 1299 snap 444 seq_num 1299) 2024-04-12T11:30:56.129 INFO:tasks.rados.rados.0.smithi029.stdout:3902: write oid 24 current snap is 494 2024-04-12T11:30:56.129 INFO:tasks.rados.rados.0.smithi029.stdout:3898: finishing write tid 2 to smithi02979277-42 2024-04-12T11:30:56.129 INFO:tasks.rados.rados.0.smithi029.stdout:3902: seq_num 1478 ranges {440746=590504,1689092=697505} 2024-04-12T11:30:56.134 INFO:tasks.rados.rados.0.smithi029.stdout:3902: writing smithi02979277-24 from 440746 to 1031250 tid 1 2024-04-12T11:30:56.142 INFO:tasks.rados.rados.0.smithi029.stdout:3902: writing smithi02979277-24 from 1689092 to 2386597 tid 2 2024-04-12T11:30:56.142 INFO:tasks.rados.rados.0.smithi029.stdout:3903: delete oid 46 current snap is 494 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3898: finishing write tid 3 to smithi02979277-42 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3898: finishing write tid 4 to smithi02979277-42 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3898: oid 42 updating version 0 to 479 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3898: oid 42 updating version 479 to 481 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3898: oid 42 version 481 is already newer than 480 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 42 v 481 (ObjNum 1475 snap 494 seq_num 1475) dirty exists 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3898: left oid 42 (ObjNum 1475 snap 494 seq_num 1475) 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3900: finishing write tid 1 to smithi02979277-27 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3900: finishing write tid 2 to smithi02979277-27 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3900: finishing write tid 3 to smithi02979277-27 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3900: finishing write tid 4 to smithi02979277-27 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3900: finishing write tid 5 to smithi02979277-27 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3900: oid 27 updating version 0 to 1252 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3900: oid 27 updating version 1252 to 1253 2024-04-12T11:30:56.143 INFO:tasks.rados.rados.0.smithi029.stdout:3900: oid 27 updating version 1253 to 1255 2024-04-12T11:30:56.144 INFO:tasks.rados.rados.0.smithi029.stdout:3900: oid 27 version 1255 is already newer than 1254 2024-04-12T11:30:56.144 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 1255 (ObjNum 1477 snap 494 seq_num 1477) dirty exists 2024-04-12T11:30:56.144 INFO:tasks.rados.rados.0.smithi029.stdout:3900: left oid 27 (ObjNum 1477 snap 494 seq_num 1477) 2024-04-12T11:30:56.144 INFO:tasks.rados.rados.0.smithi029.stdout:3901: expect (ObjNum 1299 snap 444 seq_num 1299) 2024-04-12T11:30:56.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:55 smithi087 ceph-mon[89014]: osdmap e1050: 8 total, 8 up, 8 in 2024-04-12T11:30:56.377 INFO:tasks.rados.rados.0.smithi029.stdout:3902: finishing write tid 1 to smithi02979277-24 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:3902: finishing write tid 2 to smithi02979277-24 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:3902: finishing write tid 3 to smithi02979277-24 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:3902: finishing write tid 4 to smithi02979277-24 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:3902: oid 24 updating version 0 to 2527 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:3902: oid 24 version 2527 is already newer than 2526 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:3902: oid 24 version 2527 is already newer than 2525 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 24 v 2527 (ObjNum 1478 snap 494 seq_num 1478) dirty exists 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:3902: left oid 24 (ObjNum 1478 snap 494 seq_num 1478) 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:3898: done (4 left) 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:3900: done (3 left) 2024-04-12T11:30:56.378 INFO:tasks.rados.rados.0.smithi029.stdout:3901: done (2 left) 2024-04-12T11:30:56.379 INFO:tasks.rados.rados.0.smithi029.stdout:3902: done (1 left) 2024-04-12T11:30:56.379 INFO:tasks.rados.rados.0.smithi029.stdout:3903: done (0 left) 2024-04-12T11:30:56.379 INFO:tasks.rados.rados.0.smithi029.stdout:3904: snap_create 2024-04-12T11:30:56.763 INFO:tasks.rados.rados.0.smithi029.stdout:3904: done (0 left) 2024-04-12T11:30:56.763 INFO:tasks.rados.rados.0.smithi029.stdout:3905: read oid 41 snap 491 2024-04-12T11:30:56.763 INFO:tasks.rados.rados.0.smithi029.stdout:3905: expect (ObjNum 1452 snap 487 seq_num 1452) 2024-04-12T11:30:56.763 INFO:tasks.rados.rados.0.smithi029.stdout:3906: write oid 25 current snap is 495 2024-04-12T11:30:56.763 INFO:tasks.rados.rados.0.smithi029.stdout:3906: seq_num 1480 ranges {474360=741002,1806056=701174} 2024-04-12T11:30:56.769 INFO:tasks.rados.rados.0.smithi029.stdout:3906: writing smithi02979277-25 from 474360 to 1215362 tid 1 2024-04-12T11:30:56.777 INFO:tasks.rados.rados.0.smithi029.stdout:3906: writing smithi02979277-25 from 1806056 to 2507230 tid 2 2024-04-12T11:30:56.777 INFO:tasks.rados.rados.0.smithi029.stdout:3907: snap_remove snap 492 2024-04-12T11:30:57.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:56 smithi029 ceph-mon[93650]: pgmap v1212: 89 pgs: 89 active+clean; 178 MiB data, 786 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 588 KiB/s wr, 3 op/s 2024-04-12T11:30:57.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:56 smithi029 ceph-mon[93650]: osdmap e1051: 8 total, 8 up, 8 in 2024-04-12T11:30:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:56 smithi029 ceph-mon[97787]: pgmap v1212: 89 pgs: 89 active+clean; 178 MiB data, 786 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 588 KiB/s wr, 3 op/s 2024-04-12T11:30:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:56 smithi029 ceph-mon[97787]: osdmap e1051: 8 total, 8 up, 8 in 2024-04-12T11:30:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:56 smithi087 ceph-mon[89014]: pgmap v1212: 89 pgs: 89 active+clean; 178 MiB data, 786 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 588 KiB/s wr, 3 op/s 2024-04-12T11:30:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:56 smithi087 ceph-mon[89014]: osdmap e1051: 8 total, 8 up, 8 in 2024-04-12T11:30:57.736 INFO:tasks.rados.rados.0.smithi029.stdout:3907: done (2 left) 2024-04-12T11:30:57.736 INFO:tasks.rados.rados.0.smithi029.stdout:3908: write oid 2 current snap is 495 2024-04-12T11:30:57.736 INFO:tasks.rados.rados.0.smithi029.stdout:3908: seq_num 1481 ranges {491167=416251,1464538=561185,2753570=137597} 2024-04-12T11:30:57.741 INFO:tasks.rados.rados.0.smithi029.stdout:3908: writing smithi02979277-2 from 491167 to 907418 tid 1 2024-04-12T11:30:57.748 INFO:tasks.rados.rados.0.smithi029.stdout:3908: writing smithi02979277-2 from 1464538 to 2025723 tid 2 2024-04-12T11:30:57.753 INFO:tasks.rados.rados.0.smithi029.stdout:3908: writing smithi02979277-2 from 2753570 to 2891167 tid 3 2024-04-12T11:30:57.753 INFO:tasks.rados.rados.0.smithi029.stdout:3909: read oid 39 snap -1 2024-04-12T11:30:57.754 INFO:tasks.rados.rados.0.smithi029.stdout:3909: expect (ObjNum 1088 snap 365 seq_num 1088) 2024-04-12T11:30:57.754 INFO:tasks.rados.rados.0.smithi029.stdout:3910: snap_remove snap 473 2024-04-12T11:30:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:57 smithi029 ceph-mon[93650]: osdmap e1052: 8 total, 8 up, 8 in 2024-04-12T11:30:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:57 smithi029 ceph-mon[93650]: osdmap e1053: 8 total, 8 up, 8 in 2024-04-12T11:30:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:57 smithi029 ceph-mon[97787]: osdmap e1052: 8 total, 8 up, 8 in 2024-04-12T11:30:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:57 smithi029 ceph-mon[97787]: osdmap e1053: 8 total, 8 up, 8 in 2024-04-12T11:30:58.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:57 smithi087 ceph-mon[89014]: osdmap e1052: 8 total, 8 up, 8 in 2024-04-12T11:30:58.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:57 smithi087 ceph-mon[89014]: osdmap e1053: 8 total, 8 up, 8 in 2024-04-12T11:30:58.739 INFO:tasks.rados.rados.0.smithi029.stdout:3910: done (4 left) 2024-04-12T11:30:58.740 INFO:tasks.rados.rados.0.smithi029.stdout:3911: read oid 3 snap 480 2024-04-12T11:30:58.740 INFO:tasks.rados.rados.0.smithi029.stdout:3911: expect (ObjNum 1406 snap 472 seq_num 1406) 2024-04-12T11:30:58.740 INFO:tasks.rados.rados.0.smithi029.stdout:3912: read oid 46 snap -1 2024-04-12T11:30:58.740 INFO:tasks.rados.rados.0.smithi029.stdout:3912: expect deleted 2024-04-12T11:30:58.740 INFO:tasks.rados.rados.0.smithi029.stdout:3913: write oid 18 current snap is 495 2024-04-12T11:30:58.740 INFO:tasks.rados.rados.0.smithi029.stdout:3913: seq_num 1482 ranges {507974=491500,1406667=704843,2548287=359687} 2024-04-12T11:30:58.746 INFO:tasks.rados.rados.0.smithi029.stdout:3913: writing smithi02979277-18 from 507974 to 999474 tid 1 2024-04-12T11:30:58.753 INFO:tasks.rados.rados.0.smithi029.stdout:3913: writing smithi02979277-18 from 1406667 to 2111510 tid 2 2024-04-12T11:30:58.758 INFO:tasks.rados.rados.0.smithi029.stdout:3913: writing smithi02979277-18 from 2548287 to 2907974 tid 3 2024-04-12T11:30:58.758 INFO:tasks.rados.rados.0.smithi029.stdout:3914: delete oid 38 current snap is 495 2024-04-12T11:30:58.758 INFO:tasks.rados.rados.0.smithi029.stdout:3905: expect (ObjNum 1452 snap 487 seq_num 1452) 2024-04-12T11:30:58.855 INFO:tasks.rados.rados.0.smithi029.stdout:3906: finishing write tid 1 to smithi02979277-25 2024-04-12T11:30:58.855 INFO:tasks.rados.rados.0.smithi029.stdout:3906: finishing write tid 2 to smithi02979277-25 2024-04-12T11:30:58.855 INFO:tasks.rados.rados.0.smithi029.stdout:3906: finishing write tid 3 to smithi02979277-25 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3906: finishing write tid 4 to smithi02979277-25 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3906: oid 25 updating version 0 to 2086 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3906: oid 25 version 2086 is already newer than 2085 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3906: oid 25 version 2086 is already newer than 2084 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 25 v 2086 (ObjNum 1480 snap 495 seq_num 1480) dirty exists 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3906: left oid 25 (ObjNum 1480 snap 495 seq_num 1480) 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3908: finishing write tid 1 to smithi02979277-2 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3908: finishing write tid 2 to smithi02979277-2 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3905: done (7 left) 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3906: done (6 left) 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3914: done (5 left) 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3915: rollback oid 8 current snap is 495 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 8 to 489 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3908: finishing write tid 3 to smithi02979277-2 2024-04-12T11:30:58.856 INFO:tasks.rados.rados.0.smithi029.stdout:3908: finishing write tid 4 to smithi02979277-2 2024-04-12T11:30:58.857 INFO:tasks.rados.rados.0.smithi029.stdout:3908: finishing write tid 5 to smithi02979277-2 2024-04-12T11:30:58.857 INFO:tasks.rados.rados.0.smithi029.stdout:3908: oid 2 updating version 0 to 447 2024-04-12T11:30:58.857 INFO:tasks.rados.rados.0.smithi029.stdout:3908: oid 2 updating version 447 to 448 2024-04-12T11:30:58.857 INFO:tasks.rados.rados.0.smithi029.stdout:3908: oid 2 updating version 448 to 449 2024-04-12T11:30:58.857 INFO:tasks.rados.rados.0.smithi029.stdout:3908: oid 2 updating version 449 to 450 2024-04-12T11:30:58.857 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 450 (ObjNum 1481 snap 495 seq_num 1481) dirty exists 2024-04-12T11:30:58.857 INFO:tasks.rados.rados.0.smithi029.stdout:3908: left oid 2 (ObjNum 1481 snap 495 seq_num 1481) 2024-04-12T11:30:58.857 INFO:tasks.rados.rados.0.smithi029.stdout:3909: expect (ObjNum 1088 snap 365 seq_num 1088) 2024-04-12T11:30:58.978 INFO:tasks.rados.rados.0.smithi029.stdout:3911: expect (ObjNum 1406 snap 472 seq_num 1406) 2024-04-12T11:30:59.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:58 smithi029 ceph-mon[93650]: pgmap v1215: 89 pgs: 89 active+clean; 185 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-04-12T11:30:59.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:30:58 smithi029 ceph-mon[93650]: osdmap e1054: 8 total, 8 up, 8 in 2024-04-12T11:30:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:58 smithi029 ceph-mon[97787]: pgmap v1215: 89 pgs: 89 active+clean; 185 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-04-12T11:30:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:30:58 smithi029 ceph-mon[97787]: osdmap e1054: 8 total, 8 up, 8 in 2024-04-12T11:30:59.147 INFO:tasks.rados.rados.0.smithi029.stdout:3913: finishing write tid 1 to smithi02979277-18 2024-04-12T11:30:59.147 INFO:tasks.rados.rados.0.smithi029.stdout:3913: finishing write tid 2 to smithi02979277-18 2024-04-12T11:30:59.147 INFO:tasks.rados.rados.0.smithi029.stdout:3913: finishing write tid 3 to smithi02979277-18 2024-04-12T11:30:59.147 INFO:tasks.rados.rados.0.smithi029.stdout:3913: finishing write tid 4 to smithi02979277-18 2024-04-12T11:30:59.147 INFO:tasks.rados.rados.0.smithi029.stdout:3913: finishing write tid 5 to smithi02979277-18 2024-04-12T11:30:59.147 INFO:tasks.rados.rados.0.smithi029.stdout:3913: oid 18 updating version 0 to 2098 2024-04-12T11:30:59.147 INFO:tasks.rados.rados.0.smithi029.stdout:3913: oid 18 version 2098 is already newer than 2097 2024-04-12T11:30:59.147 INFO:tasks.rados.rados.0.smithi029.stdout:3913: oid 18 updating version 2098 to 2099 2024-04-12T11:30:59.147 INFO:tasks.rados.rados.0.smithi029.stdout:3913: oid 18 updating version 2099 to 2100 2024-04-12T11:30:59.148 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 2100 (ObjNum 1482 snap 495 seq_num 1482) dirty exists 2024-04-12T11:30:59.148 INFO:tasks.rados.rados.0.smithi029.stdout:3913: left oid 18 (ObjNum 1482 snap 495 seq_num 1482) 2024-04-12T11:30:59.148 INFO:tasks.rados.rados.0.smithi029.stdout:3908: done (5 left) 2024-04-12T11:30:59.148 INFO:tasks.rados.rados.0.smithi029.stdout:3909: done (4 left) 2024-04-12T11:30:59.148 INFO:tasks.rados.rados.0.smithi029.stdout:3911: done (3 left) 2024-04-12T11:30:59.148 INFO:tasks.rados.rados.0.smithi029.stdout:3912: done (2 left) 2024-04-12T11:30:59.148 INFO:tasks.rados.rados.0.smithi029.stdout:3913: done (1 left) 2024-04-12T11:30:59.148 INFO:tasks.rados.rados.0.smithi029.stdout:3916: read oid 23 snap -1 2024-04-12T11:30:59.148 INFO:tasks.rados.rados.0.smithi029.stdout:3916: expect (ObjNum 1468 snap 492 seq_num 1468) 2024-04-12T11:30:59.148 INFO:tasks.rados.rados.0.smithi029.stdout:3917: snap_remove snap 488 2024-04-12T11:30:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:58 smithi087 ceph-mon[89014]: pgmap v1215: 89 pgs: 89 active+clean; 185 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-04-12T11:30:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:30:58 smithi087 ceph-mon[89014]: osdmap e1054: 8 total, 8 up, 8 in 2024-04-12T11:30:59.775 INFO:tasks.rados.rados.0.smithi029.stdout:3917: done (2 left) 2024-04-12T11:30:59.776 INFO:tasks.rados.rados.0.smithi029.stdout:3918: snap_remove snap 487 2024-04-12T11:31:00.781 INFO:tasks.rados.rados.0.smithi029.stdout:3918: done (2 left) 2024-04-12T11:31:00.781 INFO:tasks.rados.rados.0.smithi029.stdout:3919: snap_remove snap 484 2024-04-12T11:31:01.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:00 smithi087 ceph-mon[89014]: pgmap v1218: 89 pgs: 89 active+clean; 194 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 3.9 MiB/s wr, 38 op/s 2024-04-12T11:31:01.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:00 smithi087 ceph-mon[89014]: osdmap e1055: 8 total, 8 up, 8 in 2024-04-12T11:31:01.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:00 smithi029 ceph-mon[93650]: pgmap v1218: 89 pgs: 89 active+clean; 194 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 3.9 MiB/s wr, 38 op/s 2024-04-12T11:31:01.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:00 smithi029 ceph-mon[93650]: osdmap e1055: 8 total, 8 up, 8 in 2024-04-12T11:31:01.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:00 smithi029 ceph-mon[97787]: pgmap v1218: 89 pgs: 89 active+clean; 194 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 3.9 MiB/s wr, 38 op/s 2024-04-12T11:31:01.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:00 smithi029 ceph-mon[97787]: osdmap e1055: 8 total, 8 up, 8 in 2024-04-12T11:31:01.789 INFO:tasks.rados.rados.0.smithi029.stdout:3919: done (2 left) 2024-04-12T11:31:01.789 INFO:tasks.rados.rados.0.smithi029.stdout:3920: rollback oid 18 current snap is 495 2024-04-12T11:31:01.789 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 18 to 491 2024-04-12T11:31:01.790 INFO:tasks.rados.rados.0.smithi029.stdout:3915: finishing rollback tid 0 to smithi02979277-8 2024-04-12T11:31:01.790 INFO:tasks.rados.rados.0.smithi029.stdout:3915: finishing rollback tid 1 to smithi02979277-8 2024-04-12T11:31:01.790 INFO:tasks.rados.rados.0.smithi029.stdout:3915: finishing rollback tid 2 to smithi02979277-8 2024-04-12T11:31:01.790 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 8 v 392 (ObjNum 1395 snap 469 seq_num 1395) dirty exists 2024-04-12T11:31:01.790 INFO:tasks.rados.rados.0.smithi029.stdout:3915: done (2 left) 2024-04-12T11:31:01.790 INFO:tasks.rados.rados.0.smithi029.stdout:3921: snap_remove snap 471 2024-04-12T11:31:02.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:01 smithi087 ceph-mon[89014]: osdmap e1056: 8 total, 8 up, 8 in 2024-04-12T11:31:02.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:01 smithi029 ceph-mon[93650]: osdmap e1056: 8 total, 8 up, 8 in 2024-04-12T11:31:02.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:01 smithi029 ceph-mon[97787]: osdmap e1056: 8 total, 8 up, 8 in 2024-04-12T11:31:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:31:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:31:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:31:02.790 INFO:tasks.rados.rados.0.smithi029.stdout:3921: done (2 left) 2024-04-12T11:31:02.791 INFO:tasks.rados.rados.0.smithi029.stdout:3922: read oid 30 snap 491 2024-04-12T11:31:02.791 INFO:tasks.rados.rados.0.smithi029.stdout:3922: expect (ObjNum 1456 snap 487 seq_num 1456) 2024-04-12T11:31:02.791 INFO:tasks.rados.rados.0.smithi029.stdout:3923: snap_create 2024-04-12T11:31:02.791 INFO:tasks.rados.rados.0.smithi029.stdout:3916: expect (ObjNum 1468 snap 492 seq_num 1468) 2024-04-12T11:31:02.910 INFO:tasks.rados.rados.0.smithi029.stdout:3920: finishing rollback tid 0 to smithi02979277-18 2024-04-12T11:31:02.910 INFO:tasks.rados.rados.0.smithi029.stdout:3920: finishing rollback tid 1 to smithi02979277-18 2024-04-12T11:31:02.910 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 2114 (ObjNum 49 snap 0 seq_num 1777349616) dirty dne 2024-04-12T11:31:02.910 INFO:tasks.rados.rados.0.smithi029.stdout:3922: expect (ObjNum 1456 snap 487 seq_num 1456) 2024-04-12T11:31:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:02 smithi087 ceph-mon[89014]: pgmap v1221: 89 pgs: 89 active+clean; 196 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-04-12T11:31:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:02 smithi087 ceph-mon[89014]: osdmap e1057: 8 total, 8 up, 8 in 2024-04-12T11:31:03.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:02 smithi029 ceph-mon[97787]: pgmap v1221: 89 pgs: 89 active+clean; 196 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-04-12T11:31:03.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:02 smithi029 ceph-mon[97787]: osdmap e1057: 8 total, 8 up, 8 in 2024-04-12T11:31:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:02 smithi029 ceph-mon[93650]: pgmap v1221: 89 pgs: 89 active+clean; 196 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-04-12T11:31:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:02 smithi029 ceph-mon[93650]: osdmap e1057: 8 total, 8 up, 8 in 2024-04-12T11:31:03.790 INFO:tasks.rados.rados.0.smithi029.stdout:3916: done (3 left) 2024-04-12T11:31:03.790 INFO:tasks.rados.rados.0.smithi029.stdout:3920: done (2 left) 2024-04-12T11:31:03.790 INFO:tasks.rados.rados.0.smithi029.stdout:3922: done (1 left) 2024-04-12T11:31:03.790 INFO:tasks.rados.rados.0.smithi029.stdout:3923: done (0 left) 2024-04-12T11:31:03.791 INFO:tasks.rados.rados.0.smithi029.stdout:3924: snap_remove snap 495 2024-04-12T11:31:04.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:03 smithi087 ceph-mon[89014]: osdmap e1058: 8 total, 8 up, 8 in 2024-04-12T11:31:04.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:03 smithi087 ceph-mon[89014]: osdmap e1059: 8 total, 8 up, 8 in 2024-04-12T11:31:04.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:03 smithi029 ceph-mon[97787]: osdmap e1058: 8 total, 8 up, 8 in 2024-04-12T11:31:04.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:03 smithi029 ceph-mon[97787]: osdmap e1059: 8 total, 8 up, 8 in 2024-04-12T11:31:04.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:03 smithi029 ceph-mon[93650]: osdmap e1058: 8 total, 8 up, 8 in 2024-04-12T11:31:04.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:03 smithi029 ceph-mon[93650]: osdmap e1059: 8 total, 8 up, 8 in 2024-04-12T11:31:04.793 INFO:tasks.rados.rados.0.smithi029.stdout:3924: done (0 left) 2024-04-12T11:31:04.793 INFO:tasks.rados.rados.0.smithi029.stdout:3925: write oid 9 current snap is 496 2024-04-12T11:31:04.793 INFO:tasks.rados.rados.0.smithi029.stdout:3925: seq_num 1484 ranges {541588=758351,2039984=708512} 2024-04-12T11:31:04.801 INFO:tasks.rados.rados.0.smithi029.stdout:3925: writing smithi02979277-9 from 541588 to 1299939 tid 1 2024-04-12T11:31:04.809 INFO:tasks.rados.rados.0.smithi029.stdout:3925: writing smithi02979277-9 from 2039984 to 2748496 tid 2 2024-04-12T11:31:04.810 INFO:tasks.rados.rados.0.smithi029.stdout:3926: rollback oid 5 current snap is 496 2024-04-12T11:31:04.810 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 5 to 490 2024-04-12T11:31:04.810 INFO:tasks.rados.rados.0.smithi029.stdout:3925: finishing write tid 1 to smithi02979277-9 2024-04-12T11:31:04.810 INFO:tasks.rados.rados.0.smithi029.stdout:3927: delete oid 39 current snap is 496 2024-04-12T11:31:04.813 INFO:tasks.rados.rados.0.smithi029.stdout:3927: done (2 left) 2024-04-12T11:31:04.813 INFO:tasks.rados.rados.0.smithi029.stdout:3928: snap_remove snap 494 2024-04-12T11:31:05.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:04 smithi087 ceph-mon[89014]: pgmap v1224: 89 pgs: 89 active+clean; 196 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 701 KiB/s wr, 5 op/s 2024-04-12T11:31:05.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:04 smithi087 ceph-mon[89014]: osdmap e1060: 8 total, 8 up, 8 in 2024-04-12T11:31:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:31:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:31:04] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2024-04-12T11:31:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:04 smithi029 ceph-mon[97787]: pgmap v1224: 89 pgs: 89 active+clean; 196 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 701 KiB/s wr, 5 op/s 2024-04-12T11:31:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:04 smithi029 ceph-mon[97787]: osdmap e1060: 8 total, 8 up, 8 in 2024-04-12T11:31:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:04 smithi029 ceph-mon[93650]: pgmap v1224: 89 pgs: 89 active+clean; 196 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 701 KiB/s wr, 5 op/s 2024-04-12T11:31:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:04 smithi029 ceph-mon[93650]: osdmap e1060: 8 total, 8 up, 8 in 2024-04-12T11:31:05.805 INFO:tasks.rados.rados.0.smithi029.stdout:3928: done (2 left) 2024-04-12T11:31:05.806 INFO:tasks.rados.rados.0.smithi029.stdout:3929: write oid 50 current snap is 496 2024-04-12T11:31:05.806 INFO:tasks.rados.rados.0.smithi029.stdout:3926: finishing rollback tid 1 to smithi02979277-5 2024-04-12T11:31:05.806 INFO:tasks.rados.rados.0.smithi029.stdout:3929: seq_num 1486 ranges {575202=508849,1640595=712181} 2024-04-12T11:31:05.811 INFO:tasks.rados.rados.0.smithi029.stdout:3929: writing smithi02979277-50 from 575202 to 1084051 tid 1 2024-04-12T11:31:05.819 INFO:tasks.rados.rados.0.smithi029.stdout:3929: writing smithi02979277-50 from 1640595 to 2352776 tid 2 2024-04-12T11:31:05.819 INFO:tasks.rados.rados.0.smithi029.stdout:3930: snap_create 2024-04-12T11:31:05.819 INFO:tasks.rados.rados.0.smithi029.stdout:3926: finishing rollback tid 2 to smithi02979277-5 2024-04-12T11:31:05.819 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 5 v 1972 (ObjNum 1364 snap 462 seq_num 1364) dirty exists 2024-04-12T11:31:05.819 INFO:tasks.rados.rados.0.smithi029.stdout:3925: finishing write tid 2 to smithi02979277-9 2024-04-12T11:31:05.819 INFO:tasks.rados.rados.0.smithi029.stdout:3925: finishing write tid 3 to smithi02979277-9 2024-04-12T11:31:05.820 INFO:tasks.rados.rados.0.smithi029.stdout:3925: finishing write tid 4 to smithi02979277-9 2024-04-12T11:31:05.820 INFO:tasks.rados.rados.0.smithi029.stdout:3925: oid 9 updating version 0 to 1389 2024-04-12T11:31:05.820 INFO:tasks.rados.rados.0.smithi029.stdout:3925: oid 9 version 1389 is already newer than 1388 2024-04-12T11:31:05.820 INFO:tasks.rados.rados.0.smithi029.stdout:3925: oid 9 updating version 1389 to 1390 2024-04-12T11:31:05.820 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 9 v 1390 (ObjNum 1484 snap 496 seq_num 1484) dirty exists 2024-04-12T11:31:05.820 INFO:tasks.rados.rados.0.smithi029.stdout:3925: left oid 9 (ObjNum 1484 snap 496 seq_num 1484) 2024-04-12T11:31:05.820 INFO:tasks.rados.rados.0.smithi029.stdout:3929: finishing write tid 1 to smithi02979277-50 2024-04-12T11:31:05.824 INFO:tasks.rados.rados.0.smithi029.stdout:3929: finishing write tid 2 to smithi02979277-50 2024-04-12T11:31:05.824 INFO:tasks.rados.rados.0.smithi029.stdout:3929: finishing write tid 3 to smithi02979277-50 2024-04-12T11:31:05.825 INFO:tasks.rados.rados.0.smithi029.stdout:3929: finishing write tid 4 to smithi02979277-50 2024-04-12T11:31:05.825 INFO:tasks.rados.rados.0.smithi029.stdout:3929: oid 50 updating version 0 to 1678 2024-04-12T11:31:05.825 INFO:tasks.rados.rados.0.smithi029.stdout:3929: oid 50 version 1678 is already newer than 1677 2024-04-12T11:31:05.825 INFO:tasks.rados.rados.0.smithi029.stdout:3929: oid 50 version 1678 is already newer than 1676 2024-04-12T11:31:05.825 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 50 v 1678 (ObjNum 1486 snap 496 seq_num 1486) dirty exists 2024-04-12T11:31:05.825 INFO:tasks.rados.rados.0.smithi029.stdout:3929: left oid 50 (ObjNum 1486 snap 496 seq_num 1486) 2024-04-12T11:31:06.811 INFO:tasks.rados.rados.0.smithi029.stdout:3925: done (3 left) 2024-04-12T11:31:06.811 INFO:tasks.rados.rados.0.smithi029.stdout:3926: done (2 left) 2024-04-12T11:31:06.811 INFO:tasks.rados.rados.0.smithi029.stdout:3929: done (1 left) 2024-04-12T11:31:06.811 INFO:tasks.rados.rados.0.smithi029.stdout:3930: done (0 left) 2024-04-12T11:31:06.812 INFO:tasks.rados.rados.0.smithi029.stdout:3931: read oid 13 snap -1 2024-04-12T11:31:06.812 INFO:tasks.rados.rados.0.smithi029.stdout:3931: expect (ObjNum 1470 snap 493 seq_num 1470) 2024-04-12T11:31:06.812 INFO:tasks.rados.rados.0.smithi029.stdout:3932: write oid 45 current snap is 497 2024-04-12T11:31:06.812 INFO:tasks.rados.rados.0.smithi029.stdout:3932: seq_num 1487 ranges {592009=584098,1699077=572192,2801755=190254} 2024-04-12T11:31:06.817 INFO:tasks.rados.rados.0.smithi029.stdout:3932: writing smithi02979277-45 from 592009 to 1176107 tid 1 2024-04-12T11:31:06.823 INFO:tasks.rados.rados.0.smithi029.stdout:3932: writing smithi02979277-45 from 1699077 to 2271269 tid 2 2024-04-12T11:31:06.828 INFO:tasks.rados.rados.0.smithi029.stdout:3932: writing smithi02979277-45 from 2801755 to 2992009 tid 3 2024-04-12T11:31:06.828 INFO:tasks.rados.rados.0.smithi029.stdout:3933: snap_remove snap 485 2024-04-12T11:31:07.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:06 smithi087 ceph-mon[89014]: pgmap v1227: 89 pgs: 89 active+clean; 195 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 380 KiB/s wr, 6 op/s 2024-04-12T11:31:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:06 smithi087 ceph-mon[89014]: osdmap e1061: 8 total, 8 up, 8 in 2024-04-12T11:31:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:07.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:06 smithi029 ceph-mon[93650]: pgmap v1227: 89 pgs: 89 active+clean; 195 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 380 KiB/s wr, 6 op/s 2024-04-12T11:31:07.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:06 smithi029 ceph-mon[93650]: osdmap e1061: 8 total, 8 up, 8 in 2024-04-12T11:31:07.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:07.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:06 smithi029 ceph-mon[97787]: pgmap v1227: 89 pgs: 89 active+clean; 195 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 380 KiB/s wr, 6 op/s 2024-04-12T11:31:07.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:06 smithi029 ceph-mon[97787]: osdmap e1061: 8 total, 8 up, 8 in 2024-04-12T11:31:07.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:07.742 INFO:tasks.rados.rados.0.smithi029.stdout:3931: expect (ObjNum 1470 snap 493 seq_num 1470) 2024-04-12T11:31:07.907 INFO:tasks.rados.rados.0.smithi029.stdout:3932: finishing write tid 1 to smithi02979277-45 2024-04-12T11:31:07.907 INFO:tasks.rados.rados.0.smithi029.stdout:3932: finishing write tid 2 to smithi02979277-45 2024-04-12T11:31:07.907 INFO:tasks.rados.rados.0.smithi029.stdout:3932: finishing write tid 3 to smithi02979277-45 2024-04-12T11:31:07.907 INFO:tasks.rados.rados.0.smithi029.stdout:3932: finishing write tid 4 to smithi02979277-45 2024-04-12T11:31:07.907 INFO:tasks.rados.rados.0.smithi029.stdout:3932: finishing write tid 5 to smithi02979277-45 2024-04-12T11:31:07.907 INFO:tasks.rados.rados.0.smithi029.stdout:3932: oid 45 updating version 0 to 1964 2024-04-12T11:31:07.907 INFO:tasks.rados.rados.0.smithi029.stdout:3932: oid 45 version 1964 is already newer than 1962 2024-04-12T11:31:07.907 INFO:tasks.rados.rados.0.smithi029.stdout:3932: oid 45 version 1964 is already newer than 1963 2024-04-12T11:31:07.908 INFO:tasks.rados.rados.0.smithi029.stdout:3932: oid 45 version 1964 is already newer than 1961 2024-04-12T11:31:07.908 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1964 (ObjNum 1487 snap 497 seq_num 1487) dirty exists 2024-04-12T11:31:07.908 INFO:tasks.rados.rados.0.smithi029.stdout:3932: left oid 45 (ObjNum 1487 snap 497 seq_num 1487) 2024-04-12T11:31:07.908 INFO:tasks.rados.rados.0.smithi029.stdout:3931: done (2 left) 2024-04-12T11:31:07.908 INFO:tasks.rados.rados.0.smithi029.stdout:3932: done (1 left) 2024-04-12T11:31:07.908 INFO:tasks.rados.rados.0.smithi029.stdout:3933: done (0 left) 2024-04-12T11:31:07.908 INFO:tasks.rados.rados.0.smithi029.stdout:3934: write oid 45 current snap is 497 2024-04-12T11:31:07.908 INFO:tasks.rados.rados.0.smithi029.stdout:3934: seq_num 1488 ranges {608816=659347,1757559=715850} 2024-04-12T11:31:07.913 INFO:tasks.rados.rados.0.smithi029.stdout:3934: writing smithi02979277-45 from 608816 to 1268163 tid 1 2024-04-12T11:31:07.920 INFO:tasks.rados.rados.0.smithi029.stdout:3934: writing smithi02979277-45 from 1757559 to 2473409 tid 2 2024-04-12T11:31:07.920 INFO:tasks.rados.rados.0.smithi029.stdout:3935: rollback oid 23 current snap is 497 2024-04-12T11:31:07.920 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 23 to 477 2024-04-12T11:31:07.921 INFO:tasks.rados.rados.0.smithi029.stdout:3934: finishing write tid 1 to smithi02979277-45 2024-04-12T11:31:07.921 INFO:tasks.rados.rados.0.smithi029.stdout:3936: snap_create 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:3934: finishing write tid 2 to smithi02979277-45 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:3934: finishing write tid 3 to smithi02979277-45 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:3935: finishing rollback tid 0 to smithi02979277-23 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:3934: finishing write tid 4 to smithi02979277-45 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:3934: oid 45 updating version 0 to 1970 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:3934: oid 45 version 1970 is already newer than 1969 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:3934: oid 45 updating version 1970 to 1971 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 45 v 1971 (ObjNum 1488 snap 497 seq_num 1488) dirty exists 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:3934: left oid 45 (ObjNum 1488 snap 497 seq_num 1488) 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:3935: finishing rollback tid 1 to smithi02979277-23 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:3935: finishing rollback tid 2 to smithi02979277-23 2024-04-12T11:31:07.926 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 23 v 1686 (ObjNum 1353 snap 457 seq_num 1353) dirty exists 2024-04-12T11:31:08.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:07 smithi087 ceph-mon[89014]: osdmap e1062: 8 total, 8 up, 8 in 2024-04-12T11:31:08.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:07 smithi087 ceph-mon[89014]: osdmap e1063: 8 total, 8 up, 8 in 2024-04-12T11:31:08.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:07 smithi029 ceph-mon[97787]: osdmap e1062: 8 total, 8 up, 8 in 2024-04-12T11:31:08.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:07 smithi029 ceph-mon[97787]: osdmap e1063: 8 total, 8 up, 8 in 2024-04-12T11:31:08.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:07 smithi029 ceph-mon[93650]: osdmap e1062: 8 total, 8 up, 8 in 2024-04-12T11:31:08.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:07 smithi029 ceph-mon[93650]: osdmap e1063: 8 total, 8 up, 8 in 2024-04-12T11:31:08.821 INFO:tasks.rados.rados.0.smithi029.stdout:3934: done (2 left) 2024-04-12T11:31:08.821 INFO:tasks.rados.rados.0.smithi029.stdout:3935: done (1 left) 2024-04-12T11:31:08.821 INFO:tasks.rados.rados.0.smithi029.stdout:3936: done (0 left) 2024-04-12T11:31:08.821 INFO:tasks.rados.rados.0.smithi029.stdout:3937: write oid 20 current snap is 498 2024-04-12T11:31:08.821 INFO:tasks.rados.rados.0.smithi029.stdout:3937: seq_num 1489 ranges {625623=734596,1816041=459508,2740248=285375} 2024-04-12T11:31:08.828 INFO:tasks.rados.rados.0.smithi029.stdout:3937: writing smithi02979277-20 from 625623 to 1360219 tid 1 2024-04-12T11:31:08.833 INFO:tasks.rados.rados.0.smithi029.stdout:3937: writing smithi02979277-20 from 1816041 to 2275549 tid 2 2024-04-12T11:31:08.838 INFO:tasks.rados.rados.0.smithi029.stdout:3937: writing smithi02979277-20 from 2740248 to 3025623 tid 3 2024-04-12T11:31:08.838 INFO:tasks.rados.rados.0.smithi029.stdout:3938: write oid 34 current snap is 498 2024-04-12T11:31:08.838 INFO:tasks.rados.rados.0.smithi029.stdout:3938: seq_num 1490 ranges {642430=409845,1758170=719519} 2024-04-12T11:31:08.843 INFO:tasks.rados.rados.0.smithi029.stdout:3938: writing smithi02979277-34 from 642430 to 1052275 tid 1 2024-04-12T11:31:08.852 INFO:tasks.rados.rados.0.smithi029.stdout:3938: writing smithi02979277-34 from 1758170 to 2477689 tid 2 2024-04-12T11:31:08.852 INFO:tasks.rados.rados.0.smithi029.stdout:3939: read oid 46 snap -1 2024-04-12T11:31:08.852 INFO:tasks.rados.rados.0.smithi029.stdout:3939: expect deleted 2024-04-12T11:31:08.852 INFO:tasks.rados.rados.0.smithi029.stdout:3940: snap_create 2024-04-12T11:31:08.852 INFO:tasks.rados.rados.0.smithi029.stdout:3937: finishing write tid 1 to smithi02979277-20 2024-04-12T11:31:08.852 INFO:tasks.rados.rados.0.smithi029.stdout:3937: finishing write tid 2 to smithi02979277-20 2024-04-12T11:31:08.852 INFO:tasks.rados.rados.0.smithi029.stdout:3937: finishing write tid 3 to smithi02979277-20 2024-04-12T11:31:08.852 INFO:tasks.rados.rados.0.smithi029.stdout:3937: finishing write tid 4 to smithi02979277-20 2024-04-12T11:31:08.853 INFO:tasks.rados.rados.0.smithi029.stdout:3937: finishing write tid 5 to smithi02979277-20 2024-04-12T11:31:08.853 INFO:tasks.rados.rados.0.smithi029.stdout:3937: oid 20 updating version 0 to 1330 2024-04-12T11:31:08.853 INFO:tasks.rados.rados.0.smithi029.stdout:3937: oid 20 version 1330 is already newer than 1327 2024-04-12T11:31:08.853 INFO:tasks.rados.rados.0.smithi029.stdout:3937: oid 20 version 1330 is already newer than 1328 2024-04-12T11:31:08.853 INFO:tasks.rados.rados.0.smithi029.stdout:3937: oid 20 version 1330 is already newer than 1329 2024-04-12T11:31:08.853 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 20 v 1330 (ObjNum 1489 snap 498 seq_num 1489) dirty exists 2024-04-12T11:31:08.853 INFO:tasks.rados.rados.0.smithi029.stdout:3937: left oid 20 (ObjNum 1489 snap 498 seq_num 1489) 2024-04-12T11:31:08.853 INFO:tasks.rados.rados.0.smithi029.stdout:3938: finishing write tid 1 to smithi02979277-34 2024-04-12T11:31:08.855 INFO:tasks.rados.rados.0.smithi029.stdout:3938: finishing write tid 2 to smithi02979277-34 2024-04-12T11:31:08.855 INFO:tasks.rados.rados.0.smithi029.stdout:3938: finishing write tid 3 to smithi02979277-34 2024-04-12T11:31:08.855 INFO:tasks.rados.rados.0.smithi029.stdout:3938: finishing write tid 4 to smithi02979277-34 2024-04-12T11:31:08.856 INFO:tasks.rados.rados.0.smithi029.stdout:3938: oid 34 updating version 0 to 1789 2024-04-12T11:31:08.856 INFO:tasks.rados.rados.0.smithi029.stdout:3938: oid 34 version 1789 is already newer than 1788 2024-04-12T11:31:08.856 INFO:tasks.rados.rados.0.smithi029.stdout:3938: oid 34 version 1789 is already newer than 1787 2024-04-12T11:31:08.856 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 34 v 1789 (ObjNum 1490 snap 498 seq_num 1490) dirty exists 2024-04-12T11:31:08.856 INFO:tasks.rados.rados.0.smithi029.stdout:3938: left oid 34 (ObjNum 1490 snap 498 seq_num 1490) 2024-04-12T11:31:09.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:08 smithi087 ceph-mon[89014]: pgmap v1230: 89 pgs: 89 active+clean; 200 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 298 KiB/s wr, 5 op/s 2024-04-12T11:31:09.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:08 smithi029 ceph-mon[93650]: pgmap v1230: 89 pgs: 89 active+clean; 200 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 298 KiB/s wr, 5 op/s 2024-04-12T11:31:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:08 smithi029 ceph-mon[97787]: pgmap v1230: 89 pgs: 89 active+clean; 200 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 298 KiB/s wr, 5 op/s 2024-04-12T11:31:09.825 INFO:tasks.rados.rados.0.smithi029.stdout:3937: done (3 left) 2024-04-12T11:31:09.825 INFO:tasks.rados.rados.0.smithi029.stdout:3938: done (2 left) 2024-04-12T11:31:09.825 INFO:tasks.rados.rados.0.smithi029.stdout:3939: done (1 left) 2024-04-12T11:31:09.825 INFO:tasks.rados.rados.0.smithi029.stdout:3940: done (0 left) 2024-04-12T11:31:09.825 INFO:tasks.rados.rados.0.smithi029.stdout:3941: snap_create 2024-04-12T11:31:10.088 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:09 smithi029 ceph-mon[97787]: osdmap e1064: 8 total, 8 up, 8 in 2024-04-12T11:31:10.088 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:31:10.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:09 smithi029 ceph-mon[93650]: osdmap e1064: 8 total, 8 up, 8 in 2024-04-12T11:31:10.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:31:10.089 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:09 smithi087 ceph-mon[89014]: osdmap e1064: 8 total, 8 up, 8 in 2024-04-12T11:31:10.089 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:31:10.826 INFO:tasks.rados.rados.0.smithi029.stdout:3941: done (0 left) 2024-04-12T11:31:10.826 INFO:tasks.rados.rados.0.smithi029.stdout:3942: snap_remove snap 477 2024-04-12T11:31:11.157 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:10 smithi029 ceph-mon[97787]: pgmap v1233: 89 pgs: 89 active+clean; 204 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 985 KiB/s wr, 8 op/s 2024-04-12T11:31:11.157 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:10 smithi029 ceph-mon[97787]: osdmap e1065: 8 total, 8 up, 8 in 2024-04-12T11:31:11.157 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:10 smithi029 ceph-mon[93650]: pgmap v1233: 89 pgs: 89 active+clean; 204 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 985 KiB/s wr, 8 op/s 2024-04-12T11:31:11.157 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:10 smithi029 ceph-mon[93650]: osdmap e1065: 8 total, 8 up, 8 in 2024-04-12T11:31:11.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:10 smithi087 ceph-mon[89014]: pgmap v1233: 89 pgs: 89 active+clean; 204 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 985 KiB/s wr, 8 op/s 2024-04-12T11:31:11.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:10 smithi087 ceph-mon[89014]: osdmap e1065: 8 total, 8 up, 8 in 2024-04-12T11:31:11.832 INFO:tasks.rados.rados.0.smithi029.stdout:3942: done (0 left) 2024-04-12T11:31:11.832 INFO:tasks.rados.rados.0.smithi029.stdout:3943: write oid 6 current snap is 500 2024-04-12T11:31:11.832 INFO:tasks.rados.rados.0.smithi029.stdout:3943: seq_num 1491 ranges {659237=601447,1933005=463177} 2024-04-12T11:31:11.839 INFO:tasks.rados.rados.0.smithi029.stdout:3943: writing smithi02979277-6 from 659237 to 1260684 tid 1 2024-04-12T11:31:11.847 INFO:tasks.rados.rados.0.smithi029.stdout:3943: writing smithi02979277-6 from 1933005 to 2396182 tid 2 2024-04-12T11:31:11.847 INFO:tasks.rados.rados.0.smithi029.stdout:3944: write oid 2 current snap is 500 2024-04-12T11:31:11.847 INFO:tasks.rados.rados.0.smithi029.stdout:3943: finishing write tid 1 to smithi02979277-6 2024-04-12T11:31:11.847 INFO:tasks.rados.rados.0.smithi029.stdout:3944: seq_num 1492 ranges {676044=676696,1991487=723188} 2024-04-12T11:31:11.854 INFO:tasks.rados.rados.0.smithi029.stdout:3944: writing smithi02979277-2 from 676044 to 1352740 tid 1 2024-04-12T11:31:11.861 INFO:tasks.rados.rados.0.smithi029.stdout:3944: writing smithi02979277-2 from 1991487 to 2714675 tid 2 2024-04-12T11:31:11.861 INFO:tasks.rados.rados.0.smithi029.stdout:3945: snap_remove snap 490 2024-04-12T11:31:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:11 smithi029 ceph-mon[93650]: osdmap e1066: 8 total, 8 up, 8 in 2024-04-12T11:31:12.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:11 smithi029 ceph-mon[97787]: osdmap e1066: 8 total, 8 up, 8 in 2024-04-12T11:31:12.188 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:11 smithi087 ceph-mon[89014]: osdmap e1066: 8 total, 8 up, 8 in 2024-04-12T11:31:12.465 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:31:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:31:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:31:12.738 INFO:tasks.rados.rados.0.smithi029.stdout:3945: done (2 left) 2024-04-12T11:31:12.738 INFO:tasks.rados.rados.0.smithi029.stdout:3946: snap_remove snap 499 2024-04-12T11:31:13.143 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:12 smithi087 ceph-mon[89014]: pgmap v1236: 89 pgs: 89 active+clean; 204 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 687 KiB/s wr, 5 op/s 2024-04-12T11:31:13.144 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:12 smithi087 ceph-mon[89014]: osdmap e1067: 8 total, 8 up, 8 in 2024-04-12T11:31:13.144 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:12 smithi087 ceph-mon[89014]: osdmap e1068: 8 total, 8 up, 8 in 2024-04-12T11:31:13.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:12 smithi029 ceph-mon[93650]: pgmap v1236: 89 pgs: 89 active+clean; 204 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 687 KiB/s wr, 5 op/s 2024-04-12T11:31:13.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:12 smithi029 ceph-mon[93650]: osdmap e1067: 8 total, 8 up, 8 in 2024-04-12T11:31:13.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:12 smithi029 ceph-mon[93650]: osdmap e1068: 8 total, 8 up, 8 in 2024-04-12T11:31:13.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:12 smithi029 ceph-mon[97787]: pgmap v1236: 89 pgs: 89 active+clean; 204 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 687 KiB/s wr, 5 op/s 2024-04-12T11:31:13.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:12 smithi029 ceph-mon[97787]: osdmap e1067: 8 total, 8 up, 8 in 2024-04-12T11:31:13.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:12 smithi029 ceph-mon[97787]: osdmap e1068: 8 total, 8 up, 8 in 2024-04-12T11:31:13.739 INFO:tasks.rados.rados.0.smithi029.stdout:3946: done (2 left) 2024-04-12T11:31:13.739 INFO:tasks.rados.rados.0.smithi029.stdout:3947: snap_remove snap 497 2024-04-12T11:31:14.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:13 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:13 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:13 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:13 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:13 smithi087 ceph-mon[89014]: osdmap e1069: 8 total, 8 up, 8 in 2024-04-12T11:31:14.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:13 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:13 smithi029 ceph-mon[93650]: osdmap e1069: 8 total, 8 up, 8 in 2024-04-12T11:31:14.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:13 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:14.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:13 smithi029 ceph-mon[97787]: osdmap e1069: 8 total, 8 up, 8 in 2024-04-12T11:31:14.742 INFO:tasks.rados.rados.0.smithi029.stdout:3947: done (2 left) 2024-04-12T11:31:14.742 INFO:tasks.rados.rados.0.smithi029.stdout:3948: delete oid 16 current snap is 500 2024-04-12T11:31:14.742 INFO:tasks.rados.rados.0.smithi029.stdout:3943: finishing write tid 2 to smithi02979277-6 2024-04-12T11:31:14.742 INFO:tasks.rados.rados.0.smithi029.stdout:3943: finishing write tid 3 to smithi02979277-6 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3943: finishing write tid 4 to smithi02979277-6 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3943: oid 6 updating version 0 to 936 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3943: oid 6 version 936 is already newer than 935 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3943: oid 6 version 936 is already newer than 934 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 6 v 936 (ObjNum 1491 snap 500 seq_num 1491) dirty exists 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3943: left oid 6 (ObjNum 1491 snap 500 seq_num 1491) 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3944: finishing write tid 1 to smithi02979277-2 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3944: finishing write tid 2 to smithi02979277-2 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3944: finishing write tid 3 to smithi02979277-2 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3944: finishing write tid 4 to smithi02979277-2 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3944: oid 2 updating version 0 to 470 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3944: oid 2 version 470 is already newer than 468 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:3944: oid 2 version 470 is already newer than 469 2024-04-12T11:31:14.743 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 2 v 470 (ObjNum 1492 snap 500 seq_num 1492) dirty exists 2024-04-12T11:31:14.744 INFO:tasks.rados.rados.0.smithi029.stdout:3944: left oid 2 (ObjNum 1492 snap 500 seq_num 1492) 2024-04-12T11:31:14.746 INFO:tasks.rados.rados.0.smithi029.stdout:3943: done (2 left) 2024-04-12T11:31:14.746 INFO:tasks.rados.rados.0.smithi029.stdout:3944: done (1 left) 2024-04-12T11:31:14.746 INFO:tasks.rados.rados.0.smithi029.stdout:3948: done (0 left) 2024-04-12T11:31:14.746 INFO:tasks.rados.rados.0.smithi029.stdout:3949: write oid 40 current snap is 500 2024-04-12T11:31:14.746 INFO:tasks.rados.rados.0.smithi029.stdout:3949: seq_num 1494 ranges {709658=427194,1592098=726857,2761010=348648} 2024-04-12T11:31:14.752 INFO:tasks.rados.rados.0.smithi029.stdout:3949: writing smithi02979277-40 from 709658 to 1136852 tid 1 2024-04-12T11:31:14.759 INFO:tasks.rados.rados.0.smithi029.stdout:3949: writing smithi02979277-40 from 1592098 to 2318955 tid 2 2024-04-12T11:31:14.764 INFO:tasks.rados.rados.0.smithi029.stdout:3949: writing smithi02979277-40 from 2761010 to 3109658 tid 3 2024-04-12T11:31:14.764 INFO:tasks.rados.rados.0.smithi029.stdout:3950: read oid 20 snap -1 2024-04-12T11:31:14.764 INFO:tasks.rados.rados.0.smithi029.stdout:3950: expect (ObjNum 1489 snap 498 seq_num 1489) 2024-04-12T11:31:14.764 INFO:tasks.rados.rados.0.smithi029.stdout:3951: read oid 23 snap -1 2024-04-12T11:31:14.764 INFO:tasks.rados.rados.0.smithi029.stdout:3951: expect (ObjNum 1353 snap 457 seq_num 1353) 2024-04-12T11:31:14.764 INFO:tasks.rados.rados.0.smithi029.stdout:3952: read oid 8 snap -1 2024-04-12T11:31:14.764 INFO:tasks.rados.rados.0.smithi029.stdout:3952: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:31:14.765 INFO:tasks.rados.rados.0.smithi029.stdout:3953: read oid 24 snap -1 2024-04-12T11:31:14.765 INFO:tasks.rados.rados.0.smithi029.stdout:3953: expect (ObjNum 1478 snap 494 seq_num 1478) 2024-04-12T11:31:14.765 INFO:tasks.rados.rados.0.smithi029.stdout:3954: snap_remove snap 493 2024-04-12T11:31:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:14 smithi087 ceph-mon[89014]: pgmap v1239: 89 pgs: 89 active+clean; 202 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 336 KiB/s wr, 1 op/s 2024-04-12T11:31:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:31:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:31:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:14 smithi087 ceph-mon[89014]: osdmap e1070: 8 total, 8 up, 8 in 2024-04-12T11:31:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:31:14] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-12T11:31:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-mon[93650]: pgmap v1239: 89 pgs: 89 active+clean; 202 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 336 KiB/s wr, 1 op/s 2024-04-12T11:31:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:31:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:31:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-mon[93650]: osdmap e1070: 8 total, 8 up, 8 in 2024-04-12T11:31:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-mon[97787]: pgmap v1239: 89 pgs: 89 active+clean; 202 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 336 KiB/s wr, 1 op/s 2024-04-12T11:31:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:31:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:31:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:31:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:14 smithi029 ceph-mon[97787]: osdmap e1070: 8 total, 8 up, 8 in 2024-04-12T11:31:15.744 INFO:tasks.rados.rados.0.smithi029.stdout:3954: done (5 left) 2024-04-12T11:31:15.744 INFO:tasks.rados.rados.0.smithi029.stdout:3955: read oid 2 snap -1 2024-04-12T11:31:15.744 INFO:tasks.rados.rados.0.smithi029.stdout:3955: expect (ObjNum 1492 snap 500 seq_num 1492) 2024-04-12T11:31:15.744 INFO:tasks.rados.rados.0.smithi029.stdout:3956: delete oid 12 current snap is 500 2024-04-12T11:31:15.744 INFO:tasks.rados.rados.0.smithi029.stdout:3949: finishing write tid 1 to smithi02979277-40 2024-04-12T11:31:15.744 INFO:tasks.rados.rados.0.smithi029.stdout:3949: finishing write tid 2 to smithi02979277-40 2024-04-12T11:31:15.744 INFO:tasks.rados.rados.0.smithi029.stdout:3950: expect (ObjNum 1489 snap 498 seq_num 1489) 2024-04-12T11:31:15.942 INFO:tasks.rados.rados.0.smithi029.stdout:3949: finishing write tid 3 to smithi02979277-40 2024-04-12T11:31:15.942 INFO:tasks.rados.rados.0.smithi029.stdout:3949: finishing write tid 4 to smithi02979277-40 2024-04-12T11:31:15.942 INFO:tasks.rados.rados.0.smithi029.stdout:3949: finishing write tid 5 to smithi02979277-40 2024-04-12T11:31:15.942 INFO:tasks.rados.rados.0.smithi029.stdout:3949: oid 40 updating version 0 to 2576 2024-04-12T11:31:15.942 INFO:tasks.rados.rados.0.smithi029.stdout:3949: oid 40 version 2576 is already newer than 2574 2024-04-12T11:31:15.942 INFO:tasks.rados.rados.0.smithi029.stdout:3949: oid 40 version 2576 is already newer than 2573 2024-04-12T11:31:15.942 INFO:tasks.rados.rados.0.smithi029.stdout:3949: oid 40 version 2576 is already newer than 2575 2024-04-12T11:31:15.942 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 40 v 2576 (ObjNum 1494 snap 500 seq_num 1494) dirty exists 2024-04-12T11:31:15.942 INFO:tasks.rados.rados.0.smithi029.stdout:3949: left oid 40 (ObjNum 1494 snap 500 seq_num 1494) 2024-04-12T11:31:15.942 INFO:tasks.rados.rados.0.smithi029.stdout:3952: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:31:16.054 INFO:tasks.rados.rados.0.smithi029.stdout:3953: expect (ObjNum 1478 snap 494 seq_num 1478) 2024-04-12T11:31:16.174 INFO:tasks.rados.rados.0.smithi029.stdout:3951: expect (ObjNum 1353 snap 457 seq_num 1353) 2024-04-12T11:31:16.269 INFO:tasks.rados.rados.0.smithi029.stdout:3955: expect (ObjNum 1492 snap 500 seq_num 1492) 2024-04-12T11:31:16.452 INFO:tasks.rados.rados.0.smithi029.stdout:3949: done (6 left) 2024-04-12T11:31:16.452 INFO:tasks.rados.rados.0.smithi029.stdout:3950: done (5 left) 2024-04-12T11:31:16.452 INFO:tasks.rados.rados.0.smithi029.stdout:3951: done (4 left) 2024-04-12T11:31:16.452 INFO:tasks.rados.rados.0.smithi029.stdout:3952: done (3 left) 2024-04-12T11:31:16.452 INFO:tasks.rados.rados.0.smithi029.stdout:3953: done (2 left) 2024-04-12T11:31:16.453 INFO:tasks.rados.rados.0.smithi029.stdout:3955: done (1 left) 2024-04-12T11:31:16.453 INFO:tasks.rados.rados.0.smithi029.stdout:3956: done (0 left) 2024-04-12T11:31:16.453 INFO:tasks.rados.rados.0.smithi029.stdout:3957: rollback oid 43 current snap is 500 2024-04-12T11:31:16.453 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 43 to 491 2024-04-12T11:31:16.453 INFO:tasks.rados.rados.0.smithi029.stdout:3958: delete oid 7 current snap is 500 2024-04-12T11:31:16.453 INFO:tasks.rados.rados.0.smithi029.stdout:3957: finishing rollback tid 1 to smithi02979277-43 2024-04-12T11:31:16.453 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 0 (ObjNum 49 snap 0 seq_num 1800907824) dirty dne 2024-04-12T11:31:16.455 INFO:tasks.rados.rados.0.smithi029.stdout:3957: done (1 left) 2024-04-12T11:31:16.455 INFO:tasks.rados.rados.0.smithi029.stdout:3958: done (0 left) 2024-04-12T11:31:16.456 INFO:tasks.rados.rados.0.smithi029.stdout:3959: read oid 44 snap -1 2024-04-12T11:31:16.456 INFO:tasks.rados.rados.0.smithi029.stdout:3959: expect (ObjNum 1469 snap 492 seq_num 1469) 2024-04-12T11:31:16.456 INFO:tasks.rados.rados.0.smithi029.stdout:3960: write oid 30 current snap is 500 2024-04-12T11:31:16.456 INFO:tasks.rados.rados.0.smithi029.stdout:3960: seq_num 1497 ranges {760079=652941,2167544=474184} 2024-04-12T11:31:16.465 INFO:tasks.rados.rados.0.smithi029.stdout:3960: writing smithi02979277-30 from 760079 to 1413020 tid 1 2024-04-12T11:31:16.472 INFO:tasks.rados.rados.0.smithi029.stdout:3960: writing smithi02979277-30 from 2167544 to 2641728 tid 2 2024-04-12T11:31:16.472 INFO:tasks.rados.rados.0.smithi029.stdout:3961: read oid 18 snap -1 2024-04-12T11:31:16.472 INFO:tasks.rados.rados.0.smithi029.stdout:3961: expect deleted 2024-04-12T11:31:16.472 INFO:tasks.rados.rados.0.smithi029.stdout:3962: snap_create 2024-04-12T11:31:16.472 INFO:tasks.rados.rados.0.smithi029.stdout:3959: expect (ObjNum 1469 snap 492 seq_num 1469) 2024-04-12T11:31:16.632 INFO:tasks.rados.rados.0.smithi029.stdout:3960: finishing write tid 1 to smithi02979277-30 2024-04-12T11:31:16.633 INFO:tasks.rados.rados.0.smithi029.stdout:3960: finishing write tid 2 to smithi02979277-30 2024-04-12T11:31:16.633 INFO:tasks.rados.rados.0.smithi029.stdout:3960: finishing write tid 3 to smithi02979277-30 2024-04-12T11:31:16.633 INFO:tasks.rados.rados.0.smithi029.stdout:3960: finishing write tid 4 to smithi02979277-30 2024-04-12T11:31:16.633 INFO:tasks.rados.rados.0.smithi029.stdout:3960: oid 30 updating version 0 to 1297 2024-04-12T11:31:16.633 INFO:tasks.rados.rados.0.smithi029.stdout:3960: oid 30 version 1297 is already newer than 1296 2024-04-12T11:31:16.633 INFO:tasks.rados.rados.0.smithi029.stdout:3960: oid 30 version 1297 is already newer than 1295 2024-04-12T11:31:16.633 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 1297 (ObjNum 1497 snap 500 seq_num 1497) dirty exists 2024-04-12T11:31:16.633 INFO:tasks.rados.rados.0.smithi029.stdout:3960: left oid 30 (ObjNum 1497 snap 500 seq_num 1497) 2024-04-12T11:31:16.914 INFO:tasks.rados.rados.0.smithi029.stdout:3959: done (3 left) 2024-04-12T11:31:16.914 INFO:tasks.rados.rados.0.smithi029.stdout:3960: done (2 left) 2024-04-12T11:31:16.914 INFO:tasks.rados.rados.0.smithi029.stdout:3961: done (1 left) 2024-04-12T11:31:16.914 INFO:tasks.rados.rados.0.smithi029.stdout:3962: done (0 left) 2024-04-12T11:31:16.914 INFO:tasks.rados.rados.0.smithi029.stdout:3963: write oid 43 current snap is 501 2024-04-12T11:31:16.914 INFO:tasks.rados.rados.0.smithi029.stdout:3963: seq_num 1498 ranges {776886=444543,1826026=734195,2987055=189831} 2024-04-12T11:31:16.920 INFO:tasks.rados.rados.0.smithi029.stdout:3963: writing smithi02979277-43 from 776886 to 1221429 tid 1 2024-04-12T11:31:16.928 INFO:tasks.rados.rados.0.smithi029.stdout:3963: writing smithi02979277-43 from 1826026 to 2560221 tid 2 2024-04-12T11:31:16.931 INFO:tasks.rados.rados.0.smithi029.stdout:3963: writing smithi02979277-43 from 2987055 to 3176886 tid 3 2024-04-12T11:31:16.931 INFO:tasks.rados.rados.0.smithi029.stdout:3964: write oid 18 current snap is 501 2024-04-12T11:31:16.932 INFO:tasks.rados.rados.0.smithi029.stdout:3963: finishing write tid 1 to smithi02979277-43 2024-04-12T11:31:16.932 INFO:tasks.rados.rados.0.smithi029.stdout:3963: finishing write tid 2 to smithi02979277-43 2024-04-12T11:31:16.932 INFO:tasks.rados.rados.0.smithi029.stdout:3964: seq_num 1499 ranges {793693=519792,1884508=477853,2898125=295568} 2024-04-12T11:31:16.938 INFO:tasks.rados.rados.0.smithi029.stdout:3964: writing smithi02979277-18 from 793693 to 1313485 tid 1 2024-04-12T11:31:16.944 INFO:tasks.rados.rados.0.smithi029.stdout:3964: writing smithi02979277-18 from 1884508 to 2362361 tid 2 2024-04-12T11:31:16.948 INFO:tasks.rados.rados.0.smithi029.stdout:3964: writing smithi02979277-18 from 2898125 to 3193693 tid 3 2024-04-12T11:31:16.949 INFO:tasks.rados.rados.0.smithi029.stdout:3965: write oid 3 current snap is 501 2024-04-12T11:31:16.949 INFO:tasks.rados.rados.0.smithi029.stdout:3963: finishing write tid 3 to smithi02979277-43 2024-04-12T11:31:16.949 INFO:tasks.rados.rados.0.smithi029.stdout:3965: seq_num 1500 ranges {410500=595041,1542990=621511,2925548=284952} 2024-04-12T11:31:16.954 INFO:tasks.rados.rados.0.smithi029.stdout:3965: writing smithi02979277-3 from 410500 to 1005541 tid 1 2024-04-12T11:31:16.960 INFO:tasks.rados.rados.0.smithi029.stdout:3965: writing smithi02979277-3 from 1542990 to 2164501 tid 2 2024-04-12T11:31:16.967 INFO:tasks.rados.rados.0.smithi029.stdout:3965: writing smithi02979277-3 from 2925548 to 3210500 tid 3 2024-04-12T11:31:16.967 INFO:tasks.rados.rados.0.smithi029.stdout:3966: read oid 8 snap 500 2024-04-12T11:31:16.967 INFO:tasks.rados.rados.0.smithi029.stdout:3966: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:31:16.967 INFO:tasks.rados.rados.0.smithi029.stdout:3967: read oid 9 snap 489 2024-04-12T11:31:16.967 INFO:tasks.rados.rados.0.smithi029.stdout:3967: expect deleted 2024-04-12T11:31:16.967 INFO:tasks.rados.rados.0.smithi029.stdout:3968: snap_remove snap 480 2024-04-12T11:31:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:16 smithi029 ceph-mon[97787]: pgmap v1242: 89 pgs: 89 active+clean; 204 MiB data, 847 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-04-12T11:31:17.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:16 smithi029 ceph-mon[97787]: osdmap e1071: 8 total, 8 up, 8 in 2024-04-12T11:31:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:16 smithi029 ceph-mon[93650]: pgmap v1242: 89 pgs: 89 active+clean; 204 MiB data, 847 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-04-12T11:31:17.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:16 smithi029 ceph-mon[93650]: osdmap e1071: 8 total, 8 up, 8 in 2024-04-12T11:31:17.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:16 smithi087 ceph-mon[89014]: pgmap v1242: 89 pgs: 89 active+clean; 204 MiB data, 847 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-04-12T11:31:17.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:16 smithi087 ceph-mon[89014]: osdmap e1071: 8 total, 8 up, 8 in 2024-04-12T11:31:17.744 INFO:tasks.rados.rados.0.smithi029.stdout:3963: finishing write tid 4 to smithi02979277-43 2024-04-12T11:31:17.744 INFO:tasks.rados.rados.0.smithi029.stdout:3963: finishing write tid 5 to smithi02979277-43 2024-04-12T11:31:17.744 INFO:tasks.rados.rados.0.smithi029.stdout:3963: oid 43 updating version 0 to 1803 2024-04-12T11:31:17.744 INFO:tasks.rados.rados.0.smithi029.stdout:3963: oid 43 version 1803 is already newer than 1801 2024-04-12T11:31:17.744 INFO:tasks.rados.rados.0.smithi029.stdout:3963: oid 43 version 1803 is already newer than 1802 2024-04-12T11:31:17.744 INFO:tasks.rados.rados.0.smithi029.stdout:3963: oid 43 version 1803 is already newer than 1800 2024-04-12T11:31:17.744 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 43 v 1803 (ObjNum 1498 snap 501 seq_num 1498) dirty exists 2024-04-12T11:31:17.744 INFO:tasks.rados.rados.0.smithi029.stdout:3963: left oid 43 (ObjNum 1498 snap 501 seq_num 1498) 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3964: finishing write tid 1 to smithi02979277-18 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3964: finishing write tid 2 to smithi02979277-18 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3964: finishing write tid 3 to smithi02979277-18 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3964: finishing write tid 4 to smithi02979277-18 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3964: finishing write tid 5 to smithi02979277-18 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3964: oid 18 updating version 0 to 2139 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3964: oid 18 updating version 2139 to 2140 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3964: oid 18 updating version 2140 to 2142 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3964: oid 18 version 2142 is already newer than 2141 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 18 v 2142 (ObjNum 1499 snap 501 seq_num 1499) dirty exists 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3964: left oid 18 (ObjNum 1499 snap 501 seq_num 1499) 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3965: finishing write tid 1 to smithi02979277-3 2024-04-12T11:31:17.745 INFO:tasks.rados.rados.0.smithi029.stdout:3965: finishing write tid 2 to smithi02979277-3 2024-04-12T11:31:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3965: finishing write tid 3 to smithi02979277-3 2024-04-12T11:31:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3965: finishing write tid 4 to smithi02979277-3 2024-04-12T11:31:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3965: finishing write tid 5 to smithi02979277-3 2024-04-12T11:31:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3965: oid 3 updating version 0 to 2584 2024-04-12T11:31:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3965: oid 3 updating version 2584 to 2586 2024-04-12T11:31:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3965: oid 3 updating version 2586 to 2587 2024-04-12T11:31:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3965: oid 3 version 2587 is already newer than 2585 2024-04-12T11:31:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 3 v 2587 (ObjNum 1500 snap 501 seq_num 1500) dirty exists 2024-04-12T11:31:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3965: left oid 3 (ObjNum 1500 snap 501 seq_num 1500) 2024-04-12T11:31:17.746 INFO:tasks.rados.rados.0.smithi029.stdout:3966: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:31:17.865 INFO:tasks.rados.rados.0.smithi029.stdout:3963: done (5 left) 2024-04-12T11:31:17.865 INFO:tasks.rados.rados.0.smithi029.stdout:3964: done (4 left) 2024-04-12T11:31:17.865 INFO:tasks.rados.rados.0.smithi029.stdout:3965: done (3 left) 2024-04-12T11:31:17.865 INFO:tasks.rados.rados.0.smithi029.stdout:3966: done (2 left) 2024-04-12T11:31:17.866 INFO:tasks.rados.rados.0.smithi029.stdout:3967: done (1 left) 2024-04-12T11:31:17.866 INFO:tasks.rados.rados.0.smithi029.stdout:3968: done (0 left) 2024-04-12T11:31:17.866 INFO:tasks.rados.rados.0.smithi029.stdout:3969: snap_create 2024-04-12T11:31:18.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:17 smithi087 ceph-mon[89014]: osdmap e1072: 8 total, 8 up, 8 in 2024-04-12T11:31:18.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:17 smithi087 ceph-mon[89014]: osdmap e1073: 8 total, 8 up, 8 in 2024-04-12T11:31:18.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:17 smithi029 ceph-mon[93650]: osdmap e1072: 8 total, 8 up, 8 in 2024-04-12T11:31:18.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:17 smithi029 ceph-mon[93650]: osdmap e1073: 8 total, 8 up, 8 in 2024-04-12T11:31:18.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:17 smithi029 ceph-mon[97787]: osdmap e1072: 8 total, 8 up, 8 in 2024-04-12T11:31:18.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:17 smithi029 ceph-mon[97787]: osdmap e1073: 8 total, 8 up, 8 in 2024-04-12T11:31:18.750 INFO:tasks.rados.rados.0.smithi029.stdout:3969: done (0 left) 2024-04-12T11:31:18.750 INFO:tasks.rados.rados.0.smithi029.stdout:3970: write oid 44 current snap is 502 2024-04-12T11:31:18.750 INFO:tasks.rados.rados.0.smithi029.stdout:3970: seq_num 1501 ranges {427307=670290,1601472=481522,2552971=483056} 2024-04-12T11:31:18.756 INFO:tasks.rados.rados.0.smithi029.stdout:3970: writing smithi02979277-44 from 427307 to 1097597 tid 1 2024-04-12T11:31:18.762 INFO:tasks.rados.rados.0.smithi029.stdout:3970: writing smithi02979277-44 from 1601472 to 2082994 tid 2 2024-04-12T11:31:18.767 INFO:tasks.rados.rados.0.smithi029.stdout:3970: writing smithi02979277-44 from 2552971 to 3036027 tid 3 2024-04-12T11:31:18.768 INFO:tasks.rados.rados.0.smithi029.stdout:3971: rollback oid 33 current snap is 502 2024-04-12T11:31:18.768 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 33 to 469 2024-04-12T11:31:18.768 INFO:tasks.rados.rados.0.smithi029.stdout:3970: finishing write tid 1 to smithi02979277-44 2024-04-12T11:31:18.768 INFO:tasks.rados.rados.0.smithi029.stdout:3970: finishing write tid 2 to smithi02979277-44 2024-04-12T11:31:18.768 INFO:tasks.rados.rados.0.smithi029.stdout:3972: snap_remove snap 498 2024-04-12T11:31:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:18 smithi087 ceph-mon[89014]: pgmap v1245: 89 pgs: 89 active+clean; 207 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:31:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:18 smithi087 ceph-mon[89014]: osdmap e1074: 8 total, 8 up, 8 in 2024-04-12T11:31:19.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:18 smithi029 ceph-mon[97787]: pgmap v1245: 89 pgs: 89 active+clean; 207 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:31:19.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:18 smithi029 ceph-mon[97787]: osdmap e1074: 8 total, 8 up, 8 in 2024-04-12T11:31:19.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:18 smithi029 ceph-mon[93650]: pgmap v1245: 89 pgs: 89 active+clean; 207 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-04-12T11:31:19.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:18 smithi029 ceph-mon[93650]: osdmap e1074: 8 total, 8 up, 8 in 2024-04-12T11:31:19.752 INFO:tasks.rados.rados.0.smithi029.stdout:3972: done (2 left) 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3970: finishing write tid 3 to smithi02979277-44 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3970: finishing write tid 4 to smithi02979277-44 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3970: finishing write tid 5 to smithi02979277-44 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3970: oid 44 updating version 0 to 1993 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3970: oid 44 version 1993 is already newer than 1992 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3970: oid 44 version 1993 is already newer than 1990 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3970: oid 44 version 1993 is already newer than 1991 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 44 v 1993 (ObjNum 1501 snap 502 seq_num 1501) dirty exists 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3970: left oid 44 (ObjNum 1501 snap 502 seq_num 1501) 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3971: finishing rollback tid 0 to smithi02979277-33 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3971: finishing rollback tid 1 to smithi02979277-33 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 33 v 1705 (ObjNum 129 snap 0 seq_num 1801357232) dirty dne 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3973: read oid 5 snap -1 2024-04-12T11:31:19.753 INFO:tasks.rados.rados.0.smithi029.stdout:3973: expect (ObjNum 1364 snap 462 seq_num 1364) 2024-04-12T11:31:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:3970: done (2 left) 2024-04-12T11:31:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:3971: done (1 left) 2024-04-12T11:31:19.754 INFO:tasks.rados.rados.0.smithi029.stdout:3974: snap_create 2024-04-12T11:31:19.765 INFO:tasks.rados.rados.0.smithi029.stdout:3973: expect (ObjNum 1364 snap 462 seq_num 1364) 2024-04-12T11:31:20.753 INFO:tasks.rados.rados.0.smithi029.stdout:3973: done (1 left) 2024-04-12T11:31:20.753 INFO:tasks.rados.rados.0.smithi029.stdout:3974: done (0 left) 2024-04-12T11:31:20.753 INFO:tasks.rados.rados.0.smithi029.stdout:3975: write oid 29 current snap is 503 2024-04-12T11:31:20.754 INFO:tasks.rados.rados.0.smithi029.stdout:3975: seq_num 1502 ranges {444114=745539,1943601=625180} 2024-04-12T11:31:20.760 INFO:tasks.rados.rados.0.smithi029.stdout:3975: writing smithi02979277-29 from 444114 to 1189653 tid 1 2024-04-12T11:31:20.767 INFO:tasks.rados.rados.0.smithi029.stdout:3975: writing smithi02979277-29 from 1943601 to 2568781 tid 2 2024-04-12T11:31:20.768 INFO:tasks.rados.rados.0.smithi029.stdout:3976: snap_create 2024-04-12T11:31:20.768 INFO:tasks.rados.rados.0.smithi029.stdout:3975: finishing write tid 1 to smithi02979277-29 2024-04-12T11:31:20.771 INFO:tasks.rados.rados.0.smithi029.stdout:3975: finishing write tid 2 to smithi02979277-29 2024-04-12T11:31:20.771 INFO:tasks.rados.rados.0.smithi029.stdout:3975: finishing write tid 3 to smithi02979277-29 2024-04-12T11:31:20.772 INFO:tasks.rados.rados.0.smithi029.stdout:3975: finishing write tid 4 to smithi02979277-29 2024-04-12T11:31:20.772 INFO:tasks.rados.rados.0.smithi029.stdout:3975: oid 29 updating version 0 to 1808 2024-04-12T11:31:20.772 INFO:tasks.rados.rados.0.smithi029.stdout:3975: oid 29 updating version 1808 to 1810 2024-04-12T11:31:20.772 INFO:tasks.rados.rados.0.smithi029.stdout:3975: oid 29 version 1810 is already newer than 1809 2024-04-12T11:31:20.772 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 29 v 1810 (ObjNum 1502 snap 503 seq_num 1502) dirty exists 2024-04-12T11:31:20.772 INFO:tasks.rados.rados.0.smithi029.stdout:3975: left oid 29 (ObjNum 1502 snap 503 seq_num 1502) 2024-04-12T11:31:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:20 smithi029 ceph-mon[97787]: pgmap v1248: 89 pgs: 89 active+clean; 205 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2024-04-12T11:31:21.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:20 smithi029 ceph-mon[97787]: osdmap e1075: 8 total, 8 up, 8 in 2024-04-12T11:31:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:20 smithi029 ceph-mon[93650]: pgmap v1248: 89 pgs: 89 active+clean; 205 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2024-04-12T11:31:21.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:20 smithi029 ceph-mon[93650]: osdmap e1075: 8 total, 8 up, 8 in 2024-04-12T11:31:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:20 smithi087 ceph-mon[89014]: pgmap v1248: 89 pgs: 89 active+clean; 205 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2024-04-12T11:31:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:20 smithi087 ceph-mon[89014]: osdmap e1075: 8 total, 8 up, 8 in 2024-04-12T11:31:21.763 INFO:tasks.rados.rados.0.smithi029.stdout:3975: done (1 left) 2024-04-12T11:31:21.763 INFO:tasks.rados.rados.0.smithi029.stdout:3976: done (0 left) 2024-04-12T11:31:21.764 INFO:tasks.rados.rados.0.smithi029.stdout:3977: snap_create 2024-04-12T11:31:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:21 smithi029 ceph-mon[97787]: osdmap e1076: 8 total, 8 up, 8 in 2024-04-12T11:31:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:21 smithi029 ceph-mon[93650]: osdmap e1076: 8 total, 8 up, 8 in 2024-04-12T11:31:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:22.117 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:21 smithi087 ceph-mon[89014]: osdmap e1076: 8 total, 8 up, 8 in 2024-04-12T11:31:22.117 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:22.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:31:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:31:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:31:22.745 INFO:tasks.rados.rados.0.smithi029.stdout:3977: done (0 left) 2024-04-12T11:31:22.745 INFO:tasks.rados.rados.0.smithi029.stdout:3978: read oid 8 snap -1 2024-04-12T11:31:22.745 INFO:tasks.rados.rados.0.smithi029.stdout:3978: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:31:22.745 INFO:tasks.rados.rados.0.smithi029.stdout:3979: read oid 38 snap -1 2024-04-12T11:31:22.745 INFO:tasks.rados.rados.0.smithi029.stdout:3979: expect deleted 2024-04-12T11:31:22.745 INFO:tasks.rados.rados.0.smithi029.stdout:3980: rollback oid 26 current snap is 505 2024-04-12T11:31:22.745 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 26 to 502 2024-04-12T11:31:22.745 INFO:tasks.rados.rados.0.smithi029.stdout:3981: snap_create 2024-04-12T11:31:22.752 INFO:tasks.rados.rados.0.smithi029.stdout:3980: finishing rollback tid 0 to smithi02979277-26 2024-04-12T11:31:22.753 INFO:tasks.rados.rados.0.smithi029.stdout:3980: finishing rollback tid 1 to smithi02979277-26 2024-04-12T11:31:22.753 INFO:tasks.rados.rados.0.smithi029.stdout:3980: finishing rollback tid 2 to smithi02979277-26 2024-04-12T11:31:22.753 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 26 v 2603 (ObjNum 1473 snap 494 seq_num 1473) dirty exists 2024-04-12T11:31:22.753 INFO:tasks.rados.rados.0.smithi029.stdout:3978: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:31:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:22 smithi029 ceph-mon[93650]: pgmap v1251: 89 pgs: 89 active+clean; 205 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-04-12T11:31:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:22 smithi029 ceph-mon[93650]: osdmap e1077: 8 total, 8 up, 8 in 2024-04-12T11:31:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:22 smithi029 ceph-mon[93650]: osdmap e1078: 8 total, 8 up, 8 in 2024-04-12T11:31:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:22 smithi029 ceph-mon[97787]: pgmap v1251: 89 pgs: 89 active+clean; 205 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-04-12T11:31:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:22 smithi029 ceph-mon[97787]: osdmap e1077: 8 total, 8 up, 8 in 2024-04-12T11:31:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:22 smithi029 ceph-mon[97787]: osdmap e1078: 8 total, 8 up, 8 in 2024-04-12T11:31:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:22 smithi087 ceph-mon[89014]: pgmap v1251: 89 pgs: 89 active+clean; 205 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-04-12T11:31:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:22 smithi087 ceph-mon[89014]: osdmap e1077: 8 total, 8 up, 8 in 2024-04-12T11:31:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:22 smithi087 ceph-mon[89014]: osdmap e1078: 8 total, 8 up, 8 in 2024-04-12T11:31:23.746 INFO:tasks.rados.rados.0.smithi029.stdout:3978: done (3 left) 2024-04-12T11:31:23.747 INFO:tasks.rados.rados.0.smithi029.stdout:3979: done (2 left) 2024-04-12T11:31:23.747 INFO:tasks.rados.rados.0.smithi029.stdout:3980: done (1 left) 2024-04-12T11:31:23.747 INFO:tasks.rados.rados.0.smithi029.stdout:3981: done (0 left) 2024-04-12T11:31:23.747 INFO:tasks.rados.rados.0.smithi029.stdout:3982: read oid 35 snap -1 2024-04-12T11:31:23.747 INFO:tasks.rados.rados.0.smithi029.stdout:3982: expect (ObjNum 1426 snap 478 seq_num 1426) 2024-04-12T11:31:23.747 INFO:tasks.rados.rados.0.smithi029.stdout:3983: read oid 28 snap -1 2024-04-12T11:31:23.747 INFO:tasks.rados.rados.0.smithi029.stdout:3983: expect (ObjNum 1434 snap 481 seq_num 1434) 2024-04-12T11:31:23.747 INFO:tasks.rados.rados.0.smithi029.stdout:3984: read oid 8 snap 489 2024-04-12T11:31:23.747 INFO:tasks.rados.rados.0.smithi029.stdout:3984: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:31:23.747 INFO:tasks.rados.rados.0.smithi029.stdout:3985: write oid 36 current snap is 506 2024-04-12T11:31:23.748 INFO:tasks.rados.rados.0.smithi029.stdout:3985: seq_num 1503 ranges {460921=420788,1602083=485191,2491464=621953} 2024-04-12T11:31:23.752 INFO:tasks.rados.rados.0.smithi029.stdout:3985: writing smithi02979277-36 from 460921 to 881709 tid 1 2024-04-12T11:31:23.759 INFO:tasks.rados.rados.0.smithi029.stdout:3985: writing smithi02979277-36 from 1602083 to 2087274 tid 2 2024-04-12T11:31:23.765 INFO:tasks.rados.rados.0.smithi029.stdout:3985: writing smithi02979277-36 from 2491464 to 3113417 tid 3 2024-04-12T11:31:23.765 INFO:tasks.rados.rados.0.smithi029.stdout:3986: write oid 21 current snap is 506 2024-04-12T11:31:23.765 INFO:tasks.rados.rados.0.smithi029.stdout:3986: seq_num 1504 ranges {477728=496037,1660565=628849,2918887=358841} 2024-04-12T11:31:23.770 INFO:tasks.rados.rados.0.smithi029.stdout:3986: writing smithi02979277-21 from 477728 to 973765 tid 1 2024-04-12T11:31:23.778 INFO:tasks.rados.rados.0.smithi029.stdout:3986: writing smithi02979277-21 from 1660565 to 2289414 tid 2 2024-04-12T11:31:23.783 INFO:tasks.rados.rados.0.smithi029.stdout:3986: writing smithi02979277-21 from 2918887 to 3277728 tid 3 2024-04-12T11:31:23.783 INFO:tasks.rados.rados.0.smithi029.stdout:3987: rollback oid 41 current snap is 506 2024-04-12T11:31:23.784 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 41 to 500 2024-04-12T11:31:23.784 INFO:tasks.rados.rados.0.smithi029.stdout:3983: expect (ObjNum 1434 snap 481 seq_num 1434) 2024-04-12T11:31:23.854 INFO:tasks.rados.rados.0.smithi029.stdout:3982: expect (ObjNum 1426 snap 478 seq_num 1426) 2024-04-12T11:31:23.983 INFO:tasks.rados.rados.0.smithi029.stdout:3984: expect (ObjNum 1395 snap 469 seq_num 1395) 2024-04-12T11:31:24.096 INFO:tasks.rados.rados.0.smithi029.stdout:3985: finishing write tid 1 to smithi02979277-36 2024-04-12T11:31:24.096 INFO:tasks.rados.rados.0.smithi029.stdout:3985: finishing write tid 2 to smithi02979277-36 2024-04-12T11:31:24.096 INFO:tasks.rados.rados.0.smithi029.stdout:3985: finishing write tid 3 to smithi02979277-36 2024-04-12T11:31:24.096 INFO:tasks.rados.rados.0.smithi029.stdout:3985: finishing write tid 4 to smithi02979277-36 2024-04-12T11:31:24.096 INFO:tasks.rados.rados.0.smithi029.stdout:3985: finishing write tid 5 to smithi02979277-36 2024-04-12T11:31:24.096 INFO:tasks.rados.rados.0.smithi029.stdout:3985: oid 36 updating version 0 to 951 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3985: oid 36 updating version 951 to 953 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3985: oid 36 version 953 is already newer than 952 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3985: oid 36 version 953 is already newer than 950 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 36 v 953 (ObjNum 1503 snap 506 seq_num 1503) dirty exists 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3985: left oid 36 (ObjNum 1503 snap 506 seq_num 1503) 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3986: finishing write tid 1 to smithi02979277-21 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3986: finishing write tid 2 to smithi02979277-21 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3982: done (5 left) 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3983: done (4 left) 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3984: done (3 left) 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3985: done (2 left) 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3988: write oid 47 current snap is 506 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3986: finishing write tid 3 to smithi02979277-21 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3986: finishing write tid 4 to smithi02979277-21 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3986: finishing write tid 5 to smithi02979277-21 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3986: oid 21 updating version 0 to 1996 2024-04-12T11:31:24.097 INFO:tasks.rados.rados.0.smithi029.stdout:3986: oid 21 updating version 1996 to 1999 2024-04-12T11:31:24.098 INFO:tasks.rados.rados.0.smithi029.stdout:3986: oid 21 version 1999 is already newer than 1997 2024-04-12T11:31:24.098 INFO:tasks.rados.rados.0.smithi029.stdout:3986: oid 21 version 1999 is already newer than 1998 2024-04-12T11:31:24.098 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 21 v 1999 (ObjNum 1504 snap 506 seq_num 1504) dirty exists 2024-04-12T11:31:24.098 INFO:tasks.rados.rados.0.smithi029.stdout:3986: left oid 21 (ObjNum 1504 snap 506 seq_num 1504) 2024-04-12T11:31:24.098 INFO:tasks.rados.rados.0.smithi029.stdout:3987: finishing rollback tid 0 to smithi02979277-41 2024-04-12T11:31:24.098 INFO:tasks.rados.rados.0.smithi029.stdout:3987: finishing rollback tid 1 to smithi02979277-41 2024-04-12T11:31:24.098 INFO:tasks.rados.rados.0.smithi029.stdout:3987: finishing rollback tid 2 to smithi02979277-41 2024-04-12T11:31:24.098 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 41 v 2003 (ObjNum 1452 snap 487 seq_num 1452) dirty exists 2024-04-12T11:31:24.098 INFO:tasks.rados.rados.0.smithi029.stdout:3988: seq_num 1505 ranges {494535=571286,1719047=488860,2946310=348225} 2024-04-12T11:31:24.101 INFO:tasks.rados.rados.0.smithi029.stdout:3988: writing smithi02979277-47 from 494535 to 1065821 tid 1 2024-04-12T11:31:24.108 INFO:tasks.rados.rados.0.smithi029.stdout:3988: writing smithi02979277-47 from 1719047 to 2207907 tid 2 2024-04-12T11:31:24.114 INFO:tasks.rados.rados.0.smithi029.stdout:3988: writing smithi02979277-47 from 2946310 to 3294535 tid 3 2024-04-12T11:31:24.114 INFO:tasks.rados.rados.0.smithi029.stdout:3986: done (2 left) 2024-04-12T11:31:24.114 INFO:tasks.rados.rados.0.smithi029.stdout:3987: done (1 left) 2024-04-12T11:31:24.114 INFO:tasks.rados.rados.0.smithi029.stdout:3989: rollback oid 27 current snap is 506 2024-04-12T11:31:24.114 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 27 to 489 2024-04-12T11:31:24.115 INFO:tasks.rados.rados.0.smithi029.stdout:3988: finishing write tid 1 to smithi02979277-47 2024-04-12T11:31:24.115 INFO:tasks.rados.rados.0.smithi029.stdout:3988: finishing write tid 2 to smithi02979277-47 2024-04-12T11:31:24.115 INFO:tasks.rados.rados.0.smithi029.stdout:3990: rollback oid 17 current snap is 506 2024-04-12T11:31:24.115 INFO:tasks.rados.rados.0.smithi029.stdout:rollback oid 17 to 504 2024-04-12T11:31:24.115 INFO:tasks.rados.rados.0.smithi029.stdout:3991: delete oid 24 current snap is 506 2024-04-12T11:31:24.118 INFO:tasks.rados.rados.0.smithi029.stdout:3991: done (3 left) 2024-04-12T11:31:24.118 INFO:tasks.rados.rados.0.smithi029.stdout:3992: delete oid 28 current snap is 506 2024-04-12T11:31:24.118 INFO:tasks.rados.rados.0.smithi029.stdout:3989: finishing rollback tid 0 to smithi02979277-27 2024-04-12T11:31:24.118 INFO:tasks.rados.rados.0.smithi029.stdout:3989: finishing rollback tid 1 to smithi02979277-27 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:3988: finishing write tid 3 to smithi02979277-47 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:3989: finishing rollback tid 2 to smithi02979277-27 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 27 v 1309 (ObjNum 1440 snap 482 seq_num 1440) dirty exists 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:3988: finishing write tid 4 to smithi02979277-47 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:3988: finishing write tid 5 to smithi02979277-47 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:3988: oid 47 updating version 0 to 1991 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:3988: oid 47 updating version 1991 to 1993 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:3988: oid 47 version 1993 is already newer than 1992 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:3988: oid 47 version 1993 is already newer than 1990 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 47 v 1993 (ObjNum 1505 snap 506 seq_num 1505) dirty exists 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:3988: left oid 47 (ObjNum 1505 snap 506 seq_num 1505) 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:3990: finishing rollback tid 1 to smithi02979277-17 2024-04-12T11:31:24.119 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 17 v 2040 (ObjNum 129 snap 0 seq_num 1801266320) dirty dne 2024-04-12T11:31:24.120 INFO:tasks.rados.rados.0.smithi029.stdout:3988: done (3 left) 2024-04-12T11:31:24.120 INFO:tasks.rados.rados.0.smithi029.stdout:3989: done (2 left) 2024-04-12T11:31:24.120 INFO:tasks.rados.rados.0.smithi029.stdout:3990: done (1 left) 2024-04-12T11:31:24.120 INFO:tasks.rados.rados.0.smithi029.stdout:3992: done (0 left) 2024-04-12T11:31:24.120 INFO:tasks.rados.rados.0.smithi029.stdout:3993: read oid 12 snap -1 2024-04-12T11:31:24.120 INFO:tasks.rados.rados.0.smithi029.stdout:3993: expect deleted 2024-04-12T11:31:24.120 INFO:tasks.rados.rados.0.smithi029.stdout:3994: read oid 28 snap -1 2024-04-12T11:31:24.121 INFO:tasks.rados.rados.0.smithi029.stdout:3994: expect deleted 2024-04-12T11:31:24.121 INFO:tasks.rados.rados.0.smithi029.stdout:3995: snap_create 2024-04-12T11:31:24.766 INFO:tasks.rados.rados.0.smithi029.stdout:3993: done (2 left) 2024-04-12T11:31:24.766 INFO:tasks.rados.rados.0.smithi029.stdout:3994: done (1 left) 2024-04-12T11:31:24.766 INFO:tasks.rados.rados.0.smithi029.stdout:3995: done (0 left) 2024-04-12T11:31:24.767 INFO:tasks.rados.rados.0.smithi029.stdout:3996: write oid 15 current snap is 507 2024-04-12T11:31:24.767 INFO:tasks.rados.rados.0.smithi029.stdout:3996: seq_num 1508 ranges {544956=513386,1494493=636187,2628579=594666} 2024-04-12T11:31:24.772 INFO:tasks.rados.rados.0.smithi029.stdout:3996: writing smithi02979277-15 from 544956 to 1058342 tid 1 2024-04-12T11:31:24.778 INFO:tasks.rados.rados.0.smithi029.stdout:3996: writing smithi02979277-15 from 1494493 to 2130680 tid 2 2024-04-12T11:31:24.784 INFO:tasks.rados.rados.0.smithi029.stdout:3996: writing smithi02979277-15 from 2628579 to 3223245 tid 3 2024-04-12T11:31:24.784 INFO:tasks.rados.rados.0.smithi029.stdout:3997: read oid 41 snap -1 2024-04-12T11:31:24.784 INFO:tasks.rados.rados.0.smithi029.stdout:3997: expect (ObjNum 1452 snap 487 seq_num 1452) 2024-04-12T11:31:24.784 INFO:tasks.rados.rados.0.smithi029.stdout:3998: snap_remove snap 502 2024-04-12T11:31:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:24 smithi029 ceph-mon[97787]: pgmap v1254: 89 pgs: 89 active+clean; 193 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 399 KiB/s wr, 4 op/s 2024-04-12T11:31:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:24 smithi029 ceph-mon[97787]: osdmap e1079: 8 total, 8 up, 8 in 2024-04-12T11:31:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:24 smithi029 ceph-mon[93650]: pgmap v1254: 89 pgs: 89 active+clean; 193 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 399 KiB/s wr, 4 op/s 2024-04-12T11:31:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:24 smithi029 ceph-mon[93650]: osdmap e1079: 8 total, 8 up, 8 in 2024-04-12T11:31:25.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:31:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:31:24] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-12T11:31:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:24 smithi087 ceph-mon[89014]: pgmap v1254: 89 pgs: 89 active+clean; 193 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 399 KiB/s wr, 4 op/s 2024-04-12T11:31:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:24 smithi087 ceph-mon[89014]: osdmap e1079: 8 total, 8 up, 8 in 2024-04-12T11:31:25.773 INFO:tasks.rados.rados.0.smithi029.stdout:3998: done (2 left) 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3999: write oid 30 current snap is 507 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3996: finishing write tid 1 to smithi02979277-15 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3996: finishing write tid 2 to smithi02979277-15 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3996: finishing write tid 3 to smithi02979277-15 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3996: finishing write tid 4 to smithi02979277-15 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3996: finishing write tid 5 to smithi02979277-15 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3996: oid 15 updating version 0 to 1314 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3996: oid 15 version 1314 is already newer than 1313 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3996: oid 15 version 1314 is already newer than 1312 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:3996: oid 15 version 1314 is already newer than 1311 2024-04-12T11:31:25.774 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 15 v 1314 (ObjNum 1508 snap 507 seq_num 1508) dirty exists 2024-04-12T11:31:25.775 INFO:tasks.rados.rados.0.smithi029.stdout:3996: left oid 15 (ObjNum 1508 snap 507 seq_num 1508) 2024-04-12T11:31:25.775 INFO:tasks.rados.rados.0.smithi029.stdout:3997: expect (ObjNum 1452 snap 487 seq_num 1452) 2024-04-12T11:31:25.869 INFO:tasks.rados.rados.0.smithi029.stdout:3999: seq_num 1509 ranges {561763=588635,1552975=496198,2656002=405938} 2024-04-12T11:31:25.875 INFO:tasks.rados.rados.0.smithi029.stdout:3999: writing smithi02979277-30 from 561763 to 1150398 tid 1 2024-04-12T11:31:25.881 INFO:tasks.rados.rados.0.smithi029.stdout:3999: writing smithi02979277-30 from 1552975 to 2049173 tid 2 2024-04-12T11:31:25.886 INFO:tasks.rados.rados.0.smithi029.stdout:3999: writing smithi02979277-30 from 2656002 to 3061940 tid 3 2024-04-12T11:31:25.887 INFO:tasks.rados.rados.0.smithi029.stdout:3996: done (2 left) 2024-04-12T11:31:25.887 INFO:tasks.rados.rados.0.smithi029.stdout:3997: done (1 left) 2024-04-12T11:31:25.887 INFO:tasks.rados.rados.0.smithi029.stdout: waiting on 1 2024-04-12T11:31:25.887 INFO:tasks.rados.rados.0.smithi029.stdout:3999: finishing write tid 1 to smithi02979277-30 2024-04-12T11:31:25.887 INFO:tasks.rados.rados.0.smithi029.stdout:3999: finishing write tid 2 to smithi02979277-30 2024-04-12T11:31:25.889 INFO:tasks.rados.rados.0.smithi029.stdout:3999: finishing write tid 3 to smithi02979277-30 2024-04-12T11:31:25.889 INFO:tasks.rados.rados.0.smithi029.stdout:3999: finishing write tid 4 to smithi02979277-30 2024-04-12T11:31:25.889 INFO:tasks.rados.rados.0.smithi029.stdout:3999: finishing write tid 5 to smithi02979277-30 2024-04-12T11:31:25.890 INFO:tasks.rados.rados.0.smithi029.stdout:3999: oid 30 updating version 0 to 1320 2024-04-12T11:31:25.890 INFO:tasks.rados.rados.0.smithi029.stdout:3999: oid 30 updating version 1320 to 1323 2024-04-12T11:31:25.890 INFO:tasks.rados.rados.0.smithi029.stdout:3999: oid 30 version 1323 is already newer than 1321 2024-04-12T11:31:25.890 INFO:tasks.rados.rados.0.smithi029.stdout:3999: oid 30 version 1323 is already newer than 1322 2024-04-12T11:31:25.890 INFO:tasks.rados.rados.0.smithi029.stdout:update_object_version oid 30 v 1323 (ObjNum 1509 snap 507 seq_num 1509) dirty exists 2024-04-12T11:31:25.890 INFO:tasks.rados.rados.0.smithi029.stdout:3999: left oid 30 (ObjNum 1509 snap 507 seq_num 1509) 2024-04-12T11:31:25.890 INFO:tasks.rados.rados.0.smithi029.stdout:3999: done (0 left) 2024-04-12T11:31:25.891 INFO:tasks.rados.rados.0.smithi029.stderr:0 errors. 2024-04-12T11:31:25.891 INFO:tasks.rados.rados.0.smithi029.stderr: 2024-04-12T11:31:25.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T11:31:26.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:25 smithi029 ceph-mon[97787]: osdmap e1080: 8 total, 8 up, 8 in 2024-04-12T11:31:26.104 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:25 smithi029 ceph-mon[93650]: osdmap e1080: 8 total, 8 up, 8 in 2024-04-12T11:31:26.104 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:26.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:25 smithi087 ceph-mon[89014]: osdmap e1080: 8 total, 8 up, 8 in 2024-04-12T11:31:27.072 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:26 smithi029 ceph-mon[97787]: pgmap v1257: 89 pgs: 89 active+clean; 197 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-04-12T11:31:27.072 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:26 smithi029 ceph-mon[97787]: osdmap e1081: 8 total, 8 up, 8 in 2024-04-12T11:31:27.072 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:26 smithi029 ceph-mon[93650]: pgmap v1257: 89 pgs: 89 active+clean; 197 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-04-12T11:31:27.072 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:26 smithi029 ceph-mon[93650]: osdmap e1081: 8 total, 8 up, 8 in 2024-04-12T11:31:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:26 smithi087 ceph-mon[89014]: pgmap v1257: 89 pgs: 89 active+clean; 197 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-04-12T11:31:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:26 smithi087 ceph-mon[89014]: osdmap e1081: 8 total, 8 up, 8 in 2024-04-12T11:31:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:31:28.148 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":1082,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T11:31:27.740563+0000","last_up_change":"2024-04-12T11:23:35.611026+0000","last_in_change":"2024-04-12T11:11:02.730106+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T11:11:48.103219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T11:12:56.493343+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T11:12:57.001342+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2024-04-12T11:14:19.435735+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1081","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":584,"snap_epoch":1081,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":503,"up_from":514,"up_thru":611,"down_at":504,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6803","nonce":3899727946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6805","nonce":3899727946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6809","nonce":3899727946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6807","nonce":3899727946}]},"public_addr":"172.21.15.29:6803/3899727946","cluster_addr":"172.21.15.29:6805/3899727946","heartbeat_back_addr":"172.21.15.29:6809/3899727946","heartbeat_front_addr":"172.21.15.29:6807/3899727946","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":523,"up_from":534,"up_thru":611,"down_at":524,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6811","nonce":840649505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6813","nonce":840649505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6817","nonce":840649505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6815","nonce":840649505}]},"public_addr":"172.21.15.29:6811/840649505","cluster_addr":"172.21.15.29:6813/840649505","heartbeat_back_addr":"172.21.15.29:6817/840649505","heartbeat_front_addr":"172.21.15.29:6815/840649505","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":544,"up_from":555,"up_thru":611,"down_at":545,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6819","nonce":840062687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6821","nonce":840062687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6825","nonce":840062687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6823","nonce":840062687}]},"public_addr":"172.21.15.29:6819/840062687","cluster_addr":"172.21.15.29:6821/840062687","heartbeat_back_addr":"172.21.15.29:6825/840062687","heartbeat_front_addr":"172.21.15.29:6823/840062687","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":579,"up_from":590,"up_thru":611,"down_at":580,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6827","nonce":2609709308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6829","nonce":2609709308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6833","nonce":2609709308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6831","nonce":2609709308}]},"public_addr":"172.21.15.29:6827/2609709308","cluster_addr":"172.21.15.29:6829/2609709308","heartbeat_back_addr":"172.21.15.29:6833/2609709308","heartbeat_front_addr":"172.21.15.29:6831/2609709308","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":600,"up_from":611,"up_thru":611,"down_at":601,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6801","nonce":3230425210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6803","nonce":3230425210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6807","nonce":3230425210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6805","nonce":3230425210}]},"public_addr":"172.21.15.87:6801/3230425210","cluster_addr":"172.21.15.87:6803/3230425210","heartbeat_back_addr":"172.21.15.87:6807/3230425210","heartbeat_front_addr":"172.21.15.87:6805/3230425210","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":611,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6809","nonce":3642701185}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6811","nonce":3642701185}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6815","nonce":3642701185}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6813","nonce":3642701185}]},"public_addr":"172.21.15.87:6809/3642701185","cluster_addr":"172.21.15.87:6811/3642701185","heartbeat_back_addr":"172.21.15.87:6815/3642701185","heartbeat_front_addr":"172.21.15.87:6813/3642701185","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":611,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":611,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":503},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":523},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":544},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":579},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":600},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.9","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.87:0/943640318":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/3128860153":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/2463058069":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2946468060":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/600676740":"2024-04-13T11:17:21.160240+0000","172.21.15.29:6800/686523287":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2241771384":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/3254157714":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/659258741":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.87:6834/1005928369":"2024-04-13T11:16:36.316651+0000","172.21.15.29:6801/3931254419":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/4202867789":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.87:0/1543620546":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000","172.21.15.29:6800/3931254419":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:6801/686523287":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/1221327863":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/1314515623":"2024-04-13T11:17:21.160240+0000","172.21.15.87:6835/1005928369":"2024-04-13T11:16:36.316651+0000","172.21.15.87:0/220307911":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/4218166463":"2024-04-13T11:17:21.160240+0000","172.21.15.87:0/2414938593":"2024-04-13T11:16:36.316651+0000","172.21.15.29:6801/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6800/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.87:0/3681614798":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/364858296":"2024-04-13T11:11:58.108258+0000","172.21.15.87:0/67786467":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/683850481":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/273190388":"2024-04-13T11:17:21.160240+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":580,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":5,"snaps":[{"begin":558,"length":1},{"begin":576,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T11:31:28.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:31:29.017 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:28 smithi029 ceph-mon[93650]: pgmap v1259: 89 pgs: 89 active+clean; 200 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 877 KiB/s wr, 9 op/s 2024-04-12T11:31:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:28 smithi029 ceph-mon[93650]: osdmap e1082: 8 total, 8 up, 8 in 2024-04-12T11:31:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:28 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1829882178' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:31:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:28 smithi029 ceph-mon[97787]: pgmap v1259: 89 pgs: 89 active+clean; 200 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 877 KiB/s wr, 9 op/s 2024-04-12T11:31:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:28 smithi029 ceph-mon[97787]: osdmap e1082: 8 total, 8 up, 8 in 2024-04-12T11:31:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:28 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1829882178' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:31:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:28 smithi087 ceph-mon[89014]: pgmap v1259: 89 pgs: 89 active+clean; 200 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 877 KiB/s wr, 9 op/s 2024-04-12T11:31:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:28 smithi087 ceph-mon[89014]: osdmap e1082: 8 total, 8 up, 8 in 2024-04-12T11:31:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:28 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1829882178' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:31:30.919 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:31:30.921 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:31:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:30 smithi029 ceph-mon[93650]: pgmap v1261: 89 pgs: 89 active+clean; 206 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:31:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:30 smithi029 ceph-mon[97787]: pgmap v1261: 89 pgs: 89 active+clean; 206 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:31:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:30 smithi087 ceph-mon[89014]: pgmap v1261: 89 pgs: 89 active+clean; 206 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-04-12T11:31:31.562 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1261,"stamp":"2024-04-12T11:31:29.625210+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":216243297,"num_objects":132,"num_object_clones":60,"num_object_copies":396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":132,"num_whiteouts":13,"num_read":7557,"num_read_kb":7597445,"num_write":5970,"num_write_kb":1358977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":42251131,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":23063,"ondisk_log_size":23063,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":860016,"kb_used_data":455620,"kb_used_omap":175,"kb_used_meta":404112,"kb_avail":749101200,"statfs":{"total":767960285184,"available":767079628800,"internally_reserved":0,"allocated":466554880,"data_stored":530241424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":413810857},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":11138303,"num_objects":9,"num_object_clones":8,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":2,"num_read":47,"num_read_kb":52486,"num_write":31,"num_write_kb":7483,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.882105"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1507,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745017+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:31:23.745017+0000","last_peered":"2024-04-12T11:31:23.745017+0000","last_clean":"2024-04-12T11:31:23.745017+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:31:23.745017+0000","last_undegraded":"2024-04-12T11:31:23.745017+0000","last_fullsized":"2024-04-12T11:31:23.745017+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1521,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.745404+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:31:27.745404+0000","last_peered":"2024-04-12T11:31:27.745404+0000","last_clean":"2024-04-12T11:31:27.745404+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:31:27.745404+0000","last_undegraded":"2024-04-12T11:31:27.745404+0000","last_fullsized":"2024-04-12T11:31:27.745404+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1570,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.772938+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:31:25.772938+0000","last_peered":"2024-04-12T11:31:25.772938+0000","last_clean":"2024-04-12T11:31:25.772938+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:31:25.772938+0000","last_undegraded":"2024-04-12T11:31:25.772938+0000","last_fullsized":"2024-04-12T11:31:25.772938+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1565,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.775516+0000","last_change":"2024-04-12T11:23:15.593723+0000","last_active":"2024-04-12T11:31:25.775516+0000","last_peered":"2024-04-12T11:31:25.775516+0000","last_clean":"2024-04-12T11:31:25.775516+0000","last_became_active":"2024-04-12T11:23:15.593381+0000","last_became_peered":"2024-04-12T11:23:15.593381+0000","last_unstale":"2024-04-12T11:31:25.775516+0000","last_undegraded":"2024-04-12T11:31:25.775516+0000","last_fullsized":"2024-04-12T11:31:25.775516+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:18:06.971493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1529,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.743817+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:31:27.743817+0000","last_peered":"2024-04-12T11:31:27.743817+0000","last_clean":"2024-04-12T11:31:27.743817+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:31:27.743817+0000","last_undegraded":"2024-04-12T11:31:27.743817+0000","last_fullsized":"2024-04-12T11:31:27.743817+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1021,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.787534+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:31:24.787534+0000","last_peered":"2024-04-12T11:31:24.787534+0000","last_clean":"2024-04-12T11:31:24.787534+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:31:24.787534+0000","last_undegraded":"2024-04-12T11:31:24.787534+0000","last_fullsized":"2024-04-12T11:31:24.787534+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1561,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.775940+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:31:25.775940+0000","last_peered":"2024-04-12T11:31:25.775940+0000","last_clean":"2024-04-12T11:31:25.775940+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:31:25.775940+0000","last_undegraded":"2024-04-12T11:31:25.775940+0000","last_fullsized":"2024-04-12T11:31:25.775940+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1502,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.743726+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:31:27.743726+0000","last_peered":"2024-04-12T11:31:27.743726+0000","last_clean":"2024-04-12T11:31:27.743726+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:31:27.743726+0000","last_undegraded":"2024-04-12T11:31:27.743726+0000","last_fullsized":"2024-04-12T11:31:27.743726+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1027,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.765090+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:31:24.765090+0000","last_peered":"2024-04-12T11:31:24.765090+0000","last_clean":"2024-04-12T11:31:24.765090+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:31:24.765090+0000","last_undegraded":"2024-04-12T11:31:24.765090+0000","last_fullsized":"2024-04-12T11:31:24.765090+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1482,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.744211+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:31:27.744211+0000","last_peered":"2024-04-12T11:31:27.744211+0000","last_clean":"2024-04-12T11:31:27.744211+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:31:27.744211+0000","last_undegraded":"2024-04-12T11:31:27.744211+0000","last_fullsized":"2024-04-12T11:31:27.744211+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1036,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.926013+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:31:24.926013+0000","last_peered":"2024-04-12T11:31:24.926013+0000","last_clean":"2024-04-12T11:31:24.926013+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:31:24.926013+0000","last_undegraded":"2024-04-12T11:31:24.926013+0000","last_fullsized":"2024-04-12T11:31:24.926013+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1561,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.773061+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:31:25.773061+0000","last_peered":"2024-04-12T11:31:25.773061+0000","last_clean":"2024-04-12T11:31:25.773061+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:31:25.773061+0000","last_undegraded":"2024-04-12T11:31:25.773061+0000","last_fullsized":"2024-04-12T11:31:25.773061+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1479,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745344+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:31:23.745344+0000","last_peered":"2024-04-12T11:31:23.745344+0000","last_clean":"2024-04-12T11:31:23.745344+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:31:23.745344+0000","last_undegraded":"2024-04-12T11:31:23.745344+0000","last_fullsized":"2024-04-12T11:31:23.745344+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1470,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745264+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:31:23.745264+0000","last_peered":"2024-04-12T11:31:23.745264+0000","last_clean":"2024-04-12T11:31:23.745264+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:31:23.745264+0000","last_undegraded":"2024-04-12T11:31:23.745264+0000","last_fullsized":"2024-04-12T11:31:23.745264+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1035,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.925819+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:31:24.925819+0000","last_peered":"2024-04-12T11:31:24.925819+0000","last_clean":"2024-04-12T11:31:24.925819+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:31:24.925819+0000","last_undegraded":"2024-04-12T11:31:24.925819+0000","last_fullsized":"2024-04-12T11:31:24.925819+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1513,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.745066+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:31:27.745066+0000","last_peered":"2024-04-12T11:31:27.745066+0000","last_clean":"2024-04-12T11:31:27.745066+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:31:27.745066+0000","last_undegraded":"2024-04-12T11:31:27.745066+0000","last_fullsized":"2024-04-12T11:31:27.745066+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1563,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.775331+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:31:25.775331+0000","last_peered":"2024-04-12T11:31:25.775331+0000","last_clean":"2024-04-12T11:31:25.775331+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:31:25.775331+0000","last_undegraded":"2024-04-12T11:31:25.775331+0000","last_fullsized":"2024-04-12T11:31:25.775331+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1026,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.765123+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:31:24.765123+0000","last_peered":"2024-04-12T11:31:24.765123+0000","last_clean":"2024-04-12T11:31:24.765123+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:31:24.765123+0000","last_undegraded":"2024-04-12T11:31:24.765123+0000","last_fullsized":"2024-04-12T11:31:24.765123+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1018,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.925632+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:31:24.925632+0000","last_peered":"2024-04-12T11:31:24.925632+0000","last_clean":"2024-04-12T11:31:24.925632+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:31:24.925632+0000","last_undegraded":"2024-04-12T11:31:24.925632+0000","last_fullsized":"2024-04-12T11:31:24.925632+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1563,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.775329+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:31:25.775329+0000","last_peered":"2024-04-12T11:31:25.775329+0000","last_clean":"2024-04-12T11:31:25.775329+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:31:25.775329+0000","last_undegraded":"2024-04-12T11:31:25.775329+0000","last_fullsized":"2024-04-12T11:31:25.775329+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1029,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.787073+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:31:24.787073+0000","last_peered":"2024-04-12T11:31:24.787073+0000","last_clean":"2024-04-12T11:31:24.787073+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:31:24.787073+0000","last_undegraded":"2024-04-12T11:31:24.787073+0000","last_fullsized":"2024-04-12T11:31:24.787073+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1513,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.745626+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:31:27.745626+0000","last_peered":"2024-04-12T11:31:27.745626+0000","last_clean":"2024-04-12T11:31:27.745626+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:31:27.745626+0000","last_undegraded":"2024-04-12T11:31:27.745626+0000","last_fullsized":"2024-04-12T11:31:27.745626+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"1075'1707","reported_seq":4816,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.996362+0000","last_change":"2024-04-12T11:31:25.996730+0000","last_active":"2024-04-12T11:31:25.996362+0000","last_peered":"2024-04-12T11:31:25.996362+0000","last_clean":"2024-04-12T11:31:25.996362+0000","last_became_active":"2024-04-12T11:23:36.647505+0000","last_became_peered":"2024-04-12T11:23:36.647505+0000","last_unstale":"2024-04-12T11:31:25.996362+0000","last_undegraded":"2024-04-12T11:31:25.996362+0000","last_fullsized":"2024-04-12T11:31:25.996362+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":1707,"log_dups_size":0,"ondisk_log_size":1707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:46:26.971718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073669899999999995,"stat_sum":{"num_bytes":14641526,"num_objects":9,"num_object_clones":5,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":1,"num_read":497,"num_read_kb":491977,"num_write":434,"num_write_kb":102711,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":2725973,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"22e","length":"1"},{"start":"240","length":"1"},{"start":"244","length":"1"}]},{"pgid":"3.f","version":"68'1","reported_seq":1035,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.926227+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:31:24.926227+0000","last_peered":"2024-04-12T11:31:24.926227+0000","last_clean":"2024-04-12T11:31:24.926227+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:31:24.926227+0000","last_undegraded":"2024-04-12T11:31:24.926227+0000","last_fullsized":"2024-04-12T11:31:24.926227+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1012,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.764525+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:31:24.764525+0000","last_peered":"2024-04-12T11:31:24.764525+0000","last_clean":"2024-04-12T11:31:24.764525+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:31:24.764525+0000","last_undegraded":"2024-04-12T11:31:24.764525+0000","last_fullsized":"2024-04-12T11:31:24.764525+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"1073'598","reported_seq":4764,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:28.394614+0000","last_change":"2024-04-12T11:31:26.097271+0000","last_active":"2024-04-12T11:31:28.394614+0000","last_peered":"2024-04-12T11:31:28.394614+0000","last_clean":"2024-04-12T11:31:28.394614+0000","last_became_active":"2024-04-12T11:22:40.584535+0000","last_became_peered":"2024-04-12T11:22:40.584535+0000","last_unstale":"2024-04-12T11:31:28.394614+0000","last_undegraded":"2024-04-12T11:31:28.394614+0000","last_fullsized":"2024-04-12T11:31:28.394614+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":598,"log_dups_size":0,"ondisk_log_size":598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:22:34.265904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069622100000000001,"stat_sum":{"num_bytes":3321941,"num_objects":3,"num_object_clones":2,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":1,"num_read":133,"num_read_kb":153385,"num_write":139,"num_write_kb":34029,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1946729,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"244","length":"1"}]},{"pgid":"3.e","version":"68'2","reported_seq":1036,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.926088+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:31:24.926088+0000","last_peered":"2024-04-12T11:31:24.926088+0000","last_clean":"2024-04-12T11:31:24.926088+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:31:24.926088+0000","last_undegraded":"2024-04-12T11:31:24.926088+0000","last_fullsized":"2024-04-12T11:31:24.926088+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1492,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745879+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:31:23.745879+0000","last_peered":"2024-04-12T11:31:23.745879+0000","last_clean":"2024-04-12T11:31:23.745879+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:31:23.745879+0000","last_undegraded":"2024-04-12T11:31:23.745879+0000","last_fullsized":"2024-04-12T11:31:23.745879+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"1076'1810","reported_seq":5900,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:28.394860+0000","last_change":"2024-04-12T11:31:26.097680+0000","last_active":"2024-04-12T11:31:28.394860+0000","last_peered":"2024-04-12T11:31:28.394860+0000","last_clean":"2024-04-12T11:31:28.394860+0000","last_became_active":"2024-04-12T11:22:40.584231+0000","last_became_peered":"2024-04-12T11:22:40.584231+0000","last_unstale":"2024-04-12T11:31:28.394860+0000","last_undegraded":"2024-04-12T11:31:28.394860+0000","last_fullsized":"2024-04-12T11:31:28.394860+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":1810,"log_dups_size":0,"ondisk_log_size":1810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:18:07.111496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000865758,"stat_sum":{"num_bytes":14287699,"num_objects":6,"num_object_clones":2,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":1,"num_read":726,"num_read_kb":830305,"num_write":443,"num_write_kb":102973,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"244","length":"1"}]},{"pgid":"3.d","version":"68'1","reported_seq":1005,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.926128+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:31:24.926128+0000","last_peered":"2024-04-12T11:31:24.926128+0000","last_clean":"2024-04-12T11:31:24.926128+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:31:24.926128+0000","last_undegraded":"2024-04-12T11:31:24.926128+0000","last_fullsized":"2024-04-12T11:31:24.926128+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1021,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.787467+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:31:24.787467+0000","last_peered":"2024-04-12T11:31:24.787467+0000","last_clean":"2024-04-12T11:31:24.787467+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:31:24.787467+0000","last_undegraded":"2024-04-12T11:31:24.787467+0000","last_fullsized":"2024-04-12T11:31:24.787467+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"1075'482","reported_seq":4634,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:28.394705+0000","last_change":"2024-04-12T11:31:26.097442+0000","last_active":"2024-04-12T11:31:28.394705+0000","last_peered":"2024-04-12T11:31:28.394705+0000","last_clean":"2024-04-12T11:31:28.394705+0000","last_became_active":"2024-04-12T11:23:36.639354+0000","last_became_peered":"2024-04-12T11:23:36.639354+0000","last_unstale":"2024-04-12T11:31:28.394705+0000","last_undegraded":"2024-04-12T11:31:28.394705+0000","last_fullsized":"2024-04-12T11:31:28.394705+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":482,"log_dups_size":0,"ondisk_log_size":482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:46:37.327968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069093099999999999,"stat_sum":{"num_bytes":5977489,"num_objects":3,"num_object_clones":2,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":155,"num_read_kb":157635,"num_write":125,"num_write_kb":29358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":3893555,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"244","length":"1"}]},{"pgid":"3.c","version":"0'0","reported_seq":1028,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.764850+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:31:24.764850+0000","last_peered":"2024-04-12T11:31:24.764850+0000","last_clean":"2024-04-12T11:31:24.764850+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:31:24.764850+0000","last_undegraded":"2024-04-12T11:31:24.764850+0000","last_fullsized":"2024-04-12T11:31:24.764850+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1573,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.775409+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:31:25.775409+0000","last_peered":"2024-04-12T11:31:25.775409+0000","last_clean":"2024-04-12T11:31:25.775409+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:31:25.775409+0000","last_undegraded":"2024-04-12T11:31:25.775409+0000","last_fullsized":"2024-04-12T11:31:25.775409+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"1081'2609","reported_seq":6583,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:28.394988+0000","last_change":"2024-04-12T11:31:26.101918+0000","last_active":"2024-04-12T11:31:28.394988+0000","last_peered":"2024-04-12T11:31:28.394988+0000","last_clean":"2024-04-12T11:31:28.394988+0000","last_became_active":"2024-04-12T11:22:40.581653+0000","last_became_peered":"2024-04-12T11:22:40.581653+0000","last_unstale":"2024-04-12T11:31:28.394988+0000","last_undegraded":"2024-04-12T11:31:28.394988+0000","last_fullsized":"2024-04-12T11:31:28.394988+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":2609,"log_dups_size":0,"ondisk_log_size":2609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:02:38.632913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0045061220000000004,"stat_sum":{"num_bytes":34348563,"num_objects":15,"num_object_clones":9,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":1,"num_read":836,"num_read_kb":953583,"num_write":645,"num_write_kb":154625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":2829111,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"244","length":"1"}]},{"pgid":"3.b","version":"68'2","reported_seq":1506,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.744720+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:31:27.744720+0000","last_peered":"2024-04-12T11:31:27.744720+0000","last_clean":"2024-04-12T11:31:27.744720+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:31:27.744720+0000","last_undegraded":"2024-04-12T11:31:27.744720+0000","last_fullsized":"2024-04-12T11:31:27.744720+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1483,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745534+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:31:23.745534+0000","last_peered":"2024-04-12T11:31:23.745534+0000","last_clean":"2024-04-12T11:31:23.745534+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:31:23.745534+0000","last_undegraded":"2024-04-12T11:31:23.745534+0000","last_fullsized":"2024-04-12T11:31:23.745534+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"1073'1400","reported_seq":5511,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.774576+0000","last_change":"2024-04-12T11:31:25.774920+0000","last_active":"2024-04-12T11:31:25.774576+0000","last_peered":"2024-04-12T11:31:25.774576+0000","last_clean":"2024-04-12T11:31:25.774576+0000","last_became_active":"2024-04-12T11:23:36.636905+0000","last_became_peered":"2024-04-12T11:23:36.636905+0000","last_unstale":"2024-04-12T11:31:25.774576+0000","last_undegraded":"2024-04-12T11:31:25.774576+0000","last_fullsized":"2024-04-12T11:31:25.774576+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":1400,"log_dups_size":0,"ondisk_log_size":1400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:27:50.001207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010808269999999999,"stat_sum":{"num_bytes":15245632,"num_objects":5,"num_object_clones":2,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":414,"num_read_kb":399709,"num_write":345,"num_write_kb":82179,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":5167729,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"22e","length":"1"},{"start":"240","length":"1"},{"start":"244","length":"1"}]},{"pgid":"3.a","version":"0'0","reported_seq":1037,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.787285+0000","last_change":"2024-04-12T11:23:36.639270+0000","last_active":"2024-04-12T11:31:24.787285+0000","last_peered":"2024-04-12T11:31:24.787285+0000","last_clean":"2024-04-12T11:31:24.787285+0000","last_became_active":"2024-04-12T11:23:36.638952+0000","last_became_peered":"2024-04-12T11:23:36.638952+0000","last_unstale":"2024-04-12T11:31:24.787285+0000","last_undegraded":"2024-04-12T11:31:24.787285+0000","last_fullsized":"2024-04-12T11:31:24.787285+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:04:50.403812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1505,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745275+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:31:23.745275+0000","last_peered":"2024-04-12T11:31:23.745275+0000","last_clean":"2024-04-12T11:31:23.745275+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:31:23.745275+0000","last_undegraded":"2024-04-12T11:31:23.745275+0000","last_fullsized":"2024-04-12T11:31:23.745275+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"1073'412","reported_seq":3640,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.771076+0000","last_change":"2024-04-12T11:31:25.771433+0000","last_active":"2024-04-12T11:31:25.771076+0000","last_peered":"2024-04-12T11:31:25.771076+0000","last_clean":"2024-04-12T11:31:25.771076+0000","last_became_active":"2024-04-12T11:23:36.645150+0000","last_became_peered":"2024-04-12T11:23:36.645150+0000","last_unstale":"2024-04-12T11:31:25.771076+0000","last_undegraded":"2024-04-12T11:31:25.771076+0000","last_fullsized":"2024-04-12T11:31:25.771076+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":412,"log_dups_size":0,"ondisk_log_size":412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:32.468731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048010899999999999,"stat_sum":{"num_bytes":3445765,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":187125,"num_write":116,"num_write_kb":27951,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":2507020,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"22e","length":"1"},{"start":"240","length":"1"},{"start":"244","length":"1"}]},{"pgid":"3.9","version":"0'0","reported_seq":1491,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745895+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:31:23.745895+0000","last_peered":"2024-04-12T11:31:23.745895+0000","last_clean":"2024-04-12T11:31:23.745895+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:31:23.745895+0000","last_undegraded":"2024-04-12T11:31:23.745895+0000","last_fullsized":"2024-04-12T11:31:23.745895+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1470,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745701+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:31:23.745701+0000","last_peered":"2024-04-12T11:31:23.745701+0000","last_clean":"2024-04-12T11:31:23.745701+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:31:23.745701+0000","last_undegraded":"2024-04-12T11:31:23.745701+0000","last_fullsized":"2024-04-12T11:31:23.745701+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"1079'953","reported_seq":4933,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.769392+0000","last_change":"2024-04-12T11:31:19.754637+0000","last_active":"2024-04-12T11:31:23.769392+0000","last_peered":"2024-04-12T11:31:23.769392+0000","last_clean":"2024-04-12T11:31:23.769392+0000","last_became_active":"2024-04-12T11:23:36.643229+0000","last_became_peered":"2024-04-12T11:23:36.643229+0000","last_unstale":"2024-04-12T11:31:23.769392+0000","last_undegraded":"2024-04-12T11:31:23.769392+0000","last_fullsized":"2024-04-12T11:31:23.769392+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":953,"log_dups_size":0,"ondisk_log_size":953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:47:16.188158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0027391849999999999,"stat_sum":{"num_bytes":9752998,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":231,"num_read_kb":242983,"num_write":212,"num_write_kb":51245,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":2648951,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"22e","length":"1"},{"start":"240","length":"1"}]},{"pgid":"3.8","version":"0'0","reported_seq":1514,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.743623+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:31:27.743623+0000","last_peered":"2024-04-12T11:31:27.743623+0000","last_clean":"2024-04-12T11:31:27.743623+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:31:27.743623+0000","last_undegraded":"2024-04-12T11:31:27.743623+0000","last_fullsized":"2024-04-12T11:31:27.743623+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1571,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.773616+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:31:25.773616+0000","last_peered":"2024-04-12T11:31:25.773616+0000","last_clean":"2024-04-12T11:31:25.773616+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:31:25.773616+0000","last_undegraded":"2024-04-12T11:31:25.773616+0000","last_fullsized":"2024-04-12T11:31:25.773616+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1572,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.774322+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:31:25.774322+0000","last_peered":"2024-04-12T11:31:25.774322+0000","last_clean":"2024-04-12T11:31:25.774322+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:31:25.774322+0000","last_undegraded":"2024-04-12T11:31:25.774322+0000","last_fullsized":"2024-04-12T11:31:25.774322+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1043,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.765469+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:31:24.765469+0000","last_peered":"2024-04-12T11:31:24.765469+0000","last_clean":"2024-04-12T11:31:24.765469+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:31:24.765469+0000","last_undegraded":"2024-04-12T11:31:24.765469+0000","last_fullsized":"2024-04-12T11:31:24.765469+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1563,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.773106+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:31:25.773106+0000","last_peered":"2024-04-12T11:31:25.773106+0000","last_clean":"2024-04-12T11:31:25.773106+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:31:25.773106+0000","last_undegraded":"2024-04-12T11:31:25.773106+0000","last_fullsized":"2024-04-12T11:31:25.773106+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"1081'1995","reported_seq":6086,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.779197+0000","last_change":"2024-04-12T11:31:25.779458+0000","last_active":"2024-04-12T11:31:25.779197+0000","last_peered":"2024-04-12T11:31:25.779197+0000","last_clean":"2024-04-12T11:31:25.779197+0000","last_became_active":"2024-04-12T11:21:59.595112+0000","last_became_peered":"2024-04-12T11:21:59.595112+0000","last_unstale":"2024-04-12T11:31:25.779197+0000","last_undegraded":"2024-04-12T11:31:25.779197+0000","last_fullsized":"2024-04-12T11:31:25.779197+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":1995,"log_dups_size":0,"ondisk_log_size":1995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:38:21.683605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0037524379999999999,"stat_sum":{"num_bytes":19902707,"num_objects":9,"num_object_clones":5,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":2,"num_read":605,"num_read_kb":646121,"num_write":533,"num_write_kb":125232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":1966903,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"22e","length":"1"},{"start":"240","length":"1"},{"start":"244","length":"1"}]},{"pgid":"5.5","version":"1081'1323","reported_seq":5463,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.890073+0000","last_change":"2024-04-12T11:31:25.779358+0000","last_active":"2024-04-12T11:31:25.890073+0000","last_peered":"2024-04-12T11:31:25.890073+0000","last_clean":"2024-04-12T11:31:25.890073+0000","last_became_active":"2024-04-12T11:23:36.648558+0000","last_became_peered":"2024-04-12T11:23:36.648558+0000","last_unstale":"2024-04-12T11:31:25.890073+0000","last_undegraded":"2024-04-12T11:31:25.890073+0000","last_fullsized":"2024-04-12T11:31:25.890073+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":1323,"log_dups_size":0,"ondisk_log_size":1323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:21:48.240693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0034389350000000002,"stat_sum":{"num_bytes":17671095,"num_objects":9,"num_object_clones":6,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":331,"num_read_kb":333392,"num_write":348,"num_write_kb":82482,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":3499219,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"22e","length":"1"},{"start":"240","length":"1"},{"start":"244","length":"1"}]},{"pgid":"2.2","version":"0'0","reported_seq":1033,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.764656+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:31:24.764656+0000","last_peered":"2024-04-12T11:31:24.764656+0000","last_clean":"2024-04-12T11:31:24.764656+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:31:24.764656+0000","last_undegraded":"2024-04-12T11:31:24.764656+0000","last_fullsized":"2024-04-12T11:31:24.764656+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1491,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745138+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:31:23.745138+0000","last_peered":"2024-04-12T11:31:23.745138+0000","last_clean":"2024-04-12T11:31:23.745138+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:31:23.745138+0000","last_undegraded":"2024-04-12T11:31:23.745138+0000","last_fullsized":"2024-04-12T11:31:23.745138+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1572,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.773509+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:31:25.773509+0000","last_peered":"2024-04-12T11:31:25.773509+0000","last_clean":"2024-04-12T11:31:25.773509+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:31:25.773509+0000","last_undegraded":"2024-04-12T11:31:25.773509+0000","last_fullsized":"2024-04-12T11:31:25.773509+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"1073'1356","reported_seq":5421,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:28.394988+0000","last_change":"2024-04-12T11:31:26.098071+0000","last_active":"2024-04-12T11:31:28.394988+0000","last_peered":"2024-04-12T11:31:28.394988+0000","last_clean":"2024-04-12T11:31:28.394988+0000","last_became_active":"2024-04-12T11:22:40.581320+0000","last_became_peered":"2024-04-12T11:22:40.581320+0000","last_unstale":"2024-04-12T11:31:28.394988+0000","last_undegraded":"2024-04-12T11:31:28.394988+0000","last_fullsized":"2024-04-12T11:31:28.394988+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":1356,"log_dups_size":0,"ondisk_log_size":1356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:53.541820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063582500000000002,"stat_sum":{"num_bytes":12081731,"num_objects":7,"num_object_clones":4,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":2,"num_read":415,"num_read_kb":418035,"num_write":306,"num_write_kb":73877,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":3542942,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"244","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":1538,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:28.394562+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:31:28.394562+0000","last_peered":"2024-04-12T11:31:28.394562+0000","last_clean":"2024-04-12T11:31:28.394562+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:31:28.394562+0000","last_undegraded":"2024-04-12T11:31:28.394562+0000","last_fullsized":"2024-04-12T11:31:28.394562+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1563,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.773996+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:31:25.773996+0000","last_peered":"2024-04-12T11:31:25.773996+0000","last_clean":"2024-04-12T11:31:25.773996+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:31:25.773996+0000","last_undegraded":"2024-04-12T11:31:25.773996+0000","last_fullsized":"2024-04-12T11:31:25.773996+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1566,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.773818+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:31:25.773818+0000","last_peered":"2024-04-12T11:31:25.773818+0000","last_clean":"2024-04-12T11:31:25.773818+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:31:25.773818+0000","last_undegraded":"2024-04-12T11:31:25.773818+0000","last_fullsized":"2024-04-12T11:31:25.773818+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1033,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.764712+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:31:24.764712+0000","last_peered":"2024-04-12T11:31:24.764712+0000","last_clean":"2024-04-12T11:31:24.764712+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:31:24.764712+0000","last_undegraded":"2024-04-12T11:31:24.764712+0000","last_fullsized":"2024-04-12T11:31:24.764712+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1387,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.925588+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:31:24.925588+0000","last_peered":"2024-04-12T11:31:24.925588+0000","last_clean":"2024-04-12T11:31:24.925588+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:31:24.925588+0000","last_undegraded":"2024-04-12T11:31:24.925588+0000","last_fullsized":"2024-04-12T11:31:24.925588+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1481,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.744385+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:31:27.744385+0000","last_peered":"2024-04-12T11:31:27.744385+0000","last_clean":"2024-04-12T11:31:27.744385+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:31:27.744385+0000","last_undegraded":"2024-04-12T11:31:27.744385+0000","last_fullsized":"2024-04-12T11:31:27.744385+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1021,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.787174+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:31:24.787174+0000","last_peered":"2024-04-12T11:31:24.787174+0000","last_clean":"2024-04-12T11:31:24.787174+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:31:24.787174+0000","last_undegraded":"2024-04-12T11:31:24.787174+0000","last_fullsized":"2024-04-12T11:31:24.787174+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"1079'2149","reported_seq":5187,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.996585+0000","last_change":"2024-04-12T11:31:25.996989+0000","last_active":"2024-04-12T11:31:25.996585+0000","last_peered":"2024-04-12T11:31:25.996585+0000","last_clean":"2024-04-12T11:31:25.996585+0000","last_became_active":"2024-04-12T11:22:40.585627+0000","last_became_peered":"2024-04-12T11:22:40.585627+0000","last_unstale":"2024-04-12T11:31:25.996585+0000","last_undegraded":"2024-04-12T11:31:25.996585+0000","last_fullsized":"2024-04-12T11:31:25.996585+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":2149,"log_dups_size":0,"ondisk_log_size":2149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:16:01.663375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072102799999999997,"stat_sum":{"num_bytes":17008317,"num_objects":9,"num_object_clones":5,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":1,"num_read":609,"num_read_kb":621350,"num_write":520,"num_write_kb":128036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"22e","length":"1"},{"start":"240","length":"1"},{"start":"244","length":"1"}]},{"pgid":"5.7","version":"1075'2002","reported_seq":5078,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.770784+0000","last_change":"2024-04-12T11:31:25.771540+0000","last_active":"2024-04-12T11:31:25.770784+0000","last_peered":"2024-04-12T11:31:25.770784+0000","last_clean":"2024-04-12T11:31:25.770784+0000","last_became_active":"2024-04-12T11:22:19.607400+0000","last_became_peered":"2024-04-12T11:22:19.607400+0000","last_unstale":"2024-04-12T11:31:25.770784+0000","last_undegraded":"2024-04-12T11:31:25.770784+0000","last_fullsized":"2024-04-12T11:31:25.770784+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":2002,"log_dups_size":0,"ondisk_log_size":2002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:08:55.403630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088998400000000002,"stat_sum":{"num_bytes":23517942,"num_objects":12,"num_object_clones":7,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":2,"num_read":587,"num_read_kb":686643,"num_write":494,"num_write_kb":120831,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":3492764,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"22e","length":"1"},{"start":"240","length":"1"},{"start":"244","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":1040,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.925684+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:31:24.925684+0000","last_peered":"2024-04-12T11:31:24.925684+0000","last_clean":"2024-04-12T11:31:24.925684+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:31:24.925684+0000","last_undegraded":"2024-04-12T11:31:24.925684+0000","last_fullsized":"2024-04-12T11:31:24.925684+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1573,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.774988+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:31:25.774988+0000","last_peered":"2024-04-12T11:31:25.774988+0000","last_clean":"2024-04-12T11:31:25.774988+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:31:25.774988+0000","last_undegraded":"2024-04-12T11:31:25.774988+0000","last_fullsized":"2024-04-12T11:31:25.774988+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1573,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.775222+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:31:25.775222+0000","last_peered":"2024-04-12T11:31:25.775222+0000","last_clean":"2024-04-12T11:31:25.775222+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:31:25.775222+0000","last_undegraded":"2024-04-12T11:31:25.775222+0000","last_fullsized":"2024-04-12T11:31:25.775222+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1040,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.926000+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:31:24.926000+0000","last_peered":"2024-04-12T11:31:24.926000+0000","last_clean":"2024-04-12T11:31:24.926000+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:31:24.926000+0000","last_undegraded":"2024-04-12T11:31:24.926000+0000","last_fullsized":"2024-04-12T11:31:24.926000+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1563,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.774130+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:31:25.774130+0000","last_peered":"2024-04-12T11:31:25.774130+0000","last_clean":"2024-04-12T11:31:25.774130+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:31:25.774130+0000","last_undegraded":"2024-04-12T11:31:25.774130+0000","last_fullsized":"2024-04-12T11:31:25.774130+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1556,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.776120+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:31:25.776120+0000","last_peered":"2024-04-12T11:31:25.776120+0000","last_clean":"2024-04-12T11:31:25.776120+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:31:25.776120+0000","last_undegraded":"2024-04-12T11:31:25.776120+0000","last_fullsized":"2024-04-12T11:31:25.776120+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"1081'2005","reported_seq":5110,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.780864+0000","last_change":"2024-04-12T11:31:25.781051+0000","last_active":"2024-04-12T11:31:25.780864+0000","last_peered":"2024-04-12T11:31:25.780864+0000","last_clean":"2024-04-12T11:31:25.780864+0000","last_became_active":"2024-04-12T11:21:59.593842+0000","last_became_peered":"2024-04-12T11:21:59.593842+0000","last_unstale":"2024-04-12T11:31:25.780864+0000","last_undegraded":"2024-04-12T11:31:25.780864+0000","last_fullsized":"2024-04-12T11:31:25.780864+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":2005,"log_dups_size":0,"ondisk_log_size":2005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:40:27.350718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0023555690000000001,"stat_sum":{"num_bytes":16127836,"num_objects":9,"num_object_clones":5,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":1,"num_read":701,"num_read_kb":805583,"num_write":531,"num_write_kb":130071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":1704639,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"22e","length":"1"},{"start":"240","length":"1"},{"start":"244","length":"1"}]},{"pgid":"3.7","version":"68'2","reported_seq":1497,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.745657+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:31:27.745657+0000","last_peered":"2024-04-12T11:31:27.745657+0000","last_clean":"2024-04-12T11:31:27.745657+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:31:27.745657+0000","last_undegraded":"2024-04-12T11:31:27.745657+0000","last_fullsized":"2024-04-12T11:31:27.745657+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1563,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.773637+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:31:25.773637+0000","last_peered":"2024-04-12T11:31:25.773637+0000","last_clean":"2024-04-12T11:31:25.773637+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:31:25.773637+0000","last_undegraded":"2024-04-12T11:31:25.773637+0000","last_fullsized":"2024-04-12T11:31:25.773637+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1495,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.743583+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:31:27.743583+0000","last_peered":"2024-04-12T11:31:27.743583+0000","last_clean":"2024-04-12T11:31:27.743583+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:31:27.743583+0000","last_undegraded":"2024-04-12T11:31:27.743583+0000","last_fullsized":"2024-04-12T11:31:27.743583+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"1073'503","reported_seq":4534,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745234+0000","last_change":"2024-04-12T11:31:19.752302+0000","last_active":"2024-04-12T11:31:23.745234+0000","last_peered":"2024-04-12T11:31:23.745234+0000","last_clean":"2024-04-12T11:31:23.745234+0000","last_became_active":"2024-04-12T11:23:36.649605+0000","last_became_peered":"2024-04-12T11:23:36.649605+0000","last_unstale":"2024-04-12T11:31:23.745234+0000","last_undegraded":"2024-04-12T11:31:23.745234+0000","last_fullsized":"2024-04-12T11:31:23.745234+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":503,"log_dups_size":0,"ondisk_log_size":503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:59:14.056968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011944010000000001,"stat_sum":{"num_bytes":3676709,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":92,"num_read_kb":99896,"num_write":108,"num_write_kb":25261,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":1423026,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"22e","length":"1"},{"start":"240","length":"1"}]},{"pgid":"2.7","version":"0'0","reported_seq":1028,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.787380+0000","last_change":"2024-04-12T11:22:40.580200+0000","last_active":"2024-04-12T11:31:24.787380+0000","last_peered":"2024-04-12T11:31:24.787380+0000","last_clean":"2024-04-12T11:31:24.787380+0000","last_became_active":"2024-04-12T11:22:40.579809+0000","last_became_peered":"2024-04-12T11:22:40.579809+0000","last_unstale":"2024-04-12T11:31:24.787380+0000","last_undegraded":"2024-04-12T11:31:24.787380+0000","last_fullsized":"2024-04-12T11:31:24.787380+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:28:23.317249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1576,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.775625+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:31:25.775625+0000","last_peered":"2024-04-12T11:31:25.775625+0000","last_clean":"2024-04-12T11:31:25.775625+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:31:25.775625+0000","last_undegraded":"2024-04-12T11:31:25.775625+0000","last_fullsized":"2024-04-12T11:31:25.775625+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1478,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745279+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:31:23.745279+0000","last_peered":"2024-04-12T11:31:23.745279+0000","last_clean":"2024-04-12T11:31:23.745279+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:31:23.745279+0000","last_undegraded":"2024-04-12T11:31:23.745279+0000","last_fullsized":"2024-04-12T11:31:23.745279+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"1081'1543","reported_seq":5590,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.743756+0000","last_change":"2024-04-12T11:31:25.775803+0000","last_active":"2024-04-12T11:31:27.743756+0000","last_peered":"2024-04-12T11:31:27.743756+0000","last_clean":"2024-04-12T11:31:27.743756+0000","last_became_active":"2024-04-12T11:23:36.635759+0000","last_became_peered":"2024-04-12T11:23:36.635759+0000","last_unstale":"2024-04-12T11:31:27.743756+0000","last_undegraded":"2024-04-12T11:31:27.743756+0000","last_fullsized":"2024-04-12T11:31:27.743756+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":71,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:14:19.485873+0000","last_clean_scrub_stamp":"2024-04-12T11:14:19.485873+0000","objects_scrubbed":0,"log_size":1543,"log_dups_size":0,"ondisk_log_size":1543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:10:18.867345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0050596080000000002,"stat_sum":{"num_bytes":4639425,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":1,"num_read":559,"num_read_kb":568799,"num_write":371,"num_write_kb":85670,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":3131466,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"244","length":"1"}]},{"pgid":"3.1b","version":"68'1","reported_seq":1566,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.775991+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:31:25.775991+0000","last_peered":"2024-04-12T11:31:25.775991+0000","last_clean":"2024-04-12T11:31:25.775991+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:31:25.775991+0000","last_undegraded":"2024-04-12T11:31:25.775991+0000","last_fullsized":"2024-04-12T11:31:25.775991+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1549,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:28.395014+0000","last_change":"2024-04-12T11:23:15.593766+0000","last_active":"2024-04-12T11:31:28.395014+0000","last_peered":"2024-04-12T11:31:28.395014+0000","last_clean":"2024-04-12T11:31:28.395014+0000","last_became_active":"2024-04-12T11:23:15.593428+0000","last_became_peered":"2024-04-12T11:23:15.593428+0000","last_unstale":"2024-04-12T11:31:28.395014+0000","last_undegraded":"2024-04-12T11:31:28.395014+0000","last_fullsized":"2024-04-12T11:31:28.395014+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:12.389667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1044,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.764971+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:31:24.764971+0000","last_peered":"2024-04-12T11:31:24.764971+0000","last_clean":"2024-04-12T11:31:24.764971+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:31:24.764971+0000","last_undegraded":"2024-04-12T11:31:24.764971+0000","last_fullsized":"2024-04-12T11:31:24.764971+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1507,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.745450+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:31:23.745450+0000","last_peered":"2024-04-12T11:31:23.745450+0000","last_clean":"2024-04-12T11:31:23.745450+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:31:23.745450+0000","last_undegraded":"2024-04-12T11:31:23.745450+0000","last_fullsized":"2024-04-12T11:31:23.745450+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1029,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-04-12T11:31:24.765247+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:31:24.765247+0000","last_peered":"2024-04-12T11:31:24.765247+0000","last_clean":"2024-04-12T11:31:24.765247+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:31:24.765247+0000","last_undegraded":"2024-04-12T11:31:24.765247+0000","last_fullsized":"2024-04-12T11:31:24.765247+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1498,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.744787+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:31:23.744787+0000","last_peered":"2024-04-12T11:31:23.744787+0000","last_clean":"2024-04-12T11:31:23.744787+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:31:23.744787+0000","last_undegraded":"2024-04-12T11:31:23.744787+0000","last_fullsized":"2024-04-12T11:31:23.744787+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1496,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.745448+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:31:27.745448+0000","last_peered":"2024-04-12T11:31:27.745448+0000","last_clean":"2024-04-12T11:31:27.745448+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:31:27.745448+0000","last_undegraded":"2024-04-12T11:31:27.745448+0000","last_fullsized":"2024-04-12T11:31:27.745448+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1507,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-04-12T11:31:27.744076+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:31:27.744076+0000","last_peered":"2024-04-12T11:31:27.744076+0000","last_clean":"2024-04-12T11:31:27.744076+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:31:27.744076+0000","last_undegraded":"2024-04-12T11:31:27.744076+0000","last_fullsized":"2024-04-12T11:31:27.744076+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1570,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-04-12T11:31:25.775846+0000","last_change":"2024-04-12T11:22:40.582361+0000","last_active":"2024-04-12T11:31:25.775846+0000","last_peered":"2024-04-12T11:31:25.775846+0000","last_clean":"2024-04-12T11:31:25.775846+0000","last_became_active":"2024-04-12T11:22:40.581996+0000","last_became_peered":"2024-04-12T11:22:40.581996+0000","last_unstale":"2024-04-12T11:31:25.775846+0000","last_undegraded":"2024-04-12T11:31:25.775846+0000","last_fullsized":"2024-04-12T11:31:25.775846+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:40:43.563258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1470,"reported_epoch":1079,"state":"active+clean","last_fresh":"2024-04-12T11:31:23.744821+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:31:23.744821+0000","last_peered":"2024-04-12T11:31:23.744821+0000","last_clean":"2024-04-12T11:31:23.744821+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:31:23.744821+0000","last_undegraded":"2024-04-12T11:31:23.744821+0000","last_fullsized":"2024-04-12T11:31:23.744821+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":215647375,"num_objects":108,"num_object_clones":60,"num_object_copies":324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108,"num_whiteouts":13,"num_read":7057,"num_read_kb":7596521,"num_write":5670,"num_write_kb":1356531,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":40480027,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":397185024,"data_stored":482739180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22847,"ondisk_log_size":22847,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234228,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":111088,"kb_used_data":61924,"kb_used_omap":1,"kb_used_meta":49150,"kb_avail":93634064,"statfs":{"total":95995035648,"available":95881281536,"internally_reserved":0,"allocated":63410176,"data_stored":71212407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":50330059},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495865,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":98068,"kb_used_data":40704,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93647084,"statfs":{"total":95995035648,"available":95894614016,"internally_reserved":0,"allocated":41680896,"data_stored":46235563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":58718664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536175,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":72096,"kb_used_data":44304,"kb_used_omap":30,"kb_used_meta":27745,"kb_avail":93673056,"statfs":{"total":95995035648,"available":95921209344,"internally_reserved":0,"allocated":45367296,"data_stored":54000251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":28411202},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190259,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":136400,"kb_used_data":77956,"kb_used_omap":43,"kb_used_meta":58388,"kb_avail":93608752,"statfs":{"total":95995035648,"available":95855362048,"internally_reserved":0,"allocated":79826944,"data_stored":91266919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":59789355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849387,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":118076,"kb_used_data":63788,"kb_used_omap":36,"kb_used_meta":54235,"kb_avail":93627076,"statfs":{"total":95995035648,"available":95874125824,"internally_reserved":0,"allocated":65318912,"data_stored":72960443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":55536820},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704740,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":53156,"kb_used_data":17240,"kb_used_omap":17,"kb_used_meta":35886,"kb_avail":93691996,"statfs":{"total":95995035648,"available":95940603904,"internally_reserved":0,"allocated":17653760,"data_stored":18082202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":36747767},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225017951,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":82244,"kb_used_data":50104,"kb_used_omap":42,"kb_used_meta":32085,"kb_avail":93662908,"statfs":{"total":95995035648,"available":95910817792,"internally_reserved":0,"allocated":51306496,"data_stored":59791432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":32855864},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757501,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":188888,"kb_used_data":99600,"kb_used_omap":1,"kb_used_meta":89278,"kb_avail":93556264,"statfs":{"total":95995035648,"available":95801614336,"internally_reserved":0,"allocated":101990400,"data_stored":116692207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":91421126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":70787072,"data_stored":84960503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":36929536,"data_stored":48284807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":56872960,"data_stored":67244383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":9199616,"data_stored":12358238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":42868736,"data_stored":54088554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":93544448,"data_stored":110976123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":32632832,"data_stored":39924937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":54349824,"data_stored":64901635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:31:31.566 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-04-12T11:31:31.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-04-12T11:31:31.780 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:31.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:31 smithi029 ceph-mon[97787]: from='client.44462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:31:31.901 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:31 smithi029 ceph-mon[93650]: from='client.44462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:31:32.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:31 smithi087 ceph-mon[89014]: from='client.44462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:31:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:31:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:31:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:31:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:32 smithi029 ceph-mon[97787]: pgmap v1262: 89 pgs: 89 active+clean; 206 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 873 KiB/s wr, 10 op/s 2024-04-12T11:31:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:32 smithi029 ceph-mon[97787]: osdmap e1083: 8 total, 8 up, 8 in 2024-04-12T11:31:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:32 smithi029 ceph-mon[93650]: pgmap v1262: 89 pgs: 89 active+clean; 206 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 873 KiB/s wr, 10 op/s 2024-04-12T11:31:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:32 smithi029 ceph-mon[93650]: osdmap e1083: 8 total, 8 up, 8 in 2024-04-12T11:31:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:32 smithi087 ceph-mon[89014]: pgmap v1262: 89 pgs: 89 active+clean; 206 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 873 KiB/s wr, 10 op/s 2024-04-12T11:31:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:32 smithi087 ceph-mon[89014]: osdmap e1083: 8 total, 8 up, 8 in 2024-04-12T11:31:33.808 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_0' removed 2024-04-12T11:31:34.102 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:33 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2002448478' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:31:34.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:33 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2002448478' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:31:34.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:33 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2002448478' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:31:34.385 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-12T11:31:34.385 INFO:teuthology.task.print:**** done rados/snaps-few-objects 5-workload 2024-04-12T11:31:34.385 DEBUG:teuthology.parallel:result is None 2024-04-12T11:31:34.386 INFO:teuthology.run_tasks:Running task print... 2024-04-12T11:31:34.398 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-12T11:31:34.398 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-12T11:31:34.407 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-12T11:31:34.407 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-12T11:31:34.407 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph mgr dump --format=json 2024-04-12T11:31:34.615 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:34.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:34 smithi029 ceph-mon[97787]: pgmap v1264: 89 pgs: 89 active+clean; 206 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 762 KiB/s wr, 9 op/s 2024-04-12T11:31:34.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:34 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2002448478' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:31:34.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:34 smithi029 ceph-mon[97787]: osdmap e1084: 8 total, 8 up, 8 in 2024-04-12T11:31:34.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:34 smithi029 ceph-mon[93650]: pgmap v1264: 89 pgs: 89 active+clean; 206 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 762 KiB/s wr, 9 op/s 2024-04-12T11:31:34.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:34 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2002448478' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:31:34.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:34 smithi029 ceph-mon[93650]: osdmap e1084: 8 total, 8 up, 8 in 2024-04-12T11:31:34.996 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:31:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:31:34] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-12T11:31:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:34 smithi087 ceph-mon[89014]: pgmap v1264: 89 pgs: 89 active+clean; 206 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 762 KiB/s wr, 9 op/s 2024-04-12T11:31:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:34 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2002448478' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:31:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:34 smithi087 ceph-mon[89014]: osdmap e1084: 8 total, 8 up, 8 in 2024-04-12T11:31:36.547 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:31:37.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:36 smithi029 ceph-mon[97787]: pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 799 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:31:37.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:37.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:36 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2080413450' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T11:31:37.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:36 smithi029 ceph-mon[93650]: pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 799 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:31:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:36 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2080413450' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T11:31:37.195 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":37,"flags":0,"active_gid":25202,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":3831156113},{"type":"v1","addr":"172.21.15.29:6801","nonce":3831156113}]},"active_addr":"172.21.15.29:6801/3831156113","active_change":"2024-04-12T11:17:21.160473+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":34101,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":248,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3766170058}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":692633529}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3454151607}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":2699724311}]}]} 2024-04-12T11:31:37.199 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-12T11:31:37.199 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-12T11:31:37.199 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T11:31:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:36 smithi087 ceph-mon[89014]: pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 799 MiB used, 714 GiB / 715 GiB avail 2024-04-12T11:31:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:36 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2080413450' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T11:31:37.415 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:38.904 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:38 smithi029 ceph-mon[97787]: pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 627 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:38.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:38 smithi029 ceph-mon[93650]: pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 627 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:38 smithi087 ceph-mon[89014]: pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 627 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:39.395 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:31:39.396 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":1084,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T11:31:33.801968+0000","last_up_change":"2024-04-12T11:23:35.611026+0000","last_in_change":"2024-04-12T11:11:02.730106+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T11:11:48.103219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T11:12:56.493343+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T11:12:57.001342+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":503,"up_from":514,"up_thru":611,"down_at":504,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6803","nonce":3899727946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6805","nonce":3899727946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6809","nonce":3899727946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6807","nonce":3899727946}]},"public_addr":"172.21.15.29:6803/3899727946","cluster_addr":"172.21.15.29:6805/3899727946","heartbeat_back_addr":"172.21.15.29:6809/3899727946","heartbeat_front_addr":"172.21.15.29:6807/3899727946","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":523,"up_from":534,"up_thru":611,"down_at":524,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6811","nonce":840649505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6813","nonce":840649505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6817","nonce":840649505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6815","nonce":840649505}]},"public_addr":"172.21.15.29:6811/840649505","cluster_addr":"172.21.15.29:6813/840649505","heartbeat_back_addr":"172.21.15.29:6817/840649505","heartbeat_front_addr":"172.21.15.29:6815/840649505","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":544,"up_from":555,"up_thru":611,"down_at":545,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6819","nonce":840062687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6821","nonce":840062687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6825","nonce":840062687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6823","nonce":840062687}]},"public_addr":"172.21.15.29:6819/840062687","cluster_addr":"172.21.15.29:6821/840062687","heartbeat_back_addr":"172.21.15.29:6825/840062687","heartbeat_front_addr":"172.21.15.29:6823/840062687","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":579,"up_from":590,"up_thru":611,"down_at":580,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6827","nonce":2609709308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6829","nonce":2609709308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6833","nonce":2609709308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6831","nonce":2609709308}]},"public_addr":"172.21.15.29:6827/2609709308","cluster_addr":"172.21.15.29:6829/2609709308","heartbeat_back_addr":"172.21.15.29:6833/2609709308","heartbeat_front_addr":"172.21.15.29:6831/2609709308","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":600,"up_from":611,"up_thru":611,"down_at":601,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6801","nonce":3230425210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6803","nonce":3230425210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6807","nonce":3230425210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6805","nonce":3230425210}]},"public_addr":"172.21.15.87:6801/3230425210","cluster_addr":"172.21.15.87:6803/3230425210","heartbeat_back_addr":"172.21.15.87:6807/3230425210","heartbeat_front_addr":"172.21.15.87:6805/3230425210","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":611,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6809","nonce":3642701185}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6811","nonce":3642701185}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6815","nonce":3642701185}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6813","nonce":3642701185}]},"public_addr":"172.21.15.87:6809/3642701185","cluster_addr":"172.21.15.87:6811/3642701185","heartbeat_back_addr":"172.21.15.87:6815/3642701185","heartbeat_front_addr":"172.21.15.87:6813/3642701185","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":611,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":611,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":503},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":523},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":544},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":579},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":600},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.9","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.87:0/943640318":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/3128860153":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/2463058069":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2946468060":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/600676740":"2024-04-13T11:17:21.160240+0000","172.21.15.29:6800/686523287":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2241771384":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/3254157714":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/659258741":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.87:6834/1005928369":"2024-04-13T11:16:36.316651+0000","172.21.15.29:6801/3931254419":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/4202867789":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.87:0/1543620546":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000","172.21.15.29:6800/3931254419":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:6801/686523287":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/1221327863":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/1314515623":"2024-04-13T11:17:21.160240+0000","172.21.15.87:6835/1005928369":"2024-04-13T11:16:36.316651+0000","172.21.15.87:0/220307911":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/4218166463":"2024-04-13T11:17:21.160240+0000","172.21.15.87:0/2414938593":"2024-04-13T11:16:36.316651+0000","172.21.15.29:6801/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6800/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.87:0/3681614798":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/364858296":"2024-04-13T11:11:58.108258+0000","172.21.15.87:0/67786467":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/683850481":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/273190388":"2024-04-13T11:17:21.160240+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T11:31:40.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:39 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2799660500' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:31:40.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:39 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2799660500' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:31:40.096 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-12T11:31:40.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T11:31:40.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:39 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2799660500' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:31:40.306 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:41.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:40 smithi029 ceph-mon[93650]: pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:41.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:40 smithi029 ceph-mon[97787]: pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:40 smithi087 ceph-mon[89014]: pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:42.200 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:31:42.201 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":1084,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T11:31:33.801968+0000","last_up_change":"2024-04-12T11:23:35.611026+0000","last_in_change":"2024-04-12T11:11:02.730106+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T11:11:48.103219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T11:12:56.493343+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T11:12:57.001342+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":503,"up_from":514,"up_thru":611,"down_at":504,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6803","nonce":3899727946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6805","nonce":3899727946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6809","nonce":3899727946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6807","nonce":3899727946}]},"public_addr":"172.21.15.29:6803/3899727946","cluster_addr":"172.21.15.29:6805/3899727946","heartbeat_back_addr":"172.21.15.29:6809/3899727946","heartbeat_front_addr":"172.21.15.29:6807/3899727946","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":523,"up_from":534,"up_thru":611,"down_at":524,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6811","nonce":840649505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6813","nonce":840649505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6817","nonce":840649505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6815","nonce":840649505}]},"public_addr":"172.21.15.29:6811/840649505","cluster_addr":"172.21.15.29:6813/840649505","heartbeat_back_addr":"172.21.15.29:6817/840649505","heartbeat_front_addr":"172.21.15.29:6815/840649505","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":544,"up_from":555,"up_thru":611,"down_at":545,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6819","nonce":840062687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6821","nonce":840062687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6825","nonce":840062687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6823","nonce":840062687}]},"public_addr":"172.21.15.29:6819/840062687","cluster_addr":"172.21.15.29:6821/840062687","heartbeat_back_addr":"172.21.15.29:6825/840062687","heartbeat_front_addr":"172.21.15.29:6823/840062687","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":579,"up_from":590,"up_thru":611,"down_at":580,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6827","nonce":2609709308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6829","nonce":2609709308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6833","nonce":2609709308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2609709308},{"type":"v1","addr":"172.21.15.29:6831","nonce":2609709308}]},"public_addr":"172.21.15.29:6827/2609709308","cluster_addr":"172.21.15.29:6829/2609709308","heartbeat_back_addr":"172.21.15.29:6833/2609709308","heartbeat_front_addr":"172.21.15.29:6831/2609709308","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":600,"up_from":611,"up_thru":611,"down_at":601,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6801","nonce":3230425210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6803","nonce":3230425210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6807","nonce":3230425210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6805","nonce":3230425210}]},"public_addr":"172.21.15.87:6801/3230425210","cluster_addr":"172.21.15.87:6803/3230425210","heartbeat_back_addr":"172.21.15.87:6807/3230425210","heartbeat_front_addr":"172.21.15.87:6805/3230425210","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":611,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6809","nonce":3642701185}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6811","nonce":3642701185}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6815","nonce":3642701185}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3642701185},{"type":"v1","addr":"172.21.15.87:6813","nonce":3642701185}]},"public_addr":"172.21.15.87:6809/3642701185","cluster_addr":"172.21.15.87:6811/3642701185","heartbeat_back_addr":"172.21.15.87:6815/3642701185","heartbeat_front_addr":"172.21.15.87:6813/3642701185","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":611,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":611,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":503},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":523},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":544},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":579},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":600},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.9","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/273190388":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/683850481":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/686523287":"2024-04-13T11:17:21.160240+0000","172.21.15.87:6835/1005928369":"2024-04-13T11:16:36.316651+0000","172.21.15.87:0/220307911":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/659258741":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/2463058069":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.87:6834/1005928369":"2024-04-13T11:16:36.316651+0000","172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/3254157714":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/3931254419":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/4202867789":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.87:0/1543620546":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000","172.21.15.29:6800/3931254419":"2024-04-13T11:16:01.391032+0000","172.21.15.87:0/943640318":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/1314515623":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/1221327863":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:6800/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.87:0/2414938593":"2024-04-13T11:16:36.316651+0000","172.21.15.29:6801/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/4218166463":"2024-04-13T11:17:21.160240+0000","172.21.15.87:0/3681614798":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/2946468060":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/364858296":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.87:0/67786467":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/3128860153":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/2241771384":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/600676740":"2024-04-13T11:17:21.160240+0000","172.21.15.29:6800/686523287":"2024-04-13T11:17:21.160240+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T11:31:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:31:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:31:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:31:42.828 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-12T11:31:42.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-12T11:31:42.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-12T11:31:42.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-12T11:31:42.830 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-12T11:31:42.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-12T11:31:42.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-12T11:31:42.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-12T11:31:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:42 smithi087 ceph-mon[89014]: pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:42 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3864588295' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:31:43.240 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:42 smithi029 ceph-mon[93650]: pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:43.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:42 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3864588295' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:31:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:42 smithi029 ceph-mon[97787]: pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:42 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3864588295' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T11:31:43.363 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:43.602 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:43.602 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:43.603 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:43.603 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:43.613 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:43.614 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:31:45.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:44 smithi029 ceph-mon[93650]: pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:44 smithi029 ceph-mon[97787]: pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:45.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:31:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:31:44] "GET /metrics HTTP/1.1" 200 34008 "" "Prometheus/2.43.0" 2024-04-12T11:31:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:44 smithi087 ceph-mon[89014]: pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:46 smithi029 ceph-mon[93650]: pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:46 smithi029 ceph-mon[97787]: pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:46 smithi087 ceph-mon[89014]: pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:48.958 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:48 smithi029 ceph-mon[93650]: pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:48.958 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:48 smithi029 ceph-mon[97787]: pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:48 smithi087 ceph-mon[89014]: pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:50 smithi087 ceph-mon[89014]: pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:51.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:50 smithi029 ceph-mon[93650]: pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:50 smithi029 ceph-mon[97787]: pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:51.423 INFO:teuthology.orchestra.run.smithi029.stdout:2207613190264 2024-04-12T11:31:51.424 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-12T11:31:52.118 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:52.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:52.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:31:52.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:31:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:31:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:31:53.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:52 smithi029 ceph-mon[93650]: pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:53.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:52 smithi029 ceph-mon[97787]: pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:52 smithi087 ceph-mon[89014]: pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:55.154 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:54 smithi029 ceph-mon[97787]: pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:55.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:54 smithi029 ceph-mon[93650]: pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:55.155 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:31:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:31:54] "GET /metrics HTTP/1.1" 200 34002 "" "Prometheus/2.43.0" 2024-04-12T11:31:55.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:54 smithi087 ceph-mon[89014]: pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:57.052 INFO:teuthology.orchestra.run.smithi029.stdout:231928234233 2024-04-12T11:31:57.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-12T11:31:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:56 smithi087 ceph-mon[89014]: pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:57.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:56 smithi029 ceph-mon[93650]: pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:57.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:56 smithi029 ceph-mon[97787]: pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:57.700 INFO:teuthology.orchestra.run.smithi029.stdout:2534030704746 2024-04-12T11:31:57.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-12T11:31:58.786 INFO:teuthology.orchestra.run.smithi029.stdout:197568495871 2024-04-12T11:31:58.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-12T11:31:59.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:31:58 smithi029 ceph-mon[93650]: pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:31:58 smithi029 ceph-mon[97787]: pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:59.137 INFO:teuthology.orchestra.run.smithi029.stdout:2383706849393 2024-04-12T11:31:59.138 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-12T11:31:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:31:58 smithi087 ceph-mon[89014]: pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:31:59.485 INFO:teuthology.orchestra.run.smithi029.stdout:163208757506 2024-04-12T11:31:59.485 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-12T11:32:00.224 INFO:teuthology.orchestra.run.smithi029.stdout:2624225017958 2024-04-12T11:32:00.224 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-12T11:32:00.815 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:00 smithi029 ceph-mon[93650]: pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:01.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:00 smithi029 ceph-mon[97787]: pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:01.104 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:01.105 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:01.105 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:01.106 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:01.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:00 smithi087 ceph-mon[89014]: pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:32:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:32:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:32:02.972 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:02 smithi029 ceph-mon[93650]: pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:02 smithi029 ceph-mon[97787]: pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:02 smithi087 ceph-mon[89014]: pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:05.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:04 smithi029 ceph-mon[93650]: pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:05.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:04 smithi029 ceph-mon[97787]: pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:05.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:32:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:32:04] "GET /metrics HTTP/1.1" 200 34002 "" "Prometheus/2.43.0" 2024-04-12T11:32:05.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:04 smithi087 ceph-mon[89014]: pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:05.924 INFO:teuthology.orchestra.run.smithi029.stdout:2293512536182 2024-04-12T11:32:05.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-12T11:32:06.581 INFO:teuthology.orchestra.run.smithi029.stdout:2207613190267 2024-04-12T11:32:06.789 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:07.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:06 smithi029 ceph-mon[97787]: pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:07.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:07.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1208884950' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T11:32:07.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:06 smithi029 ceph-mon[93650]: pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:07.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:07.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1208884950' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T11:32:07.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:06 smithi087 ceph-mon[89014]: pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1208884950' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T11:32:08.494 INFO:teuthology.orchestra.run.smithi029.stdout:231928234237 2024-04-12T11:32:09.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:08 smithi087 ceph-mon[89014]: pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:09.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:08 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1726905078' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T11:32:09.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:08 smithi029 ceph-mon[93650]: pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:09.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:08 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1726905078' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T11:32:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:08 smithi029 ceph-mon[97787]: pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:08 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1726905078' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T11:32:09.321 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234233 got 231928234237 for osd.7 2024-04-12T11:32:09.322 DEBUG:teuthology.parallel:result is None 2024-04-12T11:32:09.748 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2207613190264 got 2207613190267 for osd.0 2024-04-12T11:32:09.749 DEBUG:teuthology.parallel:result is None 2024-04-12T11:32:11.153 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:10 smithi029 ceph-mon[97787]: pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:11.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:10 smithi029 ceph-mon[93650]: pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:11.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:10 smithi087 ceph-mon[89014]: pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:11.481 INFO:teuthology.orchestra.run.smithi029.stdout:2383706849396 2024-04-12T11:32:11.683 INFO:teuthology.orchestra.run.smithi029.stdout:2534030704749 2024-04-12T11:32:12.010 INFO:teuthology.orchestra.run.smithi029.stdout:2624225017961 2024-04-12T11:32:12.038 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:11 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1386847758' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T11:32:12.039 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:11 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2458476378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T11:32:12.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:11 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1386847758' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T11:32:12.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:11 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2458476378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T11:32:12.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:11 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1386847758' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T11:32:12.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:11 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2458476378' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T11:32:12.524 INFO:teuthology.orchestra.run.smithi029.stdout:197568495874 2024-04-12T11:32:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:32:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:32:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:32:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:12 smithi029 ceph-mon[93650]: pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:12 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2379461210' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T11:32:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:12 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/166415482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T11:32:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:12 smithi029 ceph-mon[97787]: pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:12 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2379461210' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T11:32:13.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:12 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/166415482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T11:32:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:12 smithi087 ceph-mon[89014]: pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:13.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:12 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2379461210' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T11:32:13.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:12 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/166415482' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T11:32:13.258 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2383706849393 got 2383706849396 for osd.2 2024-04-12T11:32:13.258 DEBUG:teuthology.parallel:result is None 2024-04-12T11:32:13.654 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:14.515 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2624225017958 got 2624225017961 for osd.4 2024-04-12T11:32:14.515 DEBUG:teuthology.parallel:result is None 2024-04-12T11:32:14.956 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2534030704746 got 2534030704749 for osd.3 2024-04-12T11:32:14.956 DEBUG:teuthology.parallel:result is None 2024-04-12T11:32:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:14 smithi087 ceph-mon[89014]: pgmap v1285: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:32:15.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:14 smithi029 ceph-mon[97787]: pgmap v1285: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:15.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:32:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:14 smithi029 ceph-mon[93650]: pgmap v1285: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:32:15.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:32:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:32:14] "GET /metrics HTTP/1.1" 200 34007 "" "Prometheus/2.43.0" 2024-04-12T11:32:16.142 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495871 got 197568495874 for osd.6 2024-04-12T11:32:16.142 DEBUG:teuthology.parallel:result is None 2024-04-12T11:32:17.015 INFO:teuthology.orchestra.run.smithi029.stdout:163208757511 2024-04-12T11:32:17.190 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:16 smithi087 ceph-mon[89014]: pgmap v1286: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:17.207 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:16 smithi029 ceph-mon[97787]: pgmap v1286: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:17.208 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:16 smithi029 ceph-mon[93650]: pgmap v1286: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:17.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:17 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2333605551' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T11:32:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:17 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2333605551' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T11:32:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:17 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2333605551' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T11:32:18.178 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757506 got 163208757511 for osd.5 2024-04-12T11:32:18.178 DEBUG:teuthology.parallel:result is None 2024-04-12T11:32:18.850 INFO:teuthology.orchestra.run.smithi029.stdout:2293512536185 2024-04-12T11:32:19.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:18 smithi029 ceph-mon[97787]: pgmap v1287: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:19.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:18 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:19.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:18 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:19.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:18 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3588043650' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T11:32:19.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:18 smithi029 ceph-mon[93650]: pgmap v1287: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:19.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:18 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:19.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:18 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:19.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:18 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3588043650' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T11:32:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:18 smithi087 ceph-mon[89014]: pgmap v1287: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:18 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:18 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:18 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3588043650' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T11:32:19.893 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2293512536182 got 2293512536185 for osd.1 2024-04-12T11:32:19.894 DEBUG:teuthology.parallel:result is None 2024-04-12T11:32:19.894 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-12T11:32:19.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:32:20.134 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:20 smithi087 ceph-mon[89014]: pgmap v1288: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:21.221 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:20 smithi029 ceph-mon[97787]: pgmap v1288: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:21.221 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:20 smithi029 ceph-mon[93650]: pgmap v1288: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:22.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:22.652 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:32:22.653 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:32:22.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:32:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:32:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:32:23.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:22 smithi029 ceph-mon[93650]: pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:22 smithi029 ceph-mon[97787]: pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:22 smithi087 ceph-mon[89014]: pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:23.383 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1289,"stamp":"2024-04-12T11:32:21.638231+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":472476,"kb_used_data":68076,"kb_used_omap":175,"kb_used_meta":404112,"kb_avail":749488740,"statfs":{"total":767960285184,"available":767476469760,"internally_reserved":0,"allocated":69709824,"data_stored":47712258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":413810857},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003319"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807039+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:31:33.807039+0000","last_peered":"2024-04-12T11:31:33.807039+0000","last_clean":"2024-04-12T11:31:33.807039+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:31:33.807039+0000","last_undegraded":"2024-04-12T11:31:33.807039+0000","last_fullsized":"2024-04-12T11:31:33.807039+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1525,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805863+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:31:33.805863+0000","last_peered":"2024-04-12T11:31:33.805863+0000","last_clean":"2024-04-12T11:31:33.805863+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:31:33.805863+0000","last_undegraded":"2024-04-12T11:31:33.805863+0000","last_fullsized":"2024-04-12T11:31:33.805863+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307246+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:31:34.307246+0000","last_peered":"2024-04-12T11:31:34.307246+0000","last_clean":"2024-04-12T11:31:34.307246+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:31:34.307246+0000","last_undegraded":"2024-04-12T11:31:34.307246+0000","last_fullsized":"2024-04-12T11:31:34.307246+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1572,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329931+0000","last_change":"2024-04-12T11:23:15.593723+0000","last_active":"2024-04-12T11:31:34.329931+0000","last_peered":"2024-04-12T11:31:34.329931+0000","last_clean":"2024-04-12T11:31:34.329931+0000","last_became_active":"2024-04-12T11:23:15.593381+0000","last_became_peered":"2024-04-12T11:23:15.593381+0000","last_unstale":"2024-04-12T11:31:34.329931+0000","last_undegraded":"2024-04-12T11:31:34.329931+0000","last_fullsized":"2024-04-12T11:31:34.329931+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:18:06.971493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1533,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804979+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:31:33.804979+0000","last_peered":"2024-04-12T11:31:33.804979+0000","last_clean":"2024-04-12T11:31:33.804979+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:31:33.804979+0000","last_undegraded":"2024-04-12T11:31:33.804979+0000","last_fullsized":"2024-04-12T11:31:33.804979+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.963036+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:31:34.963036+0000","last_peered":"2024-04-12T11:31:34.963036+0000","last_clean":"2024-04-12T11:31:34.963036+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:31:34.963036+0000","last_undegraded":"2024-04-12T11:31:34.963036+0000","last_fullsized":"2024-04-12T11:31:34.963036+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1568,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330398+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:31:34.330398+0000","last_peered":"2024-04-12T11:31:34.330398+0000","last_clean":"2024-04-12T11:31:34.330398+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:31:34.330398+0000","last_undegraded":"2024-04-12T11:31:34.330398+0000","last_fullsized":"2024-04-12T11:31:34.330398+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1506,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804876+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:31:33.804876+0000","last_peered":"2024-04-12T11:31:33.804876+0000","last_clean":"2024-04-12T11:31:33.804876+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:31:33.804876+0000","last_undegraded":"2024-04-12T11:31:33.804876+0000","last_fullsized":"2024-04-12T11:31:33.804876+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1031,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806752+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:31:33.806752+0000","last_peered":"2024-04-12T11:31:33.806752+0000","last_clean":"2024-04-12T11:31:33.806752+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:31:33.806752+0000","last_undegraded":"2024-04-12T11:31:33.806752+0000","last_fullsized":"2024-04-12T11:31:33.806752+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1486,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804993+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:31:33.804993+0000","last_peered":"2024-04-12T11:31:33.804993+0000","last_clean":"2024-04-12T11:31:33.804993+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:31:33.804993+0000","last_undegraded":"2024-04-12T11:31:33.804993+0000","last_fullsized":"2024-04-12T11:31:33.804993+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1040,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.963138+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:31:34.963138+0000","last_peered":"2024-04-12T11:31:34.963138+0000","last_clean":"2024-04-12T11:31:34.963138+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:31:34.963138+0000","last_undegraded":"2024-04-12T11:31:34.963138+0000","last_fullsized":"2024-04-12T11:31:34.963138+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1568,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307302+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:31:34.307302+0000","last_peered":"2024-04-12T11:31:34.307302+0000","last_clean":"2024-04-12T11:31:34.307302+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:31:34.307302+0000","last_undegraded":"2024-04-12T11:31:34.307302+0000","last_fullsized":"2024-04-12T11:31:34.307302+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1489,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806028+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:31:33.806028+0000","last_peered":"2024-04-12T11:31:33.806028+0000","last_clean":"2024-04-12T11:31:33.806028+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:31:33.806028+0000","last_undegraded":"2024-04-12T11:31:33.806028+0000","last_fullsized":"2024-04-12T11:31:33.806028+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806164+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:31:33.806164+0000","last_peered":"2024-04-12T11:31:33.806164+0000","last_clean":"2024-04-12T11:31:33.806164+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:31:33.806164+0000","last_undegraded":"2024-04-12T11:31:33.806164+0000","last_fullsized":"2024-04-12T11:31:33.806164+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1039,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962053+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:31:34.962053+0000","last_peered":"2024-04-12T11:31:34.962053+0000","last_clean":"2024-04-12T11:31:34.962053+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:31:34.962053+0000","last_undegraded":"2024-04-12T11:31:34.962053+0000","last_fullsized":"2024-04-12T11:31:34.962053+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805796+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:31:33.805796+0000","last_peered":"2024-04-12T11:31:33.805796+0000","last_clean":"2024-04-12T11:31:33.805796+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:31:33.805796+0000","last_undegraded":"2024-04-12T11:31:33.805796+0000","last_fullsized":"2024-04-12T11:31:33.805796+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329105+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:31:34.329105+0000","last_peered":"2024-04-12T11:31:34.329105+0000","last_clean":"2024-04-12T11:31:34.329105+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:31:34.329105+0000","last_undegraded":"2024-04-12T11:31:34.329105+0000","last_fullsized":"2024-04-12T11:31:34.329105+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806888+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:31:33.806888+0000","last_peered":"2024-04-12T11:31:33.806888+0000","last_clean":"2024-04-12T11:31:33.806888+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:31:33.806888+0000","last_undegraded":"2024-04-12T11:31:33.806888+0000","last_fullsized":"2024-04-12T11:31:33.806888+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962176+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:31:34.962176+0000","last_peered":"2024-04-12T11:31:34.962176+0000","last_clean":"2024-04-12T11:31:34.962176+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:31:34.962176+0000","last_undegraded":"2024-04-12T11:31:34.962176+0000","last_fullsized":"2024-04-12T11:31:34.962176+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1570,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329830+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:31:34.329830+0000","last_peered":"2024-04-12T11:31:34.329830+0000","last_clean":"2024-04-12T11:31:34.329830+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:31:34.329830+0000","last_undegraded":"2024-04-12T11:31:34.329830+0000","last_fullsized":"2024-04-12T11:31:34.329830+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1033,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961852+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:31:34.961852+0000","last_peered":"2024-04-12T11:31:34.961852+0000","last_clean":"2024-04-12T11:31:34.961852+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:31:34.961852+0000","last_undegraded":"2024-04-12T11:31:34.961852+0000","last_fullsized":"2024-04-12T11:31:34.961852+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805881+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:31:33.805881+0000","last_peered":"2024-04-12T11:31:33.805881+0000","last_clean":"2024-04-12T11:31:33.805881+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:31:33.805881+0000","last_undegraded":"2024-04-12T11:31:33.805881+0000","last_fullsized":"2024-04-12T11:31:33.805881+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027548+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:31:33.027548+0000","last_peered":"2024-04-12T11:31:33.027548+0000","last_clean":"2024-04-12T11:31:33.027548+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:31:33.027548+0000","last_undegraded":"2024-04-12T11:31:33.027548+0000","last_fullsized":"2024-04-12T11:31:33.027548+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1015,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488384+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:31:33.488384+0000","last_peered":"2024-04-12T11:31:33.488384+0000","last_clean":"2024-04-12T11:31:33.488384+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:31:33.488384+0000","last_undegraded":"2024-04-12T11:31:33.488384+0000","last_fullsized":"2024-04-12T11:31:33.488384+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027317+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:31:33.027317+0000","last_peered":"2024-04-12T11:31:33.027317+0000","last_clean":"2024-04-12T11:31:33.027317+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:31:33.027317+0000","last_undegraded":"2024-04-12T11:31:33.027317+0000","last_fullsized":"2024-04-12T11:31:33.027317+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1503,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806074+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:31:33.806074+0000","last_peered":"2024-04-12T11:31:33.806074+0000","last_clean":"2024-04-12T11:31:33.806074+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:31:33.806074+0000","last_undegraded":"2024-04-12T11:31:33.806074+0000","last_fullsized":"2024-04-12T11:31:33.806074+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027231+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:31:33.027231+0000","last_peered":"2024-04-12T11:31:33.027231+0000","last_clean":"2024-04-12T11:31:33.027231+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:31:33.027231+0000","last_undegraded":"2024-04-12T11:31:33.027231+0000","last_fullsized":"2024-04-12T11:31:33.027231+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.098131+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:31:33.098131+0000","last_peered":"2024-04-12T11:31:33.098131+0000","last_clean":"2024-04-12T11:31:33.098131+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:31:33.098131+0000","last_undegraded":"2024-04-12T11:31:33.098131+0000","last_fullsized":"2024-04-12T11:31:33.098131+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488787+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:31:33.488787+0000","last_peered":"2024-04-12T11:31:33.488787+0000","last_clean":"2024-04-12T11:31:33.488787+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:31:33.488787+0000","last_undegraded":"2024-04-12T11:31:33.488787+0000","last_fullsized":"2024-04-12T11:31:33.488787+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1579,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329258+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:31:34.329258+0000","last_peered":"2024-04-12T11:31:34.329258+0000","last_clean":"2024-04-12T11:31:34.329258+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:31:34.329258+0000","last_undegraded":"2024-04-12T11:31:34.329258+0000","last_fullsized":"2024-04-12T11:31:34.329258+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1511,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805669+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:31:33.805669+0000","last_peered":"2024-04-12T11:31:33.805669+0000","last_clean":"2024-04-12T11:31:33.805669+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:31:33.805669+0000","last_undegraded":"2024-04-12T11:31:33.805669+0000","last_fullsized":"2024-04-12T11:31:33.805669+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1493,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806698+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:31:33.806698+0000","last_peered":"2024-04-12T11:31:33.806698+0000","last_clean":"2024-04-12T11:31:33.806698+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:31:33.806698+0000","last_undegraded":"2024-04-12T11:31:33.806698+0000","last_fullsized":"2024-04-12T11:31:33.806698+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.098030+0000","last_change":"2024-04-12T11:23:36.639270+0000","last_active":"2024-04-12T11:31:33.098030+0000","last_peered":"2024-04-12T11:31:33.098030+0000","last_clean":"2024-04-12T11:31:33.098030+0000","last_became_active":"2024-04-12T11:23:36.638952+0000","last_became_peered":"2024-04-12T11:23:36.638952+0000","last_unstale":"2024-04-12T11:31:33.098030+0000","last_undegraded":"2024-04-12T11:31:33.098030+0000","last_fullsized":"2024-04-12T11:31:33.098030+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:04:50.403812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1515,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.809196+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:31:33.809196+0000","last_peered":"2024-04-12T11:31:33.809196+0000","last_clean":"2024-04-12T11:31:33.809196+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:31:33.809196+0000","last_undegraded":"2024-04-12T11:31:33.809196+0000","last_fullsized":"2024-04-12T11:31:33.809196+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806193+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:31:33.806193+0000","last_peered":"2024-04-12T11:31:33.806193+0000","last_clean":"2024-04-12T11:31:33.806193+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:31:33.806193+0000","last_undegraded":"2024-04-12T11:31:33.806193+0000","last_fullsized":"2024-04-12T11:31:33.806193+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807330+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:31:33.807330+0000","last_peered":"2024-04-12T11:31:33.807330+0000","last_clean":"2024-04-12T11:31:33.807330+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:31:33.807330+0000","last_undegraded":"2024-04-12T11:31:33.807330+0000","last_fullsized":"2024-04-12T11:31:33.807330+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1519,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804798+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:31:33.804798+0000","last_peered":"2024-04-12T11:31:33.804798+0000","last_clean":"2024-04-12T11:31:33.804798+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:31:33.804798+0000","last_undegraded":"2024-04-12T11:31:33.804798+0000","last_fullsized":"2024-04-12T11:31:33.804798+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307677+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:31:34.307677+0000","last_peered":"2024-04-12T11:31:34.307677+0000","last_clean":"2024-04-12T11:31:34.307677+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:31:34.307677+0000","last_undegraded":"2024-04-12T11:31:34.307677+0000","last_fullsized":"2024-04-12T11:31:34.307677+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1578,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.308030+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:31:34.308030+0000","last_peered":"2024-04-12T11:31:34.308030+0000","last_clean":"2024-04-12T11:31:34.308030+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:31:34.308030+0000","last_undegraded":"2024-04-12T11:31:34.308030+0000","last_fullsized":"2024-04-12T11:31:34.308030+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1047,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807269+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:31:33.807269+0000","last_peered":"2024-04-12T11:31:33.807269+0000","last_clean":"2024-04-12T11:31:33.807269+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:31:33.807269+0000","last_undegraded":"2024-04-12T11:31:33.807269+0000","last_fullsized":"2024-04-12T11:31:33.807269+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307257+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:31:34.307257+0000","last_peered":"2024-04-12T11:31:34.307257+0000","last_clean":"2024-04-12T11:31:34.307257+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:31:34.307257+0000","last_undegraded":"2024-04-12T11:31:34.307257+0000","last_fullsized":"2024-04-12T11:31:34.307257+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488693+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:31:33.488693+0000","last_peered":"2024-04-12T11:31:33.488693+0000","last_clean":"2024-04-12T11:31:33.488693+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:31:33.488693+0000","last_undegraded":"2024-04-12T11:31:33.488693+0000","last_fullsized":"2024-04-12T11:31:33.488693+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806006+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:31:33.806006+0000","last_peered":"2024-04-12T11:31:33.806006+0000","last_clean":"2024-04-12T11:31:33.806006+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:31:33.806006+0000","last_undegraded":"2024-04-12T11:31:33.806006+0000","last_fullsized":"2024-04-12T11:31:33.806006+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1578,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307348+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:31:34.307348+0000","last_peered":"2024-04-12T11:31:34.307348+0000","last_clean":"2024-04-12T11:31:34.307348+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:31:34.307348+0000","last_undegraded":"2024-04-12T11:31:34.307348+0000","last_fullsized":"2024-04-12T11:31:34.307348+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1542,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961088+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:31:34.961088+0000","last_peered":"2024-04-12T11:31:34.961088+0000","last_clean":"2024-04-12T11:31:34.961088+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:31:34.961088+0000","last_undegraded":"2024-04-12T11:31:34.961088+0000","last_fullsized":"2024-04-12T11:31:34.961088+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1570,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307824+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:31:34.307824+0000","last_peered":"2024-04-12T11:31:34.307824+0000","last_clean":"2024-04-12T11:31:34.307824+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:31:34.307824+0000","last_undegraded":"2024-04-12T11:31:34.307824+0000","last_fullsized":"2024-04-12T11:31:34.307824+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1572,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307812+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:31:34.307812+0000","last_peered":"2024-04-12T11:31:34.307812+0000","last_clean":"2024-04-12T11:31:34.307812+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:31:34.307812+0000","last_undegraded":"2024-04-12T11:31:34.307812+0000","last_fullsized":"2024-04-12T11:31:34.307812+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488902+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:31:33.488902+0000","last_peered":"2024-04-12T11:31:33.488902+0000","last_clean":"2024-04-12T11:31:33.488902+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:31:33.488902+0000","last_undegraded":"2024-04-12T11:31:33.488902+0000","last_fullsized":"2024-04-12T11:31:33.488902+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1391,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.026548+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:31:33.026548+0000","last_peered":"2024-04-12T11:31:33.026548+0000","last_clean":"2024-04-12T11:31:33.026548+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:31:33.026548+0000","last_undegraded":"2024-04-12T11:31:33.026548+0000","last_fullsized":"2024-04-12T11:31:33.026548+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1486,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804959+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:31:33.804959+0000","last_peered":"2024-04-12T11:31:33.804959+0000","last_clean":"2024-04-12T11:31:33.804959+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:31:33.804959+0000","last_undegraded":"2024-04-12T11:31:33.804959+0000","last_fullsized":"2024-04-12T11:31:33.804959+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962347+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:31:34.962347+0000","last_peered":"2024-04-12T11:31:34.962347+0000","last_clean":"2024-04-12T11:31:34.962347+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:31:34.962347+0000","last_undegraded":"2024-04-12T11:31:34.962347+0000","last_fullsized":"2024-04-12T11:31:34.962347+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1044,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961718+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:31:34.961718+0000","last_peered":"2024-04-12T11:31:34.961718+0000","last_clean":"2024-04-12T11:31:34.961718+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:31:34.961718+0000","last_undegraded":"2024-04-12T11:31:34.961718+0000","last_fullsized":"2024-04-12T11:31:34.961718+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1579,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329043+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:31:34.329043+0000","last_peered":"2024-04-12T11:31:34.329043+0000","last_clean":"2024-04-12T11:31:34.329043+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:31:34.329043+0000","last_undegraded":"2024-04-12T11:31:34.329043+0000","last_fullsized":"2024-04-12T11:31:34.329043+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1579,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329333+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:31:34.329333+0000","last_peered":"2024-04-12T11:31:34.329333+0000","last_clean":"2024-04-12T11:31:34.329333+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:31:34.329333+0000","last_undegraded":"2024-04-12T11:31:34.329333+0000","last_fullsized":"2024-04-12T11:31:34.329333+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1043,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027168+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:31:33.027168+0000","last_peered":"2024-04-12T11:31:33.027168+0000","last_clean":"2024-04-12T11:31:33.027168+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:31:33.027168+0000","last_undegraded":"2024-04-12T11:31:33.027168+0000","last_fullsized":"2024-04-12T11:31:33.027168+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307821+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:31:34.307821+0000","last_peered":"2024-04-12T11:31:34.307821+0000","last_clean":"2024-04-12T11:31:34.307821+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:31:34.307821+0000","last_undegraded":"2024-04-12T11:31:34.307821+0000","last_fullsized":"2024-04-12T11:31:34.307821+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1562,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330351+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:31:34.330351+0000","last_peered":"2024-04-12T11:31:34.330351+0000","last_clean":"2024-04-12T11:31:34.330351+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:31:34.330351+0000","last_undegraded":"2024-04-12T11:31:34.330351+0000","last_fullsized":"2024-04-12T11:31:34.330351+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806115+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:31:33.806115+0000","last_peered":"2024-04-12T11:31:33.806115+0000","last_clean":"2024-04-12T11:31:33.806115+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:31:33.806115+0000","last_undegraded":"2024-04-12T11:31:33.806115+0000","last_fullsized":"2024-04-12T11:31:33.806115+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307609+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:31:34.307609+0000","last_peered":"2024-04-12T11:31:34.307609+0000","last_clean":"2024-04-12T11:31:34.307609+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:31:34.307609+0000","last_undegraded":"2024-04-12T11:31:34.307609+0000","last_fullsized":"2024-04-12T11:31:34.307609+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1499,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804727+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:31:33.804727+0000","last_peered":"2024-04-12T11:31:33.804727+0000","last_clean":"2024-04-12T11:31:33.804727+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:31:33.804727+0000","last_undegraded":"2024-04-12T11:31:33.804727+0000","last_fullsized":"2024-04-12T11:31:33.804727+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.098490+0000","last_change":"2024-04-12T11:22:40.580200+0000","last_active":"2024-04-12T11:31:33.098490+0000","last_peered":"2024-04-12T11:31:33.098490+0000","last_clean":"2024-04-12T11:31:33.098490+0000","last_became_active":"2024-04-12T11:22:40.579809+0000","last_became_peered":"2024-04-12T11:22:40.579809+0000","last_unstale":"2024-04-12T11:31:33.098490+0000","last_undegraded":"2024-04-12T11:31:33.098490+0000","last_fullsized":"2024-04-12T11:31:33.098490+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:28:23.317249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1582,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329989+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:31:34.329989+0000","last_peered":"2024-04-12T11:31:34.329989+0000","last_clean":"2024-04-12T11:31:34.329989+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:31:34.329989+0000","last_undegraded":"2024-04-12T11:31:34.329989+0000","last_fullsized":"2024-04-12T11:31:34.329989+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1489,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805172+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:31:33.805172+0000","last_peered":"2024-04-12T11:31:33.805172+0000","last_clean":"2024-04-12T11:31:33.805172+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:31:33.805172+0000","last_undegraded":"2024-04-12T11:31:33.805172+0000","last_fullsized":"2024-04-12T11:31:33.805172+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1573,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330291+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:31:34.330291+0000","last_peered":"2024-04-12T11:31:34.330291+0000","last_clean":"2024-04-12T11:31:34.330291+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:31:34.330291+0000","last_undegraded":"2024-04-12T11:31:34.330291+0000","last_fullsized":"2024-04-12T11:31:34.330291+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1553,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961201+0000","last_change":"2024-04-12T11:23:15.593766+0000","last_active":"2024-04-12T11:31:34.961201+0000","last_peered":"2024-04-12T11:31:34.961201+0000","last_clean":"2024-04-12T11:31:34.961201+0000","last_became_active":"2024-04-12T11:23:15.593428+0000","last_became_peered":"2024-04-12T11:23:15.593428+0000","last_unstale":"2024-04-12T11:31:34.961201+0000","last_undegraded":"2024-04-12T11:31:34.961201+0000","last_fullsized":"2024-04-12T11:31:34.961201+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:12.389667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1047,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488797+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:31:33.488797+0000","last_peered":"2024-04-12T11:31:33.488797+0000","last_clean":"2024-04-12T11:31:33.488797+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:31:33.488797+0000","last_undegraded":"2024-04-12T11:31:33.488797+0000","last_fullsized":"2024-04-12T11:31:33.488797+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806362+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:31:33.806362+0000","last_peered":"2024-04-12T11:31:33.806362+0000","last_clean":"2024-04-12T11:31:33.806362+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:31:33.806362+0000","last_undegraded":"2024-04-12T11:31:33.806362+0000","last_fullsized":"2024-04-12T11:31:33.806362+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1032,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.489264+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:31:33.489264+0000","last_peered":"2024-04-12T11:31:33.489264+0000","last_clean":"2024-04-12T11:31:33.489264+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:31:33.489264+0000","last_undegraded":"2024-04-12T11:31:33.489264+0000","last_fullsized":"2024-04-12T11:31:33.489264+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1508,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805189+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:31:33.805189+0000","last_peered":"2024-04-12T11:31:33.805189+0000","last_clean":"2024-04-12T11:31:33.805189+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:31:33.805189+0000","last_undegraded":"2024-04-12T11:31:33.805189+0000","last_fullsized":"2024-04-12T11:31:33.805189+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1500,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805844+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:31:33.805844+0000","last_peered":"2024-04-12T11:31:33.805844+0000","last_clean":"2024-04-12T11:31:33.805844+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:31:33.805844+0000","last_undegraded":"2024-04-12T11:31:33.805844+0000","last_fullsized":"2024-04-12T11:31:33.805844+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1511,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804851+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:31:33.804851+0000","last_peered":"2024-04-12T11:31:33.804851+0000","last_clean":"2024-04-12T11:31:33.804851+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:31:33.804851+0000","last_undegraded":"2024-04-12T11:31:33.804851+0000","last_fullsized":"2024-04-12T11:31:33.804851+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1576,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329447+0000","last_change":"2024-04-12T11:22:40.582361+0000","last_active":"2024-04-12T11:31:34.329447+0000","last_peered":"2024-04-12T11:31:34.329447+0000","last_clean":"2024-04-12T11:31:34.329447+0000","last_became_active":"2024-04-12T11:22:40.581996+0000","last_became_peered":"2024-04-12T11:22:40.581996+0000","last_unstale":"2024-04-12T11:31:34.329447+0000","last_undegraded":"2024-04-12T11:31:34.329447+0000","last_fullsized":"2024-04-12T11:31:34.329447+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:40:43.563258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805035+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:31:33.805035+0000","last_peered":"2024-04-12T11:31:33.805035+0000","last_clean":"2024-04-12T11:31:33.805035+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:31:33.805035+0000","last_undegraded":"2024-04-12T11:31:33.805035+0000","last_fullsized":"2024-04-12T11:31:33.805035+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234239,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":58052,"kb_used_data":8888,"kb_used_omap":1,"kb_used_meta":49150,"kb_avail":93687100,"statfs":{"total":95995035648,"available":95935590400,"internally_reserved":0,"allocated":9101312,"data_stored":6335894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":50330059},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495876,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":66228,"kb_used_data":8876,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93678924,"statfs":{"total":95995035648,"available":95927218176,"internally_reserved":0,"allocated":9089024,"data_stored":6335748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":58718664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536186,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36068,"kb_used_data":8280,"kb_used_omap":30,"kb_used_meta":27745,"kb_avail":93709084,"statfs":{"total":95995035648,"available":95958102016,"internally_reserved":0,"allocated":8478720,"data_stored":5740566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":28411202},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190270,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67316,"kb_used_data":8868,"kb_used_omap":43,"kb_used_meta":58388,"kb_avail":93677836,"statfs":{"total":95995035648,"available":95926104064,"internally_reserved":0,"allocated":9080832,"data_stored":6331538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":59789355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849398,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":62576,"kb_used_data":8288,"kb_used_omap":36,"kb_used_meta":54235,"kb_avail":93682576,"statfs":{"total":95995035648,"available":95930957824,"internally_reserved":0,"allocated":8486912,"data_stored":5741182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":55536820},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704751,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":44204,"kb_used_data":8284,"kb_used_omap":17,"kb_used_meta":35886,"kb_avail":93700948,"statfs":{"total":95995035648,"available":95949770752,"internally_reserved":0,"allocated":8482816,"data_stored":5740656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":36747767},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225017963,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40448,"kb_used_data":8304,"kb_used_omap":42,"kb_used_meta":32085,"kb_avail":93704704,"statfs":{"total":95995035648,"available":95953616896,"internally_reserved":0,"allocated":8503296,"data_stored":5745468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":32855864},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757512,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":97584,"kb_used_data":8288,"kb_used_omap":1,"kb_used_meta":89278,"kb_avail":93647568,"statfs":{"total":95995035648,"available":95895109632,"internally_reserved":0,"allocated":8486912,"data_stored":5741206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":91421126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:32:23.387 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:32:24.206 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:23 smithi029 ceph-mon[93650]: from='client.34689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:24.206 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:23 smithi029 ceph-mon[97787]: from='client.34689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:24.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:23 smithi087 ceph-mon[89014]: from='client.34689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:24.636 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:25.029 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:24 smithi029 ceph-mon[93650]: pgmap v1290: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:25.029 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:24 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:25.029 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:24 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:25.029 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:32:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:32:24] "GET /metrics HTTP/1.1" 200 34004 "" "Prometheus/2.43.0" 2024-04-12T11:32:25.029 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:24 smithi029 ceph-mon[97787]: pgmap v1290: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:25.030 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:24 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:25.030 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:24 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:24 smithi087 ceph-mon[89014]: pgmap v1290: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:24 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:24 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:32:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:32:26.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:32:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:32:26.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:26.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:32:26.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:32:26.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:26.695 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:32:26.696 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:32:27.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:26 smithi029 ceph-mon[97787]: pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:27.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:26 smithi029 ceph-mon[93650]: pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:26 smithi087 ceph-mon[89014]: pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:27.373 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1291,"stamp":"2024-04-12T11:32:25.639004+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":472476,"kb_used_data":68076,"kb_used_omap":175,"kb_used_meta":404112,"kb_avail":749488740,"statfs":{"total":767960285184,"available":767476469760,"internally_reserved":0,"allocated":69709824,"data_stored":47712258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":413810857},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003215"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807039+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:31:33.807039+0000","last_peered":"2024-04-12T11:31:33.807039+0000","last_clean":"2024-04-12T11:31:33.807039+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:31:33.807039+0000","last_undegraded":"2024-04-12T11:31:33.807039+0000","last_fullsized":"2024-04-12T11:31:33.807039+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1525,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805863+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:31:33.805863+0000","last_peered":"2024-04-12T11:31:33.805863+0000","last_clean":"2024-04-12T11:31:33.805863+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:31:33.805863+0000","last_undegraded":"2024-04-12T11:31:33.805863+0000","last_fullsized":"2024-04-12T11:31:33.805863+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307246+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:31:34.307246+0000","last_peered":"2024-04-12T11:31:34.307246+0000","last_clean":"2024-04-12T11:31:34.307246+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:31:34.307246+0000","last_undegraded":"2024-04-12T11:31:34.307246+0000","last_fullsized":"2024-04-12T11:31:34.307246+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1572,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329931+0000","last_change":"2024-04-12T11:23:15.593723+0000","last_active":"2024-04-12T11:31:34.329931+0000","last_peered":"2024-04-12T11:31:34.329931+0000","last_clean":"2024-04-12T11:31:34.329931+0000","last_became_active":"2024-04-12T11:23:15.593381+0000","last_became_peered":"2024-04-12T11:23:15.593381+0000","last_unstale":"2024-04-12T11:31:34.329931+0000","last_undegraded":"2024-04-12T11:31:34.329931+0000","last_fullsized":"2024-04-12T11:31:34.329931+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:18:06.971493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1533,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804979+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:31:33.804979+0000","last_peered":"2024-04-12T11:31:33.804979+0000","last_clean":"2024-04-12T11:31:33.804979+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:31:33.804979+0000","last_undegraded":"2024-04-12T11:31:33.804979+0000","last_fullsized":"2024-04-12T11:31:33.804979+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.963036+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:31:34.963036+0000","last_peered":"2024-04-12T11:31:34.963036+0000","last_clean":"2024-04-12T11:31:34.963036+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:31:34.963036+0000","last_undegraded":"2024-04-12T11:31:34.963036+0000","last_fullsized":"2024-04-12T11:31:34.963036+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1568,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330398+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:31:34.330398+0000","last_peered":"2024-04-12T11:31:34.330398+0000","last_clean":"2024-04-12T11:31:34.330398+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:31:34.330398+0000","last_undegraded":"2024-04-12T11:31:34.330398+0000","last_fullsized":"2024-04-12T11:31:34.330398+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1506,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804876+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:31:33.804876+0000","last_peered":"2024-04-12T11:31:33.804876+0000","last_clean":"2024-04-12T11:31:33.804876+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:31:33.804876+0000","last_undegraded":"2024-04-12T11:31:33.804876+0000","last_fullsized":"2024-04-12T11:31:33.804876+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1031,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806752+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:31:33.806752+0000","last_peered":"2024-04-12T11:31:33.806752+0000","last_clean":"2024-04-12T11:31:33.806752+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:31:33.806752+0000","last_undegraded":"2024-04-12T11:31:33.806752+0000","last_fullsized":"2024-04-12T11:31:33.806752+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1486,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804993+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:31:33.804993+0000","last_peered":"2024-04-12T11:31:33.804993+0000","last_clean":"2024-04-12T11:31:33.804993+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:31:33.804993+0000","last_undegraded":"2024-04-12T11:31:33.804993+0000","last_fullsized":"2024-04-12T11:31:33.804993+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1040,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.963138+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:31:34.963138+0000","last_peered":"2024-04-12T11:31:34.963138+0000","last_clean":"2024-04-12T11:31:34.963138+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:31:34.963138+0000","last_undegraded":"2024-04-12T11:31:34.963138+0000","last_fullsized":"2024-04-12T11:31:34.963138+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1568,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307302+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:31:34.307302+0000","last_peered":"2024-04-12T11:31:34.307302+0000","last_clean":"2024-04-12T11:31:34.307302+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:31:34.307302+0000","last_undegraded":"2024-04-12T11:31:34.307302+0000","last_fullsized":"2024-04-12T11:31:34.307302+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1489,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806028+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:31:33.806028+0000","last_peered":"2024-04-12T11:31:33.806028+0000","last_clean":"2024-04-12T11:31:33.806028+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:31:33.806028+0000","last_undegraded":"2024-04-12T11:31:33.806028+0000","last_fullsized":"2024-04-12T11:31:33.806028+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806164+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:31:33.806164+0000","last_peered":"2024-04-12T11:31:33.806164+0000","last_clean":"2024-04-12T11:31:33.806164+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:31:33.806164+0000","last_undegraded":"2024-04-12T11:31:33.806164+0000","last_fullsized":"2024-04-12T11:31:33.806164+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1039,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962053+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:31:34.962053+0000","last_peered":"2024-04-12T11:31:34.962053+0000","last_clean":"2024-04-12T11:31:34.962053+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:31:34.962053+0000","last_undegraded":"2024-04-12T11:31:34.962053+0000","last_fullsized":"2024-04-12T11:31:34.962053+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805796+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:31:33.805796+0000","last_peered":"2024-04-12T11:31:33.805796+0000","last_clean":"2024-04-12T11:31:33.805796+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:31:33.805796+0000","last_undegraded":"2024-04-12T11:31:33.805796+0000","last_fullsized":"2024-04-12T11:31:33.805796+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329105+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:31:34.329105+0000","last_peered":"2024-04-12T11:31:34.329105+0000","last_clean":"2024-04-12T11:31:34.329105+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:31:34.329105+0000","last_undegraded":"2024-04-12T11:31:34.329105+0000","last_fullsized":"2024-04-12T11:31:34.329105+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806888+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:31:33.806888+0000","last_peered":"2024-04-12T11:31:33.806888+0000","last_clean":"2024-04-12T11:31:33.806888+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:31:33.806888+0000","last_undegraded":"2024-04-12T11:31:33.806888+0000","last_fullsized":"2024-04-12T11:31:33.806888+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962176+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:31:34.962176+0000","last_peered":"2024-04-12T11:31:34.962176+0000","last_clean":"2024-04-12T11:31:34.962176+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:31:34.962176+0000","last_undegraded":"2024-04-12T11:31:34.962176+0000","last_fullsized":"2024-04-12T11:31:34.962176+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1570,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329830+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:31:34.329830+0000","last_peered":"2024-04-12T11:31:34.329830+0000","last_clean":"2024-04-12T11:31:34.329830+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:31:34.329830+0000","last_undegraded":"2024-04-12T11:31:34.329830+0000","last_fullsized":"2024-04-12T11:31:34.329830+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1033,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961852+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:31:34.961852+0000","last_peered":"2024-04-12T11:31:34.961852+0000","last_clean":"2024-04-12T11:31:34.961852+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:31:34.961852+0000","last_undegraded":"2024-04-12T11:31:34.961852+0000","last_fullsized":"2024-04-12T11:31:34.961852+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805881+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:31:33.805881+0000","last_peered":"2024-04-12T11:31:33.805881+0000","last_clean":"2024-04-12T11:31:33.805881+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:31:33.805881+0000","last_undegraded":"2024-04-12T11:31:33.805881+0000","last_fullsized":"2024-04-12T11:31:33.805881+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027548+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:31:33.027548+0000","last_peered":"2024-04-12T11:31:33.027548+0000","last_clean":"2024-04-12T11:31:33.027548+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:31:33.027548+0000","last_undegraded":"2024-04-12T11:31:33.027548+0000","last_fullsized":"2024-04-12T11:31:33.027548+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1015,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488384+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:31:33.488384+0000","last_peered":"2024-04-12T11:31:33.488384+0000","last_clean":"2024-04-12T11:31:33.488384+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:31:33.488384+0000","last_undegraded":"2024-04-12T11:31:33.488384+0000","last_fullsized":"2024-04-12T11:31:33.488384+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027317+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:31:33.027317+0000","last_peered":"2024-04-12T11:31:33.027317+0000","last_clean":"2024-04-12T11:31:33.027317+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:31:33.027317+0000","last_undegraded":"2024-04-12T11:31:33.027317+0000","last_fullsized":"2024-04-12T11:31:33.027317+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1503,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806074+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:31:33.806074+0000","last_peered":"2024-04-12T11:31:33.806074+0000","last_clean":"2024-04-12T11:31:33.806074+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:31:33.806074+0000","last_undegraded":"2024-04-12T11:31:33.806074+0000","last_fullsized":"2024-04-12T11:31:33.806074+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027231+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:31:33.027231+0000","last_peered":"2024-04-12T11:31:33.027231+0000","last_clean":"2024-04-12T11:31:33.027231+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:31:33.027231+0000","last_undegraded":"2024-04-12T11:31:33.027231+0000","last_fullsized":"2024-04-12T11:31:33.027231+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.098131+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:31:33.098131+0000","last_peered":"2024-04-12T11:31:33.098131+0000","last_clean":"2024-04-12T11:31:33.098131+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:31:33.098131+0000","last_undegraded":"2024-04-12T11:31:33.098131+0000","last_fullsized":"2024-04-12T11:31:33.098131+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488787+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:31:33.488787+0000","last_peered":"2024-04-12T11:31:33.488787+0000","last_clean":"2024-04-12T11:31:33.488787+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:31:33.488787+0000","last_undegraded":"2024-04-12T11:31:33.488787+0000","last_fullsized":"2024-04-12T11:31:33.488787+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1579,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329258+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:31:34.329258+0000","last_peered":"2024-04-12T11:31:34.329258+0000","last_clean":"2024-04-12T11:31:34.329258+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:31:34.329258+0000","last_undegraded":"2024-04-12T11:31:34.329258+0000","last_fullsized":"2024-04-12T11:31:34.329258+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1511,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805669+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:31:33.805669+0000","last_peered":"2024-04-12T11:31:33.805669+0000","last_clean":"2024-04-12T11:31:33.805669+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:31:33.805669+0000","last_undegraded":"2024-04-12T11:31:33.805669+0000","last_fullsized":"2024-04-12T11:31:33.805669+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1493,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806698+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:31:33.806698+0000","last_peered":"2024-04-12T11:31:33.806698+0000","last_clean":"2024-04-12T11:31:33.806698+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:31:33.806698+0000","last_undegraded":"2024-04-12T11:31:33.806698+0000","last_fullsized":"2024-04-12T11:31:33.806698+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.098030+0000","last_change":"2024-04-12T11:23:36.639270+0000","last_active":"2024-04-12T11:31:33.098030+0000","last_peered":"2024-04-12T11:31:33.098030+0000","last_clean":"2024-04-12T11:31:33.098030+0000","last_became_active":"2024-04-12T11:23:36.638952+0000","last_became_peered":"2024-04-12T11:23:36.638952+0000","last_unstale":"2024-04-12T11:31:33.098030+0000","last_undegraded":"2024-04-12T11:31:33.098030+0000","last_fullsized":"2024-04-12T11:31:33.098030+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:04:50.403812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1515,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.809196+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:31:33.809196+0000","last_peered":"2024-04-12T11:31:33.809196+0000","last_clean":"2024-04-12T11:31:33.809196+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:31:33.809196+0000","last_undegraded":"2024-04-12T11:31:33.809196+0000","last_fullsized":"2024-04-12T11:31:33.809196+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806193+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:31:33.806193+0000","last_peered":"2024-04-12T11:31:33.806193+0000","last_clean":"2024-04-12T11:31:33.806193+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:31:33.806193+0000","last_undegraded":"2024-04-12T11:31:33.806193+0000","last_fullsized":"2024-04-12T11:31:33.806193+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807330+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:31:33.807330+0000","last_peered":"2024-04-12T11:31:33.807330+0000","last_clean":"2024-04-12T11:31:33.807330+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:31:33.807330+0000","last_undegraded":"2024-04-12T11:31:33.807330+0000","last_fullsized":"2024-04-12T11:31:33.807330+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1519,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804798+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:31:33.804798+0000","last_peered":"2024-04-12T11:31:33.804798+0000","last_clean":"2024-04-12T11:31:33.804798+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:31:33.804798+0000","last_undegraded":"2024-04-12T11:31:33.804798+0000","last_fullsized":"2024-04-12T11:31:33.804798+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307677+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:31:34.307677+0000","last_peered":"2024-04-12T11:31:34.307677+0000","last_clean":"2024-04-12T11:31:34.307677+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:31:34.307677+0000","last_undegraded":"2024-04-12T11:31:34.307677+0000","last_fullsized":"2024-04-12T11:31:34.307677+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1578,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.308030+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:31:34.308030+0000","last_peered":"2024-04-12T11:31:34.308030+0000","last_clean":"2024-04-12T11:31:34.308030+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:31:34.308030+0000","last_undegraded":"2024-04-12T11:31:34.308030+0000","last_fullsized":"2024-04-12T11:31:34.308030+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1047,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807269+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:31:33.807269+0000","last_peered":"2024-04-12T11:31:33.807269+0000","last_clean":"2024-04-12T11:31:33.807269+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:31:33.807269+0000","last_undegraded":"2024-04-12T11:31:33.807269+0000","last_fullsized":"2024-04-12T11:31:33.807269+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307257+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:31:34.307257+0000","last_peered":"2024-04-12T11:31:34.307257+0000","last_clean":"2024-04-12T11:31:34.307257+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:31:34.307257+0000","last_undegraded":"2024-04-12T11:31:34.307257+0000","last_fullsized":"2024-04-12T11:31:34.307257+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488693+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:31:33.488693+0000","last_peered":"2024-04-12T11:31:33.488693+0000","last_clean":"2024-04-12T11:31:33.488693+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:31:33.488693+0000","last_undegraded":"2024-04-12T11:31:33.488693+0000","last_fullsized":"2024-04-12T11:31:33.488693+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806006+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:31:33.806006+0000","last_peered":"2024-04-12T11:31:33.806006+0000","last_clean":"2024-04-12T11:31:33.806006+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:31:33.806006+0000","last_undegraded":"2024-04-12T11:31:33.806006+0000","last_fullsized":"2024-04-12T11:31:33.806006+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1578,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307348+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:31:34.307348+0000","last_peered":"2024-04-12T11:31:34.307348+0000","last_clean":"2024-04-12T11:31:34.307348+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:31:34.307348+0000","last_undegraded":"2024-04-12T11:31:34.307348+0000","last_fullsized":"2024-04-12T11:31:34.307348+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1542,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961088+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:31:34.961088+0000","last_peered":"2024-04-12T11:31:34.961088+0000","last_clean":"2024-04-12T11:31:34.961088+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:31:34.961088+0000","last_undegraded":"2024-04-12T11:31:34.961088+0000","last_fullsized":"2024-04-12T11:31:34.961088+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1570,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307824+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:31:34.307824+0000","last_peered":"2024-04-12T11:31:34.307824+0000","last_clean":"2024-04-12T11:31:34.307824+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:31:34.307824+0000","last_undegraded":"2024-04-12T11:31:34.307824+0000","last_fullsized":"2024-04-12T11:31:34.307824+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1572,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307812+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:31:34.307812+0000","last_peered":"2024-04-12T11:31:34.307812+0000","last_clean":"2024-04-12T11:31:34.307812+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:31:34.307812+0000","last_undegraded":"2024-04-12T11:31:34.307812+0000","last_fullsized":"2024-04-12T11:31:34.307812+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488902+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:31:33.488902+0000","last_peered":"2024-04-12T11:31:33.488902+0000","last_clean":"2024-04-12T11:31:33.488902+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:31:33.488902+0000","last_undegraded":"2024-04-12T11:31:33.488902+0000","last_fullsized":"2024-04-12T11:31:33.488902+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1391,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.026548+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:31:33.026548+0000","last_peered":"2024-04-12T11:31:33.026548+0000","last_clean":"2024-04-12T11:31:33.026548+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:31:33.026548+0000","last_undegraded":"2024-04-12T11:31:33.026548+0000","last_fullsized":"2024-04-12T11:31:33.026548+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1486,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804959+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:31:33.804959+0000","last_peered":"2024-04-12T11:31:33.804959+0000","last_clean":"2024-04-12T11:31:33.804959+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:31:33.804959+0000","last_undegraded":"2024-04-12T11:31:33.804959+0000","last_fullsized":"2024-04-12T11:31:33.804959+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962347+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:31:34.962347+0000","last_peered":"2024-04-12T11:31:34.962347+0000","last_clean":"2024-04-12T11:31:34.962347+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:31:34.962347+0000","last_undegraded":"2024-04-12T11:31:34.962347+0000","last_fullsized":"2024-04-12T11:31:34.962347+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1044,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961718+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:31:34.961718+0000","last_peered":"2024-04-12T11:31:34.961718+0000","last_clean":"2024-04-12T11:31:34.961718+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:31:34.961718+0000","last_undegraded":"2024-04-12T11:31:34.961718+0000","last_fullsized":"2024-04-12T11:31:34.961718+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1580,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329112+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:31:34.329112+0000","last_peered":"2024-04-12T11:31:34.329112+0000","last_clean":"2024-04-12T11:31:34.329112+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:31:34.329112+0000","last_undegraded":"2024-04-12T11:31:34.329112+0000","last_fullsized":"2024-04-12T11:31:34.329112+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1579,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329333+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:31:34.329333+0000","last_peered":"2024-04-12T11:31:34.329333+0000","last_clean":"2024-04-12T11:31:34.329333+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:31:34.329333+0000","last_undegraded":"2024-04-12T11:31:34.329333+0000","last_fullsized":"2024-04-12T11:31:34.329333+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1043,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027168+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:31:33.027168+0000","last_peered":"2024-04-12T11:31:33.027168+0000","last_clean":"2024-04-12T11:31:33.027168+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:31:33.027168+0000","last_undegraded":"2024-04-12T11:31:33.027168+0000","last_fullsized":"2024-04-12T11:31:33.027168+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307821+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:31:34.307821+0000","last_peered":"2024-04-12T11:31:34.307821+0000","last_clean":"2024-04-12T11:31:34.307821+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:31:34.307821+0000","last_undegraded":"2024-04-12T11:31:34.307821+0000","last_fullsized":"2024-04-12T11:31:34.307821+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1563,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330389+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:31:34.330389+0000","last_peered":"2024-04-12T11:31:34.330389+0000","last_clean":"2024-04-12T11:31:34.330389+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:31:34.330389+0000","last_undegraded":"2024-04-12T11:31:34.330389+0000","last_fullsized":"2024-04-12T11:31:34.330389+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806115+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:31:33.806115+0000","last_peered":"2024-04-12T11:31:33.806115+0000","last_clean":"2024-04-12T11:31:33.806115+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:31:33.806115+0000","last_undegraded":"2024-04-12T11:31:33.806115+0000","last_fullsized":"2024-04-12T11:31:33.806115+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307609+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:31:34.307609+0000","last_peered":"2024-04-12T11:31:34.307609+0000","last_clean":"2024-04-12T11:31:34.307609+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:31:34.307609+0000","last_undegraded":"2024-04-12T11:31:34.307609+0000","last_fullsized":"2024-04-12T11:31:34.307609+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1499,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804727+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:31:33.804727+0000","last_peered":"2024-04-12T11:31:33.804727+0000","last_clean":"2024-04-12T11:31:33.804727+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:31:33.804727+0000","last_undegraded":"2024-04-12T11:31:33.804727+0000","last_fullsized":"2024-04-12T11:31:33.804727+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.098490+0000","last_change":"2024-04-12T11:22:40.580200+0000","last_active":"2024-04-12T11:31:33.098490+0000","last_peered":"2024-04-12T11:31:33.098490+0000","last_clean":"2024-04-12T11:31:33.098490+0000","last_became_active":"2024-04-12T11:22:40.579809+0000","last_became_peered":"2024-04-12T11:22:40.579809+0000","last_unstale":"2024-04-12T11:31:33.098490+0000","last_undegraded":"2024-04-12T11:31:33.098490+0000","last_fullsized":"2024-04-12T11:31:33.098490+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:28:23.317249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1582,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329989+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:31:34.329989+0000","last_peered":"2024-04-12T11:31:34.329989+0000","last_clean":"2024-04-12T11:31:34.329989+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:31:34.329989+0000","last_undegraded":"2024-04-12T11:31:34.329989+0000","last_fullsized":"2024-04-12T11:31:34.329989+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1489,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805172+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:31:33.805172+0000","last_peered":"2024-04-12T11:31:33.805172+0000","last_clean":"2024-04-12T11:31:33.805172+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:31:33.805172+0000","last_undegraded":"2024-04-12T11:31:33.805172+0000","last_fullsized":"2024-04-12T11:31:33.805172+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1573,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330291+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:31:34.330291+0000","last_peered":"2024-04-12T11:31:34.330291+0000","last_clean":"2024-04-12T11:31:34.330291+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:31:34.330291+0000","last_undegraded":"2024-04-12T11:31:34.330291+0000","last_fullsized":"2024-04-12T11:31:34.330291+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1553,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961201+0000","last_change":"2024-04-12T11:23:15.593766+0000","last_active":"2024-04-12T11:31:34.961201+0000","last_peered":"2024-04-12T11:31:34.961201+0000","last_clean":"2024-04-12T11:31:34.961201+0000","last_became_active":"2024-04-12T11:23:15.593428+0000","last_became_peered":"2024-04-12T11:23:15.593428+0000","last_unstale":"2024-04-12T11:31:34.961201+0000","last_undegraded":"2024-04-12T11:31:34.961201+0000","last_fullsized":"2024-04-12T11:31:34.961201+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:12.389667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1047,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488797+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:31:33.488797+0000","last_peered":"2024-04-12T11:31:33.488797+0000","last_clean":"2024-04-12T11:31:33.488797+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:31:33.488797+0000","last_undegraded":"2024-04-12T11:31:33.488797+0000","last_fullsized":"2024-04-12T11:31:33.488797+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806362+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:31:33.806362+0000","last_peered":"2024-04-12T11:31:33.806362+0000","last_clean":"2024-04-12T11:31:33.806362+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:31:33.806362+0000","last_undegraded":"2024-04-12T11:31:33.806362+0000","last_fullsized":"2024-04-12T11:31:33.806362+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1032,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.489264+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:31:33.489264+0000","last_peered":"2024-04-12T11:31:33.489264+0000","last_clean":"2024-04-12T11:31:33.489264+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:31:33.489264+0000","last_undegraded":"2024-04-12T11:31:33.489264+0000","last_fullsized":"2024-04-12T11:31:33.489264+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1508,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805189+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:31:33.805189+0000","last_peered":"2024-04-12T11:31:33.805189+0000","last_clean":"2024-04-12T11:31:33.805189+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:31:33.805189+0000","last_undegraded":"2024-04-12T11:31:33.805189+0000","last_fullsized":"2024-04-12T11:31:33.805189+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1500,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805844+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:31:33.805844+0000","last_peered":"2024-04-12T11:31:33.805844+0000","last_clean":"2024-04-12T11:31:33.805844+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:31:33.805844+0000","last_undegraded":"2024-04-12T11:31:33.805844+0000","last_fullsized":"2024-04-12T11:31:33.805844+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1511,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804851+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:31:33.804851+0000","last_peered":"2024-04-12T11:31:33.804851+0000","last_clean":"2024-04-12T11:31:33.804851+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:31:33.804851+0000","last_undegraded":"2024-04-12T11:31:33.804851+0000","last_fullsized":"2024-04-12T11:31:33.804851+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329503+0000","last_change":"2024-04-12T11:22:40.582361+0000","last_active":"2024-04-12T11:31:34.329503+0000","last_peered":"2024-04-12T11:31:34.329503+0000","last_clean":"2024-04-12T11:31:34.329503+0000","last_became_active":"2024-04-12T11:22:40.581996+0000","last_became_peered":"2024-04-12T11:22:40.581996+0000","last_unstale":"2024-04-12T11:31:34.329503+0000","last_undegraded":"2024-04-12T11:31:34.329503+0000","last_fullsized":"2024-04-12T11:31:34.329503+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:40:43.563258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805035+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:31:33.805035+0000","last_peered":"2024-04-12T11:31:33.805035+0000","last_clean":"2024-04-12T11:31:33.805035+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:31:33.805035+0000","last_undegraded":"2024-04-12T11:31:33.805035+0000","last_fullsized":"2024-04-12T11:31:33.805035+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234240,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":58052,"kb_used_data":8888,"kb_used_omap":1,"kb_used_meta":49150,"kb_avail":93687100,"statfs":{"total":95995035648,"available":95935590400,"internally_reserved":0,"allocated":9101312,"data_stored":6335894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":50330059},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495877,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":66228,"kb_used_data":8876,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93678924,"statfs":{"total":95995035648,"available":95927218176,"internally_reserved":0,"allocated":9089024,"data_stored":6335748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":58718664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536187,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36068,"kb_used_data":8280,"kb_used_omap":30,"kb_used_meta":27745,"kb_avail":93709084,"statfs":{"total":95995035648,"available":95958102016,"internally_reserved":0,"allocated":8478720,"data_stored":5740566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":28411202},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190271,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67316,"kb_used_data":8868,"kb_used_omap":43,"kb_used_meta":58388,"kb_avail":93677836,"statfs":{"total":95995035648,"available":95926104064,"internally_reserved":0,"allocated":9080832,"data_stored":6331538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":59789355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849399,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":62576,"kb_used_data":8288,"kb_used_omap":36,"kb_used_meta":54235,"kb_avail":93682576,"statfs":{"total":95995035648,"available":95930957824,"internally_reserved":0,"allocated":8486912,"data_stored":5741182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":55536820},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704752,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":44204,"kb_used_data":8284,"kb_used_omap":17,"kb_used_meta":35886,"kb_avail":93700948,"statfs":{"total":95995035648,"available":95949770752,"internally_reserved":0,"allocated":8482816,"data_stored":5740656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":36747767},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225017964,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40448,"kb_used_data":8304,"kb_used_omap":42,"kb_used_meta":32085,"kb_avail":93704704,"statfs":{"total":95995035648,"available":95953616896,"internally_reserved":0,"allocated":8503296,"data_stored":5745468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":32855864},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757513,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":97584,"kb_used_data":8288,"kb_used_omap":1,"kb_used_meta":89278,"kb_avail":93647568,"statfs":{"total":95995035648,"available":95895109632,"internally_reserved":0,"allocated":8486912,"data_stored":5741206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":91421126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:32:27.376 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-12T11:32:27.376 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-12T11:32:27.376 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-12T11:32:27.377 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph health --format=json 2024-04-12T11:32:27.588 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:27.997 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:27 smithi029 ceph-mon[97787]: from='client.44594 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:27.998 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:27 smithi029 ceph-mon[93650]: from='client.44594 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:28.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:27 smithi087 ceph-mon[89014]: from='client.44594 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:29.164 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:28 smithi029 ceph-mon[93650]: pgmap v1292: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:29.165 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:28 smithi029 ceph-mon[97787]: pgmap v1292: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:28 smithi087 ceph-mon[89014]: pgmap v1292: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:32:29.427 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-12T11:32:29.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:29 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3849223341' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T11:32:29.999 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:29 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3849223341' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T11:32:30.021 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-12T11:32:30.022 INFO:teuthology.run_tasks:Running task print... 2024-04-12T11:32:30.032 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-04-12T11:32:30.032 INFO:teuthology.run_tasks:Running task parallel... 2024-04-12T11:32:30.040 INFO:teuthology.task.parallel:starting parallel... 2024-04-12T11:32:30.040 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-12T11:32:30.040 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-04-12T11:32:30.042 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-04-12T11:32:30.043 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'wip-rf-comb1'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'wip-rf-comb1', 'sha1': '2cee1857f8068f7ca29db3a37937b8e9f919ba95'}} 2024-04-12T11:32:30.043 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-04-12T11:32:30.043 INFO:tasks.thrashosds:Beginning thrashosds... 2024-04-12T11:32:30.043 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-12T11:32:30.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:32:30.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:29 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3849223341' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T11:32:30.251 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:30 smithi029 ceph-mon[93650]: pgmap v1293: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:30 smithi029 ceph-mon[97787]: pgmap v1293: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:31.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:30 smithi087 ceph-mon[89014]: pgmap v1293: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:32.077 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:32:32.078 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:32:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:32:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:32:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:32:32.776 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1294,"stamp":"2024-04-12T11:32:31.640303+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":472476,"kb_used_data":68076,"kb_used_omap":175,"kb_used_meta":404112,"kb_avail":749488740,"statfs":{"total":767960285184,"available":767476469760,"internally_reserved":0,"allocated":69709824,"data_stored":47712258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":413810857},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002580"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807039+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:31:33.807039+0000","last_peered":"2024-04-12T11:31:33.807039+0000","last_clean":"2024-04-12T11:31:33.807039+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:31:33.807039+0000","last_undegraded":"2024-04-12T11:31:33.807039+0000","last_fullsized":"2024-04-12T11:31:33.807039+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1525,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805863+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:31:33.805863+0000","last_peered":"2024-04-12T11:31:33.805863+0000","last_clean":"2024-04-12T11:31:33.805863+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:31:33.805863+0000","last_undegraded":"2024-04-12T11:31:33.805863+0000","last_fullsized":"2024-04-12T11:31:33.805863+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307246+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:31:34.307246+0000","last_peered":"2024-04-12T11:31:34.307246+0000","last_clean":"2024-04-12T11:31:34.307246+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:31:34.307246+0000","last_undegraded":"2024-04-12T11:31:34.307246+0000","last_fullsized":"2024-04-12T11:31:34.307246+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1572,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329931+0000","last_change":"2024-04-12T11:23:15.593723+0000","last_active":"2024-04-12T11:31:34.329931+0000","last_peered":"2024-04-12T11:31:34.329931+0000","last_clean":"2024-04-12T11:31:34.329931+0000","last_became_active":"2024-04-12T11:23:15.593381+0000","last_became_peered":"2024-04-12T11:23:15.593381+0000","last_unstale":"2024-04-12T11:31:34.329931+0000","last_undegraded":"2024-04-12T11:31:34.329931+0000","last_fullsized":"2024-04-12T11:31:34.329931+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:18:06.971493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1533,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804979+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:31:33.804979+0000","last_peered":"2024-04-12T11:31:33.804979+0000","last_clean":"2024-04-12T11:31:33.804979+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:31:33.804979+0000","last_undegraded":"2024-04-12T11:31:33.804979+0000","last_fullsized":"2024-04-12T11:31:33.804979+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.963036+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:31:34.963036+0000","last_peered":"2024-04-12T11:31:34.963036+0000","last_clean":"2024-04-12T11:31:34.963036+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:31:34.963036+0000","last_undegraded":"2024-04-12T11:31:34.963036+0000","last_fullsized":"2024-04-12T11:31:34.963036+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1568,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330398+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:31:34.330398+0000","last_peered":"2024-04-12T11:31:34.330398+0000","last_clean":"2024-04-12T11:31:34.330398+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:31:34.330398+0000","last_undegraded":"2024-04-12T11:31:34.330398+0000","last_fullsized":"2024-04-12T11:31:34.330398+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1506,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804876+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:31:33.804876+0000","last_peered":"2024-04-12T11:31:33.804876+0000","last_clean":"2024-04-12T11:31:33.804876+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:31:33.804876+0000","last_undegraded":"2024-04-12T11:31:33.804876+0000","last_fullsized":"2024-04-12T11:31:33.804876+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1031,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806752+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:31:33.806752+0000","last_peered":"2024-04-12T11:31:33.806752+0000","last_clean":"2024-04-12T11:31:33.806752+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:31:33.806752+0000","last_undegraded":"2024-04-12T11:31:33.806752+0000","last_fullsized":"2024-04-12T11:31:33.806752+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1486,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804993+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:31:33.804993+0000","last_peered":"2024-04-12T11:31:33.804993+0000","last_clean":"2024-04-12T11:31:33.804993+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:31:33.804993+0000","last_undegraded":"2024-04-12T11:31:33.804993+0000","last_fullsized":"2024-04-12T11:31:33.804993+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1040,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.963138+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:31:34.963138+0000","last_peered":"2024-04-12T11:31:34.963138+0000","last_clean":"2024-04-12T11:31:34.963138+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:31:34.963138+0000","last_undegraded":"2024-04-12T11:31:34.963138+0000","last_fullsized":"2024-04-12T11:31:34.963138+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1568,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307302+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:31:34.307302+0000","last_peered":"2024-04-12T11:31:34.307302+0000","last_clean":"2024-04-12T11:31:34.307302+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:31:34.307302+0000","last_undegraded":"2024-04-12T11:31:34.307302+0000","last_fullsized":"2024-04-12T11:31:34.307302+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1489,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806028+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:31:33.806028+0000","last_peered":"2024-04-12T11:31:33.806028+0000","last_clean":"2024-04-12T11:31:33.806028+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:31:33.806028+0000","last_undegraded":"2024-04-12T11:31:33.806028+0000","last_fullsized":"2024-04-12T11:31:33.806028+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806164+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:31:33.806164+0000","last_peered":"2024-04-12T11:31:33.806164+0000","last_clean":"2024-04-12T11:31:33.806164+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:31:33.806164+0000","last_undegraded":"2024-04-12T11:31:33.806164+0000","last_fullsized":"2024-04-12T11:31:33.806164+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1039,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962053+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:31:34.962053+0000","last_peered":"2024-04-12T11:31:34.962053+0000","last_clean":"2024-04-12T11:31:34.962053+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:31:34.962053+0000","last_undegraded":"2024-04-12T11:31:34.962053+0000","last_fullsized":"2024-04-12T11:31:34.962053+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805796+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:31:33.805796+0000","last_peered":"2024-04-12T11:31:33.805796+0000","last_clean":"2024-04-12T11:31:33.805796+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:31:33.805796+0000","last_undegraded":"2024-04-12T11:31:33.805796+0000","last_fullsized":"2024-04-12T11:31:33.805796+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329105+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:31:34.329105+0000","last_peered":"2024-04-12T11:31:34.329105+0000","last_clean":"2024-04-12T11:31:34.329105+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:31:34.329105+0000","last_undegraded":"2024-04-12T11:31:34.329105+0000","last_fullsized":"2024-04-12T11:31:34.329105+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806888+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:31:33.806888+0000","last_peered":"2024-04-12T11:31:33.806888+0000","last_clean":"2024-04-12T11:31:33.806888+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:31:33.806888+0000","last_undegraded":"2024-04-12T11:31:33.806888+0000","last_fullsized":"2024-04-12T11:31:33.806888+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962176+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:31:34.962176+0000","last_peered":"2024-04-12T11:31:34.962176+0000","last_clean":"2024-04-12T11:31:34.962176+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:31:34.962176+0000","last_undegraded":"2024-04-12T11:31:34.962176+0000","last_fullsized":"2024-04-12T11:31:34.962176+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1570,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329830+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:31:34.329830+0000","last_peered":"2024-04-12T11:31:34.329830+0000","last_clean":"2024-04-12T11:31:34.329830+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:31:34.329830+0000","last_undegraded":"2024-04-12T11:31:34.329830+0000","last_fullsized":"2024-04-12T11:31:34.329830+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1033,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961852+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:31:34.961852+0000","last_peered":"2024-04-12T11:31:34.961852+0000","last_clean":"2024-04-12T11:31:34.961852+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:31:34.961852+0000","last_undegraded":"2024-04-12T11:31:34.961852+0000","last_fullsized":"2024-04-12T11:31:34.961852+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805881+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:31:33.805881+0000","last_peered":"2024-04-12T11:31:33.805881+0000","last_clean":"2024-04-12T11:31:33.805881+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:31:33.805881+0000","last_undegraded":"2024-04-12T11:31:33.805881+0000","last_fullsized":"2024-04-12T11:31:33.805881+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027548+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:31:33.027548+0000","last_peered":"2024-04-12T11:31:33.027548+0000","last_clean":"2024-04-12T11:31:33.027548+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:31:33.027548+0000","last_undegraded":"2024-04-12T11:31:33.027548+0000","last_fullsized":"2024-04-12T11:31:33.027548+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1015,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488384+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:31:33.488384+0000","last_peered":"2024-04-12T11:31:33.488384+0000","last_clean":"2024-04-12T11:31:33.488384+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:31:33.488384+0000","last_undegraded":"2024-04-12T11:31:33.488384+0000","last_fullsized":"2024-04-12T11:31:33.488384+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027317+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:31:33.027317+0000","last_peered":"2024-04-12T11:31:33.027317+0000","last_clean":"2024-04-12T11:31:33.027317+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:31:33.027317+0000","last_undegraded":"2024-04-12T11:31:33.027317+0000","last_fullsized":"2024-04-12T11:31:33.027317+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1503,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806074+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:31:33.806074+0000","last_peered":"2024-04-12T11:31:33.806074+0000","last_clean":"2024-04-12T11:31:33.806074+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:31:33.806074+0000","last_undegraded":"2024-04-12T11:31:33.806074+0000","last_fullsized":"2024-04-12T11:31:33.806074+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027231+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:31:33.027231+0000","last_peered":"2024-04-12T11:31:33.027231+0000","last_clean":"2024-04-12T11:31:33.027231+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:31:33.027231+0000","last_undegraded":"2024-04-12T11:31:33.027231+0000","last_fullsized":"2024-04-12T11:31:33.027231+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.098131+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:31:33.098131+0000","last_peered":"2024-04-12T11:31:33.098131+0000","last_clean":"2024-04-12T11:31:33.098131+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:31:33.098131+0000","last_undegraded":"2024-04-12T11:31:33.098131+0000","last_fullsized":"2024-04-12T11:31:33.098131+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488787+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:31:33.488787+0000","last_peered":"2024-04-12T11:31:33.488787+0000","last_clean":"2024-04-12T11:31:33.488787+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:31:33.488787+0000","last_undegraded":"2024-04-12T11:31:33.488787+0000","last_fullsized":"2024-04-12T11:31:33.488787+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1579,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329258+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:31:34.329258+0000","last_peered":"2024-04-12T11:31:34.329258+0000","last_clean":"2024-04-12T11:31:34.329258+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:31:34.329258+0000","last_undegraded":"2024-04-12T11:31:34.329258+0000","last_fullsized":"2024-04-12T11:31:34.329258+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1511,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805669+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:31:33.805669+0000","last_peered":"2024-04-12T11:31:33.805669+0000","last_clean":"2024-04-12T11:31:33.805669+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:31:33.805669+0000","last_undegraded":"2024-04-12T11:31:33.805669+0000","last_fullsized":"2024-04-12T11:31:33.805669+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1493,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806698+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:31:33.806698+0000","last_peered":"2024-04-12T11:31:33.806698+0000","last_clean":"2024-04-12T11:31:33.806698+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:31:33.806698+0000","last_undegraded":"2024-04-12T11:31:33.806698+0000","last_fullsized":"2024-04-12T11:31:33.806698+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.098030+0000","last_change":"2024-04-12T11:23:36.639270+0000","last_active":"2024-04-12T11:31:33.098030+0000","last_peered":"2024-04-12T11:31:33.098030+0000","last_clean":"2024-04-12T11:31:33.098030+0000","last_became_active":"2024-04-12T11:23:36.638952+0000","last_became_peered":"2024-04-12T11:23:36.638952+0000","last_unstale":"2024-04-12T11:31:33.098030+0000","last_undegraded":"2024-04-12T11:31:33.098030+0000","last_fullsized":"2024-04-12T11:31:33.098030+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:04:50.403812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1515,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.809196+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:31:33.809196+0000","last_peered":"2024-04-12T11:31:33.809196+0000","last_clean":"2024-04-12T11:31:33.809196+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:31:33.809196+0000","last_undegraded":"2024-04-12T11:31:33.809196+0000","last_fullsized":"2024-04-12T11:31:33.809196+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806193+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:31:33.806193+0000","last_peered":"2024-04-12T11:31:33.806193+0000","last_clean":"2024-04-12T11:31:33.806193+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:31:33.806193+0000","last_undegraded":"2024-04-12T11:31:33.806193+0000","last_fullsized":"2024-04-12T11:31:33.806193+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807330+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:31:33.807330+0000","last_peered":"2024-04-12T11:31:33.807330+0000","last_clean":"2024-04-12T11:31:33.807330+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:31:33.807330+0000","last_undegraded":"2024-04-12T11:31:33.807330+0000","last_fullsized":"2024-04-12T11:31:33.807330+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1519,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804798+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:31:33.804798+0000","last_peered":"2024-04-12T11:31:33.804798+0000","last_clean":"2024-04-12T11:31:33.804798+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:31:33.804798+0000","last_undegraded":"2024-04-12T11:31:33.804798+0000","last_fullsized":"2024-04-12T11:31:33.804798+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307677+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:31:34.307677+0000","last_peered":"2024-04-12T11:31:34.307677+0000","last_clean":"2024-04-12T11:31:34.307677+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:31:34.307677+0000","last_undegraded":"2024-04-12T11:31:34.307677+0000","last_fullsized":"2024-04-12T11:31:34.307677+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1578,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.308030+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:31:34.308030+0000","last_peered":"2024-04-12T11:31:34.308030+0000","last_clean":"2024-04-12T11:31:34.308030+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:31:34.308030+0000","last_undegraded":"2024-04-12T11:31:34.308030+0000","last_fullsized":"2024-04-12T11:31:34.308030+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1047,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807269+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:31:33.807269+0000","last_peered":"2024-04-12T11:31:33.807269+0000","last_clean":"2024-04-12T11:31:33.807269+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:31:33.807269+0000","last_undegraded":"2024-04-12T11:31:33.807269+0000","last_fullsized":"2024-04-12T11:31:33.807269+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307257+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:31:34.307257+0000","last_peered":"2024-04-12T11:31:34.307257+0000","last_clean":"2024-04-12T11:31:34.307257+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:31:34.307257+0000","last_undegraded":"2024-04-12T11:31:34.307257+0000","last_fullsized":"2024-04-12T11:31:34.307257+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488693+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:31:33.488693+0000","last_peered":"2024-04-12T11:31:33.488693+0000","last_clean":"2024-04-12T11:31:33.488693+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:31:33.488693+0000","last_undegraded":"2024-04-12T11:31:33.488693+0000","last_fullsized":"2024-04-12T11:31:33.488693+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806006+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:31:33.806006+0000","last_peered":"2024-04-12T11:31:33.806006+0000","last_clean":"2024-04-12T11:31:33.806006+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:31:33.806006+0000","last_undegraded":"2024-04-12T11:31:33.806006+0000","last_fullsized":"2024-04-12T11:31:33.806006+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1578,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307348+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:31:34.307348+0000","last_peered":"2024-04-12T11:31:34.307348+0000","last_clean":"2024-04-12T11:31:34.307348+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:31:34.307348+0000","last_undegraded":"2024-04-12T11:31:34.307348+0000","last_fullsized":"2024-04-12T11:31:34.307348+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1542,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961088+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:31:34.961088+0000","last_peered":"2024-04-12T11:31:34.961088+0000","last_clean":"2024-04-12T11:31:34.961088+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:31:34.961088+0000","last_undegraded":"2024-04-12T11:31:34.961088+0000","last_fullsized":"2024-04-12T11:31:34.961088+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1570,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307824+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:31:34.307824+0000","last_peered":"2024-04-12T11:31:34.307824+0000","last_clean":"2024-04-12T11:31:34.307824+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:31:34.307824+0000","last_undegraded":"2024-04-12T11:31:34.307824+0000","last_fullsized":"2024-04-12T11:31:34.307824+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1572,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307812+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:31:34.307812+0000","last_peered":"2024-04-12T11:31:34.307812+0000","last_clean":"2024-04-12T11:31:34.307812+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:31:34.307812+0000","last_undegraded":"2024-04-12T11:31:34.307812+0000","last_fullsized":"2024-04-12T11:31:34.307812+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488902+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:31:33.488902+0000","last_peered":"2024-04-12T11:31:33.488902+0000","last_clean":"2024-04-12T11:31:33.488902+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:31:33.488902+0000","last_undegraded":"2024-04-12T11:31:33.488902+0000","last_fullsized":"2024-04-12T11:31:33.488902+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1391,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.026548+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:31:33.026548+0000","last_peered":"2024-04-12T11:31:33.026548+0000","last_clean":"2024-04-12T11:31:33.026548+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:31:33.026548+0000","last_undegraded":"2024-04-12T11:31:33.026548+0000","last_fullsized":"2024-04-12T11:31:33.026548+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1486,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804959+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:31:33.804959+0000","last_peered":"2024-04-12T11:31:33.804959+0000","last_clean":"2024-04-12T11:31:33.804959+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:31:33.804959+0000","last_undegraded":"2024-04-12T11:31:33.804959+0000","last_fullsized":"2024-04-12T11:31:33.804959+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962347+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:31:34.962347+0000","last_peered":"2024-04-12T11:31:34.962347+0000","last_clean":"2024-04-12T11:31:34.962347+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:31:34.962347+0000","last_undegraded":"2024-04-12T11:31:34.962347+0000","last_fullsized":"2024-04-12T11:31:34.962347+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1044,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961718+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:31:34.961718+0000","last_peered":"2024-04-12T11:31:34.961718+0000","last_clean":"2024-04-12T11:31:34.961718+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:31:34.961718+0000","last_undegraded":"2024-04-12T11:31:34.961718+0000","last_fullsized":"2024-04-12T11:31:34.961718+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1580,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329112+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:31:34.329112+0000","last_peered":"2024-04-12T11:31:34.329112+0000","last_clean":"2024-04-12T11:31:34.329112+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:31:34.329112+0000","last_undegraded":"2024-04-12T11:31:34.329112+0000","last_fullsized":"2024-04-12T11:31:34.329112+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1579,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329333+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:31:34.329333+0000","last_peered":"2024-04-12T11:31:34.329333+0000","last_clean":"2024-04-12T11:31:34.329333+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:31:34.329333+0000","last_undegraded":"2024-04-12T11:31:34.329333+0000","last_fullsized":"2024-04-12T11:31:34.329333+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1043,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027168+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:31:33.027168+0000","last_peered":"2024-04-12T11:31:33.027168+0000","last_clean":"2024-04-12T11:31:33.027168+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:31:33.027168+0000","last_undegraded":"2024-04-12T11:31:33.027168+0000","last_fullsized":"2024-04-12T11:31:33.027168+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307821+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:31:34.307821+0000","last_peered":"2024-04-12T11:31:34.307821+0000","last_clean":"2024-04-12T11:31:34.307821+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:31:34.307821+0000","last_undegraded":"2024-04-12T11:31:34.307821+0000","last_fullsized":"2024-04-12T11:31:34.307821+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1563,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330389+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:31:34.330389+0000","last_peered":"2024-04-12T11:31:34.330389+0000","last_clean":"2024-04-12T11:31:34.330389+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:31:34.330389+0000","last_undegraded":"2024-04-12T11:31:34.330389+0000","last_fullsized":"2024-04-12T11:31:34.330389+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806115+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:31:33.806115+0000","last_peered":"2024-04-12T11:31:33.806115+0000","last_clean":"2024-04-12T11:31:33.806115+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:31:33.806115+0000","last_undegraded":"2024-04-12T11:31:33.806115+0000","last_fullsized":"2024-04-12T11:31:33.806115+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307609+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:31:34.307609+0000","last_peered":"2024-04-12T11:31:34.307609+0000","last_clean":"2024-04-12T11:31:34.307609+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:31:34.307609+0000","last_undegraded":"2024-04-12T11:31:34.307609+0000","last_fullsized":"2024-04-12T11:31:34.307609+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1499,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804727+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:31:33.804727+0000","last_peered":"2024-04-12T11:31:33.804727+0000","last_clean":"2024-04-12T11:31:33.804727+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:31:33.804727+0000","last_undegraded":"2024-04-12T11:31:33.804727+0000","last_fullsized":"2024-04-12T11:31:33.804727+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.098490+0000","last_change":"2024-04-12T11:22:40.580200+0000","last_active":"2024-04-12T11:31:33.098490+0000","last_peered":"2024-04-12T11:31:33.098490+0000","last_clean":"2024-04-12T11:31:33.098490+0000","last_became_active":"2024-04-12T11:22:40.579809+0000","last_became_peered":"2024-04-12T11:22:40.579809+0000","last_unstale":"2024-04-12T11:31:33.098490+0000","last_undegraded":"2024-04-12T11:31:33.098490+0000","last_fullsized":"2024-04-12T11:31:33.098490+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:28:23.317249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1582,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329989+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:31:34.329989+0000","last_peered":"2024-04-12T11:31:34.329989+0000","last_clean":"2024-04-12T11:31:34.329989+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:31:34.329989+0000","last_undegraded":"2024-04-12T11:31:34.329989+0000","last_fullsized":"2024-04-12T11:31:34.329989+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1489,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805172+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:31:33.805172+0000","last_peered":"2024-04-12T11:31:33.805172+0000","last_clean":"2024-04-12T11:31:33.805172+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:31:33.805172+0000","last_undegraded":"2024-04-12T11:31:33.805172+0000","last_fullsized":"2024-04-12T11:31:33.805172+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1573,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330291+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:31:34.330291+0000","last_peered":"2024-04-12T11:31:34.330291+0000","last_clean":"2024-04-12T11:31:34.330291+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:31:34.330291+0000","last_undegraded":"2024-04-12T11:31:34.330291+0000","last_fullsized":"2024-04-12T11:31:34.330291+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1553,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961201+0000","last_change":"2024-04-12T11:23:15.593766+0000","last_active":"2024-04-12T11:31:34.961201+0000","last_peered":"2024-04-12T11:31:34.961201+0000","last_clean":"2024-04-12T11:31:34.961201+0000","last_became_active":"2024-04-12T11:23:15.593428+0000","last_became_peered":"2024-04-12T11:23:15.593428+0000","last_unstale":"2024-04-12T11:31:34.961201+0000","last_undegraded":"2024-04-12T11:31:34.961201+0000","last_fullsized":"2024-04-12T11:31:34.961201+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:12.389667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1047,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488797+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:31:33.488797+0000","last_peered":"2024-04-12T11:31:33.488797+0000","last_clean":"2024-04-12T11:31:33.488797+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:31:33.488797+0000","last_undegraded":"2024-04-12T11:31:33.488797+0000","last_fullsized":"2024-04-12T11:31:33.488797+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806362+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:31:33.806362+0000","last_peered":"2024-04-12T11:31:33.806362+0000","last_clean":"2024-04-12T11:31:33.806362+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:31:33.806362+0000","last_undegraded":"2024-04-12T11:31:33.806362+0000","last_fullsized":"2024-04-12T11:31:33.806362+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1032,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.489264+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:31:33.489264+0000","last_peered":"2024-04-12T11:31:33.489264+0000","last_clean":"2024-04-12T11:31:33.489264+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:31:33.489264+0000","last_undegraded":"2024-04-12T11:31:33.489264+0000","last_fullsized":"2024-04-12T11:31:33.489264+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1508,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805189+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:31:33.805189+0000","last_peered":"2024-04-12T11:31:33.805189+0000","last_clean":"2024-04-12T11:31:33.805189+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:31:33.805189+0000","last_undegraded":"2024-04-12T11:31:33.805189+0000","last_fullsized":"2024-04-12T11:31:33.805189+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1500,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805844+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:31:33.805844+0000","last_peered":"2024-04-12T11:31:33.805844+0000","last_clean":"2024-04-12T11:31:33.805844+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:31:33.805844+0000","last_undegraded":"2024-04-12T11:31:33.805844+0000","last_fullsized":"2024-04-12T11:31:33.805844+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1511,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804851+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:31:33.804851+0000","last_peered":"2024-04-12T11:31:33.804851+0000","last_clean":"2024-04-12T11:31:33.804851+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:31:33.804851+0000","last_undegraded":"2024-04-12T11:31:33.804851+0000","last_fullsized":"2024-04-12T11:31:33.804851+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329503+0000","last_change":"2024-04-12T11:22:40.582361+0000","last_active":"2024-04-12T11:31:34.329503+0000","last_peered":"2024-04-12T11:31:34.329503+0000","last_clean":"2024-04-12T11:31:34.329503+0000","last_became_active":"2024-04-12T11:22:40.581996+0000","last_became_peered":"2024-04-12T11:22:40.581996+0000","last_unstale":"2024-04-12T11:31:34.329503+0000","last_undegraded":"2024-04-12T11:31:34.329503+0000","last_fullsized":"2024-04-12T11:31:34.329503+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:40:43.563258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805035+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:31:33.805035+0000","last_peered":"2024-04-12T11:31:33.805035+0000","last_clean":"2024-04-12T11:31:33.805035+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:31:33.805035+0000","last_undegraded":"2024-04-12T11:31:33.805035+0000","last_fullsized":"2024-04-12T11:31:33.805035+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234241,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":58052,"kb_used_data":8888,"kb_used_omap":1,"kb_used_meta":49150,"kb_avail":93687100,"statfs":{"total":95995035648,"available":95935590400,"internally_reserved":0,"allocated":9101312,"data_stored":6335894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":50330059},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495878,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":66228,"kb_used_data":8876,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93678924,"statfs":{"total":95995035648,"available":95927218176,"internally_reserved":0,"allocated":9089024,"data_stored":6335748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":58718664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536188,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36068,"kb_used_data":8280,"kb_used_omap":30,"kb_used_meta":27745,"kb_avail":93709084,"statfs":{"total":95995035648,"available":95958102016,"internally_reserved":0,"allocated":8478720,"data_stored":5740566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":28411202},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190272,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67316,"kb_used_data":8868,"kb_used_omap":43,"kb_used_meta":58388,"kb_avail":93677836,"statfs":{"total":95995035648,"available":95926104064,"internally_reserved":0,"allocated":9080832,"data_stored":6331538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":59789355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849400,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":62576,"kb_used_data":8288,"kb_used_omap":36,"kb_used_meta":54235,"kb_avail":93682576,"statfs":{"total":95995035648,"available":95930957824,"internally_reserved":0,"allocated":8486912,"data_stored":5741182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":55536820},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":44204,"kb_used_data":8284,"kb_used_omap":17,"kb_used_meta":35886,"kb_avail":93700948,"statfs":{"total":95995035648,"available":95949770752,"internally_reserved":0,"allocated":8482816,"data_stored":5740656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":36747767},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225017965,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40448,"kb_used_data":8304,"kb_used_omap":42,"kb_used_meta":32085,"kb_avail":93704704,"statfs":{"total":95995035648,"available":95953616896,"internally_reserved":0,"allocated":8503296,"data_stored":5745468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":32855864},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757514,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":97584,"kb_used_data":8288,"kb_used_omap":1,"kb_used_meta":89278,"kb_avail":93647568,"statfs":{"total":95995035648,"available":95895109632,"internally_reserved":0,"allocated":8486912,"data_stored":5741206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":91421126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:32:32.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:32:32.989 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:33.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:32 smithi029 ceph-mon[97787]: pgmap v1294: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:33.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:32 smithi029 ceph-mon[97787]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:33.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:32 smithi029 ceph-mon[93650]: pgmap v1294: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:33.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:32 smithi029 ceph-mon[93650]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:32 smithi087 ceph-mon[89014]: pgmap v1294: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:32 smithi087 ceph-mon[89014]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:34.840 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:32:34.841 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:32:35.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:34 smithi029 ceph-mon[97787]: pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:35.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:34 smithi029 ceph-mon[93650]: pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:35.159 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:32:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:32:34] "GET /metrics HTTP/1.1" 200 34004 "" "Prometheus/2.43.0" 2024-04-12T11:32:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:34 smithi087 ceph-mon[89014]: pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:35.490 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1295,"stamp":"2024-04-12T11:32:33.640757+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":472476,"kb_used_data":68076,"kb_used_omap":175,"kb_used_meta":404112,"kb_avail":749488740,"statfs":{"total":767960285184,"available":767476469760,"internally_reserved":0,"allocated":69709824,"data_stored":47712258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":413810857},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002526"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807039+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:31:33.807039+0000","last_peered":"2024-04-12T11:31:33.807039+0000","last_clean":"2024-04-12T11:31:33.807039+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:31:33.807039+0000","last_undegraded":"2024-04-12T11:31:33.807039+0000","last_fullsized":"2024-04-12T11:31:33.807039+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1525,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805863+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:31:33.805863+0000","last_peered":"2024-04-12T11:31:33.805863+0000","last_clean":"2024-04-12T11:31:33.805863+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:31:33.805863+0000","last_undegraded":"2024-04-12T11:31:33.805863+0000","last_fullsized":"2024-04-12T11:31:33.805863+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307246+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:31:34.307246+0000","last_peered":"2024-04-12T11:31:34.307246+0000","last_clean":"2024-04-12T11:31:34.307246+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:31:34.307246+0000","last_undegraded":"2024-04-12T11:31:34.307246+0000","last_fullsized":"2024-04-12T11:31:34.307246+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1572,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329931+0000","last_change":"2024-04-12T11:23:15.593723+0000","last_active":"2024-04-12T11:31:34.329931+0000","last_peered":"2024-04-12T11:31:34.329931+0000","last_clean":"2024-04-12T11:31:34.329931+0000","last_became_active":"2024-04-12T11:23:15.593381+0000","last_became_peered":"2024-04-12T11:23:15.593381+0000","last_unstale":"2024-04-12T11:31:34.329931+0000","last_undegraded":"2024-04-12T11:31:34.329931+0000","last_fullsized":"2024-04-12T11:31:34.329931+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:18:06.971493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1533,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804979+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:31:33.804979+0000","last_peered":"2024-04-12T11:31:33.804979+0000","last_clean":"2024-04-12T11:31:33.804979+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:31:33.804979+0000","last_undegraded":"2024-04-12T11:31:33.804979+0000","last_fullsized":"2024-04-12T11:31:33.804979+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.963036+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:31:34.963036+0000","last_peered":"2024-04-12T11:31:34.963036+0000","last_clean":"2024-04-12T11:31:34.963036+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:31:34.963036+0000","last_undegraded":"2024-04-12T11:31:34.963036+0000","last_fullsized":"2024-04-12T11:31:34.963036+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1568,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330398+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:31:34.330398+0000","last_peered":"2024-04-12T11:31:34.330398+0000","last_clean":"2024-04-12T11:31:34.330398+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:31:34.330398+0000","last_undegraded":"2024-04-12T11:31:34.330398+0000","last_fullsized":"2024-04-12T11:31:34.330398+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1506,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804876+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:31:33.804876+0000","last_peered":"2024-04-12T11:31:33.804876+0000","last_clean":"2024-04-12T11:31:33.804876+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:31:33.804876+0000","last_undegraded":"2024-04-12T11:31:33.804876+0000","last_fullsized":"2024-04-12T11:31:33.804876+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1031,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806752+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:31:33.806752+0000","last_peered":"2024-04-12T11:31:33.806752+0000","last_clean":"2024-04-12T11:31:33.806752+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:31:33.806752+0000","last_undegraded":"2024-04-12T11:31:33.806752+0000","last_fullsized":"2024-04-12T11:31:33.806752+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1486,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804993+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:31:33.804993+0000","last_peered":"2024-04-12T11:31:33.804993+0000","last_clean":"2024-04-12T11:31:33.804993+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:31:33.804993+0000","last_undegraded":"2024-04-12T11:31:33.804993+0000","last_fullsized":"2024-04-12T11:31:33.804993+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1040,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.963138+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:31:34.963138+0000","last_peered":"2024-04-12T11:31:34.963138+0000","last_clean":"2024-04-12T11:31:34.963138+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:31:34.963138+0000","last_undegraded":"2024-04-12T11:31:34.963138+0000","last_fullsized":"2024-04-12T11:31:34.963138+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1568,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307302+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:31:34.307302+0000","last_peered":"2024-04-12T11:31:34.307302+0000","last_clean":"2024-04-12T11:31:34.307302+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:31:34.307302+0000","last_undegraded":"2024-04-12T11:31:34.307302+0000","last_fullsized":"2024-04-12T11:31:34.307302+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1489,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806028+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:31:33.806028+0000","last_peered":"2024-04-12T11:31:33.806028+0000","last_clean":"2024-04-12T11:31:33.806028+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:31:33.806028+0000","last_undegraded":"2024-04-12T11:31:33.806028+0000","last_fullsized":"2024-04-12T11:31:33.806028+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806164+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:31:33.806164+0000","last_peered":"2024-04-12T11:31:33.806164+0000","last_clean":"2024-04-12T11:31:33.806164+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:31:33.806164+0000","last_undegraded":"2024-04-12T11:31:33.806164+0000","last_fullsized":"2024-04-12T11:31:33.806164+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1039,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962053+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:31:34.962053+0000","last_peered":"2024-04-12T11:31:34.962053+0000","last_clean":"2024-04-12T11:31:34.962053+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:31:34.962053+0000","last_undegraded":"2024-04-12T11:31:34.962053+0000","last_fullsized":"2024-04-12T11:31:34.962053+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805796+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:31:33.805796+0000","last_peered":"2024-04-12T11:31:33.805796+0000","last_clean":"2024-04-12T11:31:33.805796+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:31:33.805796+0000","last_undegraded":"2024-04-12T11:31:33.805796+0000","last_fullsized":"2024-04-12T11:31:33.805796+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329105+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:31:34.329105+0000","last_peered":"2024-04-12T11:31:34.329105+0000","last_clean":"2024-04-12T11:31:34.329105+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:31:34.329105+0000","last_undegraded":"2024-04-12T11:31:34.329105+0000","last_fullsized":"2024-04-12T11:31:34.329105+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806888+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:31:33.806888+0000","last_peered":"2024-04-12T11:31:33.806888+0000","last_clean":"2024-04-12T11:31:33.806888+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:31:33.806888+0000","last_undegraded":"2024-04-12T11:31:33.806888+0000","last_fullsized":"2024-04-12T11:31:33.806888+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962176+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:31:34.962176+0000","last_peered":"2024-04-12T11:31:34.962176+0000","last_clean":"2024-04-12T11:31:34.962176+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:31:34.962176+0000","last_undegraded":"2024-04-12T11:31:34.962176+0000","last_fullsized":"2024-04-12T11:31:34.962176+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1570,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329830+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:31:34.329830+0000","last_peered":"2024-04-12T11:31:34.329830+0000","last_clean":"2024-04-12T11:31:34.329830+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:31:34.329830+0000","last_undegraded":"2024-04-12T11:31:34.329830+0000","last_fullsized":"2024-04-12T11:31:34.329830+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1033,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961852+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:31:34.961852+0000","last_peered":"2024-04-12T11:31:34.961852+0000","last_clean":"2024-04-12T11:31:34.961852+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:31:34.961852+0000","last_undegraded":"2024-04-12T11:31:34.961852+0000","last_fullsized":"2024-04-12T11:31:34.961852+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805881+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:31:33.805881+0000","last_peered":"2024-04-12T11:31:33.805881+0000","last_clean":"2024-04-12T11:31:33.805881+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:31:33.805881+0000","last_undegraded":"2024-04-12T11:31:33.805881+0000","last_fullsized":"2024-04-12T11:31:33.805881+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027548+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:31:33.027548+0000","last_peered":"2024-04-12T11:31:33.027548+0000","last_clean":"2024-04-12T11:31:33.027548+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:31:33.027548+0000","last_undegraded":"2024-04-12T11:31:33.027548+0000","last_fullsized":"2024-04-12T11:31:33.027548+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1016,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805263+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:31:33.805263+0000","last_peered":"2024-04-12T11:31:33.805263+0000","last_clean":"2024-04-12T11:31:33.805263+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:31:33.805263+0000","last_undegraded":"2024-04-12T11:31:33.805263+0000","last_fullsized":"2024-04-12T11:31:33.805263+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027317+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:31:33.027317+0000","last_peered":"2024-04-12T11:31:33.027317+0000","last_clean":"2024-04-12T11:31:33.027317+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:31:33.027317+0000","last_undegraded":"2024-04-12T11:31:33.027317+0000","last_fullsized":"2024-04-12T11:31:33.027317+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1503,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806074+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:31:33.806074+0000","last_peered":"2024-04-12T11:31:33.806074+0000","last_clean":"2024-04-12T11:31:33.806074+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:31:33.806074+0000","last_undegraded":"2024-04-12T11:31:33.806074+0000","last_fullsized":"2024-04-12T11:31:33.806074+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027231+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:31:33.027231+0000","last_peered":"2024-04-12T11:31:33.027231+0000","last_clean":"2024-04-12T11:31:33.027231+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:31:33.027231+0000","last_undegraded":"2024-04-12T11:31:33.027231+0000","last_fullsized":"2024-04-12T11:31:33.027231+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.963327+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:31:34.963327+0000","last_peered":"2024-04-12T11:31:34.963327+0000","last_clean":"2024-04-12T11:31:34.963327+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:31:34.963327+0000","last_undegraded":"2024-04-12T11:31:34.963327+0000","last_fullsized":"2024-04-12T11:31:34.963327+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488787+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:31:33.488787+0000","last_peered":"2024-04-12T11:31:33.488787+0000","last_clean":"2024-04-12T11:31:33.488787+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:31:33.488787+0000","last_undegraded":"2024-04-12T11:31:33.488787+0000","last_fullsized":"2024-04-12T11:31:33.488787+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1579,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329258+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:31:34.329258+0000","last_peered":"2024-04-12T11:31:34.329258+0000","last_clean":"2024-04-12T11:31:34.329258+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:31:34.329258+0000","last_undegraded":"2024-04-12T11:31:34.329258+0000","last_fullsized":"2024-04-12T11:31:34.329258+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1511,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805669+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:31:33.805669+0000","last_peered":"2024-04-12T11:31:33.805669+0000","last_clean":"2024-04-12T11:31:33.805669+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:31:33.805669+0000","last_undegraded":"2024-04-12T11:31:33.805669+0000","last_fullsized":"2024-04-12T11:31:33.805669+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1493,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806698+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:31:33.806698+0000","last_peered":"2024-04-12T11:31:33.806698+0000","last_clean":"2024-04-12T11:31:33.806698+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:31:33.806698+0000","last_undegraded":"2024-04-12T11:31:33.806698+0000","last_fullsized":"2024-04-12T11:31:33.806698+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1041,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962841+0000","last_change":"2024-04-12T11:23:36.639270+0000","last_active":"2024-04-12T11:31:34.962841+0000","last_peered":"2024-04-12T11:31:34.962841+0000","last_clean":"2024-04-12T11:31:34.962841+0000","last_became_active":"2024-04-12T11:23:36.638952+0000","last_became_peered":"2024-04-12T11:23:36.638952+0000","last_unstale":"2024-04-12T11:31:34.962841+0000","last_undegraded":"2024-04-12T11:31:34.962841+0000","last_fullsized":"2024-04-12T11:31:34.962841+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:04:50.403812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1515,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.809196+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:31:33.809196+0000","last_peered":"2024-04-12T11:31:33.809196+0000","last_clean":"2024-04-12T11:31:33.809196+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:31:33.809196+0000","last_undegraded":"2024-04-12T11:31:33.809196+0000","last_fullsized":"2024-04-12T11:31:33.809196+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806193+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:31:33.806193+0000","last_peered":"2024-04-12T11:31:33.806193+0000","last_clean":"2024-04-12T11:31:33.806193+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:31:33.806193+0000","last_undegraded":"2024-04-12T11:31:33.806193+0000","last_fullsized":"2024-04-12T11:31:33.806193+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807330+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:31:33.807330+0000","last_peered":"2024-04-12T11:31:33.807330+0000","last_clean":"2024-04-12T11:31:33.807330+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:31:33.807330+0000","last_undegraded":"2024-04-12T11:31:33.807330+0000","last_fullsized":"2024-04-12T11:31:33.807330+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1519,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804798+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:31:33.804798+0000","last_peered":"2024-04-12T11:31:33.804798+0000","last_clean":"2024-04-12T11:31:33.804798+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:31:33.804798+0000","last_undegraded":"2024-04-12T11:31:33.804798+0000","last_fullsized":"2024-04-12T11:31:33.804798+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307677+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:31:34.307677+0000","last_peered":"2024-04-12T11:31:34.307677+0000","last_clean":"2024-04-12T11:31:34.307677+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:31:34.307677+0000","last_undegraded":"2024-04-12T11:31:34.307677+0000","last_fullsized":"2024-04-12T11:31:34.307677+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1578,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.308030+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:31:34.308030+0000","last_peered":"2024-04-12T11:31:34.308030+0000","last_clean":"2024-04-12T11:31:34.308030+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:31:34.308030+0000","last_undegraded":"2024-04-12T11:31:34.308030+0000","last_fullsized":"2024-04-12T11:31:34.308030+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1047,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.807269+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:31:33.807269+0000","last_peered":"2024-04-12T11:31:33.807269+0000","last_clean":"2024-04-12T11:31:33.807269+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:31:33.807269+0000","last_undegraded":"2024-04-12T11:31:33.807269+0000","last_fullsized":"2024-04-12T11:31:33.807269+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307257+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:31:34.307257+0000","last_peered":"2024-04-12T11:31:34.307257+0000","last_clean":"2024-04-12T11:31:34.307257+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:31:34.307257+0000","last_undegraded":"2024-04-12T11:31:34.307257+0000","last_fullsized":"2024-04-12T11:31:34.307257+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488693+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:31:33.488693+0000","last_peered":"2024-04-12T11:31:33.488693+0000","last_clean":"2024-04-12T11:31:33.488693+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:31:33.488693+0000","last_undegraded":"2024-04-12T11:31:33.488693+0000","last_fullsized":"2024-04-12T11:31:33.488693+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806006+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:31:33.806006+0000","last_peered":"2024-04-12T11:31:33.806006+0000","last_clean":"2024-04-12T11:31:33.806006+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:31:33.806006+0000","last_undegraded":"2024-04-12T11:31:33.806006+0000","last_fullsized":"2024-04-12T11:31:33.806006+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1578,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307348+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:31:34.307348+0000","last_peered":"2024-04-12T11:31:34.307348+0000","last_clean":"2024-04-12T11:31:34.307348+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:31:34.307348+0000","last_undegraded":"2024-04-12T11:31:34.307348+0000","last_fullsized":"2024-04-12T11:31:34.307348+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1542,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961088+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:31:34.961088+0000","last_peered":"2024-04-12T11:31:34.961088+0000","last_clean":"2024-04-12T11:31:34.961088+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:31:34.961088+0000","last_undegraded":"2024-04-12T11:31:34.961088+0000","last_fullsized":"2024-04-12T11:31:34.961088+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1570,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307824+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:31:34.307824+0000","last_peered":"2024-04-12T11:31:34.307824+0000","last_clean":"2024-04-12T11:31:34.307824+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:31:34.307824+0000","last_undegraded":"2024-04-12T11:31:34.307824+0000","last_fullsized":"2024-04-12T11:31:34.307824+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1572,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307812+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:31:34.307812+0000","last_peered":"2024-04-12T11:31:34.307812+0000","last_clean":"2024-04-12T11:31:34.307812+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:31:34.307812+0000","last_undegraded":"2024-04-12T11:31:34.307812+0000","last_fullsized":"2024-04-12T11:31:34.307812+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488902+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:31:33.488902+0000","last_peered":"2024-04-12T11:31:33.488902+0000","last_clean":"2024-04-12T11:31:33.488902+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:31:33.488902+0000","last_undegraded":"2024-04-12T11:31:33.488902+0000","last_fullsized":"2024-04-12T11:31:33.488902+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1391,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.026548+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:31:33.026548+0000","last_peered":"2024-04-12T11:31:33.026548+0000","last_clean":"2024-04-12T11:31:33.026548+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:31:33.026548+0000","last_undegraded":"2024-04-12T11:31:33.026548+0000","last_fullsized":"2024-04-12T11:31:33.026548+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1486,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804959+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:31:33.804959+0000","last_peered":"2024-04-12T11:31:33.804959+0000","last_clean":"2024-04-12T11:31:33.804959+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:31:33.804959+0000","last_undegraded":"2024-04-12T11:31:33.804959+0000","last_fullsized":"2024-04-12T11:31:33.804959+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.962347+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:31:34.962347+0000","last_peered":"2024-04-12T11:31:34.962347+0000","last_clean":"2024-04-12T11:31:34.962347+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:31:34.962347+0000","last_undegraded":"2024-04-12T11:31:34.962347+0000","last_fullsized":"2024-04-12T11:31:34.962347+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1044,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961718+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:31:34.961718+0000","last_peered":"2024-04-12T11:31:34.961718+0000","last_clean":"2024-04-12T11:31:34.961718+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:31:34.961718+0000","last_undegraded":"2024-04-12T11:31:34.961718+0000","last_fullsized":"2024-04-12T11:31:34.961718+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1580,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329112+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:31:34.329112+0000","last_peered":"2024-04-12T11:31:34.329112+0000","last_clean":"2024-04-12T11:31:34.329112+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:31:34.329112+0000","last_undegraded":"2024-04-12T11:31:34.329112+0000","last_fullsized":"2024-04-12T11:31:34.329112+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1579,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329333+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:31:34.329333+0000","last_peered":"2024-04-12T11:31:34.329333+0000","last_clean":"2024-04-12T11:31:34.329333+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:31:34.329333+0000","last_undegraded":"2024-04-12T11:31:34.329333+0000","last_fullsized":"2024-04-12T11:31:34.329333+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1043,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.027168+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:31:33.027168+0000","last_peered":"2024-04-12T11:31:33.027168+0000","last_clean":"2024-04-12T11:31:33.027168+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:31:33.027168+0000","last_undegraded":"2024-04-12T11:31:33.027168+0000","last_fullsized":"2024-04-12T11:31:33.027168+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307821+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:31:34.307821+0000","last_peered":"2024-04-12T11:31:34.307821+0000","last_clean":"2024-04-12T11:31:34.307821+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:31:34.307821+0000","last_undegraded":"2024-04-12T11:31:34.307821+0000","last_fullsized":"2024-04-12T11:31:34.307821+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1563,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330389+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:31:34.330389+0000","last_peered":"2024-04-12T11:31:34.330389+0000","last_clean":"2024-04-12T11:31:34.330389+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:31:34.330389+0000","last_undegraded":"2024-04-12T11:31:34.330389+0000","last_fullsized":"2024-04-12T11:31:34.330389+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1502,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806115+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:31:33.806115+0000","last_peered":"2024-04-12T11:31:33.806115+0000","last_clean":"2024-04-12T11:31:33.806115+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:31:33.806115+0000","last_undegraded":"2024-04-12T11:31:33.806115+0000","last_fullsized":"2024-04-12T11:31:33.806115+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1569,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.307609+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:31:34.307609+0000","last_peered":"2024-04-12T11:31:34.307609+0000","last_clean":"2024-04-12T11:31:34.307609+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:31:34.307609+0000","last_undegraded":"2024-04-12T11:31:34.307609+0000","last_fullsized":"2024-04-12T11:31:34.307609+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1499,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804727+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:31:33.804727+0000","last_peered":"2024-04-12T11:31:33.804727+0000","last_clean":"2024-04-12T11:31:33.804727+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:31:33.804727+0000","last_undegraded":"2024-04-12T11:31:33.804727+0000","last_fullsized":"2024-04-12T11:31:33.804727+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.098490+0000","last_change":"2024-04-12T11:22:40.580200+0000","last_active":"2024-04-12T11:31:33.098490+0000","last_peered":"2024-04-12T11:31:33.098490+0000","last_clean":"2024-04-12T11:31:33.098490+0000","last_became_active":"2024-04-12T11:22:40.579809+0000","last_became_peered":"2024-04-12T11:22:40.579809+0000","last_unstale":"2024-04-12T11:31:33.098490+0000","last_undegraded":"2024-04-12T11:31:33.098490+0000","last_fullsized":"2024-04-12T11:31:33.098490+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:28:23.317249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1582,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329989+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:31:34.329989+0000","last_peered":"2024-04-12T11:31:34.329989+0000","last_clean":"2024-04-12T11:31:34.329989+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:31:34.329989+0000","last_undegraded":"2024-04-12T11:31:34.329989+0000","last_fullsized":"2024-04-12T11:31:34.329989+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1489,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805172+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:31:33.805172+0000","last_peered":"2024-04-12T11:31:33.805172+0000","last_clean":"2024-04-12T11:31:33.805172+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:31:33.805172+0000","last_undegraded":"2024-04-12T11:31:33.805172+0000","last_fullsized":"2024-04-12T11:31:33.805172+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1573,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.330291+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:31:34.330291+0000","last_peered":"2024-04-12T11:31:34.330291+0000","last_clean":"2024-04-12T11:31:34.330291+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:31:34.330291+0000","last_undegraded":"2024-04-12T11:31:34.330291+0000","last_fullsized":"2024-04-12T11:31:34.330291+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1553,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.961201+0000","last_change":"2024-04-12T11:23:15.593766+0000","last_active":"2024-04-12T11:31:34.961201+0000","last_peered":"2024-04-12T11:31:34.961201+0000","last_clean":"2024-04-12T11:31:34.961201+0000","last_became_active":"2024-04-12T11:23:15.593428+0000","last_became_peered":"2024-04-12T11:23:15.593428+0000","last_unstale":"2024-04-12T11:31:34.961201+0000","last_undegraded":"2024-04-12T11:31:34.961201+0000","last_fullsized":"2024-04-12T11:31:34.961201+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:12.389667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1047,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.488797+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:31:33.488797+0000","last_peered":"2024-04-12T11:31:33.488797+0000","last_clean":"2024-04-12T11:31:33.488797+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:31:33.488797+0000","last_undegraded":"2024-04-12T11:31:33.488797+0000","last_fullsized":"2024-04-12T11:31:33.488797+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1517,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.806362+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:31:33.806362+0000","last_peered":"2024-04-12T11:31:33.806362+0000","last_clean":"2024-04-12T11:31:33.806362+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:31:33.806362+0000","last_undegraded":"2024-04-12T11:31:33.806362+0000","last_fullsized":"2024-04-12T11:31:33.806362+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1032,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.489264+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:31:33.489264+0000","last_peered":"2024-04-12T11:31:33.489264+0000","last_clean":"2024-04-12T11:31:33.489264+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:31:33.489264+0000","last_undegraded":"2024-04-12T11:31:33.489264+0000","last_fullsized":"2024-04-12T11:31:33.489264+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1508,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805189+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:31:33.805189+0000","last_peered":"2024-04-12T11:31:33.805189+0000","last_clean":"2024-04-12T11:31:33.805189+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:31:33.805189+0000","last_undegraded":"2024-04-12T11:31:33.805189+0000","last_fullsized":"2024-04-12T11:31:33.805189+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1500,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805844+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:31:33.805844+0000","last_peered":"2024-04-12T11:31:33.805844+0000","last_clean":"2024-04-12T11:31:33.805844+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:31:33.805844+0000","last_undegraded":"2024-04-12T11:31:33.805844+0000","last_fullsized":"2024-04-12T11:31:33.805844+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1511,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.804851+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:31:33.804851+0000","last_peered":"2024-04-12T11:31:33.804851+0000","last_clean":"2024-04-12T11:31:33.804851+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:31:33.804851+0000","last_undegraded":"2024-04-12T11:31:33.804851+0000","last_fullsized":"2024-04-12T11:31:33.804851+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1577,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:34.329503+0000","last_change":"2024-04-12T11:22:40.582361+0000","last_active":"2024-04-12T11:31:34.329503+0000","last_peered":"2024-04-12T11:31:34.329503+0000","last_clean":"2024-04-12T11:31:34.329503+0000","last_became_active":"2024-04-12T11:22:40.581996+0000","last_became_peered":"2024-04-12T11:22:40.581996+0000","last_unstale":"2024-04-12T11:31:34.329503+0000","last_undegraded":"2024-04-12T11:31:34.329503+0000","last_fullsized":"2024-04-12T11:31:34.329503+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:40:43.563258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1480,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-04-12T11:31:33.805035+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:31:33.805035+0000","last_peered":"2024-04-12T11:31:33.805035+0000","last_clean":"2024-04-12T11:31:33.805035+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:31:33.805035+0000","last_undegraded":"2024-04-12T11:31:33.805035+0000","last_fullsized":"2024-04-12T11:31:33.805035+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234242,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":58052,"kb_used_data":8888,"kb_used_omap":1,"kb_used_meta":49150,"kb_avail":93687100,"statfs":{"total":95995035648,"available":95935590400,"internally_reserved":0,"allocated":9101312,"data_stored":6335894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":50330059},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495879,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":66228,"kb_used_data":8876,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93678924,"statfs":{"total":95995035648,"available":95927218176,"internally_reserved":0,"allocated":9089024,"data_stored":6335748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":58718664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536189,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36068,"kb_used_data":8280,"kb_used_omap":30,"kb_used_meta":27745,"kb_avail":93709084,"statfs":{"total":95995035648,"available":95958102016,"internally_reserved":0,"allocated":8478720,"data_stored":5740566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":28411202},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190273,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67316,"kb_used_data":8868,"kb_used_omap":43,"kb_used_meta":58388,"kb_avail":93677836,"statfs":{"total":95995035648,"available":95926104064,"internally_reserved":0,"allocated":9080832,"data_stored":6331538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":59789355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849400,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":62576,"kb_used_data":8288,"kb_used_omap":36,"kb_used_meta":54235,"kb_avail":93682576,"statfs":{"total":95995035648,"available":95930957824,"internally_reserved":0,"allocated":8486912,"data_stored":5741182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":55536820},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":44204,"kb_used_data":8284,"kb_used_omap":17,"kb_used_meta":35886,"kb_avail":93700948,"statfs":{"total":95995035648,"available":95949770752,"internally_reserved":0,"allocated":8482816,"data_stored":5740656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":36747767},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225017965,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":40448,"kb_used_data":8304,"kb_used_omap":42,"kb_used_meta":32085,"kb_avail":93704704,"statfs":{"total":95995035648,"available":95953616896,"internally_reserved":0,"allocated":8503296,"data_stored":5745468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":32855864},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757515,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":97584,"kb_used_data":8288,"kb_used_omap":1,"kb_used_meta":89278,"kb_avail":93647568,"statfs":{"total":95995035648,"available":95895109632,"internally_reserved":0,"allocated":8486912,"data_stored":5741206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":91421126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:32:35.493 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-12T11:32:35.493 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump 2024-04-12T11:32:35.703 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:36.160 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:35 smithi029 ceph-mon[97787]: from='client.34713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:36.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:35 smithi029 ceph-mon[93650]: from='client.34713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:36.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:35 smithi087 ceph-mon[89014]: from='client.34713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:32:37.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:36 smithi029 ceph-mon[93650]: pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:37.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:37.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:36 smithi029 ceph-mon[97787]: pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:37.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:36 smithi087 ceph-mon[89014]: pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:37.699 INFO:teuthology.orchestra.run.smithi029.stdout:epoch 1084 2024-04-12T11:32:37.699 INFO:teuthology.orchestra.run.smithi029.stdout:fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T11:32:37.699 INFO:teuthology.orchestra.run.smithi029.stdout:created 2024-04-12T11:06:06.410593+0000 2024-04-12T11:32:37.699 INFO:teuthology.orchestra.run.smithi029.stdout:modified 2024-04-12T11:31:33.801968+0000 2024-04-12T11:32:37.699 INFO:teuthology.orchestra.run.smithi029.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-04-12T11:32:37.699 INFO:teuthology.orchestra.run.smithi029.stdout:crush_version 23 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:full_ratio 0.95 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:backfillfull_ratio 0.9 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:nearfull_ratio 0.85 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:require_min_compat_client luminous 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:min_compat_client luminous 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:require_osd_release reef 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:stretch_mode_enabled false 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 62 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 67 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:max_osd 8 2024-04-12T11:32:37.700 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 up in weight 1 up_from 514 up_thru 611 down_at 504 last_clean_interval [8,503) [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946] [v2:172.21.15.29:6804/3899727946,v1:172.21.15.29:6805/3899727946] exists,up 4c3c7583-fa1d-4a1d-95ea-77be7e31737e 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 up in weight 1 up_from 534 up_thru 611 down_at 524 last_clean_interval [13,523) [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505] [v2:172.21.15.29:6812/840649505,v1:172.21.15.29:6813/840649505] exists,up 58da0c82-e2ec-4067-82d5-05c379001ae5 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 up in weight 1 up_from 555 up_thru 611 down_at 545 last_clean_interval [18,544) [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687] [v2:172.21.15.29:6820/840062687,v1:172.21.15.29:6821/840062687] exists,up 53b6c920-2e05-4905-be45-0f33131d358d 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 up in weight 1 up_from 590 up_thru 611 down_at 580 last_clean_interval [25,579) [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308] [v2:172.21.15.29:6828/2609709308,v1:172.21.15.29:6829/2609709308] exists,up 5c33079c-a506-4d4f-97bc-11df1245e5ba 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 up in weight 1 up_from 611 up_thru 611 down_at 601 last_clean_interval [30,600) [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210] [v2:172.21.15.87:6802/3230425210,v1:172.21.15.87:6803/3230425210] exists,up f776683c-bddc-4a6f-a85d-a43639afadee 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 up in weight 1 up_from 38 up_thru 611 down_at 0 last_clean_interval [0,0) [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185] [v2:172.21.15.87:6810/3642701185,v1:172.21.15.87:6811/3642701185] exists,up ca436dff-e509-439d-a17b-0f4efe4d4693 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 up in weight 1 up_from 46 up_thru 611 down_at 0 last_clean_interval [0,0) [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062] [v2:172.21.15.87:6818/1395988062,v1:172.21.15.87:6819/1395988062] exists,up 3daf2021-0642-495d-803d-ad11f134a765 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 up in weight 1 up_from 54 up_thru 611 down_at 0 last_clean_interval [0,0) [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515] [v2:172.21.15.87:6826/2616284515,v1:172.21.15.87:6827/2616284515] exists,up 2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:pg_upmap_items 4.9 [1,5] 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:pg_upmap_items 4.10 [1,2] 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:pg_upmap_items 4.1b [1,2] 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/273190388 expires 2024-04-13T11:17:21.160240+0000 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/683850481 expires 2024-04-13T11:17:21.160240+0000 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/2957874151 expires 2024-04-13T11:07:09.398471+0000 2024-04-12T11:32:37.701 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6801/686523287 expires 2024-04-13T11:17:21.160240+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.87:6835/1005928369 expires 2024-04-13T11:16:36.316651+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.87:0/220307911 expires 2024-04-13T11:16:36.316651+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/659258741 expires 2024-04-13T11:11:58.108258+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/2463058069 expires 2024-04-13T11:16:01.391032+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6800/4219816068 expires 2024-04-13T11:07:09.398471+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.87:6834/1005928369 expires 2024-04-13T11:16:36.316651+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6801/4219816068 expires 2024-04-13T11:07:09.398471+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/3638696241 expires 2024-04-13T11:07:09.398471+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/3254157714 expires 2024-04-13T11:11:58.108258+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6801/3931254419 expires 2024-04-13T11:16:01.391032+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/4202867789 expires 2024-04-13T11:16:01.391032+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6800/848294730 expires 2024-04-13T11:06:44.508932+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.87:0/1543620546 expires 2024-04-13T11:16:36.316651+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/4278995423 expires 2024-04-13T11:06:44.508932+0000 2024-04-12T11:32:37.702 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6800/3931254419 expires 2024-04-13T11:16:01.391032+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.87:0/943640318 expires 2024-04-13T11:16:36.316651+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/4198974780 expires 2024-04-13T11:07:09.398471+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/1314515623 expires 2024-04-13T11:17:21.160240+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/1221327863 expires 2024-04-13T11:16:01.391032+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6801/848294730 expires 2024-04-13T11:06:44.508932+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6800/1422272459 expires 2024-04-13T11:11:58.108258+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.87:0/2414938593 expires 2024-04-13T11:16:36.316651+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6801/1422272459 expires 2024-04-13T11:11:58.108258+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/4218166463 expires 2024-04-13T11:17:21.160240+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.87:0/3681614798 expires 2024-04-13T11:16:36.316651+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/2946468060 expires 2024-04-13T11:11:58.108258+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/2511162340 expires 2024-04-13T11:06:44.508932+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/364858296 expires 2024-04-13T11:11:58.108258+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/3763274316 expires 2024-04-13T11:06:44.508932+0000 2024-04-12T11:32:37.703 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.87:0/67786467 expires 2024-04-13T11:16:36.316651+0000 2024-04-12T11:32:37.704 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/3128860153 expires 2024-04-13T11:16:01.391032+0000 2024-04-12T11:32:37.704 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/2241771384 expires 2024-04-13T11:17:21.160240+0000 2024-04-12T11:32:37.704 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/600676740 expires 2024-04-13T11:17:21.160240+0000 2024-04-12T11:32:37.704 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6800/686523287 expires 2024-04-13T11:17:21.160240+0000 2024-04-12T11:32:38.008 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:37 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/329023931' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-12T11:32:38.008 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:37 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/329023931' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-12T11:32:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:37 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/329023931' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-12T11:32:38.370 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 514 up_thru 611 down_at 504 last_clean_interval [8,503) [v2:172.21.15.29:6802/3899727946,v1:172.21.15.29:6803/3899727946] [v2:172.21.15.29:6804/3899727946,v1:172.21.15.29:6805/3899727946] exists,up 4c3c7583-fa1d-4a1d-95ea-77be7e31737e', 'osd.1 up in weight 1 up_from 534 up_thru 611 down_at 524 last_clean_interval [13,523) [v2:172.21.15.29:6810/840649505,v1:172.21.15.29:6811/840649505] [v2:172.21.15.29:6812/840649505,v1:172.21.15.29:6813/840649505] exists,up 58da0c82-e2ec-4067-82d5-05c379001ae5', 'osd.2 up in weight 1 up_from 555 up_thru 611 down_at 545 last_clean_interval [18,544) [v2:172.21.15.29:6818/840062687,v1:172.21.15.29:6819/840062687] [v2:172.21.15.29:6820/840062687,v1:172.21.15.29:6821/840062687] exists,up 53b6c920-2e05-4905-be45-0f33131d358d', 'osd.3 up in weight 1 up_from 590 up_thru 611 down_at 580 last_clean_interval [25,579) [v2:172.21.15.29:6826/2609709308,v1:172.21.15.29:6827/2609709308] [v2:172.21.15.29:6828/2609709308,v1:172.21.15.29:6829/2609709308] exists,up 5c33079c-a506-4d4f-97bc-11df1245e5ba', 'osd.4 up in weight 1 up_from 611 up_thru 611 down_at 601 last_clean_interval [30,600) [v2:172.21.15.87:6800/3230425210,v1:172.21.15.87:6801/3230425210] [v2:172.21.15.87:6802/3230425210,v1:172.21.15.87:6803/3230425210] exists,up f776683c-bddc-4a6f-a85d-a43639afadee', 'osd.5 up in weight 1 up_from 38 up_thru 611 down_at 0 last_clean_interval [0,0) [v2:172.21.15.87:6808/3642701185,v1:172.21.15.87:6809/3642701185] [v2:172.21.15.87:6810/3642701185,v1:172.21.15.87:6811/3642701185] exists,up ca436dff-e509-439d-a17b-0f4efe4d4693', 'osd.6 up in weight 1 up_from 46 up_thru 611 down_at 0 last_clean_interval [0,0) [v2:172.21.15.87:6816/1395988062,v1:172.21.15.87:6817/1395988062] [v2:172.21.15.87:6818/1395988062,v1:172.21.15.87:6819/1395988062] exists,up 3daf2021-0642-495d-803d-ad11f134a765', 'osd.7 up in weight 1 up_from 54 up_thru 611 down_at 0 last_clean_interval [0,0) [v2:172.21.15.87:6824/2616284515,v1:172.21.15.87:6825/2616284515] [v2:172.21.15.87:6826/2616284515,v1:172.21.15.87:6827/2616284515] exists,up 2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e'] 2024-04-12T11:32:38.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon mon.a config show 2024-04-12T11:32:38.580 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:38.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:38 smithi029 ceph-mon[93650]: pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:38.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:38 smithi029 ceph-mon[97787]: pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:38 smithi087 ceph-mon[89014]: pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "name": "mon.a", 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "cluster": "ceph", 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "admin_socket_mode": "", 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_client_required": "cephx, none", 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_cluster_required": "cephx", 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_debug": "true", 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-04-12T11:32:40.307 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_service_required": "cephx", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_service_ticket_ttl": "240.000000", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_supported": "", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_aio": "true", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_aio_max_queue_depth": "1024", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_aio_poll_ms": "250", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_aio_reap_max": "16", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_async_discard_threads": "0", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_block_size": "4096", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_debug_aio": "false", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_debug_inflight_ios": "false", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_enable_discard": "false", 2024-04-12T11:32:40.308 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_flock_retry": "3", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_flock_retry_interval": "0.100000", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_inject_crash": "0", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_inject_crash_flush_delay": "2", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_ioring": "false", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_ioring_hipri": "false", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_ioring_sqthread_poll": "false", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_read_buffer_alignment": "4096", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_type": "", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_alloc_size": "1048576", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_allocator": "hybrid", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_buffered_io": "true", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_check_for_zeros": "false", 2024-04-12T11:32:40.309 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_check_volume_selector_often": "false", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_compact_log_sync": "false", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_log_compact_min_size": "16777216", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_log_replay_check_allocations": "true", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_max_log_runway": "4194304", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_max_prefetch": "1048576", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_min_flush_size": "524288", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_min_log_runway": "1048576", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_replay_recovery": "false", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_shared_alloc_size": "65536", 2024-04-12T11:32:40.310 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_sync_write": "false", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_allocation_from_file": "true", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_allocator": "hybrid", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_blobid_prealloc": "10240", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_create": "true", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_db_create": "false", 2024-04-12T11:32:40.311 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_db_path": "", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_db_size": "0", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_path": "", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_preallocate_file": "false", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_size": "107374182400", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_wal_create": "false", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_wal_path": "", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_wal_size": "100663296", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bluefs": "true", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bluefs_env_mirror": "false", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-04-12T11:32:40.312 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_autotune": "true", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_size": "0", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_type": "2q", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-04-12T11:32:40.313 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_clone_cow": "true", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_algorithm": "snappy", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_max_blob_size": "0", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_min_blob_size": "0", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_mode": "none", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_csum_type": "crc32c", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_enforce_settings": "default", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_freelist": "false", 2024-04-12T11:32:40.314 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_fsck_abort": "false", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_inject_read_err": "false", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_legacy_omap": "false", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_prefill": "0.000000", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_small_allocations": "0", 2024-04-12T11:32:40.315 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_default_buffered_read": "true", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_default_buffered_write": "false", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_deferred_batch_ops": "0", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_elastic_shared_blobs": "true", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fail_eio": "false", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-04-12T11:32:40.316 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_mkfs": "true", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_mount": "false", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_umount": "false", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-04-12T11:32:40.317 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_ignore_data_csum": "false", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_kvbackend": "rocksdb", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_log_op_age": "5.000000", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_alloc_size": "0", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_blob_size": "0", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_defer_interval": "3.000000", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_deferred_txc": "32", 2024-04-12T11:32:40.318 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_min_alloc_size": "0", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_nid_prealloc": "1024", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_prefer_deferred_size": "0", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_qfsck_on_mount": "true", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_retry_disk_reads": "3", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_rocksdb_cf": "true", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_rocksdb_options_annex": "", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_spdk_coremask": "0x1", 2024-04-12T11:32:40.319 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_spdk_io_sleep": "5", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_spdk_max_io_completion": "0", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_spdk_mem": "512", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_sync_submit_transaction": "false", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_bytes": "67108864", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_cost_per_io": "0", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_tracing": "false", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_volume_selection_reserved": "0", 2024-04-12T11:32:40.320 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_zero_block_detection": "false", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_action_update_interval": "2", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_mount_timeout": "10", 2024-04-12T11:32:40.321 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_cluster_require_signatures": "false", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_cluster_require_version": "2", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_require_signatures": "false", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_require_version": "2", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_service_require_signatures": "false", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_service_require_version": "2", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_sign_messages": "true", 2024-04-12T11:32:40.322 INFO:teuthology.orchestra.run.smithi029.stdout: "chdir": "", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_acl_type": "", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_asio_thread_count": "2", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_cache_mid": "0.750000", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_cache_size": "16384", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_caps_release_delay": "5", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_check_pool_perm": "true", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_collect_and_send_global_metrics": "false", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_debug_force_sync_read": "false", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_debug_getattr_caps": "false", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_debug_inject_tick_delay": "0", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_die_on_failed_remount": "false", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_dirsize_rbytes": "true", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_force_lazyio": "false", 2024-04-12T11:32:40.323 INFO:teuthology.orchestra.run.smithi029.stdout: "client_fs": "", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_inject_fixed_oldest_tid": "false", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_inject_release_failure": "false", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_max_inline_size": "4096", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_max_retries_on_remount_failure": "5", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_mds_namespace": "", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_metadata": "", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_mount_gid": "-1", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_mount_timeout": "300", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_mount_uid": "-1", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_mountpoint": "/", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_notify_timeout": "10", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc": "true", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc_max_dirty": "104857600", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc_max_dirty_age": "5.000000", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc_max_objects": "1000", 2024-04-12T11:32:40.324 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc_size": "209715200", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc_target_dirty": "8388608", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_permissions": "true", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_quota": "true", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_quota_df": "true", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_readahead_max_bytes": "0", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_readahead_max_periods": "4", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_readahead_min": "131072", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_reconnect_stale": "false", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_shutdown_timeout": "30", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_snapdir": ".snap", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_tick_interval": "1", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_trace": "", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_try_dentry_invalidate": "false", 2024-04-12T11:32:40.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_use_faked_inos": "false", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "client_use_random_mds": "false", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_graylog": "false", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_graylog_port": "12201", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_monitors": "default=true", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_syslog": "false", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_syslog_level": "info", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "cluster_addr": "-", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "cluster_network": "", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "cluster_network_interface": "", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "compressor_zlib_isal": "false", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "compressor_zlib_level": "5", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "compressor_zlib_winsize": "-15", 2024-04-12T11:32:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "compressor_zstd_level": "1", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "crimson_alien_op_num_threads": "6", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "crimson_alien_thread_cpu_cores": "", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "crimson_osd_obc_lru_size": "512", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "crimson_seastar_cpu_cores": "", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "crush_location": "", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "crush_location_hook": "", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "crush_location_hook_timeout": "10", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "daemonize": "false", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "daos_pool": "tank", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-04-12T11:32:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "dbstore_db_name_prefix": "dbstore", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_alienstore": "0/5", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_allow_any_pool_priority": "false", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_asok": "1/5", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_asok_assert_abort": "false", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_asserts_on_shutdown": "true", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_auth": "1/5", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_bdev": "1/3", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_bluefs": "1/5", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_bluestore": "1/5", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_buffer": "0/1", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_ceph_exporter": "1/5", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_cephfs_mirror": "0/5", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_cephsqlite": "0/5", 2024-04-12T11:32:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_client": "0/5", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_compressor": "1/5", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_context": "0/1", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_crimson_interrupt": "0/5", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_crush": "1/1", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_crypto": "1/5", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_cyanstore": "0/5", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_deliberately_leak_memory": "false", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_disable_randomized_ping": "false", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_dpdk": "1/5", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_eventtrace": "1/5", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_filer": "0/1", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_filestore": "1/3", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_finisher": "1/1", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_fuse": "1/5", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_heartbeat_testing_span": "0", 2024-04-12T11:32:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_heartbeatmap": "1/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_immutable_obj_cache": "0/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_javaclient": "1/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_journal": "1/3", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_journaler": "0/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_kstore": "1/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_lockdep": "0/1", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mclock": "1/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds": "1/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_balancer": "1/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_locker": "1/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_log": "1/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_log_expire": "1/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_migrator": "1/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_quiesce": "3/5", 2024-04-12T11:32:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_memstore": "1/5", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mgr": "2/5", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mgrc": "1/5", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mon": "20/20", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_monc": "0/10", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_ms": "1/1", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_none": "0/5", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_objclass": "0/5", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_objectcacher": "0/5", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_objecter": "0/1", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_optracker": "0/5", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_osd": "1/5", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_paxos": "20/20", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_perfcounter": "1/5", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_prioritycache": "1/5", 2024-04-12T11:32:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rados": "0/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rbd": "0/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rbd_mirror": "0/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rbd_pwl": "0/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rbd_replay": "0/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_refs": "0/0", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_reserver": "1/1", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw": "1/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_access": "1/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_datacache": "1/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_dbstore": "1/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_flight": "1/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_lifecycle": "1/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_notification": "1/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_sync": "1/5", 2024-04-12T11:32:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rocksdb": "4/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_backref": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_cache": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_cleaner": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_device": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_epm": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_journal": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_lba": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_odata": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_omap": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_onode": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_t": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_tm": "0/5", 2024-04-12T11:32:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_striper": "0/1", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_test": "0/5", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_throttle": "1/1", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_timer": "0/1", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_tp": "0/5", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_trace": "1/5", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "device_failure_prediction_mode": "none", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "err_to_graylog": "false", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "err_to_journald": "false", 2024-04-12T11:32:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "err_to_stderr": "true", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "err_to_syslog": "false", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "event_tracing": "false", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_addr": "0.0.0.0", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_http_port": "9926", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_prio_limit": "5", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_sort_metrics": "true", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_stats_period": "5", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "fake_statfs_for_testing": "0", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "fatal_signal_handlers": "true", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "filer_max_purge_ops": "10", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "filer_max_truncate_ops": "128", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_apply_finisher_threads": "1", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_blackhole": "false", 2024-04-12T11:32:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_btrfs_clone_range": "true", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_btrfs_snap": "true", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_caller_concurrency": "10", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_collect_device_partition_information": "true", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_commit_timeout": "600.000000", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_debug_inject_read_err": "false", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_debug_omap_check": "false", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_debug_verify_split": "false", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_dump_file": "", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fadvise": "true", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fail_eio": "true", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fd_cache_shards": "16", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fd_cache_size": "128", 2024-04-12T11:32:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fiemap": "false", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fiemap_threshold": "4096", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_index_retry_probability": "0.000000", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_inject_stall": "0", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_journal_parallel": "false", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_journal_trailing": "false", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_journal_writeahead": "false", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_kill_at": "0", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattr_size": "0", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-04-12T11:32:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattrs": "0", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattrs_other": "2", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_sync_interval": "5.000000", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_xattr_value_size": "0", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_merge_threshold": "-10", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_min_sync_interval": "0.010000", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_odsync_write": "false", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_omap_backend": "rocksdb", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_omap_backend_path": "", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_omap_header_cache_size": "1024", 2024-04-12T11:32:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_ondisk_finisher_threads": "1", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_op_thread_timeout": "60", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_op_threads": "2", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_punch_hole": "false", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_max_bytes": "104857600", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_max_ops": "50", 2024-04-12T11:32:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_seek_data_hole": "false", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_sloppy_crc": "false", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_splice": "false", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_split_multiple": "2", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_split_rand_factor": "20", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_update_to": "1000", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_enable": "true", 2024-04-12T11:32:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_xfs_extsize": "false", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_zfs_snap": "false", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "fio_dir": "/tmp/fio", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "fsid": "80efffc6-f8bc-11ee-bc8c-c7b262605968", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_allow_other": "true", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_atomic_o_trunc": "true", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_big_writes": "true", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_debug": "false", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_default_permissions": "false", 2024-04-12T11:32:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_disable_pagecache": "false", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_max_write": "0", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_multithreaded": "true", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_require_active_mds": "true", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_set_user_groups": "true", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_splice_move": "true", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_splice_read": "true", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_splice_write": "true", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_syncfs_on_mksnap": "true", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_use_invalidate_cb": "true", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "gss_target_name": "ceph", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "heartbeat_file": "", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "heartbeat_inject_failure": "0", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "heartbeat_interval": "5", 2024-04-12T11:32:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "host": "smithi029", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_max_size": "1073741824", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_watermark": "0.900000", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "inject_early_sigterm": "false", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "jaeger_agent_port": "6799", 2024-04-12T11:32:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "jaeger_tracing_enable": "false", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_aio": "true", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_align_min_size": "65536", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_block_align": "true", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_block_size": "4096", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_dio": "true", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_discard": "false", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_force_aio": "false", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_ignore_corruption": "false", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_max_write_bytes": "10485760", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_max_write_entries": "100", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_replay_from": "0", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_throttle_high_multiple": "0.000000", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-04-12T11:32:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_throttle_max_multiple": "0.000000", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_write_header_frequency": "0", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_zero_on_create": "false", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "journaler_prefetch_periods": "10", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "journaler_prezero_periods": "5", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "journaler_write_head_interval": "15", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "key": "", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "keyfile": "", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_backend": "rocksdb", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_default_stripe_size": "65536", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_fsck_on_mount": "false", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_fsck_on_mount_deep": "true", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_max_bytes": "67108864", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_max_ops": "512", 2024-04-12T11:32:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_nid_prealloc": "1024", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_onode_map_size": "1024", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_sync_submit_transaction": "false", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_sync_transaction": "false", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "librados_thread_count": "2", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "lockdep": "false", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "lockdep_force_backtrace": "false", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "log_coarse_timestamps": "true", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "log_flush_on_exit": "false", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "log_graylog_host": "127.0.0.1", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "log_graylog_port": "12201", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "log_max_new": "1000", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "log_max_recent": "10000", 2024-04-12T11:32:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "log_stderr_prefix": "", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "log_stop_at_utilization": "0.970000", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "log_to_file": "true", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "log_to_graylog": "false", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "log_to_journald": "false", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "log_to_stderr": "false", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "log_to_syslog": "false", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "max_rotating_auth_attempts": "10", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_action_on_write_error": "1", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_alternate_name_max": "8192", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_asio_thread_count": "2", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_export_pin": "true", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_fragment_dirs": "true", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_fragment_interval": "5", 2024-04-12T11:32:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_fragment_size_max": "100000", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_idle_threshold": "0.000000", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_interval": "10", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_max": "-1", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_max_until": "-1", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_merge_size": "50", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_midchunk": "0.300000", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_min_rebalance": "0.100000", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_min_start": "0.200000", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_minchunk": "0.001000", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_mode": "0", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_need_max": "1.200000", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_need_min": "0.800000", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_overload_epochs": "2", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-04-12T11:32:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_sample_interval": "3.000000", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_split_bits": "3", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_split_rd": "25000.000000", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_split_size": "10000", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_split_wr": "10000.000000", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_target_decay": "10.000000", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_beacon_grace": "15.000000", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_beacon_interval": "4.000000", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_beacon_mon_down_grace": "60", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_memory_limit": "4294967296", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_mid": "0.700000", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_quiesce_delay": "0", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_quiesce_sleep": "200", 2024-04-12T11:32:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_quiesce_splitauth": "true", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_quiesce_threshold": "524288", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_release_free_interval": "10", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_reservation": "0.050000", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_trim_interval": "1", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_trim_threshold": "262144", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_client_delegate_inos_pct": "50", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_client_prealloc_inos": "1000", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_connect_bootstrapping": "false", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_damage_table_max_entries": "10000", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_debug_auth_pins": "false", 2024-04-12T11:32:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_debug_frag": "false", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_debug_scatterstat": "false", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_debug_subtrees": "false", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_decay_halflife": "5.000000", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_default_dir_hash": "2", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_defer_session_stale": "true", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_deny_all_reconnect": "false", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dir_keys_per_op": "16384", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dir_max_commit_size": "10", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dir_max_entries": "0", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dir_prefetch": "true", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dirstat_min_interval": "1.000000", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dump_cache_after_rejoin": "false", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dump_cache_on_map": "false", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dump_cache_threshold_file": "0", 2024-04-12T11:32:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_early_reply": "true", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_enable_op_tracker": "true", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_enforce_unique_name": "true", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_export_ephemeral_distributed": "true", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_export_ephemeral_random": "true", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_forward_all_requests_to_auth": "false", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_health_cache_threshold": "1.500000", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_health_summarize_threshold": "10", 2024-04-12T11:32:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_heartbeat_grace": "15.000000", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_heartbeat_reset_grace": "1000", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_health_dummy": "false", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_migrator_session_race": "false", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_join_fs": "", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_journal_format": "1", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_create_at": "0", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_export_at": "0", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_import_at": "0", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_journal_at": "0", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_journal_expire_at": "0", 2024-04-12T11:32:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_journal_replay_at": "0", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_link_at": "0", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_mdstable_at": "0", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_openc_at": "0", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_rename_at": "0", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_shutdown_at": "0", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_skip_replaying_inotable": "false", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_event_large_threshold": "524288", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_events_per_segment": "1024", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_major_segment_event_ratio": "12", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_max_events": "-1", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_max_segments": "128", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_pause": "false", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_segment_size": "0", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_skip_corrupt_events": "false", 2024-04-12T11:32:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_skip_unbounded_events": "false", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_trim_threshold": "128", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_warn_factor": "2.000000", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_caps_per_client": "1048576", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_completed_flushes": "100000", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_completed_requests": "100000", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_export_size": "20971520", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_file_recover": "32", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_purge_files": "64", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_purge_ops": "8192", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_snaps_per_dir": "100", 2024-04-12T11:32:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_metrics_update_interval": "2", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_min_caps_per_client": "100", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_min_caps_working_set": "10000", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_numa_node": "-1", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_op_complaint_time": "30.000000", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_op_history_duration": "600", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_op_history_size": "20", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_op_history_slow_op_size": "20", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_op_history_slow_op_threshold": "10", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_op_log_threshold": "5", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_ping_grace": "15", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_ping_interval": "5", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-04-12T11:32:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_max_caps": "30000", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_max_decay_threshold": "131072", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_warning_threshold": "262144", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_reconnect_timeout": "45.000000", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_replay_interval": "1.000000", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_root_ino_gid": "0", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_root_ino_uid": "0", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_blocklist_on_evict": "true", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_blocklist_on_timeout": "true", 2024-04-12T11:32:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_metadata_threshold": "16777216", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_shutdown_check": "0", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_skip_ino": "0", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_sleep_rank_change": "0.000000", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_snap_max_uid": "4294967294", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_snap_min_uid": "0", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_snap_rstat": "false", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_standby_replay_damaged": "false", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_symlink_recovery": "true", 2024-04-12T11:32:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_task_status_update_interval": "2.000000", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_thrash_exports": "0", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_thrash_fragments": "0", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_tick_interval": "5.000000", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_valgrind_exit": "false", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_verify_backtrace": "1", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_verify_scatter": "false", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_wipe_ino_prealloc": "false", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_wipe_sessions": "false", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mempool_debug": "false", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "memstore_debug_omit_block_device_write": "false", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "memstore_device_bytes": "1073741824", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "memstore_page_set": "false", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "memstore_page_size": "65536", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_client_bytes": "134217728", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_client_messages": "512", 2024-04-12T11:32:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_connect_retry_interval": "1.000000", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_disabled_modules": "", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_max_pg_creating": "1024", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_max_pg_num_change": "128", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_mds_bytes": "134217728", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_mds_messages": "128", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_mon_bytes": "134217728", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_mon_messages": "128", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_osd_bytes": "536870912", 2024-04-12T11:32:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_osd_messages": "8192", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_pool": "true", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_service_beacon_grace": "60.000000", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_standby_modules": "true", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_stats_period": "5", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_stats_threshold": "5", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_tick_period": "2", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_accept_timeout_factor": "2.000000", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_allow_pool_delete": "true", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_allow_pool_size_one": "false", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_auth_validate_all_caps": "true", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_bytes": "104857600", 2024-04-12T11:32:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_directed_command_retry": "2", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_hunt_interval": "3.000000", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_hunt_parallel": "3", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_log_interval": "1.000000", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_ping_interval": "10.000000", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_ping_timeout": "30.000000", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_target_rank": "-1", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_clock_drift_allowed": "1.000000", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_file_level": "debug", 2024-04-12T11:32:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_file": "true", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_graylog": "false", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_journald": "true", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_stderr": "false", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_syslog_level": "info", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_compact_on_bootstrap": "false", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_compact_on_start": "false", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_compact_on_trim": "true", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_con_tracker_persist_interval": "10", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_con_tracker_score_halflife": "43200", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_config_key_max_entry_size": "65536", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cpu_threads": "4", 2024-04-12T11:32:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_crush_min_required_version": "hammer", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_daemon_bytes": "419430400", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_data_avail_crit": "5", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_data_avail_warn": "5", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_data_size_warn": "16106127360", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_block_osdmap_trim": "false", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_dump_json": "false", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_dump_transactions": "false", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_extra_checks": "false", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_no_require_reef": "false", 2024-04-12T11:32:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_no_require_squid": "false", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_delta_reset_interval": "10.000000", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_dns_srv_name": "ceph-mon", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_down_mkfs_grace": "60", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_down_uptime_grace": "60", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_election_default_strategy": "3", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_election_timeout": "5.000000", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_elector_ping_divisor": "2", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_elector_ping_timeout": "2.000000", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_enable_op_tracker": "true", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_fake_pool_delete": "false", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_force_quorum_join": "false", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_globalid_prealloc": "10000", 2024-04-12T11:32:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_detail_to_clog": "true", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_log_update_period": "5", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_max_detail": "50", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_to_clog": "true", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_to_clog_interval": "600", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_host": "[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_host_override": "", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_initial_members": "", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_keyvaluedb": "rocksdb", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_lease": "5.000000", 2024-04-12T11:32:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_log_full_interval": "50", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_log_max": "10000", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_log_max_summary": "50", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_log_entries_per_event": "4096", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_log_epochs": "500", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_mdsmap_epochs": "500", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_mgrmap_epochs": "500", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_osd": "10000", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_pg_per_osd": "10000", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_pool_pg_num": "65536", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mds_force_trim_to": "0", 2024-04-12T11:32:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mds_skip_sanity": "true", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_memory_autotune": "true", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_memory_target": "2147483648", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_beacon_grace": "30", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_digest_period": "5", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_inactive_grace": "60", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_mkfs_grace": "240", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_min_osdmap_epochs": "500", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_complaint_time": "30", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_history_duration": "600", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_history_size": "20", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_history_slow_op_size": "20", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_history_slow_op_threshold": "10", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_log_threshold": "5", 2024-04-12T11:32:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_auto_mark_in": "false", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_auto_mark_new_in": "true", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_cache_size": "500", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_cache_size_min": "134217728", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_crush_smoke_test": "true", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_destroyed_out_interval": "600", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_down_out_interval": "600", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_force_trim_to": "0", 2024-04-12T11:32:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_full_ratio": "0.950000", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_laggy_halflife": "3600", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_laggy_max_interval": "300", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_laggy_weight": "0.300000", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_max_initial_pgs": "1024", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_min_down_reporters": "2", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_prime_pg_temp": "true", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-04-12T11:32:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_report_timeout": "900", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_warn_num_repaired": "10", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_warn_op_age": "32.000000", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osdmap_full_prune_interval": "10", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osdmap_full_prune_min": "10000", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_stuck_threshold": "60", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_warn_min_objects": "10000", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_warn_min_per_osd": "0", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-04-12T11:32:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pool_quota_crit_threshold": "0", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pool_quota_warn_threshold": "0", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_probe_timeout": "2.000000", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_reweight_max_change": "0.050000", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_reweight_max_osds": "4", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_scrub_interval": "86400", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_scrub_max_keys": "100", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_scrub_timeout": "300", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_session_timeout": "300", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_smart_report_timeout": "5", 2024-04-12T11:32:40.373 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_stat_smooth_intervals": "6", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_stretch_pool_min_size": "2", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_stretch_pool_size": "4", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_subscribe_interval": "86400.000000", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_debug": "false", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_max_payload_keys": "2000", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_max_payload_size": "1048576", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_provider_kill_at": "0", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_requester_kill_at": "0", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_timeout": "60.000000", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_target_pg_per_osd": "100", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_tick_interval": "5", 2024-04-12T11:32:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_timecheck_interval": "300.000000", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_use_min_delay_socket": "false", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_older_version_delay": "604800", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_filestore_osds": "true", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_misplaced": "false", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_older_version": "true", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-04-12T11:32:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_pool_no_app": "true", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_too_few_osds": "false", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "monmap": "", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-04-12T11:32:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_tracing_enabled": "false", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_op_threads": "3", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_buffer_size": "131072", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_cm": "false", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_device_name": "", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_dscp": "96", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_gid_idx": "0", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_local_gid": "", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_polling_us": "1000", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_port_num": "1", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_roce_ver": "1", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_send_buffers": "1024", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_sl": "3", 2024-04-12T11:32:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_support_srq": "true", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_type": "ib", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_reap_threshold": "5", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_before_connect": "false", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_ipv4": "true", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_ipv6": "false", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_msgr1": "true", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_msgr2": "true", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_port_max": "7568", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_port_min": "6800", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_prefer_ipv4": "false", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_retry_count": "3", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_retry_delay": "5", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_blackhole_client": "false", 2024-04-12T11:32:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_blackhole_mds": "false", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_blackhole_mgr": "false", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_blackhole_mon": "false", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_blackhole_osd": "false", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_client_mode": "crc secure", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_cluster_mode": "crc secure", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_cluster_type": "", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_compress_secure": "false", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_connection_idle_timeout": "900", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_connection_ready_timeout": "10", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_crc_data": "true", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_crc_header": "true", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_die_on_bad_msg": "false", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_die_on_bug": "true", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_die_on_old_message": "true", 2024-04-12T11:32:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_die_on_skipped_message": "false", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_die_on_unhandled_msg": "false", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_coremask": "0xF", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_devs_allowlist": "", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_enable_tso": "true", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_hugepages": "", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_hw_flow_control": "true", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_lro": "true", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_memory_channel": "4", 2024-04-12T11:32:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_pmd": "", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_port_id": "0", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dump_corrupt_message_level": "1", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dump_on_send": "false", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_initial_backoff": "0.200000", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_delay_max": "1.000000", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_delay_probability": "0.000000", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_delay_type": "", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_internal_delays": "0.000000", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_network_congestion": "0", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_socket_failures": "0", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_learn_addr_from_peer": "true", 2024-04-12T11:32:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_max_accept_failures": "4", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_max_backoff": "15.000000", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_mon_client_mode": "secure crc", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_mon_cluster_mode": "secure crc", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_mon_service_mode": "secure crc", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_osd_compress_min_size": "1024", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_osd_compress_mode": "none", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_osd_compression_algorithm": "snappy", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_pq_min_cost": "65536", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_public_type": "", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_service_mode": "crc secure", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_tcp_listen_backlog": "512", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_tcp_nodelay": "true", 2024-04-12T11:32:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_tcp_rcvbuf": "0", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_type": "async+posix", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "no_config_file": "false", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_completion_locks_per_session": "32", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_debug_inject_relock_delay": "false", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_inflight_op_bytes": "104857600", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_inflight_ops": "1024", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_inject_no_watch_ping": "false", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_tick_interval": "5.000000", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_timeout": "10.000000", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "objectstore_blackhole": "false", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "openssl_engine_opts": "", 2024-04-12T11:32:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_delay_time": "5.000000", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_hist_halflife": "1000", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_max_low_ops": "2", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_max_ops": "4", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_quantize_effort": "0.100000", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_slop": "0.020000", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_allow_recovery_below_min_size": "true", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_asio_thread_count": "2", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_async_recovery_min_cost": "100", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_auto_mark_unfound_lost": "false", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backfill_retry_interval": "30.000000", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backfill_scan_max": "512", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backfill_scan_min": "64", 2024-04-12T11:32:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backoff_on_degraded": "false", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backoff_on_peering": "false", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backoff_on_unfound": "true", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_beacon_report_interval": "300", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_bench_duration": "30", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_bench_max_block_size": "67108864", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_bench_small_size_max_iops": "100", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_blkin_trace_all": "false", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_blocked_scrub_grace_period": "120", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_check_for_log_corruption": "false", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-04-12T11:32:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_class_update_on_start": "true", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_client_message_cap": "256", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_client_message_size_cap": "524288000", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_client_op_priority": "63", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_client_watch_timeout": "30", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_command_max_records": "256", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_compact_on_start": "false", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_crush_chooseleaf_type": "1", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_crush_initial_weight": "-1.000000", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_crush_update_on_start": "true", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_crush_update_weight_set": "true", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-04-12T11:32:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_drop_ping_duration": "0", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_feed_pullee": "-1", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_misdirected_ops": "false", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_no_acting_change": "false", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_no_purge_strays": "false", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_op_order": "false", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_pg_log_writeout": "false", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_pretend_recovery_active": "false", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-04-12T11:32:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_shutdown": "false", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_trim_objects": "false", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_verify_cached_snaps": "false", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_verify_missing_on_start": "false", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_verify_snaps": "false", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_keys": "1024", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_stride": "524288", 2024-04-12T11:32:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_default_data_pool_replay_window": "45", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_default_notify_timeout": "30", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_delete_sleep": "0.000000", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_discard_disconnected_ops": "true", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_enable_op_tracker": "true", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_extblkdev_plugins": "vdo", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_fast_info": "true", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_fast_shutdown": "true", 2024-04-12T11:32:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_fast_shutdown_timeout": "15", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_function_tracing": "false", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_grace": "20", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_interval": "6", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_min_peers": "10", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_min_size": "2000", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_stale": "600", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_hit_set_max_size": "100000", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_hit_set_min_size": "1000", 2024-04-12T11:32:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_journal_flush_on_shutdown": "true", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_journal_size": "5120", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_kill_backfill_at": "0", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_loop_before_reset_tphandle": "64", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_map_cache_size": "50", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_map_dedup": "true", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_map_message_max": "40", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_map_message_max_bytes": "10485760", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_map_share_max_epochs": "40", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_attr_name_len": "100", 2024-04-12T11:32:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_attr_size": "0", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_backfills": "1", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_markdown_count": "5", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_markdown_period": "600", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_object_name_len": "2048", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_object_namespace_len": "256", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_object_size": "134217728", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_omap_entries_per_request": "1024", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_pg_blocked_by": "16", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_pg_log_entries": "10000", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_pgls": "1024", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_push_cost": "8388608", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_push_objects": "10", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_scrubs": "3", 2024-04-12T11:32:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_trimming_pgs": "2", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_write_op_reply_len": "64", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_write_size": "90", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_override_recovery_settings": "false", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_profile": "balanced", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-04-12T11:32:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_skip_benchmark": "false", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_base": "805306368", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_cache_min": "134217728", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_target": "4294967296", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_target_autotune": "false", 2024-04-12T11:32:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_min_pg_log_entries": "250", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_min_recovery_priority": "0", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mon_heartbeat_interval": "30", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mon_report_interval": "5", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mon_report_max_in_flight": "2", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_num_cache_shards": "32", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_num_op_tracker_shard": "32", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_numa_auto_affinity": "true", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_numa_node": "-1", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_numa_prefer_iface": "true", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_objecter_finishers": "1", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_objectstore": "bluestore", 2024-04-12T11:32:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_objectstore_fuse": "false", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_objectstore_tracing": "false", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_complaint_time": "30.000000", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_history_duration": "600", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_history_size": "20", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_history_slow_op_size": "20", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_log_threshold": "5", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_shards": "0", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_shards_hdd": "5", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_shards_ssd": "8", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_threads_per_shard": "0", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-04-12T11:32:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_pq_min_cost": "65536", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_queue": "mclock_scheduler", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_queue_cut_off": "high", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_thread_suicide_timeout": "150", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_thread_timeout": "15", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_open_classes_on_start": "true", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_os_flags": "0", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_peering_op_priority": "255", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_delete_cost": "1048576", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_delete_priority": "5", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_log_dups_tracked": "3000", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_log_trim_max": "10000", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_log_trim_min": "100", 2024-04-12T11:32:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_object_context_cache_count": "64", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_crimson": "false", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_crush_rule": "-1", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_ec_fast_read": "false", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flag_bulk": "false", 2024-04-12T11:32:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-04-12T11:32:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flag_nodelete": "false", 2024-04-12T11:32:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-04-12T11:32:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-04-12T11:32:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flags": "0", 2024-04-12T11:32:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-04-12T11:32:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_min_size": "0", 2024-04-12T11:32:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-04-12T11:32:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_pg_num": "32", 2024-04-12T11:32:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_pgp_num": "0", 2024-04-12T11:32:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-04-12T11:32:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_read_ratio": "70", 2024-04-12T11:32:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_size": "3", 2024-04-12T11:32:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_type": "replicated", 2024-04-12T11:32:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-04-12T11:32:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_use_gmt_hitset": "true", 2024-04-12T11:32:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_push_per_object_cost": "1000", 2024-04-12T11:32:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_read_ec_check_for_errors": "false", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recover_clone_overlap": "true", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recover_clone_overlap_limit": "10", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_cost": "20971520", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_delay_start": "0.000000", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_active": "0", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_active_hdd": "3", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_active_ssd": "10", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_chunk": "8388608", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_single_start": "1", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_op_priority": "3", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_op_warn_multiple": "16", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_priority": "5", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_retry_interval": "30.000000", 2024-04-12T11:32:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_sleep": "0.000000", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_repair_during_recovery": "false", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_requested_scrub_priority": "120", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_rollback_to_cluster_snap": "", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_auto_repair": "false", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_begin_hour": "0", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_begin_week_day": "0", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_chunk_max": "25", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_chunk_min": "5", 2024-04-12T11:32:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_cost": "52428800", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_during_recovery": "false", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_end_hour": "0", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_end_week_day": "0", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_event_cost": "4096", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_invalid_stats": "true", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_load_threshold": "0.500000", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_max_interval": "604800.000000", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_max_preemptions": "5", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_min_interval": "86400.000000", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_priority": "5", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_sleep": "0.000000", 2024-04-12T11:32:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_shutdown_pgref_assert": "false", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_skip_data_digest": "false", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_smart_report_timeout": "5", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_cost": "1048576", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_priority": "5", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_sleep": "0.000000", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_stats_update_period_scrubbing": "15", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_target_transaction_size": "30", 2024-04-12T11:32:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_mode": "writeback", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tracing": "false", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_use_stale_snap": "false", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "osdc_blkin_trace_all": "false", 2024-04-12T11:32:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_kill_at": "0", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_max_join_drift": "10", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_min": "500", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_min_wait": "0.050000", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_propose_interval": "1.000000", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_service_trim_max": "500", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_service_trim_max_multiplier": "20", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_service_trim_min": "250", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_stash_full_interval": "25", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_trim_max": "500", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_trim_min": "250", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "perf": "true", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "pid_file": "", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-04-12T11:32:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "public_addr": "-", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "public_addrv": "", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "public_bind_addr": "-", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "public_network": "172.21.0.0/20", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "public_network_interface": "", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "qat_compressor_enabled": "false", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "qat_compressor_session_max_number": "256", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "rados_mon_op_timeout": "0", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "rados_osd_op_timeout": "0", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "rados_tracing": "false", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_atime_update_interval": "60", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_balance_parent_reads": "false", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_balance_snap_reads": "false", 2024-04-12T11:32:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_blkin_trace_all": "false", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_blocklist_expire_seconds": "0", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_blocklist_on_break_lock": "true", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache": "true", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_block_writes_upfront": "false", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_max_dirty": "25165824", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_max_dirty_object": "0", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_policy": "writearound", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_size": "33554432", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_target_dirty": "16777216", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_clone_copy_on_read": "false", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_compression_hint": "none", 2024-04-12T11:32:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_concurrent_management_ops": "10", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_clone_format": "auto", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_data_pool": "", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_features": "61", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_format": "2", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_map_options": "", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_order": "22", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_pool": "rbd", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_stripe_count": "0", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_stripe_unit": "0", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_disable_zero_copy_writes": "true", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_discard_granularity_bytes": "65536", 2024-04-12T11:32:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_enable_alloc_hint": "true", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_io_scheduler": "simple", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_commit_age": "5.000000", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_object_flush_interval": "0", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_order": "24", 2024-04-12T11:32:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_pool": "", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_splay_width": "4", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_localize_parent_reads": "false", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_localize_snap_reads": "false", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_policy_type": "simple", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-04-12T11:32:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_autotune": "true", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_base": "805306368", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_target": "4294967296", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirroring_delete_delay": "0", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-04-12T11:32:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirroring_replay_delay": "0", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_move_to_trash_on_remove": "false", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mtime_update_interval": "60", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_non_blocking_aio": "true", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_op_thread_timeout": "60", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_op_threads": "1", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_parent_cache_enabled": "false", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_persistent_cache_mode": "disabled", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_persistent_cache_path": "/tmp", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_persistent_cache_size": "1073741824", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_plugins": "", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_bps_burst": "0", 2024-04-12T11:32:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_bps_limit": "0", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_exclude_ops": "0", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_iops_burst": "0", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_iops_limit": "0", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_bps_burst": "0", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_bps_limit": "0", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_iops_burst": "0", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_iops_limit": "0", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_schedule_tick_min": "50", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_bps_burst": "0", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-04-12T11:32:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_bps_limit": "0", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_iops_burst": "0", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_iops_limit": "0", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_quiesce_notification_attempts": "10", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_read_from_replica_policy": "default", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_readahead_max_bytes": "524288", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_readahead_trigger_requests": "10", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_request_timed_out_seconds": "30", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_skip_partial_discard": "true", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_tracing": "false", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_validate_names": "true", 2024-04-12T11:32:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_validate_pool": "true", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "restapi_base_url": "", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "restapi_log_level": "", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_acl_grants_max_num": "100", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_admin_entry": "admin", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_backend_store": "rados", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_barbican_url": "", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_beast_enable_async": "true", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_counters_cache": "false", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_index_max_aio": "128", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-04-12T11:32:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_quota_ttl": "600", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_cache_enabled": "true", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_cache_expiry_interval": "900", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_cache_lru_size": "10000", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_config_store": "rados", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_content_length_compat": "false", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_copy_obj_progress": "true", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_cors_rules_max_num": "100", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_cross_domain_policy": "", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_default_encryption_key": "", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_addr": "", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_ca_path": "", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_client_cert": "", 2024-04-12T11:32:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_client_key": "", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_password": "", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_username": "", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_require_ssl": "true", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-04-12T11:32:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_suppress_logs": "true", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_addr": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_auth": "token", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_namespace": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_prefix": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_token_file": "", 2024-04-12T11:32:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_curl_buffersize": "524288", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_curl_low_speed_limit": "1024", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_curl_low_speed_time": "300", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_curl_tcp_keepalive": "0", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_fadvise": "4", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d4n_host": "127.0.0.1", 2024-04-12T11:32:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d4n_l1_fadvise": "4", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d4n_port": "6379", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_log_changes_size": "1000", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_log_num_shards": "128", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_log_window": "30", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_notify_interval_msec": "0", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_sync_poll_interval": "20", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_sync_spawn_window": "20", 2024-04-12T11:32:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_default_data_log_backing": "fifo", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_default_region_info_oid": "default.region", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_defer_to_bucket_acls": "", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-04-12T11:32:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_data_res": "500.000000", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dns_name": "", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dns_s3website_name": "", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dynamic_resharding": "true", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_gc_threads": "true", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_lc_threads": "true", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_ops_log": "false", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_quota_threads": "true", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_static_website": "false", 2024-04-12T11:32:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_usage_log": "false", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enforce_swift_acls": "true", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_exit_timeout_secs": "120", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_expose_bucket": "false", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_extended_http_attrs": "", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_filter": "none", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_frontends": "beast port=7480", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_concurrent_io": "10", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_deferred": "50", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_objs": "32", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_queue_size": "134213632", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_trim_chunk": "16", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_obj_min_wait": "7200", 2024-04-12T11:32:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_processor_max_time": "3600", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_processor_period": "3600", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_get_obj_window_size": "16777216", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_healthcheck_disabling_path": "", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ignore_get_invalid_range": "false", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_init_timeout": "300", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_kafka_connection_idle": "300", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_kafka_message_timeout": "5000", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_kafka_sleep_timeout": "10", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-04-12T11:32:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_domain": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_password": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_password_path": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_project": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_tenant": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_token": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_token_path": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_user": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_api_version": "2", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_barbican_domain": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_barbican_password": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_barbican_project": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_barbican_tenant": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_barbican_user": "", 2024-04-12T11:32:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_implicit_tenants": "false", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_service_token_enabled": "false", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_token_cache_size": "10000", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_url": "", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_verify_ssl": "true", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_debug_interval": "-1", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_lock_max_time": "90", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_max_objs": "32", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_max_rules": "1000", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_max_worker": "3", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_max_wp_worker": "3", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_thread_delay": "0", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-04-12T11:32:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_dnattr": "uid", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_searchfilter": "", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_uri": "", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lfuda_sync_frequency": "60", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_local_cache_address": "127.0.0.1:6379", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_log_http_headers": "", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_log_nonexistent_bucket": "false", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_log_object_name_utc": "false", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-04-12T11:32:40.428 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_attr_name_len": "0", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_attr_size": "0", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_attrs_num_in_req": "0", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_chunk_size": "4194304", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_concurrent_requests": "1024", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_dynamic_shards": "1999", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_listing_results": "1000", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_notify_retries": "10", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_objs_per_shard": "100000", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_put_param_size": "1048576", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_put_size": "5368709120", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_slo_entries": "1000", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_md_log_max_shards": "64", 2024-04-12T11:32:40.429 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_md_notify_interval_msec": "200", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_meta_sync_poll_interval": "20", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_meta_sync_spawn_window": "20", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_mp_lock_max_time": "600", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_multipart_min_part_size": "5242880", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_fhcache_size": "2017", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_lru_lanes": "5", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_max_gc": "300", 2024-04-12T11:32:40.430 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_run_gc_threads": "false", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_run_lc_threads": "false", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_run_quota_threads": "false", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_run_sync_thread": "false", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_num_async_rados_threads": "32", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_num_control_oids": "8", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_numa_node": "-1", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_obj_stripe_size": "4194304", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_objexp_chunk_size": "100", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_objexp_gc_interval": "600", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_objexp_hints_num_shards": "127", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-04-12T11:32:40.431 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_op_thread_timeout": "600", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_op_tracing": "false", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_opa_token": "", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_opa_url": "", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_opa_verify_ssl": "true", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ops_log_rados": "false", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ops_log_socket_path": "", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_period_push_interval": "2.000000", 2024-04-12T11:32:40.432 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_period_push_interval_max": "30.000000", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_period_root_pool": ".rgw.root", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_posix_cache_lanes": "3", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_posix_cache_max_buckets": "100", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_posix_cache_partitions": "3", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_print_continue": "true", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_print_prohibited_content_length": "false", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-04-12T11:32:40.433 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_rados_tracing": "false", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_realm": "", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_realm_id": "", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_region": "", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_region_root_pool": ".rgw.root", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_relaxed_region_enforcement": "false", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_request_uri": "", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_reshard_batch_size": "64", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_reshard_max_aio": "128", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_reshard_num_logs": "16", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_reshard_thread_interval": "600", 2024-04-12T11:32:40.434 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_resolve_cname": "false", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_rest_getusage_op_compat": "false", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_run_sync_thread": "true", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_use_keystone": "false", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_use_ldap": "false", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_use_rados": "true", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_use_sts": "false", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_success_create_obj_status": "0", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_scheduler_type": "throttler", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_script_uri": "", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_service_provider_name": "", 2024-04-12T11:32:40.435 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_client_id": "", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_client_secret": "", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_entry": "sts", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_key": "sts", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_max_session_duration": "43200", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_min_session_duration": "900", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_token_introspection_url": "", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_account_in_url": "false", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_auth_entry": "auth", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_auth_url": "", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_custom_header": "", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_enforce_content_length": "false", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_need_stats": "true", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_tenant_name": "", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_token_expiration": "86400", 2024-04-12T11:32:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_url": "", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_url_prefix": "swift", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_versioning_enabled": "false", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_lease_period": "120", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_log_trim_interval": "1200", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_obj_etag_verify": "false", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_trace_history_size": "4096", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_thread_pool_size": "512", 2024-04-12T11:32:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_topic_persistency_max_retries": "0", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_topic_require_publish_policy": "false", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_comment": "", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_createby": "", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_encoding": "", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_flag": "false", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_max_size": "5368709120", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_origin": "", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_sha_unit": "524288", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_tracker": "", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_trust_forwarded_https": "false", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-04-12T11:32:40.438 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_usage_log_tick_interval": "30", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_usage_max_shards": "32", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_usage_max_user_shards": "1", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_use_opa_authz": "false", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_counters_cache": "false", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_counters_cache_size": "10000", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_default_quota_max_size": "-1", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_max_buckets": "1000", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_policies_max_num": "100", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_quota_sync_interval": "86400", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_unique_email": "true", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_verify_ssl": "true", 2024-04-12T11:32:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_website_routing_rules_max_num": "50", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zone": "", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zone_id": "", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zonegroup": "", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zonegroup_id": "", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rgwlc_auto_session_clear": "true", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rgwlc_skip_bucket_step": "false", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_block_size": "4096", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-04-12T11:32:40.440 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_shard_bits": "4", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_size": "536870912", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_type": "binned_lru", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_collect_compaction_stats": "false", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_collect_extended_stats": "false", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_collect_memory_stats": "false", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_index_type": "binary_search", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_log_to_ceph_log": "true", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_metadata_block_size": "4096", 2024-04-12T11:32:40.441 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_partition_filters": "false", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_perf": "false", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "rotating_keys_renewal_timeout": "10", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "run_dir": "/var/run/ceph", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_block_create": "true", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_cache_lru_size": "67108864", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_cbjournal_size": "5368709120", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_data_delta_based_overwrite": "0", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_default_max_object_size": "16777216", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_device_size": "53687091200", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_journal_batch_capacity": "16", 2024-04-12T11:32:40.442 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_journal_iodepth_limit": "5", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_main_device_type": "SSD", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_max_concurrent_transactions": "8", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_segment_size": "67108864", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "set_keepcaps": "false", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "setgroup": "ceph", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "setuser": "ceph", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "setuser_match_path": "", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "target_max_misplaced_ratio": "0.050000", 2024-04-12T11:32:40.443 INFO:teuthology.orchestra.run.smithi029.stdout: "thp": "false", 2024-04-12T11:32:40.444 INFO:teuthology.orchestra.run.smithi029.stdout: "threadpool_default_timeout": "60", 2024-04-12T11:32:40.444 INFO:teuthology.orchestra.run.smithi029.stdout: "threadpool_empty_queue_max_wait": "2", 2024-04-12T11:32:40.444 INFO:teuthology.orchestra.run.smithi029.stdout: "throttler_perf_counter": "true" 2024-04-12T11:32:40.444 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:32:41.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-04-12T11:32:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:40 smithi087 ceph-mon[89014]: pgmap v1298: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:41.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:40 smithi029 ceph-mon[93650]: pgmap v1298: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:41.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:40 smithi029 ceph-mon[97787]: pgmap v1298: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:41.336 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:32:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:32:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:32:42.971 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a: {} 2024-04-12T11:32:42.981 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b: {} 2024-04-12T11:32:42.982 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:42 smithi029 ceph-mon[97787]: pgmap v1299: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:42.982 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:42 smithi029 ceph-mon[93650]: pgmap v1299: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:42.992 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c: {} 2024-04-12T11:32:42.992 INFO:teuthology.orchestra.run.smithi029.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-04-12T11:32:42.992 INFO:teuthology.orchestra.run.smithi029.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-04-12T11:32:42.992 INFO:teuthology.orchestra.run.smithi029.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-04-12T11:32:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:42 smithi087 ceph-mon[89014]: pgmap v1299: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:43.758 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-04-12T11:32:43.758 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-04-12T11:32:43.758 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-12T11:32:43.760 INFO:tasks.radosbench:Beginning radosbench... 2024-04-12T11:32:43.760 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-04-12T11:32:43.760 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-04-12T11:32:43.761 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-04-12T11:32:43.761 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:32:43.761 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:32:43.761 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-04-12T11:32:43.762 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:43.762 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-04-12T11:32:43.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:32:43.762 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-04-12T11:32:43.763 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-04-12T11:32:43.763 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-04-12T11:32:43.763 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:32:43.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:43.794 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:43.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:43.895 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:43.922 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:43.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:43.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:44.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:43 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3580232546' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T11:32:44.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:43 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3580232546' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T11:32:44.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:44.052 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:44.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:44.075 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:44.075 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:44.075 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:44.076 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:44.154 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:44.181 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:44.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:44.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:44.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:43 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3580232546' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T11:32:44.283 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:44.311 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:44.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:44.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:44.413 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:44.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:44.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:44.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:44.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:44.563 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:44.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:44.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:44.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:44.688 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:44.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:44.790 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:44.817 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:44.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:44.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:44.919 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:44.946 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:44.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:44.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:45.048 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:45.075 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:45.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:45.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:45.176 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:45.203 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:45.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:45.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:44 smithi087 ceph-mon[89014]: pgmap v1300: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:45.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:44 smithi029 ceph-mon[97787]: pgmap v1300: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:44 smithi029 ceph-mon[93650]: pgmap v1300: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:32:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:32:44] "GET /metrics HTTP/1.1" 200 34004 "" "Prometheus/2.43.0" 2024-04-12T11:32:45.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:45.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:45.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:45.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:45.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:32:45.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:45.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:45.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:32:45.557 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:45.584 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:45.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:45.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:32:45.711 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:45.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:45.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:32:45.813 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:45.837 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:45.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:45.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:45.939 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:45.967 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:45.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:45.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:46.068 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:46.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:46.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:46.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:46.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:46.222 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:46.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:46.324 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:46.351 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:46.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:46.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:46.453 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:46.481 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:46.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:46.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:46.583 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:46.610 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:46.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:46.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:46.712 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:46.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:46.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:46.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:46.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:46.867 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:46.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:46.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:46.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:46 smithi029 ceph-mon[97787]: pgmap v1301: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:46.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:46 smithi029 ceph-mon[93650]: pgmap v1301: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:46.968 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:46.996 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:46.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:46.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:47.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:47.124 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:47.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:47.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:46 smithi087 ceph-mon[89014]: pgmap v1301: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:47.225 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:47.251 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:47.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:47.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:47.353 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:47.379 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:47.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:47.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:47.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:47.507 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:47.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:47.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:47.608 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:47.635 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:47.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:47.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:47.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:32:47.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:47.765 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:47.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:32:47.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:47.890 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:47.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:47.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:47.966 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:32:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=cleanup t=2024-04-12T11:32:47.544756149Z level=info msg="Completed cleanup jobs" duration=11.287711ms 2024-04-12T11:32:47.985 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:32:47.985 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:32:47.985 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2024-04-12T11:32:47.986 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:32:47.992 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:48.020 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:48.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:48.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:48.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:48.149 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:48.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:48.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:48.250 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:48.275 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:48.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:48.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:48.376 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:48.404 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:48.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:48.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:48.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:48.530 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:48.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:48.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:48.655 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:48.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:48.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:48.757 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:48.785 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:48.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:48.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:48.887 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:48.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:48.915 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:48.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:48.993 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:48 smithi029 ceph-mon[93650]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:48.994 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:48 smithi029 ceph-mon[97787]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:49.015 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:49.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:49.043 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:49.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:49.144 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:49.169 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:49.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:49.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:48 smithi087 ceph-mon[89014]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:49.270 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:49.294 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:49.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:49.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:49.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:49.418 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:49.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:49.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:49.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:49.543 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:49.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:49.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:49.645 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:32:49.667 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:49.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:32:49.769 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:49.792 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:49.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:49.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:49.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:49.917 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:49.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:49.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:50.018 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:50.046 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:50.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:50.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:50.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:50.173 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:50.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:50.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:50.274 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:50.303 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:50.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:50.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:50.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:50.428 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:50.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:50.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:50.530 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:50.557 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:50.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:50.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:50.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-04-12T11:32:50.658 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:50.686 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:50.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:50.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:50.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:50.811 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:50.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:50.913 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:50.936 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:50.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:50.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:50.980 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:32:51.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:51.062 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:51.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:51.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:51.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:51.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:51.188 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:51.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:50 smithi087 ceph-mon[89014]: pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:50 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1597725237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:32:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:50 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:32:51.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:50 smithi029 ceph-mon[93650]: pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:51.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:50 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1597725237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:32:51.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:50 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:32:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:50 smithi029 ceph-mon[97787]: pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:50 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1597725237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:32:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:50 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:32:51.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:51.317 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:51.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:51.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:51.419 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:51.423 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:32:51.443 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:51.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:51.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:51.445 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:32:51.464 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:32:51.485 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:32:51.505 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:32:51.525 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:32:51.545 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:51.546 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:32:51.567 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:32:51.568 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-04-12T11:32:51.568 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:32:51.568 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:32:51.568 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-04-12T11:32:51.569 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:32:51.569 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:32:51.569 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:32:51.569 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:32:51.573 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:51.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:51.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:51.675 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:51.701 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:51.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:51.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:51.803 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:51.827 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:51.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:51.929 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:51.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:51.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:51.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:52.059 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:52.086 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:52.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:52.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:52.188 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:32:52.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:52.214 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:52.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:32:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:51 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:32:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:51 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:32:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:51 smithi087 ceph-mon[89014]: osdmap e1085: 8 total, 8 up, 8 in 2024-04-12T11:32:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:51 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1658598009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:32:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:51 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3948112021' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-12T11:32:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:51 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-12T11:32:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:32:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:32:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[93650]: osdmap e1085: 8 total, 8 up, 8 in 2024-04-12T11:32:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1658598009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:32:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3948112021' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-12T11:32:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-12T11:32:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:32:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:32:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[97787]: osdmap e1085: 8 total, 8 up, 8 in 2024-04-12T11:32:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:32:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:32:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1658598009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:32:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3948112021' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-12T11:32:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:51 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-12T11:32:52.287 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_1' created 2024-04-12T11:32:52.315 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:52.339 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:52.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:52.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:52.463 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:52.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:52.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:52.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:32:52.588 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:52.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:52.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:32:52.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:52.713 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:52.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:52.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:52.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:32:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:32:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:32:52.814 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:52.843 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:52.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:52.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:52.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:32:52.968 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:52.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:52.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:32:52.999 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:53.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:53.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:53.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:53.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:53.194 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:52 smithi087 ceph-mon[89014]: pgmap v1305: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:52 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-12T11:32:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:52 smithi087 ceph-mon[89014]: osdmap e1086: 8 total, 8 up, 8 in 2024-04-12T11:32:53.218 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:53.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:53.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:53.250 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:52 smithi029 ceph-mon[97787]: pgmap v1305: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:53.251 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:52 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-12T11:32:53.251 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:52 smithi029 ceph-mon[97787]: osdmap e1086: 8 total, 8 up, 8 in 2024-04-12T11:32:53.251 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:52 smithi029 ceph-mon[93650]: pgmap v1305: 73 pgs: 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:53.251 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:52 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-12T11:32:53.251 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:52 smithi029 ceph-mon[93650]: osdmap e1086: 8 total, 8 up, 8 in 2024-04-12T11:32:53.320 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:53.343 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:53.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:53.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:53.445 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:53.473 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:53.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:53.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:53.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:53.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:53.599 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:53.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:53.698 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-04-12T11:32:53.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:53.763 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:53.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:53.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:53.865 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:53.893 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:53.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:53.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:53.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:54.019 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:54.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:54.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:54.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:54.145 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:54.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:54.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:54.247 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:54.274 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:54.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:54.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:54.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:32:54.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:32:54.399 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:54.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:54.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:32:54.501 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:54.526 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:54.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:54.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:54.572 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:54 smithi029 ceph-mon[97787]: osdmap e1087: 8 total, 8 up, 8 in 2024-04-12T11:32:54.573 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:54 smithi029 ceph-mon[97787]: pgmap v1308: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:54.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:54 smithi029 ceph-mon[93650]: osdmap e1087: 8 total, 8 up, 8 in 2024-04-12T11:32:54.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:54 smithi029 ceph-mon[93650]: pgmap v1308: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:54.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:54.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:54.655 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:54.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:54.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:54 smithi087 ceph-mon[89014]: osdmap e1087: 8 total, 8 up, 8 in 2024-04-12T11:32:54.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:54 smithi087 ceph-mon[89014]: pgmap v1308: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 461 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:54.756 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:54.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:54.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:54.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:54.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:32:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:54.906 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:54.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:32:54.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:32:55.007 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:55.035 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:55.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:55.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:55.136 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:55.136 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:55.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:55.144 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:32:55.144 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:32:55.145 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:32:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:32:54] "GET /metrics HTTP/1.1" 200 34071 "" "Prometheus/2.43.0" 2024-04-12T11:32:55.165 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:55.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:55.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:55.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:55.294 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:55.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:55.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:55.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:55.423 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:55.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:55.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:55.473 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:55 smithi029 ceph-mon[97787]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:32:55.473 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:55 smithi029 ceph-mon[97787]: osdmap e1088: 8 total, 8 up, 8 in 2024-04-12T11:32:55.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:55 smithi029 ceph-mon[93650]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:32:55.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:55 smithi029 ceph-mon[93650]: osdmap e1088: 8 total, 8 up, 8 in 2024-04-12T11:32:55.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:55.550 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:55.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:55.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:55.652 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:55.681 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:55.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:55.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:55.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:55 smithi087 ceph-mon[89014]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:32:55.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:55 smithi087 ceph-mon[89014]: osdmap e1088: 8 total, 8 up, 8 in 2024-04-12T11:32:55.783 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:55.806 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:55.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:55.908 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:55.932 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:55.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:56.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:56.061 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:56.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:56.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:56.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:56.186 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:56.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:56.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:56.288 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:56.316 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:56.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:56.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:56.417 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:56.445 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:56.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:56.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:56.547 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:56.575 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:56.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:56.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:32:56.620 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:56 smithi029 ceph-mon[93650]: pgmap v1310: 89 pgs: 7 creating+peering, 3 unknown, 79 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:56.621 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:56 smithi029 ceph-mon[97787]: pgmap v1310: 89 pgs: 7 creating+peering, 3 unknown, 79 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:56.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:56.700 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:56.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:56.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:56.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:56 smithi087 ceph-mon[89014]: pgmap v1310: 89 pgs: 7 creating+peering, 3 unknown, 79 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:56.802 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:56.829 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:56.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:56.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:56.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:56.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:56.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:56.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:57.057 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:57.085 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:57.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:57.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:57.187 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:57.211 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:57.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:57.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:57.313 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:57.341 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:57.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:57.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:57.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:57.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:57.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:57.573 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:57.601 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:57.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:57.692 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:32:57.702 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:57.714 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:32:57.729 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:57.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:57.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:57.736 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:32:57.756 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:32:57.776 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:32:57.796 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:32:57.816 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:32:57.831 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:57.836 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:32:57.837 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:32:57.837 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:32:57.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:32:57.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:32:57.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:32:57.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:32:57.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:32:57.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:32:57.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:57.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:57.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:57.961 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:57.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:57 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1394865617' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:32:57.985 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:57.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:57 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1394865617' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:32:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:57 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1394865617' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:32:58.087 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:58.115 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:58.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:58.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:58.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-04-12T11:32:58.217 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:32:58.244 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:58.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:58.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:32:58.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:58.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:58.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:58.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:58.473 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:58.500 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:58.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:58.602 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:58.629 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:58.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:58.731 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:58.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:58.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:58.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:58.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:32:58 smithi029 ceph-mon[93650]: pgmap v1311: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:58.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:32:58 smithi029 ceph-mon[97787]: pgmap v1311: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:58.819 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:32:58.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:32:58.885 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:58.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:58.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:32:58.987 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:59.016 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:59.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:59.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:59.118 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:59.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:59.147 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:59.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:32:58 smithi087 ceph-mon[89014]: pgmap v1311: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:32:59.249 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:32:59.272 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:59.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:59.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:32:59.374 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:32:59.398 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:59.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:59.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:32:59.499 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:32:59.527 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:59.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:32:59.629 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:32:59.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:32:59.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:59.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:32:59.755 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:32:59.783 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:59.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:59.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:32:59.885 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:32:59.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:32:59.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:32:59.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:00.014 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:00.042 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:00.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:00.144 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:00.168 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:00.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:00.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:00.270 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:00.298 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:00.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:00.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:00.364 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:00.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:00.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:00.424 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:00.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:00.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:00.549 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:00.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:00.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:00.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:00.673 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:00.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:00.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:00.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:00.799 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:00.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:00.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:00.802 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:33:00.864 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:00 smithi029 ceph-mon[97787]: pgmap v1312: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:00.864 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:00 smithi029 ceph-mon[93650]: pgmap v1312: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:00.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:00.929 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:00.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:00.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:01.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:01.056 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:01.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:01.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:01.158 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:01.182 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:01.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:01.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:01.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:00 smithi087 ceph-mon[89014]: pgmap v1312: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:01.284 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:01.312 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:01.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:01.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:01.413 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:01.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:01.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:01.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:01.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:01.563 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:01.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:01.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:01.665 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:01.692 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:01.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:01.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:01.728 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:33:01.794 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:01.820 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:01.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:01.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:01.949 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:01.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:01.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:02.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:01 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1371869018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:33:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:01 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1371869018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:33:02.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:02.078 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:02.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:02.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:02.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:02.194 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:02.207 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:02.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:02.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:01 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1371869018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:33:02.309 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:02.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:02.337 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:02.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:02.438 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:02.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:02.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:02.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:02.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:02.589 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:02.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:02.690 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:33:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:33:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:33:02.717 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:02.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:02.733 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-04-12T11:33:02.734 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[93650]: pgmap v1313: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:02.734 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:33:02.734 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1371869018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:33:02.734 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[93650]: osdmap e1089: 8 total, 8 up, 8 in 2024-04-12T11:33:02.734 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3787615427' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:33:02.734 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:33:02.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[97787]: pgmap v1313: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:02.735 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:33:02.735 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1371869018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:33:02.735 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[97787]: osdmap e1089: 8 total, 8 up, 8 in 2024-04-12T11:33:02.735 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3787615427' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:33:02.735 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:02 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:33:02.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:02.842 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:02.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:02.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:02.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:02.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:02.968 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:02.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:03.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:03.092 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:03.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:03.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:03.194 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:02 smithi087 ceph-mon[89014]: pgmap v1313: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:02 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:33:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:02 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1371869018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:33:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:02 smithi087 ceph-mon[89014]: osdmap e1089: 8 total, 8 up, 8 in 2024-04-12T11:33:03.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:02 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3787615427' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:33:03.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:02 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:33:03.222 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:03.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:03.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:03.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:03.348 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:03.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:03.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:03.450 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:03.477 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:03.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:03.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:03.579 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:03.607 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:03.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:03.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:03.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:03 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:33:03.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:03 smithi029 ceph-mon[93650]: osdmap e1090: 8 total, 8 up, 8 in 2024-04-12T11:33:03.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:03 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:33:03.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:03 smithi029 ceph-mon[97787]: osdmap e1090: 8 total, 8 up, 8 in 2024-04-12T11:33:03.860 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:03.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:03.962 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:03.991 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:03.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:03.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:04.081 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:33:04.081 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:33:04.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.2.0:176 5.b e602/555 5:d887ce1c:::smithi02979277-43:157 v 602'764, mlcod=598'760)", 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:23:26.601480+0000", 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 577.47814313399999, 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012994712, 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.082 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:26.601480+0000", 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:26.601480+0000", 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:26.601480+0000", 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7300000000000002e-07 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:26.601488+0000", 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9000000000000006e-06 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:26.607824+0000", 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0063359820000000004 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:26.607834+0000", 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2620000000000004e-06 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:26.611101+0000", 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032670289999999999 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:26.611122+0000", 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1491000000000001e-05 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:26.614437+0000", 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033145660000000001 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:26.614475+0000", 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8209e-05 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:5391 5.5 5:a500c83d:::smithi02979277-27:head [append 0~0] snapc 15f=[15f,15e,15d,15c,15b,153,151,14f,14d,14c,147,140,13c] ondisk+write+known_if_redirected+supports_pool_eio e612)", 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:23:36.629909+0000", 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 567.44971397500001, 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025846563999999999, 2024-04-12T11:33:04.086 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 5391 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629909+0000", 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629909+0000", 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629911+0000", 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9230000000000001e-06 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629920+0000", 2024-04-12T11:33:04.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8019999999999998e-06 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629937+0000", 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6935999999999998e-05 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629950+0000", 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2561999999999999e-05 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.630983+0000", 2024-04-12T11:33:04.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001033639 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.630995+0000", 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1674e-05 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.649643+0000", 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018648191000000001 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.649789+0000", 2024-04-12T11:33:04.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014579 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.649954+0000", 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016560699999999999 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.652406+0000", 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024517789999999999 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.654088+0000", 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016816400000000001 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.655624+0000", 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015357909999999999 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.655695+0000", 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0878999999999997e-05 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.655756+0000", 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1351000000000003e-05 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:5392 5.5 5:a500c83d:::smithi02979277-27:head [rollback 13c] snapc 15f=[15f,15e,15d,15c,15b,153,151,14f,14d,14c,147,140,13c] ondisk+write+known_if_redirected+supports_pool_eio e612)", 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:23:36.629969+0000", 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 567.44965393400003, 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026623997, 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.093 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 5392 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629969+0000", 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629969+0000", 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629970+0000", 2024-04-12T11:33:04.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0400000000000002e-07 2024-04-12T11:33:04.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.095 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:04.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629983+0000", 2024-04-12T11:33:04.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3586e-05 2024-04-12T11:33:04.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629991+0000", 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.391e-06 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.629997+0000", 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9209999999999997e-06 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.631020+0000", 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010231120000000001 2024-04-12T11:33:04.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.631028+0000", 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5890000000000003e-06 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.651489+0000", 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020460359000000001 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.651607+0000", 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118476 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.651840+0000", 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023250899999999999 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.653334+0000", 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014939809999999999 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.655448+0000", 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021142629999999999 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.656437+0000", 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00098930999999999997 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.656502+0000", 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5067999999999994e-05 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:23:36.656593+0000", 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1027999999999997e-05 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:7895 5.5 5:accaecb1:::smithi02979277-30:head [rollback 1b3] snapc 1d6=[1d6,1d5,1d4,1d3,1d2,1d0,1cf,1cb,1ca,1c9,1c6,1c4,1c3,1bf,1be,1bc,1b8,1b7,1b3,1a3] ondisk+write+known_if_redirected+supports_pool_eio e843)", 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:27:27.657658+0000", 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 336.42196576999999, 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011443261, 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.101 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 7895 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.657658+0000", 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.657658+0000", 2024-04-12T11:33:04.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.657658+0000", 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9500000000000001e-07 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.657663+0000", 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8930000000000004e-06 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.657674+0000", 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0608000000000001e-05 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.657683+0000", 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2450000000000004e-06 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.657744+0000", 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1268999999999999e-05 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.657751+0000", 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6479999999999997e-06 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.662189+0000", 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044382880000000003 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.662292+0000", 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010306099999999999 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:33:04.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.663439+0000", 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011462200000000001 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.665094+0000", 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016558880000000001 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.667792+0000", 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026975889999999998 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.669014+0000", 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012221700000000001 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.669054+0000", 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9920999999999999e-05 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:27.669101+0000", 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6665999999999997e-05 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.107 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:7988 5.3 5:cc16ccc4:::smithi02979277-12:head [write 746541~400243] snapc 1d8=[1d8,1d7,1d6,1d5,1d4,1d3,1d2,1d0,1cf,1cb,1ca,1c9,1c6,1c4,1c3,1be,1bc,1b8,1b3,1a3] ondisk+write+known_if_redirected+supports_pool_eio e847)", 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:27:31.722991+0000", 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 332.35663274699999, 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012148166, 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 7988 2024-04-12T11:33:04.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.722991+0000", 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.722991+0000", 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.722993+0000", 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.95e-06 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.723265+0000", 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000272716 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.723286+0000", 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1112e-05 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.723296+0000", 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3500000000000003e-06 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.723365+0000", 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9762000000000001e-05 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.723483+0000", 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117996 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.723675+0000", 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019195599999999999 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.725421+0000", 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017450949999999999 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.729470+0000", 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040492339999999996 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.734984+0000", 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055140629999999996 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.735080+0000", 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6123000000000001e-05 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:27:31.735139+0000", 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8808999999999999e-05 2024-04-12T11:33:04.113 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:8511 5.3 5:cc16ccc4:::smithi02979277-12:head [append 0~0] snapc 1f2=[1f2,1f1,1f0,1ef,1ee,1ec,1e9,1e8,1e7,1e6,1e2,1de,1dc,1d9,1d2,1b3] ondisk+write+known_if_redirected+supports_pool_eio e903)", 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:28:27.675125+0000", 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 276.40449803500002, 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016453189, 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.114 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 8511 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675125+0000", 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675125+0000", 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675128+0000", 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.605e-06 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675138+0000", 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6039999999999999e-06 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675172+0000", 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4452000000000002e-05 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675185+0000", 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3107e-05 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675277+0000", 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2045000000000004e-05 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675283+0000", 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0190000000000003e-06 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.679258+0000", 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039751750000000001 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.679677+0000", 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041894300000000002 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.680092+0000", 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041523800000000001 2024-04-12T11:33:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.685148+0000", 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050558490000000003 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.688619+0000", 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034702909999999999 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.691446+0000", 2024-04-12T11:33:04.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002827651 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.691518+0000", 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1848999999999996e-05 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.691578+0000", 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0361e-05 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:8512 5.3 5:cc16ccc4:::smithi02979277-12:head [rollback 1b3] snapc 1f2=[1f2,1f1,1f0,1ef,1ee,1ec,1e9,1e8,1e7,1e6,1e2,1de,1dc,1d9,1d2,1b3] ondisk+write+known_if_redirected+supports_pool_eio e903)", 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:28:27.675220+0000", 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 276.404402936, 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016664583, 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 8512 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675220+0000", 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675220+0000", 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675221+0000", 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2900000000000003e-07 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675226+0000", 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9239999999999998e-06 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675237+0000", 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1141000000000001e-05 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675245+0000", 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8010000000000004e-06 2024-04-12T11:33:04.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675341+0000", 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5855999999999998e-05 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.675345+0000", 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1180000000000002e-06 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.681492+0000", 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061475269999999999 2024-04-12T11:33:04.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.681603+0000", 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011079299999999999 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.681856+0000", 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00025320900000000001 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.685159+0000", 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033020969999999999 2024-04-12T11:33:04.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.691050+0000", 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058910880000000001 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.691652+0000", 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00060184599999999996 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.691829+0000", 2024-04-12T11:33:04.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000177904 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:27.691885+0000", 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5550000000000002e-05 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:8554 5.3 5:c6d0755e:::smithi02979277-47:head [append 0~0] snapc 1f6=[1f6,1f5,1f3,1f2,1f1,1f0,1ef,1ee,1ec,1e7,1e6,1e2,1dc,1d9,1d2,1b3] ondisk+write+known_if_redirected+supports_pool_eio e911)", 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:28:35.703433+0000", 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.37619007500001, 2024-04-12T11:33:04.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018341564000000001, 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 8554 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703433+0000", 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703433+0000", 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703436+0000", 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.48e-06 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703444+0000", 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5809999999999997e-06 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703470+0000", 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5411000000000001e-05 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703482+0000", 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2391999999999999e-05 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703627+0000", 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014480600000000001 2024-04-12T11:33:04.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703634+0000", 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5190000000000003e-06 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.706968+0000", 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003333201 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.707097+0000", 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012971599999999999 2024-04-12T11:33:04.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.707324+0000", 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022683600000000001 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.719830+0000", 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012506264 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.720216+0000", 2024-04-12T11:33:04.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00038578400000000002 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.721633+0000", 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001417092 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.721707+0000", 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3481999999999999e-05 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.721775+0000", 2024-04-12T11:33:04.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7999999999999999e-05 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:8555 5.3 5:c6d0755e:::smithi02979277-47:head [rollback 1ee] snapc 1f6=[1f6,1f5,1f3,1f2,1f1,1f0,1ef,1ee,1ec,1e7,1e6,1e2,1dc,1d9,1d2,1b3] ondisk+write+known_if_redirected+supports_pool_eio e911)", 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:28:35.703513+0000", 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.37611017699999, 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018523191000000001, 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.134 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.135 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.135 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 8555 2024-04-12T11:33:04.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.135 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703513+0000", 2024-04-12T11:33:04.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703513+0000", 2024-04-12T11:33:04.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703514+0000", 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2600000000000002e-07 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703518+0000", 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2150000000000003e-06 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703527+0000", 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2010000000000005e-06 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703535+0000", 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5580000000000003e-06 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703715+0000", 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017966999999999999 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703721+0000", 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.353e-06 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.711108+0000", 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073872449999999997 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.711480+0000", 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00037167299999999999 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.711863+0000", 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00038273400000000003 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.721785+0000", 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099228130000000008 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.721831+0000", 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5642000000000001e-05 2024-04-12T11:33:04.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.721918+0000", 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7127999999999997e-05 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.721984+0000", 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5712999999999999e-05 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.722036+0000", 2024-04-12T11:33:04.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2519999999999999e-05 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:8556 5.3 5:c6d0755e:::smithi02979277-47:head [append 0~0] snapc 1f6=[1f6,1f5,1f3,1f2,1f1,1f0,1ef,1ee,1ec,1e7,1e6,1e2,1dc,1d9,1d2,1b3] ondisk+write+known_if_redirected+supports_pool_eio e911)", 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:28:35.703545+0000", 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.37607806699998, 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019104100999999998, 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.142 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 8556 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703545+0000", 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703545+0000", 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703546+0000", 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8400000000000004e-07 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703549+0000", 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2509999999999999e-06 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703556+0000", 2024-04-12T11:33:04.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.754e-06 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703562+0000", 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9150000000000001e-06 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703944+0000", 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00038256999999999999 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.703952+0000", 2024-04-12T11:33:04.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4440000000000002e-06 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.718484+0000", 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014532201 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.718612+0000", 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012817800000000001 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:33:04.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.718821+0000", 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020847800000000001 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.721863+0000", 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030424779999999999 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.722483+0000", 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00062012099999999995 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.722532+0000", 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8640999999999997e-05 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.722597+0000", 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5080999999999996e-05 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:28:35.722649+0000", 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2404999999999997e-05 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:8817 5.3 5:c6d0755e:::smithi02979277-47:head [append 0~0] snapc 204=[204,203,202,201,200,1ff,1fe,1fc,1fb,1fa,1f7,1f6,1f0,1e7,1e2] ondisk+write+known_if_redirected+supports_pool_eio e940)", 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:29:04.702821+0000", 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 239.37680267900001, 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013204036000000001, 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 8817 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702821+0000", 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702821+0000", 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702823+0000", 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.553e-06 2024-04-12T11:33:04.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702844+0000", 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0296999999999999e-05 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702865+0000", 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1977e-05 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702878+0000", 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2046e-05 2024-04-12T11:33:04.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.703015+0000", 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013760500000000001 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.703040+0000", 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4839999999999999e-05 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.708963+0000", 2024-04-12T11:33:04.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059235149999999999 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.709038+0000", 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4191999999999997e-05 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.709387+0000", 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034884 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.714843+0000", 2024-04-12T11:33:04.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054565459999999996 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.714871+0000", 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7832999999999999e-05 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.715886+0000", 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010153289999999999 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.715960+0000", 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3679999999999999e-05 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.716025+0000", 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4783000000000006e-05 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:8818 5.3 5:c6d0755e:::smithi02979277-47:head [rollback 1ff] snapc 204=[204,203,202,201,200,1ff,1fe,1fc,1fb,1fa,1f7,1f6,1f0,1e7,1e2] ondisk+write+known_if_redirected+supports_pool_eio e940)", 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:29:04.702909+0000", 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 239.37671412099999, 2024-04-12T11:33:04.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013799011, 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 8818 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702909+0000", 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702909+0000", 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702910+0000", 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1999999999999999e-07 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702914+0000", 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8589999999999998e-06 2024-04-12T11:33:04.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702937+0000", 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3643999999999999e-05 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.702947+0000", 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2150000000000007e-06 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.703112+0000", 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016543300000000001 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.703119+0000", 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2969999999999998e-06 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.710627+0000", 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075080809999999998 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.710733+0000", 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010568 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.710984+0000", 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00025138800000000001 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.714877+0000", 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038929490000000001 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.716070+0000", 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011929219999999999 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.716625+0000", 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00055474799999999998 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.716665+0000", 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0295e-05 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:29:04.716708+0000", 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2880000000000003e-05 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.14811.0:9900 5.3 5:c8fcb981:::smithi02979277-45:head [stat,delete] snapc 236=[236,235,233,232,22f,22e,22b,227,225,223] ondisk+write+known_if_redirected+supports_pool_eio e1045)", 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:30:49.737676+0000", 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 134.34194701199999, 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016920786, 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:33:04.162 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.14811", 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2526178583", 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 9900 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.737676+0000", 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:33:04.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.737676+0000", 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.737679+0000", 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.221e-06 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.737706+0000", 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7712000000000002e-05 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.737725+0000", 2024-04-12T11:33:04.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8597000000000002e-05 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.737736+0000", 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1588e-05 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.737815+0000", 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9028000000000003e-05 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2024-04-12T11:33:04.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.737823+0000", 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4540000000000001e-06 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.739151+0000", 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001327885 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.739271+0000", 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000119838 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:33:04.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.739514+0000", 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00024373999999999999 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.743010+0000", 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034956140000000002 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.747204+0000", 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041937270000000004 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.754457+0000", 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072536019999999996 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.754535+0000", 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7309000000000001e-05 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:30:49.754597+0000", 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2471000000000005e-05 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.168 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.169 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:33:04.169 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:33:04.169 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:33:04.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:04.170 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:04.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:04.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:03 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:33:04.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:03 smithi087 ceph-mon[89014]: osdmap e1090: 8 total, 8 up, 8 in 2024-04-12T11:33:04.271 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:04.299 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:04.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:04.401 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:04.428 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:04.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:04.530 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:04.553 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:04.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:04.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:04.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:33:04.655 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:04.684 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:04.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:04.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:04.786 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:04.813 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:04.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:04.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:04.915 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:04 smithi029 ceph-mon[93650]: pgmap v1316: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:04.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:04 smithi029 ceph-mon[93650]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:33:04.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:04 smithi029 ceph-mon[97787]: pgmap v1316: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:04.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:04 smithi029 ceph-mon[97787]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:33:04.943 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:04.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:04.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:05.045 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:05.073 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:05.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:05.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:05.175 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:05.203 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:05.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:05.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:04 smithi087 ceph-mon[89014]: pgmap v1316: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:05.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:04 smithi087 ceph-mon[89014]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:33:05.240 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:33:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:33:04] "GET /metrics HTTP/1.1" 200 34071 "" "Prometheus/2.43.0" 2024-04-12T11:33:05.304 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:05.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:05.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:05.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:05.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:05.455 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:05.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:05.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:05.558 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:05.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:05.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:05.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:05.688 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:05.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:05.716 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:05.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:05.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:05.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:05.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:05.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:05.972 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:05.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:05.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:05.997 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-04-12T11:33:06.074 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:06.104 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:06.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:06.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:06.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:06.229 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:06.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:06.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:06.331 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:06.359 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:06.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:06.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:06.461 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:06.486 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:06.488 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:06.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:06.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:06.497 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:06.556 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-04-12T11:33:06.557 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2024-04-12T11:33:06.557 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:33:06.559 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-12T11:33:06.560 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-12T11:33:06.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:06.610 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2024-04-12T11:33:06.610 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-12T11:33:06.610 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_138569 2024-04-12T11:33:06.610 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:33:06.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2024-04-12T11:33:06.640 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:06.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:06.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:06.742 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:06.783 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:06.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:06.884 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:06.925 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:06.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:06.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:06.980 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:06 smithi029 ceph-mon[97787]: pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:06.980 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:06.981 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:06.983 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:06 smithi029 ceph-mon[93650]: pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:06.983 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:06.983 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:07.028 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:07.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:07.068 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:07.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:07.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:07.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:07.211 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:07.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:07.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:06 smithi087 ceph-mon[89014]: pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-04-12T11:33:07.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:07.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:07.313 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:07.335 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:33:07.345 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:07.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:07.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:07.385 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:33:07.429 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:33:07.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:07.478 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:33:07.488 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:07.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:07.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:07.522 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:33:07.570 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:33:07.590 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:07.610 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3426 3410 213.11 213.125 0.00392033 0.00467803 2024-04-12T11:33:07.614 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:33:07.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:07.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:07.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:07.656 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:33:07.661 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:33:07.661 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:33:07.661 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:33:07.661 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:33:07.661 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:33:07.661 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:33:07.661 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:33:07.661 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:33:07.724 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:07.758 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:07.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:07.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:07.860 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:07.898 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:07.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:07.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:08.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:08.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:07 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/854151609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:33:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:07 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/854151609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:33:08.039 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:08.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:08.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:08.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:08.179 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:08.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:08.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:07 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/854151609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:33:08.280 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:08.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:08.317 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:08.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:08.418 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:08.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:08.457 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:08.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:08.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:08.597 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:08.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:08.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:08.610 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 7020 7004 218.853 224.625 0.00225503 0.00455428 2024-04-12T11:33:08.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:08.738 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:08.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:08.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:08.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:08.845 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:08 smithi029 ceph-mon[93650]: pgmap v1318: 89 pgs: 89 active+clean; 52 MiB data, 515 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s wr, 102 op/s 2024-04-12T11:33:08.845 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:08 smithi029 ceph-mon[97787]: pgmap v1318: 89 pgs: 89 active+clean; 52 MiB data, 515 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s wr, 102 op/s 2024-04-12T11:33:08.883 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:08.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:08.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:08.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:09.024 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:09.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:09.126 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:09.162 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:09.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:09.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:09.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:08 smithi087 ceph-mon[89014]: pgmap v1318: 89 pgs: 89 active+clean; 52 MiB data, 515 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s wr, 102 op/s 2024-04-12T11:33:09.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:09.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:09.302 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:09.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:09.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:09.446 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:09.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:09.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:09.548 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:09.583 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:09.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:09.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:09.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 10386 10370 216.017 210.375 0.00452392 0.00462036 2024-04-12T11:33:09.685 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:09.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:09.728 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:09.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:09.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:09.862 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:09.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:09.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:09.998 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:09.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:09.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:10.100 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:10.132 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:10.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:10.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:10.226 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:10.234 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:10.265 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:10.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:10.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:10.366 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:10.409 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:10.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:10.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:10.512 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:10.546 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:10.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:10.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:10.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 13458 13442 210.008 192 0.00690048 0.00475539 2024-04-12T11:33:10.625 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:33:10.648 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:10.710 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:10.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:10.811 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:10.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:10.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:10.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:10.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:10.970 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:10.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:10.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:10 smithi029 ceph-mon[93650]: pgmap v1319: 89 pgs: 89 active+clean; 221 MiB data, 953 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s wr, 439 op/s 2024-04-12T11:33:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:10 smithi029 ceph-mon[97787]: pgmap v1319: 89 pgs: 89 active+clean; 221 MiB data, 953 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s wr, 439 op/s 2024-04-12T11:33:11.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:11.110 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:11.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:11.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:11.212 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:11.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:10 smithi087 ceph-mon[89014]: pgmap v1319: 89 pgs: 89 active+clean; 221 MiB data, 953 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s wr, 439 op/s 2024-04-12T11:33:11.250 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:11.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:11.352 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:11.397 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:11.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:11.499 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:11.531 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:11.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:11.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:11.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 15741 15725 196.542 142.688 0.00598145 0.00508198 2024-04-12T11:33:11.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:11.673 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:11.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:11.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:11.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:11.799 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:11.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:11.900 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:11.924 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:11.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:11.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:12.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:12.063 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:12.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:12.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:12.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:12.204 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:12.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:12.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:12.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:12.346 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:12.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:12.447 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:12.482 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:12.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:12.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:12.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:12.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 17956 17940 186.856 138.438 0.00459943 0.0053453 2024-04-12T11:33:12.625 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:12.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:12.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:33:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:33:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:33:12.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:12.767 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:12.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:12.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:12.869 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:12.905 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:12.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:12.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:13.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:12 smithi029 ceph-mon[93650]: pgmap v1320: 89 pgs: 89 active+clean; 303 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s wr, 486 op/s 2024-04-12T11:33:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:12 smithi029 ceph-mon[97787]: pgmap v1320: 89 pgs: 89 active+clean; 303 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s wr, 486 op/s 2024-04-12T11:33:13.046 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:13.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:13.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:13.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:13.189 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:13.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:13.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:12 smithi087 ceph-mon[89014]: pgmap v1320: 89 pgs: 89 active+clean; 303 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s wr, 486 op/s 2024-04-12T11:33:13.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:13.330 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:13.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:13.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:13.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:13.457 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:13.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:13.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:13.559 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:13.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:13.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:13.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:13.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 20177 20161 179.991 138.812 0.174011 0.00553319 2024-04-12T11:33:13.684 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:13.708 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:13.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:13.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:13.810 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:13.841 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:13.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:13.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:13.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:13.986 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:13.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:14.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:14.125 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:14.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:14.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:14.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:13 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1627409499' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:33:14.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:13 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:33:14.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:14.258 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:14.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:14.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:13 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1627409499' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:33:14.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:13 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:33:14.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:13 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1627409499' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:33:14.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:13 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:33:14.359 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:14.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:14.391 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:14.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:14.492 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:14.530 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:14.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:14.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:14.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 21719 21703 169.537 96.375 0.00194425 0.00584789 2024-04-12T11:33:14.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:14.660 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:14.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:14.762 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:14.797 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:14.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:14.899 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:14.936 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:14.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:14.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:15.038 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:15.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:15.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:15.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:15.174 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:15.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:15.205 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:15.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:14 smithi087 ceph-mon[89014]: pgmap v1321: 89 pgs: 89 active+clean; 881 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-04-12T11:33:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:14 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:33:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:14 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:33:15.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:14 smithi087 ceph-mon[89014]: osdmap e1091: 8 total, 8 up, 8 in 2024-04-12T11:33:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:33:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:33:14] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2024-04-12T11:33:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:14 smithi029 ceph-mon[93650]: pgmap v1321: 89 pgs: 89 active+clean; 881 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-04-12T11:33:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:14 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:33:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:14 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:33:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:14 smithi029 ceph-mon[93650]: osdmap e1091: 8 total, 8 up, 8 in 2024-04-12T11:33:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:14 smithi029 ceph-mon[97787]: pgmap v1321: 89 pgs: 89 active+clean; 881 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-04-12T11:33:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:14 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:33:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:14 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:33:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:14 smithi029 ceph-mon[97787]: osdmap e1091: 8 total, 8 up, 8 in 2024-04-12T11:33:15.306 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:15.333 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:15.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:15.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:15.435 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:15.474 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:15.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:15.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:15.572 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:33:15.577 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:15.611 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:15.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:15.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:15.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 23655 23639 164.143 121 0.0038707 0.00608653 2024-04-12T11:33:15.713 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:15.736 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:15.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:15.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:15.837 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:15.863 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:15.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:15.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:15.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:16.005 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:16.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:16.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:16.107 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:16.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:16.142 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:16.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:16.243 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:16.278 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:16.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:16.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:16.380 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:16.414 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:16.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:16.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:16.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:16.555 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:16.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:16.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:16.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 25616 25600 159.984 122.562 0.00775567 0.00624466 2024-04-12T11:33:16.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:16.696 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:16.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:16.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:16.798 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:16.832 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:16.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:16.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:16.934 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:16.970 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:16.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:17.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:16 smithi029 ceph-mon[93650]: pgmap v1323: 89 pgs: 89 active+clean; 1.1 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-04-12T11:33:17.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:16 smithi029 ceph-mon[97787]: pgmap v1323: 89 pgs: 89 active+clean; 1.1 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-04-12T11:33:17.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:17.112 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:17.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:17.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:17.214 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:17.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:16 smithi087 ceph-mon[89014]: pgmap v1323: 89 pgs: 89 active+clean; 1.1 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-04-12T11:33:17.240 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:17.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:17.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:17.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:17.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:17.383 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:17.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:17.484 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:17.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:17.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:17.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 28034 28018 159.177 151.125 0.00233957 0.00627609 2024-04-12T11:33:17.620 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:17.643 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:17.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:17.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:17.782 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:17.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:17.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:17.884 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:17.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:17.923 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:17.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:18.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:18.060 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:18.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:18.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:18.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:18.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:18.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:18.288 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:18.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:18.324 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:18.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:18.425 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:18.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:18.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:18.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:18.561 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:18.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:18.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:18.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:18.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 29825 29809 155.24 111.938 0.00881464 0.00643669 2024-04-12T11:33:18.691 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:18.731 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:18.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:18.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:18.833 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:18.866 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:18.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:18.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:18.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:19.007 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:19.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:19.108 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:19.130 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:19.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:19.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:18 smithi087 ceph-mon[89014]: pgmap v1324: 89 pgs: 89 active+clean; 1.4 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-04-12T11:33:19.233 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:19.261 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:19.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:19.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:19.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:18 smithi029 ceph-mon[97787]: pgmap v1324: 89 pgs: 89 active+clean; 1.4 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-04-12T11:33:19.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:18 smithi029 ceph-mon[93650]: pgmap v1324: 89 pgs: 89 active+clean; 1.4 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-04-12T11:33:19.363 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:19.401 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:19.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:19.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:19.502 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:19.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:19.536 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:19.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:19.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 31652 31636 152.081 114.188 0.00485321 0.0065704 2024-04-12T11:33:19.637 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:19.670 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:19.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:19.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:19.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:19.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:19.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:19.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:19.916 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:19.939 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:19.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:19.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:20.041 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:20.076 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:20.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:20.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:20.177 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:20.211 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:20.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:20.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:20.313 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:20.350 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:20.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:20.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:20.452 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:20.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:20.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:20.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:20.611 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 33656 33640 150.164 125.25 0.00688491 0.0066537 2024-04-12T11:33:20.624 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:20.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:20.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:20.765 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:20.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:20.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:20.906 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:20.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:20.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:21.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:21.048 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:21.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:21.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:21.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:21.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:21.189 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:21.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:20 smithi087 ceph-mon[89014]: pgmap v1325: 89 pgs: 89 active+clean; 1.7 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2024-04-12T11:33:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:20 smithi029 ceph-mon[97787]: pgmap v1325: 89 pgs: 89 active+clean; 1.7 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2024-04-12T11:33:21.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:20 smithi029 ceph-mon[93650]: pgmap v1325: 89 pgs: 89 active+clean; 1.7 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2024-04-12T11:33:21.290 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:21.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:21.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:21.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:21.470 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:21.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:21.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:21.572 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:21.598 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:21.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:21.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 35746 35730 148.86 130.625 0.0100257 0.0067116 2024-04-12T11:33:21.699 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:21.734 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:21.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:21.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:21.835 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:21.870 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:21.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:21.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:21.972 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:22.006 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:22.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:22.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:22.108 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:22.117 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:22.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:22.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:22.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:22.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:22.256 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:22.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:22.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:22.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:22.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:22.358 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:22.393 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:22.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:22.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:22.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:33:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:33:22.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:33:22.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:22.532 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:22.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:22.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 37356 37340 145.845 100.625 0.00414996 0.00685148 2024-04-12T11:33:22.634 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:22.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:22.672 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:22.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:22.774 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:22.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:22.810 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:22.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:22.911 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:22.934 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:22.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:22.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:23.036 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:23.060 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:23.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:23.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:23.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:23.185 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:23.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:23.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:22 smithi087 ceph-mon[89014]: pgmap v1326: 89 pgs: 89 active+clean; 1.7 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-04-12T11:33:23.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:22 smithi029 ceph-mon[93650]: pgmap v1326: 89 pgs: 89 active+clean; 1.7 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-04-12T11:33:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:22 smithi029 ceph-mon[97787]: pgmap v1326: 89 pgs: 89 active+clean; 1.7 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-04-12T11:33:23.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:23.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:23.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:23.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:23.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:23.453 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:23.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:23.554 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:23.581 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:23.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:23.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:23.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 38431 38415 141.218 67.1875 0.00326172 0.00705783 2024-04-12T11:33:23.683 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:23.706 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:23.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:23.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:23.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:23.840 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:23.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:23.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:23.942 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:23.982 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:23.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:23.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:24.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:24.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:24.113 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:24.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:24.214 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:24.238 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:24.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:24.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:24.340 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:24.376 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:24.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:24.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:24.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:24.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:24.515 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:24.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:24.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 39516 39500 137.14 67.8125 0.0045861 0.00728684 2024-04-12T11:33:24.616 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:24.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:24.651 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:24.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:24.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:24.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:24.793 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:24.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:24.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:24.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:24.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:24.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:25.036 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:25.059 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:25.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:25.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:25.161 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:25.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:25.185 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:25.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:24 smithi087 ceph-mon[89014]: pgmap v1327: 89 pgs: 89 active+clean; 2.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-12T11:33:25.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:24 smithi029 ceph-mon[97787]: pgmap v1327: 89 pgs: 89 active+clean; 2.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-12T11:33:25.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:24 smithi029 ceph-mon[93650]: pgmap v1327: 89 pgs: 89 active+clean; 2.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-12T11:33:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:33:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:33:24] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.43.0" 2024-04-12T11:33:25.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:25.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:25.327 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:25.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:25.415 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:33:25.415 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:33:25.428 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:25.460 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:25.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:25.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:25.562 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:25.600 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:25.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:25.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:25.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 40943 40927 134.615 89.1875 0.0104177 0.00742183 2024-04-12T11:33:25.703 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:25.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:25.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:25.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:25.838 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:25.889 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:25.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:25.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:25.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:26.002 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:33:26.002 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:33:26.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:26.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:26.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:26.132 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:26.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:26.174 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:26.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:26.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:33:26.275 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:26.313 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:26.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:26.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:26.416 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:26.455 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:26.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:26.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:26.557 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:26.595 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:26.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:26.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:26.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:33:26.613129+0000 min lat: 0.00141572 max lat: 0.336491 avg lat: 0.00752652 2024-04-12T11:33:26.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:33:26.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 42503 42487 132.759 97.5 0.00656818 0.00752652 2024-04-12T11:33:26.698 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:26.731 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:26.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:26.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:26.832 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:26.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:26.874 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:26.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:26.975 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:27.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:27.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:27.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:27.103 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:27.141 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:27.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:27.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:26 smithi087 ceph-mon[89014]: pgmap v1328: 89 pgs: 89 active+clean; 2.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-04-12T11:33:27.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:27.253 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:26 smithi029 ceph-mon[93650]: pgmap v1328: 89 pgs: 89 active+clean; 2.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-04-12T11:33:27.254 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:26 smithi029 ceph-mon[97787]: pgmap v1328: 89 pgs: 89 active+clean; 2.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-04-12T11:33:27.280 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:27.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:27.382 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:27.414 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:27.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:27.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:27.516 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:27.551 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:27.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:27.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:27.612 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 44074 44058 131.112 98.1875 0.00438828 0.00762181 2024-04-12T11:33:27.653 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:27.685 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:27.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:27.787 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:27.829 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:27.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:27.893 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:27.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:28.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:28.002 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:28.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:28.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:28.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:28.154 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:28.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:28.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:28.296 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:28.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:28.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:28.436 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:28.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:28.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:28.481 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-04-12T11:33:28.538 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:28.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:28.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:28.613 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 45835 45819 130.155 110.062 0.0109964 0.00767863 2024-04-12T11:33:28.683 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:28.711 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:28.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:28.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:28.813 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:28.832 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:28 smithi087 ceph-mon[89014]: pgmap v1329: 89 pgs: 89 active+clean; 2.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-04-12T11:33:28.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:28.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:28.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:28.955 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:28 smithi029 ceph-mon[93650]: pgmap v1329: 89 pgs: 89 active+clean; 2.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-04-12T11:33:28.956 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:28 smithi029 ceph-mon[97787]: pgmap v1329: 89 pgs: 89 active+clean; 2.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-04-12T11:33:28.961 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:28.995 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:28.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:28.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:29.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:29.135 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:29.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:29.236 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:29.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:29.272 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:29.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:29.373 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:29.412 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:29.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:33:29.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:29.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:29.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:29.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:29.565 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:29.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:29.574 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:33:29.587 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:33:29.613 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 15 47402 47387 128.756 98 0.00399739 0.00776063 2024-04-12T11:33:29.666 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:29.699 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:29.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:29.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:29.801 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:29.850 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:29.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:29.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:29.996 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:29.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:29.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:30.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:30.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:30.137 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:30.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:30.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:30.278 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:30.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:30.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:30.380 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:30.412 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:30.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:30.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:30.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:30.553 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:30.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:30.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:30.613 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 49129 49113 127.886 107.875 0.00786952 0.00781367 2024-04-12T11:33:30.654 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:30.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:30.689 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:30.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:30.790 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:30.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:30.835 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:30.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:30.936 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:30.979 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:30.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:30.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1329,"stamp":"2024-04-12T11:33:27.655628+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2640582610,"num_objects":40307,"num_object_clones":0,"num_object_copies":120921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40307,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":40583,"num_write_kb":2580558,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":40499,"ondisk_log_size":40499,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8229644,"kb_used_data":7667148,"kb_used_omap":175,"kb_used_meta":562256,"kb_avail":741731572,"statfs":{"total":767960285184,"available":759533129728,"internally_reserved":0,"allocated":7851159552,"data_stored":7828944866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":575750313},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1413087232,"num_objects":21562,"num_object_clones":0,"num_object_copies":64686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21562,"num_write_kb":1379968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004527"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1531,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.779521+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:33:13.779521+0000","last_peered":"2024-04-12T11:33:13.779521+0000","last_clean":"2024-04-12T11:33:13.779521+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:33:13.779521+0000","last_undegraded":"2024-04-12T11:33:13.779521+0000","last_fullsized":"2024-04-12T11:33:13.779521+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1540,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.781205+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:33:13.781205+0000","last_peered":"2024-04-12T11:33:13.781205+0000","last_clean":"2024-04-12T11:33:13.781205+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:33:13.781205+0000","last_undegraded":"2024-04-12T11:33:13.781205+0000","last_fullsized":"2024-04-12T11:33:13.781205+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1591,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.786987+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:33:13.786987+0000","last_peered":"2024-04-12T11:33:13.786987+0000","last_clean":"2024-04-12T11:33:13.786987+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:33:13.786987+0000","last_undegraded":"2024-04-12T11:33:13.786987+0000","last_fullsized":"2024-04-12T11:33:13.786987+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1586,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788152+0000","last_change":"2024-04-12T11:23:15.593723+0000","last_active":"2024-04-12T11:33:13.788152+0000","last_peered":"2024-04-12T11:33:13.788152+0000","last_clean":"2024-04-12T11:33:13.788152+0000","last_became_active":"2024-04-12T11:23:15.593381+0000","last_became_peered":"2024-04-12T11:23:15.593381+0000","last_unstale":"2024-04-12T11:33:13.788152+0000","last_undegraded":"2024-04-12T11:33:13.788152+0000","last_fullsized":"2024-04-12T11:33:13.788152+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:18:06.971493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1548,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.781389+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:33:13.781389+0000","last_peered":"2024-04-12T11:33:13.781389+0000","last_clean":"2024-04-12T11:33:13.781389+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:33:13.781389+0000","last_undegraded":"2024-04-12T11:33:13.781389+0000","last_fullsized":"2024-04-12T11:33:13.781389+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1032,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.800967+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:33:02.800967+0000","last_peered":"2024-04-12T11:33:02.800967+0000","last_clean":"2024-04-12T11:33:02.800967+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:33:02.800967+0000","last_undegraded":"2024-04-12T11:33:02.800967+0000","last_fullsized":"2024-04-12T11:33:02.800967+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1582,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788516+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:33:13.788516+0000","last_peered":"2024-04-12T11:33:13.788516+0000","last_clean":"2024-04-12T11:33:13.788516+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:33:13.788516+0000","last_undegraded":"2024-04-12T11:33:13.788516+0000","last_fullsized":"2024-04-12T11:33:13.788516+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1521,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.781293+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:33:13.781293+0000","last_peered":"2024-04-12T11:33:13.781293+0000","last_clean":"2024-04-12T11:33:13.781293+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:33:13.781293+0000","last_undegraded":"2024-04-12T11:33:13.781293+0000","last_fullsized":"2024-04-12T11:33:13.781293+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1038,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.729917+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:33:02.729917+0000","last_peered":"2024-04-12T11:33:02.729917+0000","last_clean":"2024-04-12T11:33:02.729917+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:33:02.729917+0000","last_undegraded":"2024-04-12T11:33:02.729917+0000","last_fullsized":"2024-04-12T11:33:02.729917+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1501,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.780677+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:33:13.780677+0000","last_peered":"2024-04-12T11:33:13.780677+0000","last_clean":"2024-04-12T11:33:13.780677+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:33:13.780677+0000","last_undegraded":"2024-04-12T11:33:13.780677+0000","last_fullsized":"2024-04-12T11:33:13.780677+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1047,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.801036+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:33:02.801036+0000","last_peered":"2024-04-12T11:33:02.801036+0000","last_clean":"2024-04-12T11:33:02.801036+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:33:02.801036+0000","last_undegraded":"2024-04-12T11:33:02.801036+0000","last_fullsized":"2024-04-12T11:33:02.801036+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1582,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.787184+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:33:13.787184+0000","last_peered":"2024-04-12T11:33:13.787184+0000","last_clean":"2024-04-12T11:33:13.787184+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:33:13.787184+0000","last_undegraded":"2024-04-12T11:33:13.787184+0000","last_fullsized":"2024-04-12T11:33:13.787184+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1504,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.779935+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:33:13.779935+0000","last_peered":"2024-04-12T11:33:13.779935+0000","last_clean":"2024-04-12T11:33:13.779935+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:33:13.779935+0000","last_undegraded":"2024-04-12T11:33:13.779935+0000","last_fullsized":"2024-04-12T11:33:13.779935+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1495,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.779579+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:33:13.779579+0000","last_peered":"2024-04-12T11:33:13.779579+0000","last_clean":"2024-04-12T11:33:13.779579+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:33:13.779579+0000","last_undegraded":"2024-04-12T11:33:13.779579+0000","last_fullsized":"2024-04-12T11:33:13.779579+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1046,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.801536+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:33:02.801536+0000","last_peered":"2024-04-12T11:33:02.801536+0000","last_clean":"2024-04-12T11:33:02.801536+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:33:02.801536+0000","last_undegraded":"2024-04-12T11:33:02.801536+0000","last_fullsized":"2024-04-12T11:33:02.801536+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1532,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.781831+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:33:13.781831+0000","last_peered":"2024-04-12T11:33:13.781831+0000","last_clean":"2024-04-12T11:33:13.781831+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:33:13.781831+0000","last_undegraded":"2024-04-12T11:33:13.781831+0000","last_fullsized":"2024-04-12T11:33:13.781831+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1583,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.787718+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:33:13.787718+0000","last_peered":"2024-04-12T11:33:13.787718+0000","last_clean":"2024-04-12T11:33:13.787718+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:33:13.787718+0000","last_undegraded":"2024-04-12T11:33:13.787718+0000","last_fullsized":"2024-04-12T11:33:13.787718+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1037,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.730283+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:33:02.730283+0000","last_peered":"2024-04-12T11:33:02.730283+0000","last_clean":"2024-04-12T11:33:02.730283+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:33:02.730283+0000","last_undegraded":"2024-04-12T11:33:02.730283+0000","last_fullsized":"2024-04-12T11:33:02.730283+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1029,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.800367+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:33:02.800367+0000","last_peered":"2024-04-12T11:33:02.800367+0000","last_clean":"2024-04-12T11:33:02.800367+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:33:02.800367+0000","last_undegraded":"2024-04-12T11:33:02.800367+0000","last_fullsized":"2024-04-12T11:33:02.800367+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1584,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788047+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:33:13.788047+0000","last_peered":"2024-04-12T11:33:13.788047+0000","last_clean":"2024-04-12T11:33:13.788047+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:33:13.788047+0000","last_undegraded":"2024-04-12T11:33:13.788047+0000","last_fullsized":"2024-04-12T11:33:13.788047+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1040,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.800189+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:33:02.800189+0000","last_peered":"2024-04-12T11:33:02.800189+0000","last_clean":"2024-04-12T11:33:02.800189+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:33:02.800189+0000","last_undegraded":"2024-04-12T11:33:02.800189+0000","last_fullsized":"2024-04-12T11:33:02.800189+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1532,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.781355+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:33:13.781355+0000","last_peered":"2024-04-12T11:33:13.781355+0000","last_clean":"2024-04-12T11:33:13.781355+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:33:13.781355+0000","last_undegraded":"2024-04-12T11:33:13.781355+0000","last_fullsized":"2024-04-12T11:33:13.781355+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"1091'2641","reported_seq":2655,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:27.170250+0000","last_change":"2024-04-12T11:32:53.289719+0000","last_active":"2024-04-12T11:33:27.170250+0000","last_peered":"2024-04-12T11:33:27.170250+0000","last_clean":"2024-04-12T11:33:27.170250+0000","last_became_active":"2024-04-12T11:32:53.289368+0000","last_became_peered":"2024-04-12T11:32:53.289368+0000","last_unstale":"2024-04-12T11:33:27.170250+0000","last_undegraded":"2024-04-12T11:33:27.170250+0000","last_fullsized":"2024-04-12T11:33:27.170250+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2641,"log_dups_size":0,"ondisk_log_size":2641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:35:04.099561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173080576,"num_objects":2641,"num_object_clones":0,"num_object_copies":7923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2641,"num_write_kb":169024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1046,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.784135+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:33:13.784135+0000","last_peered":"2024-04-12T11:33:13.784135+0000","last_clean":"2024-04-12T11:33:13.784135+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:33:13.784135+0000","last_undegraded":"2024-04-12T11:33:13.784135+0000","last_fullsized":"2024-04-12T11:33:13.784135+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1023,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.729689+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:33:02.729689+0000","last_peered":"2024-04-12T11:33:02.729689+0000","last_clean":"2024-04-12T11:33:02.729689+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:33:02.729689+0000","last_undegraded":"2024-04-12T11:33:02.729689+0000","last_fullsized":"2024-04-12T11:33:02.729689+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1091'2369","reported_seq":2388,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:23.783009+0000","last_change":"2024-04-12T11:32:53.292425+0000","last_active":"2024-04-12T11:33:23.783009+0000","last_peered":"2024-04-12T11:33:23.783009+0000","last_clean":"2024-04-12T11:33:23.783009+0000","last_became_active":"2024-04-12T11:32:53.292129+0000","last_became_peered":"2024-04-12T11:32:53.292129+0000","last_unstale":"2024-04-12T11:33:23.783009+0000","last_undegraded":"2024-04-12T11:33:23.783009+0000","last_fullsized":"2024-04-12T11:33:23.783009+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2369,"log_dups_size":0,"ondisk_log_size":2369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:04:59.069449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155254784,"num_objects":2369,"num_object_clones":0,"num_object_copies":7107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2369,"num_write_kb":151616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1047,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.784188+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:33:13.784188+0000","last_peered":"2024-04-12T11:33:13.784188+0000","last_clean":"2024-04-12T11:33:13.784188+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:33:13.784188+0000","last_undegraded":"2024-04-12T11:33:13.784188+0000","last_fullsized":"2024-04-12T11:33:13.784188+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1517,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.780244+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:33:13.780244+0000","last_peered":"2024-04-12T11:33:13.780244+0000","last_clean":"2024-04-12T11:33:13.780244+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:33:13.780244+0000","last_undegraded":"2024-04-12T11:33:13.780244+0000","last_fullsized":"2024-04-12T11:33:13.780244+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1091'2750","reported_seq":2764,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:27.272417+0000","last_change":"2024-04-12T11:32:53.294371+0000","last_active":"2024-04-12T11:33:27.272417+0000","last_peered":"2024-04-12T11:33:27.272417+0000","last_clean":"2024-04-12T11:33:27.272417+0000","last_became_active":"2024-04-12T11:32:53.293969+0000","last_became_peered":"2024-04-12T11:32:53.293969+0000","last_unstale":"2024-04-12T11:33:27.272417+0000","last_undegraded":"2024-04-12T11:33:27.272417+0000","last_fullsized":"2024-04-12T11:33:27.272417+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2750,"log_dups_size":0,"ondisk_log_size":2750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:22:06.691519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180224000,"num_objects":2750,"num_object_clones":0,"num_object_copies":8250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2750,"num_write_kb":176000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1016,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:12.300333+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:33:12.300333+0000","last_peered":"2024-04-12T11:33:12.300333+0000","last_clean":"2024-04-12T11:33:12.300333+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:33:12.300333+0000","last_undegraded":"2024-04-12T11:33:12.300333+0000","last_fullsized":"2024-04-12T11:33:12.300333+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1032,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.800792+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:33:02.800792+0000","last_peered":"2024-04-12T11:33:02.800792+0000","last_clean":"2024-04-12T11:33:02.800792+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:33:02.800792+0000","last_undegraded":"2024-04-12T11:33:02.800792+0000","last_fullsized":"2024-04-12T11:33:02.800792+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1091'2375","reported_seq":2395,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:22.720644+0000","last_change":"2024-04-12T11:32:53.293287+0000","last_active":"2024-04-12T11:33:22.720644+0000","last_peered":"2024-04-12T11:33:22.720644+0000","last_clean":"2024-04-12T11:33:22.720644+0000","last_became_active":"2024-04-12T11:32:53.292980+0000","last_became_peered":"2024-04-12T11:32:53.292980+0000","last_unstale":"2024-04-12T11:33:22.720644+0000","last_undegraded":"2024-04-12T11:33:22.720644+0000","last_fullsized":"2024-04-12T11:33:22.720644+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2375,"log_dups_size":0,"ondisk_log_size":2375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:22:34.921585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155648000,"num_objects":2375,"num_object_clones":0,"num_object_copies":7125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2375,"num_write_kb":152000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1039,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:12.184469+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:33:12.184469+0000","last_peered":"2024-04-12T11:33:12.184469+0000","last_clean":"2024-04-12T11:33:12.184469+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:33:12.184469+0000","last_undegraded":"2024-04-12T11:33:12.184469+0000","last_fullsized":"2024-04-12T11:33:12.184469+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1594,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.787619+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:33:13.787619+0000","last_peered":"2024-04-12T11:33:13.787619+0000","last_clean":"2024-04-12T11:33:13.787619+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:33:13.787619+0000","last_undegraded":"2024-04-12T11:33:13.787619+0000","last_fullsized":"2024-04-12T11:33:13.787619+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1091'2484","reported_seq":2504,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:24.686037+0000","last_change":"2024-04-12T11:32:53.288446+0000","last_active":"2024-04-12T11:33:24.686037+0000","last_peered":"2024-04-12T11:33:24.686037+0000","last_clean":"2024-04-12T11:33:24.686037+0000","last_became_active":"2024-04-12T11:32:53.288017+0000","last_became_peered":"2024-04-12T11:32:53.288017+0000","last_unstale":"2024-04-12T11:33:24.686037+0000","last_undegraded":"2024-04-12T11:33:24.686037+0000","last_fullsized":"2024-04-12T11:33:24.686037+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2484,"log_dups_size":0,"ondisk_log_size":2484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:20:27.103744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162791424,"num_objects":2484,"num_object_clones":0,"num_object_copies":7452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2484,"num_write_kb":158976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1525,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.783708+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:33:13.783708+0000","last_peered":"2024-04-12T11:33:13.783708+0000","last_clean":"2024-04-12T11:33:13.783708+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:33:13.783708+0000","last_undegraded":"2024-04-12T11:33:13.783708+0000","last_fullsized":"2024-04-12T11:33:13.783708+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1508,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.780047+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:33:13.780047+0000","last_peered":"2024-04-12T11:33:13.780047+0000","last_clean":"2024-04-12T11:33:13.780047+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:33:13.780047+0000","last_undegraded":"2024-04-12T11:33:13.780047+0000","last_fullsized":"2024-04-12T11:33:13.780047+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1091'2430","reported_seq":2450,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:23.547767+0000","last_change":"2024-04-12T11:32:53.290372+0000","last_active":"2024-04-12T11:33:23.547767+0000","last_peered":"2024-04-12T11:33:23.547767+0000","last_clean":"2024-04-12T11:33:23.547767+0000","last_became_active":"2024-04-12T11:32:53.289969+0000","last_became_peered":"2024-04-12T11:32:53.289969+0000","last_unstale":"2024-04-12T11:33:23.547767+0000","last_undegraded":"2024-04-12T11:33:23.547767+0000","last_fullsized":"2024-04-12T11:33:23.547767+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2430,"log_dups_size":0,"ondisk_log_size":2430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:12:57.581355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159252480,"num_objects":2430,"num_object_clones":0,"num_object_copies":7290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2430,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1048,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.800766+0000","last_change":"2024-04-12T11:23:36.639270+0000","last_active":"2024-04-12T11:33:02.800766+0000","last_peered":"2024-04-12T11:33:02.800766+0000","last_clean":"2024-04-12T11:33:02.800766+0000","last_became_active":"2024-04-12T11:23:36.638952+0000","last_became_peered":"2024-04-12T11:23:36.638952+0000","last_unstale":"2024-04-12T11:33:02.800766+0000","last_undegraded":"2024-04-12T11:33:02.800766+0000","last_fullsized":"2024-04-12T11:33:02.800766+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:04:50.403812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1530,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.780162+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:33:13.780162+0000","last_peered":"2024-04-12T11:33:13.780162+0000","last_clean":"2024-04-12T11:33:13.780162+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:33:13.780162+0000","last_undegraded":"2024-04-12T11:33:13.780162+0000","last_fullsized":"2024-04-12T11:33:13.780162+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1091'2429","reported_seq":2449,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:23.567658+0000","last_change":"2024-04-12T11:32:53.288700+0000","last_active":"2024-04-12T11:33:23.567658+0000","last_peered":"2024-04-12T11:33:23.567658+0000","last_clean":"2024-04-12T11:33:23.567658+0000","last_became_active":"2024-04-12T11:32:53.288384+0000","last_became_peered":"2024-04-12T11:32:53.288384+0000","last_unstale":"2024-04-12T11:33:23.567658+0000","last_undegraded":"2024-04-12T11:33:23.567658+0000","last_fullsized":"2024-04-12T11:33:23.567658+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2429,"log_dups_size":0,"ondisk_log_size":2429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:31.496066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159186944,"num_objects":2429,"num_object_clones":0,"num_object_copies":7287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2429,"num_write_kb":155456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1516,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.780242+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:33:13.780242+0000","last_peered":"2024-04-12T11:33:13.780242+0000","last_clean":"2024-04-12T11:33:13.780242+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:33:13.780242+0000","last_undegraded":"2024-04-12T11:33:13.780242+0000","last_fullsized":"2024-04-12T11:33:13.780242+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1495,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.779800+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:33:13.779800+0000","last_peered":"2024-04-12T11:33:13.779800+0000","last_clean":"2024-04-12T11:33:13.779800+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:33:13.779800+0000","last_undegraded":"2024-04-12T11:33:13.779800+0000","last_fullsized":"2024-04-12T11:33:13.779800+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1091'2741","reported_seq":2755,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:27.179227+0000","last_change":"2024-04-12T11:32:53.289738+0000","last_active":"2024-04-12T11:33:27.179227+0000","last_peered":"2024-04-12T11:33:27.179227+0000","last_clean":"2024-04-12T11:33:27.179227+0000","last_became_active":"2024-04-12T11:32:53.289241+0000","last_became_peered":"2024-04-12T11:32:53.289241+0000","last_unstale":"2024-04-12T11:33:27.179227+0000","last_undegraded":"2024-04-12T11:33:27.179227+0000","last_fullsized":"2024-04-12T11:33:27.179227+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2741,"log_dups_size":0,"ondisk_log_size":2741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:39:02.390027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179634176,"num_objects":2741,"num_object_clones":0,"num_object_copies":8223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2741,"num_write_kb":175424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1533,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.781036+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:33:13.781036+0000","last_peered":"2024-04-12T11:33:13.781036+0000","last_clean":"2024-04-12T11:33:13.781036+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:33:13.781036+0000","last_undegraded":"2024-04-12T11:33:13.781036+0000","last_fullsized":"2024-04-12T11:33:13.781036+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1591,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.787316+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:33:13.787316+0000","last_peered":"2024-04-12T11:33:13.787316+0000","last_clean":"2024-04-12T11:33:13.787316+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:33:13.787316+0000","last_undegraded":"2024-04-12T11:33:13.787316+0000","last_fullsized":"2024-04-12T11:33:13.787316+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1091'2408","reported_seq":2428,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:22.708209+0000","last_change":"2024-04-12T11:32:53.292103+0000","last_active":"2024-04-12T11:33:22.708209+0000","last_peered":"2024-04-12T11:33:22.708209+0000","last_clean":"2024-04-12T11:33:22.708209+0000","last_became_active":"2024-04-12T11:32:53.291792+0000","last_became_peered":"2024-04-12T11:32:53.291792+0000","last_unstale":"2024-04-12T11:33:22.708209+0000","last_undegraded":"2024-04-12T11:33:22.708209+0000","last_fullsized":"2024-04-12T11:33:22.708209+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2408,"log_dups_size":0,"ondisk_log_size":2408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:36:23.973553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157810688,"num_objects":2408,"num_object_clones":0,"num_object_copies":7224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2408,"num_write_kb":154112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1593,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788607+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:33:13.788607+0000","last_peered":"2024-04-12T11:33:13.788607+0000","last_clean":"2024-04-12T11:33:13.788607+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:33:13.788607+0000","last_undegraded":"2024-04-12T11:33:13.788607+0000","last_fullsized":"2024-04-12T11:33:13.788607+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1054,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.730375+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:33:02.730375+0000","last_peered":"2024-04-12T11:33:02.730375+0000","last_clean":"2024-04-12T11:33:02.730375+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:33:02.730375+0000","last_undegraded":"2024-04-12T11:33:02.730375+0000","last_fullsized":"2024-04-12T11:33:02.730375+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1584,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.787229+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:33:13.787229+0000","last_peered":"2024-04-12T11:33:13.787229+0000","last_clean":"2024-04-12T11:33:13.787229+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:33:13.787229+0000","last_undegraded":"2024-04-12T11:33:13.787229+0000","last_fullsized":"2024-04-12T11:33:13.787229+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1091'2439","reported_seq":2459,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:23.540693+0000","last_change":"2024-04-12T11:32:53.293447+0000","last_active":"2024-04-12T11:33:23.540693+0000","last_peered":"2024-04-12T11:33:23.540693+0000","last_clean":"2024-04-12T11:33:23.540693+0000","last_became_active":"2024-04-12T11:32:53.293123+0000","last_became_peered":"2024-04-12T11:32:53.293123+0000","last_unstale":"2024-04-12T11:33:23.540693+0000","last_undegraded":"2024-04-12T11:33:23.540693+0000","last_fullsized":"2024-04-12T11:33:23.540693+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2439,"log_dups_size":0,"ondisk_log_size":2439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:22:25.118434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159842304,"num_objects":2439,"num_object_clones":0,"num_object_copies":7317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2439,"num_write_kb":156096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1044,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.730125+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:33:02.730125+0000","last_peered":"2024-04-12T11:33:02.730125+0000","last_clean":"2024-04-12T11:33:02.730125+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:33:02.730125+0000","last_undegraded":"2024-04-12T11:33:02.730125+0000","last_fullsized":"2024-04-12T11:33:02.730125+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1516,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.779479+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:33:13.779479+0000","last_peered":"2024-04-12T11:33:13.779479+0000","last_clean":"2024-04-12T11:33:13.779479+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:33:13.779479+0000","last_undegraded":"2024-04-12T11:33:13.779479+0000","last_fullsized":"2024-04-12T11:33:13.779479+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1593,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788231+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:33:13.788231+0000","last_peered":"2024-04-12T11:33:13.788231+0000","last_clean":"2024-04-12T11:33:13.788231+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:33:13.788231+0000","last_undegraded":"2024-04-12T11:33:13.788231+0000","last_fullsized":"2024-04-12T11:33:13.788231+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"1091'2690","reported_seq":2704,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:27.301949+0000","last_change":"2024-04-12T11:32:53.294617+0000","last_active":"2024-04-12T11:33:27.301949+0000","last_peered":"2024-04-12T11:33:27.301949+0000","last_clean":"2024-04-12T11:33:27.301949+0000","last_became_active":"2024-04-12T11:32:53.294285+0000","last_became_peered":"2024-04-12T11:32:53.294285+0000","last_unstale":"2024-04-12T11:33:27.301949+0000","last_undegraded":"2024-04-12T11:33:27.301949+0000","last_fullsized":"2024-04-12T11:33:27.301949+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2690,"log_dups_size":0,"ondisk_log_size":2690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:28:19.592399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176291840,"num_objects":2690,"num_object_clones":0,"num_object_copies":8070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2690,"num_write_kb":172160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1557,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788466+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:33:13.788466+0000","last_peered":"2024-04-12T11:33:13.788466+0000","last_clean":"2024-04-12T11:33:13.788466+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:33:13.788466+0000","last_undegraded":"2024-04-12T11:33:13.788466+0000","last_fullsized":"2024-04-12T11:33:13.788466+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1584,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.786800+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:33:13.786800+0000","last_peered":"2024-04-12T11:33:13.786800+0000","last_clean":"2024-04-12T11:33:13.786800+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:33:13.786800+0000","last_undegraded":"2024-04-12T11:33:13.786800+0000","last_fullsized":"2024-04-12T11:33:13.786800+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1587,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.787387+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:33:13.787387+0000","last_peered":"2024-04-12T11:33:13.787387+0000","last_clean":"2024-04-12T11:33:13.787387+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:33:13.787387+0000","last_undegraded":"2024-04-12T11:33:13.787387+0000","last_fullsized":"2024-04-12T11:33:13.787387+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1091'2751","reported_seq":2765,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:27.165232+0000","last_change":"2024-04-12T11:32:53.291594+0000","last_active":"2024-04-12T11:33:27.165232+0000","last_peered":"2024-04-12T11:33:27.165232+0000","last_clean":"2024-04-12T11:33:27.165232+0000","last_became_active":"2024-04-12T11:32:53.291267+0000","last_became_peered":"2024-04-12T11:32:53.291267+0000","last_unstale":"2024-04-12T11:33:27.165232+0000","last_undegraded":"2024-04-12T11:33:27.165232+0000","last_fullsized":"2024-04-12T11:33:27.165232+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2751,"log_dups_size":0,"ondisk_log_size":2751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:25:29.085176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180289536,"num_objects":2751,"num_object_clones":0,"num_object_copies":8253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2751,"num_write_kb":176064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1044,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.729732+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:33:02.729732+0000","last_peered":"2024-04-12T11:33:02.729732+0000","last_clean":"2024-04-12T11:33:02.729732+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:33:02.729732+0000","last_undegraded":"2024-04-12T11:33:02.729732+0000","last_fullsized":"2024-04-12T11:33:02.729732+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1398,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.800208+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:33:02.800208+0000","last_peered":"2024-04-12T11:33:02.800208+0000","last_clean":"2024-04-12T11:33:02.800208+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:33:02.800208+0000","last_undegraded":"2024-04-12T11:33:02.800208+0000","last_fullsized":"2024-04-12T11:33:02.800208+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1500,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.780901+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:33:13.780901+0000","last_peered":"2024-04-12T11:33:13.780901+0000","last_clean":"2024-04-12T11:33:13.780901+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:33:13.780901+0000","last_undegraded":"2024-04-12T11:33:13.780901+0000","last_fullsized":"2024-04-12T11:33:13.780901+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1032,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.800694+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:33:02.800694+0000","last_peered":"2024-04-12T11:33:02.800694+0000","last_clean":"2024-04-12T11:33:02.800694+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:33:02.800694+0000","last_undegraded":"2024-04-12T11:33:02.800694+0000","last_fullsized":"2024-04-12T11:33:02.800694+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1091'2362","reported_seq":2382,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:22.650848+0000","last_change":"2024-04-12T11:32:53.290316+0000","last_active":"2024-04-12T11:33:22.650848+0000","last_peered":"2024-04-12T11:33:22.650848+0000","last_clean":"2024-04-12T11:33:22.650848+0000","last_became_active":"2024-04-12T11:32:53.289996+0000","last_became_peered":"2024-04-12T11:32:53.289996+0000","last_unstale":"2024-04-12T11:33:22.650848+0000","last_undegraded":"2024-04-12T11:33:22.650848+0000","last_fullsized":"2024-04-12T11:33:22.650848+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2362,"log_dups_size":0,"ondisk_log_size":2362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:37:16.480758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154796032,"num_objects":2362,"num_object_clones":0,"num_object_copies":7086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2362,"num_write_kb":151168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1051,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.800528+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:33:02.800528+0000","last_peered":"2024-04-12T11:33:02.800528+0000","last_clean":"2024-04-12T11:33:02.800528+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:33:02.800528+0000","last_undegraded":"2024-04-12T11:33:02.800528+0000","last_fullsized":"2024-04-12T11:33:02.800528+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1594,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788749+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:33:13.788749+0000","last_peered":"2024-04-12T11:33:13.788749+0000","last_clean":"2024-04-12T11:33:13.788749+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:33:13.788749+0000","last_undegraded":"2024-04-12T11:33:13.788749+0000","last_fullsized":"2024-04-12T11:33:13.788749+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1594,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.787949+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:33:13.787949+0000","last_peered":"2024-04-12T11:33:13.787949+0000","last_clean":"2024-04-12T11:33:13.787949+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:33:13.787949+0000","last_undegraded":"2024-04-12T11:33:13.787949+0000","last_fullsized":"2024-04-12T11:33:13.787949+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1091'2305","reported_seq":2325,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:22.693022+0000","last_change":"2024-04-12T11:32:53.288166+0000","last_active":"2024-04-12T11:33:22.693022+0000","last_peered":"2024-04-12T11:33:22.693022+0000","last_clean":"2024-04-12T11:33:22.693022+0000","last_became_active":"2024-04-12T11:32:53.287852+0000","last_became_peered":"2024-04-12T11:32:53.287852+0000","last_unstale":"2024-04-12T11:33:22.693022+0000","last_undegraded":"2024-04-12T11:33:22.693022+0000","last_fullsized":"2024-04-12T11:33:22.693022+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2305,"log_dups_size":0,"ondisk_log_size":2305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:30:02.625450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151060480,"num_objects":2305,"num_object_clones":0,"num_object_copies":6915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2305,"num_write_kb":147520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1051,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.801127+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:33:02.801127+0000","last_peered":"2024-04-12T11:33:02.801127+0000","last_clean":"2024-04-12T11:33:02.801127+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:33:02.801127+0000","last_undegraded":"2024-04-12T11:33:02.801127+0000","last_fullsized":"2024-04-12T11:33:02.801127+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1584,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788451+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:33:13.788451+0000","last_peered":"2024-04-12T11:33:13.788451+0000","last_clean":"2024-04-12T11:33:13.788451+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:33:13.788451+0000","last_undegraded":"2024-04-12T11:33:13.788451+0000","last_fullsized":"2024-04-12T11:33:13.788451+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1577,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788731+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:33:13.788731+0000","last_peered":"2024-04-12T11:33:13.788731+0000","last_clean":"2024-04-12T11:33:13.788731+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:33:13.788731+0000","last_undegraded":"2024-04-12T11:33:13.788731+0000","last_fullsized":"2024-04-12T11:33:13.788731+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1091'2398","reported_seq":2418,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:24.684297+0000","last_change":"2024-04-12T11:32:53.290050+0000","last_active":"2024-04-12T11:33:24.684297+0000","last_peered":"2024-04-12T11:33:24.684297+0000","last_clean":"2024-04-12T11:33:24.684297+0000","last_became_active":"2024-04-12T11:32:53.288647+0000","last_became_peered":"2024-04-12T11:32:53.288647+0000","last_unstale":"2024-04-12T11:33:24.684297+0000","last_undegraded":"2024-04-12T11:33:24.684297+0000","last_fullsized":"2024-04-12T11:33:24.684297+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2398,"log_dups_size":0,"ondisk_log_size":2398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:13:07.256170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157155328,"num_objects":2398,"num_object_clones":0,"num_object_copies":7194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2398,"num_write_kb":153472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1516,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.781446+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:33:13.781446+0000","last_peered":"2024-04-12T11:33:13.781446+0000","last_clean":"2024-04-12T11:33:13.781446+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:33:13.781446+0000","last_undegraded":"2024-04-12T11:33:13.781446+0000","last_fullsized":"2024-04-12T11:33:13.781446+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1584,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.787886+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:33:13.787886+0000","last_peered":"2024-04-12T11:33:13.787886+0000","last_clean":"2024-04-12T11:33:13.787886+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:33:13.787886+0000","last_undegraded":"2024-04-12T11:33:13.787886+0000","last_fullsized":"2024-04-12T11:33:13.787886+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1514,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.780761+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:33:13.780761+0000","last_peered":"2024-04-12T11:33:13.780761+0000","last_clean":"2024-04-12T11:33:13.780761+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:33:13.780761+0000","last_undegraded":"2024-04-12T11:33:13.780761+0000","last_fullsized":"2024-04-12T11:33:13.780761+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1091'2711","reported_seq":2725,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:27.296755+0000","last_change":"2024-04-12T11:32:53.292930+0000","last_active":"2024-04-12T11:33:27.296755+0000","last_peered":"2024-04-12T11:33:27.296755+0000","last_clean":"2024-04-12T11:33:27.296755+0000","last_became_active":"2024-04-12T11:32:53.292611+0000","last_became_peered":"2024-04-12T11:32:53.292611+0000","last_unstale":"2024-04-12T11:33:27.296755+0000","last_undegraded":"2024-04-12T11:33:27.296755+0000","last_fullsized":"2024-04-12T11:33:27.296755+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:32:52.272733+0000","last_clean_scrub_stamp":"2024-04-12T11:32:52.272733+0000","objects_scrubbed":0,"log_size":2711,"log_dups_size":0,"ondisk_log_size":2711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:38:21.601732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177668096,"num_objects":2711,"num_object_clones":0,"num_object_copies":8133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2711,"num_write_kb":173504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1039,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.785042+0000","last_change":"2024-04-12T11:22:40.580200+0000","last_active":"2024-04-12T11:33:13.785042+0000","last_peered":"2024-04-12T11:33:13.785042+0000","last_clean":"2024-04-12T11:33:13.785042+0000","last_became_active":"2024-04-12T11:22:40.579809+0000","last_became_peered":"2024-04-12T11:22:40.579809+0000","last_unstale":"2024-04-12T11:33:13.785042+0000","last_undegraded":"2024-04-12T11:33:13.785042+0000","last_fullsized":"2024-04-12T11:33:13.785042+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:28:23.317249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1597,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788243+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:33:13.788243+0000","last_peered":"2024-04-12T11:33:13.788243+0000","last_clean":"2024-04-12T11:33:13.788243+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:33:13.788243+0000","last_undegraded":"2024-04-12T11:33:13.788243+0000","last_fullsized":"2024-04-12T11:33:13.788243+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1503,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.780075+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:33:13.780075+0000","last_peered":"2024-04-12T11:33:13.780075+0000","last_clean":"2024-04-12T11:33:13.780075+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:33:13.780075+0000","last_undegraded":"2024-04-12T11:33:13.780075+0000","last_fullsized":"2024-04-12T11:33:13.780075+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1587,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.787783+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:33:13.787783+0000","last_peered":"2024-04-12T11:33:13.787783+0000","last_clean":"2024-04-12T11:33:13.787783+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:33:13.787783+0000","last_undegraded":"2024-04-12T11:33:13.787783+0000","last_fullsized":"2024-04-12T11:33:13.787783+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1568,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.784537+0000","last_change":"2024-04-12T11:23:15.593766+0000","last_active":"2024-04-12T11:33:13.784537+0000","last_peered":"2024-04-12T11:33:13.784537+0000","last_clean":"2024-04-12T11:33:13.784537+0000","last_became_active":"2024-04-12T11:23:15.593428+0000","last_became_peered":"2024-04-12T11:23:15.593428+0000","last_unstale":"2024-04-12T11:33:13.784537+0000","last_undegraded":"2024-04-12T11:33:13.784537+0000","last_fullsized":"2024-04-12T11:33:13.784537+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:12.389667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1055,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.730125+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:33:02.730125+0000","last_peered":"2024-04-12T11:33:02.730125+0000","last_clean":"2024-04-12T11:33:02.730125+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:33:02.730125+0000","last_undegraded":"2024-04-12T11:33:02.730125+0000","last_fullsized":"2024-04-12T11:33:02.730125+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1531,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.779662+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:33:13.779662+0000","last_peered":"2024-04-12T11:33:13.779662+0000","last_clean":"2024-04-12T11:33:13.779662+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:33:13.779662+0000","last_undegraded":"2024-04-12T11:33:13.779662+0000","last_fullsized":"2024-04-12T11:33:13.779662+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1040,"reported_epoch":1090,"state":"active+clean","last_fresh":"2024-04-12T11:33:02.730471+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:33:02.730471+0000","last_peered":"2024-04-12T11:33:02.730471+0000","last_clean":"2024-04-12T11:33:02.730471+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:33:02.730471+0000","last_undegraded":"2024-04-12T11:33:02.730471+0000","last_fullsized":"2024-04-12T11:33:02.730471+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1522,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.779392+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:33:13.779392+0000","last_peered":"2024-04-12T11:33:13.779392+0000","last_clean":"2024-04-12T11:33:13.779392+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:33:13.779392+0000","last_undegraded":"2024-04-12T11:33:13.779392+0000","last_fullsized":"2024-04-12T11:33:13.779392+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1515,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.783672+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:33:13.783672+0000","last_peered":"2024-04-12T11:33:13.783672+0000","last_clean":"2024-04-12T11:33:13.783672+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:33:13.783672+0000","last_undegraded":"2024-04-12T11:33:13.783672+0000","last_fullsized":"2024-04-12T11:33:13.783672+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1526,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.780801+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:33:13.780801+0000","last_peered":"2024-04-12T11:33:13.780801+0000","last_clean":"2024-04-12T11:33:13.780801+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:33:13.780801+0000","last_undegraded":"2024-04-12T11:33:13.780801+0000","last_fullsized":"2024-04-12T11:33:13.780801+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1591,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.788424+0000","last_change":"2024-04-12T11:22:40.582361+0000","last_active":"2024-04-12T11:33:13.788424+0000","last_peered":"2024-04-12T11:33:13.788424+0000","last_clean":"2024-04-12T11:33:13.788424+0000","last_became_active":"2024-04-12T11:22:40.581996+0000","last_became_peered":"2024-04-12T11:22:40.581996+0000","last_unstale":"2024-04-12T11:33:13.788424+0000","last_undegraded":"2024-04-12T11:33:13.788424+0000","last_fullsized":"2024-04-12T11:33:13.788424+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:40:43.563258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1495,"reported_epoch":1091,"state":"active+clean","last_fresh":"2024-04-12T11:33:13.779410+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:33:13.779410+0000","last_peered":"2024-04-12T11:33:13.779410+0000","last_clean":"2024-04-12T11:33:13.779410+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:33:13.779410+0000","last_undegraded":"2024-04-12T11:33:13.779410+0000","last_fullsized":"2024-04-12T11:33:13.779410+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2639986688,"num_objects":40283,"num_object_clones":0,"num_object_copies":120849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":40283,"num_write_kb":2578112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7880310784,"data_stored":7880310784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":40283,"ondisk_log_size":40283,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234253,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1131732,"kb_used_data":1050636,"kb_used_omap":1,"kb_used_meta":81086,"kb_avail":92613420,"statfs":{"total":95995035648,"available":94836142080,"internally_reserved":0,"allocated":1075851264,"data_stored":1073058706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":83032523},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495890,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":928904,"kb_used_data":853568,"kb_used_omap":1,"kb_used_meta":75326,"kb_avail":92816248,"statfs":{"total":95995035648,"available":95043837952,"internally_reserved":0,"allocated":874053632,"data_stored":871273216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":77134280},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536199,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":796916,"kb_used_data":742764,"kb_used_omap":30,"kb_used_meta":54113,"kb_avail":92948236,"statfs":{"total":95995035648,"available":95178993664,"internally_reserved":0,"allocated":760590336,"data_stored":757825042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":55412034},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190283,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":658688,"kb_used_data":600248,"kb_used_omap":43,"kb_used_meta":58388,"kb_avail":93086464,"statfs":{"total":95995035648,"available":95320539136,"internally_reserved":0,"allocated":614653952,"data_stored":611877518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":59789355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849411,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1318076,"kb_used_data":1231924,"kb_used_omap":36,"kb_used_meta":86107,"kb_avail":92427076,"statfs":{"total":95995035648,"available":94645325824,"internally_reserved":0,"allocated":1261490176,"data_stored":1258717306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":88173748},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704764,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1585208,"kb_used_data":1531248,"kb_used_omap":17,"kb_used_meta":53934,"kb_avail":92159944,"statfs":{"total":95995035648,"available":94371782656,"internally_reserved":0,"allocated":1567997952,"data_stored":1565228652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":55228919},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225017976,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":847116,"kb_used_data":801092,"kb_used_omap":42,"kb_used_meta":45973,"kb_avail":92898036,"statfs":{"total":95995035648,"available":95127588864,"internally_reserved":0,"allocated":820318208,"data_stored":817533240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":47077176},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757526,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":963004,"kb_used_data":855668,"kb_used_omap":1,"kb_used_meta":107326,"kb_avail":92782148,"statfs":{"total":95995035648,"available":95008919552,"internally_reserved":0,"allocated":876204032,"data_stored":873431186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":109902278},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":619773952,"data_stored":619773952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":767033344,"data_stored":767033344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1259667456,"data_stored":1259667456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1578696704,"data_stored":1578696704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":813498368,"data_stored":813498368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":890830848,"data_stored":890830848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":881197056,"data_stored":881197056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1069613056,"data_stored":1069613056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:33:31.001 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-04-12T11:33:31.002 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool set unique_pool_1 pg_num 26 2024-04-12T11:33:31.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:31.135 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:31.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:31.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:31.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:30 smithi029 ceph-mon[93650]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:33:31.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:30 smithi029 ceph-mon[93650]: pgmap v1330: 89 pgs: 89 active+clean; 2.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-04-12T11:33:31.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:31.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:30 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:31.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:30 smithi029 ceph-mon[97787]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:33:31.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:30 smithi029 ceph-mon[97787]: pgmap v1330: 89 pgs: 89 active+clean; 2.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-04-12T11:33:31.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:30 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:31.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:30 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:30 smithi087 ceph-mon[89014]: from='client.44693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:33:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:30 smithi087 ceph-mon[89014]: pgmap v1330: 89 pgs: 89 active+clean; 2.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-04-12T11:33:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:30 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:30 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:31.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:31.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:31.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:31.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:31.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:31.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:31.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:31.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:31.523 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:31.558 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:31.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:31.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:31.613 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 50713 50697 126.73 99 0.00502972 0.00788535 2024-04-12T11:33:31.660 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:31.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:31.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:31.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:31.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:31.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:31.868 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:31.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:31.969 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:32.003 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:32.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:32.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:32.105 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:32.141 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:32.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:32.242 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:32.287 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:32.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:32.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:32.388 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:32.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:32.413 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:32.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:32.514 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:32.551 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:32.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:32.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:32.613 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 52250 52234 125.55 96.0625 0.00817117 0.00796001 2024-04-12T11:33:32.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:32.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:32.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:32.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:33:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:33:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:33:32.793 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:32.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:32.817 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:32.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:32.918 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:32.953 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:32.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:32.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:33.055 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:33.091 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:33.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:33.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:33.193 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:32 smithi087 ceph-mon[89014]: pgmap v1331: 89 pgs: 89 active+clean; 2.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-04-12T11:33:33.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:33.217 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:33.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:33.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:32 smithi029 ceph-mon[93650]: pgmap v1331: 89 pgs: 89 active+clean; 2.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-04-12T11:33:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:32 smithi029 ceph-mon[97787]: pgmap v1331: 89 pgs: 89 active+clean; 2.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-04-12T11:33:33.317 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:33.341 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:33.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:33.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:33.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:33.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:33.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:33.586 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:33.613 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 53560 53544 123.932 81.875 0.0107873 0.00806328 2024-04-12T11:33:33.621 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:33.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:33.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:33.722 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:33.753 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:33.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:33.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:33.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:33.898 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:33.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:33.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:33.999 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:34.000 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:33:34.000 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:33:34.000 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:33:34.000 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:33:34.000 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:33:34.012 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:33:34.032 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:34.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:34.055 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:33:34.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:33 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3558941044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:33:34.098 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:33 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3558941044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:33:34.098 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:33:34.104 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:33:34.105 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:33:34.105 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:33:34.105 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:33:34.105 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:33:34.105 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:33:34.105 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:33:34.105 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:33:34.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:34.197 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:34.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:34.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:34.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:33 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3558941044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:33:34.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:34.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:34.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:34.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:34.444 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:34.446 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:34.448 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:34.449 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:34.480 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:34.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:34.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:34.582 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:34.613 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 54870 54854 122.429 81.875 0.00651198 0.0081634 2024-04-12T11:33:34.616 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:34.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:34.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:34.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:34.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:34.765 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:34.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:34.866 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:34.898 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:34.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:34.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:35.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:35.126 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:34 smithi029 ceph-mon[97787]: pgmap v1332: 89 pgs: 89 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-04-12T11:33:35.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:34 smithi029 ceph-mon[93650]: pgmap v1332: 89 pgs: 89 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-04-12T11:33:35.126 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:33:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:33:34] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.43.0" 2024-04-12T11:33:35.128 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:35.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:35.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:34 smithi087 ceph-mon[89014]: pgmap v1332: 89 pgs: 89 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-04-12T11:33:35.232 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:35.261 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:35.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:35.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:35.363 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:35.406 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:35.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:35.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:35.508 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:35.545 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:35.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:35.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:35.614 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 56541 56525 121.808 104.438 0.00770373 0.0082041 2024-04-12T11:33:35.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:35.686 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:35.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:35.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:35.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:35.832 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:35.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:35.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:35.934 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:35.968 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:35.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:35.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:36.070 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:36.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:36.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:36.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:36.204 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:36.241 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:36.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:36.343 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:36.384 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:36.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:36.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:36.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:36.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:36.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:36.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:36.618 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 58153 58137 121.088 100.75 0.0120504 0.0082533 2024-04-12T11:33:36.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:36.654 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:36.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:36.756 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:36.797 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:36.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:36.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:36.899 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:36.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:36.932 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:36.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:37.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:37.068 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:37.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:37.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:37.125 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:36 smithi029 ceph-mon[97787]: pgmap v1333: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-12T11:33:37.125 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:37.125 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:36 smithi029 ceph-mon[93650]: pgmap v1333: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-12T11:33:37.125 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:37.170 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:37.205 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:37.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:37.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:36 smithi087 ceph-mon[89014]: pgmap v1333: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-12T11:33:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:37.308 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:37.351 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:37.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:37.453 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:37.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:37.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:37.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:37.586 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:37.618 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:37.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:37.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:37.619 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 59745 59729 120.387 99.5 0.00521451 0.00830111 2024-04-12T11:33:37.720 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:37.757 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:37.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:37.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:37.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:37.921 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:37.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:37.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:38.023 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:38.071 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:38.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:38.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:38.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:38.216 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:38.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:38.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:38.319 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:38.351 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:38.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:38.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:38.453 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:38.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:38.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:38.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:38.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:33:38.586 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:38.614 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:38.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:38.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:38.619 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 61176 61160 119.419 89.4375 0.0146985 0.00836676 2024-04-12T11:33:38.716 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:38.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:38.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:38.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:38.871 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:38.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:38.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:38.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:38.999 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:39.024 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:39.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:39.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:39.126 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:39.153 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:39.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:39.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:39.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:38 smithi029 ceph-mon[93650]: pgmap v1334: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-12T11:33:39.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:38 smithi029 ceph-mon[97787]: pgmap v1334: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-12T11:33:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:38 smithi087 ceph-mon[89014]: pgmap v1334: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-12T11:33:39.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:39.293 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:39.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:39.395 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:39.427 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:39.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:39.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:39.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:39.569 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:39.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:39.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:39.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 62179 62163 117.7 62.6875 0.0111991 0.00849014 2024-04-12T11:33:39.671 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:39.698 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:39.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:39.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:39.756 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:39.800 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:39.830 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:39.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:39.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:39.878 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:39 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3013175390' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:33:39.879 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:39 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1358130183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:33:39.879 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:39 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:33:39.901 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:33:39.902 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 6 pg_num to 26 2024-04-12T11:33:39.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:39.977 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:39.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:39.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:40.079 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:40.104 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:40.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:40.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:40.130 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:39 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3013175390' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:33:40.130 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:39 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1358130183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:33:40.130 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:39 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:33:40.206 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:40.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:39 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3013175390' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:33:40.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:39 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1358130183' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:33:40.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:39 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:33:40.239 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:40.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:40.341 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:40.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:40.380 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:40.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:40.481 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:40.531 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:40.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:40.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 63316 63300 116.329 71.0625 0.0185842 0.00858982 2024-04-12T11:33:40.632 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:40.663 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:40.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:40.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:40.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:40.803 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:40.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:40.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:40.904 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:40.946 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:40.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:40.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:41.048 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:41.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[93650]: pgmap v1335: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-04-12T11:33:41.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:33:41.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:33:41.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3013175390' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-04-12T11:33:41.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:33:41.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[93650]: osdmap e1092: 8 total, 8 up, 8 in 2024-04-12T11:33:41.081 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[97787]: pgmap v1335: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-04-12T11:33:41.081 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:33:41.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:33:41.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3013175390' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-04-12T11:33:41.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:33:41.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:40 smithi029 ceph-mon[97787]: osdmap e1092: 8 total, 8 up, 8 in 2024-04-12T11:33:41.089 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:41.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:41.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:41.191 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:40 smithi087 ceph-mon[89014]: pgmap v1335: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-04-12T11:33:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:40 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:33:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:40 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:33:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:40 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3013175390' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2024-04-12T11:33:41.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:40 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:33:41.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:40 smithi087 ceph-mon[89014]: osdmap e1092: 8 total, 8 up, 8 in 2024-04-12T11:33:41.225 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:41.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:41.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:41.326 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:41.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:41.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:41.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:41.459 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:41.500 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:41.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:41.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:41.602 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:41.619 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 64808 64792 115.669 93.25 0.0100497 0.00864046 2024-04-12T11:33:41.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:41.636 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:41.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:41.736 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:41.771 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:41.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:41.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:41.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:41.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:41.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:41.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2024-04-12T11:33:41.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2024-04-12T11:33:42.005 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:42.038 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:42.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:42.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:42.127 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:33:42.128 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:33:42.128 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:33:42.128 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:33:42.128 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:33:42.140 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:42.164 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:42.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:42.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2024-04-12T11:33:42.266 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:42.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:42.290 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:42.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:42.391 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:42.414 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:42.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:42.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:42.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:42.540 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:42.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:42.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:42.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 65220 65204 113.172 25.75 0.132509 0.00865341 2024-04-12T11:33:42.642 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:42.666 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:42.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:33:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:33:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:33:42.768 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:42.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:42.793 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:42.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:42.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:33:42.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:42.961 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:42.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:42.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-04-12T11:33:43.062 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:43.093 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:43.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:43.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:43.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:42 smithi029 ceph-mon[93650]: pgmap v1337: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-12T11:33:43.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-04-12T11:33:43.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:42 smithi029 ceph-mon[93650]: osdmap e1093: 8 total, 8 up, 8 in 2024-04-12T11:33:43.161 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:42 smithi029 ceph-mon[97787]: pgmap v1337: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-12T11:33:43.161 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-04-12T11:33:43.161 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:42 smithi029 ceph-mon[97787]: osdmap e1093: 8 total, 8 up, 8 in 2024-04-12T11:33:43.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:42 smithi087 ceph-mon[89014]: pgmap v1337: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-12T11:33:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2024-04-12T11:33:43.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:42 smithi087 ceph-mon[89014]: osdmap e1093: 8 total, 8 up, 8 in 2024-04-12T11:33:43.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:43.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:43.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:43.329 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:43.361 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:43.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:43.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:43.463 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:43.492 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:43.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:43.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:43.594 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:43.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 66216 66200 111.795 62.25 0.0118375 0.00893856 2024-04-12T11:33:43.625 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:43.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:43.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:43.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:43.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:43.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:43.861 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:43.892 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:43.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:43.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:43.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:44.029 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:44.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:44.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:44.131 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:44.172 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:44.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:44.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:44.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:43 smithi087 ceph-mon[89014]: osdmap e1094: 8 total, 8 up, 8 in 2024-04-12T11:33:44.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:43 smithi087 ceph-mon[89014]: 6.1 scrub starts 2024-04-12T11:33:44.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:43 smithi087 ceph-mon[89014]: 6.2 scrub starts 2024-04-12T11:33:44.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:43 smithi087 ceph-mon[89014]: 6.1 scrub ok 2024-04-12T11:33:44.251 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:44.263 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:33:44.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:43 smithi029 ceph-mon[93650]: osdmap e1094: 8 total, 8 up, 8 in 2024-04-12T11:33:44.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:43 smithi029 ceph-mon[93650]: 6.1 scrub starts 2024-04-12T11:33:44.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:43 smithi029 ceph-mon[93650]: 6.2 scrub starts 2024-04-12T11:33:44.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:43 smithi029 ceph-mon[93650]: 6.1 scrub ok 2024-04-12T11:33:44.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:43 smithi029 ceph-mon[97787]: osdmap e1094: 8 total, 8 up, 8 in 2024-04-12T11:33:44.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:43 smithi029 ceph-mon[97787]: 6.1 scrub starts 2024-04-12T11:33:44.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:43 smithi029 ceph-mon[97787]: 6.2 scrub starts 2024-04-12T11:33:44.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:43 smithi029 ceph-mon[97787]: 6.1 scrub ok 2024-04-12T11:33:44.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:44.331 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:44.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:44.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:44.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:44.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:44.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:44.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:44.565 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:44.596 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:44.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:44.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:44.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 67701 67685 111.296 92.8125 0.00757554 0.00897906 2024-04-12T11:33:44.698 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:44.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:44.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:44.844 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:44.867 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:44.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:44.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:44.969 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:44.992 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:44.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:44.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:45.094 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:45.133 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:45.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:45.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:44 smithi087 ceph-mon[89014]: 6.9 scrub starts 2024-04-12T11:33:45.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.6 scrub starts 2024-04-12T11:33:45.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: pgmap v1340: 99 pgs: 8 peering, 10 unknown, 81 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T11:33:45.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.9 scrub ok 2024-04-12T11:33:45.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.6 scrub ok 2024-04-12T11:33:45.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.2 scrub ok 2024-04-12T11:33:45.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.11 scrub starts 2024-04-12T11:33:45.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.12 scrub starts 2024-04-12T11:33:45.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.11 scrub ok 2024-04-12T11:33:45.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:45.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[97787]: 6.9 scrub starts 2024-04-12T11:33:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[97787]: 6.6 scrub starts 2024-04-12T11:33:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[97787]: pgmap v1340: 99 pgs: 8 peering, 10 unknown, 81 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T11:33:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[97787]: 6.9 scrub ok 2024-04-12T11:33:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[97787]: 6.6 scrub ok 2024-04-12T11:33:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[97787]: 6.2 scrub ok 2024-04-12T11:33:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[97787]: 6.11 scrub starts 2024-04-12T11:33:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[97787]: 6.12 scrub starts 2024-04-12T11:33:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[97787]: 6.11 scrub ok 2024-04-12T11:33:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[93650]: 6.9 scrub starts 2024-04-12T11:33:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[93650]: 6.6 scrub starts 2024-04-12T11:33:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[93650]: pgmap v1340: 99 pgs: 8 peering, 10 unknown, 81 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T11:33:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[93650]: 6.9 scrub ok 2024-04-12T11:33:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[93650]: 6.6 scrub ok 2024-04-12T11:33:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[93650]: 6.2 scrub ok 2024-04-12T11:33:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[93650]: 6.11 scrub starts 2024-04-12T11:33:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[93650]: 6.12 scrub starts 2024-04-12T11:33:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-mon[93650]: 6.11 scrub ok 2024-04-12T11:33:45.270 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:33:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:33:44] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2024-04-12T11:33:45.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:45.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:45.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:45.371 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:45.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:45.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:45.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:45.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:45.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:45.551 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:45.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:45.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 68882 68866 110.335 73.8125 0.00518688 0.00905709 2024-04-12T11:33:45.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:45.692 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:45.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:45.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:45.794 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:45.826 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:45.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:45.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:45.928 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:45.959 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:45.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:45.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:46.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:46.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:46.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:46.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:46.197 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.0 scrub starts 2024-04-12T11:33:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.16 scrub starts 2024-04-12T11:33:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.0 scrub ok 2024-04-12T11:33:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.4 scrub starts 2024-04-12T11:33:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.12 scrub ok 2024-04-12T11:33:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:45 smithi087 ceph-mon[89014]: 6.8 scrub starts 2024-04-12T11:33:46.237 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:46.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:46.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[97787]: 6.0 scrub starts 2024-04-12T11:33:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[97787]: 6.16 scrub starts 2024-04-12T11:33:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[97787]: 6.0 scrub ok 2024-04-12T11:33:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[97787]: 6.4 scrub starts 2024-04-12T11:33:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[97787]: 6.12 scrub ok 2024-04-12T11:33:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[97787]: 6.8 scrub starts 2024-04-12T11:33:46.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[93650]: 6.0 scrub starts 2024-04-12T11:33:46.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[93650]: 6.16 scrub starts 2024-04-12T11:33:46.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[93650]: 6.0 scrub ok 2024-04-12T11:33:46.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[93650]: 6.4 scrub starts 2024-04-12T11:33:46.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[93650]: 6.12 scrub ok 2024-04-12T11:33:46.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:45 smithi029 ceph-mon[93650]: 6.8 scrub starts 2024-04-12T11:33:46.339 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:46.373 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:46.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:46.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:46.475 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:46.510 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:46.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:46.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:46.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:46.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:33:46.621063+0000 min lat: 0.00141572 max lat: 1.09431 avg lat: 0.00907829 2024-04-12T11:33:46.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:33:46.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 70480 70464 110.073 99.875 0.00670664 0.00907829 2024-04-12T11:33:46.651 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:46.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:46.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:46.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:46.776 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:46.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:46.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:46.878 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:46.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:46.918 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:46.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:46.974 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:33:46.974 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:33:46.974 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-04-12T11:33:46.975 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.730674 2024-04-12T11:33:46.975 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 6 0.7306737697738339 2024-04-12T11:33:47.018 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:47.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:47.050 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:47.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:47.151 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:47.181 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:47.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:47.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:46 smithi087 ceph-mon[89014]: 6.19 scrub starts 2024-04-12T11:33:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:46 smithi087 ceph-mon[89014]: 6.16 scrub ok 2024-04-12T11:33:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:46 smithi087 ceph-mon[89014]: pgmap v1341: 99 pgs: 2 active+clean+scrubbing, 8 peering, 8 unknown, 81 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2024-04-12T11:33:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:46 smithi087 ceph-mon[89014]: 6.8 scrub ok 2024-04-12T11:33:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:46 smithi087 ceph-mon[89014]: 6.4 scrub ok 2024-04-12T11:33:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:46 smithi087 ceph-mon[89014]: 6.19 scrub ok 2024-04-12T11:33:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:46 smithi087 ceph-mon[89014]: 6.14 scrub starts 2024-04-12T11:33:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[97787]: 6.19 scrub starts 2024-04-12T11:33:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[97787]: 6.16 scrub ok 2024-04-12T11:33:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[97787]: pgmap v1341: 99 pgs: 2 active+clean+scrubbing, 8 peering, 8 unknown, 81 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2024-04-12T11:33:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[97787]: 6.8 scrub ok 2024-04-12T11:33:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[97787]: 6.4 scrub ok 2024-04-12T11:33:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[97787]: 6.19 scrub ok 2024-04-12T11:33:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[97787]: 6.14 scrub starts 2024-04-12T11:33:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[93650]: 6.19 scrub starts 2024-04-12T11:33:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[93650]: 6.16 scrub ok 2024-04-12T11:33:47.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[93650]: pgmap v1341: 99 pgs: 2 active+clean+scrubbing, 8 peering, 8 unknown, 81 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2024-04-12T11:33:47.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[93650]: 6.8 scrub ok 2024-04-12T11:33:47.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[93650]: 6.4 scrub ok 2024-04-12T11:33:47.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[93650]: 6.19 scrub ok 2024-04-12T11:33:47.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:46 smithi029 ceph-mon[93650]: 6.14 scrub starts 2024-04-12T11:33:47.283 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:47.307 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:47.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:47.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:47.408 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:47.443 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:47.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:47.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:47.545 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:47.581 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:47.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:47.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:47.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 71443 71427 108.856 60.1875 0.0114348 0.00918116 2024-04-12T11:33:47.683 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:47.717 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:47.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:47.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:47.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:47.845 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:47.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:47.947 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:47.970 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:47.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:48.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:48.109 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:48.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:48.211 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:48.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:47 smithi087 ceph-mon[89014]: 6.7 scrub starts 2024-04-12T11:33:48.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:47 smithi087 ceph-mon[89014]: 6.10 scrub starts 2024-04-12T11:33:48.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:47 smithi087 ceph-mon[89014]: 6.7 scrub ok 2024-04-12T11:33:48.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:47 smithi087 ceph-mon[89014]: 6.14 scrub ok 2024-04-12T11:33:48.245 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:48.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:48.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:48.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:47 smithi029 ceph-mon[93650]: 6.7 scrub starts 2024-04-12T11:33:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:47 smithi029 ceph-mon[93650]: 6.10 scrub starts 2024-04-12T11:33:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:47 smithi029 ceph-mon[93650]: 6.7 scrub ok 2024-04-12T11:33:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:47 smithi029 ceph-mon[93650]: 6.14 scrub ok 2024-04-12T11:33:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:47 smithi029 ceph-mon[97787]: 6.7 scrub starts 2024-04-12T11:33:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:47 smithi029 ceph-mon[97787]: 6.10 scrub starts 2024-04-12T11:33:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:47 smithi029 ceph-mon[97787]: 6.7 scrub ok 2024-04-12T11:33:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:47 smithi029 ceph-mon[97787]: 6.14 scrub ok 2024-04-12T11:33:48.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:48.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:48.382 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:48.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:48.483 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:48.507 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:48.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:48.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:48.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:48.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 72553 72537 107.916 69.375 0.00282576 0.00923495 2024-04-12T11:33:48.632 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:48.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:48.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:48.734 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:48.770 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:48.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:48.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:48.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:48.909 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:48.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:48.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:49.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:49.050 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:49.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:49.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:49.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:49.176 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:49.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:49.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:48 smithi087 ceph-mon[89014]: 6.10 scrub ok 2024-04-12T11:33:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:48 smithi087 ceph-mon[89014]: pgmap v1342: 99 pgs: 2 active+clean+scrubbing, 4 peering, 4 unknown, 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 11 MiB/s wr, 169 op/s 2024-04-12T11:33:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:48 smithi029 ceph-mon[93650]: 6.10 scrub ok 2024-04-12T11:33:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:48 smithi029 ceph-mon[93650]: pgmap v1342: 99 pgs: 2 active+clean+scrubbing, 4 peering, 4 unknown, 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 11 MiB/s wr, 169 op/s 2024-04-12T11:33:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:48 smithi029 ceph-mon[97787]: 6.10 scrub ok 2024-04-12T11:33:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:48 smithi029 ceph-mon[97787]: pgmap v1342: 99 pgs: 2 active+clean+scrubbing, 4 peering, 4 unknown, 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 11 MiB/s wr, 169 op/s 2024-04-12T11:33:49.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:49.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:49.312 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:49.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:49.414 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:49.438 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:49.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:49.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:49.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:49.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:49.589 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:49.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:49.621 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 73666 73650 107.024 69.5625 0.00979807 0.00933868 2024-04-12T11:33:49.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:49.724 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:49.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:49.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:49.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:49.848 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:49.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:49.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:49.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:49.984 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:49.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:49.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:50.085 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:50.112 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:50.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:50.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:50.214 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:50.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:49 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]: dispatch 2024-04-12T11:33:50.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:50.238 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:50.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:50.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:49 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]: dispatch 2024-04-12T11:33:50.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:49 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]: dispatch 2024-04-12T11:33:50.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:50.362 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:50.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:50.463 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:50.487 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:50.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:50.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:50.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:50.612 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:50.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:50.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:50.620 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 74287 74271 105.474 38.8125 0.00193819 0.00935414 2024-04-12T11:33:50.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:50.738 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:50.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:50.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:50.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:50.864 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:50.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:50.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:50.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:50.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:50.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:50.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:51.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:50 smithi029 ceph-mon[93650]: 6.17 scrub starts 2024-04-12T11:33:51.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:50 smithi029 ceph-mon[97787]: 6.17 scrub starts 2024-04-12T11:33:51.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:51.114 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:51.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:51.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:51.215 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:50 smithi087 ceph-mon[89014]: 6.17 scrub starts 2024-04-12T11:33:51.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:50 smithi087 ceph-mon[89014]: 6.17 scrub ok 2024-04-12T11:33:51.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:50 smithi087 ceph-mon[89014]: pgmap v1343: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2024-04-12T11:33:51.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:50 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-04-12T11:33:51.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:50 smithi087 ceph-mon[89014]: osdmap e1095: 8 total, 8 up, 8 in 2024-04-12T11:33:51.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:50 smithi087 ceph-mon[89014]: 6.3 scrub starts 2024-04-12T11:33:51.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:50 smithi087 ceph-mon[89014]: 6.3 scrub ok 2024-04-12T11:33:51.240 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:51.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:51.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:51.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:51.367 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:51.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:51.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:51.469 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:51.493 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:51.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:51.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:51.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[93650]: 6.17 scrub ok 2024-04-12T11:33:51.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[93650]: pgmap v1343: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2024-04-12T11:33:51.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-04-12T11:33:51.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[93650]: osdmap e1095: 8 total, 8 up, 8 in 2024-04-12T11:33:51.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[93650]: 6.3 scrub starts 2024-04-12T11:33:51.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[93650]: 6.3 scrub ok 2024-04-12T11:33:51.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[97787]: 6.17 scrub ok 2024-04-12T11:33:51.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[97787]: pgmap v1343: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2024-04-12T11:33:51.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "17"}]': finished 2024-04-12T11:33:51.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[97787]: osdmap e1095: 8 total, 8 up, 8 in 2024-04-12T11:33:51.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[97787]: 6.3 scrub starts 2024-04-12T11:33:51.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[97787]: 6.3 scrub ok 2024-04-12T11:33:51.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:51.621 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 74287 74271 103.13 0 - 0.00935414 2024-04-12T11:33:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:51.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:51.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:51.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:51.748 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:51.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:51.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:51.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:51.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:51.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:51.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:51.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:52.002 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:52.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:52.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:52.104 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:52.129 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:52.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:52.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:51 smithi087 ceph-mon[89014]: osdmap e1096: 8 total, 8 up, 8 in 2024-04-12T11:33:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]: dispatch 2024-04-12T11:33:52.231 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:52.258 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:52.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:52.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[97787]: osdmap e1096: 8 total, 8 up, 8 in 2024-04-12T11:33:52.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:52.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:52.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]: dispatch 2024-04-12T11:33:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[93650]: osdmap e1096: 8 total, 8 up, 8 in 2024-04-12T11:33:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:33:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]: dispatch 2024-04-12T11:33:52.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:52.390 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:52.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:52.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:52.525 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:52.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:52.621 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 74345 74329 100.968 1.8125 2.53234 0.0097998 2024-04-12T11:33:52.626 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:52.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:52.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:52.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:52.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:33:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:33:52.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:33:52.769 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:52.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:52.794 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:52.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:52.895 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:52.918 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:52.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:53.021 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:53.053 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:53.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:53.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:53.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:53.189 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:53.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:53.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:52 smithi087 ceph-mon[89014]: pgmap v1346: 99 pgs: 99 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2024-04-12T11:33:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-04-12T11:33:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:52 smithi087 ceph-mon[89014]: osdmap e1097: 8 total, 8 up, 8 in 2024-04-12T11:33:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:52 smithi087 ceph-mon[89014]: 6.5 deep-scrub starts 2024-04-12T11:33:53.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:52 smithi087 ceph-mon[89014]: osdmap e1098: 8 total, 8 up, 8 in 2024-04-12T11:33:53.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:52 smithi029 ceph-mon[97787]: pgmap v1346: 99 pgs: 99 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2024-04-12T11:33:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-04-12T11:33:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:52 smithi029 ceph-mon[97787]: osdmap e1097: 8 total, 8 up, 8 in 2024-04-12T11:33:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:52 smithi029 ceph-mon[97787]: 6.5 deep-scrub starts 2024-04-12T11:33:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:52 smithi029 ceph-mon[97787]: osdmap e1098: 8 total, 8 up, 8 in 2024-04-12T11:33:53.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:52 smithi029 ceph-mon[93650]: pgmap v1346: 99 pgs: 99 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2024-04-12T11:33:53.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "18"}]': finished 2024-04-12T11:33:53.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:52 smithi029 ceph-mon[93650]: osdmap e1097: 8 total, 8 up, 8 in 2024-04-12T11:33:53.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:52 smithi029 ceph-mon[93650]: 6.5 deep-scrub starts 2024-04-12T11:33:53.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:52 smithi029 ceph-mon[93650]: osdmap e1098: 8 total, 8 up, 8 in 2024-04-12T11:33:53.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:53.318 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:53.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:53.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:53.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:53.443 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:53.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:53.545 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:53.569 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:53.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:53.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:53.621 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 74899 74883 99.5561 34.625 0.0030946 0.00989937 2024-04-12T11:33:53.671 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:53.695 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:53.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:53.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:53.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:53.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:53.829 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:53.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:53.930 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:53.959 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:53.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:53.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:54.061 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:54.085 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:54.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:54.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:54.186 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:54.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:54.211 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:54.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]: dispatch 2024-04-12T11:33:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-04-12T11:33:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:53 smithi029 ceph-mon[93650]: osdmap e1099: 8 total, 8 up, 8 in 2024-04-12T11:33:54.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]: dispatch 2024-04-12T11:33:54.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-04-12T11:33:54.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:53 smithi029 ceph-mon[97787]: osdmap e1099: 8 total, 8 up, 8 in 2024-04-12T11:33:54.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:54.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:54.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:54.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:54.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:54.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]: dispatch 2024-04-12T11:33:54.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "19"}]': finished 2024-04-12T11:33:54.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:53 smithi087 ceph-mon[89014]: osdmap e1099: 8 total, 8 up, 8 in 2024-04-12T11:33:54.472 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:54.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:54.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:54.574 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:54.605 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:54.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:54.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:54.621 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 75337 75321 98.0525 27.375 0.00222419 0.0101538 2024-04-12T11:33:54.707 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:54.732 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:54.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:54.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:54.834 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:54.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:54.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:54.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:54.961 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:54.985 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:54.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:54.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:55.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:55.111 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:55.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:55.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:55.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:54 smithi029 ceph-mon[93650]: pgmap v1349: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 97 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 4802/221550 objects misplaced (2.167%); 80 KiB/s, 1 objects/s recovering 2024-04-12T11:33:55.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:54 smithi029 ceph-mon[97787]: pgmap v1349: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 97 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 4802/221550 objects misplaced (2.167%); 80 KiB/s, 1 objects/s recovering 2024-04-12T11:33:55.185 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:33:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:33:54] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2024-04-12T11:33:55.213 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:55.237 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:55.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:55.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:55.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:55.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:55.365 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:55.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:55.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:54 smithi087 ceph-mon[89014]: pgmap v1349: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 97 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 4802/221550 objects misplaced (2.167%); 80 KiB/s, 1 objects/s recovering 2024-04-12T11:33:55.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:54 smithi087 ceph-mon[89014]: osdmap e1100: 8 total, 8 up, 8 in 2024-04-12T11:33:55.467 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:55.491 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:55.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:55.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:55.517 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:55 smithi029 ceph-mon[93650]: osdmap e1100: 8 total, 8 up, 8 in 2024-04-12T11:33:55.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:55 smithi029 ceph-mon[97787]: osdmap e1100: 8 total, 8 up, 8 in 2024-04-12T11:33:55.593 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:55.618 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:55.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:55.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:55.621 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 75387 75371 96.1154 3.125 0.00245511 0.0101505 2024-04-12T11:33:55.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:55.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:55.747 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:55.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:55.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:55.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:55.874 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:55.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:55.974 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:56.007 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:56.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:56.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:56.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:55 smithi029 ceph-mon[93650]: 6.15 scrub starts 2024-04-12T11:33:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]: dispatch 2024-04-12T11:33:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-04-12T11:33:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:55 smithi029 ceph-mon[93650]: osdmap e1101: 8 total, 8 up, 8 in 2024-04-12T11:33:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:55 smithi029 ceph-mon[97787]: 6.15 scrub starts 2024-04-12T11:33:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]: dispatch 2024-04-12T11:33:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-04-12T11:33:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:55 smithi029 ceph-mon[97787]: osdmap e1101: 8 total, 8 up, 8 in 2024-04-12T11:33:56.109 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:56.142 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:56.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:56.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:56.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:56.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:56.282 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:56.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:56.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:56.407 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:56.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:56.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:56.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:55 smithi087 ceph-mon[89014]: 6.15 scrub starts 2024-04-12T11:33:56.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]: dispatch 2024-04-12T11:33:56.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "20"}]': finished 2024-04-12T11:33:56.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:55 smithi087 ceph-mon[89014]: osdmap e1101: 8 total, 8 up, 8 in 2024-04-12T11:33:56.509 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:56.542 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:56.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:56.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:56.621 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 75818 75802 94.732 26.9375 0.00265102 0.0104677 2024-04-12T11:33:56.644 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:56.669 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:56.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:56.771 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:56.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:56.798 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:56.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:56.899 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:56.921 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:56.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:56.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:57.023 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:57.046 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:57.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:57.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:57.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:57.175 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:57.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:57.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:57.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:56 smithi029 ceph-mon[93650]: pgmap v1352: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 97 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 4802/223476 objects misplaced (2.149%); 80 KiB/s, 1 objects/s recovering 2024-04-12T11:33:57.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:57 smithi029 ceph-mon[93650]: 6.15 scrub ok 2024-04-12T11:33:57.264 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:57 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:57.264 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:57 smithi029 ceph-mon[93650]: osdmap e1102: 8 total, 8 up, 8 in 2024-04-12T11:33:57.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:56 smithi029 ceph-mon[97787]: pgmap v1352: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 97 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 4802/223476 objects misplaced (2.149%); 80 KiB/s, 1 objects/s recovering 2024-04-12T11:33:57.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:57 smithi029 ceph-mon[97787]: 6.15 scrub ok 2024-04-12T11:33:57.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:57 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:57.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:57 smithi029 ceph-mon[97787]: osdmap e1102: 8 total, 8 up, 8 in 2024-04-12T11:33:57.276 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:57.301 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:57.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:57.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:57.426 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:57.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:57.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:56 smithi087 ceph-mon[89014]: pgmap v1352: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 97 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 4802/223476 objects misplaced (2.149%); 80 KiB/s, 1 objects/s recovering 2024-04-12T11:33:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:56 smithi087 ceph-mon[89014]: 6.15 scrub ok 2024-04-12T11:33:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:56 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:33:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:56 smithi087 ceph-mon[89014]: osdmap e1102: 8 total, 8 up, 8 in 2024-04-12T11:33:57.528 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:57.551 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:57.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:57.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:57.621 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 75821 75805 92.8784 0.1875 0.00921782 0.0104676 2024-04-12T11:33:57.653 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:57.679 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:57.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:57.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:57.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:57.806 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:57.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:57.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:57.908 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:57.931 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:57.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:57.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:58.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:57 smithi029 ceph-mon[97787]: 6.5 deep-scrub ok 2024-04-12T11:33:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:57 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]: dispatch 2024-04-12T11:33:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:57 smithi029 ceph-mon[93650]: 6.5 deep-scrub ok 2024-04-12T11:33:58.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:57 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]: dispatch 2024-04-12T11:33:58.033 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:58.060 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:58.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:58.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:58.185 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:58.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:58.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:58.287 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:58.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:58.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:58.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:58.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:58.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:57 smithi087 ceph-mon[89014]: 6.5 deep-scrub ok 2024-04-12T11:33:58.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:57 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]: dispatch 2024-04-12T11:33:58.467 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:58.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:58.569 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:33:58.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:58.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:58.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:33:58.621 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 76260 76244 91.62 27.4375 0.00481494 0.0108914 2024-04-12T11:33:58.701 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:33:58.726 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:58.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:58.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:33:58.828 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:58.853 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:58.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:58.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:58.954 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:58.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:58.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:58.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:59.084 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:33:59.108 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:59.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:59.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:33:59.210 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:33:59.233 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:59.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:59.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:33:59.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:58 smithi029 ceph-mon[93650]: pgmap v1355: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 4802/225054 objects misplaced (2.134%) 2024-04-12T11:33:59.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:58 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-04-12T11:33:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:33:58 smithi029 ceph-mon[93650]: osdmap e1103: 8 total, 8 up, 8 in 2024-04-12T11:33:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:58 smithi029 ceph-mon[97787]: pgmap v1355: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 4802/225054 objects misplaced (2.134%) 2024-04-12T11:33:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:58 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-04-12T11:33:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:33:58 smithi029 ceph-mon[97787]: osdmap e1103: 8 total, 8 up, 8 in 2024-04-12T11:33:59.334 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:33:59.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:33:59.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:59.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:33:59.459 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:59.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:58 smithi087 ceph-mon[89014]: pgmap v1355: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 4802/225054 objects misplaced (2.134%) 2024-04-12T11:33:59.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:58 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "21"}]': finished 2024-04-12T11:33:59.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:33:58 smithi087 ceph-mon[89014]: osdmap e1103: 8 total, 8 up, 8 in 2024-04-12T11:33:59.482 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:59.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:59.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:59.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:33:59.609 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:59.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:33:59.621 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 76444 76428 90.1085 11.5 0.00359266 0.0108855 2024-04-12T11:33:59.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:59.737 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:59.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:59.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:33:59.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:33:59.865 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:59.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:59.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:33:59.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:33:59.995 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:33:59.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:33:59.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:00.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:00.125 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:00.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:00.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:00.227 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:00.251 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:00.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:00.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:00.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:00 smithi029 ceph-mon[97787]: osdmap e1104: 8 total, 8 up, 8 in 2024-04-12T11:34:00.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:00 smithi029 ceph-mon[97787]: osdmap e1105: 8 total, 8 up, 8 in 2024-04-12T11:34:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:00 smithi029 ceph-mon[93650]: osdmap e1104: 8 total, 8 up, 8 in 2024-04-12T11:34:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:00 smithi029 ceph-mon[93650]: osdmap e1105: 8 total, 8 up, 8 in 2024-04-12T11:34:00.354 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:00.377 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:00.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:00.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:00 smithi087 ceph-mon[89014]: osdmap e1104: 8 total, 8 up, 8 in 2024-04-12T11:34:00.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:00 smithi087 ceph-mon[89014]: osdmap e1105: 8 total, 8 up, 8 in 2024-04-12T11:34:00.479 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:00.517 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:00.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:00.619 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:00.622 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 76717 76701 88.7559 17.0625 0.0057004 0.0112613 2024-04-12T11:34:00.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:00.654 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:00.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:00.755 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:00.791 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:00.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:00.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:00.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:00.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:00.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:00.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:01.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:01.067 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:01.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:01.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:01.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:01.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:01.213 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:01.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:01.254 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:01 smithi029 ceph-mon[97787]: pgmap v1358: 99 pgs: 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 6867/220701 objects degraded (3.111%); 10803/220701 objects misplaced (4.895%); 9.7 MiB/s, 155 objects/s recovering 2024-04-12T11:34:01.254 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:01 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 6867/220701 objects degraded (3.111%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:01.254 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:01 smithi029 ceph-mon[93650]: pgmap v1358: 99 pgs: 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 6867/220701 objects degraded (3.111%); 10803/220701 objects misplaced (4.895%); 9.7 MiB/s, 155 objects/s recovering 2024-04-12T11:34:01.255 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:01 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 6867/220701 objects degraded (3.111%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:01.314 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:01.365 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:01.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:01.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:01.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:01.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:01 smithi087 ceph-mon[89014]: pgmap v1358: 99 pgs: 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 6867/220701 objects degraded (3.111%); 10803/220701 objects misplaced (4.895%); 9.7 MiB/s, 155 objects/s recovering 2024-04-12T11:34:01.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:01 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 6867/220701 objects degraded (3.111%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:01.496 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:01.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:01.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:01.597 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:01.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:01.621 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:01.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:01.622 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 77983 77967 88.5807 79.125 0.0139869 0.0112731 2024-04-12T11:34:01.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:01.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:01.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:01.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:01.847 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:01.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:01.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:01.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:01.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:02.026 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:02.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:02.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:02.154 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:02.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:02.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:02.256 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:02.290 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:02.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:02.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:02.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:02.436 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:02.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:02.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:02.538 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:02.576 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:02.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:02.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:02.622 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 78974 78958 88.1049 61.9375 0.00966648 0.0113446 2024-04-12T11:34:02.679 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:34:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:34:02.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:34:02.722 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:02.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:02.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:02.824 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:02.853 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:02.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:02.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:02.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:02.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:02.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:02.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:03.089 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:03.128 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:03.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:03.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:03.229 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:03.261 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:03.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:03.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:03.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:03 smithi029 ceph-mon[97787]: pgmap v1360: 99 pgs: 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 6867/221037 objects degraded (3.107%); 10803/221037 objects misplaced (4.887%); 8.0 MiB/s, 127 objects/s recovering 2024-04-12T11:34:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:03 smithi029 ceph-mon[93650]: pgmap v1360: 99 pgs: 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 6867/221037 objects degraded (3.107%); 10803/221037 objects misplaced (4.887%); 8.0 MiB/s, 127 objects/s recovering 2024-04-12T11:34:03.363 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:03.386 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:03.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:03.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:03 smithi087 ceph-mon[89014]: pgmap v1360: 99 pgs: 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 6867/221037 objects degraded (3.107%); 10803/221037 objects misplaced (4.887%); 8.0 MiB/s, 127 objects/s recovering 2024-04-12T11:34:03.488 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:03.511 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:03.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:03.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:03.613 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:03.622 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 80050 80034 87.7389 67.25 0.00746731 0.0113922 2024-04-12T11:34:03.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:03.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:03.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:03.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:03.792 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:03.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:03.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:03.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:03.936 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:03.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:03.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:04.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:04.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:04.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:04.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:04.178 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:04.202 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:04.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:04.303 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:04.326 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:04.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:04.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:04.428 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:04.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:04.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:04.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:04.560 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:04.591 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:04.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:04.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:04.622 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 81000 80984 87.2498 59.375 0.0162683 0.0114568 2024-04-12T11:34:04.692 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:04.726 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:04.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:04.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:04.828 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:04.852 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:04.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:04.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:04.953 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:04.983 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:04.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:05.084 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:05.114 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:05.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:05.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:05.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:05.255 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:05.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:05.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:05.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:05 smithi029 ceph-mon[97787]: pgmap v1361: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 29 MiB/s wr, 459 op/s; 11625/233334 objects degraded (4.982%); 12594/233334 objects misplaced (5.397%); 9.5 MiB/s, 152 objects/s recovering 2024-04-12T11:34:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:05 smithi029 ceph-mon[93650]: pgmap v1361: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 29 MiB/s wr, 459 op/s; 11625/233334 objects degraded (4.982%); 12594/233334 objects misplaced (5.397%); 9.5 MiB/s, 152 objects/s recovering 2024-04-12T11:34:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:34:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:34:04] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2024-04-12T11:34:05.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:05.397 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:05.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:05.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:05.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:05 smithi087 ceph-mon[89014]: pgmap v1361: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 29 MiB/s wr, 459 op/s; 11625/233334 objects degraded (4.982%); 12594/233334 objects misplaced (5.397%); 9.5 MiB/s, 152 objects/s recovering 2024-04-12T11:34:05.499 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:05.523 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:05.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:05.622 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 81852 81836 86.6735 53.25 0.00201761 0.0114962 2024-04-12T11:34:05.625 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:05.647 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:05.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:05.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:05.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:05.789 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:05.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:05.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:05.890 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:05.922 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:05.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:05.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:06.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:06.047 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:06.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:06.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:06.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:06.345 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:06.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:06.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:06.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:06.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:06 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11625/238362 objects degraded (4.877%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:06.495 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:06.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:06.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:06.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:06 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11625/238362 objects degraded (4.877%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:06.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:06 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11625/238362 objects degraded (4.877%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:06.597 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:06.621 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:06.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:06.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:06.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:34:06.623045+0000 min lat: 0.00141572 max lat: 2.5959 avg lat: 0.0116379 2024-04-12T11:34:06.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:34:06.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 82400 82384 85.7998 34.25 0.00209421 0.0116379 2024-04-12T11:34:06.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:06.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:06.749 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:06.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:06.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:06.874 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:06.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:06.975 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:07.009 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:07.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:07.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:07.111 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:07.150 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:07.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:07.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:07.252 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:07.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:07.305 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:07.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:07.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:07 smithi029 ceph-mon[97787]: pgmap v1362: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 11625/238362 objects degraded (4.877%); 12051/238362 objects misplaced (5.056%); 9.5 MiB/s, 152 objects/s recovering 2024-04-12T11:34:07.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:07.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:07.381 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:07 smithi029 ceph-mon[93650]: pgmap v1362: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 11625/238362 objects degraded (4.877%); 12051/238362 objects misplaced (5.056%); 9.5 MiB/s, 152 objects/s recovering 2024-04-12T11:34:07.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:07.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:07.406 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:07.439 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:07.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:07.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:07.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:07 smithi087 ceph-mon[89014]: pgmap v1362: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 33 MiB/s wr, 522 op/s; 11625/238362 objects degraded (4.877%); 12051/238362 objects misplaced (5.056%); 9.5 MiB/s, 152 objects/s recovering 2024-04-12T11:34:07.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:07.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:07.541 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:07.571 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:07.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:07.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:07.622 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 83374 83358 85.3911 60.875 0.0217345 0.0117056 2024-04-12T11:34:07.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:07.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:07.698 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:07.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:07.799 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:07.824 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:07.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:07.925 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:07.948 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:07.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:07.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:08.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:08.092 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:08.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:08.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:08.194 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:08.236 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:08.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:08.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:08.337 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:08.361 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:08.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:08.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:08.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:08.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:08.488 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:08.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:08.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:08.622 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 83900 83884 84.5441 32.875 0.00736552 0.0118226 2024-04-12T11:34:08.623 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:08.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:08.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:08.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:08.766 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:08.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:08.868 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:08.898 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:08.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:08.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:09.000 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:09.023 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:09.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:09.126 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:09.151 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:09.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:09.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:09.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:09.292 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:09.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:09.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:09.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:09.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:09.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:09.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:09 smithi087 ceph-mon[89014]: pgmap v1363: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 37 MiB/s wr, 597 op/s; 11625/242910 objects degraded (4.786%); 12051/242910 objects misplaced (4.961%); 6.0 MiB/s, 96 objects/s recovering 2024-04-12T11:34:09.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:09 smithi029 ceph-mon[97787]: pgmap v1363: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 37 MiB/s wr, 597 op/s; 11625/242910 objects degraded (4.786%); 12051/242910 objects misplaced (4.961%); 6.0 MiB/s, 96 objects/s recovering 2024-04-12T11:34:09.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:09 smithi029 ceph-mon[93650]: pgmap v1363: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 37 MiB/s wr, 597 op/s; 11625/242910 objects degraded (4.786%); 12051/242910 objects misplaced (4.961%); 6.0 MiB/s, 96 objects/s recovering 2024-04-12T11:34:09.541 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:09.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:09.585 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:09.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:09.622 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 84855 84839 84.1495 59.6875 0.0160586 0.0118784 2024-04-12T11:34:09.686 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:09.718 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:09.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:09.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:09.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:09.857 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:09.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:09.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:09.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:09.998 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:09.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:09.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:10.100 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:10.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:10.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:10.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:10.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:10.277 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:10.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:10.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:10.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:10.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:10.416 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:10.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:10.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:10.561 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:10.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:10.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:10.622 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 86027 86011 83.9791 73.25 0.00351105 0.0118933 2024-04-12T11:34:10.663 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:10.687 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:10.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:10.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:10.788 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:10.812 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:10.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:10.914 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:10.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:10.944 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:10.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:11.045 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:11.077 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:11.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:11.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:11.178 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:11.201 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:11.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:11.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:11.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:11.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:11.340 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:11.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:11.441 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:11.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:11 smithi087 ceph-mon[89014]: pgmap v1364: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 11625/249807 objects degraded (4.654%); 11359/249807 objects misplaced (4.547%); 8.6 MiB/s, 137 objects/s recovering 2024-04-12T11:34:11.477 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:11.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:11.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:11 smithi029 ceph-mon[97787]: pgmap v1364: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 11625/249807 objects degraded (4.654%); 11359/249807 objects misplaced (4.547%); 8.6 MiB/s, 137 objects/s recovering 2024-04-12T11:34:11.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:11 smithi029 ceph-mon[93650]: pgmap v1364: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 11625/249807 objects degraded (4.654%); 11359/249807 objects misplaced (4.547%); 8.6 MiB/s, 137 objects/s recovering 2024-04-12T11:34:11.579 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:11.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 86709 86693 83.3429 42.625 0.0225028 0.0119937 2024-04-12T11:34:11.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:11.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:11.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:11.725 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:11.761 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:11.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:11.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:11.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:11.886 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:11.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:11.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:12.012 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:12.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:12.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:12.156 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:12.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:12.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:12.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:12.297 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:12.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:12.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:12.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:12.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:12.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:12.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:12 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11625/250887 objects degraded (4.634%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:12.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:34:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:34:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:34:12.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:12 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11625/250887 objects degraded (4.634%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:12.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:12 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11625/250887 objects degraded (4.634%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:12.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:12.581 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:12.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:12.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 87708 87692 83.0261 62.4375 0.0208769 0.012039 2024-04-12T11:34:12.683 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:12.714 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:12.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:12.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:12.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:12.838 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:12.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:12.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:12.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:12.983 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:12.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:12.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:13.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:13.123 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:13.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:13.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:13.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:13.277 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:13.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:13.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:13.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:13.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:13.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:13.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:13 smithi087 ceph-mon[89014]: pgmap v1365: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s; 11625/250887 objects degraded (4.634%); 11359/250887 objects misplaced (4.528%); 7.4 MiB/s, 117 objects/s recovering 2024-04-12T11:34:13.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:13.517 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:13 smithi029 ceph-mon[93650]: pgmap v1365: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s; 11625/250887 objects degraded (4.634%); 11359/250887 objects misplaced (4.528%); 7.4 MiB/s, 117 objects/s recovering 2024-04-12T11:34:13.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:13 smithi029 ceph-mon[97787]: pgmap v1365: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s; 11625/250887 objects degraded (4.634%); 11359/250887 objects misplaced (4.528%); 7.4 MiB/s, 117 objects/s recovering 2024-04-12T11:34:13.527 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:13.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:13.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:13.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 88295 88279 82.3345 36.6875 0.0088598 0.0121007 2024-04-12T11:34:13.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:13.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:13.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:13.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:13.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:13.799 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:13.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:13.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:13.901 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:13.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:13.946 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:13.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:14.047 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:14.070 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:14.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:14.172 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:14.206 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:14.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:14.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:14.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:14.341 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:14.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:14.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:14.442 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:14.473 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:14.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:14.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:14.575 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:14.599 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:14.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:14.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:14.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 89303 89287 82.0501 63 0.0115195 0.0121642 2024-04-12T11:34:14.701 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:14.732 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:14.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:14.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:14.834 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:14.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:14.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:14.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:14.962 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:14.992 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:14.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:14.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:15.094 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:15.126 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:15.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:15.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:15.228 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:15.261 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:15.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:15.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:15.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:15 smithi029 ceph-mon[97787]: pgmap v1366: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 11625/258660 objects degraded (4.494%); 11021/258660 objects misplaced (4.261%); 8.1 MiB/s, 128 objects/s recovering 2024-04-12T11:34:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:15 smithi029 ceph-mon[93650]: pgmap v1366: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 11625/258660 objects degraded (4.494%); 11021/258660 objects misplaced (4.261%); 8.1 MiB/s, 128 objects/s recovering 2024-04-12T11:34:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:34:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:34:14] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.43.0" 2024-04-12T11:34:15.363 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:15.396 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:15.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:15.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:15.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:15 smithi087 ceph-mon[89014]: pgmap v1366: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 11625/258660 objects degraded (4.494%); 11021/258660 objects misplaced (4.261%); 8.1 MiB/s, 128 objects/s recovering 2024-04-12T11:34:15.498 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:15.521 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:15.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:15.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:15.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:15.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 90098 90082 81.581 49.6875 0.00603936 0.0122195 2024-04-12T11:34:15.647 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:15.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:15.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:15.748 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:15.786 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:15.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:15.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:15.888 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:15.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:15.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:15.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:16.012 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:16.044 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:16.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:16.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:16.146 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:16.190 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:16.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:16.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:16.292 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:16.315 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:16.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:16.416 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:16.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:16.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:16.563 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:16.596 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:16.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:16.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 91052 91036 81.2673 59.625 0.0104286 0.0122999 2024-04-12T11:34:16.697 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:16.728 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:16.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:16.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:16.830 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:16.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:16.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:16.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:16.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:17.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:17.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:17.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:17.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:17.163 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:17.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:17.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:17.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:17.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:17.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:17.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:17.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:17.452 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:17.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:17.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:17 smithi087 ceph-mon[89014]: pgmap v1367: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 11625/263898 objects degraded (4.405%); 10580/263898 objects misplaced (4.009%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:34:17.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:17 smithi029 ceph-mon[93650]: pgmap v1367: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 11625/263898 objects degraded (4.405%); 10580/263898 objects misplaced (4.009%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:34:17.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:17 smithi029 ceph-mon[97787]: pgmap v1367: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 11625/263898 objects degraded (4.405%); 10580/263898 objects misplaced (4.009%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:34:17.553 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:17.577 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:17.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:17.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 92107 92091 81.0512 65.9375 0.00817226 0.0123057 2024-04-12T11:34:17.679 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:17.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:17.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:17.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:17.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:17.845 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:17.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:17.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:17.947 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:17.977 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:17.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:17.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:18.079 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:18.112 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:18.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:18.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:18.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:18.238 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:18.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:18.339 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:18.364 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:18.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:18.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:18.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:18.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11625/263898 objects degraded (4.405%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:18.489 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:18.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:18.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:18.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11625/263898 objects degraded (4.405%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:18.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11625/263898 objects degraded (4.405%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:18.591 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:18.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 92771 92755 80.5019 41.5 0.00876336 0.0124172 2024-04-12T11:34:18.624 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:18.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:18.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:18.726 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:18.763 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:18.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:18.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:18.865 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:18.896 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:18.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:18.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:18.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:19.039 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:19.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:19.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:19.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:19.182 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:19.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:19.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:19.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:19.328 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:19.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:19.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:19.430 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:19.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:19.454 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:19.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:19.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:19 smithi087 ceph-mon[89014]: pgmap v1368: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 11625/268692 objects degraded (4.327%); 10580/268692 objects misplaced (3.938%); 6.0 MiB/s, 95 objects/s recovering 2024-04-12T11:34:19.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:19 smithi029 ceph-mon[93650]: pgmap v1368: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 11625/268692 objects degraded (4.327%); 10580/268692 objects misplaced (3.938%); 6.0 MiB/s, 95 objects/s recovering 2024-04-12T11:34:19.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:19 smithi029 ceph-mon[97787]: pgmap v1368: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 11625/268692 objects degraded (4.327%); 10580/268692 objects misplaced (3.938%); 6.0 MiB/s, 95 objects/s recovering 2024-04-12T11:34:19.555 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:19.594 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:19.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:19.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 93848 93832 80.3212 67.3125 0.015867 0.012445 2024-04-12T11:34:19.696 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:19.728 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:19.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:19.830 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:19.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:19.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:19.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:19.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:20.004 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:20.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:20.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:20.106 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:20.137 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:20.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:20.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:20.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:20.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:20.263 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:20.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:20.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:20.406 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:20.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:20.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:20.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:20 smithi087 ceph-mon[89014]: pgmap v1369: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 11625/276303 objects degraded (4.207%); 9702/276303 objects misplaced (3.511%); 9.5 MiB/s, 151 objects/s recovering 2024-04-12T11:34:20.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:20 smithi087 ceph-mon[89014]: osdmap e1106: 8 total, 8 up, 8 in 2024-04-12T11:34:20.509 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:20.517 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:20 smithi029 ceph-mon[97787]: pgmap v1369: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 11625/276303 objects degraded (4.207%); 9702/276303 objects misplaced (3.511%); 9.5 MiB/s, 151 objects/s recovering 2024-04-12T11:34:20.517 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:20 smithi029 ceph-mon[97787]: osdmap e1106: 8 total, 8 up, 8 in 2024-04-12T11:34:20.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:20 smithi029 ceph-mon[93650]: pgmap v1369: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 11625/276303 objects degraded (4.207%); 9702/276303 objects misplaced (3.511%); 9.5 MiB/s, 151 objects/s recovering 2024-04-12T11:34:20.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:20 smithi029 ceph-mon[93650]: osdmap e1106: 8 total, 8 up, 8 in 2024-04-12T11:34:20.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:20.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:20.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:20.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 94732 94716 79.9823 55.25 0.0036449 0.0124621 2024-04-12T11:34:20.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:20.665 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:20.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:20.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:20.766 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:20.790 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:20.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:20.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:20.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:20.916 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:20.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:20.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:21.018 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:21.041 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:21.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:21.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:21.143 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:21.170 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:21.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:21.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:21.272 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:21.311 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:21.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:21.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:21.453 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:21.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:21.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:21.555 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:21.588 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:21.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:21.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:21.623 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 95239 95223 79.3384 31.6875 0.0123994 0.0125992 2024-04-12T11:34:21.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:21.729 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:21.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:21.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:21.830 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:21.853 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:21.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:21.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:21.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:21.983 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:21.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:21.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:22.085 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:22.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:22.121 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:22.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:22.222 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:22.266 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:22.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:22.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:22.368 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:22.392 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:22.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:22.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:22.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:34:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:34:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:34:22.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:22 smithi087 ceph-mon[89014]: osdmap e1107: 8 total, 8 up, 8 in 2024-04-12T11:34:22.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:22.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:22 smithi087 ceph-mon[89014]: pgmap v1372: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 11625/277665 objects degraded (4.187%); 9702/277665 objects misplaced (3.494%); 8.7 MiB/s, 138 objects/s recovering 2024-04-12T11:34:22.494 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:22.517 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:22 smithi029 ceph-mon[97787]: osdmap e1107: 8 total, 8 up, 8 in 2024-04-12T11:34:22.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:22.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:22 smithi029 ceph-mon[97787]: pgmap v1372: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 11625/277665 objects degraded (4.187%); 9702/277665 objects misplaced (3.494%); 8.7 MiB/s, 138 objects/s recovering 2024-04-12T11:34:22.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:22 smithi029 ceph-mon[93650]: osdmap e1107: 8 total, 8 up, 8 in 2024-04-12T11:34:22.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:22.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:22 smithi029 ceph-mon[93650]: pgmap v1372: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 11625/277665 objects degraded (4.187%); 9702/277665 objects misplaced (3.494%); 8.7 MiB/s, 138 objects/s recovering 2024-04-12T11:34:22.538 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:22.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:22.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:22.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 96078 96062 78.9844 52.4375 0.0139963 0.0126556 2024-04-12T11:34:22.640 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:22.674 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:22.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:22.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:22.776 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:22.808 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:22.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:22.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:22.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:22.933 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:22.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:22.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:23.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:23.075 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:23.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:23.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:23.177 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:23.205 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:23.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:23.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:23.307 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:23.351 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:23.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:23.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:23.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:23.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11625/277665 objects degraded (4.187%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:23.502 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:23.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:23.517 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11625/277665 objects degraded (4.187%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:23.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11625/277665 objects degraded (4.187%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:23.604 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:23.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 97103 97087 78.7906 64.0625 0.00929965 0.0126862 2024-04-12T11:34:23.634 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:23.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:23.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:23.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:23.780 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:23.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:23.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:23.882 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:23.926 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:23.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:23.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:24.028 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:24.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:24.068 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:24.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:24.169 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:24.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:24.193 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:24.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:24.294 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:24.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:24.333 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:24.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:24.434 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:24.465 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:24.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:24.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:24.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:24 smithi087 ceph-mon[89014]: 6.18 deep-scrub starts 2024-04-12T11:34:24.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:24 smithi087 ceph-mon[89014]: pgmap v1373: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 11625/285012 objects degraded (4.079%); 9027/285012 objects misplaced (3.167%); 8.3 MiB/s, 133 objects/s recovering 2024-04-12T11:34:24.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:24 smithi029 ceph-mon[97787]: 6.18 deep-scrub starts 2024-04-12T11:34:24.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:24 smithi029 ceph-mon[97787]: pgmap v1373: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 11625/285012 objects degraded (4.079%); 9027/285012 objects misplaced (3.167%); 8.3 MiB/s, 133 objects/s recovering 2024-04-12T11:34:24.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:24 smithi029 ceph-mon[93650]: 6.18 deep-scrub starts 2024-04-12T11:34:24.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:24 smithi029 ceph-mon[93650]: pgmap v1373: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 11625/285012 objects degraded (4.079%); 9027/285012 objects misplaced (3.167%); 8.3 MiB/s, 133 objects/s recovering 2024-04-12T11:34:24.567 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:24.593 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:24.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:24.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:24.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 98036 98020 78.528 58.3125 0.0297254 0.0127209 2024-04-12T11:34:24.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:24.717 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:24.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:24.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:24.860 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:24.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:24.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:24.962 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:25.008 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:25.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:25.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:25.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:25.133 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:25.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:25.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:25.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:25.265 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:25.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:25.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:34:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:34:24] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-04-12T11:34:25.366 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:25.413 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:25.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:25.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:25.514 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:25.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:25.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:25.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:25.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 98744 98728 78.0941 44.25 0.0142774 0.0127993 2024-04-12T11:34:25.649 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:25.678 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:25.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:25.780 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:25.812 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:25.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:25.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:25.914 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:25.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:25.952 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:25.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:26.053 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:26.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:26.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:26.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:26.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:26.219 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:26.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:26.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:26.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:26.344 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:26.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:26.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:26.446 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:26.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:26.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:26.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:26.577 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:26.609 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:26.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:26.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:26.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:34:26.624992+0000 min lat: 0.00141572 max lat: 2.5959 avg lat: 0.0128591 2024-04-12T11:34:26.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:34:26.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 99524 99508 77.7272 48.75 0.0147187 0.0128591 2024-04-12T11:34:26.711 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:26.751 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:26.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:26.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:26.852 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:26.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:26.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:26.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:26.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:27.011 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:27.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:27.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:27.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:26 smithi029 ceph-mon[97787]: pgmap v1374: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 11625/289788 objects degraded (4.012%); 9027/289788 objects misplaced (3.115%); 8.3 MiB/s, 133 objects/s recovering 2024-04-12T11:34:27.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:26 smithi029 ceph-mon[93650]: pgmap v1374: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 11625/289788 objects degraded (4.012%); 9027/289788 objects misplaced (3.115%); 8.3 MiB/s, 133 objects/s recovering 2024-04-12T11:34:27.112 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:27.140 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:27.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:27.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:26 smithi087 ceph-mon[89014]: pgmap v1374: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 11625/289788 objects degraded (4.012%); 9027/289788 objects misplaced (3.115%); 8.3 MiB/s, 133 objects/s recovering 2024-04-12T11:34:27.242 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:27.265 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:27.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:27.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:27.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:27.408 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:27.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:27.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:27.510 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:27.547 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:27.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:27.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 100271 100255 77.344 46.6875 0.0590044 0.0129226 2024-04-12T11:34:27.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:27.672 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:27.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:27.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:27.773 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:27.797 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:27.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:27.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:27.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:27.922 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:27.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:27.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:28.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:28.050 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:28.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:28.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:28.190 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:28.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:28.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:28.292 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:28.326 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:28.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:28.428 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:28.473 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:28.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:28.574 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:28.598 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:28.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:28.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:28.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 100740 100724 76.7583 29.3125 0.0316456 0.0130008 2024-04-12T11:34:28.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:28.728 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:28.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:28.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:28.830 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:28.864 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:28.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:28.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:28.966 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:29.012 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:29.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:29.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:28 smithi029 ceph-mon[97787]: pgmap v1375: 99 pgs: 1 active+clean+scrubbing+deep, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 757 op/s; 11625/294492 objects degraded (3.947%); 9027/294492 objects misplaced (3.065%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:34:29.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11625/294492 objects degraded (3.947%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:28 smithi029 ceph-mon[93650]: pgmap v1375: 99 pgs: 1 active+clean+scrubbing+deep, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 757 op/s; 11625/294492 objects degraded (3.947%); 9027/294492 objects misplaced (3.065%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:34:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11625/294492 objects degraded (3.947%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:29.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:29.152 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:29.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:29.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:28 smithi087 ceph-mon[89014]: pgmap v1375: 99 pgs: 1 active+clean+scrubbing+deep, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 757 op/s; 11625/294492 objects degraded (3.947%); 9027/294492 objects misplaced (3.065%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:34:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11625/294492 objects degraded (3.947%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:29.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:29.276 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:29.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:29.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:29.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:29.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:29.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:29.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:29.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:29.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:29.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:29.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 101387 101371 76.3207 40.4375 0.0112627 0.0130969 2024-04-12T11:34:29.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:29.662 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:29.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:29.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:29.764 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:29.804 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:29.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:29.906 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:29.936 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:29.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:29.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:30.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:30.081 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:30.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:30.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:30.225 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:30.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:30.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:30.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:30.368 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:30.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:30.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:30.470 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:30.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:30.503 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:30.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:30.604 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:30.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 102362 102346 76.1375 60.9375 0.00408239 0.0131193 2024-04-12T11:34:30.627 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:30.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:30.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:30.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:30.752 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:30.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:30.854 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:30.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:30.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:30.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:30.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:30 smithi029 ceph-mon[93650]: pgmap v1376: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s; 11625/300354 objects degraded (3.870%); 8577/300354 objects misplaced (2.856%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:34:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:30 smithi029 ceph-mon[97787]: pgmap v1376: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s; 11625/300354 objects degraded (3.870%); 8577/300354 objects misplaced (2.856%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:34:31.029 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:31.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:31.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:31.130 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:31.160 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:31.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:31.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:30 smithi087 ceph-mon[89014]: pgmap v1376: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s; 11625/300354 objects degraded (3.870%); 8577/300354 objects misplaced (2.856%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:34:31.262 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:31.304 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:31.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:31.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:31.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:31.449 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:31.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:31.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:31.551 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:31.591 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:31.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:31.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:31.624 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 103236 103220 75.8844 54.625 0.0155547 0.013172 2024-04-12T11:34:31.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:31.736 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:31.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:31.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:31.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:31.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:31.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:31.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:31.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:31.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:31.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:32.090 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:32.122 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:32.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:32.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:32.224 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:32.254 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:32.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:32.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:32.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:32.398 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:32.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:32.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:32.500 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:32.524 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:32.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:32.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:32.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 103829 103813 75.4329 37.0625 0.0767078 0.0132429 2024-04-12T11:34:32.625 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:32.662 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:32.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:34:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:34:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:34:32.764 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:32.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:32.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:32.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:32.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:32.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:33.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:32 smithi029 ceph-mon[93650]: pgmap v1377: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 11625/301323 objects degraded (3.858%); 8577/301323 objects misplaced (2.846%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:34:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:32 smithi029 ceph-mon[97787]: pgmap v1377: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 11625/301323 objects degraded (3.858%); 8577/301323 objects misplaced (2.846%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:34:33.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:33.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:33.082 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:33.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:33.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:32 smithi087 ceph-mon[89014]: pgmap v1377: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 11625/301323 objects degraded (3.858%); 8577/301323 objects misplaced (2.846%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:34:33.224 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:33.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:33.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:33.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:33.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:33.373 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:33.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:33.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:33.507 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:33.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:33.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:33.608 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:33.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 104585 104569 75.109 47.25 0.00828015 0.013292 2024-04-12T11:34:33.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:33.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:33.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:33.733 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:33.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:33.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:33.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:33.860 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:33.883 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:33.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:33.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:33.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:34.008 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:34.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:34.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:34.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11625/301323 objects degraded (3.858%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:34.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11625/301323 objects degraded (3.858%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:34.109 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:34.132 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:34.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:34.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:34.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11625/301323 objects degraded (3.858%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:34.235 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:34.258 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:34.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:34.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:34.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:34.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:34.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:34.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:34.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:34.536 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:34.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:34.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:34.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 104953 104937 74.5168 23 0.00847627 0.0134145 2024-04-12T11:34:34.638 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:34.684 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:34.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:34.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:34.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:34.825 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:34.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:34.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:34.927 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:34.957 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:34.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:35.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:34 smithi029 ceph-mon[97787]: pgmap v1378: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 11625/308520 objects degraded (3.768%); 7859/308520 objects misplaced (2.547%); 5.1 MiB/s, 81 objects/s recovering 2024-04-12T11:34:35.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:34 smithi029 ceph-mon[93650]: pgmap v1378: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 11625/308520 objects degraded (3.768%); 7859/308520 objects misplaced (2.547%); 5.1 MiB/s, 81 objects/s recovering 2024-04-12T11:34:35.022 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:34:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:34:34] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-04-12T11:34:35.059 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:35.093 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:35.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:35.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:35.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:34 smithi087 ceph-mon[89014]: pgmap v1378: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 11625/308520 objects degraded (3.768%); 7859/308520 objects misplaced (2.547%); 5.1 MiB/s, 81 objects/s recovering 2024-04-12T11:34:35.249 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:35.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:35.350 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:35.382 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:35.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:35.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:35.530 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:35.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:35.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:35.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 105925 105909 74.3621 60.75 0.0359041 0.0134423 2024-04-12T11:34:35.631 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:35.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:35.701 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:35.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:35.802 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:35.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:35.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:35.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:35.965 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:35.997 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:35.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:36.015 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:34:36.015 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:34:36.015 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:34:36.056 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:34:36.099 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:34:36.099 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:36.139 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:34:36.142 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:36.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:36.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:36.175 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:34:36.214 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:34:36.227 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:34:36.227 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:34:36.227 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:34:36.227 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:34:36.227 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:34:36.227 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:34:36.227 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:34:36.227 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:34:36.245 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:36.273 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:36.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:36.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:36.375 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:36.404 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:36.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:36.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:36.558 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:36.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:36.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:36.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 90 4 106822 106818 74.167 56.8125 0.0419491 0.0134763 2024-04-12T11:34:36.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:36.690 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:36.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:36.722 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:34:36.722 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:34:36.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:36.815 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:36.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:36.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:36.945 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:36.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:36.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:36 smithi029 ceph-mon[97787]: pgmap v1379: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s; 11625/312567 objects degraded (3.719%); 7859/312567 objects misplaced (2.514%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T11:34:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:36 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2158526893' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:34:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:36 smithi029 ceph-mon[93650]: pgmap v1379: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s; 11625/312567 objects degraded (3.719%); 7859/312567 objects misplaced (2.514%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T11:34:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:36 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2158526893' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:34:37.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:37.047 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:37.071 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:37.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:37.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:37.196 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:37.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:37.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:36 smithi087 ceph-mon[89014]: pgmap v1379: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 765 op/s; 11625/312567 objects degraded (3.719%); 7859/312567 objects misplaced (2.514%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T11:34:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:36 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2158526893' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:34:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:37.298 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:37.322 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:37.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:37.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:37.424 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:37.448 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:37.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:37.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:37.550 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:37.577 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:37.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:37.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:37.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.027 2024-04-12T11:34:37.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 106822 2024-04-12T11:34:37.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2024-04-12T11:34:37.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2024-04-12T11:34:37.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 74.1597 2024-04-12T11:34:37.625 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 45.1611 2024-04-12T11:34:37.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 224.625 2024-04-12T11:34:37.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2024-04-12T11:34:37.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1186 2024-04-12T11:34:37.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 722.578 2024-04-12T11:34:37.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3594 2024-04-12T11:34:37.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2024-04-12T11:34:37.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0134776 2024-04-12T11:34:37.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0584632 2024-04-12T11:34:37.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 2.5959 2024-04-12T11:34:37.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00141572 2024-04-12T11:34:37.632 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2024-04-12T11:34:37.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:37.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:37.725 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:37.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:37.826 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:37.874 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:37.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:37.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:37.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:37 smithi029 ceph-mon[93650]: osdmap e1108: 8 total, 8 up, 8 in 2024-04-12T11:34:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:37 smithi029 ceph-mon[97787]: osdmap e1108: 8 total, 8 up, 8 in 2024-04-12T11:34:37.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:38.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:38.026 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:38.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:38.127 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:38.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:38.156 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:38.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:37 smithi087 ceph-mon[89014]: osdmap e1108: 8 total, 8 up, 8 in 2024-04-12T11:34:38.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:38.319 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:38.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:38.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:38.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:38.452 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:38.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:38.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:38.553 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:38.596 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:38.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:38.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:38.698 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:38.729 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:38.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:38.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:38.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:38.876 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:38.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:38.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:38.978 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:39.005 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:39.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[97787]: pgmap v1381: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s; 11625/316122 objects degraded (3.677%); 7859/316122 objects misplaced (2.486%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:34:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[97787]: osdmap e1109: 8 total, 8 up, 8 in 2024-04-12T11:34:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11625/316122 objects degraded (3.677%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[97787]: 6.18 deep-scrub ok 2024-04-12T11:34:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[93650]: pgmap v1381: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s; 11625/316122 objects degraded (3.677%); 7859/316122 objects misplaced (2.486%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:34:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[93650]: osdmap e1109: 8 total, 8 up, 8 in 2024-04-12T11:34:39.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11625/316122 objects degraded (3.677%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:39.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[93650]: 6.18 deep-scrub ok 2024-04-12T11:34:39.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:39.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:38 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:39.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:39.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:39.164 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:39.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:38 smithi087 ceph-mon[89014]: pgmap v1381: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s; 11625/316122 objects degraded (3.677%); 7859/316122 objects misplaced (2.486%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:34:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:38 smithi087 ceph-mon[89014]: osdmap e1109: 8 total, 8 up, 8 in 2024-04-12T11:34:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11625/316122 objects degraded (3.677%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:38 smithi087 ceph-mon[89014]: 6.18 deep-scrub ok 2024-04-12T11:34:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:38 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:38 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:39.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:39.267 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:34:39.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:39.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:39.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:39.464 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:39.489 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:39.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:39.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:39.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:39.625 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:39.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:39.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:39.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:39.766 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:39.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:39.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:39.802 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:34:39.868 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:39.900 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:39.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:39.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:40.002 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:40.036 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:40.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:40.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:40.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:40.164 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:40.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:40.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:40.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:39 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/317702488' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:34:40.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:39 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:34:40.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:39 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/317702488' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:34:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:39 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:34:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:39 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/317702488' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:34:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:39 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:34:40.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:40.292 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:40.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:40.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:40.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:40.421 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:40.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:40.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:40.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:40.551 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:40.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:40.652 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:40.686 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:40.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:40.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:40.787 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:40.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:40.812 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:40.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:40.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:40.951 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:40.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:40.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:41.054 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:41.088 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:41.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:41.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:41.190 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:40 smithi087 ceph-mon[89014]: pgmap v1383: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 834 op/s; 7127/318663 objects degraded (2.237%); 11241/318663 objects misplaced (3.528%); 6.8 MiB/s, 108 objects/s recovering 2024-04-12T11:34:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:40 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:34:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:40 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:34:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:40 smithi087 ceph-mon[89014]: osdmap e1110: 8 total, 8 up, 8 in 2024-04-12T11:34:41.235 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:41.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:41.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:41.266 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:40 smithi029 ceph-mon[97787]: pgmap v1383: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 834 op/s; 7127/318663 objects degraded (2.237%); 11241/318663 objects misplaced (3.528%); 6.8 MiB/s, 108 objects/s recovering 2024-04-12T11:34:41.266 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:40 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:34:41.266 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:40 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:34:41.266 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:40 smithi029 ceph-mon[97787]: osdmap e1110: 8 total, 8 up, 8 in 2024-04-12T11:34:41.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:40 smithi029 ceph-mon[93650]: pgmap v1383: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 834 op/s; 7127/318663 objects degraded (2.237%); 11241/318663 objects misplaced (3.528%); 6.8 MiB/s, 108 objects/s recovering 2024-04-12T11:34:41.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:40 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:34:41.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:40 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:34:41.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:40 smithi029 ceph-mon[93650]: osdmap e1110: 8 total, 8 up, 8 in 2024-04-12T11:34:41.308 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-04-12T11:34:41.336 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:41.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:41.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:41.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:41.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:41.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:41.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:41.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:41.630 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:41.658 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:41.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:41.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:41.760 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:41.789 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:41.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:41.892 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:41.924 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:41.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:41.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:34:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:34:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:42.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:34:42.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:34:42.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:42.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:42.059 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:42.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:42.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:42.203 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:42.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:42.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:42.208 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:34:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:34:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:34:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:42.269 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:34:42.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:42.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:42.353 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:42.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:42.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:42.496 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:42.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:42.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:42.597 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:42.628 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:42.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:42.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:34:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:34:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:34:42.730 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:42.774 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:42.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:42.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:42.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:42.914 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:42.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:42.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:42.975 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:42 smithi029 ceph-mon[97787]: pgmap v1385: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 27 MiB/s wr, 543 op/s; 7127/318624 objects degraded (2.237%); 11241/318624 objects misplaced (3.528%); 5.3 MiB/s, 85 objects/s recovering 2024-04-12T11:34:42.975 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:42 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 7127/318624 objects degraded (2.237%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:42.976 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:42 smithi029 ceph-mon[93650]: pgmap v1385: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 27 MiB/s wr, 543 op/s; 7127/318624 objects degraded (2.237%); 11241/318624 objects misplaced (3.528%); 5.3 MiB/s, 85 objects/s recovering 2024-04-12T11:34:42.976 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:42 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 7127/318624 objects degraded (2.237%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:43.016 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:43.064 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:43.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:43.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:43.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:43.210 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:43.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:42 smithi087 ceph-mon[89014]: pgmap v1385: 99 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 27 MiB/s wr, 543 op/s; 7127/318624 objects degraded (2.237%); 11241/318624 objects misplaced (3.528%); 5.3 MiB/s, 85 objects/s recovering 2024-04-12T11:34:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:42 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 7127/318624 objects degraded (2.237%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:43.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:43.345 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:43.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:43.371 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:34:43.447 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:43.476 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:43.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:43.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:43.578 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:43.610 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:43.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:43.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:43.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:43.753 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:43.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:43.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:43.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:43.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:43.896 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:43.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:43.997 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:44.031 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:44.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:44.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:44.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:34:44.174 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:44.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:44.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:44.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:44.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:44.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:44.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:44.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:44.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:44.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:44.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:44.566 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:44.598 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:44.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:44.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:44.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:44.755 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:44.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:44.856 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:44.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:44 smithi029 ceph-mon[93650]: pgmap v1386: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 147 B/s rd, 0 B/s wr, 411 op/s; 6853/310392 objects degraded (2.208%); 10710/310392 objects misplaced (3.450%); 7.1 MiB/s, 112 objects/s recovering 2024-04-12T11:34:44.893 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:44 smithi029 ceph-mon[97787]: pgmap v1386: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 147 B/s rd, 0 B/s wr, 411 op/s; 6853/310392 objects degraded (2.208%); 10710/310392 objects misplaced (3.450%); 7.1 MiB/s, 112 objects/s recovering 2024-04-12T11:34:44.922 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:44.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:44.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:45.024 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:45.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:45.073 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:45.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:45.174 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:45.201 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:45.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:45.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:44 smithi087 ceph-mon[89014]: pgmap v1386: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 147 B/s rd, 0 B/s wr, 411 op/s; 6853/310392 objects degraded (2.208%); 10710/310392 objects misplaced (3.450%); 7.1 MiB/s, 112 objects/s recovering 2024-04-12T11:34:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:34:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:34:44] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.43.0" 2024-04-12T11:34:45.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:45.327 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:45.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:45.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:45.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:45.466 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:45.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:45.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:45.568 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:45.593 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:45.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:45.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:45.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:45.734 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:45.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:45.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:45.836 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:45.872 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:45.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:45.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:46.022 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:46.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:46.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:46.124 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:46.164 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:46.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:46.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:46.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:46.298 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:46.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:46.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:46.400 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:46.437 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:46.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:46.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:46.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:46.562 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:46.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:46.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:46.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:46.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:46.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:46.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:46.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:46.844 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:46.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:46.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:46.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:46.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:47.090 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:47.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:47.122 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:47.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:46 smithi087 ceph-mon[89014]: pgmap v1387: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 0 B/s wr, 432 op/s; 6853/305019 objects degraded (2.247%); 10710/305019 objects misplaced (3.511%); 6.1 MiB/s, 97 objects/s recovering 2024-04-12T11:34:47.224 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:47.263 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:47.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:47.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:47.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:46 smithi029 ceph-mon[97787]: pgmap v1387: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 0 B/s wr, 432 op/s; 6853/305019 objects degraded (2.247%); 10710/305019 objects misplaced (3.511%); 6.1 MiB/s, 97 objects/s recovering 2024-04-12T11:34:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:46 smithi029 ceph-mon[93650]: pgmap v1387: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 0 B/s wr, 432 op/s; 6853/305019 objects degraded (2.247%); 10710/305019 objects misplaced (3.511%); 6.1 MiB/s, 97 objects/s recovering 2024-04-12T11:34:47.365 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:47.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:47.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:47.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:47.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:47.534 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:47.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:47.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:47.636 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:47.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:47.661 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:47.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:47.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:47.805 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:47.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:47.907 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:47.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:47.939 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:47.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:48.040 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:48.071 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:48.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:48.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:48.206 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:48.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:48.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:48.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:47 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 6853/305019 objects degraded (2.247%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:48.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:47 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 6853/305019 objects degraded (2.247%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:48.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:47 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 6853/305019 objects degraded (2.247%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:48.308 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:48.332 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:48.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:48.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:48.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:48.476 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:48.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:48.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:48.578 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:48.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:48.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:48.717 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:48.744 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:48.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:48.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:48.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:48.869 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:48.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:48.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:48.970 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:49.013 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:49.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:49.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:49.115 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:49.144 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:49.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:49.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:48 smithi087 ceph-mon[89014]: pgmap v1388: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 103 B/s rd, 0 B/s wr, 547 op/s; 6853/299112 objects degraded (2.291%); 10710/299112 objects misplaced (3.581%); 4.9 MiB/s, 78 objects/s recovering 2024-04-12T11:34:49.247 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:49.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:48 smithi029 ceph-mon[93650]: pgmap v1388: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 103 B/s rd, 0 B/s wr, 547 op/s; 6853/299112 objects degraded (2.291%); 10710/299112 objects misplaced (3.581%); 4.9 MiB/s, 78 objects/s recovering 2024-04-12T11:34:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:48 smithi029 ceph-mon[97787]: pgmap v1388: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 103 B/s rd, 0 B/s wr, 547 op/s; 6853/299112 objects degraded (2.291%); 10710/299112 objects misplaced (3.581%); 4.9 MiB/s, 78 objects/s recovering 2024-04-12T11:34:49.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:49.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:49.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:49.371 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:49.402 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:49.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:49.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:49.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:49.533 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:49.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:49.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:49.635 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:49.658 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:49.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:49.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:49.760 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:49.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:49.789 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:49.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:49.890 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:49.919 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:49.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:49.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:49.994 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:34:50.021 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:50.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:50.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:50.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:50.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:50.180 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:50.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:50.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:50.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:50.305 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:50.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:50.407 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:50.454 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:50.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:50.555 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:50.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:50.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:50.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:50.688 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:50.728 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:50.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:50.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:50.830 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:50.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:50.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:50.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:50.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:51.005 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:51.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:51.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:51.107 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:51.130 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:51.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:51.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:50 smithi087 ceph-mon[89014]: pgmap v1389: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 625 op/s; 6533/291381 objects degraded (2.242%); 10092/291381 objects misplaced (3.464%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:34:51.232 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:51.255 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:51.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:51.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:51.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:50 smithi029 ceph-mon[93650]: pgmap v1389: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 625 op/s; 6533/291381 objects degraded (2.242%); 10092/291381 objects misplaced (3.464%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:34:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:50 smithi029 ceph-mon[97787]: pgmap v1389: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 625 op/s; 6533/291381 objects degraded (2.242%); 10092/291381 objects misplaced (3.464%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:34:51.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:51.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:51.393 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:51.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:51.494 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:51.540 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:51.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:51.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:51.642 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:51.665 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:51.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:51.766 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:51.801 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:51.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:51.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:51.903 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:51.931 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:51.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:51.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:52.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:52.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:52.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:52.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:52.206 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:52.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:52.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:52.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:52.343 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:52.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:52.445 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:52.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:52.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:52.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:52.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:52.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:52 smithi029 ceph-mon[93650]: pgmap v1390: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 86 B/s rd, 0 B/s wr, 533 op/s; 6533/290139 objects degraded (2.252%); 10092/290139 objects misplaced (3.478%); 3.1 MiB/s, 48 objects/s recovering 2024-04-12T11:34:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:52 smithi029 ceph-mon[97787]: pgmap v1390: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 86 B/s rd, 0 B/s wr, 533 op/s; 6533/290139 objects degraded (2.252%); 10092/290139 objects misplaced (3.478%); 3.1 MiB/s, 48 objects/s recovering 2024-04-12T11:34:52.586 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:52.609 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:52.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:52.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:52.711 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:52.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:34:52.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:34:52.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:52 smithi087 ceph-mon[89014]: pgmap v1390: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 86 B/s rd, 0 B/s wr, 533 op/s; 6533/290139 objects degraded (2.252%); 10092/290139 objects misplaced (3.478%); 3.1 MiB/s, 48 objects/s recovering 2024-04-12T11:34:52.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:34:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:34:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:34:52.743 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:52.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:52.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:52.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:52.888 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:52.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:52.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:52.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:53.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:53.037 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:53.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:53.138 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:53.178 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:53.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:53.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:53.320 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:53.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:53.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:53.422 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:53.455 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:53.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:53.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:53.557 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:53.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:53.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:53.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:53.688 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:53.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 6533/290139 objects degraded (2.252%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:53.721 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:53.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:53.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 6533/290139 objects degraded (2.252%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:53.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 6533/290139 objects degraded (2.252%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:34:53.822 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:53.856 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:53.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:53.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:53.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:53.999 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:54.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:54.102 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:54.131 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:54.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:54.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:54.233 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:54.262 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:54.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:54.364 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:54.395 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:54.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:54.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:54.547 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:54.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:54.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:54.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:54 smithi029 ceph-mon[93650]: pgmap v1391: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 0 B/s wr, 748 op/s; 6287/282201 objects degraded (2.228%); 9642/282201 objects misplaced (3.417%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:34:54.571 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:54 smithi029 ceph-mon[97787]: pgmap v1391: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 0 B/s wr, 748 op/s; 6287/282201 objects degraded (2.228%); 9642/282201 objects misplaced (3.417%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:34:54.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:54.689 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:54.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:54.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:54.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:54 smithi087 ceph-mon[89014]: pgmap v1391: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 0 B/s wr, 748 op/s; 6287/282201 objects degraded (2.228%); 9642/282201 objects misplaced (3.417%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:34:54.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:54.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:54.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:54.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:54.958 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:54.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:54.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:55.060 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:55.089 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:55.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:55.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:55.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:55.206 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:34:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:34:54] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.43.0" 2024-04-12T11:34:55.230 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:55.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:55.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:55.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:55.359 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:55.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:55.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:55.461 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:55.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:55.501 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:55.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:55.602 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:55.632 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:55.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:55.734 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:55.761 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:55.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:55.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:55.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:55.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:55.912 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:55.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:56.013 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:56.036 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:56.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:56.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:56.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:56.162 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:56.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:56.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:56.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:56.289 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:56.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:56.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:56.392 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:56.420 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:56.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:56.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:56.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:56.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:56.563 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:56.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:56.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:56.710 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:56.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:56.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:56.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:56.854 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:56.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:56.956 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:56.978 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:56.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:56.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:57.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:56 smithi029 ceph-mon[93650]: pgmap v1392: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 6287/277068 objects degraded (2.269%); 9642/277068 objects misplaced (3.480%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T11:34:57.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:56 smithi029 ceph-mon[97787]: pgmap v1392: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 6287/277068 objects degraded (2.269%); 9642/277068 objects misplaced (3.480%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T11:34:57.080 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:57.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:57.104 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:57.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:57.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:56 smithi087 ceph-mon[89014]: pgmap v1392: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 6287/277068 objects degraded (2.269%); 9642/277068 objects misplaced (3.480%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T11:34:57.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:57.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:57.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:57.346 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:57.385 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:57.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:57.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:57.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:57.509 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:57.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:57.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:57.635 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:57.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:57.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:57.737 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:34:57.761 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:57.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:34:57.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:57.902 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:57.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:58.004 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:58.034 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:58.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:58.135 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:58.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:58.159 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:58.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:58.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:58.293 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:58.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:58.395 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:58.426 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:58.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:34:58.528 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:34:58.561 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:58.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:58.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:34:58.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:58.703 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:58.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:58.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:58.805 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:58.833 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:58.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:58.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:58.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:58.980 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:58.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:58 smithi029 ceph-mon[97787]: pgmap v1393: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 6287/272286 objects degraded (2.309%); 9642/272286 objects misplaced (3.541%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T11:34:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 6287/272286 objects degraded (2.309%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:34:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:58 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2848982093' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.7306737697738339}]: dispatch 2024-04-12T11:34:59.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:58 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.7306737697738339}]: dispatch 2024-04-12T11:34:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:58 smithi029 ceph-mon[93650]: pgmap v1393: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 6287/272286 objects degraded (2.309%); 9642/272286 objects misplaced (3.541%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T11:34:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 6287/272286 objects degraded (2.309%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:34:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:58 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2848982093' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.7306737697738339}]: dispatch 2024-04-12T11:34:59.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:58 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.7306737697738339}]: dispatch 2024-04-12T11:34:59.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:34:59.104 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:59.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:59.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:34:59.206 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:34:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:58 smithi087 ceph-mon[89014]: pgmap v1393: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 6287/272286 objects degraded (2.309%); 9642/272286 objects misplaced (3.541%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T11:34:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 6287/272286 objects degraded (2.309%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:34:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:58 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2848982093' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.7306737697738339}]: dispatch 2024-04-12T11:34:59.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:58 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.7306737697738339}]: dispatch 2024-04-12T11:34:59.236 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:59.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:59.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:34:59.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:34:59.378 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:59.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:34:59.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:59.519 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:59.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:59.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:59.620 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:34:59.656 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:59.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:59.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:34:59.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:34:59.781 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:34:59.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:59.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:34:59.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:34:59.916 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:34:59.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:34:59.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:00.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:59 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.7306737697738339}]': finished 2024-04-12T11:35:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:59 smithi029 ceph-mon[97787]: osdmap e1111: 8 total, 8 up, 8 in 2024-04-12T11:35:00.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:34:59 smithi029 ceph-mon[97787]: osdmap e1112: 8 total, 8 up, 8 in 2024-04-12T11:35:00.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:59 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.7306737697738339}]': finished 2024-04-12T11:35:00.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:59 smithi029 ceph-mon[93650]: osdmap e1111: 8 total, 8 up, 8 in 2024-04-12T11:35:00.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:34:59 smithi029 ceph-mon[93650]: osdmap e1112: 8 total, 8 up, 8 in 2024-04-12T11:35:00.053 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:00.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:00.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:00.154 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:00.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:00.178 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:00.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:00.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:59 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.7306737697738339}]': finished 2024-04-12T11:35:00.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:59 smithi087 ceph-mon[89014]: osdmap e1111: 8 total, 8 up, 8 in 2024-04-12T11:35:00.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:34:59 smithi087 ceph-mon[89014]: osdmap e1112: 8 total, 8 up, 8 in 2024-04-12T11:35:00.267 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.6 primary-affinity to 0.730674 (8478852) 2024-04-12T11:35:00.279 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:00.310 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:00.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:00.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:00.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:00.455 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:00.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:00.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:00.557 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:00.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:00.581 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:00.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:00.682 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:00.715 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:00.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:00.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:00.852 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:00.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:00.953 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:00.976 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:00.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:00.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:01.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:00 smithi029 ceph-mon[97787]: pgmap v1395: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 6007/265779 objects degraded (2.260%); 9133/265779 objects misplaced (3.436%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:35:01.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:00 smithi029 ceph-mon[93650]: pgmap v1395: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 6007/265779 objects degraded (2.260%); 9133/265779 objects misplaced (3.436%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:35:01.078 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:01.110 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:01.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:01.212 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:01.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:00 smithi087 ceph-mon[89014]: pgmap v1395: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 6007/265779 objects degraded (2.260%); 9133/265779 objects misplaced (3.436%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:35:01.240 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:01.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:01.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:01.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:01.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:01.385 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:01.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:01.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:01.515 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:01.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:01.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:01.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:01.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:01.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:01.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:01.767 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:01.799 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:01.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:01.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:01.900 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:01.935 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:01.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:01.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:02.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:01 smithi029 ceph-mon[97787]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:35:02.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:01 smithi029 ceph-mon[93650]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:35:02.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:02.074 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:02.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:02.175 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:02.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:02.216 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:02.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:02.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:01 smithi087 ceph-mon[89014]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:35:02.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:02.341 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:02.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:02.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:02.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:02.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:02.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:02.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:02.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:02.621 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:02.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:02.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:35:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:35:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:35:02.723 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:02.752 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:02.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:02.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:02.854 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:02.884 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:02.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:02.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:02.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:03.132 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:03.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:03.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:02 smithi087 ceph-mon[89014]: pgmap v1397: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6007/264705 objects degraded (2.269%); 9133/264705 objects misplaced (3.450%); 2.1 MiB/s, 32 objects/s recovering 2024-04-12T11:35:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:02 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 6007/264705 objects degraded (2.269%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:03.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:03.266 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:03.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:03.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:03.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:02 smithi029 ceph-mon[97787]: pgmap v1397: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6007/264705 objects degraded (2.269%); 9133/264705 objects misplaced (3.450%); 2.1 MiB/s, 32 objects/s recovering 2024-04-12T11:35:03.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:02 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 6007/264705 objects degraded (2.269%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:03.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:02 smithi029 ceph-mon[93650]: pgmap v1397: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6007/264705 objects degraded (2.269%); 9133/264705 objects misplaced (3.450%); 2.1 MiB/s, 32 objects/s recovering 2024-04-12T11:35:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:02 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 6007/264705 objects degraded (2.269%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:03.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:03.396 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:03.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:03.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:03.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:03.540 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:03.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:03.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:03.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:03.681 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:03.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:03.783 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:03.813 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:03.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:03.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:03.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:03.953 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:03.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:04.054 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:04.084 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:04.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:04.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:04.221 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:04.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:04.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:04.323 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:04.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:04.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:04.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:04.465 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:04.503 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:04.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:04.605 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:04.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:04.639 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:04.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:04.740 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:04.771 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:04.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:04.873 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:04.902 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:04.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:04.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:05.004 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:05.065 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:05.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:05.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:05.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:04 smithi029 ceph-mon[93650]: pgmap v1398: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 5737/256719 objects degraded (2.235%); 8578/256719 objects misplaced (3.341%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:05.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:04 smithi029 ceph-mon[97787]: pgmap v1398: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 5737/256719 objects degraded (2.235%); 8578/256719 objects misplaced (3.341%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:05.116 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:35:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:35:04] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.43.0" 2024-04-12T11:35:05.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:05.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:04 smithi087 ceph-mon[89014]: pgmap v1398: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 5737/256719 objects degraded (2.235%); 8578/256719 objects misplaced (3.341%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:05.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:05.219 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:05.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:05.254 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:35:05.254 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:35:05.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:05.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:05.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:05.465 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:05.493 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:05.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:05.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:05.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:05.625 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:05.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:05.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:05.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:05.768 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:05.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:05.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:05.870 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:05.898 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:05.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:05.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:06.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:06.044 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:06.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:06.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:06.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:05 smithi029 ceph-mon[93650]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T11:35:06.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:05 smithi029 ceph-mon[97787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T11:35:06.146 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:06.178 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:06.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:06.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:06.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:05 smithi087 ceph-mon[89014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T11:35:06.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:06.327 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:06.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:06.429 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:06.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:06.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:06.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:06.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:06.606 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:06.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:06.707 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:06.750 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:06.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:06.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:06.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:06.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:06.894 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:06.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:06.953 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:06 smithi029 ceph-mon[93650]: pgmap v1399: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 5737/251472 objects degraded (2.281%); 8578/251472 objects misplaced (3.411%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:06.953 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:06.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:06 smithi029 ceph-mon[97787]: pgmap v1399: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 5737/251472 objects degraded (2.281%); 8578/251472 objects misplaced (3.411%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:06.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:06.992 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-04-12T11:35:06.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:07.023 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:07.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:07.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:07.125 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:07.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:07.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:07.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:07.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:06 smithi087 ceph-mon[89014]: pgmap v1399: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 5737/251472 objects degraded (2.281%); 8578/251472 objects misplaced (3.411%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:07.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:07.254 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:07.281 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:07.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:07.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:07.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:07.424 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:07.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:07.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:07.526 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:07.553 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:07.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:07.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:07.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:07.692 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:07.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:07.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:07.794 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:07.822 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:07.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:07.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:07.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:07.965 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:07.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:08.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:07 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5737/251472 objects degraded (2.281%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:08.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:07 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5737/251472 objects degraded (2.281%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:08.067 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:08.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:08.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:08.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:08.099 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:35:08.149 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:35:08.194 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:35:08.196 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:08.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:07 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5737/251472 objects degraded (2.281%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:08.239 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:08.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:08.241 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:35:08.293 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:35:08.337 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:35:08.340 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:08.374 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:35:08.385 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:08.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:08.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:08.410 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:35:08.420 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:35:08.420 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:35:08.420 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:35:08.420 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:35:08.420 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:35:08.421 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:35:08.421 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:35:08.421 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:35:08.487 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:08.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:08.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:08.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:08.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:08.649 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:08.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:08.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:08.751 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:08.779 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:08.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:08.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:08.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:08.934 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:08.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:08.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:08 smithi029 ceph-mon[97787]: pgmap v1400: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 5737/246177 objects degraded (2.330%); 8578/246177 objects misplaced (3.484%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:35:09.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:08 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3948010514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:35:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:08 smithi029 ceph-mon[93650]: pgmap v1400: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 5737/246177 objects degraded (2.330%); 8578/246177 objects misplaced (3.484%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:35:09.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:08 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3948010514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:35:09.036 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:09.072 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:09.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:09.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:09.174 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:09.204 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:09.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:09.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:09.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:08 smithi087 ceph-mon[89014]: pgmap v1400: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 5737/246177 objects degraded (2.330%); 8578/246177 objects misplaced (3.484%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:35:09.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:08 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3948010514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:35:09.290 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:35:09.307 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:09.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:09.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:09.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:09.438 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:09.481 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:09.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:09.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:09.582 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:09.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:09.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:09.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:09.715 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:09.745 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:09.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:09.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:09.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:09.884 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:09.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:09.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:09.913 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:35:09.986 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:10.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3063818842' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:35:10.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3063818842' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:35:10.029 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:10.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:10.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:10.131 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:10.162 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:10.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:10.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:10.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3063818842' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:35:10.264 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:10.309 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:10.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:10.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:10.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:10.459 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:10.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:10.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:10.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:10.598 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:10.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:10.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:10.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:10.729 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:10.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:10.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:10.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:10.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:10.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:10.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:11.017 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:10 smithi029 ceph-mon[97787]: pgmap v1401: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 5441/238461 objects degraded (2.282%); 7946/238461 objects misplaced (3.332%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:35:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:10 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:35:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3063818842' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:35:11.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:10 smithi029 ceph-mon[97787]: osdmap e1113: 8 total, 8 up, 8 in 2024-04-12T11:35:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:10 smithi029 ceph-mon[93650]: pgmap v1401: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 5441/238461 objects degraded (2.282%); 7946/238461 objects misplaced (3.332%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:35:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:10 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:35:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3063818842' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:35:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:10 smithi029 ceph-mon[93650]: osdmap e1113: 8 total, 8 up, 8 in 2024-04-12T11:35:11.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:11.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:11.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:11.163 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:11.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:11.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:11.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:10 smithi087 ceph-mon[89014]: pgmap v1401: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 910 op/s; 5441/238461 objects degraded (2.282%); 7946/238461 objects misplaced (3.332%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:35:11.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:10 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:35:11.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3063818842' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:35:11.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:10 smithi087 ceph-mon[89014]: osdmap e1113: 8 total, 8 up, 8 in 2024-04-12T11:35:11.265 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:11.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:35:11.292 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:11.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:11.394 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:11.423 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:11.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:11.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:11.525 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:11.556 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:11.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:11.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:11.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:11.716 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:11.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:11.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:11.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:11.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:11.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:11.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:11.979 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:35:11.993 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:35:11.993 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:35:11.993 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:35:11.993 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:35:11.993 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:35:12.003 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:12.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:12.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:12.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:12.149 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:12.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:12.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:12.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:12.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:12.286 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:12.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:12.336 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:35:12.336 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:35:12.336 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-04-12T11:35:12.337 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-04-12T11:35:12.387 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:12.420 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:12.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:12.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:12.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:12.549 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:12.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:12.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:12.651 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:12.684 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:12.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:12.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:35:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:35:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:35:12.786 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:12.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:35:12.819 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:12.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:12.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:12.921 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:12.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:12.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:12.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:12 smithi029 ceph-mon[97787]: pgmap v1403: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 5441/237093 objects degraded (2.295%); 7946/237093 objects misplaced (3.351%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:35:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:12 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5441/237093 objects degraded (2.295%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:13.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:12 smithi029 ceph-mon[93650]: pgmap v1403: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 5441/237093 objects degraded (2.295%); 7946/237093 objects misplaced (3.351%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:35:13.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:12 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5441/237093 objects degraded (2.295%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:13.071 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:13.137 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:13.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:13.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:12 smithi087 ceph-mon[89014]: pgmap v1403: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 5441/237093 objects degraded (2.295%); 7946/237093 objects misplaced (3.351%); 3.6 MiB/s, 58 objects/s recovering 2024-04-12T11:35:13.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:12 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5441/237093 objects degraded (2.295%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:13.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:13.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:13.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:13.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:13.399 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:13.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:13.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:13.547 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:13.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:13.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:13.649 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:13.681 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:13.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:13.782 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:13.816 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:13.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:13.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:13.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:13.959 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:13.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:13.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:14.060 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:14.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-04-12T11:35:14.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:14.091 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:14.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:14.192 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:14.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:14.257 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:14.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:14.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:14.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:14.425 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:14.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:14.526 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:14.554 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:14.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:14.656 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:14.684 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:14.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:14.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:14.732 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:35:14.739 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:35:14.739 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:35:14.755 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:35:14.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:14.811 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:35:14.861 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:35:14.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:14.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:14.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:14.903 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:35:14.949 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:35:14.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:14.987 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:35:15.010 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:15.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:15.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:15.029 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:35:15.072 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:35:15.077 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:35:15.077 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:35:15.077 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:35:15.077 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:35:15.077 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:35:15.077 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:35:15.077 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:35:15.077 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:35:15.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:15.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:15.156 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:15.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:15.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:14 smithi029 ceph-mon[97787]: pgmap v1404: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 5141/228369 objects degraded (2.251%); 7260/228369 objects misplaced (3.179%); 4.0 MiB/s, 63 objects/s recovering 2024-04-12T11:35:15.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:14 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1450806504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:35:15.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:14 smithi029 ceph-mon[93650]: pgmap v1404: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 5141/228369 objects degraded (2.251%); 7260/228369 objects misplaced (3.179%); 4.0 MiB/s, 63 objects/s recovering 2024-04-12T11:35:15.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:14 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1450806504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:35:15.193 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:35:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:35:14] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2024-04-12T11:35:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:14 smithi087 ceph-mon[89014]: pgmap v1404: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 5141/228369 objects degraded (2.251%); 7260/228369 objects misplaced (3.179%); 4.0 MiB/s, 63 objects/s recovering 2024-04-12T11:35:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:14 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1450806504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:35:15.257 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:15.286 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:15.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:15.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:15.388 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:15.431 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:15.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:15.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:15.533 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:15.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:15.572 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:15.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:15.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:15.716 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:15.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:15.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:15.840 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:15.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:15.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:15.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:15.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:15.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:15.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:16.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:16.103 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:16.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:16.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:16.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:16.248 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:16.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:16.350 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:16.381 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:16.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:16.483 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:16.524 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:16.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:16.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:16.626 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:16.661 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:16.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:16.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:16.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:16.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:16.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:16.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:16.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:16.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:16.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:16.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:17.056 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:17.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:17.086 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:17.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:17.187 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:17.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:16 smithi087 ceph-mon[89014]: pgmap v1405: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 5141/222888 objects degraded (2.307%); 7260/222888 objects misplaced (3.257%); 4.0 MiB/s, 63 objects/s recovering 2024-04-12T11:35:17.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:17.231 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:17.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:17.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:16 smithi029 ceph-mon[93650]: pgmap v1405: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 5141/222888 objects degraded (2.307%); 7260/222888 objects misplaced (3.257%); 4.0 MiB/s, 63 objects/s recovering 2024-04-12T11:35:17.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:16 smithi029 ceph-mon[97787]: pgmap v1405: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 5141/222888 objects degraded (2.307%); 7260/222888 objects misplaced (3.257%); 4.0 MiB/s, 63 objects/s recovering 2024-04-12T11:35:17.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:17.383 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:17.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:17.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:17.548 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:17.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:17.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:17.700 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:17.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:17.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:17.802 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:17.839 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:17.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:17.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:17.941 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:17.971 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:17.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:17.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:18.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:18.114 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:18.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:18.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:18.212 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:17 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5141/222888 objects degraded (2.307%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:18.213 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:17 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5141/222888 objects degraded (2.307%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:18.216 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:18.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:17 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5141/222888 objects degraded (2.307%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:18.246 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:18.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:18.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:18.348 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:18.381 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:18.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:18.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:18.482 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:18.510 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:18.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:18.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:18.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:18.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:18.654 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:18.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:18.755 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:18.817 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:18.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:18.918 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:18.946 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:18.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:18.966 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:35:19.048 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:19.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:19.091 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:19.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:19.133 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:18 smithi029 ceph-mon[97787]: pgmap v1406: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 5141/217416 objects degraded (2.365%); 7260/217416 objects misplaced (3.339%); 4.0 MiB/s, 63 objects/s recovering 2024-04-12T11:35:19.133 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:18 smithi029 ceph-mon[93650]: pgmap v1406: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 5141/217416 objects degraded (2.365%); 7260/217416 objects misplaced (3.339%); 4.0 MiB/s, 63 objects/s recovering 2024-04-12T11:35:19.193 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:18 smithi087 ceph-mon[89014]: pgmap v1406: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 5141/217416 objects degraded (2.365%); 7260/217416 objects misplaced (3.339%); 4.0 MiB/s, 63 objects/s recovering 2024-04-12T11:35:19.222 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:19.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:19.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:19.324 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:19.354 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:19.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:19.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:19.456 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:19.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:19.502 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:19.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:19.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:19.640 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:19.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:19.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:19.742 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:19.785 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:19.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:19.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:19.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:19.926 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:19.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:19.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:19.949 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:35:20.028 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:20.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:20.075 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:20.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:20.106 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:19 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/377322941' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:35:20.107 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:19 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/377322941' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:35:20.177 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:20.206 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:20.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:20.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:19 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/377322941' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:35:20.307 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:20.360 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:20.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:20.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:20.462 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:20.493 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:20.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:20.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:20.594 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:20.623 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:20.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:20.725 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:20.757 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:20.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:20.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:20.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:20.891 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:20.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:20.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:20.993 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:21.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:21.031 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:21.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:21.131 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:21.175 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:21.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:21.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:20 smithi087 ceph-mon[89014]: pgmap v1407: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 4831/209718 objects degraded (2.304%); 6544/209718 objects misplaced (3.120%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:20 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:35:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:20 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/377322941' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:35:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:20 smithi087 ceph-mon[89014]: osdmap e1114: 8 total, 8 up, 8 in 2024-04-12T11:35:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:20 smithi029 ceph-mon[97787]: pgmap v1407: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 4831/209718 objects degraded (2.304%); 6544/209718 objects misplaced (3.120%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:20 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:35:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:20 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/377322941' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:35:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:20 smithi029 ceph-mon[97787]: osdmap e1114: 8 total, 8 up, 8 in 2024-04-12T11:35:21.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:20 smithi029 ceph-mon[93650]: pgmap v1407: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 4831/209718 objects degraded (2.304%); 6544/209718 objects misplaced (3.120%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:21.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:20 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:35:21.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:20 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/377322941' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:35:21.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:20 smithi029 ceph-mon[93650]: osdmap e1114: 8 total, 8 up, 8 in 2024-04-12T11:35:21.277 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:21.310 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:21.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:21.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:21.434 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:21.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:21.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:21.536 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:21.573 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:21.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:21.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:21.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:21.721 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:21.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:21.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:21.823 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:21.850 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:21.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:21.952 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:21.983 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:21.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:21.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:22.085 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:22.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:22.116 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:22.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:22.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:22.216 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:22.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:22.238 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:22.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:22.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:22.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:22.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:22.395 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:22.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:22.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:22.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:22.521 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:22.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:22.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:22.623 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:22.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:22.656 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:22.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:22.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:35:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:35:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:35:22.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:22.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:22.793 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:22.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:22.894 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:22.927 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:22.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:22.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:23.029 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:23.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:23.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:23.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:23.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:23.193 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:23.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:23.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:22 smithi087 ceph-mon[89014]: pgmap v1409: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 4831/208338 objects degraded (2.319%); 6544/208338 objects misplaced (3.141%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:22 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 4831/208338 objects degraded (2.319%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:23.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:22 smithi029 ceph-mon[93650]: pgmap v1409: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 4831/208338 objects degraded (2.319%); 6544/208338 objects misplaced (3.141%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:23.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:22 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 4831/208338 objects degraded (2.319%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:22 smithi029 ceph-mon[97787]: pgmap v1409: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 958 op/s; 4831/208338 objects degraded (2.319%); 6544/208338 objects misplaced (3.141%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:35:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:22 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 4831/208338 objects degraded (2.319%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:23.295 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:23.329 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:23.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:23.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:23.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:23.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:23.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:23.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:23.558 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:23.592 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:23.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:23.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:23.694 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:23.723 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:23.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:23.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:23.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:23.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:23.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:23.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:23.963 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:23.992 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:23.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:23.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:24.094 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:24.123 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:24.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:24.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:24.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:24.262 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:24.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:24.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:24.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:35:24.364 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:24.387 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:24.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:24.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:24.543 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:24.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:24.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:24.645 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:24.684 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:24.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:24.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:24.786 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:24.817 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:24.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:24.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:24.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:24.971 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:24.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:25.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:24 smithi029 ceph-mon[97787]: pgmap v1410: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 4573/199779 objects degraded (2.289%); 5851/199779 objects misplaced (2.929%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:35:25.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:24 smithi029 ceph-mon[93650]: pgmap v1410: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 4573/199779 objects degraded (2.289%); 5851/199779 objects misplaced (2.929%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:35:25.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:35:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:35:24] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2024-04-12T11:35:25.072 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:25.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:25.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:25.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:25.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:24 smithi087 ceph-mon[89014]: pgmap v1410: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 952 op/s; 4573/199779 objects degraded (2.289%); 5851/199779 objects misplaced (2.929%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:35:25.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:25.226 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:25.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:25.327 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:25.356 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:25.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:25.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:25.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:25.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:25.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:25.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:25.646 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:25.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:25.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:25.748 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:25.778 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:25.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:25.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:25.880 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:25.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:25.916 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:25.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:26.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:26.056 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:26.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:26.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:26.158 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:26.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:26.191 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:26.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:26.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:26.341 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:26.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:26.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:26.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:26.491 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:26.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:26.592 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:26.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:26.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:26.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:26.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:26.755 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:26.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:26.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:26.857 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:26.887 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:26.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:26.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:26.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:27.016 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:27.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:27.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:27.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:27.157 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:27.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:27.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:26 smithi087 ceph-mon[89014]: pgmap v1411: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 4573/194889 objects degraded (2.346%); 5851/194889 objects misplaced (3.002%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:35:27.259 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:27.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:26 smithi029 ceph-mon[93650]: pgmap v1411: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 4573/194889 objects degraded (2.346%); 5851/194889 objects misplaced (3.002%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:35:27.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:26 smithi029 ceph-mon[97787]: pgmap v1411: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 4573/194889 objects degraded (2.346%); 5851/194889 objects misplaced (3.002%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:35:27.282 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:27.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:27.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:27.384 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:27.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:27.415 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:27.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:27.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:27.561 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:27.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:27.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:27.662 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:27.692 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:27.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:27.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:27.794 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:27.836 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:27.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:27.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:27.938 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:27.967 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:27.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:27.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:28.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:28.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:28.121 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:28.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:28.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:27 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 4573/194889 objects degraded (2.346%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:28.223 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:28.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:28.254 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:28.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:28.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:27 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 4573/194889 objects degraded (2.346%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:28.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:27 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 4573/194889 objects degraded (2.346%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:28.355 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:28.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:28.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:28.496 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:28.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:28.542 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:28.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:28.643 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:28.666 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:28.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:28.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:28.768 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:28.791 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:28.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:28.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:28.893 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:28.927 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:28.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:28.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:29.029 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:29.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:29.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:29.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:29.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:29.190 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:29.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:29.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:28 smithi087 ceph-mon[89014]: pgmap v1412: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4573/190308 objects degraded (2.403%); 5851/190308 objects misplaced (3.074%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:35:29.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:28 smithi029 ceph-mon[97787]: pgmap v1412: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4573/190308 objects degraded (2.403%); 5851/190308 objects misplaced (3.074%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:35:29.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:28 smithi029 ceph-mon[93650]: pgmap v1412: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4573/190308 objects degraded (2.403%); 5851/190308 objects misplaced (3.074%); 4.1 MiB/s, 65 objects/s recovering 2024-04-12T11:35:29.291 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:29.316 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:29.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:29.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:29.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:29.441 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:29.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:29.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:29.543 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:29.585 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:29.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:29.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:29.687 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:29.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:29.724 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:29.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:29.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:29.856 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:29.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:29.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:29.958 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:29.981 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:29.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:29.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:30.083 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:30.114 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:30.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:30.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:30.256 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:30.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:30.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:30.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:30.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:30.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:30.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:30.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:30.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:30.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:30.629 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:30.659 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:30.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:30.760 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:30.798 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:30.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:30.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:30.899 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:30.930 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:30.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:31.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:31.075 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:31.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:31.176 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:31.199 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:31.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:31.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:30 smithi087 ceph-mon[89014]: pgmap v1413: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 4307/184149 objects degraded (2.339%); 5211/184149 objects misplaced (2.830%); 3.9 MiB/s, 61 objects/s recovering 2024-04-12T11:35:31.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:30 smithi029 ceph-mon[97787]: pgmap v1413: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 4307/184149 objects degraded (2.339%); 5211/184149 objects misplaced (2.830%); 3.9 MiB/s, 61 objects/s recovering 2024-04-12T11:35:31.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:30 smithi029 ceph-mon[93650]: pgmap v1413: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 4307/184149 objects degraded (2.339%); 5211/184149 objects misplaced (2.830%); 3.9 MiB/s, 61 objects/s recovering 2024-04-12T11:35:31.301 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:31.335 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:31.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:31.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:31.437 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:31.465 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:31.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:31.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:31.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:31.602 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:31.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:31.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:31.727 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:31.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:31.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:31.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:31.869 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:31.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:31.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:31.971 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:32.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:32.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:32.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:32.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:32.449 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:32.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:32.551 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:32.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:32.591 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:32.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:32.692 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:35:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:35:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:35:32.723 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:32.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:32.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:32.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:32.850 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:32.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:32.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:32.952 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:32.978 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:32.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:32.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:33.081 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:33.114 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:33.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:33.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:33.216 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:33.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:32 smithi087 ceph-mon[89014]: pgmap v1414: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 4307/183105 objects degraded (2.352%); 5211/183105 objects misplaced (2.846%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T11:35:33.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:32 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 4307/183105 objects degraded (2.352%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:33.245 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:33.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:33.263 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:32 smithi029 ceph-mon[97787]: pgmap v1414: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 4307/183105 objects degraded (2.352%); 5211/183105 objects misplaced (2.846%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T11:35:33.263 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 4307/183105 objects degraded (2.352%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:33.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:32 smithi029 ceph-mon[93650]: pgmap v1414: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 4307/183105 objects degraded (2.352%); 5211/183105 objects misplaced (2.846%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T11:35:33.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 4307/183105 objects degraded (2.352%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:33.347 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:33.377 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:33.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:33.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:33.479 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:33.512 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:33.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:33.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:33.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:33.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:33.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:33.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:33.750 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:33.793 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:33.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:33.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:33.895 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:33.929 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:33.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:33.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:34.031 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:34.059 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:34.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:34.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:34.161 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:34.203 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:34.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:34.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:34.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:34.328 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:34.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:34.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:34.467 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:34.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:34.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:34.569 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:34.617 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:34.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:34.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:34.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:34.761 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:34.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:34.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:34.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:34.902 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:34.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:34.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:35.004 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:35.033 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:35.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:35.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:35.135 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:35.162 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:35.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:35.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:35.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:35.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:34 smithi029 ceph-mon[97787]: pgmap v1415: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 4088/176676 objects degraded (2.314%); 4663/176676 objects misplaced (2.639%); 4.6 MiB/s, 72 objects/s recovering 2024-04-12T11:35:35.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:34 smithi029 ceph-mon[93650]: pgmap v1415: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 4088/176676 objects degraded (2.314%); 4663/176676 objects misplaced (2.639%); 4.6 MiB/s, 72 objects/s recovering 2024-04-12T11:35:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:35:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:35:34] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2024-04-12T11:35:35.297 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:35.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:35.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:35.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:35.422 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:35.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:35.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:35.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:34 smithi087 ceph-mon[89014]: pgmap v1415: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 4088/176676 objects degraded (2.314%); 4663/176676 objects misplaced (2.639%); 4.6 MiB/s, 72 objects/s recovering 2024-04-12T11:35:35.524 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:35.549 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:35.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:35.651 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:35.674 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:35.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:35.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:35.776 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:35.807 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:35.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:35.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:35.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:35.939 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:35.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:35.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:36.041 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:36.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:36.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:36.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:36.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:36.197 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:36.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:36.299 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:36.322 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:36.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:36.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:36.424 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:36.451 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:36.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:36.553 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:36.589 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:36.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:36.690 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:36.724 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:36.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:36.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:36.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:36.848 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:36.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:36.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:36.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:36.975 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:36.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:36.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:37.076 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:37.100 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:37.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:37.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:37.202 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:37.234 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:37.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:36 smithi029 ceph-mon[93650]: pgmap v1416: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 4088/172842 objects degraded (2.365%); 4663/172842 objects misplaced (2.698%); 2.9 MiB/s, 46 objects/s recovering 2024-04-12T11:35:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260'}"}]: dispatch 2024-04-12T11:35:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:36 smithi029 ceph-mon[97787]: pgmap v1416: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 4088/172842 objects degraded (2.365%); 4663/172842 objects misplaced (2.698%); 2.9 MiB/s, 46 objects/s recovering 2024-04-12T11:35:37.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:37.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260'}"}]: dispatch 2024-04-12T11:35:37.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:37.371 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:37.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:37.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:37.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:36 smithi087 ceph-mon[89014]: pgmap v1416: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 4088/172842 objects degraded (2.365%); 4663/172842 objects misplaced (2.698%); 2.9 MiB/s, 46 objects/s recovering 2024-04-12T11:35:37.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:37.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260'}"}]: dispatch 2024-04-12T11:35:37.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:37.510 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:37.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:37.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:37.612 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:37.635 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:37.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:37.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:37.736 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:37.760 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:37.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:37.862 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:37.890 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:37.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:37.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:37.992 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:38.034 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:38.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:38.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:38.136 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:38.167 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:38.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:38.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:38.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[97787]: from='client.44744 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:35:38.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260'}"}]': finished 2024-04-12T11:35:38.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[97787]: osdmap e1115: 8 total, 8 up, 8 in 2024-04-12T11:35:38.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 4088/172842 objects degraded (2.365%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:38.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[93650]: from='client.44744 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:35:38.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260'}"}]': finished 2024-04-12T11:35:38.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[93650]: osdmap e1115: 8 total, 8 up, 8 in 2024-04-12T11:35:38.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:37 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 4088/172842 objects degraded (2.365%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:38.269 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:38.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:38.301 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:38.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:38.402 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:38.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:38.432 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:38.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:38.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:37 smithi087 ceph-mon[89014]: from='client.44744 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:35:38.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'62260','3':'62260'}"}]': finished 2024-04-12T11:35:38.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:37 smithi087 ceph-mon[89014]: osdmap e1115: 8 total, 8 up, 8 in 2024-04-12T11:35:38.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:37 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 4088/172842 objects degraded (2.365%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:38.533 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:38.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:38.557 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:38.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:38.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:38.682 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:38.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:38.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:38.784 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:38.817 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:38.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:38.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:38.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:38.962 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:38.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:39.060 INFO:teuthology.orchestra.run.smithi029.stdout:moved 1 / 297 (0.3367%) 2024-04-12T11:35:39.060 INFO:teuthology.orchestra.run.smithi029.stdout:avg 37.125 2024-04-12T11:35:39.060 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 5.79736 -> 5.88297 (expected baseline 5.69951) 2024-04-12T11:35:39.061 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.5 with 29 -> 29 pgs (0.781145 -> 0.781145 * mean) 2024-04-12T11:35:39.061 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.3 with 47 -> 47 pgs (1.26599 -> 1.26599 * mean) 2024-04-12T11:35:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:35:39.061 INFO:teuthology.orchestra.run.smithi029.stdout:oload 110 2024-04-12T11:35:39.061 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 0.05 2024-04-12T11:35:39.061 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2024-04-12T11:35:39.061 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0176 2024-04-12T11:35:39.061 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0193 2024-04-12T11:35:39.061 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 weight 1.0000 -> 0.9500 2024-04-12T11:35:39.061 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 weight 1.0000 -> 0.9500 2024-04-12T11:35:39.064 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:39.094 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:39.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:39.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:39.196 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:39.223 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:39.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:39.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:38 smithi029 ceph-mon[93650]: pgmap v1418: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 4088/168945 objects degraded (2.420%); 4663/168945 objects misplaced (2.760%); 3.5 MiB/s, 55 objects/s recovering 2024-04-12T11:35:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:38 smithi029 ceph-mon[93650]: osdmap e1116: 8 total, 8 up, 8 in 2024-04-12T11:35:39.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:38 smithi029 ceph-mon[97787]: pgmap v1418: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 4088/168945 objects degraded (2.420%); 4663/168945 objects misplaced (2.760%); 3.5 MiB/s, 55 objects/s recovering 2024-04-12T11:35:39.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:38 smithi029 ceph-mon[97787]: osdmap e1116: 8 total, 8 up, 8 in 2024-04-12T11:35:39.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:39.365 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:39.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:39.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:38 smithi087 ceph-mon[89014]: pgmap v1418: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 4088/168945 objects degraded (2.420%); 4663/168945 objects misplaced (2.760%); 3.5 MiB/s, 55 objects/s recovering 2024-04-12T11:35:39.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:38 smithi087 ceph-mon[89014]: osdmap e1116: 8 total, 8 up, 8 in 2024-04-12T11:35:39.467 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:39.497 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:39.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:39.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:39.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:39.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:39.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:39.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:39.724 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:39.746 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:39.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:39.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:39.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:39.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:39.890 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:39.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:39.991 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:40.021 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:40.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:40.123 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:40.146 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:40.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:40.248 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:40.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:40.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:40.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:40.373 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:40.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:40.425 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:40.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:40.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:40.566 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:40.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:40.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:40.667 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:40.702 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:40.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:40.803 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:40.826 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:40.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:40.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:40.928 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:40.951 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:40.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:40.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:41.053 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:41.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:41.094 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:41.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:41.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:41.230 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:41.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:41.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:41.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:40 smithi029 ceph-mon[97787]: pgmap v1420: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 3854/163419 objects degraded (2.358%); 3992/163419 objects misplaced (2.443%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T11:35:41.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:40 smithi029 ceph-mon[93650]: pgmap v1420: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 3854/163419 objects degraded (2.358%); 3992/163419 objects misplaced (2.443%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T11:35:41.331 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:41.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:41.362 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:41.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:41.463 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:41.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:40 smithi087 ceph-mon[89014]: pgmap v1420: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 3854/163419 objects degraded (2.358%); 3992/163419 objects misplaced (2.443%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T11:35:41.492 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:41.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:41.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:41.594 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:41.621 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:41.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:41.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:41.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:41.746 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:41.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:41.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:41.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:41.888 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:41.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:41.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:41.990 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:42.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:42.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:42.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:42.115 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:42.150 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:42.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:42.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:42.252 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:42.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:35:42.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:35:42.293 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:42.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:42.314 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:35:42.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:42.450 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:42.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:42.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:42.608 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:42.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:42.710 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:42.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:35:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:35:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:35:42.742 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:42.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:42.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:42.843 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:42.871 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:42.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:42.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:42.973 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:43.008 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:43.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:43.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:43.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:43.141 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:43.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:43.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:42 smithi029 ceph-mon[93650]: pgmap v1421: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3854/162489 objects degraded (2.372%); 3992/162489 objects misplaced (2.457%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:35:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:42 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 3854/162489 objects degraded (2.372%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:43.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:42 smithi029 ceph-mon[97787]: pgmap v1421: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3854/162489 objects degraded (2.372%); 3992/162489 objects misplaced (2.457%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:35:43.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:42 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 3854/162489 objects degraded (2.372%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:43.281 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:43.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:43.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:43.287 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:42 smithi087 ceph-mon[89014]: pgmap v1421: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3854/162489 objects degraded (2.372%); 3992/162489 objects misplaced (2.457%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:35:43.287 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 3854/162489 objects degraded (2.372%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:43.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:43.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:43.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:43.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:43.523 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:43.564 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:43.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:43.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:43.666 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:43.695 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:43.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:43.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:43.834 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:43.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:43.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:43.936 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:43.988 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:43.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:43.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:44.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:44.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:44.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:44.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:44.233 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:44.264 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:44.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:44.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:44.366 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:44.410 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:44.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:44.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:44.513 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:44.545 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:44.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:44.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:44.647 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:44.676 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:44.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:44.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:44.778 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:44.809 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:44.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:44.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:44.911 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:44.936 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:44.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:45.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:45.063 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:45.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:45.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:45.164 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:45.197 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:45.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:45.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:45.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:45 smithi029 ceph-mon[97787]: pgmap v1422: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 3643/156033 objects degraded (2.335%); 3241/156033 objects misplaced (2.077%); 4.9 MiB/s, 77 objects/s recovering 2024-04-12T11:35:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:45 smithi029 ceph-mon[93650]: pgmap v1422: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 3643/156033 objects degraded (2.335%); 3241/156033 objects misplaced (2.077%); 4.9 MiB/s, 77 objects/s recovering 2024-04-12T11:35:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:35:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:35:44] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2024-04-12T11:35:45.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:45.299 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:45 smithi087 ceph-mon[89014]: pgmap v1422: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 3643/156033 objects degraded (2.335%); 3241/156033 objects misplaced (2.077%); 4.9 MiB/s, 77 objects/s recovering 2024-04-12T11:35:45.335 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:45.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:45.437 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:45.468 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:45.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:45.570 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:45.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:45.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:45.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:45.701 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:45.727 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:45.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:45.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:45.741 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-04-12T11:35:45.741 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-04-12T11:35:45.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:45.853 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:45.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:45.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:45.955 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:45.982 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:45.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:45.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:46.084 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:46.118 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:46.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:46.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:46.220 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:46.249 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:46.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:46.267 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:46 smithi087 ceph-mon[89014]: osdmap e1117: 8 total, 8 up, 8 in 2024-04-12T11:35:46.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:46 smithi029 ceph-mon[93650]: osdmap e1117: 8 total, 8 up, 8 in 2024-04-12T11:35:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:46 smithi029 ceph-mon[97787]: osdmap e1117: 8 total, 8 up, 8 in 2024-04-12T11:35:46.351 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:46.375 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:46.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:46.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:46.477 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:46.513 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:46.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:46.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:46.542 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:35:46.542 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:35:46.615 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:46.646 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:46.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:46.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:46.748 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:46.767 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:35:46.777 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:46.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:46.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:46.878 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:46.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:46.920 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:46.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:47.021 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:47.053 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:47.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:47.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:47.078 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:47 smithi087 ceph-mon[89014]: pgmap v1424: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 3643/151701 objects degraded (2.401%); 3241/151701 objects misplaced (2.136%); 4.9 MiB/s, 77 objects/s recovering 2024-04-12T11:35:47.078 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:47 smithi087 ceph-mon[89014]: osdmap e1118: 8 total, 8 up, 8 in 2024-04-12T11:35:47.155 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:47.186 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:47.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:47.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:47.235 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:47 smithi029 ceph-mon[97787]: pgmap v1424: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 3643/151701 objects degraded (2.401%); 3241/151701 objects misplaced (2.136%); 4.9 MiB/s, 77 objects/s recovering 2024-04-12T11:35:47.235 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:47 smithi029 ceph-mon[97787]: osdmap e1118: 8 total, 8 up, 8 in 2024-04-12T11:35:47.236 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:47 smithi029 ceph-mon[93650]: pgmap v1424: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 3643/151701 objects degraded (2.401%); 3241/151701 objects misplaced (2.136%); 4.9 MiB/s, 77 objects/s recovering 2024-04-12T11:35:47.236 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:47 smithi029 ceph-mon[93650]: osdmap e1118: 8 total, 8 up, 8 in 2024-04-12T11:35:47.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-04-12T11:35:47.287 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:47.371 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:47.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:47.472 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:47.506 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:47.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:47.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:47.544 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:35:47.608 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:47.644 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:47.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:47.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:47.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:47.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:47.804 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:47.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:47.905 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:47.948 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:47.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:48.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:48.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:48.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:48.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:48.154 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 3643/151701 objects degraded (2.401%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:48.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 3643/151701 objects degraded (2.401%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:48.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:48.238 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:48.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:48.340 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:48.377 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:48.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:48.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 3643/151701 objects degraded (2.401%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:48.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:48.513 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:48.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:48.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:48.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:48.656 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:48.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:48.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:48.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:48.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:48.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:48.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:48.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:48.944 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:48.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:49.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:49.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:49.103 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:49.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:49.203 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:49.239 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:49.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:49.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:49.258 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:49 smithi029 ceph-mon[93650]: pgmap v1426: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 3643/147306 objects degraded (2.473%); 3241/147306 objects misplaced (2.200%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T11:35:49.258 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:49 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:35:49.258 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:49 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:35:49.259 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:49 smithi029 ceph-mon[97787]: pgmap v1426: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 3643/147306 objects degraded (2.473%); 3241/147306 objects misplaced (2.200%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T11:35:49.259 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:49 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:35:49.259 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:49 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:35:49.341 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:49.368 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:49.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:49.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:49.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:49 smithi087 ceph-mon[89014]: pgmap v1426: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 3643/147306 objects degraded (2.473%); 3241/147306 objects misplaced (2.200%); 2.5 MiB/s, 40 objects/s recovering 2024-04-12T11:35:49.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:49 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:35:49.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:49 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:35:49.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:49.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:49.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:49.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:49.640 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:49.670 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:49.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:49.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:49.771 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:49.800 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:49.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:49.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:49.886 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:35:49.902 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:49.931 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:49.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:49.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:50.033 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:50.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:50.092 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:50.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:50.193 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:50.237 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:50.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:50.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:50.339 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:50.369 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:50.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:50.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:50.470 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:50.504 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:50.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:50.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:50.606 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:50.637 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:50.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:50.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:50.738 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:50.762 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:50.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:50.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:50.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:50.905 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:50.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:50.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:51.006 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:51.038 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:51.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:51.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:51.140 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:51.175 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:51.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:51.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:51.277 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:51.301 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:51.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:51.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:51.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:51.435 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:51.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:51.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:51 smithi087 ceph-mon[89014]: pgmap v1427: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 1797/141078 objects degraded (1.274%); 3606/141078 objects misplaced (2.556%); 6.9 MiB/s, 109 objects/s recovering 2024-04-12T11:35:51.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:51 smithi029 ceph-mon[97787]: pgmap v1427: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 1797/141078 objects degraded (1.274%); 3606/141078 objects misplaced (2.556%); 6.9 MiB/s, 109 objects/s recovering 2024-04-12T11:35:51.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:51 smithi029 ceph-mon[93650]: pgmap v1427: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 1797/141078 objects degraded (1.274%); 3606/141078 objects misplaced (2.556%); 6.9 MiB/s, 109 objects/s recovering 2024-04-12T11:35:51.537 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:51.565 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:51.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:51.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:51.667 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:51.694 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:51.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:51.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:51.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:51.820 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:51.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:51.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:51.921 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:51.945 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:51.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:52.047 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:52.079 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:52.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:52.181 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:52.221 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:52.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:52.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:52.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:52.316 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:52 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2067492603' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:35:52.316 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:52 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:35:52.322 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:52.346 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:52.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:52.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:52.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:52.483 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:52.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:52.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:52 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2067492603' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:35:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:52 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:35:52.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:35:52.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:52 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2067492603' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:35:52.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:52 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:35:52.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:52.624 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:52.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:52.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:52.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:35:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:35:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:35:52.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:52.768 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:52.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:52.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:52.870 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:52.893 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:52.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:52.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:52.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:53.034 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:53.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:53.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:53.136 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:53.174 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:53.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:53.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:53.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:53.315 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:53.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:53.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:53.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:53.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:53.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:53.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:53 smithi087 ceph-mon[89014]: pgmap v1428: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1797/139947 objects degraded (1.284%); 3606/139947 objects misplaced (2.577%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T11:35:53.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:53 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:35:53.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:53 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:35:53.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:53 smithi087 ceph-mon[89014]: osdmap e1119: 8 total, 8 up, 8 in 2024-04-12T11:35:53.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 1797/139947 objects degraded (1.284%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:53 smithi029 ceph-mon[97787]: pgmap v1428: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1797/139947 objects degraded (1.284%); 3606/139947 objects misplaced (2.577%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T11:35:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:53 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:35:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:53 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:35:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:53 smithi029 ceph-mon[97787]: osdmap e1119: 8 total, 8 up, 8 in 2024-04-12T11:35:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 1797/139947 objects degraded (1.284%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:53.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:53 smithi029 ceph-mon[93650]: pgmap v1428: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 1797/139947 objects degraded (1.284%); 3606/139947 objects misplaced (2.577%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T11:35:53.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:53 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:35:53.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:53 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:35:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:53 smithi029 ceph-mon[93650]: osdmap e1119: 8 total, 8 up, 8 in 2024-04-12T11:35:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 1797/139947 objects degraded (1.284%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:53.558 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:53.596 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:53.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:53.697 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:53.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:53.736 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:53.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:53.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:53.867 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:53.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:53.969 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:54.007 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:54.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:54.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:54.109 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:54.121 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:35:54.143 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:54.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:54.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:54.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:54.267 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:54.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:54.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:54.369 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:54.392 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:54.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:54.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:54.494 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:54.530 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:54.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:54.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:54.631 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:54.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:54.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:54.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:54.767 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:54.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:54.806 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:54.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:54.907 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:54.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:54.943 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:54.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:55.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:55.079 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:55.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:55.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:55.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:55.231 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:55.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:55.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:55 smithi029 ceph-mon[97787]: pgmap v1430: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 1797/133179 objects degraded (1.349%); 2871/133179 objects misplaced (2.156%); 6.3 MiB/s, 100 objects/s recovering 2024-04-12T11:35:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:55 smithi029 ceph-mon[93650]: pgmap v1430: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 1797/133179 objects degraded (1.349%); 2871/133179 objects misplaced (2.156%); 6.3 MiB/s, 100 objects/s recovering 2024-04-12T11:35:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:35:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:35:54] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2024-04-12T11:35:55.333 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:55.366 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:55.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:55.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:55.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:55 smithi087 ceph-mon[89014]: pgmap v1430: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 1797/133179 objects degraded (1.349%); 2871/133179 objects misplaced (2.156%); 6.3 MiB/s, 100 objects/s recovering 2024-04-12T11:35:55.468 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:55.494 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:55.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:55.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:55.596 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:55.619 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:55.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:55.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:55.721 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:55.751 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:55.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:55.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:55.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:55.879 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:55.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:55.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:55.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:56.004 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:56.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:56.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:56.105 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:56.129 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:56.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:56.231 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:56.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:56.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:56.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:56.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:56.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:56.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:56.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:56.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:56 smithi087 ceph-mon[89014]: osdmap e1120: 8 total, 8 up, 8 in 2024-04-12T11:35:56.493 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:56.517 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:56 smithi029 ceph-mon[93650]: osdmap e1120: 8 total, 8 up, 8 in 2024-04-12T11:35:56.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:56 smithi029 ceph-mon[97787]: osdmap e1120: 8 total, 8 up, 8 in 2024-04-12T11:35:56.526 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:56.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:56.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:56.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:56.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:56.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:56.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:56.790 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:56.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:56.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:56.892 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:56.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:56.928 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:56.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:57.029 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:57.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:57.079 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:57.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:57.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:35:57.203 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:57.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:57.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:35:57.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:57.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:57.336 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:57.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:57.437 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:57.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:57 smithi087 ceph-mon[89014]: pgmap v1432: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 1797/129072 objects degraded (1.392%); 2641/129072 objects misplaced (2.046%); 7.2 MiB/s, 114 objects/s recovering 2024-04-12T11:35:57.470 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:57 smithi087 ceph-mon[89014]: osdmap e1121: 8 total, 8 up, 8 in 2024-04-12T11:35:57.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:57.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:57.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:57.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:57 smithi029 ceph-mon[93650]: pgmap v1432: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 1797/129072 objects degraded (1.392%); 2641/129072 objects misplaced (2.046%); 7.2 MiB/s, 114 objects/s recovering 2024-04-12T11:35:57.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:57 smithi029 ceph-mon[93650]: osdmap e1121: 8 total, 8 up, 8 in 2024-04-12T11:35:57.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:57 smithi029 ceph-mon[97787]: pgmap v1432: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 1797/129072 objects degraded (1.392%); 2641/129072 objects misplaced (2.046%); 7.2 MiB/s, 114 objects/s recovering 2024-04-12T11:35:57.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:57 smithi029 ceph-mon[97787]: osdmap e1121: 8 total, 8 up, 8 in 2024-04-12T11:35:57.572 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:57.610 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:57.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:57.711 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:57.751 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:57.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:57.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:35:57.853 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:57.888 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:57.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:57.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:57.990 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:58.022 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:58.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:58.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:58.124 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:58.160 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:58.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:58.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:58.262 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:58.293 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:58.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:58.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:35:58.434 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:58.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:58.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:35:58.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 1797/129072 objects degraded (1.392%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 1797/129072 objects degraded (1.392%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:58.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 1797/129072 objects degraded (1.392%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:35:58.536 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:58.568 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:58.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:58.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:35:58.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:58.709 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:58.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:35:58.810 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:58.834 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:58.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:58.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:58.935 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:35:58.967 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:58.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:35:59.068 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:59.100 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:59.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:59.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:59.202 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:35:59.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:59.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:59.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:35:59.329 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:59.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:59.367 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:59.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:59.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:35:59 smithi087 ceph-mon[89014]: pgmap v1434: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 1797/125199 objects degraded (1.435%); 2641/125199 objects misplaced (2.109%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:35:59.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:59.504 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:59.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:59.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:59.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:35:59 smithi029 ceph-mon[97787]: pgmap v1434: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 1797/125199 objects degraded (1.435%); 2641/125199 objects misplaced (2.109%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:35:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:35:59 smithi029 ceph-mon[93650]: pgmap v1434: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 1797/125199 objects degraded (1.435%); 2641/125199 objects misplaced (2.109%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:35:59.606 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:35:59.635 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:35:59.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:59.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:35:59.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:35:59.781 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:59.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:59.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:35:59.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:35:59.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:35:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:35:59.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:00.035 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:00.069 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:00.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:00.170 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:00.201 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:00.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:00.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:00.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:00 smithi029 ceph-mon[97787]: pgmap v1435: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 1797/118965 objects degraded (1.511%); 2018/118965 objects misplaced (1.696%); 5.1 MiB/s, 81 objects/s recovering 2024-04-12T11:36:00.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:00 smithi029 ceph-mon[93650]: pgmap v1435: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 1797/118965 objects degraded (1.511%); 2018/118965 objects misplaced (1.696%); 5.1 MiB/s, 81 objects/s recovering 2024-04-12T11:36:00.302 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:00.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:00.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:00.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:00.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:00.458 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:00.462 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:00.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:00.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:00 smithi087 ceph-mon[89014]: pgmap v1435: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 1797/118965 objects degraded (1.511%); 2018/118965 objects misplaced (1.696%); 5.1 MiB/s, 81 objects/s recovering 2024-04-12T11:36:00.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:00.619 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:00.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:00.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:00.721 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:00.753 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:00.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:00.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:00.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:00.900 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:00.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:00.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:01.002 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:01.033 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:01.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:01.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:01.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:01.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:01.195 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:01.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:01.296 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:01.323 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:01.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:01.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:01.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:01.466 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:01.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:01.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:01.568 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:01.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:01.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:01.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:01.701 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:01.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:01.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:01.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:01.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:01.870 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:01.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:01.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:01.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:02.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:02.023 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:02.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:02.081 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:36:02.124 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:02.129 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:36:02.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:02.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:02.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:02.181 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:36:02.233 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:36:02.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:02.275 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:36:02.297 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:02.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:02.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:02.330 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:36:02.373 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:36:02.387 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:36:02.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:02.426 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:36:02.441 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:36:02.441 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:36:02.441 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:36:02.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:36:02.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:36:02.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:36:02.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:36:02.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:36:02.472 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:02.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:02.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:02.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:02.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:02.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:02.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:36:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:36:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:36:02.732 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:moved 3 / 297 (1.0101%) 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:avg 37.125 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 5.88297 -> 5.92532 (expected baseline 5.69951) 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.5 with 29 -> 29 pgs (0.781145 -> 0.781145 * mean) 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.3 with 47 -> 47 pgs (1.26599 -> 1.26599 * mean) 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:oload 110 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 0.05 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0132 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0145 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 weight 0.9500 -> 0.9000 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 weight 0.9500 -> 0.9000 2024-04-12T11:36:02.779 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 weight 1.0000 -> 0.9500 2024-04-12T11:36:02.783 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:02.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:02.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:02.885 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:02 smithi029 ceph-mon[97787]: pgmap v1436: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 1797/117765 objects degraded (1.526%); 2018/117765 objects misplaced (1.714%); 3.0 MiB/s, 47 objects/s recovering 2024-04-12T11:36:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:02 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/349693981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:36:03.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:02 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'62260'}"}]: dispatch 2024-04-12T11:36:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:02 smithi029 ceph-mon[93650]: pgmap v1436: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 1797/117765 objects degraded (1.526%); 2018/117765 objects misplaced (1.714%); 3.0 MiB/s, 47 objects/s recovering 2024-04-12T11:36:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:02 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/349693981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:36:03.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:02 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'62260'}"}]: dispatch 2024-04-12T11:36:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:03.029 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:03.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:03.130 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:03.177 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:03.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:03.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:02 smithi087 ceph-mon[89014]: pgmap v1436: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 1797/117765 objects degraded (1.526%); 2018/117765 objects misplaced (1.714%); 3.0 MiB/s, 47 objects/s recovering 2024-04-12T11:36:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:02 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/349693981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:36:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:02 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'62260'}"}]: dispatch 2024-04-12T11:36:03.279 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:03.314 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:03.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:03.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:03.416 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:03.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:03.454 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:03.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:03.555 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:03.598 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:03.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:03.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:03.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:03.724 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:03.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:03.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:03.826 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:03.849 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:03.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:03.951 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:03.985 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:03.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:03.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:04.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:04.102 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:03 smithi029 ceph-mon[97787]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:36:04.102 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'62260'}"}]': finished 2024-04-12T11:36:04.102 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:03 smithi029 ceph-mon[97787]: osdmap e1122: 8 total, 8 up, 8 in 2024-04-12T11:36:04.102 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 1797/117765 objects degraded (1.526%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:04.103 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:03 smithi029 ceph-mon[93650]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:36:04.103 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'62260'}"}]': finished 2024-04-12T11:36:04.103 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:03 smithi029 ceph-mon[93650]: osdmap e1122: 8 total, 8 up, 8 in 2024-04-12T11:36:04.103 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 1797/117765 objects degraded (1.526%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:04.129 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:04.130 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:03 smithi087 ceph-mon[89014]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:36:04.130 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'58984','3':'58984','7':'62260'}"}]': finished 2024-04-12T11:36:04.130 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:03 smithi087 ceph-mon[89014]: osdmap e1122: 8 total, 8 up, 8 in 2024-04-12T11:36:04.130 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 1797/117765 objects degraded (1.526%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:04.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:04.231 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:04.262 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:04.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:04.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:04.364 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:04.403 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:04.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:04.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:04.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:04.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:04.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:04.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:04.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:04.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:04.686 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:04.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:04.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:04.826 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:04.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:04.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:04.928 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:04.962 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:04.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:04.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:05.064 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:05.096 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:05.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:05.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:05.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:05.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:04 smithi087 ceph-mon[89014]: pgmap v1438: 99 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 1797/109545 objects degraded (1.640%); 2018/109545 objects misplaced (1.842%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:36:05.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:04 smithi087 ceph-mon[89014]: osdmap e1123: 8 total, 8 up, 8 in 2024-04-12T11:36:05.240 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:05.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:05.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:04 smithi029 ceph-mon[97787]: pgmap v1438: 99 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 1797/109545 objects degraded (1.640%); 2018/109545 objects misplaced (1.842%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:36:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:04 smithi029 ceph-mon[97787]: osdmap e1123: 8 total, 8 up, 8 in 2024-04-12T11:36:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:04 smithi029 ceph-mon[93650]: pgmap v1438: 99 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 1797/109545 objects degraded (1.640%); 2018/109545 objects misplaced (1.842%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:36:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:04 smithi029 ceph-mon[93650]: osdmap e1123: 8 total, 8 up, 8 in 2024-04-12T11:36:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:36:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:36:04] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2024-04-12T11:36:05.342 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:05.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:05.373 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:05.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:05.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:05.503 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:05.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:05.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:05.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:05.628 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:05.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:05.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:05.730 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:05.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:05.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:05.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:05.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:05.901 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:05.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:06.003 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:06.027 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:06.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:06.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:06.166 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:06.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:06.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:06.268 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:06.311 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:06.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:06.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:06.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:06.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:06.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:06.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:06.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:06.566 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:06.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:06.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:06.667 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:06.699 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:06.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:06.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:06.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:06.838 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:06.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:06.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:06.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:06.979 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:06.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:07.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:07.098 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:06 smithi029 ceph-mon[93650]: pgmap v1440: 99 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 1752/103791 objects degraded (1.688%); 1752/103791 objects misplaced (1.688%); 2.9 MiB/s, 52 objects/s recovering 2024-04-12T11:36:07.099 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:07.099 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:07.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:06 smithi029 ceph-mon[97787]: pgmap v1440: 99 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 1752/103791 objects degraded (1.688%); 1752/103791 objects misplaced (1.688%); 2.9 MiB/s, 52 objects/s recovering 2024-04-12T11:36:07.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:07.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:07.149 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:07.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:07.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:06 smithi087 ceph-mon[89014]: pgmap v1440: 99 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 1752/103791 objects degraded (1.688%); 1752/103791 objects misplaced (1.688%); 2.9 MiB/s, 52 objects/s recovering 2024-04-12T11:36:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:07.250 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:07.280 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:07.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:07.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:07.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:07.423 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:07.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:07.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:07.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:07.549 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:07.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:07.651 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:07.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:07.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:07.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:07.780 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:07.817 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:07.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:07.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:07.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:07.959 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:07.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:07.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:08.060 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:08.091 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:08.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:08.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:08.193 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:08.216 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:08.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:08.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:08.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:08.358 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:08.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:08.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:08.460 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:08.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:08.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:08.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:08.592 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:08.624 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:08.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:08.726 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:08.757 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:08.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:08.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:08.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:08.897 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:08.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:08.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:08.999 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:09.032 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:09.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:09.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:09.134 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:09.168 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:09.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:09.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:09.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:08 smithi087 ceph-mon[89014]: pgmap v1441: 99 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 1752/99111 objects degraded (1.768%); 1752/99111 objects misplaced (1.768%); 888 KiB/s, 19 objects/s recovering 2024-04-12T11:36:09.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 1752/99111 objects degraded (1.768%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:08 smithi029 ceph-mon[97787]: pgmap v1441: 99 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 1752/99111 objects degraded (1.768%); 1752/99111 objects misplaced (1.768%); 888 KiB/s, 19 objects/s recovering 2024-04-12T11:36:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 1752/99111 objects degraded (1.768%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:09.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:08 smithi029 ceph-mon[93650]: pgmap v1441: 99 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 1752/99111 objects degraded (1.768%); 1752/99111 objects misplaced (1.768%); 888 KiB/s, 19 objects/s recovering 2024-04-12T11:36:09.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 1752/99111 objects degraded (1.768%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:09.271 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:09.315 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:09.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:09.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:09.417 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:09.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:09.441 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:09.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:09.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:09.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:09.581 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:09.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:09.682 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:09.709 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:09.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:09.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:09.811 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:09.846 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:09.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:09.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:09.947 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:09.971 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:09.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:10.073 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:10.095 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:10.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:10.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:10.198 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:10.226 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:10.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:10.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:10.328 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:10.362 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:10.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:10.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:10.464 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:10.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:10.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:10.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:10.592 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:10.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:10.625 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:10.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:10.726 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:10.757 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:10.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:10.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:10.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:10.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:10.984 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:11.017 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:11.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:11.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:11.118 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:11.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:11.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:11.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:11.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:10 smithi087 ceph-mon[89014]: pgmap v1442: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 1612/92589 objects degraded (1.741%); 1612/92589 objects misplaced (1.741%); 888 KiB/s, 37 objects/s recovering 2024-04-12T11:36:11.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:11.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:10 smithi029 ceph-mon[97787]: pgmap v1442: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 1612/92589 objects degraded (1.741%); 1612/92589 objects misplaced (1.741%); 888 KiB/s, 37 objects/s recovering 2024-04-12T11:36:11.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:10 smithi029 ceph-mon[93650]: pgmap v1442: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 1612/92589 objects degraded (1.741%); 1612/92589 objects misplaced (1.741%); 888 KiB/s, 37 objects/s recovering 2024-04-12T11:36:11.295 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:11.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:11.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:11.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:11.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:11.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:11.521 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:11.552 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:11.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:11.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:11.654 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:11.684 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:11.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:11.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:11.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:11.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:11.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:11.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:11.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:11.975 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:11.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:11.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:12.076 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:12.126 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:12.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:12.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:12.228 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:12.258 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:12.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:12.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:12.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:12.417 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:12.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:12.519 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:12.565 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:12.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:12.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:12.667 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:12.711 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:12.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:12.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:36:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:36:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:36:12.813 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:12.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:12.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:12.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:12.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:12.985 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:12.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:12.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:12 smithi029 ceph-mon[93650]: pgmap v1443: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 1612/91425 objects degraded (1.763%); 1612/91425 objects misplaced (1.763%); 796 KiB/s, 33 objects/s recovering 2024-04-12T11:36:13.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:12 smithi029 ceph-mon[97787]: pgmap v1443: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 1612/91425 objects degraded (1.763%); 1612/91425 objects misplaced (1.763%); 796 KiB/s, 33 objects/s recovering 2024-04-12T11:36:13.088 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:13.117 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:13.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:13.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:12 smithi087 ceph-mon[89014]: pgmap v1443: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 1612/91425 objects degraded (1.763%); 1612/91425 objects misplaced (1.763%); 796 KiB/s, 33 objects/s recovering 2024-04-12T11:36:13.219 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:13.228 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-04-12T11:36:13.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight-by-utilization 1000 3.0 2024-04-12T11:36:13.247 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:13.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:13.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:13.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:13.399 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:13.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:13.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:13.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:13.537 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:13.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:13.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:13.638 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:13.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:13.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:13.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:13.768 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:13.778 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:13.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:13.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:13.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:13.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 1612/91425 objects degraded (1.763%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:13.934 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 1612/91425 objects degraded (1.763%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:13.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:14.008 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:14.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:14.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:14.122 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:14.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:14.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:14.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:14.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 1612/91425 objects degraded (1.763%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:14.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:14.267 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:14.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:14.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:14.419 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:14.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:14.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:14.520 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:14.554 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:14.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:14.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:14.656 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:14.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:36:14.690 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:14.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:14.791 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:14.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:14.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:14.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:14.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:14.961 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:14.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:14.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:14 smithi029 ceph-mon[93650]: pgmap v1444: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 1612/84123 objects degraded (1.916%); 1612/84123 objects misplaced (1.916%); 710 KiB/s, 29 objects/s recovering 2024-04-12T11:36:15.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:36:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:36:14] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.43.0" 2024-04-12T11:36:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:14 smithi029 ceph-mon[97787]: pgmap v1444: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 1612/84123 objects degraded (1.916%); 1612/84123 objects misplaced (1.916%); 710 KiB/s, 29 objects/s recovering 2024-04-12T11:36:15.063 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:15.115 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:15.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:15.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:14 smithi087 ceph-mon[89014]: pgmap v1444: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 1612/84123 objects degraded (1.916%); 1612/84123 objects misplaced (1.916%); 710 KiB/s, 29 objects/s recovering 2024-04-12T11:36:15.217 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:15.245 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:15.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:15.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:15.347 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:15.384 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:15.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:15.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:15.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:15.530 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:15.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:15.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:15.631 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:15.671 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:15.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:15.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:15.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:15.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:15.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:15.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:15.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:15.940 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:15.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:15.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:16.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:16.083 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:16.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:16.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:16.185 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:16.222 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:16.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:16.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:16.324 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:16.356 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:16.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:16.444 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:36:16.444 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:189753 6.b 6:dc6e111b:::benchmark_data_smithi029_138569_object82928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1123)", 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:36:15.846012+0000", 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.052261770999999999, 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.052298070000000002, 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 189753 2024-04-12T11:36:16.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.846012+0000", 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.846012+0000", 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.846013+0000", 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2279999999999999e-06 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.846017+0000", 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.228e-06 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.846026+0000", 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1740000000000006e-06 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.846033+0000", 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1350000000000003e-06 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.853500+0000", 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074664529999999996 2024-04-12T11:36:16.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.853628+0000", 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000128641 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.853749+0000", 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000120418 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857278+0000", 2024-04-12T11:36:16.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035291110000000001 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.860933+0000", 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036546830000000002 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:189762 6.11 6:8981ca81:::benchmark_data_smithi029_138569_object82937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1123)", 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:36:15.856974+0000", 2024-04-12T11:36:16.450 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.041299791000000002, 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.041398063999999998, 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 189762 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.856974+0000", 2024-04-12T11:36:16.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.856974+0000", 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.856975+0000", 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6309999999999999e-06 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857042+0000", 2024-04-12T11:36:16.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7119000000000006e-05 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857056+0000", 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3614e-05 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857064+0000", 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4980000000000006e-06 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:36:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857121+0000", 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6378000000000001e-05 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857260+0000", 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013959500000000001 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.863065+0000", 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058047139999999999 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.872480+0000", 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.009415428 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873836+0000", 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013556670000000001 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.455 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:189763 6.b 6:d1e106b8:::benchmark_data_smithi029_138569_object82938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1123)", 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:36:15.857088+0000", 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.041185479999999997, 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.041335589999999998, 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 189763 2024-04-12T11:36:16.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857088+0000", 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857088+0000", 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857088+0000", 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6700000000000003e-07 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857092+0000", 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0369999999999999e-06 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:36:16.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857099+0000", 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4379999999999998e-06 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857105+0000", 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5209999999999999e-06 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857302+0000", 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019689600000000001 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857400+0000", 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7598999999999998e-05 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.857493+0000", 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2983000000000001e-05 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.869608+0000", 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012114908000000001 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.869882+0000", 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00027448799999999998 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:189767 6.b 6:da35e128:::benchmark_data_smithi029_138569_object82942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1123)", 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:36:15.858391+0000", 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.039882769999999998, 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.040084015000000001, 2024-04-12T11:36:16.461 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 189767 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.858391+0000", 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.858391+0000", 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.858391+0000", 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6899999999999997e-07 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.858395+0000", 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8210000000000003e-06 2024-04-12T11:36:16.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.858402+0000", 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1400000000000002e-06 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.858408+0000", 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2179999999999996e-06 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.860964+0000", 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025557179999999998 2024-04-12T11:36:16.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.861081+0000", 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011730000000000001 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.861212+0000", 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013087799999999999 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:16.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.869646+0000", 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084339359999999995 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.869890+0000", 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000243699 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:189771 6.c 6:3a41f196:::benchmark_data_smithi029_138569_object82946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1123)", 2024-04-12T11:36:16.466 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:36:15.862817+0000", 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.035455898, 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.035706847999999999, 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 189771 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:36:16.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.862817+0000", 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.862817+0000", 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.862819+0000", 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4670000000000001e-06 2024-04-12T11:36:16.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.862824+0000", 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2209999999999996e-06 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.862834+0000", 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0368e-05 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:36:16.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.862842+0000", 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3490000000000003e-06 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.862888+0000", 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6560999999999997e-05 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.863000+0000", 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011137999999999999 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.863112+0000", 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000112173 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.866619+0000", 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035066350000000001 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873669+0000", 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0070499999999999998 2024-04-12T11:36:16.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873757+0000", 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8279e-05 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.874002+0000", 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00024560800000000002 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.472 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.34776.0:189778 6.7 e1123/1093 6:e7051ee1:::benchmark_data_smithi029_138569_object82953:head v 1123'8002, mlcod=1123'8000)", 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:36:15.873648+0000", 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.024625613000000001, 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024933278999999999, 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873648+0000", 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873648+0000", 2024-04-12T11:36:16.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873648+0000", 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5999999999999999e-07 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873653+0000", 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3279999999999996e-06 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873660+0000", 2024-04-12T11:36:16.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2469999999999997e-06 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873665+0000", 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1109999999999999e-06 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873764+0000", 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9300000000000001e-05 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:36:16.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873785+0000", 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0744e-05 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.34776.0:189771 6.c e1123/1086 ondisk, result = 0)", 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:36:15.873679+0000", 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.024594794999999999, 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024937530999999999, 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:36:16.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873679+0000", 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873679+0000", 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873679+0000", 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5800000000000001e-07 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873681+0000", 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0339999999999999e-06 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873686+0000", 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5990000000000001e-06 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873691+0000", 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7879999999999997e-06 2024-04-12T11:36:16.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873746+0000", 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4527000000000002e-05 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:36:15.873752+0000", 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2430000000000002e-06 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:36:16.479 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2024-04-12T11:36:16.480 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 7 2024-04-12T11:36:16.480 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:36:16.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:16.518 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:16.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:16.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:16.620 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:16.659 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:16.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:16.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:16.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:16.801 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:16.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:16.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:16.903 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:16.934 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:16.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:16.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:17.036 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:17.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:17.068 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:17.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:17.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:16 smithi029 ceph-mon[93650]: pgmap v1445: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 1452/78315 objects degraded (1.854%); 1452/78315 objects misplaced (1.854%); 596 KiB/s, 38 objects/s recovering 2024-04-12T11:36:17.077 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:16 smithi029 ceph-mon[97787]: pgmap v1445: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 1452/78315 objects degraded (1.854%); 1452/78315 objects misplaced (1.854%); 596 KiB/s, 38 objects/s recovering 2024-04-12T11:36:17.169 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:17.199 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:17.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:17.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:17.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:16 smithi087 ceph-mon[89014]: pgmap v1445: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 1452/78315 objects degraded (1.854%); 1452/78315 objects misplaced (1.854%); 596 KiB/s, 38 objects/s recovering 2024-04-12T11:36:17.302 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:17.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:17.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:17.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:17.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:17.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:17.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:17.577 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:17.606 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:17.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:17.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:17.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:17.742 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:17.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:17.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:17.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:17.887 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:17.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:17.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:18.028 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:18.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:18.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:18.130 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:18.196 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:18.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:18.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:18.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:17 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 1452/78315 objects degraded (1.854%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:18.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:17 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 1452/78315 objects degraded (1.854%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:18.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:17 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 1452/78315 objects degraded (1.854%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:18.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:18.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:18.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:18.449 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:18.478 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:18.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:18.579 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:18.619 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:18.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:18.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:18.721 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:18.747 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:18.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:18.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:18.849 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:18.878 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:18.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:18.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:18.950 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-04-12T11:36:18.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:19.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:19.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:19.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:19.142 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:18 smithi029 ceph-mon[97787]: pgmap v1446: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 1452/73296 objects degraded (1.981%); 1452/73296 objects misplaced (1.981%); 48 B/s, 25 objects/s recovering 2024-04-12T11:36:19.143 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:18 smithi029 ceph-mon[93650]: pgmap v1446: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 1452/73296 objects degraded (1.981%); 1452/73296 objects misplaced (1.981%); 48 B/s, 25 objects/s recovering 2024-04-12T11:36:19.153 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:19.180 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:19.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:19.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:18 smithi087 ceph-mon[89014]: pgmap v1446: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 1452/73296 objects degraded (1.981%); 1452/73296 objects misplaced (1.981%); 48 B/s, 25 objects/s recovering 2024-04-12T11:36:19.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:19.332 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:19.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:19.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:19.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:19.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:19.479 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:19.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:19.581 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:19.611 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:19.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:19.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:19.712 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:19.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:19.747 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:19.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:19.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:19.885 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:19.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:19.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:19.987 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:20.017 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:20.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:20.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:20.119 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:20.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:20.169 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:20.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:20.270 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:20.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:20.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:20.413 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:20.460 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:20.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:20.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:20.562 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:20.602 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:20.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:20.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:20.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:20.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:20.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:20.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:20.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:20.873 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:20.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:20.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:20.975 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:21.042 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:21.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:21.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:21.055 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:20 smithi029 ceph-mon[93650]: pgmap v1447: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 1278/66015 objects degraded (1.936%); 1278/66015 objects misplaced (1.936%); 48 B/s, 39 objects/s recovering 2024-04-12T11:36:21.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:20 smithi029 ceph-mon[97787]: pgmap v1447: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 1278/66015 objects degraded (1.936%); 1278/66015 objects misplaced (1.936%); 48 B/s, 39 objects/s recovering 2024-04-12T11:36:21.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:21.193 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:21.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:21.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:20 smithi087 ceph-mon[89014]: pgmap v1447: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 1278/66015 objects degraded (1.936%); 1278/66015 objects misplaced (1.936%); 48 B/s, 39 objects/s recovering 2024-04-12T11:36:21.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:21.332 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:21.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:21.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:21.434 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:21.461 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:21.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:21.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:21.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:21.606 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:21.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:21.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:21.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:21.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:21.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:21.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:21.772 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:21.840 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:21.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:21.870 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:21.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:21.897 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:36:21.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:22.009 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:22.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:22.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:21 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/279978738' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:36:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:21 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:36:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:21 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/279978738' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:36:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:21 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:36:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:22.111 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:22.117 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:21 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/279978738' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:36:22.117 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:21 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:36:22.117 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:22.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:22.145 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:22.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:22.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:22.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:22.293 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:22.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:22.394 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:22.425 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:22.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:22.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:22.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:36:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:36:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:36:22.528 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:22.560 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:22.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:22.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:22.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:22.705 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:22.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:22.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:22.807 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:22.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:22.858 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:22.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:22.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:22 smithi029 ceph-mon[97787]: pgmap v1448: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 1278/64743 objects degraded (1.974%); 1278/64743 objects misplaced (1.974%); 0 B/s, 27 objects/s recovering 2024-04-12T11:36:22.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:22 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:36:22.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:22 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:36:22.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:22 smithi029 ceph-mon[97787]: osdmap e1124: 8 total, 8 up, 8 in 2024-04-12T11:36:22.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:22 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 1278/64743 objects degraded (1.974%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:22.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:22 smithi029 ceph-mon[93650]: pgmap v1448: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 1278/64743 objects degraded (1.974%); 1278/64743 objects misplaced (1.974%); 0 B/s, 27 objects/s recovering 2024-04-12T11:36:22.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:22 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:36:22.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:22 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:36:22.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:22 smithi029 ceph-mon[93650]: osdmap e1124: 8 total, 8 up, 8 in 2024-04-12T11:36:22.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:22 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 1278/64743 objects degraded (1.974%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:22.959 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:22.991 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:22.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:22.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:23.079 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 297 (0%) 2024-04-12T11:36:23.080 INFO:teuthology.orchestra.run.smithi029.stdout:avg 37.125 2024-04-12T11:36:23.080 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 5.92532 -> 5.92532 (expected baseline 5.69951) 2024-04-12T11:36:23.080 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 29 -> 29 pgs (0.781145 -> 0.781145 * mean) 2024-04-12T11:36:23.080 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.3 with 47 -> 47 pgs (1.26599 -> 1.26599 * mean) 2024-04-12T11:36:23.080 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:36:23.080 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2024-04-12T11:36:23.080 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 3 2024-04-12T11:36:23.080 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2024-04-12T11:36:23.080 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0084 2024-04-12T11:36:23.080 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0836 2024-04-12T11:36:23.093 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:23.094 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2024-04-12T11:36:23.127 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:23.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:23.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:22 smithi087 ceph-mon[89014]: pgmap v1448: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 1278/64743 objects degraded (1.974%); 1278/64743 objects misplaced (1.974%); 0 B/s, 27 objects/s recovering 2024-04-12T11:36:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:22 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:36:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:22 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:36:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:22 smithi087 ceph-mon[89014]: osdmap e1124: 8 total, 8 up, 8 in 2024-04-12T11:36:23.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:22 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 1278/64743 objects degraded (1.974%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:23.228 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:23.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:23.259 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:23.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:23.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:23.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:23.399 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:23.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:23.500 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:23.543 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:23.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:23.645 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:23.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:23.680 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:23.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:23.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:23.824 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:23.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:23.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:23.922 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-04-12T11:36:23.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd test-reweight-by-utilization 110 3.0 2024-04-12T11:36:23.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:24.009 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:24.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:24.111 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:24.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:24.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:24.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:24.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:23 smithi087 ceph-mon[89014]: from='client.54565 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:36:24.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:24.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:23 smithi029 ceph-mon[97787]: from='client.54565 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:36:24.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:23 smithi029 ceph-mon[93650]: from='client.54565 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:36:24.290 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:24.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:24.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:24.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:24.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:24.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:24.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:24.538 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:24.570 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:24.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:24.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:24.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:24.686 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:24.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:24.725 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:24.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:24.826 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:24.870 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:24.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:24.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:24.972 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:25.003 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:25.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:25.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:25.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:25.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:25.140 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:25.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:24 smithi087 ceph-mon[89014]: pgmap v1450: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 1278/57180 objects degraded (2.235%); 1278/57180 objects misplaced (2.235%); 0 B/s, 33 objects/s recovering 2024-04-12T11:36:25.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:25.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:24 smithi029 ceph-mon[93650]: pgmap v1450: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 1278/57180 objects degraded (2.235%); 1278/57180 objects misplaced (2.235%); 0 B/s, 33 objects/s recovering 2024-04-12T11:36:25.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:24 smithi029 ceph-mon[97787]: pgmap v1450: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 1278/57180 objects degraded (2.235%); 1278/57180 objects misplaced (2.235%); 0 B/s, 33 objects/s recovering 2024-04-12T11:36:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:36:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:36:24] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2024-04-12T11:36:25.287 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:25.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:25.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:25.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:25.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:25.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:25.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:25.538 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:25.562 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:25.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:25.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:25.663 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:25.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:25.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:25.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:25.794 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:25.834 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:25.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:25.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:25.936 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:25.978 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:25.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:25.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:26.080 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:26.117 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:26.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:26.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:26.242 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:26.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:26.344 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:26.373 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:26.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:26.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:26.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:26.504 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:26.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:26.605 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:26.629 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:26.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:26.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:26.731 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:26.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:26.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:26.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:26.873 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:26.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:26.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:26.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:27.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:27.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:27.042 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:27.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:27.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:27.185 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:27.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:27.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:26 smithi087 ceph-mon[89014]: pgmap v1451: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 1115/51522 objects degraded (2.164%); 1115/51522 objects misplaced (2.164%); 0 B/s, 33 objects/s recovering 2024-04-12T11:36:27.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:26 smithi029 ceph-mon[97787]: pgmap v1451: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 1115/51522 objects degraded (2.164%); 1115/51522 objects misplaced (2.164%); 0 B/s, 33 objects/s recovering 2024-04-12T11:36:27.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:26 smithi029 ceph-mon[93650]: pgmap v1451: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 1115/51522 objects degraded (2.164%); 1115/51522 objects misplaced (2.164%); 0 B/s, 33 objects/s recovering 2024-04-12T11:36:27.287 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:27.319 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:27.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:27.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:27.420 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:27.455 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:27.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:27.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:27.556 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:27.602 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:27.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:27.703 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:27.733 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:27.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:27.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:27.835 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:27.881 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:27.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:27.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:27.983 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:28.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:28.011 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:28.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:28.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:27 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 1115/51522 objects degraded (2.164%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:28.016 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:27 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 1115/51522 objects degraded (2.164%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:28.111 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:28.141 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:28.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:28.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:27 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 1115/51522 objects degraded (2.164%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:28.244 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:28.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:28.273 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:28.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:28.374 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:28.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:28.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:28.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:28.453 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:36:28.453 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:36:28.454 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:36:28.454 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:36:28.454 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:36:28.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:28.555 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:28.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:28.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:28.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:28.698 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:28.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:28.800 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:28.833 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:28.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:28.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:28.935 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:28.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:28.968 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:28.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:29.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:29.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:29.103 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:29.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:29.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:28 smithi087 ceph-mon[89014]: pgmap v1452: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 974 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 1115/46803 objects degraded (2.382%); 1115/46803 objects misplaced (2.382%); 0 B/s, 33 objects/s recovering 2024-04-12T11:36:29.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:29.246 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:29.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:29.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:28 smithi029 ceph-mon[93650]: pgmap v1452: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 974 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 1115/46803 objects degraded (2.382%); 1115/46803 objects misplaced (2.382%); 0 B/s, 33 objects/s recovering 2024-04-12T11:36:29.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:28 smithi029 ceph-mon[97787]: pgmap v1452: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 974 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 1115/46803 objects degraded (2.382%); 1115/46803 objects misplaced (2.382%); 0 B/s, 33 objects/s recovering 2024-04-12T11:36:29.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:29.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:29.399 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:29.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:29.500 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:29.536 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:29.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:29.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:29.638 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:29.671 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:29.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:29.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:29.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:29.778 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:36:29.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:29.813 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:29.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:29.914 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:29.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-04-12T11:36:29.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:29.966 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:29.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:30.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:30.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:30.132 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:30.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:30.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:30.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:30.299 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:30.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:30.400 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:30.428 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:30.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:30.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:30.530 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:30.585 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:30.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:30.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:30.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:30.731 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:30.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:30.832 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:30.865 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:30.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:30.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:30.967 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:31.003 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:31.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:31.019 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:31.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:31.118 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:31.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:31.152 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:31.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:31.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:30 smithi029 ceph-mon[97787]: pgmap v1453: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 827 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 916/39741 objects degraded (2.305%); 916/39741 objects misplaced (2.305%); 0 B/s, 36 objects/s recovering 2024-04-12T11:36:31.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:30 smithi029 ceph-mon[93650]: pgmap v1453: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 827 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 916/39741 objects degraded (2.305%); 916/39741 objects misplaced (2.305%); 0 B/s, 36 objects/s recovering 2024-04-12T11:36:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:30 smithi087 ceph-mon[89014]: pgmap v1453: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 827 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 916/39741 objects degraded (2.305%); 916/39741 objects misplaced (2.305%); 0 B/s, 36 objects/s recovering 2024-04-12T11:36:31.253 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:31.287 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:31.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:31.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:31.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:31.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:31.435 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:31.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:31.536 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:31.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:31.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:31.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:31.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:31.709 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:31.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:31.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:31.855 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:31.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:31.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:31.881 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:31.957 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:31.986 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:31.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:31.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:32.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:32.113 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:32.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:32.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:32.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:32.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:32.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:32.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:32.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:32.386 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:32.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:32.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:32.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:32.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:32.542 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:32.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:32.643 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:32.675 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:32.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:32.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:36:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:36:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:36:32.777 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:32.809 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:32.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:32.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:32.911 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:32.950 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:32.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:33.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:33.087 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:33.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:33.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:33.189 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:32 smithi087 ceph-mon[89014]: pgmap v1454: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 802 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 916/38541 objects degraded (2.377%); 916/38541 objects misplaced (2.377%); 0 B/s, 36 objects/s recovering 2024-04-12T11:36:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:32 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 916/38541 objects degraded (2.377%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:33.227 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:33.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:33.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:32 smithi029 ceph-mon[97787]: pgmap v1454: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 802 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 916/38541 objects degraded (2.377%); 916/38541 objects misplaced (2.377%); 0 B/s, 36 objects/s recovering 2024-04-12T11:36:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:32 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 916/38541 objects degraded (2.377%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:33.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:32 smithi029 ceph-mon[93650]: pgmap v1454: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 802 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 916/38541 objects degraded (2.377%); 916/38541 objects misplaced (2.377%); 0 B/s, 36 objects/s recovering 2024-04-12T11:36:33.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:32 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 916/38541 objects degraded (2.377%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:33.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:33.364 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:33.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:33.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:33.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:33.508 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:33.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:33.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:33.633 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:33.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:33.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:33.735 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:33.767 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:33.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:33.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:33.908 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:33.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:33.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:34.010 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:34.041 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:34.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:34.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:34.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:34.165 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:34.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:34.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:34.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:34.300 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:34.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:34.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:34.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:34.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:34.445 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:34.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:34.546 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:34.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:34.584 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:34.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:34.685 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:34.726 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:34.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:34.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:34.828 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:34.856 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:34.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:34.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:34.958 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:34.988 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:34.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:35.090 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:35.122 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:35.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:35.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:34 smithi087 ceph-mon[89014]: pgmap v1455: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 643 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 951 op/s; 916/30921 objects degraded (2.962%); 916/30921 objects misplaced (2.962%); 0 B/s, 30 objects/s recovering 2024-04-12T11:36:35.224 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:35.248 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:35.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:35.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:34 smithi029 ceph-mon[97787]: pgmap v1455: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 643 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 951 op/s; 916/30921 objects degraded (2.962%); 916/30921 objects misplaced (2.962%); 0 B/s, 30 objects/s recovering 2024-04-12T11:36:35.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:34 smithi029 ceph-mon[93650]: pgmap v1455: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 643 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 951 op/s; 916/30921 objects degraded (2.962%); 916/30921 objects misplaced (2.962%); 0 B/s, 30 objects/s recovering 2024-04-12T11:36:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:36:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:36:34] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2024-04-12T11:36:35.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:35.392 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:35.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:35.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:35.494 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:35.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:35.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:35.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:35.621 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:35.657 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:35.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:35.759 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:35.796 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:35.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:35.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:35.898 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:35.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:35.933 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:35.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:36.034 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:36.070 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:36.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:36.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:36.172 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:36.206 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:36.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:36.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:36.307 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:36.337 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:36.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:36.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:36.439 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:36.463 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:36.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:36.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:36.602 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:36.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:36.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:36.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:36.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:36.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:36.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:36.844 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:36.879 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:36.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:36.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:36.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:37.022 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:37.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:37.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:37.124 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:37.171 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:37.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:37.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:36 smithi087 ceph-mon[89014]: pgmap v1456: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 523 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 733/25158 objects degraded (2.914%); 733/25158 objects misplaced (2.914%); 0 B/s, 45 objects/s recovering 2024-04-12T11:36:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:36 smithi029 ceph-mon[93650]: pgmap v1456: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 523 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 733/25158 objects degraded (2.914%); 733/25158 objects misplaced (2.914%); 0 B/s, 45 objects/s recovering 2024-04-12T11:36:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:36 smithi029 ceph-mon[97787]: pgmap v1456: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 523 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 733/25158 objects degraded (2.914%); 733/25158 objects misplaced (2.914%); 0 B/s, 45 objects/s recovering 2024-04-12T11:36:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:37.272 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:37.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:37.297 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:37.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:37.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:37.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:37.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:37.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:37.523 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:37.556 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:37.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:37.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:37.658 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:37.686 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:37.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:37.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:37.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:37.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:37.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:37.942 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:37.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:37.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:38.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:38.080 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:38.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:38.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:38.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:37 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 733/25158 objects degraded (2.914%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:38.225 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:38.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:38.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:38.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:37 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 733/25158 objects degraded (2.914%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:38.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:37 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 733/25158 objects degraded (2.914%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:38.327 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:38.351 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:38.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:38.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:38.452 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:38.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:38.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:38.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:38.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:38.625 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:38.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:38.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:38.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:38.767 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:38.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:38.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:38.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:38.910 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:38.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:39.012 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:39.041 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:39.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:39.143 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:39.176 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:39.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:39.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:38 smithi087 ceph-mon[89014]: pgmap v1457: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 421 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 733/20268 objects degraded (3.617%); 733/20268 objects misplaced (3.617%); 0 B/s, 31 objects/s recovering 2024-04-12T11:36:39.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:38 smithi029 ceph-mon[93650]: pgmap v1457: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 421 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 733/20268 objects degraded (3.617%); 733/20268 objects misplaced (3.617%); 0 B/s, 31 objects/s recovering 2024-04-12T11:36:39.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:38 smithi029 ceph-mon[97787]: pgmap v1457: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 421 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 733/20268 objects degraded (3.617%); 733/20268 objects misplaced (3.617%); 0 B/s, 31 objects/s recovering 2024-04-12T11:36:39.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:39.303 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:39.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:39.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:39.442 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:39.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:39.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:39.544 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:39.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:39.583 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:39.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:39.684 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:39.713 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:39.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:39.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:39.815 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:39.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:39.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:39.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:39.945 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:39.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:39.979 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:39.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:40.080 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:40.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:40.124 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:40.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:40.225 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:40.254 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:40.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:40.355 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:40.391 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:40.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:40.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:40.515 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:40.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:40.617 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:40.649 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:40.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:40.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:40.751 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:40.781 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:40.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:40.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:40.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:40.906 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:40.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:40.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:41.007 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:41.039 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:41.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:41.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:41.141 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:41.175 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:41.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:41.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:40 smithi087 ceph-mon[89014]: pgmap v1458: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 295 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 564/14229 objects degraded (3.964%); 564/14229 objects misplaced (3.964%); 0 B/s, 45 objects/s recovering 2024-04-12T11:36:41.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:40 smithi029 ceph-mon[93650]: pgmap v1458: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 295 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 564/14229 objects degraded (3.964%); 564/14229 objects misplaced (3.964%); 0 B/s, 45 objects/s recovering 2024-04-12T11:36:41.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:40 smithi029 ceph-mon[97787]: pgmap v1458: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 295 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 564/14229 objects degraded (3.964%); 564/14229 objects misplaced (3.964%); 0 B/s, 45 objects/s recovering 2024-04-12T11:36:41.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:41.299 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:41.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:41.401 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:41.433 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:41.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:41.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:41.535 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:41.583 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:41.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:41.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:41.685 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:41.730 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:41.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:41.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:41.832 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:41.873 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:41.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:41.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:41.974 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:42.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:42.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:42.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:42.164 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:42.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:42.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:42.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:42.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:42.289 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:42.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:42.390 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:42.414 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:42.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:42.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:42.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:42.573 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:42.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:42.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:42.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:42.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:42.713 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:42.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:36:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:36:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:36:42.782 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:36:42.814 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:42.831 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:36:42.857 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:42.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:42.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:42.881 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:36:42.922 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:36:42.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:42.970 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:36:43.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:43.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:43.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:43.015 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:36:43.054 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:36:43.099 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:36:43.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:42 smithi029 ceph-mon[97787]: pgmap v1459: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 272 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 564/13110 objects degraded (4.302%); 564/13110 objects misplaced (4.302%); 0 B/s, 29 objects/s recovering 2024-04-12T11:36:43.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:42 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3501780762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:36:43.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:42 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 564/13110 objects degraded (4.302%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:43.099 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:42 smithi029 ceph-mon[93650]: pgmap v1459: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 272 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 564/13110 objects degraded (4.302%); 564/13110 objects misplaced (4.302%); 0 B/s, 29 objects/s recovering 2024-04-12T11:36:43.099 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:42 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3501780762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:36:43.099 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:42 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 564/13110 objects degraded (4.302%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:43.106 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:36:43.106 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:36:43.106 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:36:43.106 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:36:43.106 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:36:43.106 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:36:43.106 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:36:43.106 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:36:43.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:43.156 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:43.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:43.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:42 smithi087 ceph-mon[89014]: pgmap v1459: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 272 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 564/13110 objects degraded (4.302%); 564/13110 objects misplaced (4.302%); 0 B/s, 29 objects/s recovering 2024-04-12T11:36:43.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:42 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3501780762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:36:43.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:42 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 564/13110 objects degraded (4.302%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:43.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:43.303 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:43.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:43.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:43.405 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:43.457 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:43.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:43.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:43.559 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:43.599 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:43.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:43.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:43.701 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:43.735 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:43.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:43.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:43.826 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 106822 objects 2024-04-12T11:36:43.826 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :126.193 2024-04-12T11:36:43.837 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:43.860 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:43.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:43.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:43.961 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:43.987 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:43.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:44.089 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:44.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:44.116 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:44.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:44.217 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:44.244 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:44.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:44.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:44.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:44.372 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:44.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:44.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:44.474 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:44.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:44.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:44.601 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:44.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:44.629 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:44.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:44.730 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:44.757 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:44.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:44.803 INFO:teuthology.orchestra.run.smithi029.stdout:moved 34 / 297 (11.4478%) 2024-04-12T11:36:44.803 INFO:teuthology.orchestra.run.smithi029.stdout:avg 37.125 2024-04-12T11:36:44.803 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 5.92532 -> 8.038 (expected baseline 5.69951) 2024-04-12T11:36:44.803 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 29 -> 21 pgs (0.781145 -> 0.565657 * mean) 2024-04-12T11:36:44.803 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.3 with 47 -> 41 pgs (1.26599 -> 1.10438 * mean) 2024-04-12T11:36:44.803 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:36:44.804 INFO:teuthology.orchestra.run.smithi029.stdout:oload 110 2024-04-12T11:36:44.804 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 3 2024-04-12T11:36:44.804 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2024-04-12T11:36:44.804 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0032 2024-04-12T11:36:44.804 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0035 2024-04-12T11:36:44.804 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 weight 0.9000 -> 0.5380 2024-04-12T11:36:44.804 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 weight 0.9500 -> 0.6906 2024-04-12T11:36:44.804 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 weight 0.9000 -> 0.6765 2024-04-12T11:36:44.804 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2024-04-12T11:36:44.805 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:36:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:36:44.521+0000 7f4c1f740700 -1 mgr.server reply reply (3) No such process no change 2024-04-12T11:36:44.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:44.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:44.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:44.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:45.015 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:45.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:45.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:45.144 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:45.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:45.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:44 smithi087 ceph-mon[89014]: pgmap v1460: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 135 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 564/6528 objects degraded (8.640%); 564/6528 objects misplaced (8.640%); 0 B/s, 29 objects/s recovering 2024-04-12T11:36:45.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:45.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:44 smithi029 ceph-mon[97787]: pgmap v1460: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 135 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 564/6528 objects degraded (8.640%); 564/6528 objects misplaced (8.640%); 0 B/s, 29 objects/s recovering 2024-04-12T11:36:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:44 smithi029 ceph-mon[93650]: pgmap v1460: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 135 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 564/6528 objects degraded (8.640%); 564/6528 objects misplaced (8.640%); 0 B/s, 29 objects/s recovering 2024-04-12T11:36:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:36:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:36:44] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2024-04-12T11:36:45.270 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:45.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:45.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:45.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:45.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:45.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:45.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:45.519 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:45.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:45.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:45.621 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:45.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:45.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:45.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:45.772 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:45.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:45.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:45.874 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:45.901 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:45.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:45.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:46.003 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:46.024 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:36:46.030 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:46.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:46.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:46.131 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:46.157 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:46.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:46.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:46.206 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-04-12T11:36:46.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-04-12T11:36:46.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:45 smithi087 ceph-mon[89014]: from='client.44780 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:36:46.230 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:45 smithi029 ceph-mon[97787]: from='client.44780 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:36:46.230 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:45 smithi029 ceph-mon[93650]: from='client.44780 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:36:46.259 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:46.285 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:46.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:46.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:46.387 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:46.410 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:46.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:46.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:46.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:46.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:46.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:46.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:46.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:46.660 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:46.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:46.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:46.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:46.786 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:46.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:46.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:46.888 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:46.916 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:46.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:46.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:47.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:47.045 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:47.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:47.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:47.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:47.172 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:47.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:47.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:46 smithi087 ceph-mon[89014]: pgmap v1461: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 37 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 194/1827 objects degraded (10.619%); 194/1827 objects misplaced (10.619%); 0 B/s, 60 objects/s recovering 2024-04-12T11:36:47.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:46 smithi087 ceph-mon[89014]: osdmap e1125: 8 total, 8 up, 8 in 2024-04-12T11:36:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:46 smithi029 ceph-mon[97787]: pgmap v1461: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 37 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 194/1827 objects degraded (10.619%); 194/1827 objects misplaced (10.619%); 0 B/s, 60 objects/s recovering 2024-04-12T11:36:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:47 smithi029 ceph-mon[97787]: osdmap e1125: 8 total, 8 up, 8 in 2024-04-12T11:36:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:46 smithi029 ceph-mon[93650]: pgmap v1461: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 37 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 194/1827 objects degraded (10.619%); 194/1827 objects misplaced (10.619%); 0 B/s, 60 objects/s recovering 2024-04-12T11:36:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:47 smithi029 ceph-mon[93650]: osdmap e1125: 8 total, 8 up, 8 in 2024-04-12T11:36:47.274 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:47.302 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:47.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:47.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:47.403 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:47.431 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:47.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:47.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:47.533 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:47.559 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:47.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:47.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:47.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:47.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:47.688 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:47.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:47.789 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:47.817 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:47.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:47.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:47.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:47.948 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:47.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:47.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:48.049 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:48.077 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:48.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:48.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:48.179 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:48.201 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:47 smithi029 ceph-mon[97787]: osdmap e1126: 8 total, 8 up, 8 in 2024-04-12T11:36:48.201 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 194/1827 objects degraded (10.619%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:48.201 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:47 smithi029 ceph-mon[93650]: osdmap e1126: 8 total, 8 up, 8 in 2024-04-12T11:36:48.201 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 194/1827 objects degraded (10.619%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:48.206 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:48.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:48.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:48.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:47 smithi087 ceph-mon[89014]: osdmap e1126: 8 total, 8 up, 8 in 2024-04-12T11:36:48.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:47 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 194/1827 objects degraded (10.619%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:36:48.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:48.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:48.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:48.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:48.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:48.465 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:48.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:48.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:48.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:48.594 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:48.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:48.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:48.696 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:48.723 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:48.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:48.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:48.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:48.852 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:48.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:48.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:48.954 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:48.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:48.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:48.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:49.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:49.112 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:49.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:49.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:49.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:48 smithi087 ceph-mon[89014]: pgmap v1464: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.8 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 194/417 objects degraded (46.523%); 194/417 objects misplaced (46.523%); 0 B/s, 46 objects/s recovering 2024-04-12T11:36:49.241 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:49.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:49.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:49.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:48 smithi029 ceph-mon[93650]: pgmap v1464: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.8 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 194/417 objects degraded (46.523%); 194/417 objects misplaced (46.523%); 0 B/s, 46 objects/s recovering 2024-04-12T11:36:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:48 smithi029 ceph-mon[97787]: pgmap v1464: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 98 active+clean; 7.8 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 194/417 objects degraded (46.523%); 194/417 objects misplaced (46.523%); 0 B/s, 46 objects/s recovering 2024-04-12T11:36:49.343 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:49.369 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:49.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:49.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:49.471 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:49.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:49.501 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:49.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:49.602 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:49.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:49.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:49.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:49.732 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:49.765 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:49.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:49.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:49.835 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-04-12T11:36:49.835 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight-by-utilization 1000 3.0 2024-04-12T11:36:49.866 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:49.894 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:49.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:49.996 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:50.023 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:50.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:50.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:50.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:50.149 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:50.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:50.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:49 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]: dispatch 2024-04-12T11:36:50.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:50.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:49 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]: dispatch 2024-04-12T11:36:50.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:49 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]: dispatch 2024-04-12T11:36:50.278 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:50.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:50.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:50.379 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:50.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:50.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:50.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:50.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:50.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:50.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:50.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:50.592 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:50.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:50.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:50.652 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:50.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:50.753 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:50.781 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:50.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:50.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:50.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:50.906 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:50.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:50.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:51.008 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:51.034 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:51.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:51.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:51.136 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:51.161 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:51.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:51.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:51.202 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:36:51.202 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:36:51.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[97787]: pgmap v1465: 99 pgs: 99 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 0 B/s, 70 objects/s recovering 2024-04-12T11:36:51.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 194/417 objects degraded (46.523%), 1 pg degraded, 1 pg undersized) 2024-04-12T11:36:51.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-04-12T11:36:51.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[97787]: osdmap e1127: 8 total, 8 up, 8 in 2024-04-12T11:36:51.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:51.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:51.203 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[93650]: pgmap v1465: 99 pgs: 99 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 0 B/s, 70 objects/s recovering 2024-04-12T11:36:51.203 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 194/417 objects degraded (46.523%), 1 pg degraded, 1 pg undersized) 2024-04-12T11:36:51.204 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-04-12T11:36:51.204 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[93650]: osdmap e1127: 8 total, 8 up, 8 in 2024-04-12T11:36:51.204 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:51.204 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:50 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:50 smithi087 ceph-mon[89014]: pgmap v1465: 99 pgs: 99 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 0 B/s, 70 objects/s recovering 2024-04-12T11:36:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:50 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 194/417 objects degraded (46.523%), 1 pg degraded, 1 pg undersized) 2024-04-12T11:36:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:50 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num_actual", "val": "26"}]': finished 2024-04-12T11:36:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:50 smithi087 ceph-mon[89014]: osdmap e1127: 8 total, 8 up, 8 in 2024-04-12T11:36:51.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:50 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:51.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:50 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:51.263 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:51.288 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:51.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:51.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:51.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:51.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:51.414 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:51.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:51.515 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:51.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:51.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:51.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:51.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:51.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:51.669 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:51.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:51.770 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:51.797 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:51.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:51.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:51.899 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:51.923 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:51.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:51.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:52.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:52.049 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:52.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:52.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:52.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[97787]: osdmap e1128: 8 total, 8 up, 8 in 2024-04-12T11:36:52.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:36:52.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:36:52.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:52.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:52.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:52.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[97787]: osdmap e1129: 8 total, 8 up, 8 in 2024-04-12T11:36:52.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[93650]: osdmap e1128: 8 total, 8 up, 8 in 2024-04-12T11:36:52.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:36:52.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:36:52.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:52.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:52.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:52.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:51 smithi029 ceph-mon[93650]: osdmap e1129: 8 total, 8 up, 8 in 2024-04-12T11:36:52.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-04-12T11:36:52.151 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:52.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:52.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:52.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:51 smithi087 ceph-mon[89014]: osdmap e1128: 8 total, 8 up, 8 in 2024-04-12T11:36:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:36:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:36:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:36:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:51 smithi087 ceph-mon[89014]: osdmap e1129: 8 total, 8 up, 8 in 2024-04-12T11:36:52.302 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:52.328 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:52.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:52.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:52.339 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:36:52.430 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:52.457 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:52.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:52.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:52.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:52.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:52.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:52.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:52.617 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:36:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:36:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:36:52.684 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:52.710 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:52.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:52.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:52.812 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:52.838 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:52.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:52.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:52.940 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:52.965 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:52.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:53.067 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:53.093 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:53.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:53.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:53.214 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:52 smithi087 ceph-mon[89014]: pgmap v1468: 99 pgs: 99 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 0 B/s, 33 objects/s recovering 2024-04-12T11:36:53.214 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:52 smithi087 ceph-mon[89014]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 1124'8782 2024-04-12T11:36:53.214 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:52 smithi087 ceph-mon[89014]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 1124'11209 2024-04-12T11:36:53.214 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:52 smithi087 ceph-mon[89014]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 1124'11209 2024-04-12T11:36:53.214 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:52 smithi087 ceph-mon[89014]: osdmap e1130: 8 total, 8 up, 8 in 2024-04-12T11:36:53.219 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:53.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:53.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:52 smithi029 ceph-mon[97787]: pgmap v1468: 99 pgs: 99 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 0 B/s, 33 objects/s recovering 2024-04-12T11:36:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:52 smithi029 ceph-mon[97787]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 1124'8782 2024-04-12T11:36:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:52 smithi029 ceph-mon[97787]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 1124'11209 2024-04-12T11:36:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:52 smithi029 ceph-mon[97787]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 1124'11209 2024-04-12T11:36:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:52 smithi029 ceph-mon[97787]: osdmap e1130: 8 total, 8 up, 8 in 2024-04-12T11:36:53.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:52 smithi029 ceph-mon[93650]: pgmap v1468: 99 pgs: 99 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 0 B/s, 33 objects/s recovering 2024-04-12T11:36:53.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:52 smithi029 ceph-mon[93650]: 6.16 starting backfill to osd.0 from (0'0,0'0] MAX to 1124'8782 2024-04-12T11:36:53.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:52 smithi029 ceph-mon[93650]: 6.18 starting backfill to osd.0 from (0'0,0'0] MAX to 1124'11209 2024-04-12T11:36:53.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:52 smithi029 ceph-mon[93650]: 6.18 starting backfill to osd.1 from (0'0,0'0] MAX to 1124'11209 2024-04-12T11:36:53.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:52 smithi029 ceph-mon[93650]: osdmap e1130: 8 total, 8 up, 8 in 2024-04-12T11:36:53.322 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:53.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:53.349 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:53.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:53.450 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:53.476 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:53.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:53.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:53.578 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:53.587 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:53.603 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:53.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:53.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:53.730 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:53.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:53.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:53.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:53.850 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_1' removed 2024-04-12T11:36:53.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:53.867 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:53.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:53.968 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:53.999 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:53.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:54.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:54.100 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:54.126 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:54.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:54.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:53 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3860855651' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:36:54.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:53 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:36:54.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:53 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:36:54.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:53 smithi029 ceph-mon[93650]: osdmap e1131: 8 total, 8 up, 8 in 2024-04-12T11:36:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:53 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3860855651' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:36:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:53 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:36:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:53 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:36:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:53 smithi029 ceph-mon[97787]: osdmap e1131: 8 total, 8 up, 8 in 2024-04-12T11:36:54.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:53 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3860855651' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:36:54.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:53 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:36:54.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:53 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:36:54.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:53 smithi087 ceph-mon[89014]: osdmap e1131: 8 total, 8 up, 8 in 2024-04-12T11:36:54.228 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:54.252 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:54.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:54.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:54.264 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:36:54.264 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:36:54.354 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:54.378 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:54.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:54.480 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:54.504 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:54.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:54.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:54.606 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:54.630 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:54.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:54.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:54.674 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-12T11:36:54.674 INFO:tasks.radosbench:Beginning radosbench... 2024-04-12T11:36:54.675 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-04-12T11:36:54.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-04-12T11:36:54.731 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:54.754 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:54.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:54.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:54.856 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:54.862 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-04-12T11:36:54.879 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:54.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:54.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:54.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:55.005 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:55.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:55.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:55.070 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:36:55.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:55.130 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:55.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:55.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:55.192 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:54 smithi087 ceph-mon[89014]: pgmap v1471: 99 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 1.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6716/123 objects degraded (5460.163%); 3358/123 objects misplaced (2730.081%); 0 B/s, 7 objects/s recovering 2024-04-12T11:36:55.192 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:54 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 6716/123 objects degraded (5460.163%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:36:55.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:55.255 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:55.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:55.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:55.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:36:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:36:54] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.43.0" 2024-04-12T11:36:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:54 smithi029 ceph-mon[97787]: pgmap v1471: 99 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 1.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6716/123 objects degraded (5460.163%); 3358/123 objects misplaced (2730.081%); 0 B/s, 7 objects/s recovering 2024-04-12T11:36:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:54 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 6716/123 objects degraded (5460.163%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:36:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:54 smithi029 ceph-mon[93650]: pgmap v1471: 99 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 95 active+clean; 1.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6716/123 objects degraded (5460.163%); 3358/123 objects misplaced (2730.081%); 0 B/s, 7 objects/s recovering 2024-04-12T11:36:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:54 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 6716/123 objects degraded (5460.163%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:36:55.357 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:55.381 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:55.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:55.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:55.483 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:55.506 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:55.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:55.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:55.608 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:55.632 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:55.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:55.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:55.733 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:55.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:55.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:55.860 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:55.884 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:55.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:55.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:55.986 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:56.009 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:56.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:56.111 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:56.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:56.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:56.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:56.156 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:56 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6716/123 objects degraded (5460.163%), 1 pg degraded) 2024-04-12T11:36:56.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:56.261 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:56.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:56.288 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:56 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6716/123 objects degraded (5460.163%), 1 pg degraded) 2024-04-12T11:36:56.288 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:56 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6716/123 objects degraded (5460.163%), 1 pg degraded) 2024-04-12T11:36:56.361 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:56.385 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:56.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:56.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:56.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:56.510 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:56.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:56.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:56.612 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:56.636 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:56.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:56.737 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:56.761 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:56.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:56.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:56.774 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:36:56.774 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:36:56.863 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:56.870 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:56.871 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:36:56.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:56.888 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:56.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:56.911 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:36:56.931 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:36:56.950 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:36:56.970 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:36:56.990 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:56.990 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:36:57.009 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:36:57.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:57.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:57.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:57.037 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:36:57.038 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:36:57.058 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:36:57.060 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:36:57.060 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:36:57.060 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:36:57.060 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:36:57.060 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:36:57.060 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:36:57.060 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:36:57.060 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:36:57.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:57.139 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:57.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:57.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:57.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:57 smithi029 ceph-mon[97787]: pgmap v1473: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:36:57.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:57 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/164509638' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:36:57.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:57 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:57.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:57 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4053359353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:36:57.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:57 smithi029 ceph-mon[93650]: pgmap v1473: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:36:57.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:57 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/164509638' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:36:57.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:57 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:57.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:57 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4053359353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:36:57.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:57.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:57.265 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:57.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:57.366 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:57.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:57.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:57.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:57 smithi087 ceph-mon[89014]: pgmap v1473: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:36:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:57 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/164509638' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:36:57.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:57 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:36:57.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:57 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4053359353' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:36:57.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:57.516 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:57.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:57.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:57.534 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-04-12T11:36:57.617 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:57.641 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:57.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:57.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:57.739 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:36:57.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:57.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:57.768 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:57.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:57.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:36:57.892 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:57.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:57.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:36:57.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:58.017 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:58.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:58.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:58.119 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:58.143 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:58.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:58.151 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:58 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:36:58.151 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:58 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/164509638' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:36:58.151 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:58 smithi087 ceph-mon[89014]: osdmap e1132: 8 total, 8 up, 8 in 2024-04-12T11:36:58.245 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:58.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:58 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:36:58.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:58 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/164509638' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:36:58.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:58 smithi029 ceph-mon[97787]: osdmap e1132: 8 total, 8 up, 8 in 2024-04-12T11:36:58.257 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:58 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:36:58.257 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:58 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/164509638' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:36:58.257 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:58 smithi029 ceph-mon[93650]: osdmap e1132: 8 total, 8 up, 8 in 2024-04-12T11:36:58.269 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:58.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:58.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:58.371 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:58.395 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:58.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:58.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:58.496 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:58.520 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:58.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:58.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:58.622 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:36:58.645 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:58.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:58.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:36:58.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:58.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:58.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:58.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:58.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:58.896 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:58.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:58.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:58.997 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:59.021 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:59.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:59.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:59.123 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:36:59.147 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:59.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:59.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:65197 6.12 6:496fa9ff:::benchmark_data_smithi029_138569_object65196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:41.884945+0000", 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 197.28098735200001, 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0453005319999999, 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:36:59.168 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 65197 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.884945+0000", 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.884945+0000", 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.169 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.884947+0000", 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.714e-06 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.884998+0000", 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0593000000000002e-05 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.885009+0000", 2024-04-12T11:36:59.170 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1875000000000001e-05 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.885019+0000", 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4550000000000001e-06 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.944403+0000", 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.059384563000000001 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.944413+0000", 2024-04-12T11:36:59.171 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.5899999999999997e-06 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.930158+0000", 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98574526900000004 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.930246+0000", 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7472999999999994e-05 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.172 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:75032 6.2 6:442ab1f4:::benchmark_data_smithi029_138569_object75031:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:54.020269+0000", 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.14566385699999, 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.354982931, 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75032 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.173 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.020269+0000", 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.020269+0000", 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.020271+0000", 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9089999999999998e-06 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.174 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.175 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.175 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.020316+0000", 2024-04-12T11:36:59.175 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5015000000000002e-05 2024-04-12T11:36:59.175 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.175 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.175 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.175 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.020337+0000", 2024-04-12T11:36:59.175 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1407999999999999e-05 2024-04-12T11:36:59.175 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.175 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.020348+0000", 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0682999999999999e-05 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.020421+0000", 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3417999999999994e-05 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.020565+0000", 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014350800000000001 2024-04-12T11:36:59.176 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.020808+0000", 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000243613 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.022500+0000", 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0016918199999999999 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.022744+0000", 2024-04-12T11:36:59.177 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00024399900000000001 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.375150+0000", 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.35240580799999999 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.375199+0000", 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9257e-05 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.375252+0000", 2024-04-12T11:36:59.178 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2493e-05 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:77970 6.e 6:7226b447:::benchmark_data_smithi029_138569_object77969:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:01.561377+0000", 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 177.60455601999999, 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.248735024, 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.179 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 77970 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.561377+0000", 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.561377+0000", 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.561378+0000", 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.449e-06 2024-04-12T11:36:59.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.181 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.181 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.181 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.561432+0000", 2024-04-12T11:36:59.181 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3631000000000002e-05 2024-04-12T11:36:59.181 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.561446+0000", 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4229e-05 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.561454+0000", 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0700000000000007e-06 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.182 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.561508+0000", 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4008999999999997e-05 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.561671+0000", 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016319199999999999 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.561828+0000", 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015663299999999999 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.183 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.564605+0000", 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027767460000000001 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.566533+0000", 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0019283779999999999 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.810013+0000", 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.24348036200000001 2024-04-12T11:36:59.184 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.810066+0000", 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2269999999999999e-05 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:01.810112+0000", 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6054999999999999e-05 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.185 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:82168 6.e 6:70803b4a:::benchmark_data_smithi029_138569_object82167:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:05.935883+0000", 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 173.230049431, 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30422452100000003, 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 82168 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.186 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.935883+0000", 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.935883+0000", 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.935884+0000", 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1939999999999999e-06 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.187 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.935923+0000", 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8884999999999998e-05 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.935935+0000", 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1999e-05 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.935944+0000", 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6119999999999999e-06 2024-04-12T11:36:59.188 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.937701+0000", 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00175731 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.937860+0000", 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000158395 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.189 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.938023+0000", 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000163012 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.940376+0000", 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023536759999999999 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.946751+0000", 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0063752089999999997 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.190 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.239986+0000", 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29323418600000001 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.240053+0000", 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7558000000000006e-05 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.240108+0000", 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4484999999999998e-05 2024-04-12T11:36:59.191 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:82177 6.e 6:77b176da:::benchmark_data_smithi029_138569_object82176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:05.942896+0000", 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 173.22303651199999, 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.480965741, 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.192 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 82177 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942896+0000", 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942896+0000", 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.193 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942898+0000", 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4699999999999999e-06 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942945+0000", 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7042999999999997e-05 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942959+0000", 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.454e-05 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.194 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942967+0000", 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.909e-06 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.943018+0000", 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1025e-05 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.195 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.943187+0000", 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000169271 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.943367+0000", 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018001099999999999 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.946093+0000", 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027254150000000001 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.196 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.240144+0000", 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29405152200000001 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.423798+0000", 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.183654027 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.423833+0000", 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5037999999999999e-05 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.197 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.423862+0000", 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.847e-05 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:82186 6.2 6:444ecc6a:::benchmark_data_smithi029_138569_object82185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:05.949311+0000", 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 173.21662135899999, 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.47286100399999997, 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.198 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 82186 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949311+0000", 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.199 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949311+0000", 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949313+0000", 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.711e-06 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949362+0000", 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8847999999999997e-05 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.200 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949376+0000", 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4411e-05 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949385+0000", 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6989999999999992e-06 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949448+0000", 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3156000000000003e-05 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.201 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949618+0000", 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016943399999999999 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949790+0000", 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017233799999999999 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.951406+0000", 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001616159 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.202 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.951830+0000", 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00042385600000000002 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.422053+0000", 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.47022343300000002 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.422120+0000", 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6230999999999997e-05 2024-04-12T11:36:59.203 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.422172+0000", 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2728000000000001e-05 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:82364 6.e 6:74542616:::benchmark_data_smithi029_138569_object82363:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:06.546111+0000", 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 172.61982183999999, 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.28586065900000002, 2024-04-12T11:36:59.204 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 82364 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.546111+0000", 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.205 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.546111+0000", 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.546112+0000", 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.674e-06 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.546151+0000", 2024-04-12T11:36:59.206 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9032000000000002e-05 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.546172+0000", 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0886999999999998e-05 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.546182+0000", 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0098999999999999e-05 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.546265+0000", 2024-04-12T11:36:59.207 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2818000000000006e-05 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.546440+0000", 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017424500000000001 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.546592+0000", 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015228900000000001 2024-04-12T11:36:59.208 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.209 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:36:59 smithi087 ceph-mon[89014]: pgmap v1475: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:36:59.210 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.210 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.210 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.549598+0000", 2024-04-12T11:36:59.210 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003005733 2024-04-12T11:36:59.210 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.210 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.210 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.210 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.550386+0000", 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00078884300000000001 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.831886+0000", 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.281499675 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.831933+0000", 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7299000000000002e-05 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.211 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.831971+0000", 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8065000000000002e-05 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:82376 6.e 6:7a8c6711:::benchmark_data_smithi029_138569_object82375:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:06.555647+0000", 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 172.61028547199999, 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27848541999999998, 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.212 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 82376 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.555647+0000", 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.555647+0000", 2024-04-12T11:36:59.213 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.555649+0000", 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6130000000000001e-06 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.555690+0000", 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0908999999999997e-05 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.555707+0000", 2024-04-12T11:36:59.214 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7697999999999999e-05 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.555718+0000", 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0322999999999999e-05 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.555789+0000", 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1285999999999995e-05 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.215 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.555939+0000", 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015005899999999999 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.556136+0000", 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019713200000000001 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.558604+0000", 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0024674979999999998 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.216 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.559170+0000", 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00056641500000000002 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.834057+0000", 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27488698099999997 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.834091+0000", 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4093e-05 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.217 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.834133+0000", 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1412999999999998e-05 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:83729 6.e 6:72d435e2:::benchmark_data_smithi029_138569_object83728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:08.259152+0000", 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 170.906781056, 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.24054112799999999, 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.218 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 83729 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.259152+0000", 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.259152+0000", 2024-04-12T11:36:59.219 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.259153+0000", 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6029999999999999e-06 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.259277+0000", 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012340499999999999 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.220 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.259295+0000", 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8637000000000001e-05 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.259304+0000", 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7260000000000008e-06 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.259376+0000", 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2373000000000007e-05 2024-04-12T11:36:59.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.259539+0000", 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016280600000000001 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.259690+0000", 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015070099999999999 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.262167+0000", 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0024771229999999999 2024-04-12T11:36:59.222 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.262378+0000", 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021102100000000001 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.499606+0000", 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.23722800599999999 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.499646+0000", 2024-04-12T11:36:59.223 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9889999999999999e-05 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.499693+0000", 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6836999999999999e-05 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:84312 6.2 6:43b1a48e:::benchmark_data_smithi029_138569_object84311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:08.946270+0000", 2024-04-12T11:36:59.224 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 170.219662984, 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.256072833, 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 84312 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946270+0000", 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.225 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946270+0000", 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946271+0000", 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.471e-06 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946326+0000", 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5038999999999999e-05 2024-04-12T11:36:59.226 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946344+0000", 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8199e-05 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946353+0000", 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6289999999999999e-06 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946420+0000", 2024-04-12T11:36:59.227 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7378000000000004e-05 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946579+0000", 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015882799999999999 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946721+0000", 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014186799999999999 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.950506+0000", 2024-04-12T11:36:59.228 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0037849649999999999 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.951937+0000", 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0014312319999999999 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:09.202261+0000", 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.25032351000000003 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:09.202305+0000", 2024-04-12T11:36:59.229 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4348999999999998e-05 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:09.202342+0000", 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.7364999999999999e-05 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:84313 6.2 6:403401d1:::benchmark_data_smithi029_138569_object84312:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:08.946414+0000", 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 170.21951846799999, 2024-04-12T11:36:59.230 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.263449296, 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 84313 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946414+0000", 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.231 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946414+0000", 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946415+0000", 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5199999999999996e-07 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946500+0000", 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5371999999999995e-05 2024-04-12T11:36:59.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946510+0000", 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0138e-05 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.946518+0000", 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3719999999999999e-06 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.948482+0000", 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0019641749999999999 2024-04-12T11:36:59.233 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.948652+0000", 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016994199999999999 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.948915+0000", 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000263486 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.951462+0000", 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0025464250000000002 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:08.951978+0000", 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00051643800000000003 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:09.209750+0000", 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.257771584 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:09.209813+0000", 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2843999999999997e-05 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.235 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:09.209863+0000", 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0767999999999999e-05 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:86009 6.2 6:44ba1091:::benchmark_data_smithi029_138569_object86008:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:10.564705+0000", 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 168.601227655, 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.374530211, 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.236 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86009 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564705+0000", 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.237 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564705+0000", 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564708+0000", 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6290000000000002e-06 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564752+0000", 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4400000000000002e-05 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.238 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564782+0000", 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.0159e-05 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564798+0000", 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5739999999999998e-05 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564859+0000", 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1538000000000006e-05 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.565019+0000", 2024-04-12T11:36:59.239 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000159075 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.565189+0000", 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017095899999999999 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.566932+0000", 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001742221 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.240 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.568999+0000", 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0020677600000000001 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.939086+0000", 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.370086839 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.939167+0000", 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0799999999999999e-05 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.939235+0000", 2024-04-12T11:36:59.241 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8090999999999996e-05 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:100270 6.2 6:427fccc1:::benchmark_data_smithi029_138569_object100269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:27.623979+0000", 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 151.54195371500001, 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.45260584799999998, 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.242 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 100270 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.623979+0000", 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.623979+0000", 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.243 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.623981+0000", 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4509999999999999e-06 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624035+0000", 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3378999999999998e-05 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624060+0000", 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5372e-05 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.244 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624074+0000", 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3611000000000001e-05 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624139+0000", 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5037999999999996e-05 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624262+0000", 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012352500000000001 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:36:59.245 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624361+0000", 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8776000000000004e-05 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.625898+0000", 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0015374049999999999 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:28.068661+0000", 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.44276276199999998 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.246 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:28.076436+0000", 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0077744729999999996 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:28.076524+0000", 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8150000000000001e-05 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:28.076585+0000", 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0906000000000003e-05 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.247 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:103429 6.2 6:445c7116:::benchmark_data_smithi029_138569_object103428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:31.826607+0000", 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 147.33932535299999, 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.24315764400000001, 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 103429 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.248 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.826607+0000", 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.826607+0000", 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.826608+0000", 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6000000000000003e-07 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.826665+0000", 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6879999999999998e-05 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.826675+0000", 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9350000000000006e-06 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.826681+0000", 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9920000000000004e-06 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.250 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.826756+0000", 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5207e-05 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.826881+0000", 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012520200000000001 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.827022+0000", 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014028399999999999 2024-04-12T11:36:59.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.828773+0000", 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0017509870000000001 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:31.829211+0000", 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00043874000000000002 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:32.069608+0000", 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.24039713200000001 2024-04-12T11:36:59.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:32.069714+0000", 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000105157 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:32.069765+0000", 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1368e-05 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:104581 6.e 6:7bc98b91:::benchmark_data_smithi029_138569_object104580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:33.512510+0000", 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 145.65342214500001, 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.65829958200000005, 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 104581 2024-04-12T11:36:59.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512510+0000", 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512510+0000", 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.255 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512512+0000", 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.113e-06 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512558+0000", 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6186999999999997e-05 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512574+0000", 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6209e-05 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.256 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512584+0000", 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7079999999999992e-06 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512748+0000", 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016473099999999999 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512895+0000", 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014678900000000001 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.513040+0000", 2024-04-12T11:36:59.257 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014458400000000001 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.516481+0000", 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034407140000000001 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.516562+0000", 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1169999999999997e-05 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.258 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170688+0000", 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.65412592700000005 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170757+0000", 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9015000000000002e-05 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170810+0000", 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3434999999999999e-05 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.259 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:104586 6.e 6:7eb2d00d:::benchmark_data_smithi029_138569_object104585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:33.664460+0000", 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 145.501472191, 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50644868300000001, 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 104586 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.260 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664460+0000", 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664460+0000", 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664462+0000", 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1969999999999999e-06 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664536+0000", 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4225000000000002e-05 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664556+0000", 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9970999999999999e-05 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664565+0000", 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1290000000000002e-06 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.262 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664627+0000", 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1921999999999994e-05 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664780+0000", 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000153327 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664948+0000", 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000167553 2024-04-12T11:36:59.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.671103+0000", 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0061548640000000003 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.671158+0000", 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5615999999999997e-05 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170846+0000", 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.49968758699999999 2024-04-12T11:36:59.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170882+0000", 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6291000000000002e-05 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170909+0000", 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.7001000000000001e-05 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:104587 6.e 6:714ec73f:::benchmark_data_smithi029_138569_object104586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:33.664586+0000", 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 145.501346536, 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50648105600000004, 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 104587 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.266 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664586+0000", 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664586+0000", 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664586+0000", 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9200000000000002e-07 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664613+0000", 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6037000000000002e-05 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664619+0000", 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3849999999999999e-06 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664623+0000", 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0260000000000001e-06 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.268 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.667984+0000", 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0033609600000000001 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.668098+0000", 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011424799999999999 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.668314+0000", 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021553699999999999 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.671214+0000", 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0029004730000000002 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.671333+0000", 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000119077 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170958+0000", 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.49962431400000001 2024-04-12T11:36:59.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.171014+0000", 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6456999999999998e-05 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.171067+0000", 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3149999999999998e-05 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.271 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:104603 6.e 6:7fbcb5a1:::benchmark_data_smithi029_138569_object104602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:33.820992+0000", 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 145.34494023799999, 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.350155944, 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 104603 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.820992+0000", 2024-04-12T11:36:59.272 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.820992+0000", 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.820994+0000", 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.407e-06 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.821053+0000", 2024-04-12T11:36:59.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9240000000000002e-05 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.821067+0000", 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3719e-05 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.821074+0000", 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5730000000000002e-06 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.821126+0000", 2024-04-12T11:36:59.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1836000000000003e-05 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.821279+0000", 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015275600000000001 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.821496+0000", 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000217149 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.275 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.828487+0000", 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0069907299999999997 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.830766+0000", 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002279583 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.171094+0000", 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34032732399999999 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.276 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.171124+0000", 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.0239e-05 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.171148+0000", 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4388000000000001e-05 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:104607 6.e 6:77e2827d:::benchmark_data_smithi029_138569_object104606:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:33.825031+0000", 2024-04-12T11:36:59.277 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 145.34090191999999, 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.35368556099999998, 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 104607 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.825031+0000", 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.825031+0000", 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.825032+0000", 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7799999999999999e-06 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.825072+0000", 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9279999999999997e-05 2024-04-12T11:36:59.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.825090+0000", 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8178000000000001e-05 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.825099+0000", 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1139999999999995e-06 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.825184+0000", 2024-04-12T11:36:59.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5344000000000001e-05 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.825357+0000", 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017217799999999999 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.825531+0000", 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017429500000000001 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.828623+0000", 2024-04-12T11:36:59.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030925459999999998 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.830886+0000", 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0022626980000000001 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.178596+0000", 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34771033899999998 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.178661+0000", 2024-04-12T11:36:59.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4284999999999999e-05 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.178716+0000", 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5523999999999999e-05 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:209193 6.e 6:70898020:::benchmark_data_smithi029_138569_object102368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1124)", 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:38.233044+0000", 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 20.932888460000001, 2024-04-12T11:36:59.283 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.24246484300000001, 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 209193 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.233044+0000", 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.284 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.233044+0000", 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.233045+0000", 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.207e-06 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.233051+0000", 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5860000000000004e-06 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.233062+0000", 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1069e-05 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.233071+0000", 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4469999999999995e-06 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.233127+0000", 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5328000000000003e-05 2024-04-12T11:36:59.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.233280+0000", 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000153089 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.233413+0000", 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013336700000000001 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.236360+0000", 2024-04-12T11:36:59.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0029471229999999998 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.236464+0000", 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000103495 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.475374+0000", 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.23891049 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.288 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.475442+0000", 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7365000000000003e-05 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:38.475509+0000", 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7277e-05 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:36:59.289 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:36:59.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:36:59.313 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:59.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:36:59.415 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:36:59.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:59.439 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:59.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:36:59.473 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:36:59 smithi029 ceph-mon[97787]: pgmap v1475: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:36:59.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:36:59 smithi029 ceph-mon[93650]: pgmap v1475: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:36:59.540 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:36:59.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:36:59.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:59.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:36:59.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:36:59.689 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:59.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:59.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:36:59.791 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:59.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:59.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:59.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:59.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:36:59.941 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:36:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:36:59.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:36:59.951 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-04-12T11:36:59.955 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:37:00.042 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:00.066 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:00.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:00.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:00.155 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:37:00.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:00.191 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:00.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:00.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:00.293 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:00.317 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:00.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:00.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:00.419 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:00.443 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:00.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:00.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:00.545 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:00.569 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:00.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:00.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:00.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:00.695 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:00.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:00.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:00.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:00.823 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:00.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:00.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:00.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:00.950 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:00.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:01.054 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:01.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:01 smithi029 ceph-mon[97787]: pgmap v1476: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:01.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:01 smithi029 ceph-mon[93650]: pgmap v1476: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:01.079 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:01.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:01.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:01.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:01.205 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:01.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:01.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:01.300 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:01 smithi087 ceph-mon[89014]: pgmap v1476: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:01.307 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:01.332 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:01.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:01.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:01.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:01.458 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:01.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:01.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:01.559 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:01.584 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:01.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:01.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:01.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:01.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:01.710 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:01.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:01.782 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:01.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:37:01.811 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:01.827 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:37:01.827 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:37:01.827 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T11:37:01.828 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:37:01.835 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:01.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:01.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 219 (0%) 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:avg 27.375 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 5.33708 -> 5.33708 (expected baseline 4.89419) 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 17 -> 17 pgs (0.621005 -> 0.621005 * mean) 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 33 -> 33 pgs (1.20548 -> 1.20548 * mean) 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 3 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0022 2024-04-12T11:37:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0215 2024-04-12T11:37:01.843 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2024-04-12T11:37:01.937 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:01.960 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:01.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:01.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:02.052 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_2' created 2024-04-12T11:37:02.061 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:02.084 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:02.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:02.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:02.186 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:02.209 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:02.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:02.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:02.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:02 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4002316755' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-12T11:37:02.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:02 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-12T11:37:02.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:02 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4002316755' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-12T11:37:02.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:02 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-12T11:37:02.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:02 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4002316755' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-12T11:37:02.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:02 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-12T11:37:02.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:02.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:02.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:02.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:02.437 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:02.457 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-04-12T11:37:02.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:02.462 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:02.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:02.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:02.588 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:02.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:02.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:02.662 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:37:02.690 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:02.713 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:02.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:02.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:37:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:37:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:37:02.816 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:02.840 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:02.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:02.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:02.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:02.967 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:02.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:02.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:03.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:03.092 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:03.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:03.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:03.194 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:03.220 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:03.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:03.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:03 smithi029 ceph-mon[93650]: pgmap v1477: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:03 smithi029 ceph-mon[93650]: from='client.34911 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:37:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:03 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-12T11:37:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:03 smithi029 ceph-mon[93650]: osdmap e1133: 8 total, 8 up, 8 in 2024-04-12T11:37:03.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:03 smithi029 ceph-mon[97787]: pgmap v1477: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:03.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:03 smithi029 ceph-mon[97787]: from='client.34911 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:37:03.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:03 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-12T11:37:03.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:03 smithi029 ceph-mon[97787]: osdmap e1133: 8 total, 8 up, 8 in 2024-04-12T11:37:03.320 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-04-12T11:37:03.322 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:03.346 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:03.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:03.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:03.448 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:03 smithi087 ceph-mon[89014]: pgmap v1477: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:03 smithi087 ceph-mon[89014]: from='client.34911 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:37:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:03 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-12T11:37:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:03 smithi087 ceph-mon[89014]: osdmap e1133: 8 total, 8 up, 8 in 2024-04-12T11:37:03.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:03.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:03.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:03.573 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:03.597 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:03.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:03.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:03.699 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:03.723 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:03.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:03.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:03.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:03.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:03.849 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:03.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:03.950 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:03.974 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:03.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:03.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:04.076 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:04.099 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:04.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:04.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:04.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:04 smithi029 ceph-mon[97787]: osdmap e1134: 8 total, 8 up, 8 in 2024-04-12T11:37:04.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:04 smithi029 ceph-mon[93650]: osdmap e1134: 8 total, 8 up, 8 in 2024-04-12T11:37:04.201 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:04.226 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:04.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:04.278 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:37:04.278 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:37:04.278 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:37:04.278 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:37:04.278 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:37:04.287 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:04 smithi087 ceph-mon[89014]: osdmap e1134: 8 total, 8 up, 8 in 2024-04-12T11:37:04.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:04.352 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:04.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:04.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:04.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:04.477 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:04.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:04.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:04.578 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:04.602 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:04.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:04.704 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:04.728 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:04.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:04.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:04.830 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:04.853 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:04.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:04.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:04.895 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-04-12T11:37:04.956 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:04.980 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:04.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:04.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:05.082 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:05.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:05 smithi029 ceph-mon[93650]: pgmap v1480: 89 pgs: 8 creating+peering, 7 unknown, 74 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:05.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:05 smithi029 ceph-mon[93650]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:37:05.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:05 smithi029 ceph-mon[93650]: osdmap e1135: 8 total, 8 up, 8 in 2024-04-12T11:37:05.542 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:05 smithi029 ceph-mon[97787]: pgmap v1480: 89 pgs: 8 creating+peering, 7 unknown, 74 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:05.542 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:05 smithi029 ceph-mon[97787]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:37:05.542 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:05 smithi029 ceph-mon[97787]: osdmap e1135: 8 total, 8 up, 8 in 2024-04-12T11:37:05.542 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:37:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:37:04] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.43.0" 2024-04-12T11:37:05.543 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:37:05.543 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:37:05.543 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:37:05.543 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:05.543 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:37:05.544 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:05.544 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:37:05.544 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:37:05.544 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:37:05.545 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:37:05.545 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:37:05.545 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:37:05.545 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:37:05.545 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:37:05.545 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:37:05.545 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:37:05.545 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:37:05.545 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:37:05.546 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:37:05.546 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:05 smithi087 ceph-mon[89014]: pgmap v1480: 89 pgs: 8 creating+peering, 7 unknown, 74 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:05.546 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:05 smithi087 ceph-mon[89014]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:37:05.546 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:05 smithi087 ceph-mon[89014]: osdmap e1135: 8 total, 8 up, 8 in 2024-04-12T11:37:05.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:05.570 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:05.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:05.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:05.697 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:05.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:05.798 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:05.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:05.823 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:05.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:05.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:05.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:05.950 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:05.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:06.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:06.075 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:06.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:06.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:06.177 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:06.201 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:06.202 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2224830721' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:37:06.202 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2224830721' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:37:06.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:06.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:06.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:06.328 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:06.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:06.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:06.394 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2224830721' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:37:06.430 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:06.454 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:06.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:06.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:06.556 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:06.579 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:06.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:06.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:06.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:06.704 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:06.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:06.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:06.793 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:37:06.793 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:37:06.807 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:06.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:06.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:06.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:06.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:06.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:06.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:06.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:07.057 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:07.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:07.081 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:07.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:07.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:07.205 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:07.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:07.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:07.307 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:07.319 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:07 smithi029 ceph-mon[97787]: pgmap v1482: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:07.319 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:07.319 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:07.319 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:07 smithi029 ceph-mon[93650]: pgmap v1482: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:07.319 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:07.319 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:07.330 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:07 smithi087 ceph-mon[89014]: pgmap v1482: 89 pgs: 8 creating+peering, 81 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:07.330 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:07.330 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:07.331 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:07.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:07.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:07.351 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-04-12T11:37:07.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:07.456 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:07.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:07.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:07.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:07.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:07.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:07.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:07.595 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:37:07.684 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:07.707 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:07.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:07.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:07.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:07.833 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:07.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:07.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:07.935 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:07.960 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:07.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:07.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:08.062 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:08.086 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:08.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:08.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:08.188 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:08.212 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:08.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:08.223 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:37:08.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:08.338 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:08.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:08.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:08.439 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:08.463 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:08.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:08.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:08.565 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:08.589 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:08.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:08.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:08.691 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:08.714 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:08.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:08.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:08.816 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:08.839 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:08.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:08.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:08.941 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:08.964 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:08.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:08.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:09.065 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:37:09.066 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:09.089 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:09.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:09.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:09.091 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:37:09.091 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:37:09.110 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:09 smithi029 ceph-mon[97787]: pgmap v1483: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:09.110 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1569154466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:37:09.110 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:09 smithi029 ceph-mon[93650]: pgmap v1483: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:09.110 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1569154466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:37:09.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:09.215 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:09.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:09.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:09.317 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:09.341 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:09.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:09.394 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:09 smithi087 ceph-mon[89014]: pgmap v1483: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:09.394 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1569154466' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:37:09.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:09.467 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:09.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:09.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:09.569 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:09.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:09.594 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:09.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:09.695 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:09.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-04-12T11:37:09.718 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:09.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:09.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:09.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:09.845 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:09.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:09.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:09.918 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:37:09.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 0 1 2024-04-12T11:37:09.919 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:37:09.947 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:09.971 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:09.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:09.983 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:10.073 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:10.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:10.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:10.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:10.198 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:10.221 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:10.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:10.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:10.248 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:10 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:37:10.248 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1569154466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:37:10.248 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:10 smithi029 ceph-mon[97787]: osdmap e1136: 8 total, 8 up, 8 in 2024-04-12T11:37:10.248 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:10 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:37:10.248 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1569154466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:37:10.249 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:10 smithi029 ceph-mon[93650]: osdmap e1136: 8 total, 8 up, 8 in 2024-04-12T11:37:10.323 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:10.346 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:10.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:10.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:10.383 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:10 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:37:10.383 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1569154466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:37:10.383 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:10 smithi087 ceph-mon[89014]: osdmap e1136: 8 total, 8 up, 8 in 2024-04-12T11:37:10.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:10.471 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:10.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:10.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:10.573 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:10.596 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:10.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:10.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:10.697 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:10.723 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:10.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:10.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:10.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:10.849 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:10.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:10.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:10.950 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:10.974 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:10.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:10.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:11.075 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-04-12T11:37:11.077 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:11.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:11.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:11.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:11.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:11.227 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:11.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:11.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:11.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:11 smithi029 ceph-mon[93650]: pgmap v1485: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:11.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:11 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2761336921' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:37:11.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:11 smithi029 ceph-mon[97787]: pgmap v1485: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:11.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:11 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2761336921' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:37:11.312 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:11 smithi087 ceph-mon[89014]: pgmap v1485: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:11.312 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:11 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2761336921' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:37:11.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:11.353 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:11.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:11.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:11.455 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:11.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:11.482 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:11.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:11.548 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:37:11.548 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:37:11.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:11.607 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:11.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:11.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:11.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:11.733 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:11.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:11.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:11.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:37:11.835 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:11.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:11.867 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:11.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:11.968 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:11.994 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:11.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:11.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:12.089 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:37:12.089 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:37:12.089 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:37:12.089 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:37:12.089 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:37:12.089 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:37:12.090 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:37:12.090 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:37:12.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:37:12.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:37:12.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:37:12.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:37:12.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:37:12.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:37:12.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:37:12.091 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:37:12.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:12.120 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:12.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:12.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:12.222 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:12.246 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:12.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:12.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:12.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:12.363 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:37:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:37:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:37:12.363 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:12 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2761336921' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:37:12.363 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:12 smithi087 ceph-mon[89014]: osdmap e1137: 8 total, 8 up, 8 in 2024-04-12T11:37:12.364 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:12 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3920440709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:37:12.373 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:12.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:12.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:12.388 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-04-12T11:37:12.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:12.497 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:12 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2761336921' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:37:12.497 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:12 smithi029 ceph-mon[97787]: osdmap e1137: 8 total, 8 up, 8 in 2024-04-12T11:37:12.497 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:12 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3920440709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:37:12.497 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:12 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2761336921' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:37:12.497 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:12 smithi029 ceph-mon[93650]: osdmap e1137: 8 total, 8 up, 8 in 2024-04-12T11:37:12.497 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:12 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3920440709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:37:12.501 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:12.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:12.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:12.598 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:37:12.604 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:12.628 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:12.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:12.730 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:12.753 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:12.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:12.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:12.855 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:12.878 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:12.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:12.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:12.980 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:13.004 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:13.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:13.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:13.106 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:13.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:13.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:13.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:13.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:13.256 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:13.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:13.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:13.358 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:13.382 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:13.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:13.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:13.417 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:13 smithi029 ceph-mon[97787]: pgmap v1487: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:13.417 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:13 smithi029 ceph-mon[97787]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:37:13.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:13 smithi029 ceph-mon[93650]: pgmap v1487: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:13.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:13 smithi029 ceph-mon[93650]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:37:13.424 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:13.461 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:13.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:13 smithi087 ceph-mon[89014]: pgmap v1487: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:13.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:13 smithi087 ceph-mon[89014]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:37:13.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:13.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:13.508 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:13.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:13.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:13.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:13.632 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:13.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:13.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:13.756 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:13.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:13.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:13.857 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:13.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:13.882 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:13.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:13.983 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:14.005 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:14.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:14.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:14.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:14.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:14.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:14.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:14.213 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:37:14.213 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:37:14.213 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:37:14.213 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:37:14.213 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.213 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:60694 6.c e1091/1086 6:37dbcab9:::benchmark_data_smithi029_138569_object60693:head v 1091'3786, mlcod=1091'3783)", 2024-04-12T11:37:14.213 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:38.250701+0000", 2024-04-12T11:37:14.213 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 215.96039431200001, 2024-04-12T11:37:14.213 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.045178997999999998, 2024-04-12T11:37:14.213 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.250701+0000", 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.250701+0000", 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.250700+0000", 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.292400+0000", 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.041699845999999999 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.292419+0000", 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9477999999999999e-05 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.292427+0000", 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2230000000000001e-06 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.292500+0000", 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3726000000000005e-05 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.292520+0000", 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9262999999999998e-05 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.295807+0000", 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0032874290000000001 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.295880+0000", 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2614000000000006e-05 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:60696 6.1 e1091/1086 6:8bf4e70d:::benchmark_data_smithi029_138569_object60695:head v 1091'3666, mlcod=1091'3663)", 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:38.257845+0000", 2024-04-12T11:37:14.217 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 215.953249715, 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.022158885999999999, 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.257845+0000", 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.257845+0000", 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.257843+0000", 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999971 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.277153+0000", 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.019310227999999999 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.277169+0000", 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5784e-05 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.219 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.277176+0000", 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2339999999999997e-06 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.277227+0000", 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0816000000000003e-05 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.277248+0000", 2024-04-12T11:37:14.220 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1364000000000001e-05 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.279964+0000", 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027161450000000001 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:38.280004+0000", 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0040999999999998e-05 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:62686 6.c e1092/1086 6:3c3ec0df:::benchmark_data_smithi029_138569_object62685:head v 1092'3895, mlcod=1092'3892)", 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:40.117275+0000", 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 214.09382021600001, 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.017772112999999999, 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:40.117275+0000", 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.222 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:40.117275+0000", 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:40.117274+0000", 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:40.130499+0000", 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.013225196 2024-04-12T11:37:14.223 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:40.130519+0000", 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0514e-05 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:40.130529+0000", 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0510000000000002e-06 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:40.130578+0000", 2024-04-12T11:37:14.224 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9758e-05 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:40.130600+0000", 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1948999999999999e-05 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:40.134708+0000", 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0041078809999999999 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:40.135047+0000", 2024-04-12T11:37:14.225 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000338804 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:74642 6.5 e1098/1093 6:a3301385:::benchmark_data_smithi029_138569_object74641:head v 1098'4517, mlcod=1098'4514)", 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:53.054131+0000", 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 201.15696417199999, 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.022447485, 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.226 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.054131+0000", 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.054131+0000", 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.227 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.054130+0000", 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.054157+0000", 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6488000000000001e-05 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.054168+0000", 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1824000000000001e-05 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.228 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.054176+0000", 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.7209999999999994e-06 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.054212+0000", 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5484000000000001e-05 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.054241+0000", 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.9303999999999999e-05 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.229 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.076536+0000", 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.022295315999999999 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.076578+0000", 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2148999999999999e-05 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.230 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:74649 6.15 e1098/1093 6:abe5830b:::benchmark_data_smithi029_138569_object74648:head v 1098'4375, mlcod=1097'4373)", 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:53.057122+0000", 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 201.15397350800001, 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.019545208000000001, 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.057122+0000", 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.057122+0000", 2024-04-12T11:37:14.231 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.057121+0000", 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.057147+0000", 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6003000000000002e-05 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.057156+0000", 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7070000000000004e-06 2024-04-12T11:37:14.232 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.057162+0000", 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5709999999999997e-06 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.057200+0000", 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.7985999999999998e-05 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.233 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.057233+0000", 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.2938999999999998e-05 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.076583+0000", 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.019349900999999999 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.076667+0000", 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3901999999999997e-05 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.234 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:75572 6.5 e1101/1093 6:a50daeb3:::benchmark_data_smithi029_138569_object75571:head v 1101'5378, mlcod=1101'5376)", 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:56.156043+0000", 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 198.05505218299999, 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.017114977999999999, 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:56.156043+0000", 2024-04-12T11:37:14.235 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:56.156043+0000", 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:56.156042+0000", 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.236 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:56.156068+0000", 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5843e-05 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:56.156081+0000", 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3113e-05 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:56.156089+0000", 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6699999999999994e-06 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.237 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:56.156135+0000", 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6292000000000003e-05 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:56.156164+0000", 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.8997e-05 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:56.173132+0000", 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.016967949 2024-04-12T11:37:14.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:56.173158+0000", 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6016999999999998e-05 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:105764 6.13 e1107/1102 6:cb7def67:::benchmark_data_smithi029_138569_object105763:head v 1107'5423, mlcod=0'0)", 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:35.441085+0000", 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 158.770010423, 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014661495, 2024-04-12T11:37:14.239 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.240 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.240 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.240 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.240 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.240 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:35.441085+0000", 2024-04-12T11:37:14.240 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.240 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.240 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.240 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.240 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:35.441085+0000", 2024-04-12T11:37:14.240 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:14.242 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.242 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.242 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.242 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.242 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:35.441084+0000", 2024-04-12T11:37:14.242 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.242 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.242 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.242 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:35.452925+0000", 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.011841057 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:35.452972+0000", 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7268999999999998e-05 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:35.452980+0000", 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.7729999999999999e-06 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.243 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:35.453021+0000", 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1601999999999999e-05 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:35.453048+0000", 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6831000000000001e-05 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:35.455704+0000", 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002655339 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:35.455746+0000", 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2571999999999998e-05 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:179147 6.15 e1122/1093 6:a8e3d397:::benchmark_data_smithi029_138569_object72322:head v 1122'7693, mlcod=1122'7690)", 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:03.438236+0000", 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 70.772858744999994, 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014855101000000001, 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.245 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438236+0000", 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438236+0000", 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.246 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438235+0000", 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438243+0000", 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.242e-06 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438257+0000", 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3477e-05 2024-04-12T11:37:14.247 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438265+0000", 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4290000000000001e-06 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438338+0000", 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2766000000000001e-05 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438360+0000", 2024-04-12T11:37:14.248 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.2093999999999999e-05 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.453041+0000", 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014680281999999999 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.453091+0000", 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0838999999999999e-05 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.249 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:179149 6.a e1122/1086 6:5c5d74e5:::benchmark_data_smithi029_138569_object72324:head v 1122'11194, mlcod=1122'11192)", 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:03.438256+0000", 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 70.772839042000001, 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.016638408, 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438256+0000", 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438256+0000", 2024-04-12T11:37:14.250 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438255+0000", 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438263+0000", 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9019999999999999e-06 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438274+0000", 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1147e-05 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438281+0000", 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2949999999999996e-06 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438336+0000", 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4489e-05 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.438357+0000", 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0877e-05 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.454823+0000", 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.016465732 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:03.454894+0000", 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1829999999999995e-05 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.253 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:183767 6.a e1123/1086 6:5a973ac2:::benchmark_data_smithi029_138569_object76942:head v 1123'11483, mlcod=1123'11480)", 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:09.052795+0000", 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 65.158299513000003, 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.017557776000000001, 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:09.052795+0000", 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.254 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:09.052795+0000", 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:09.052795+0000", 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:09.052802+0000", 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6539999999999999e-06 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:09.052817+0000", 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5082000000000001e-05 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:09.052825+0000", 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6340000000000001e-06 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:09.052881+0000", 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6236000000000002e-05 2024-04-12T11:37:14.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:09.052901+0000", 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9582e-05 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:09.070279+0000", 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.017377841000000002 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.257 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:09.070353+0000", 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4616000000000005e-05 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:184723 6.a e1123/1086 6:5837c0a3:::benchmark_data_smithi029_138569_object77898:head v 1123'11544, mlcod=1123'11541)", 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:10.220138+0000", 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 63.990957479999999, 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015679667000000001, 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.258 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:10.220138+0000", 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:10.220138+0000", 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:10.220137+0000", 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:10.220144+0000", 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0770000000000002e-06 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:10.220154+0000", 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0669000000000001e-05 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:10.220161+0000", 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6189999999999996e-06 2024-04-12T11:37:14.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:10.220224+0000", 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2873999999999995e-05 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:10.220243+0000", 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8805e-05 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:10.235665+0000", 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015421851 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:10.235817+0000", 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015262899999999999 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:186430 6.a e1123/1086 6:5f628dff:::benchmark_data_smithi029_138569_object79605:head v 1123'11651, mlcod=1123'11648)", 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:12.148682+0000", 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 62.062412944999998, 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015921867999999999, 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.262 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:12.148682+0000", 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:12.148682+0000", 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:12.148681+0000", 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:12.148689+0000", 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6920000000000002e-06 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:12.148704+0000", 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4905999999999999e-05 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:12.148711+0000", 2024-04-12T11:37:14.264 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4240000000000004e-06 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:12.148771+0000", 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9305000000000002e-05 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:12.148796+0000", 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5071000000000001e-05 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:12.164121+0000", 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015324924 2024-04-12T11:37:14.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:12.164604+0000", 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00048336599999999999 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:191197 6.a e1123/1086 6:5740926a:::benchmark_data_smithi029_138569_object84372:head v 1123'11957, mlcod=1123'11955)", 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:17.436156+0000", 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 56.774939107999998, 2024-04-12T11:37:14.266 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015532377, 2024-04-12T11:37:14.267 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.267 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.267 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.267 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:17.436156+0000", 2024-04-12T11:37:14.267 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.267 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:17.436156+0000", 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:17.436155+0000", 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:17.436163+0000", 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2779999999999996e-06 2024-04-12T11:37:14.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:17.436175+0000", 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2751999999999999e-05 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:17.436182+0000", 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8390000000000001e-06 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.269 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:17.436248+0000", 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5874e-05 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:17.436270+0000", 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1625999999999999e-05 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:17.451581+0000", 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015311526000000001 2024-04-12T11:37:14.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:17.451688+0000", 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000107157 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:196708 6.c e1124/1086 6:3431a65f:::benchmark_data_smithi029_138569_object89883:head v 1124'12122, mlcod=1124'12117)", 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:23.700189+0000", 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 50.510906273000003, 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014677010000000001, 2024-04-12T11:37:14.271 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.700189+0000", 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.700189+0000", 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.700188+0000", 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.272 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.700197+0000", 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2650000000000002e-06 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.700212+0000", 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4809e-05 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.700225+0000", 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2508e-05 2024-04-12T11:37:14.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.703585+0000", 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0033601770000000002 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.703608+0000", 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3450000000000001e-05 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.714724+0000", 2024-04-12T11:37:14.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.011115869 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.714866+0000", 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014172300000000001 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:196713 6.c e1124/1086 6:38d4e788:::benchmark_data_smithi029_138569_object89888:head v 1124'12124, mlcod=1124'12118)", 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:23.710112+0000", 2024-04-12T11:37:14.275 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 50.500983419999997, 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014935789, 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.710112+0000", 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.710112+0000", 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.710111+0000", 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.710122+0000", 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1141999999999999e-05 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.710133+0000", 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1054e-05 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.710140+0000", 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1099999999999997e-06 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.711579+0000", 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0014393590000000001 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.711602+0000", 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3152000000000001e-05 2024-04-12T11:37:14.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.279 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.279 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.279 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.724961+0000", 2024-04-12T11:37:14.279 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.013358339 2024-04-12T11:37:14.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.279 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.279 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.279 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:23.725047+0000", 2024-04-12T11:37:14.279 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6645999999999997e-05 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:199735 6.a e1124/1086 6:54668364:::benchmark_data_smithi029_138569_object92910:head v 1124'12493, mlcod=1124'12490)", 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:27.213946+0000", 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 46.997149164, 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.016796220000000001, 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.280 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.213946+0000", 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.213946+0000", 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.213945+0000", 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.213953+0000", 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3010000000000007e-06 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.213968+0000", 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4358999999999999e-05 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.213976+0000", 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1019999999999997e-06 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.214025+0000", 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9184000000000002e-05 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.214046+0000", 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0917e-05 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.230709+0000", 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.016662999000000001 2024-04-12T11:37:14.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.230742+0000", 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3291999999999999e-05 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:199838 6.5 e1124/1093 6:a37b460d:::benchmark_data_smithi029_138569_object93013:head v 1124'9501, mlcod=1124'9498)", 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:27.310699+0000", 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 46.900395584000002, 2024-04-12T11:37:14.284 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014660136000000001, 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.310699+0000", 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.310699+0000", 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.310699+0000", 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.310706+0000", 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.481e-06 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.310721+0000", 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4895000000000001e-05 2024-04-12T11:37:14.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.310729+0000", 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.7060000000000004e-06 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.310787+0000", 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8254000000000002e-05 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.310805+0000", 2024-04-12T11:37:14.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8363e-05 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.325316+0000", 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014510199 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:27.325360+0000", 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3993999999999999e-05 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:204701 6.a e1124/1086 6:57f6a803:::benchmark_data_smithi029_138569_object97876:head v 1124'12810, mlcod=1124'12806)", 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:32.708812+0000", 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 41.502283237, 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.016496884999999999, 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:32.708812+0000", 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.289 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:32.708812+0000", 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:32.708811+0000", 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:32.708818+0000", 2024-04-12T11:37:14.290 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3250000000000002e-06 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:32.708832+0000", 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3263e-05 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:32.708838+0000", 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8419999999999999e-06 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:32.711940+0000", 2024-04-12T11:37:14.291 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003101548 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:32.711959+0000", 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9296e-05 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:32.725236+0000", 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.013276908 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:32.725309+0000", 2024-04-12T11:37:14.292 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2447000000000006e-05 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:206365 6.a e1124/1086 6:5961b6ce:::benchmark_data_smithi029_138569_object99540:head v 1124'12922, mlcod=1124'12920)", 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:34.692337+0000", 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 39.518758130000002, 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015152437, 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.293 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:34.692337+0000", 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:34.692337+0000", 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:34.692336+0000", 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:34.692345+0000", 2024-04-12T11:37:14.294 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3540000000000006e-06 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:34.692361+0000", 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5891000000000001e-05 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:34.692369+0000", 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0620000000000001e-06 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:34.692419+0000", 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9778999999999999e-05 2024-04-12T11:37:14.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:34.692442+0000", 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.2751999999999998e-05 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:34.707442+0000", 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015000517999999999 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:34.707489+0000", 2024-04-12T11:37:14.296 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7188000000000003e-05 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.34776.0:211720 6.a e1124/1086 6:56917869:::benchmark_data_smithi029_138569_object104895:head v 1124'13280, mlcod=1124'13277)", 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:36:41.489338+0000", 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 32.721756911, 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.016724662000000001, 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.297 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:41.489338+0000", 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:41.489338+0000", 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:41.489337+0000", 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:14.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:41.489345+0000", 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1740000000000003e-06 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:41.489358+0000", 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3241000000000001e-05 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:41.489365+0000", 2024-04-12T11:37:14.299 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9310000000000002e-06 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:41.489528+0000", 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016292300000000001 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:41.489575+0000", 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7491999999999999e-05 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:41.505981+0000", 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.016406069999999998 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:14.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:14.301 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:14.301 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:36:41.506063+0000", 2024-04-12T11:37:14.301 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1570000000000007e-05 2024-04-12T11:37:14.301 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.301 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.301 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.301 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:14.301 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:14.301 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:37:14.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:14.325 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:14.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:14.426 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:14.447 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-04-12T11:37:14.449 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-12T11:37:14.449 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-12T11:37:14.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:14.450 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:14.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:14.498 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2024-04-12T11:37:14.498 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-12T11:37:14.498 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_161157 2024-04-12T11:37:14.499 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:37:14.499 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2024-04-12T11:37:14.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:14.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:14.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:14.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:14.687 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:14.720 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:14.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:14.822 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:14.856 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:14.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:14.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:14.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:15.001 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:15.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:15.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:15.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:37:15.103 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:15.134 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:15.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:15.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:15.185 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-04-12T11:37:15.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:15 smithi029 ceph-mon[93650]: pgmap v1488: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:37:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:37:14] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.43.0" 2024-04-12T11:37:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:15 smithi029 ceph-mon[97787]: pgmap v1488: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:15.283 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:15.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:15.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:15.384 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:15.431 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:15.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:15.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:15.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:15 smithi087 ceph-mon[89014]: pgmap v1488: 89 pgs: 89 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-04-12T11:37:15.493 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:37:15.500 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3313 3297 205.86 206.062 0.00567694 0.0048397 2024-04-12T11:37:15.533 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:15.569 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:15.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:15.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:15.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:15.712 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:15.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:15.813 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:15.853 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:15.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:15.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:15.955 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:16.001 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:16.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:16.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:16.103 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:16.133 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:16.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:16.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:16.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:16.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:16.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:16.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:16.404 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:16.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:16.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:16.500 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 6078 6062 189.336 172.812 0.00417769 0.00527029 2024-04-12T11:37:16.506 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:16.538 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:16.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:16.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:16.634 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:16.640 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:16.670 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:16.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:16.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:16.771 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:16.802 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:16.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:16.904 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:16.936 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:16.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:16.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:17.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:17.077 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:17.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:17.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:17.179 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:17.202 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:17.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:17.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:17.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:17 smithi029 ceph-mon[97787]: pgmap v1489: 89 pgs: 89 active+clean; 7.3 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 864 KiB/s wr, 13 op/s 2024-04-12T11:37:17.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:17 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:17.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:17 smithi029 ceph-mon[93650]: pgmap v1489: 89 pgs: 89 active+clean; 7.3 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 864 KiB/s wr, 13 op/s 2024-04-12T11:37:17.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:17 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:17.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:17.338 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:17.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:17.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:17.388 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:17 smithi087 ceph-mon[89014]: pgmap v1489: 89 pgs: 89 active+clean; 7.3 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 864 KiB/s wr, 13 op/s 2024-04-12T11:37:17.388 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:17 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:17.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:17.474 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:17.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:17.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:17.499 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 8036 8020 167.018 122.375 0.00356497 0.00590133 2024-04-12T11:37:17.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:17.614 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:17.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:17.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:17.716 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:17.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:17.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:17.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:17.841 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:17.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:17.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:17.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:18.002 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:18.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:18.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:18.104 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:18.127 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:18.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:18.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:18.172 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.0 to 1 (10000) 2024-04-12T11:37:18.175 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:37:18.229 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:18.253 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:18.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:18.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:18.354 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:18.388 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:18.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:18.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:18.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:18 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/990527088' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:37:18.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:18 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:37:18.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:18 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3922912109' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T11:37:18.470 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:18 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/990527088' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:37:18.470 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:18 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:37:18.470 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:18 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3922912109' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T11:37:18.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:18 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/990527088' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:37:18.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:18 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:37:18.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:18 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3922912109' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T11:37:18.490 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:18.500 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 9564 9548 149.14 95.5 0.00526879 0.00669514 2024-04-12T11:37:18.522 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:18.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:18.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:18.624 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:18.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:18.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:18.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:18.749 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:18.786 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:18.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:18.887 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:18.920 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:18.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:19.022 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:19.054 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:19.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:19.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:19.156 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:19.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:19.183 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:19.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[93650]: pgmap v1490: 89 pgs: 89 active+clean; 251 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 462 op/s 2024-04-12T11:37:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:37:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:37:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:37:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3922912109' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-04-12T11:37:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[93650]: osdmap e1138: 8 total, 8 up, 8 in 2024-04-12T11:37:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[97787]: pgmap v1490: 89 pgs: 89 active+clean; 251 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 462 op/s 2024-04-12T11:37:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:37:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:37:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:37:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3922912109' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-04-12T11:37:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:19 smithi029 ceph-mon[97787]: osdmap e1138: 8 total, 8 up, 8 in 2024-04-12T11:37:19.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:19.325 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:19.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:19.426 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:19.450 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:19 smithi087 ceph-mon[89014]: pgmap v1490: 89 pgs: 89 active+clean; 251 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 462 op/s 2024-04-12T11:37:19.450 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:19 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:37:19.450 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:19 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:37:19.450 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:19 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:37:19.451 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:19 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3922912109' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-04-12T11:37:19.451 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:19 smithi087 ceph-mon[89014]: osdmap e1138: 8 total, 8 up, 8 in 2024-04-12T11:37:19.454 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:19.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:19.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:19.500 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 11870 11854 148.134 144.125 0.00136393 0.00665695 2024-04-12T11:37:19.556 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:19.580 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:19.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:19.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:19.682 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:19.709 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:37:19.709 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:12174 7.5 7:a6b3daf3:::benchmark_data_smithi029_161157_object12173:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1138)", 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:19.707014+0000", 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.001600628, 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001644822, 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 12174 2024-04-12T11:37:19.710 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707014+0000", 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707014+0000", 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707012+0000", 2024-04-12T11:37:19.711 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:37:19.712 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.712 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.712 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:19.712 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707146+0000", 2024-04-12T11:37:19.712 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013377899999999999 2024-04-12T11:37:19.712 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.712 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.712 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:19.713 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707165+0000", 2024-04-12T11:37:19.713 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.876e-05 2024-04-12T11:37:19.713 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.713 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.713 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:19.713 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707174+0000", 2024-04-12T11:37:19.714 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0890000000000006e-06 2024-04-12T11:37:19.714 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.714 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.714 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:19.714 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707241+0000", 2024-04-12T11:37:19.714 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7041999999999999e-05 2024-04-12T11:37:19.714 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.714 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.715 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:19.715 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707370+0000", 2024-04-12T11:37:19.715 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012900699999999999 2024-04-12T11:37:19.715 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.715 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.715 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:37:19.716 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707524+0000", 2024-04-12T11:37:19.716 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000153516 2024-04-12T11:37:19.716 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.716 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.716 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:19.716 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.708323+0000", 2024-04-12T11:37:19.716 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00079934099999999996 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.44843.0:12172 7.c e1138/1133 7:34748399:::benchmark_data_smithi029_161157_object12171:head v 1138'795, mlcod=1138'793)", 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:19.707768+0000", 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.00084653599999999995, 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0010116330000000001, 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:37:19.717 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707768+0000", 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707768+0000", 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:19.718 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707757+0000", 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999989 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707816+0000", 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8536000000000003e-05 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707832+0000", 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6382999999999999e-05 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.719 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707840+0000", 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4909999999999999e-06 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707899+0000", 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9658999999999999e-05 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.707957+0000", 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7742999999999999e-05 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:19.720 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:19.721 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:19.721 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:19.721 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2024-04-12T11:37:19.721 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 2 2024-04-12T11:37:19.721 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:37:19.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:19.722 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:19.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:19.823 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:19.855 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:19.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:19.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:19.980 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:19.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:20.082 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:20.122 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:20.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:20.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:20.224 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:20.265 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:20.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:20.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:20.367 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:20.401 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:20.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:20.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:20.500 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 13888 13872 144.465 126.125 0.00288226 0.00691514 2024-04-12T11:37:20.503 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:20.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:20.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:20.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:20.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:20.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:20.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:20.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:20.712 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-04-12T11:37:20.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:20.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:20.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:20.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:20.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:20.925 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:20.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:20.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:20.959 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:37:21.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:21.068 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:21.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:21.170 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:21.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:21.198 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:21.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:21.300 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:21.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:21.325 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:21.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:21.426 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:21.430 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:21 smithi087 ceph-mon[89014]: pgmap v1492: 89 pgs: 89 active+clean; 560 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-04-12T11:37:21.466 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:21.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:21.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:21.500 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 14870 14854 132.596 61.375 0.00288039 0.00753021 2024-04-12T11:37:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:21 smithi029 ceph-mon[93650]: pgmap v1492: 89 pgs: 89 active+clean; 560 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-04-12T11:37:21.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:21 smithi029 ceph-mon[97787]: pgmap v1492: 89 pgs: 89 active+clean; 560 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-04-12T11:37:21.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:21.592 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:21.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:21.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:21.716 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:21.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:21.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:21.856 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:21.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:21.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:21.958 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:21.994 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:21.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:21.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:22.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:22.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:22.123 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:22.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:22.224 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:22.248 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:22.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:22.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:22.349 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:22.377 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:37:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:37:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:37:22.379 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:22.379 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:22.379 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-12T11:37:22.379 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:22 smithi087 ceph-mon[89014]: pgmap v1493: 89 pgs: 89 active+clean; 607 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-04-12T11:37:22.385 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:22.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:22.487 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:22.500 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 16220 16204 126.568 84.375 0.00321911 0.00789369 2024-04-12T11:37:22.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:22.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:22.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:22.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:22.653 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:22.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:22.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:22.755 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:22.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:22.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:22.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-12T11:37:22.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:22 smithi029 ceph-mon[97787]: pgmap v1493: 89 pgs: 89 active+clean; 607 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-04-12T11:37:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-12T11:37:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:22 smithi029 ceph-mon[93650]: pgmap v1493: 89 pgs: 89 active+clean; 607 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-04-12T11:37:22.795 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:22.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:22.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:22.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:22.921 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:22.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:22.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:23.022 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:23.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:23.057 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:23.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:23.090 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:37:23.090 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:37:23.091 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:37:23.091 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:37:23.091 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:37:23.158 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:23.194 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:23.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:23.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:23.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:23.324 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:23.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:23.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:23.426 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:23.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:23.462 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:23.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:23.500 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 15 17959 17944 124.587 108.75 0.00638306 0.00802054 2024-04-12T11:37:23.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:23.601 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:23.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:23.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:23.703 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:23.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-12T11:37:23.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:23 smithi087 ceph-mon[89014]: osdmap e1139: 8 total, 8 up, 8 in 2024-04-12T11:37:23.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:23.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:23.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:23.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-12T11:37:23.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:23 smithi029 ceph-mon[97787]: osdmap e1139: 8 total, 8 up, 8 in 2024-04-12T11:37:23.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-12T11:37:23.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:23 smithi029 ceph-mon[93650]: osdmap e1139: 8 total, 8 up, 8 in 2024-04-12T11:37:23.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-04-12T11:37:23.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:23.870 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:23.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:23.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:23.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:24.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:24.011 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:24.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:24.077 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:37:24.113 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:24.155 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:24.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:24.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:24.297 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:24.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:24.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:24.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:24.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:24.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:24.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:24.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:24 smithi087 ceph-mon[89014]: osdmap e1140: 8 total, 8 up, 8 in 2024-04-12T11:37:24.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:24 smithi087 ceph-mon[89014]: pgmap v1496: 89 pgs: 89 active+clean; 889 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-04-12T11:37:24.500 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 20161 20145 125.883 137.562 0.00304802 0.0079374 2024-04-12T11:37:24.540 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:24.579 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:24.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:24.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:24.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:24.719 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:24 smithi029 ceph-mon[97787]: osdmap e1140: 8 total, 8 up, 8 in 2024-04-12T11:37:24.719 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:24 smithi029 ceph-mon[97787]: pgmap v1496: 89 pgs: 89 active+clean; 889 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-04-12T11:37:24.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:24 smithi029 ceph-mon[93650]: osdmap e1140: 8 total, 8 up, 8 in 2024-04-12T11:37:24.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:24 smithi029 ceph-mon[93650]: pgmap v1496: 89 pgs: 89 active+clean; 889 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-04-12T11:37:24.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:24.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:24.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:24.824 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:24.848 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:24.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:24.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:24.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:24.984 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:24.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:25.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:37:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:37:24] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2024-04-12T11:37:25.086 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:25.120 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:25.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:25.222 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:25.255 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:25.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:25.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:25.357 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:25.393 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:25.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:25.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:25.500 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 22371 22355 126.995 138.125 0.00674173 0.0078679 2024-04-12T11:37:25.537 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:25.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:25.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:25.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:25.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:25.677 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:25.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:25.778 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:25.809 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:25.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:25.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:25.911 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:25.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:25.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:25.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:26.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:26.062 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:26.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:26.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:26.164 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:26.201 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:26.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:26.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:26.231 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:65216 6.15 6:a8cb9977:::benchmark_data_smithi029_138569_object65215:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:41.885262+0000", 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 224.34214393799999, 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0513185970000001, 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.232 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 65216 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.885262+0000", 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.885262+0000", 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.233 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.885260+0000", 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.885948+0000", 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00068804900000000004 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.885966+0000", 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7139e-05 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.234 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.885975+0000", 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0589999999999992e-06 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.032997+0000", 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.147022705 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.033007+0000", 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1290000000000002e-06 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.235 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.936486+0000", 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.90347967600000001 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.936580+0000", 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4159000000000004e-05 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.236 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:65217 6.13 6:ce2d3ca1:::benchmark_data_smithi029_138569_object65216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:41.890158+0000", 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 224.337247757, 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.066303131, 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 65217 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.237 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.890158+0000", 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.890158+0000", 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.890156+0000", 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999976 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.238 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.890194+0000", 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8451999999999997e-05 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.890220+0000", 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6166999999999999e-05 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.890231+0000", 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0997e-05 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.028506+0000", 2024-04-12T11:37:26.239 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.13827504099999999 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.028517+0000", 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0771000000000001e-05 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.941001+0000", 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.91248363099999996 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.941130+0000", 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012901500000000001 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.240 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.941281+0000", 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000151068 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.942866+0000", 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0015847529999999999 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.943040+0000", 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017416099999999999 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.241 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.956400+0000", 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.013360001 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.956434+0000", 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4663000000000003e-05 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.956461+0000", 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6641000000000001e-05 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.242 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:74935 6.12 6.a65e7032 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:53.946948+0000", 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 212.280457571, 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.836861521, 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 74935 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.243 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.946948+0000", 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.946948+0000", 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.946947+0000", 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.244 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.947014+0000", 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7148000000000002e-05 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.947026+0000", 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1875000000000001e-05 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.947034+0000", 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0399999999999993e-06 2024-04-12T11:37:26.245 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.947502+0000", 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00046788999999999999 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:53.947511+0000", 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.816e-06 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.783799+0000", 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.83628844499999999 2024-04-12T11:37:26.246 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.783810+0000", 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0402e-05 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:75023 6.12 6.3cd307d2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:54.016564+0000", 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 212.21084155700001, 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.76734606599999999, 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:37:26.247 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75023 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.016564+0000", 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.016564+0000", 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.248 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.016563+0000", 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.016616+0000", 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3121999999999997e-05 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.016628+0000", 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.184e-05 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.249 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.016634+0000", 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8170000000000004e-06 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.016670+0000", 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5763000000000003e-05 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.016676+0000", 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6499999999999999e-06 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.250 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.783899+0000", 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.767222293 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.783910+0000", 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1661999999999999e-05 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:75030 6.12 6.22e14312 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1099)", 2024-04-12T11:37:26.251 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:54.019381+0000", 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 212.208025139, 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.76456187600000003, 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75030 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.019381+0000", 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.252 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.019381+0000", 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.019380+0000", 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.019439+0000", 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9474e-05 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.253 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.019449+0000", 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6069999999999997e-06 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.019456+0000", 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5070000000000003e-06 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.019505+0000", 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8508999999999999e-05 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.254 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.019512+0000", 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2509999999999997e-06 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.783937+0000", 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.76442507400000004 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:54.783942+0000", 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4269999999999999e-06 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.255 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94502 6.12 6:480475be:::benchmark_data_smithi029_138569_object94501:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.330377+0000", 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.897028404, 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.86137602199999996, 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94502 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330377+0000", 2024-04-12T11:37:26.256 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.257 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.257 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.257 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.257 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330377+0000", 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330377+0000", 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330484+0000", 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010715800000000001 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.258 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330494+0000", 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.063e-05 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330502+0000", 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.413e-06 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330555+0000", 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3108e-05 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.259 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330566+0000", 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0606e-05 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.167377+0000", 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.83681172800000003 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for missing object", 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.167497+0000", 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000119428 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.180913+0000", 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.013416104 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.180956+0000", 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3302e-05 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.191628+0000", 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.010671491999999999 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.191753+0000", 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000125699 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94511 6.12 6:4c1a5c06:::benchmark_data_smithi029_138569_object94510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.330827+0000", 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.89657821899999, 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.85342306700000004, 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.262 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94511 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330827+0000", 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330827+0000", 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.263 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330827+0000", 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330918+0000", 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1451000000000002e-05 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330935+0000", 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6566999999999999e-05 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.264 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330943+0000", 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9920000000000007e-06 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.330997+0000", 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4364999999999999e-05 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.331010+0000", 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3269e-05 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.167792+0000", 2024-04-12T11:37:26.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.83678202300000004 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.167886+0000", 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3790999999999997e-05 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.168003+0000", 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000116737 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.266 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.171745+0000", 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0037421440000000002 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.171757+0000", 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1864e-05 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.184102+0000", 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012344925 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.184152+0000", 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9610000000000001e-05 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.184251+0000", 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9018999999999995e-05 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.268 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94520 6.12 6:4bfece9d:::benchmark_data_smithi029_138569_object94519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.343422+0000", 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.883984048, 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.84318247099999999, 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94520 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.343422+0000", 2024-04-12T11:37:26.269 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.343422+0000", 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.343420+0000", 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.343466+0000", 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6236999999999998e-05 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.343479+0000", 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3020000000000001e-05 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.343488+0000", 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0389999999999993e-06 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.343547+0000", 2024-04-12T11:37:26.271 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8474000000000003e-05 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.343561+0000", 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3900000000000001e-05 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.169356+0000", 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.82579481799999999 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.169474+0000", 2024-04-12T11:37:26.272 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011858600000000001 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.169605+0000", 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013128200000000001 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.171765+0000", 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002159041 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.171820+0000", 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5912999999999998e-05 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.186431+0000", 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.01461085 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.186488+0000", 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6795000000000001e-05 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.186604+0000", 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000116026 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94530 6.12 6:49f4f679:::benchmark_data_smithi029_138569_object94529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.347047+0000", 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.88035916999999, 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.83979825299999999, 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.275 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94530 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.347047+0000", 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.347047+0000", 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.347045+0000", 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.347708+0000", 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00066269099999999997 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.347718+0000", 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7340000000000003e-06 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.277 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.347726+0000", 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8900000000000007e-06 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.351704+0000", 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0039779660000000003 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.351715+0000", 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0937e-05 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.278 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.170499+0000", 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.81878455699999997 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.170632+0000", 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000132683 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.170808+0000", 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000175778 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.279 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.172970+0000", 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002162855 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.173970+0000", 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00099978799999999994 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.186693+0000", 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012723144 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.186742+0000", 2024-04-12T11:37:26.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8310999999999999e-05 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.186845+0000", 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010306099999999999 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94532 6.12 6:4c19f449:::benchmark_data_smithi029_138569_object94531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.350170+0000", 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.877235426, 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.83683815900000003, 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.281 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94532 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.350170+0000", 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.350170+0000", 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.350169+0000", 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.350260+0000", 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1154000000000002e-05 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.350279+0000", 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8961e-05 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.350290+0000", 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0981999999999999e-05 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.351742+0000", 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0014514809999999999 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.351749+0000", 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2760000000000003e-06 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.284 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.171859+0000", 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.820109596 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.171992+0000", 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013296 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.172164+0000", 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000172036 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.174030+0000", 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0018660149999999999 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.175938+0000", 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0019082459999999999 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.186883+0000", 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.010944914 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.286 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.186923+0000", 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9690000000000001e-05 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.187008+0000", 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5927999999999995e-05 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94549 6.12 6:4894c59d:::benchmark_data_smithi029_138569_object94548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.368199+0000", 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.85920633200001, 2024-04-12T11:37:26.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.821100512, 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94549 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.368199+0000", 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.368199+0000", 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.368199+0000", 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.368291+0000", 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2281999999999995e-05 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.289 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.368303+0000", 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1996e-05 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.368309+0000", 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3690000000000003e-06 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.368366+0000", 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6403000000000002e-05 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.290 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.368376+0000", 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8660000000000001e-06 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.173007+0000", 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.80463176599999997 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.173105+0000", 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7900999999999997e-05 2024-04-12T11:37:26.291 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.173250+0000", 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014468700000000001 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.175978+0000", 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027277130000000001 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.176041+0000", 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3375999999999998e-05 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.189126+0000", 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.013084913 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.189179+0000", 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2710000000000002e-05 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.189300+0000", 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000121302 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.293 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94560 6.12 6:49c99ef1:::benchmark_data_smithi029_138569_object94559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.377252+0000", 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.85015407099999, 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.81222074600000005, 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.294 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94560 2024-04-12T11:37:26.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.295 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.295 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.295 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.377252+0000", 2024-04-12T11:37:26.295 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.296 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.296 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.377252+0000", 2024-04-12T11:37:26.296 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.296 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.296 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.296 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.296 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.377251+0000", 2024-04-12T11:37:26.296 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.296 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.296 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.377300+0000", 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9662e-05 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.377312+0000", 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1334e-05 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.377319+0000", 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2099999999999996e-06 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.297 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.377382+0000", 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3223000000000001e-05 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.377398+0000", 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5931e-05 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.174064+0000", 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.79666598399999999 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.298 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.174153+0000", 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8926999999999997e-05 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.174285+0000", 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013178899999999999 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.176031+0000", 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0017464469999999999 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.299 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.176048+0000", 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7053000000000001e-05 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.189339+0000", 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.013290512000000001 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.189372+0000", 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3012999999999997e-05 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.300 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.189472+0000", 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010056300000000001 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94567 6.12 6:4ee62a17:::benchmark_data_smithi029_138569_object94566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.384638+0000", 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.84276791100001, 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.80528957599999995, 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.301 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94567 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.384638+0000", 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.384638+0000", 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.302 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.384637+0000", 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.384662+0000", 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4746999999999999e-05 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.384668+0000", 2024-04-12T11:37:26.303 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7219999999999996e-06 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.384672+0000", 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0779999999999997e-06 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.384724+0000", 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2370000000000002e-05 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.384732+0000", 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.627e-06 2024-04-12T11:37:26.304 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.174963+0000", 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.79023119600000002 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.175093+0000", 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012972799999999999 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.175273+0000", 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018062699999999999 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.305 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.177982+0000", 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027084779999999998 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.178030+0000", 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7920000000000002e-05 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.189724+0000", 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.011693795 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.306 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.189811+0000", 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7701000000000006e-05 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.189927+0000", 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000115973 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94570 6.12 6:4c950aea:::benchmark_data_smithi029_138569_object94569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.386912+0000", 2024-04-12T11:37:26.307 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.84049360700001, 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.80325827100000002, 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94570 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.386912+0000", 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.308 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.386912+0000", 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.386911+0000", 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.386953+0000", 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1665000000000002e-05 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.309 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.386966+0000", 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3019e-05 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.386973+0000", 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8960000000000003e-06 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.387013+0000", 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9419000000000003e-05 2024-04-12T11:37:26.310 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.387021+0000", 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6070000000000002e-06 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.176064+0000", 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.78904362900000002 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.176149+0000", 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5279999999999997e-05 2024-04-12T11:37:26.311 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.176285+0000", 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013518499999999999 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.178083+0000", 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001797968 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.178094+0000", 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1063999999999999e-05 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.312 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.190007+0000", 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.011913563 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.190060+0000", 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3068000000000001e-05 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.190170+0000", 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000110099 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.313 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94619 6.12 6:49432030:::benchmark_data_smithi029_138569_object94618:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.432073+0000", 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.79533222800001, 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.760980667, 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94619 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.314 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.432073+0000", 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.432073+0000", 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.432073+0000", 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.315 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.432123+0000", 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0447e-05 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.432133+0000", 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0307e-05 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.432139+0000", 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5589999999999996e-06 2024-04-12T11:37:26.316 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.432184+0000", 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5518e-05 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.432197+0000", 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.225e-05 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.176960+0000", 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.74476377699999996 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.317 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.177090+0000", 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012925100000000001 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.177279+0000", 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018891099999999999 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.179912+0000", 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0026338379999999999 2024-04-12T11:37:26.318 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.179975+0000", 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2254000000000004e-05 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.192864+0000", 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012889004000000001 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.192958+0000", 2024-04-12T11:37:26.319 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4159000000000004e-05 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.193054+0000", 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6267999999999994e-05 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94623 6.12 6:4d8b855b:::benchmark_data_smithi029_138569_object94622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.437607+0000", 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.78979903199999, 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.75569048100000002, 2024-04-12T11:37:26.320 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94623 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.437607+0000", 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.321 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.437607+0000", 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.437605+0000", 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.437660+0000", 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5244999999999997e-05 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.322 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.437677+0000", 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7156999999999999e-05 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.437685+0000", 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2529999999999998e-06 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.437741+0000", 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6657000000000003e-05 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.437750+0000", 2024-04-12T11:37:26.323 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8899999999999996e-06 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.178109+0000", 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.74035837699999996 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.178222+0000", 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011348799999999999 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.178323+0000", 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000100921 2024-04-12T11:37:26.324 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.180887+0000", 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002564256 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.180980+0000", 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3045000000000001e-05 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.193141+0000", 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012161126 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.325 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.193184+0000", 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2185999999999999e-05 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.193297+0000", 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000113476 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94650 6.12 6:4e180316:::benchmark_data_smithi029_138569_object94649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.461043+0000", 2024-04-12T11:37:26.326 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.76636318000001, 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.73700693399999995, 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94650 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.461043+0000", 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.327 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.461043+0000", 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.461042+0000", 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.461074+0000", 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.1854999999999999e-05 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.328 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.461082+0000", 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.892e-06 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.461087+0000", 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6450000000000002e-06 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.461142+0000", 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4959e-05 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.329 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.461149+0000", 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8229999999999997e-06 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.179027+0000", 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.71787771 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.179135+0000", 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000108182 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.330 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.179243+0000", 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000107868 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.181072+0000", 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001829301 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.181084+0000", 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1717e-05 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.331 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.197905+0000", 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.016821446 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.198008+0000", 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000102902 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.198049+0000", 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1384000000000002e-05 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.332 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94655 6.12 6:4ce725d2:::benchmark_data_smithi029_138569_object94654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.464369+0000", 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.76303708099999, 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.73382173699999997, 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94655 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.333 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.464369+0000", 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.464369+0000", 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.464368+0000", 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.334 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.464426+0000", 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8811000000000003e-05 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.464436+0000", 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6369999999999994e-06 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.464443+0000", 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1500000000000002e-06 2024-04-12T11:37:26.335 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.464498+0000", 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4784000000000003e-05 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.464508+0000", 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0219e-05 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.336 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.181130+0000", 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.71662173900000004 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.181212+0000", 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2429000000000006e-05 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.181314+0000", 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000101802 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.337 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.184261+0000", 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0029468160000000001 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.186661+0000", 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0024004339999999999 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.198109+0000", 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.011447344999999999 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.338 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.198155+0000", 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5868000000000002e-05 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.198190+0000", 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5772000000000002e-05 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:94690 6.12 6:48d4b514:::benchmark_data_smithi029_138569_object94689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:20.512764+0000", 2024-04-12T11:37:26.339 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 185.714641462, 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.685570134, 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94690 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.512764+0000", 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.340 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.512764+0000", 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.512764+0000", 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.512825+0000", 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1068999999999994e-05 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.341 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.512833+0000", 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9869999999999999e-06 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.512837+0000", 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5809999999999998e-06 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.512881+0000", 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3414999999999997e-05 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:37:26.342 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:20.512889+0000", 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.473e-06 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.181963+0000", 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.66907429600000001 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.182093+0000", 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013009499999999999 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.182272+0000", 2024-04-12T11:37:26.343 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000178524 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.186619+0000", 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0043473490000000004 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.187049+0000", 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00043016700000000002 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.198246+0000", 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.011196162000000001 2024-04-12T11:37:26.344 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.198300+0000", 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4209999999999998e-05 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:21.198334+0000", 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4518999999999999e-05 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.345 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:104577 6.8 6:148f6229:::benchmark_data_smithi029_138569_object104576:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:33.512040+0000", 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 172.71536605899999, 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.66700306799999998, 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 104577 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.346 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512040+0000", 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512040+0000", 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512039+0000", 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.347 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512070+0000", 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.1389000000000001e-05 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512081+0000", 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0577000000000001e-05 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512088+0000", 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4240000000000004e-06 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.348 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.521444+0000", 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0093561540000000002 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.521550+0000", 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000105441 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.521639+0000", 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9282000000000003e-05 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.349 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.523070+0000", 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0014312509999999999 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.674362+0000", 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.15129168900000001 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.178924+0000", 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50456236600000004 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.350 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.178998+0000", 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3790999999999999e-05 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.179043+0000", 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4506999999999999e-05 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:37:26.351 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:37:26.351 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:26.389 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:26.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:26.491 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:26.501 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 23780 23764 123.75 88.0625 0.00190796 0.00802253 2024-04-12T11:37:26.517 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:26.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:26.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:26.619 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:26.647 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:26.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:26.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:26.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:26.791 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:26.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:26.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:26.893 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:26.924 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:26.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:27.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:27.049 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:27.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:27.134 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:26 smithi087 ceph-mon[89014]: pgmap v1497: 89 pgs: 89 active+clean; 1.1 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-04-12T11:37:27.150 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:27.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-04-12T11:37:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:27.183 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:27.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:27.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:26 smithi029 ceph-mon[97787]: pgmap v1497: 89 pgs: 89 active+clean; 1.1 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-04-12T11:37:27.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:26 smithi029 ceph-mon[93650]: pgmap v1497: 89 pgs: 89 active+clean; 1.1 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-04-12T11:37:27.283 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:27.307 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:27.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:27.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:27.408 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:27.442 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:27.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:27.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:27.501 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 25136 25120 120.75 84.75 0.0111838 0.00827528 2024-04-12T11:37:27.543 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:27.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:27.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:27.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:27.684 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:27.707 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:27.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:27.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:27.839 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:27.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:27.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:27.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:27.978 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:27.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:27.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:28.080 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:28.119 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:28.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:28.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:28.221 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:28.256 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:28.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:28.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:28.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:28.397 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:28.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:28.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:28.499 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:28.501 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 27094 27078 120.865 122.375 0.00903075 0.00826772 2024-04-12T11:37:28.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:28.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:28.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:28.680 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:28.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:28.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:28.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:28.811 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:28.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:28.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:28.913 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:28.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:28.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:28.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:29.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:28 smithi029 ceph-mon[93650]: pgmap v1498: 89 pgs: 89 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-04-12T11:37:29.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:28 smithi029 ceph-mon[97787]: pgmap v1498: 89 pgs: 89 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-04-12T11:37:29.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:29.070 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:29.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:29.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:29.172 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:29.213 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:29.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:29.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:28 smithi087 ceph-mon[89014]: pgmap v1498: 89 pgs: 89 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-04-12T11:37:29.315 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:29.338 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:29.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:29.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:29.440 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:29.472 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:29.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:29.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:29.501 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 28660 28644 119.332 97.875 0.00615654 0.00837469 2024-04-12T11:37:29.574 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:29.605 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:29.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:29.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:29.707 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:29.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:29.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:29.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:29.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:29.864 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:29.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:29.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:29.966 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:30.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:30.006 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:30.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:30.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:30.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:30.146 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:30.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:30.247 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:30.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:30.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:30.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:30.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:30.404 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:30.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:30.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:30.501 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 30337 30321 118.424 104.812 0.00594117 0.00843832 2024-04-12T11:37:30.505 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:30.537 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:30.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:30.639 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:30.670 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:30.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:30.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:30.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:30.822 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:30.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:30.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:30.955 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:30.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:30.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:31.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:30 smithi029 ceph-mon[97787]: pgmap v1499: 89 pgs: 89 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-12T11:37:31.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:30 smithi029 ceph-mon[93650]: pgmap v1499: 89 pgs: 89 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-12T11:37:31.057 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:31.089 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:31.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:31.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:31.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:30 smithi087 ceph-mon[89014]: pgmap v1499: 89 pgs: 89 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-12T11:37:31.228 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:31.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:31.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:31.330 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:31.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:31.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:31.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:31.459 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:31.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:31.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:31.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:31.501 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 32426 32410 119.137 130.562 0.00911254 0.00838828 2024-04-12T11:37:31.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:31.629 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:31.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:31.730 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:31.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:31.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:31.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:31.860 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 1 1 2024-04-12T11:37:31.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:31.957 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:31.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:31.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:32.058 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:32.089 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:32.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:32.190 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:32.217 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:32.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:32.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:32.319 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:32.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:32.351 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:32.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:32.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:32.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:32.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:32.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:32.501 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 34246 34230 118.836 113.75 0.0117295 0.008405 2024-04-12T11:37:32.574 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:32.588 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:32.600 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:32.621 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:37:32.627 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:32.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:32.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:32.665 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:37:32.704 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:37:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:37:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:37:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:37:32.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:32.758 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:37:32.775 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:32.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:32.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:32.803 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:37:32.844 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:37:32.877 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:32.886 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:37:32.905 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:32.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:32.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:32.924 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:37:32.935 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:37:32.936 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:37:32.936 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:37:32.936 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:37:32.936 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:37:32.936 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:37:32.936 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:37:32.936 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:37:33.006 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:32 smithi029 ceph-mon[93650]: pgmap v1500: 89 pgs: 89 active+clean; 1.6 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-04-12T11:37:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:32 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3640639595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:37:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:32 smithi029 ceph-mon[97787]: pgmap v1500: 89 pgs: 89 active+clean; 1.6 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-04-12T11:37:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:32 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3640639595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:37:33.038 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:33.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:33.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:33.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:33.178 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:33.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:33.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:32 smithi087 ceph-mon[89014]: pgmap v1500: 89 pgs: 89 active+clean; 1.6 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-04-12T11:37:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:32 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3640639595' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:37:33.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:33.316 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:33.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:33.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:33.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:37:33.418 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:33.487 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:33.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:33.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:33.501 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 35988 35972 118.311 108.875 0.004942 0.00844706 2024-04-12T11:37:33.590 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:33.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:33.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:33.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:33.724 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:33.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:33.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:33.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:33.896 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:33.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:33.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:33.997 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:34.025 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:34.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:34.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:34.127 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:34.165 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:34.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:34.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:34.267 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:34.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:34.291 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:34.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:34.392 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:34.425 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:34.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:34.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:34.502 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:37:34.502229+0000 min lat: 0.00111552 max lat: 0.350312 avg lat: 0.00850426 2024-04-12T11:37:34.502 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:37:34.502 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 37625 37609 117.51 102.312 0.00544514 0.00850426 2024-04-12T11:37:34.527 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:34.557 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:34.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:34.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:34.691 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:34.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:34.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:34.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:34.824 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:34.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:34.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:34.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:34.966 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:34.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:35.066 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:35.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:35.101 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:35.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:35.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:34 smithi029 ceph-mon[93650]: pgmap v1501: 89 pgs: 89 active+clean; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-04-12T11:37:35.124 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:37:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:37:34] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2024-04-12T11:37:35.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:34 smithi029 ceph-mon[97787]: pgmap v1501: 89 pgs: 89 active+clean; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-04-12T11:37:35.201 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:34 smithi087 ceph-mon[89014]: pgmap v1501: 89 pgs: 89 active+clean; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-04-12T11:37:35.235 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:35.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:35.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:35.336 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:35.367 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:35.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:35.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:35.469 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:35.493 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:35.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:35.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:35.501 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 39450 39434 117.346 114.062 0.00567409 0.00848915 2024-04-12T11:37:35.595 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:35.618 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:35.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:35.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:35.719 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:35.749 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:35.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:35.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:35.851 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:35.883 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:35.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:35.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:35.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:36.023 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:36.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:36.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:36.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:36.148 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:36.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:36.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:36.249 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:36.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:36.274 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:36.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:36.375 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:36.639 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 40645 40629 115.407 74.6875 0.00765419 0.00865949 2024-04-12T11:37:36.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:36.681 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:36.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:36.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:36.823 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:36.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:36.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:36.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:36.948 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:36.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:37.049 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:37.075 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:37.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:37.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:37.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:37.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:37.216 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:37.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:36 smithi087 ceph-mon[89014]: pgmap v1502: 89 pgs: 89 active+clean; 2.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-04-12T11:37:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:36 smithi029 ceph-mon[97787]: pgmap v1502: 89 pgs: 89 active+clean; 2.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-04-12T11:37:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:36 smithi029 ceph-mon[93650]: pgmap v1502: 89 pgs: 89 active+clean; 2.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-04-12T11:37:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:37.316 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:37.343 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:37.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:37.445 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:37.485 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:37.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:37.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:37.502 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 15 42136 42121 114.443 93.25 0.0168647 0.00873114 2024-04-12T11:37:37.588 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:37.626 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:37.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:37.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:37.728 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:37.768 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:37.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:37.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:37.870 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:37.907 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:37.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:37.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:38.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:38.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:38.047 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:38.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:38.148 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:38.181 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:38.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:38.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:38.320 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:38.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:38.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:38.459 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:38.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:38.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:38.502 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 15 43938 43923 114.367 112.625 0.0156159 0.00873822 2024-04-12T11:37:38.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:38.601 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:38.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:38.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:38.703 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:38.733 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:38.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:38.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:38.834 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:38.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:38.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:38.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:39.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:38 smithi029 ceph-mon[93650]: pgmap v1503: 89 pgs: 89 active+clean; 2.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:37:39.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:38 smithi029 ceph-mon[97787]: pgmap v1503: 89 pgs: 89 active+clean; 2.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:37:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:39.020 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:39.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:39.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:39.151 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:39.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:39.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:38 smithi087 ceph-mon[89014]: pgmap v1503: 89 pgs: 89 active+clean; 2.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:37:39.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:39.301 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:39.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:39.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:39.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:39.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:39.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:39.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:39.502 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 45589 45573 113.917 103.125 0.0206123 0.00876947 2024-04-12T11:37:39.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:39.607 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:39.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:39.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:39.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:39.742 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:39.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:39.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:39.844 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:39.872 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:39.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:39.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:39.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:40.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:40.021 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:40.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:40.122 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:40.151 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:40.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:40.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:40.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:40.303 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:40.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:40.404 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:40.432 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:40.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:40.471 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:40.481 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:37:40.481 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:37:40.503 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 47081 47065 113.116 93.25 0.0197509 0.00883496 2024-04-12T11:37:40.534 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:40.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:40.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:40.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:40.669 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:40.698 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:40.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:40.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:40.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:40.825 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.1 to 1 (10000) 2024-04-12T11:37:40.843 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:40.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:40.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:40.946 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:40.978 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:40.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:41.080 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:41.112 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:41.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:41.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:41.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:40 smithi087 ceph-mon[89014]: pgmap v1504: 89 pgs: 89 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-12T11:37:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:40 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1885059152' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:37:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:40 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:37:41.254 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:41.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:41.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:41.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:40 smithi029 ceph-mon[93650]: pgmap v1504: 89 pgs: 89 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-12T11:37:41.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:40 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1885059152' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:37:41.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:40 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:37:41.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:40 smithi029 ceph-mon[97787]: pgmap v1504: 89 pgs: 89 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-04-12T11:37:41.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:40 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1885059152' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:37:41.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:40 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:37:41.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:41.396 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:41.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:41.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:41.498 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:41.503 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 48651 48635 112.56 98.125 0.00198643 0.00886299 2024-04-12T11:37:41.522 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:41.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:41.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:41.618 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:37:41.625 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:41.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:41.708 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:41.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:41.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:41.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:41.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:41.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:41.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:41.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:41.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:41.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:41 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-04-12T11:37:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:41 smithi029 ceph-mon[97787]: osdmap e1141: 8 total, 8 up, 8 in 2024-04-12T11:37:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:41 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-04-12T11:37:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:41 smithi029 ceph-mon[93650]: osdmap e1141: 8 total, 8 up, 8 in 2024-04-12T11:37:42.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:42.122 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:42.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:42.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:41 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-04-12T11:37:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:41 smithi087 ceph-mon[89014]: osdmap e1141: 8 total, 8 up, 8 in 2024-04-12T11:37:42.224 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:42.262 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:42.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:42.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:42.363 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:42.387 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:42.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:42.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:42.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:42.503 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 49915 49899 111.361 79 0.0141959 0.00897476 2024-04-12T11:37:42.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:42.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:42.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:42.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:42.657 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:42.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:42.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:37:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:37:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:37:42.759 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:42.792 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:42.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:42.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:42.894 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:42.917 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:42.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:42.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:43.018 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:43.055 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:43.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:43.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:43.156 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:43.189 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:43.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:43.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:42 smithi087 ceph-mon[89014]: pgmap v1506: 89 pgs: 89 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-04-12T11:37:43.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:42 smithi029 ceph-mon[97787]: pgmap v1506: 89 pgs: 89 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-04-12T11:37:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:42 smithi029 ceph-mon[93650]: pgmap v1506: 89 pgs: 89 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-04-12T11:37:43.290 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:43.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:43.315 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:43.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:43.416 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:43.439 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:43.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:43.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:43.504 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 51047 51031 109.961 70.75 0.00501605 0.00908803 2024-04-12T11:37:43.540 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:43.571 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:43.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:43.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:43.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:43.715 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:43.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:43.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:43.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:43.840 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:43.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:43.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:43.965 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:43.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:43.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:44.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:44.091 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:44.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:44.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:44.192 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:44.224 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:44.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:44.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:44.368 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:44.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:44.470 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:44.504 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 52223 52207 108.745 73.5 0.0063929 0.00918898 2024-04-12T11:37:44.509 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:44.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:44.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:44.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:44.654 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:44.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:44.756 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:44.789 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:44.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:44.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:44.890 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:44.920 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:44.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:44.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:45.022 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:45.052 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:45.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:45.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:45.155 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:45.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:45.179 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:45.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:44 smithi087 ceph-mon[89014]: pgmap v1507: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-12T11:37:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:44 smithi029 ceph-mon[93650]: pgmap v1507: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-12T11:37:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:44 smithi029 ceph-mon[97787]: pgmap v1507: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-12T11:37:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:37:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:37:44] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2024-04-12T11:37:45.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:45.302 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:45.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:45.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:45.404 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:45.427 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:45.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:45.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:45.504 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 15 53305 53290 107.421 67.6875 0.00631345 0.00929755 2024-04-12T11:37:45.529 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:45.560 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:45.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:45.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:45.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:45.688 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:45.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:45.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:45.790 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:45.822 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:45.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:45.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:45.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:45.956 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:45.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:45.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:46.058 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:46.097 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:46.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:46.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:46.199 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:46.223 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:46.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:46.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:46.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:46.365 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:46.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:46.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:46.503 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:46.504 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 54437 54421 106.273 70.6875 0.0117625 0.00940469 2024-04-12T11:37:46.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:46.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:46.605 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:46.637 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:46.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:46.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:46.738 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:46.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:46.762 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:46.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:46.863 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:46.893 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:46.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:46.993 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:47.024 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:47.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:47.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:47.126 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:47.158 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:47.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:47.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:46 smithi087 ceph-mon[89014]: pgmap v1508: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-04-12T11:37:47.260 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:46 smithi029 ceph-mon[97787]: pgmap v1508: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-04-12T11:37:47.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:46 smithi029 ceph-mon[93650]: pgmap v1508: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-04-12T11:37:47.283 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:47.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:47.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:47.384 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:47.410 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:47.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:47.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:47.504 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 55442 55426 104.956 62.8125 0.00294173 0.00947208 2024-04-12T11:37:47.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:47.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:47.550 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:47.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:47.651 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:47.682 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:47.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:47.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:47.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:47.812 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:47.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:47.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:47.914 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:47.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:47.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:48.039 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:48.070 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:48.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:48.172 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:48.202 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:48.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:48.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:48.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:48.338 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:48.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:48.438 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:48.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:48.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:48.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:48.504 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 56483 56467 103.782 65.0625 0.00212186 0.00960562 2024-04-12T11:37:48.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:48.587 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:48.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:48.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:48.689 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:48.721 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:48.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:48.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:48.823 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:48.854 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:48.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:48.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:48.986 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:48.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:48.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:49.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-04-12T11:37:49.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:49.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:48 smithi029 ceph-mon[97787]: pgmap v1509: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-12T11:37:49.125 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:48 smithi029 ceph-mon[93650]: pgmap v1509: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-12T11:37:49.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:49.169 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:49.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:48 smithi087 ceph-mon[89014]: pgmap v1509: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-12T11:37:49.270 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:49.300 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:49.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:49.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:49.401 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:49.442 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:49.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:49.505 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 57767 57751 103.107 80.25 0.0107971 0.00969278 2024-04-12T11:37:49.544 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:49.597 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:49.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:49.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:49.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:49.740 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:49.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:49.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:49.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:49.881 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:49.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:49.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:49.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:50.021 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:50.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:50.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:50.123 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:50.154 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:50.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:50.256 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:50.297 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:50.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:50.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:50.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:50.443 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:50.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:50.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:50.505 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 59174 59158 102.685 87.9375 0.00365543 0.00971744 2024-04-12T11:37:50.545 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:50.576 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:50.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:50.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:50.678 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:50.708 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:50.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:50.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:50.810 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:50.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 2 1 2024-04-12T11:37:50.839 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:50.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:50.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:50.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:50.984 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:50.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:50.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:51.087 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:51.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:51.119 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:51.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:51.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:50 smithi087 ceph-mon[89014]: pgmap v1510: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T11:37:51.221 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:50 smithi029 ceph-mon[97787]: pgmap v1510: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T11:37:51.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:50 smithi029 ceph-mon[93650]: pgmap v1510: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T11:37:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:51.282 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:51.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:51.383 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:51.412 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:51.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:51.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:51.506 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 60459 60443 102.081 80.3125 0.025015 0.00978827 2024-04-12T11:37:51.514 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:51.528 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:51.538 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:51.542 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:51.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:51.568 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:37:51.644 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:51.675 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:51.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:51.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:51.777 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:51.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:51.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:51.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:51.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:51.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:51.994 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:51.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:51.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:37:51.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:52.000 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:37:52.001 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:52.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:52.117 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:37:52.117 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:37:52.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:52.151 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:52.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:52.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:52.293 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:52.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:52.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:52.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:52.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:52.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:52.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:52.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:37:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:37:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:37:52.506 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 61678 61662 101.397 76.1875 0.0150593 0.00985683 2024-04-12T11:37:52.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:52.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:52.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:52.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:52.668 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:52.704 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:52.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:52.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:52.806 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:52.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:52.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:52.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:52.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:52.984 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:52.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:53.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:53.126 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:53.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:53.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:53.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:52 smithi087 ceph-mon[89014]: pgmap v1511: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-12T11:37:53.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:53.250 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:52 smithi029 ceph-mon[97787]: pgmap v1511: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-12T11:37:53.250 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:52 smithi029 ceph-mon[93650]: pgmap v1511: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-12T11:37:53.269 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:53.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:53.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:53.408 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:53.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:53.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:53.506 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 63010 62994 100.931 83.25 0.0168144 0.00990178 2024-04-12T11:37:53.510 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:53.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:53.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:53.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:53.648 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:53.686 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:53.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:53.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:53.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:53.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:53.827 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:53.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:53.889 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:37:53.928 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:53.957 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:53.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:53.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:54.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:54.082 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:54.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:54.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:54.183 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:54.214 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:54.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:54.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:54.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:53 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2140833487' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:37:54.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:53 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:37:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:53 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2140833487' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:37:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:53 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:37:54.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:53 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2140833487' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:37:54.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:53 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:37:54.316 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:54.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:54.350 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:54.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:54.451 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:54.480 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:54.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:54.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:54.506 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:37:54.507242+0000 min lat: 0.00111552 max lat: 0.350312 avg lat: 0.00997042 2024-04-12T11:37:54.507 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:37:54.507 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 64188 64172 100.249 73.625 0.0129775 0.00997042 2024-04-12T11:37:54.581 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:54.621 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:54.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:54.722 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:54.752 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:54.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:54.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:54.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:54.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:54.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:54.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:54.980 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:55.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:55.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:55.112 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:55.144 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:55.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:55.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:55.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:54 smithi087 ceph-mon[89014]: pgmap v1512: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-12T11:37:55.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:54 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:37:55.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:54 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:37:55.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:54 smithi087 ceph-mon[89014]: osdmap e1142: 8 total, 8 up, 8 in 2024-04-12T11:37:55.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:54 smithi029 ceph-mon[97787]: pgmap v1512: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-12T11:37:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:54 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:37:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:54 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:37:55.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:54 smithi029 ceph-mon[97787]: osdmap e1142: 8 total, 8 up, 8 in 2024-04-12T11:37:55.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:37:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:37:54] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2024-04-12T11:37:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:54 smithi029 ceph-mon[93650]: pgmap v1512: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-12T11:37:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:54 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:37:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:54 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:37:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:54 smithi029 ceph-mon[93650]: osdmap e1142: 8 total, 8 up, 8 in 2024-04-12T11:37:55.274 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:55.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:55.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:55.376 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:55.400 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:55.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:55.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:55.501 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:55.507 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 65198 65182 99.3414 63.125 0.00940992 0.0100605 2024-04-12T11:37:55.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:55.542 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:55.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:55.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:37:55.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:55.682 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:55.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:37:55.783 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:55.806 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:55.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:55.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:55.907 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:55.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:55.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:55.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:56.040 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:56.073 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:56.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:56.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:56.175 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:56.198 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:56.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:56.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:56.300 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:56.323 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:56.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:56.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:56.425 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:56.453 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:56.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:56.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:56.507 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 66027 66011 98.2098 51.8125 0.0078751 0.010177 2024-04-12T11:37:56.556 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:56.588 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:56.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:56.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:56.690 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:56.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:56.715 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:56.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:56.816 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:56.839 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:56.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:56.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:56.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:56.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:56.978 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:56.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:57.079 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:57.116 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:57.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:57.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:57.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:56 smithi087 ceph-mon[89014]: pgmap v1514: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-12T11:37:57.218 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:57.241 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:57.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:57.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:56 smithi029 ceph-mon[93650]: pgmap v1514: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-12T11:37:57.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:56 smithi029 ceph-mon[97787]: pgmap v1514: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-12T11:37:57.343 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:57.377 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:57.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:57.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:57.479 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:57.508 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 66842 66826 97.1105 50.9375 0.00885024 0.010292 2024-04-12T11:37:57.516 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:57.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:57.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:57.618 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:57.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:57.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:57.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:57.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:57.777 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:57.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:57.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:57.879 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:57.902 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:57.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:58.004 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:58.046 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:58.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:58.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:58.148 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:58.183 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:58.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:58.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:58.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:58.307 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:58.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:58.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:58.409 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:58.432 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:58.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:58.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:58.508 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 67616 67600 96.0029 48.375 0.00393549 0.0104 2024-04-12T11:37:58.534 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:58.558 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:58.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:58.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:58.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:58.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:58.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:58.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:58.831 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:58.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:58.933 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:58.972 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:58.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:58.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:59.074 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:37:59.110 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:59.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:59.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:37:59.211 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:37:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:58 smithi087 ceph-mon[89014]: pgmap v1515: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T11:37:59.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:58 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:59.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:37:58 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:59.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:59.249 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:59.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:37:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:58 smithi029 ceph-mon[97787]: pgmap v1515: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T11:37:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:58 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:37:58 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:58 smithi029 ceph-mon[93650]: pgmap v1515: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T11:37:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:58 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:37:58 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:37:59.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:37:59.374 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:37:59.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:59.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:37:59.476 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:37:59.500 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:59.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:37:59.508 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 68662 68646 95.3221 65.375 0.00390756 0.0104436 2024-04-12T11:37:59.602 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:37:59.635 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:59.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:59.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:37:59.737 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:37:59.768 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:59.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:59.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:37:59.870 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:37:59.893 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:37:59.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:37:59.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:37:59.995 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:00.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:00.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:00.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:00.161 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:00.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:00.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:00.263 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:00.293 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:00.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:00.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:00.395 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:00.426 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:00.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:00.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:00.508 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 69702 69686 94.6628 65 0.00287053 0.01055 2024-04-12T11:38:00.527 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:00.554 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:00.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:00.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:00.656 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:00.680 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:00.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:00.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:00.816 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:00.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:00.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:00.920 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:00.953 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:00.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:00.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:01.055 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:01.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:01.098 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:01.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:01.199 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:01.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:00 smithi087 ceph-mon[89014]: pgmap v1516: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-12T11:38:01.225 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:01.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:01.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:00 smithi029 ceph-mon[93650]: pgmap v1516: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-12T11:38:01.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:00 smithi029 ceph-mon[97787]: pgmap v1516: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-12T11:38:01.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:01.350 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:01.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:01.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:01.452 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:01.482 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:01.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:01.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:01.508 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 70588 70572 93.8269 55.375 0.0139703 0.0106533 2024-04-12T11:38:01.584 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:01.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:01.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:01.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:01.716 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:01.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:01.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:01.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:01.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:01.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:01.872 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:01.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:01.973 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:02.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:02.015 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:02.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:02.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:02.156 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:02.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:02.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:02.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:02.296 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:02.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:02.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:02.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:02.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:02.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:02.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:02.508 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 71639 71623 93.2406 65.6875 0.0111716 0.0107198 2024-04-12T11:38:02.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:02.571 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:02.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:02.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:02.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:02.716 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:38:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:38:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:38:02.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:02.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:02.818 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:02.865 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:02.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:02.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:02.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:03.018 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:03.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:03.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:03.119 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:03.158 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:03.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:03.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:02 smithi087 ceph-mon[89014]: pgmap v1517: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-12T11:38:03.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:03.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:02 smithi029 ceph-mon[97787]: pgmap v1517: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-12T11:38:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:02 smithi029 ceph-mon[93650]: pgmap v1517: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-12T11:38:03.289 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:03.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:03.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:03.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:03.427 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:03.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:03.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:03.508 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 72912 72896 92.9614 79.5625 0.0140839 0.0107522 2024-04-12T11:38:03.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:03.569 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:03.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:03.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:03.671 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:03.697 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:03.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:03.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:03.799 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:03.832 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:03.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:03.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:03.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:03.983 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:03.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:03.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:04.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:04.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:04.126 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:04.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:04.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:04.256 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:04.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:04.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:04.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:04.404 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:04.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:04.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:04.508 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 74071 74055 92.5508 72.4375 0.014375 0.0107994 2024-04-12T11:38:04.547 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:04.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:04.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:04.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:04.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:04.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:04.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:04.793 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:04.823 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:04.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:04.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:04.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:04.970 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:04.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:05.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:04 smithi029 ceph-mon[97787]: pgmap v1518: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:38:05.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:04 smithi029 ceph-mon[93650]: pgmap v1518: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:38:05.057 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:38:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:38:04] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2024-04-12T11:38:05.072 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:05.103 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:05.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:05.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:05.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:05.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:04 smithi087 ceph-mon[89014]: pgmap v1518: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:38:05.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:05.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:05.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:05.345 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:05.376 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:05.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:05.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:05.508 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 75259 75243 92.1919 74.25 0.012549 0.0108413 2024-04-12T11:38:05.518 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:05.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:05.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:05.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:05.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:05.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:05.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:05.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:05.782 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:05.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:05.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:05.823 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 1 (10000) 2024-04-12T11:38:05.884 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:05.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:05.932 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:05.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:06.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:05 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1356431645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T11:38:06.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:05 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1356431645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T11:38:06.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:06.062 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:06.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:06.164 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:06.210 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:06.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:06.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:06.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:05 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1356431645' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T11:38:06.312 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:06.360 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:06.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:06.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:06.462 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:06.491 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:06.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:06.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:06.509 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 76376 76360 91.7614 69.8125 0.00384508 0.010889 2024-04-12T11:38:06.593 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:06.616 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:06.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:06.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:06.755 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:06.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:06.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:06.820 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:38:06.820 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:38:06.857 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:06.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:06.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:06.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:06.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:07.021 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:07.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:07.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:07.122 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:07.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:06 smithi029 ceph-mon[97787]: pgmap v1519: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-12T11:38:07.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1356431645' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-04-12T11:38:07.139 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:06 smithi029 ceph-mon[97787]: osdmap e1143: 8 total, 8 up, 8 in 2024-04-12T11:38:07.139 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:07.139 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:07.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:06 smithi029 ceph-mon[93650]: pgmap v1519: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-12T11:38:07.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1356431645' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-04-12T11:38:07.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:06 smithi029 ceph-mon[93650]: osdmap e1143: 8 total, 8 up, 8 in 2024-04-12T11:38:07.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:07.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:07.151 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:07.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:07.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:06 smithi087 ceph-mon[89014]: pgmap v1519: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-12T11:38:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1356431645' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-04-12T11:38:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:06 smithi087 ceph-mon[89014]: osdmap e1143: 8 total, 8 up, 8 in 2024-04-12T11:38:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:07.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:07.253 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:07.284 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:07.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:07.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:07.386 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:07.415 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:07.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:07.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:07.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:38:07.509 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 77500 77484 91.3554 70.25 0.00716182 0.0109401 2024-04-12T11:38:07.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:07.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:07.567 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:07.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:07.604 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-04-12T11:38:07.667 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:07.695 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:07.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:07.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:07.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:07.846 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:07.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:07.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:07.948 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:07.949 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:38:07.994 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:38:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:07.998 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:07.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:08.042 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:38:08.084 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:38:08.085 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:07 smithi029 ceph-mon[97787]: osdmap e1144: 8 total, 8 up, 8 in 2024-04-12T11:38:08.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:07 smithi029 ceph-mon[93650]: osdmap e1144: 8 total, 8 up, 8 in 2024-04-12T11:38:08.099 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:08.127 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:08.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:08.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:08.129 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:38:08.172 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:38:08.215 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:38:08.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:07 smithi087 ceph-mon[89014]: osdmap e1144: 8 total, 8 up, 8 in 2024-04-12T11:38:08.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:08.262 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:38:08.271 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:38:08.271 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:38:08.271 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:38:08.271 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:38:08.272 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:38:08.272 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:38:08.272 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:38:08.272 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:38:08.272 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:08.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:08.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:08.374 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 3 1 2024-04-12T11:38:08.374 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:08.434 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:08.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:08.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:08.509 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 78521 78505 90.8444 63.8125 0.0107988 0.0110034 2024-04-12T11:38:08.536 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:08.592 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:08.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:08.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:08.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:08.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:08.740 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:08.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:08.840 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:08.871 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:08.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:08.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:09.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:09.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:09.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:09.116 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:09.146 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:09.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:09.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:09.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:08 smithi087 ceph-mon[89014]: pgmap v1522: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2024-04-12T11:38:09.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:08 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/31982060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:38:09.248 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:08 smithi029 ceph-mon[97787]: pgmap v1522: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2024-04-12T11:38:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:08 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/31982060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:38:09.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:08 smithi029 ceph-mon[93650]: pgmap v1522: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2024-04-12T11:38:09.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:08 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/31982060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:38:09.277 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:09.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:09.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:09.379 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:09.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:09.417 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:09.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:09.509 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 79702 79686 90.5346 73.8125 0.0175324 0.0110399 2024-04-12T11:38:09.519 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:09.563 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:09.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:09.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:09.665 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:09.695 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:09.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:09.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:09.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:09.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:09.832 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:09.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:09.933 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:09.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:09.975 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:09.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:10.077 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:10.108 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:10.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:10.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:10.210 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:10.254 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:10.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:10.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:10.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:10.385 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:10.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:10.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:10.487 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:10.509 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 80837 80821 90.1846 70.9375 0.0186618 0.0110831 2024-04-12T11:38:10.522 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:10.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:10.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:10.624 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:10.665 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:10.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:10.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:10.766 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:10.799 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:10.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:10.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:10.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:10.928 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:10.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:10.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:11.006 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:11.030 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:11.104 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:11.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:11.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:11.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:10 smithi029 ceph-mon[97787]: pgmap v1523: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 72 B/s, 0 objects/s recovering 2024-04-12T11:38:11.151 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:10 smithi029 ceph-mon[93650]: pgmap v1523: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 72 B/s, 0 objects/s recovering 2024-04-12T11:38:11.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:11.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:10 smithi087 ceph-mon[89014]: pgmap v1523: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 72 B/s, 0 objects/s recovering 2024-04-12T11:38:11.252 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:11.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:11.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:11.354 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:11.384 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:11.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:11.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:11.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:11.509 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 15 81950 81935 89.8238 69.625 0.0148133 0.0111266 2024-04-12T11:38:11.516 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:11.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:11.618 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:11.650 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:11.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:11.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:11.753 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:11.785 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:11.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:11.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:11.809 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:11.809 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:11.887 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:11.915 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:11.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:11.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:12.018 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:12.078 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:12.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:12.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:12.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:12.210 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:12.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:12.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:12.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:12.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:12.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:12.444 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:12.477 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:12.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:12.509 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 83120 83104 89.5348 73.0625 0.0161969 0.0111633 2024-04-12T11:38:12.578 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:12.620 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:12.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:12.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:38:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:38:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:38:12.722 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:12.752 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:12.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:12.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:12.854 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:12.889 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:12.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:12.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:12.942 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:38:12.991 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:13.017 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:13.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:13.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:13.119 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:13.170 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:13.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:12 smithi087 ceph-mon[89014]: pgmap v1524: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 72 B/s, 0 objects/s recovering 2024-04-12T11:38:13.229 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:12 smithi029 ceph-mon[93650]: pgmap v1524: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 72 B/s, 0 objects/s recovering 2024-04-12T11:38:13.230 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:12 smithi029 ceph-mon[97787]: pgmap v1524: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 72 B/s, 0 objects/s recovering 2024-04-12T11:38:13.271 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:13.316 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:13.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:13.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:13.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:13.469 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:13.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:13.510 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 84186 84170 89.1459 66.625 0.0101027 0.011211 2024-04-12T11:38:13.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:13.617 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:13.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:13.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:13.718 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:13.747 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:13.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:13.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:13.849 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:13.878 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:13.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:13.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:13.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:14.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:14.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:14.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:14.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:14.171 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:14.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:14.272 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:14.301 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:14.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:14.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:14.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:14.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:14.427 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:14.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:14.510 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:38:14.510786+0000 min lat: 0.00111552 max lat: 0.350312 avg lat: 0.0112537 2024-04-12T11:38:14.510 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:38:14.510 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 85117 85101 88.6298 58.1875 0.0114052 0.0112537 2024-04-12T11:38:14.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:14.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:14.566 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:14.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:14.667 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:14.707 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:14.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:14.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:14.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:14.837 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:14.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:14.876 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:38:14.938 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:14.974 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:14.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:14.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:14 smithi029 ceph-mon[93650]: pgmap v1525: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 72 B/s, 0 objects/s recovering 2024-04-12T11:38:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:14 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/772335093' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:38:15.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:38:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:38:14] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2024-04-12T11:38:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:14 smithi029 ceph-mon[97787]: pgmap v1525: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 72 B/s, 0 objects/s recovering 2024-04-12T11:38:15.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:14 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/772335093' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:38:15.078 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:15.124 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:15.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:15.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:14 smithi087 ceph-mon[89014]: pgmap v1525: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 72 B/s, 0 objects/s recovering 2024-04-12T11:38:15.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:14 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/772335093' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:38:15.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:15.270 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:15.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:15.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:15.373 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:15.428 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:15.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:15.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:15.510 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 86191 86175 88.2772 67.125 0.0168313 0.0113229 2024-04-12T11:38:15.530 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:15.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:15.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:15.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:15.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:15.706 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:15.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:15.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:15.807 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:15.841 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:15.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:15.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:15.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:15.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:15.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:16.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:16.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:16.105 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:16.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:16.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:16.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:15 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:38:16.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:15 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/772335093' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:38:16.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:15 smithi087 ceph-mon[89014]: osdmap e1145: 8 total, 8 up, 8 in 2024-04-12T11:38:16.248 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:16.249 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:15 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:38:16.249 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:15 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/772335093' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:38:16.249 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:15 smithi029 ceph-mon[93650]: osdmap e1145: 8 total, 8 up, 8 in 2024-04-12T11:38:16.249 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:15 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:38:16.249 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:15 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/772335093' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:38:16.249 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:15 smithi029 ceph-mon[97787]: osdmap e1145: 8 total, 8 up, 8 in 2024-04-12T11:38:16.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:16.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:16.351 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:16.393 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:16.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:16.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:16.495 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:16.510 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 87309 87293 87.9802 69.875 0.0166438 0.0113602 2024-04-12T11:38:16.523 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:16.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:16.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:16.625 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:16.668 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:16.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:16.769 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:16.798 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:16.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:16.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:16.899 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:16.930 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:16.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:16.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:17.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:17.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:17.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:17.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:17.178 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:17.208 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:17.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:17.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:17.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:16 smithi087 ceph-mon[89014]: pgmap v1527: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 65 B/s, 0 objects/s recovering 2024-04-12T11:38:17.232 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:16 smithi029 ceph-mon[93650]: pgmap v1527: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 65 B/s, 0 objects/s recovering 2024-04-12T11:38:17.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:16 smithi029 ceph-mon[97787]: pgmap v1527: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 65 B/s, 0 objects/s recovering 2024-04-12T11:38:17.310 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:17.364 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:17.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:17.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:17.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:17.504 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:17.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:17.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:17.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 88349 88333 87.6154 65 0.00856656 0.0114085 2024-04-12T11:38:17.606 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:17.646 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:17.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:17.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:17.747 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:17.779 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:17.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:17.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:17.881 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:17.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:17.915 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:17.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:18.016 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:18.054 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:18.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:18.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:18.095 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:18.157 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:18.165 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:38:18.165 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:38:18.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:65203 6.0 6:07c53962:::benchmark_data_smithi029_138569_object65202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:41.885151+0000", 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 276.27172924600001, 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0560479599999999, 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 65203 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.166 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885151+0000", 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885151+0000", 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885151+0000", 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9999999999999998e-07 2024-04-12T11:38:18.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885170+0000", 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8949000000000001e-05 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885177+0000", 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.267e-06 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885183+0000", 2024-04-12T11:38:18.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9660000000000001e-06 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885578+0000", 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00039433799999999999 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885583+0000", 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8529999999999997e-06 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.987044+0000", 2024-04-12T11:38:18.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10146031599999999 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.987059+0000", 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4939e-05 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.941120+0000", 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95406113699999995 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.941198+0000", 2024-04-12T11:38:18.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8695000000000005e-05 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:65207 6.9 6:96248252:::benchmark_data_smithi029_138569_object65206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:41.885198+0000", 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 276.27168142400001, 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.053096273, 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.171 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 65207 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885198+0000", 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885198+0000", 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885199+0000", 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5900000000000002e-07 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885218+0000", 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8804000000000002e-05 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885224+0000", 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6069999999999996e-06 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885230+0000", 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4430000000000002e-06 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.888070+0000", 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028407049999999998 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.888075+0000", 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7010000000000004e-06 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.966584+0000", 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.078509147000000001 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.966590+0000", 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2650000000000002e-06 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.938205+0000", 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97161434800000002 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.938295+0000", 2024-04-12T11:38:18.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9794000000000001e-05 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:65209 6.0 6:0577e78d:::benchmark_data_smithi029_138569_object65208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:41.885237+0000", 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 276.271642778, 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.05603618, 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.176 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 65209 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885237+0000", 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885237+0000", 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885237+0000", 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.72e-07 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885255+0000", 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8048999999999999e-05 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885261+0000", 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9379999999999997e-06 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885267+0000", 2024-04-12T11:38:18.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1939999999999997e-06 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885680+0000", 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041343199999999998 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885685+0000", 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6800000000000001e-06 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.987381+0000", 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.101695913 2024-04-12T11:38:18.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.987394+0000", 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.312e-05 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.941222+0000", 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.953828448 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.941273+0000", 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1034e-05 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.180 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:65210 6.10 6:0eeac94d:::benchmark_data_smithi029_138569_object65209:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:41.885273+0000", 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 276.271606367, 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0858900460000001, 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.181 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 65210 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885273+0000", 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885273+0000", 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885274+0000", 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6100000000000002e-07 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885297+0000", 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3295e-05 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885304+0000", 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5150000000000003e-06 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885309+0000", 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1760000000000001e-06 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.987577+0000", 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10226782700000001 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.987584+0000", 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6890000000000004e-06 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.971067+0000", 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98348310500000002 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.971163+0000", 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6077999999999997e-05 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74004 6.10 6:0a3d569c:::benchmark_data_smithi029_138569_object74003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:49.970319+0000", 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.18656062299999, 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.596887578, 2024-04-12T11:38:18.185 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74004 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.970319+0000", 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.970319+0000", 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.970321+0000", 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4219999999999999e-06 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.970341+0000", 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0591e-05 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.970356+0000", 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5311e-05 2024-04-12T11:38:18.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.970364+0000", 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6590000000000007e-06 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.970427+0000", 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3035999999999998e-05 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.970437+0000", 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0293e-05 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.058871+0000", 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.088433523 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.058884+0000", 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3224000000000001e-05 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.560076+0000", 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5011915330000001 2024-04-12T11:38:18.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.560266+0000", 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000190046 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.560398+0000", 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013183800000000001 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.565744+0000", 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053459759999999997 2024-04-12T11:38:18.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.565818+0000", 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4169000000000001e-05 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.565839+0000", 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0948e-05 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.565894+0000", 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5571e-05 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.567207+0000", 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001312438 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74032 6.10 6:0927b140:::benchmark_data_smithi029_138569_object74031:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:49.983656+0000", 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.17322328500001, 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5896239410000002, 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.192 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74032 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.983656+0000", 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.983656+0000", 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.983658+0000", 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5260000000000001e-06 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.983680+0000", 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1603e-05 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.983697+0000", 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7864e-05 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.983706+0000", 2024-04-12T11:38:18.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2369999999999992e-06 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.983782+0000", 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5803999999999995e-05 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.983794+0000", 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2213e-05 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.058951+0000", 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.075157122 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.058959+0000", 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4889999999999995e-06 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.563063+0000", 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5041036239999999 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.563205+0000", 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014194899999999999 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.563331+0000", 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000126348 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.567308+0000", 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039764379999999997 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.567434+0000", 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000126336 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.570856+0000", 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034219770000000001 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.570916+0000", 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9914000000000003e-05 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.573280+0000", 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023644970000000001 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.198 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74034 6.10 6:0ba95da4:::benchmark_data_smithi029_138569_object74033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:49.986904+0000", 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.169976195, 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5874031629999998, 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74034 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.986904+0000", 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.986904+0000", 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.986905+0000", 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.609e-06 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.986927+0000", 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1763000000000002e-05 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.986942+0000", 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4755e-05 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.986951+0000", 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9539999999999993e-06 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.987055+0000", 2024-04-12T11:38:18.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010398100000000001 2024-04-12T11:38:18.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:49.987067+0000", 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2123999999999999e-05 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.058987+0000", 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.07192072 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.058997+0000", 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.346e-06 2024-04-12T11:38:18.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.564541+0000", 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.505544558 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.564651+0000", 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010935200000000001 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.564761+0000", 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109826 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.567316+0000", 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002555112 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.573361+0000", 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060456119999999997 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.573404+0000", 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2975000000000001e-05 2024-04-12T11:38:18.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.573449+0000", 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4273000000000001e-05 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.574307+0000", 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00085820299999999998 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74066 6.10 6:0dbe002d:::benchmark_data_smithi029_138569_object74065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.206 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.003405+0000", 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.15347444399998, 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5723538389999998, 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74066 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.003405+0000", 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.003405+0000", 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.003406+0000", 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1689999999999999e-06 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.003427+0000", 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0285e-05 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.003438+0000", 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1265e-05 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.003446+0000", 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6580000000000002e-06 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.003646+0000", 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019989699999999999 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.003656+0000", 2024-04-12T11:38:18.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9890000000000004e-06 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059024+0000", 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0553682470000001 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059031+0000", 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7479999999999996e-06 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.567495+0000", 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.508464808 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.567675+0000", 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000179846 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.567872+0000", 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019697599999999999 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.574320+0000", 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0064481160000000003 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.574454+0000", 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000134077 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.574585+0000", 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013032699999999999 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.574631+0000", 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6146000000000002e-05 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.575759+0000", 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011282849999999999 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74083 6.10 6:0968df7a:::benchmark_data_smithi029_138569_object74082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.213 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.013001+0000", 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.14387870000002, 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5638613779999999, 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74083 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013001+0000", 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013001+0000", 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013002+0000", 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3179999999999999e-06 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013024+0000", 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1996000000000001e-05 2024-04-12T11:38:18.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013040+0000", 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6078000000000001e-05 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013128+0000", 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7329000000000004e-05 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013451+0000", 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032300299999999998 2024-04-12T11:38:18.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013464+0000", 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2768e-05 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059054+0000", 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.045590204 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059061+0000", 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7880000000000001e-06 2024-04-12T11:38:18.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.569100+0000", 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.510039632 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.569446+0000", 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034539 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.569642+0000", 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019663699999999999 2024-04-12T11:38:18.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.574326+0000", 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046834470000000003 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.574529+0000", 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000203685 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.575814+0000", 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001284344 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.575853+0000", 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9057999999999998e-05 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.576862+0000", 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001009701 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74084 6.10 6:0973a29e:::benchmark_data_smithi029_138569_object74083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.220 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.013275+0000", 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.14360487699997, 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5838953880000002, 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74084 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013275+0000", 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013275+0000", 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013276+0000", 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0200000000000001e-07 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013305+0000", 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9074e-05 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013315+0000", 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0199e-05 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013322+0000", 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0550000000000002e-06 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013491+0000", 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016894300000000001 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.013500+0000", 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4239999999999999e-06 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059085+0000", 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.045584866 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059092+0000", 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9240000000000001e-06 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.577873+0000", 2024-04-12T11:38:18.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5187812350000001 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.578033+0000", 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000159937 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.578232+0000", 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000199094 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.590275+0000", 2024-04-12T11:38:18.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012043023 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.595378+0000", 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051022400000000001 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.595404+0000", 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6608999999999999e-05 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.595465+0000", 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0466e-05 2024-04-12T11:38:18.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.597170+0000", 2024-04-12T11:38:18.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017055970000000001 2024-04-12T11:38:18.227 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.227 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.227 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:18.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:18.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:18.228 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74091 6.10 6:0fe5cc03:::benchmark_data_smithi029_138569_object74090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.228 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.017630+0000", 2024-04-12T11:38:18.228 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.13924944500002, 2024-04-12T11:38:18.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.581963156, 2024-04-12T11:38:18.228 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.228 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74091 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.017630+0000", 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.017630+0000", 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.017632+0000", 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0360000000000001e-06 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.017655+0000", 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2555e-05 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.017672+0000", 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6699999999999999e-05 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.017680+0000", 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8279999999999992e-06 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.017753+0000", 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2602999999999996e-05 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.017770+0000", 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6801000000000001e-05 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059116+0000", 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0413465049999999 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059123+0000", 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8889999999999996e-06 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.579369+0000", 2024-04-12T11:38:18.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5202462130000001 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.579526+0000", 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015704 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.579707+0000", 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000180464 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.590354+0000", 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010646551000000001 2024-04-12T11:38:18.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.597189+0000", 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0068356609999999998 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.597312+0000", 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012252500000000001 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.597380+0000", 2024-04-12T11:38:18.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8447000000000004e-05 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.599593+0000", 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002213338 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74099 6.10 6:0c64a6a3:::benchmark_data_smithi029_138569_object74098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.021021+0000", 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.13585827600002, 2024-04-12T11:38:18.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5799493170000001, 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74099 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.021021+0000", 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.021021+0000", 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.021023+0000", 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.531e-06 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.021047+0000", 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.402e-05 2024-04-12T11:38:18.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.021062+0000", 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5058000000000001e-05 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.021072+0000", 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7310000000000005e-06 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.021157+0000", 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4747999999999996e-05 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.021169+0000", 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2308e-05 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059155+0000", 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.03798656 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059163+0000", 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1430000000000001e-06 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.580945+0000", 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.521782006 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.581099+0000", 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000154215 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.581307+0000", 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020861299999999999 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.595300+0000", 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01399267 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.597196+0000", 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018955599999999999 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.599776+0000", 2024-04-12T11:38:18.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025804159999999999 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.599841+0000", 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5282000000000003e-05 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.600971+0000", 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001129456 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74113 6.10 6:0a01cafe:::benchmark_data_smithi029_138569_object74112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.027512+0000", 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.12936801000001, 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5748582359999999, 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74113 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.243 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.027512+0000", 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.027512+0000", 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.027513+0000", 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6099999999999999e-07 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.027534+0000", 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1406000000000001e-05 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.027544+0000", 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0063e-05 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.027551+0000", 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2119999999999997e-06 2024-04-12T11:38:18.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.027608+0000", 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6347000000000001e-05 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.027615+0000", 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1420000000000004e-06 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059188+0000", 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.031573531 2024-04-12T11:38:18.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059197+0000", 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3650000000000004e-06 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.582461+0000", 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5232644070000001 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.582614+0000", 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015251800000000001 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.582770+0000", 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000155883 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.597202+0000", 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014432571999999999 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.599727+0000", 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025247889999999999 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.601059+0000", 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013322290000000001 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.601123+0000", 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3999999999999997e-05 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.602370+0000", 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001246811 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.249 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74175 6.10 6:0860a487:::benchmark_data_smithi029_138569_object74174:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.064574+0000", 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.09230537799999, 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.552667166, 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74175 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.064574+0000", 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.064574+0000", 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.064576+0000", 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.176e-06 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.064599+0000", 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3555999999999999e-05 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.064614+0000", 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4456e-05 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.064622+0000", 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5439999999999999e-06 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.064706+0000", 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3961999999999993e-05 2024-04-12T11:38:18.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.253 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.064718+0000", 2024-04-12T11:38:18.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2099000000000001e-05 2024-04-12T11:38:18.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.253 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059280+0000", 2024-04-12T11:38:18.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99456175300000005 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059288+0000", 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5429999999999994e-06 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.583903+0000", 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.524614551 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.584079+0000", 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017599099999999999 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.584260+0000", 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018055299999999999 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.597209+0000", 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01294909 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.602430+0000", 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052216770000000001 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.615419+0000", 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012988211 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.615478+0000", 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9209000000000002e-05 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.617242+0000", 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017637950000000001 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74182 6.10 6:0d4b4b7f:::benchmark_data_smithi029_138569_object74181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.068299+0000", 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.08858071700001, 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5504282489999999, 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.257 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74182 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.068299+0000", 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.068299+0000", 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.068301+0000", 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.897e-06 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.068325+0000", 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4278e-05 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.068346+0000", 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0403000000000001e-05 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.068356+0000", 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0332e-05 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.068430+0000", 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4396999999999999e-05 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.068443+0000", 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2659e-05 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059318+0000", 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99087493599999998 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059326+0000", 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1729999999999995e-06 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.585429+0000", 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5261031869999999 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.585571+0000", 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014131500000000001 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.585755+0000", 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018441599999999999 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.597216+0000", 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011460748999999999 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.602479+0000", 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052631780000000003 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.617301+0000", 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014822035000000001 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.617371+0000", 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9517999999999994e-05 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.618727+0000", 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013567760000000001 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74191 6.10 6:0de37973:::benchmark_data_smithi029_138569_object74190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.074261+0000", 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.08261857100001, 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5459981479999998, 2024-04-12T11:38:18.264 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74191 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.074261+0000", 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.074261+0000", 2024-04-12T11:38:18.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.074263+0000", 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.311e-06 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.074283+0000", 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0607000000000001e-05 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.074298+0000", 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5282999999999999e-05 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.074307+0000", 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7870000000000007e-06 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.074379+0000", 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2225999999999996e-05 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.074393+0000", 2024-04-12T11:38:18.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3164e-05 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059355+0000", 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98496194400000003 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059363+0000", 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1939999999999998e-06 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.586996+0000", 2024-04-12T11:38:18.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5276334309999999 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.587163+0000", 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016671200000000001 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.587345+0000", 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000182444 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.597242+0000", 2024-04-12T11:38:18.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0098964280000000005 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.615337+0000", 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018095080999999999 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.618804+0000", 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034674939999999998 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.618874+0000", 2024-04-12T11:38:18.270 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9911000000000002e-05 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.620259+0000", 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013851309999999999 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74210 6.10 6:0cad4053:::benchmark_data_smithi029_138569_object74209:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.088627+0000", 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.06825262199999, 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5332704339999998, 2024-04-12T11:38:18.271 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74210 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.088627+0000", 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.088627+0000", 2024-04-12T11:38:18.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.088629+0000", 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7039999999999999e-06 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.088651+0000", 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2218999999999999e-05 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.088669+0000", 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8130000000000001e-05 2024-04-12T11:38:18.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.088678+0000", 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4200000000000007e-06 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.088757+0000", 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9285999999999999e-05 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.088771+0000", 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3669000000000001e-05 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059391+0000", 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97062063099999996 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059399+0000", 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1270000000000003e-06 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.588556+0000", 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.529156768 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.588708+0000", 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015210700000000001 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.588888+0000", 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018016199999999999 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.599651+0000", 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010763056999999999 2024-04-12T11:38:18.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.620342+0000", 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020690185 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.620732+0000", 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00039023900000000002 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.620793+0000", 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1587000000000005e-05 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.278 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.278 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.621898+0000", 2024-04-12T11:38:18.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011041429999999999 2024-04-12T11:38:18.278 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.278 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.278 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74246 6.10 6:0941320f:::benchmark_data_smithi029_138569_object74245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.124093+0000", 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.03278687800002, 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4990017460000002, 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74246 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.279 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.124093+0000", 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.124093+0000", 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.124094+0000", 2024-04-12T11:38:18.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0809999999999999e-06 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.124114+0000", 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9692999999999999e-05 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.124122+0000", 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.32e-06 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.124128+0000", 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7039999999999999e-06 2024-04-12T11:38:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.124188+0000", 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0430999999999998e-05 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.124199+0000", 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0934999999999999e-05 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059428+0000", 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93522923199999997 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059436+0000", 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1300000000000001e-06 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.590398+0000", 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5309616369999999 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.590554+0000", 2024-04-12T11:38:18.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015619599999999999 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.590756+0000", 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000201249 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.599660+0000", 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089040569999999999 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.620400+0000", 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020740708 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.621953+0000", 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001552629 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.622011+0000", 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7729000000000002e-05 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.623095+0000", 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001084015 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74262 6.10 6:0bb7fa86:::benchmark_data_smithi029_138569_object74261:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.145051+0000", 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 268.01182866699997, 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4792057029999999, 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74262 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.145051+0000", 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.145051+0000", 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.145052+0000", 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3039999999999999e-06 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.145072+0000", 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9732e-05 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.145085+0000", 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2818e-05 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.145092+0000", 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5819999999999998e-06 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.145167+0000", 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5706999999999999e-05 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.145179+0000", 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1839e-05 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059464+0000", 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.91428498899999999 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059472+0000", 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0660000000000004e-06 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.592039+0000", 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5325672670000001 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.592201+0000", 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016165 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.592359+0000", 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000158109 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.599666+0000", 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073071519999999999 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.620467+0000", 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020800341 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.623160+0000", 2024-04-12T11:38:18.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026936579999999998 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.623223+0000", 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2896000000000002e-05 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.624257+0000", 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001033593 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.292 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:74287 6.10 6:0b348da2:::benchmark_data_smithi029_138569_object74286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:50.196086+0000", 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 267.96079402700002, 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4295599490000002, 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 74287 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.196086+0000", 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.196086+0000", 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.196087+0000", 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.404e-06 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.196109+0000", 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1401999999999999e-05 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.196122+0000", 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3151e-05 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.196129+0000", 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2579999999999998e-06 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.196197+0000", 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7607999999999994e-05 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:50.196204+0000", 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6410000000000002e-06 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059499+0000", 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86329526000000001 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:51.059508+0000", 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3650000000000004e-06 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.593622+0000", 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5341145489999999 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.593795+0000", 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000173012 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.594001+0000", 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000205174 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:38:18.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.599673+0000", 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056726040000000004 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.620606+0000", 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020933111000000001 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.624367+0000", 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037603210000000001 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.624432+0000", 2024-04-12T11:38:18.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5593999999999996e-05 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:52.625646+0000", 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001213495 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:38:18.299 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:38:18.328 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:18.360 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:18.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:18.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:18.461 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:18.488 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:18.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:18.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:18.510 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 89369 89353 87.2424 63.75 0.00909514 0.0114566 2024-04-12T11:38:18.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:18.629 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:18.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:18.731 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:18.765 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:18.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:18.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:18.867 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:18.896 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:18.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:18.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:18.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:19.041 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:19.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:19.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:19.143 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:19.173 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:19.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:19.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:18 smithi087 ceph-mon[89014]: pgmap v1528: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 58 B/s, 0 objects/s recovering 2024-04-12T11:38:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:18 smithi029 ceph-mon[93650]: pgmap v1528: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 58 B/s, 0 objects/s recovering 2024-04-12T11:38:19.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:18 smithi029 ceph-mon[97787]: pgmap v1528: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 58 B/s, 0 objects/s recovering 2024-04-12T11:38:19.276 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:19.307 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:19.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:19.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:19.409 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:19.439 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:19.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:19.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:19.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 90467 90451 86.956 68.625 0.00904373 0.0114945 2024-04-12T11:38:19.541 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:19.580 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:19.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:19.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:19.682 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:19.725 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:19.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:19.827 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:19.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:38:19.855 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:19.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:19.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:19.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:19.980 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:19.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:20.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:20.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:20.136 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:20.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:20.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:20.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:20.273 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:20.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:20.374 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:20.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:20.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:20.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:20.506 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:20.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 91392 91376 86.5143 57.8125 0.0113467 0.0115535 2024-04-12T11:38:20.537 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:20.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:20.639 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:20.669 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:20.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:20.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:20.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:20.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:20.810 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:20.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:20.910 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:20.937 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:20.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:20.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:21.039 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:21.064 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:21.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:21.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:21.166 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:21.189 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:21.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:21.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:20 smithi087 ceph-mon[89014]: pgmap v1529: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-12T11:38:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:20 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1400839550' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T11:38:21.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:20 smithi029 ceph-mon[93650]: pgmap v1529: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-12T11:38:21.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:20 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1400839550' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T11:38:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:20 smithi029 ceph-mon[97787]: pgmap v1529: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-12T11:38:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:20 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1400839550' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T11:38:21.291 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:21.314 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:21.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:21.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:21.415 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:21.440 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:21.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:21.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:21.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 91980 91964 85.7716 36.75 0.00334393 0.0115654 2024-04-12T11:38:21.523 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.3 to 1 (10000) 2024-04-12T11:38:21.541 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:21.565 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:21.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:21.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:21.667 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:21.691 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:21.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:21.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:21.793 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:21.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:21.817 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:21.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:21.918 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:21.945 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:21.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:21.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:22.047 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:22.075 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:22.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:22.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:22.176 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:22.203 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:22.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:22.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:22.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:21 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1400839550' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-04-12T11:38:22.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:21 smithi087 ceph-mon[89014]: osdmap e1146: 8 total, 8 up, 8 in 2024-04-12T11:38:22.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:22.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:21 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1400839550' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-04-12T11:38:22.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:21 smithi029 ceph-mon[93650]: osdmap e1146: 8 total, 8 up, 8 in 2024-04-12T11:38:22.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:22.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:21 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1400839550' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-04-12T11:38:22.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:21 smithi029 ceph-mon[97787]: osdmap e1146: 8 total, 8 up, 8 in 2024-04-12T11:38:22.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:22.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:22.329 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:22.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:22.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:22.430 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:22.453 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:22.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:22.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:22.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 91980 91964 84.5103 0 - 0.0115654 2024-04-12T11:38:22.555 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:22.583 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:22.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:22.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:22.685 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:22.708 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:22.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:22.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:38:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:38:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:38:22.810 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:22.833 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:22.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:22.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:22.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:22.962 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:22.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:23.064 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:23.086 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:23.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:23.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:23.188 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:23.212 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:23.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:22 smithi087 ceph-mon[89014]: pgmap v1531: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2024-04-12T11:38:23.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:22 smithi087 ceph-mon[89014]: osdmap e1147: 8 total, 8 up, 8 in 2024-04-12T11:38:23.237 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:22 smithi029 ceph-mon[97787]: pgmap v1531: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2024-04-12T11:38:23.237 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:22 smithi029 ceph-mon[93650]: pgmap v1531: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2024-04-12T11:38:23.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:23 smithi029 ceph-mon[93650]: osdmap e1147: 8 total, 8 up, 8 in 2024-04-12T11:38:23.313 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:23.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:23.337 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:23.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:23.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:23.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:23.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:23.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:23.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 92036 92020 83.3363 1.75 0.00829866 0.0119136 2024-04-12T11:38:23.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:23 smithi029 ceph-mon[97787]: osdmap e1147: 8 total, 8 up, 8 in 2024-04-12T11:38:23.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:23.611 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:23.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:23.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:23.713 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:23.743 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:23.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:23.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:23.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:23.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:23.873 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:23.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:23.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:24.013 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:24.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:24.114 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:24.154 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:24.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:24.256 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:24.279 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:24.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:24.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:24.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:24.422 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:24.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:24.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:24.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 92770 92754 82.8011 45.875 0.0124088 0.0120702 2024-04-12T11:38:24.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:24 smithi029 ceph-mon[97787]: osdmap e1148: 8 total, 8 up, 8 in 2024-04-12T11:38:24.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:24 smithi029 ceph-mon[97787]: pgmap v1534: 89 pgs: 1 remapped+peering, 88 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-04-12T11:38:24.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:24 smithi029 ceph-mon[93650]: osdmap e1148: 8 total, 8 up, 8 in 2024-04-12T11:38:24.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:24 smithi029 ceph-mon[93650]: pgmap v1534: 89 pgs: 1 remapped+peering, 88 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-04-12T11:38:24.523 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:24.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:24.566 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:24.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:24.667 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:24.708 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:24.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:24.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:24.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:24 smithi087 ceph-mon[89014]: osdmap e1148: 8 total, 8 up, 8 in 2024-04-12T11:38:24.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:24 smithi087 ceph-mon[89014]: pgmap v1534: 89 pgs: 1 remapped+peering, 88 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-04-12T11:38:24.810 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:24.833 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:24.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:24.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:24.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:24.958 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:24.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:24.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:25.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:25.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:25.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:25.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:25.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:25.226 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:25.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:25.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:38:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:38:24] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2024-04-12T11:38:25.328 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:25.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:25.354 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:25.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:25.455 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:25.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:25.479 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:25.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:25.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 93269 93253 82.0742 31.1875 0.00711991 0.0121534 2024-04-12T11:38:25.580 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:25.607 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:25.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:25.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:25.709 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:25.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:25.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:25.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:25.838 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:25.868 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:25.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:25.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:25.970 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:25.993 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:25.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:25.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:26.095 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:26.123 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:26.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:26.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:26.225 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:26.253 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:26.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:26.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:26.355 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:26.378 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:26.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:26.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:26.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:26.503 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:26.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:26.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 93753 93737 81.3544 30.25 0.00326465 0.0122555 2024-04-12T11:38:26.605 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:26.632 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:26.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:26.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:26.734 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:26.757 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:26.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:26.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:26.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:26.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:26.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:26.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:26.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:27.026 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:27.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:27.130 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:27.161 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:27.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:26 smithi087 ceph-mon[89014]: pgmap v1535: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 538 op/s; 5688/277143 objects misplaced (2.052%); 288 KiB/s, 4 objects/s recovering 2024-04-12T11:38:27.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:27.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:26 smithi029 ceph-mon[93650]: pgmap v1535: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 538 op/s; 5688/277143 objects misplaced (2.052%); 288 KiB/s, 4 objects/s recovering 2024-04-12T11:38:27.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:26 smithi029 ceph-mon[97787]: pgmap v1535: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 538 op/s; 5688/277143 objects misplaced (2.052%); 288 KiB/s, 4 objects/s recovering 2024-04-12T11:38:27.293 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:27.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:27.394 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:27.426 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:27.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:27.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 94365 94349 80.7639 38.25 0.00831197 0.0123621 2024-04-12T11:38:27.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:27.551 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:27.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:27.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:27.694 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:27.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:27.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:27.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:27.840 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:27.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:27.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:27.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:27.968 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:27.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:28.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:28.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:28.100 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:28.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:28.201 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:28.229 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:28.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:28.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:28.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-04-12T11:38:28.331 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:28.356 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:28.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:28.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:28.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:28.511 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 94962 94946 80.1767 37.3125 0.0579847 0.0124657 2024-04-12T11:38:28.520 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:28.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:28.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:28.621 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:28.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:28.668 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:28.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:28.769 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:28.799 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:28.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:28.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:28.923 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:28.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:28.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:29.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:29.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:29.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:29.152 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:29.183 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:29.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:29.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:28 smithi087 ceph-mon[89014]: pgmap v1536: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 87 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 32 MiB/s wr, 508 op/s; 5688/278895 objects misplaced (2.039%); 253 KiB/s, 3 objects/s recovering 2024-04-12T11:38:29.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:28 smithi029 ceph-mon[97787]: pgmap v1536: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 87 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 32 MiB/s wr, 508 op/s; 5688/278895 objects misplaced (2.039%); 253 KiB/s, 3 objects/s recovering 2024-04-12T11:38:29.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:28 smithi029 ceph-mon[93650]: pgmap v1536: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 87 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 32 MiB/s wr, 508 op/s; 5688/278895 objects misplaced (2.039%); 253 KiB/s, 3 objects/s recovering 2024-04-12T11:38:29.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:29.331 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:29.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:29.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:29.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:29.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:29.467 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:29.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:29.512 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 95692 95676 79.716 45.625 0.0076698 0.0125388 2024-04-12T11:38:29.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:29.616 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:29.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:29.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:29.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:29.759 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:29.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:29.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:29.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:29.884 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:29.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:29.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:29.986 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:30.014 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:30.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:30.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:30.161 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:30.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:30.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:30.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:29 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 5641/284685 objects degraded (1.981%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:30.263 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:30.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:29 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 5641/284685 objects degraded (1.981%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:30.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:29 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 5641/284685 objects degraded (1.981%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:30.289 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:30.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:30.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:30.307 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 4 1 2024-04-12T11:38:30.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:30.421 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:30.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:30.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:30.512 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 96392 96376 79.2427 43.75 0.0123033 0.0126132 2024-04-12T11:38:30.523 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:30.551 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:30.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:30.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:30.653 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:30.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:30.699 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:30.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:30.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:30.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:30.888 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:30.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:30.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:31.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:31.040 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:31.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:31.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:31.183 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:31.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:31.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:30 smithi087 ceph-mon[89014]: pgmap v1537: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s; 5641/284685 objects degraded (1.981%); 5688/284685 objects misplaced (1.998%); 216 KiB/s, 3 objects/s recovering 2024-04-12T11:38:31.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:30 smithi029 ceph-mon[97787]: pgmap v1537: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s; 5641/284685 objects degraded (1.981%); 5688/284685 objects misplaced (1.998%); 216 KiB/s, 3 objects/s recovering 2024-04-12T11:38:31.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:30 smithi029 ceph-mon[93650]: pgmap v1537: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s; 5641/284685 objects degraded (1.981%); 5688/284685 objects misplaced (1.998%); 216 KiB/s, 3 objects/s recovering 2024-04-12T11:38:31.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:31.323 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:31.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:31.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:31.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:31.451 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:31.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:31.472 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:31.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:31.512 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 97206 97190 78.8742 50.875 0.0810476 0.0126683 2024-04-12T11:38:31.523 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:31.573 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:31.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:31.627 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:31.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:31.728 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:31.756 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:31.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:31.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:31.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:31.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:31.926 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:31.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:32.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:32.083 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:32.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:32.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:32.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:32.220 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:32.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:32.320 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:32.349 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:32.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:32.451 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:32.480 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:32.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:32.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:32.512 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 98044 98028 78.5345 52.375 0.024286 0.0127277 2024-04-12T11:38:32.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:32.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:32.670 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:32.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:38:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:38:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:38:32.771 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:32.802 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:32.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:32.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:32.904 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:32.929 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:32.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:32.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:32 smithi029 ceph-mon[97787]: pgmap v1538: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 566 op/s; 5641/285174 objects degraded (1.978%); 5584/285174 objects misplaced (1.958%); 853 KiB/s, 13 objects/s recovering 2024-04-12T11:38:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:32 smithi029 ceph-mon[93650]: pgmap v1538: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 566 op/s; 5641/285174 objects degraded (1.978%); 5584/285174 objects misplaced (1.958%); 853 KiB/s, 13 objects/s recovering 2024-04-12T11:38:33.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:33.055 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:33.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:33.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:33.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:33.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:33.216 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:32 smithi087 ceph-mon[89014]: pgmap v1538: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 566 op/s; 5641/285174 objects degraded (1.978%); 5584/285174 objects misplaced (1.958%); 853 KiB/s, 13 objects/s recovering 2024-04-12T11:38:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:33.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:33.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:33.359 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:33.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:33.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:33.461 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:33.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:33.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:33.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:33.512 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 98863 98847 78.1883 51.1875 0.0208066 0.0127835 2024-04-12T11:38:33.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:33.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:33.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:33.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:33.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:33.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:33.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:33.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:33.880 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:33.910 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:33.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:33.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:34.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:34.049 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:34.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:34.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:34.151 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:34.182 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:34.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:34.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:34.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:34.328 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:34.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:34.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:34.474 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:34.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:34.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:34.512 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:38:34.512923+0000 min lat: 0.00111552 max lat: 2.53616 avg lat: 0.0128352 2024-04-12T11:38:34.512 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:38:34.512 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 99719 99703 77.8796 53.5 0.0429961 0.0128352 2024-04-12T11:38:34.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:34.627 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:34.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:34.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:34.729 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:34.760 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:34.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:34.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:34.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:34.902 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:34.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:34.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:35.004 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:35.046 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:35.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:35.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:35.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:34 smithi029 ceph-mon[93650]: pgmap v1539: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 35 MiB/s wr, 566 op/s; 5641/291564 objects degraded (1.935%); 5584/291564 objects misplaced (1.915%); 797 KiB/s, 12 objects/s recovering 2024-04-12T11:38:35.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:34 smithi029 ceph-mon[97787]: pgmap v1539: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 35 MiB/s wr, 566 op/s; 5641/291564 objects degraded (1.935%); 5584/291564 objects misplaced (1.915%); 797 KiB/s, 12 objects/s recovering 2024-04-12T11:38:35.186 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:35.186 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:38:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:38:34] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2024-04-12T11:38:35.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:35.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:34 smithi087 ceph-mon[89014]: pgmap v1539: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 35 MiB/s wr, 566 op/s; 5641/291564 objects degraded (1.935%); 5584/291564 objects misplaced (1.915%); 797 KiB/s, 12 objects/s recovering 2024-04-12T11:38:35.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:35.338 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:35.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:35.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:35.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:35.512 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 100559 100543 77.5663 52.5 0.0228288 0.0128851 2024-04-12T11:38:35.515 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:35.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:35.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:35.617 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:35.653 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:35.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:35.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:35.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:35.794 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:35.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:35.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:35.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:35.945 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:35.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:35.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:36.046 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:36.074 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:36.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:36.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:36.175 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:36.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:35 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5641/296847 objects degraded (1.900%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:36.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:36.234 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:36.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:36.261 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:35 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5641/296847 objects degraded (1.900%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:36.262 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:35 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5641/296847 objects degraded (1.900%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:36.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:36.383 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:36.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:36.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:36.485 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:36.512 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:36.513 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 101397 101381 77.259 52.375 0.00996928 0.0129379 2024-04-12T11:38:36.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:36.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:36.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:36.655 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:36.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:36.756 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:36.785 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:36.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:36.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:36.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:36.936 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:36.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:37.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:37.084 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:37.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:37.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:37.104 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:36 smithi029 ceph-mon[97787]: pgmap v1540: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s; 5641/296847 objects degraded (1.900%); 5441/296847 objects misplaced (1.833%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T11:38:37.104 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:37.104 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:37.104 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:36 smithi029 ceph-mon[93650]: pgmap v1540: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s; 5641/296847 objects degraded (1.900%); 5441/296847 objects misplaced (1.833%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T11:38:37.104 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:37.104 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:37.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:36 smithi087 ceph-mon[89014]: pgmap v1540: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s; 5641/296847 objects degraded (1.900%); 5441/296847 objects misplaced (1.833%); 1.4 MiB/s, 22 objects/s recovering 2024-04-12T11:38:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:37.234 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:37.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:37.335 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:37.363 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:37.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:37.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:37.465 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:37.489 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:37.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:37.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:37.512 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 102017 102001 76.795 38.75 0.0432406 0.0130142 2024-04-12T11:38:37.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:37.655 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:37.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:37.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:37.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:37.793 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:37.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:37.893 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:37.923 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:37.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:37.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:37.969 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:38:38.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:38.052 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:38.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:38.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:38.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:38.193 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:38.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:38.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:38.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:37 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3503621736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:38:38.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:37 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3534774469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:38:38.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:37 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:38:38.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:37 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3503621736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:38:38.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:37 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3534774469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:38:38.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:37 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:38:38.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:37 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3503621736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:38:38.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:37 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3534774469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:38:38.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:37 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:38:38.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:38.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:38.334 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:38.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:38.435 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:38.438 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:38:38.438 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:38:38.438 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:38:38.438 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:38:38.438 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:38:38.438 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:38:38.438 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:38:38.438 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:38:38.439 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:38:38.439 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:38:38.439 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:38:38.439 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:38:38.439 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:38:38.439 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:38:38.439 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:38:38.439 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:38:38.466 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:38.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:38.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:38.513 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 102719 102703 76.4031 43.875 0.0195771 0.0130812 2024-04-12T11:38:38.567 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:38.596 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:38.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:38.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:38.698 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:38.729 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:38.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:38.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:38.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:38.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:38.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:38.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:38.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:38.982 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 1 (10000) 2024-04-12T11:38:39.004 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:39.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:39.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[93650]: pgmap v1541: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 5641/299907 objects degraded (1.881%); 5441/299907 objects misplaced (1.814%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T11:38:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:38:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:38:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[93650]: osdmap e1149: 8 total, 8 up, 8 in 2024-04-12T11:38:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1962826994' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T11:38:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T11:38:39.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[97787]: pgmap v1541: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 5641/299907 objects degraded (1.881%); 5441/299907 objects misplaced (1.814%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T11:38:39.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:38:39.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:38:39.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[97787]: osdmap e1149: 8 total, 8 up, 8 in 2024-04-12T11:38:39.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1962826994' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T11:38:39.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:38 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T11:38:39.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:39.146 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:39.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:39.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:38 smithi087 ceph-mon[89014]: pgmap v1541: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 5641/299907 objects degraded (1.881%); 5441/299907 objects misplaced (1.814%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T11:38:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:38 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:38:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:38 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:38:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:38 smithi087 ceph-mon[89014]: osdmap e1149: 8 total, 8 up, 8 in 2024-04-12T11:38:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:38 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1962826994' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T11:38:39.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:38 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T11:38:39.248 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:39.275 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:39.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:39.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:39.376 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:39.410 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:39.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:39.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:39.513 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 103585 103569 76.141 54.125 0.0162038 0.0131286 2024-04-12T11:38:39.558 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:39.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:39.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:39.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:39.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:39.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:39.827 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:39.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:39.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:39.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:39.970 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:39.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:39.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:40.072 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:40.101 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:40.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:40.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:40.202 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:40.233 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:40.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:40.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:39 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-04-12T11:38:40.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:39 smithi029 ceph-mon[97787]: osdmap e1150: 8 total, 8 up, 8 in 2024-04-12T11:38:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:39 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-04-12T11:38:40.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:39 smithi029 ceph-mon[93650]: osdmap e1150: 8 total, 8 up, 8 in 2024-04-12T11:38:40.335 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:40.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:40.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:40.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:40.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:39 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-04-12T11:38:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:39 smithi087 ceph-mon[89014]: osdmap e1150: 8 total, 8 up, 8 in 2024-04-12T11:38:40.495 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:40.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:40.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:40.513 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 104430 104414 75.8697 52.8125 0.00550538 0.0131753 2024-04-12T11:38:40.597 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:40.626 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:40.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:40.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:40.728 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:40.768 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:40.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:40.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:40.870 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:40.898 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:40.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:40.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:41.000 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:41.031 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:41.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:41.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:41.168 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:41.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:41.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:41.242 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:40 smithi029 ceph-mon[93650]: pgmap v1544: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 5641/307947 objects degraded (1.832%); 5441/307947 objects misplaced (1.767%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T11:38:41.242 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:40 smithi029 ceph-mon[97787]: pgmap v1544: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 5641/307947 objects degraded (1.832%); 5441/307947 objects misplaced (1.767%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T11:38:41.270 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:41.297 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:41.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:41.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:41.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:41.455 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:41.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:41.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:40 smithi087 ceph-mon[89014]: pgmap v1544: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 5641/307947 objects degraded (1.832%); 5441/307947 objects misplaced (1.767%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T11:38:41.513 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 105238 105222 75.578 50.5 0.0112197 0.0132252 2024-04-12T11:38:41.558 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:41.587 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:41.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:41.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:41.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:41.734 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:41.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:41.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:41.868 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:41.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:41.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:41.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:38:41.970 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:42.000 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:42.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:42.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:42.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 5 1 2024-04-12T11:38:42.102 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:42.129 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:42.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:42.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:42.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:42 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5641/308589 objects degraded (1.828%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:42.202 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:42 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5641/308589 objects degraded (1.828%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:42.230 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:42.258 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:42.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:42.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:42.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:42 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5641/308589 objects degraded (1.828%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:42.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:42.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:42.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:42.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 106013 105997 75.2691 48.4375 0.0144932 0.0132805 2024-04-12T11:38:42.538 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:42.565 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:42.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:42.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:42.667 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:42.696 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:42.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:42.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:38:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:38:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:38:42.764 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:38:42.798 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:42.824 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:42.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:42.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:42.925 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:42.951 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:42.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:42.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:43.014 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:38:43.014 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:38:43.053 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:43.079 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:43.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:43.181 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:43.211 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:43.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:43.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:43 smithi029 ceph-mon[93650]: pgmap v1545: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 5641/308589 objects degraded (1.828%); 5298/308589 objects misplaced (1.717%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T11:38:43.244 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:43 smithi029 ceph-mon[97787]: pgmap v1545: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 5641/308589 objects degraded (1.828%); 5298/308589 objects misplaced (1.717%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T11:38:43.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:43.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:43.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:43.444 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:43.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:43 smithi087 ceph-mon[89014]: pgmap v1545: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 708 op/s; 5641/308589 objects degraded (1.828%); 5298/308589 objects misplaced (1.717%); 2.2 MiB/s, 35 objects/s recovering 2024-04-12T11:38:43.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:43.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:43.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 106763 106747 74.95 46.875 0.00996685 0.0133368 2024-04-12T11:38:43.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:43.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:43.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:43.732 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:43.776 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:43.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:43.878 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:43.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:43.915 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:43.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:44.016 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:44.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:44.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:44.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:44.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:44.203 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:44.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:44.297 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:44.303 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:44.327 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:44.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:44.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:44.341 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-04-12T11:38:44.358 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:44.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:44.491 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:44.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:44.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 90 3 107500 107497 74.6381 46.875 0.0372972 0.0133919 2024-04-12T11:38:44.592 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:44.615 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:44.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:44.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:44.740 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:44.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:44.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:44.842 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:44.866 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:44.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:44.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:44.968 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:44.992 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:44.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:44.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:45.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:45.118 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:45.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:45.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:45.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:45.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:45.243 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:45.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:45.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:45 smithi029 ceph-mon[93650]: pgmap v1546: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 5641/315354 objects degraded (1.789%); 5298/315354 objects misplaced (1.680%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T11:38:45.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:45 smithi029 ceph-mon[97787]: pgmap v1546: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 5641/315354 objects degraded (1.789%); 5298/315354 objects misplaced (1.680%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T11:38:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:38:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:38:44] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.43.0" 2024-04-12T11:38:45.344 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:45.367 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:45.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:45.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:45.447 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:45.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:45 smithi087 ceph-mon[89014]: pgmap v1546: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 5641/315354 objects degraded (1.789%); 5298/315354 objects misplaced (1.680%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T11:38:45.469 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:45.492 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:45.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:45.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:45.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.022 2024-04-12T11:38:45.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 107500 2024-04-12T11:38:45.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2024-04-12T11:38:45.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2024-04-12T11:38:45.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 74.6345 2024-04-12T11:38:45.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 33.0221 2024-04-12T11:38:45.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 206.062 2024-04-12T11:38:45.514 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2024-04-12T11:38:45.515 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1194 2024-04-12T11:38:45.515 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 528.354 2024-04-12T11:38:45.515 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3297 2024-04-12T11:38:45.515 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2024-04-12T11:38:45.515 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0133928 2024-04-12T11:38:45.515 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0360094 2024-04-12T11:38:45.515 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 2.53616 2024-04-12T11:38:45.515 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00111552 2024-04-12T11:38:45.521 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2024-04-12T11:38:45.594 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:45.641 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:45.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:45.742 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:45.793 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:45.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:45.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:45.895 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:45.923 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:45.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:45.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:46.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:46.053 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:46.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:46.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:46.155 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:46.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:46.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:46.301 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:46.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:46.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:46.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:46.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:46.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:46.470 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:46.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:46.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:46.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:46.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:46.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:46.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:46.729 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:46.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:46.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:46.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:46.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:46.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:46.961 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:46.999 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:46.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:47.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:47.100 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:47.157 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:47.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:47.257 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:47.283 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:47.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:47.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:47.294 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:47 smithi029 ceph-mon[93650]: pgmap v1547: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 5641/320196 objects degraded (1.762%); 5141/320196 objects misplaced (1.606%); 2.3 MiB/s, 37 objects/s recovering 2024-04-12T11:38:47.294 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:47.294 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:47.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:47 smithi029 ceph-mon[97787]: pgmap v1547: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 5641/320196 objects degraded (1.762%); 5141/320196 objects misplaced (1.606%); 2.3 MiB/s, 37 objects/s recovering 2024-04-12T11:38:47.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:47.294 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:47.385 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:47.412 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:47.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:47.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:47.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:47 smithi087 ceph-mon[89014]: pgmap v1547: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 5641/320196 objects degraded (1.762%); 5141/320196 objects misplaced (1.606%); 2.3 MiB/s, 37 objects/s recovering 2024-04-12T11:38:47.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:47.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:47.556 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:47.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:47.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:47.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:47.711 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:47.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:47.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:47.813 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:47.839 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:47.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:47.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:47.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:47.981 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:47.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:47.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:48.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:48.123 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:48.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:48.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:48.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:48.261 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:48.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:48.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:48.363 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:48.375 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:38:48.375 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:38:48.375 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:48.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5641/320196 objects degraded (1.762%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:48.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:38:48.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:38:48.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:48.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5641/320196 objects degraded (1.762%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:48.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:48.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:48.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:48.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:38:48.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:38:48.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:38:48.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5641/320196 objects degraded (1.762%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:48.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:48.521 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:48.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:48.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:48.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:48.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:48.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:48.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:48.813 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:48.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:48.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:48.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:48.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:48.960 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:48.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:49.061 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:49.107 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:49.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:49.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:49.209 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:49.252 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:49.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:49.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:49.344 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:49 smithi029 ceph-mon[93650]: pgmap v1548: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 104 B/s rd, 40 MiB/s wr, 669 op/s; 5641/320295 objects degraded (1.761%); 5141/320295 objects misplaced (1.605%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:38:49.347 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:49 smithi029 ceph-mon[97787]: pgmap v1548: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 104 B/s rd, 40 MiB/s wr, 669 op/s; 5641/320295 objects degraded (1.761%); 5141/320295 objects misplaced (1.605%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:38:49.354 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:49.382 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:49.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:49.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:49.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:49 smithi087 ceph-mon[89014]: pgmap v1548: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 104 B/s rd, 40 MiB/s wr, 669 op/s; 5641/320295 objects degraded (1.761%); 5141/320295 objects misplaced (1.605%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:38:49.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:49.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:49.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:49.629 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:49.661 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:49.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:49.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:49.763 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:49.790 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:49.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:49.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:49.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:49.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:49.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:49.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:50.033 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:50.067 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:50.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:50.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:50.169 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:50.196 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:50.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:50.297 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:50.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:50.327 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:50.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:50.428 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:50.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:50.500 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:50.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:50.601 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:50.658 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:50.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:50.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:50.760 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:50.790 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:50.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:50.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:50.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:50.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:50.930 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:50.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:51.031 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:51.057 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:51.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:51.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:51.112 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.5 to 1 (10000) 2024-04-12T11:38:51.158 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:51.185 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:51.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:51.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:51 smithi029 ceph-mon[97787]: pgmap v1549: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 94 B/s rd, 37 MiB/s wr, 793 op/s; 5641/315993 objects degraded (1.785%); 5141/315993 objects misplaced (1.627%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:38:51.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:51 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2953998744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T11:38:51.287 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:51.296 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:51 smithi029 ceph-mon[93650]: pgmap v1549: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 94 B/s rd, 37 MiB/s wr, 793 op/s; 5641/315993 objects degraded (1.785%); 5141/315993 objects misplaced (1.627%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:38:51.296 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:51 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2953998744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T11:38:51.358 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:51.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:51.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:51.459 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:51.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:51 smithi087 ceph-mon[89014]: pgmap v1549: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 94 B/s rd, 37 MiB/s wr, 793 op/s; 5641/315993 objects degraded (1.785%); 5141/315993 objects misplaced (1.627%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:38:51.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:51 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2953998744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T11:38:51.485 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:51.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:51.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:51.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:51.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:51.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:51.660 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:38:51.708 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:38:51.732 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:51.757 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:38:51.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:51.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:51.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:51.814 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:38:51.859 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:38:51.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:51.903 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:51.905 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:38:51.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:51.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:51.957 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:38:51.975 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:38:52.004 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:38:52.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:52.018 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:38:52.018 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:38:52.018 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:38:52.018 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:38:52.018 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:38:52.018 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:38:52.018 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:38:52.019 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:38:52.065 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:52.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:52.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:52.167 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:52.195 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:52.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:52.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:52 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2953998744' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-04-12T11:38:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:52 smithi029 ceph-mon[93650]: osdmap e1151: 8 total, 8 up, 8 in 2024-04-12T11:38:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:52 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2941981651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:38:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:52 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2953998744' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-04-12T11:38:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:52 smithi029 ceph-mon[97787]: osdmap e1151: 8 total, 8 up, 8 in 2024-04-12T11:38:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:52.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:52 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2941981651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:38:52.296 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:52.324 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:52.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:52.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:52.368 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:38:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:38:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:38:52.369 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:52 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2953998744' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-04-12T11:38:52.369 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:52 smithi087 ceph-mon[89014]: osdmap e1151: 8 total, 8 up, 8 in 2024-04-12T11:38:52.369 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:38:52.369 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:52 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2941981651' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:38:52.426 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:52.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 6 1 2024-04-12T11:38:52.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:52.473 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:52.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:52.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:52.637 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:52.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:52.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:52.739 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:52.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:52.771 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:52.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:52.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:52.899 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:52.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:52.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:53.001 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:53.040 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:53.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:53.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:53.127 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:38:53.141 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:53.167 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:53.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:53.268 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:53.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:53.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:53.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:53.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:53.451 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:53.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:53.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:53 smithi087 ceph-mon[89014]: pgmap v1551: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 29 MiB/s wr, 707 op/s; 5641/315327 objects degraded (1.789%); 4967/315327 objects misplaced (1.575%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:38:53.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:53 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4100775908' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:38:53.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5641/315327 objects degraded (1.789%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:53.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:53 smithi029 ceph-mon[97787]: pgmap v1551: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 29 MiB/s wr, 707 op/s; 5641/315327 objects degraded (1.789%); 4967/315327 objects misplaced (1.575%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:38:53.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:53 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4100775908' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:38:53.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5641/315327 objects degraded (1.789%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:53.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:53 smithi029 ceph-mon[93650]: pgmap v1551: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 29 MiB/s wr, 707 op/s; 5641/315327 objects degraded (1.789%); 4967/315327 objects misplaced (1.575%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:38:53.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:53 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4100775908' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:38:53.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5641/315327 objects degraded (1.789%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:53.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:53.591 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:53.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:53.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:53.693 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:53.722 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:53.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:53.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:53.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:53.847 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:53.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:53.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:53.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:53.978 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:53.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:53.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:54.080 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:54.109 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:54.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:54.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:54.211 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:54.234 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:54.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:54.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:54.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:54 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:38:54.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:54 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4100775908' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:38:54.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:54 smithi029 ceph-mon[97787]: osdmap e1152: 8 total, 8 up, 8 in 2024-04-12T11:38:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:54 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:38:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:54 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4100775908' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:38:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:54 smithi029 ceph-mon[93650]: osdmap e1152: 8 total, 8 up, 8 in 2024-04-12T11:38:54.336 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:54.367 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:54.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:54.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:54.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:54 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:38:54.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:54 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4100775908' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:38:54.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:54 smithi087 ceph-mon[89014]: osdmap e1152: 8 total, 8 up, 8 in 2024-04-12T11:38:54.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:54.507 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:54.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:54.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:54.632 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:54.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:54.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:54.734 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:54.762 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:54.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:54.864 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:54.899 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:54.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:54.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:55.001 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:55.039 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:55.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:55.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:55.140 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:55.173 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:55.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:55.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:55 smithi029 ceph-mon[93650]: pgmap v1553: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 6.2 MiB/s wr, 665 op/s; 5641/308991 objects degraded (1.826%); 4967/308991 objects misplaced (1.607%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T11:38:55.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:38:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:38:54] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2024-04-12T11:38:55.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:55 smithi029 ceph-mon[97787]: pgmap v1553: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 6.2 MiB/s wr, 665 op/s; 5641/308991 objects degraded (1.826%); 4967/308991 objects misplaced (1.607%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T11:38:55.275 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:55.298 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:55.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:55.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:55.400 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:55.424 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:55.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:55.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:55 smithi087 ceph-mon[89014]: pgmap v1553: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 6.2 MiB/s wr, 665 op/s; 5641/308991 objects degraded (1.826%); 4967/308991 objects misplaced (1.607%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T11:38:55.526 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:55.557 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:55.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:55.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:55.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:55.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:55.790 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:55.813 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:55.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:55.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:55.914 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:55.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:55.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:55.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:56.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:56.083 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:56.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:56.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:56.185 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:56.211 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:56.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:56.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:56.313 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:56.349 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:56.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:56.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:56.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:56.474 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:56.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:56.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:56.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:56.616 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:56.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:56.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:56.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:56.761 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:56.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:56.862 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:56.891 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:56.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:56.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:56.993 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:57.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:57.021 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:57.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:57.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:57.150 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:57.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:57.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:57.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:57.276 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:57.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:57.377 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:57.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:57.401 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:57.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:57 smithi087 ceph-mon[89014]: pgmap v1554: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 3.1 MiB/s wr, 777 op/s; 5641/304014 objects degraded (1.856%); 4835/304014 objects misplaced (1.590%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:38:57.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:57.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:57 smithi029 ceph-mon[97787]: pgmap v1554: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 3.1 MiB/s wr, 777 op/s; 5641/304014 objects degraded (1.856%); 4835/304014 objects misplaced (1.590%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:38:57.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:57 smithi029 ceph-mon[93650]: pgmap v1554: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 3.1 MiB/s wr, 777 op/s; 5641/304014 objects degraded (1.856%); 4835/304014 objects misplaced (1.590%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:38:57.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:57.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:57.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:57.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:57.663 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:57.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:57.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:57.765 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:57.788 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:57.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:38:57.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:38:57.936 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:57.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:57.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:38:58.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:58.068 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:58.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:58.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:58.170 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:58.200 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:58.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:58.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:58.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5641/304014 objects degraded (1.856%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:58.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5641/304014 objects degraded (1.856%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:58.302 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:58.331 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:58.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:58.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:58.432 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:58.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5641/304014 objects degraded (1.856%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:38:58.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:58.471 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:58.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:58.572 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:58.604 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:58.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:58.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:58.706 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:58.734 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:58.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:58.836 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:38:58.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:58.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:38:58.963 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:38:58.991 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:58.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:58.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:38:59.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:38:59.135 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:59.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:59.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:38:59.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:59.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:59.274 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:59.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:59.375 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:59.403 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:59.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:59.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:59.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:38:59 smithi087 ceph-mon[89014]: pgmap v1555: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 5641/301587 objects degraded (1.870%); 4835/301587 objects misplaced (1.603%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:38:59.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:38:59.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:38:59 smithi029 ceph-mon[97787]: pgmap v1555: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 5641/301587 objects degraded (1.870%); 4835/301587 objects misplaced (1.603%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:38:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:38:59 smithi029 ceph-mon[93650]: pgmap v1555: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 5641/301587 objects degraded (1.870%); 4835/301587 objects misplaced (1.603%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:38:59.528 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:59.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:38:59.629 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:38:59.656 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:38:59.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:59.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:38:59.758 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:38:59.783 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:59.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:38:59.885 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:38:59.914 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:38:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:38:59.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:00.016 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:00.050 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:00.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:00.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:00.152 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:00.176 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:00.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:00.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:00.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:00.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:00.320 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:00.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:00.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:00.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:00.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:00.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:00.610 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:00.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:00.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:00.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:00.750 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:00.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:00.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:00.852 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:00.890 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:00.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:00.992 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:01.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:01.042 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:01.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:01.143 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:01.174 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:01.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:01.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:01.275 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:01.307 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:01.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:01.409 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:01.453 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:01.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:01.457 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:01 smithi029 ceph-mon[97787]: pgmap v1556: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 5641/295395 objects degraded (1.910%); 4835/295395 objects misplaced (1.637%); 973 KiB/s, 15 objects/s recovering 2024-04-12T11:39:01.458 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:01 smithi029 ceph-mon[93650]: pgmap v1556: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 5641/295395 objects degraded (1.910%); 4835/295395 objects misplaced (1.637%); 973 KiB/s, 15 objects/s recovering 2024-04-12T11:39:01.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:01 smithi087 ceph-mon[89014]: pgmap v1556: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 5641/295395 objects degraded (1.910%); 4835/295395 objects misplaced (1.637%); 973 KiB/s, 15 objects/s recovering 2024-04-12T11:39:01.555 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:01.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:01.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:01.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:01.688 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:01.717 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:01.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:01.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:01.820 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:01.847 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:01.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:01.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:01.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:01.980 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:01.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:01.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:02.082 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:02.128 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:02.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:02.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:02.230 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:02.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:02.275 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:02.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:02.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:02.419 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:02.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:02.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:02.521 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:02.548 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:02.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:02.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:02.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:02.693 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:02.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:02.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:39:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:39:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:39:02.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:02.811 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:02.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:02.844 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:02.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:02.921 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:39:02.921 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:39:02.921 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:39:02.921 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:39:02.922 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:39:02.945 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:02.971 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:02.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:02.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:03.073 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:03.100 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:03.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:03.202 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:03.230 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:03.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:03.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:03.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:03.374 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:03.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:03.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:03 smithi087 ceph-mon[89014]: pgmap v1557: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 5641/294861 objects degraded (1.913%); 4718/294861 objects misplaced (1.600%); 1.6 MiB/s, 24 objects/s recovering 2024-04-12T11:39:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5641/294861 objects degraded (1.913%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:03.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:03.510 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:03 smithi029 ceph-mon[97787]: pgmap v1557: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 5641/294861 objects degraded (1.913%); 4718/294861 objects misplaced (1.600%); 1.6 MiB/s, 24 objects/s recovering 2024-04-12T11:39:03.510 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5641/294861 objects degraded (1.913%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:03.511 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:03 smithi029 ceph-mon[93650]: pgmap v1557: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 5641/294861 objects degraded (1.913%); 4718/294861 objects misplaced (1.600%); 1.6 MiB/s, 24 objects/s recovering 2024-04-12T11:39:03.511 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5641/294861 objects degraded (1.913%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:03.518 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:03.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:03.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:03.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:03.650 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:03.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:03.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:03.752 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:03.782 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:03.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:03.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:03.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:39:03.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-04-12T11:39:03.884 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:03.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:03.964 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:03.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:04.065 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:04.091 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:04.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:04.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:04.193 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:04.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:04.242 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:04.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:04.343 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:04.371 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:04.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:04.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:04.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:04.533 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:04.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:04.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:04.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:39:04.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:04.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:04.694 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:04.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:04.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:04.849 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:04.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:04.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:04.915 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:04.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:04.953 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:04.976 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:04.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:05.078 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:05.108 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:05.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:05.144 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:39:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:39:04] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2024-04-12T11:39:05.209 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:05.241 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:05.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:05.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:05.343 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:05.373 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:05.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:05.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:05.402 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:05 smithi029 ceph-mon[97787]: pgmap v1558: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 5641/288984 objects degraded (1.952%); 4718/288984 objects misplaced (1.633%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T11:39:05.403 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:05 smithi029 ceph-mon[93650]: pgmap v1558: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 5641/288984 objects degraded (1.952%); 4718/288984 objects misplaced (1.633%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T11:39:05.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:05 smithi087 ceph-mon[89014]: pgmap v1558: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 5641/288984 objects degraded (1.952%); 4718/288984 objects misplaced (1.633%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T11:39:05.475 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:05.500 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:05.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:05.602 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:05.634 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:05.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:05.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:05.736 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:05.765 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:05.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:05.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:05.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:05.911 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:05.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:05.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:06.013 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:06.041 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:06.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:06.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:06.146 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:06.180 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:06.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:06.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:06.282 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:06.310 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:06.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:06.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:06.412 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:06.437 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:06.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:06.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:06.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:06.564 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:06.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:06.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:06.666 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:06.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:06.695 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:06.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:06.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:06.819 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:06.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:06.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:06.921 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:06.956 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:06.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:07.058 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:07.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:07.088 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:07.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:07.189 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:07.213 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:07.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:07.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:07.315 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:07.341 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:07.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:07.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:07.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:07.470 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:07 smithi087 ceph-mon[89014]: pgmap v1559: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 5641/284070 objects degraded (1.986%); 4583/284070 objects misplaced (1.613%); 2.0 MiB/s, 31 objects/s recovering 2024-04-12T11:39:07.470 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:07.470 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:07.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:07.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:07.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:07.501 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:07 smithi029 ceph-mon[97787]: pgmap v1559: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 5641/284070 objects degraded (1.986%); 4583/284070 objects misplaced (1.613%); 2.0 MiB/s, 31 objects/s recovering 2024-04-12T11:39:07.501 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:07.501 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:07.502 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:07 smithi029 ceph-mon[93650]: pgmap v1559: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 5641/284070 objects degraded (1.986%); 4583/284070 objects misplaced (1.613%); 2.0 MiB/s, 31 objects/s recovering 2024-04-12T11:39:07.502 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:07.502 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:07.572 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:07.600 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:07.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:07.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:07.702 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:07.727 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:07.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:07.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:07.779 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:07.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:07.855 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:07.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:07.957 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:07.985 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:07.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:08.087 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:08.116 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:08.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:08.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:08.218 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:08.268 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:08.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:08.370 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:08.410 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:08.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:08.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5641/284070 objects degraded (1.986%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:08.502 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5641/284070 objects degraded (1.986%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:08.503 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5641/284070 objects degraded (1.986%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:08.512 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:08.539 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:08.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:08.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:08.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:08.668 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:08.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:08.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:08.770 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:08.824 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:08.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:08.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:08.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:08.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:08.984 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:08.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:09.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:09.125 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:09.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:09.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:09.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:09.252 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:09.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:09.354 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:09.361 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:09 smithi029 ceph-mon[93650]: pgmap v1560: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5641/281553 objects degraded (2.004%); 4583/281553 objects misplaced (1.628%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T11:39:09.362 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:09 smithi029 ceph-mon[97787]: pgmap v1560: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5641/281553 objects degraded (2.004%); 4583/281553 objects misplaced (1.628%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T11:39:09.435 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:09.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:09.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:09.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:09 smithi087 ceph-mon[89014]: pgmap v1560: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5641/281553 objects degraded (2.004%); 4583/281553 objects misplaced (1.628%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T11:39:09.537 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:09.564 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:09.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:09.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:09.665 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:09.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:09.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:09.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:09.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:09.848 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:09.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:09.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:09.950 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:09.995 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:09.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:09.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:10.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:10.140 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:10.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:10.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:10.153 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:39:10.197 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:39:10.242 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:10.242 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:39:10.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:10.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:10.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:10.287 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:39:10.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1927671730' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:39:10.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1927671730' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:39:10.328 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:39:10.370 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:39:10.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:10.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:10.398 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:10.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:10.420 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:39:10.461 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:39:10.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1927671730' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:39:10.468 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:39:10.468 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:39:10.468 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:39:10.468 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:39:10.468 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:39:10.468 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:39:10.468 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:39:10.468 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:39:10.499 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:10.527 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:10.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:10.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:10.629 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:10.658 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:10.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:10.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:10.760 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:10.798 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:10.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:10.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:10.900 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:10.950 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:10.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:10.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:11.051 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:11.088 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:11.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:11.190 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:11.191 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.6 to 1 (10000) 2024-04-12T11:39:11.217 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:11.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:11.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:11.319 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:11.346 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:11.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:11.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:11.450 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:11.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:11 smithi087 ceph-mon[89014]: pgmap v1561: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 5641/274248 objects degraded (2.057%); 4583/274248 objects misplaced (1.671%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T11:39:11.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:11 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1818558046' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T11:39:11.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:11 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T11:39:11.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:11.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:11.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:11 smithi029 ceph-mon[97787]: pgmap v1561: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 5641/274248 objects degraded (2.057%); 4583/274248 objects misplaced (1.671%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T11:39:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:11 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1818558046' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T11:39:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:11 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T11:39:11.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:11 smithi029 ceph-mon[93650]: pgmap v1561: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 5641/274248 objects degraded (2.057%); 4583/274248 objects misplaced (1.671%); 1.3 MiB/s, 20 objects/s recovering 2024-04-12T11:39:11.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:11 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1818558046' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T11:39:11.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:11 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T11:39:11.600 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:11.626 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:11.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:11.728 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:11.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:11.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:11.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:11.811 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:39:11.811 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:39:11.856 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:11.883 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:11.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:11.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:11.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:12.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:12.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:12.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:12.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:12.158 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:12.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:12.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:12.213 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 7 1 2024-04-12T11:39:12.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:12.290 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:12.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:12.391 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:12.418 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:12.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:12.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:39:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:39:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:39:12.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:12 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-04-12T11:39:12.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:12 smithi087 ceph-mon[89014]: osdmap e1153: 8 total, 8 up, 8 in 2024-04-12T11:39:12.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:12 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-04-12T11:39:12.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:12 smithi029 ceph-mon[97787]: osdmap e1153: 8 total, 8 up, 8 in 2024-04-12T11:39:12.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:12 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-04-12T11:39:12.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:12 smithi029 ceph-mon[93650]: osdmap e1153: 8 total, 8 up, 8 in 2024-04-12T11:39:12.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:12.573 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:12.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:12.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:12.675 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:12.703 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:12.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:12.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:12.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:12.847 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:12.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:12.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:12.948 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:12.995 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:12.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:13.097 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:13.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:13.133 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:13.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:13.234 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:13.261 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:13.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:13.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:39:13.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:13.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:13.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:13.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:13.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:13 smithi087 ceph-mon[89014]: pgmap v1563: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5641/273642 objects degraded (2.061%); 4422/273642 objects misplaced (1.616%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:13.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5641/273642 objects degraded (2.061%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:13.470 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:13 smithi029 ceph-mon[93650]: pgmap v1563: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5641/273642 objects degraded (2.061%); 4422/273642 objects misplaced (1.616%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:13.470 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5641/273642 objects degraded (2.061%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:13.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:13 smithi029 ceph-mon[97787]: pgmap v1563: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5641/273642 objects degraded (2.061%); 4422/273642 objects misplaced (1.616%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:13.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5641/273642 objects degraded (2.061%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:13.524 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:13.550 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:13.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:13.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:13.652 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:13.683 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:13.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:13.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:13.785 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:13.812 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:13.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:13.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:13.914 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:13.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-04-12T11:39:13.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:13.987 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:13.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:14.088 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:14.115 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:14.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:14.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:14.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:14.269 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:14.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:14.369 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:14.398 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:14.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:14.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:14.500 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:14.526 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:14.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:14.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:14.577 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:14.598 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:14.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:14.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:14.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:14.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:14.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:14.813 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:14.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:14.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:14.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:14.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:14.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:14.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:14.985 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:39:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:39:14] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2024-04-12T11:39:15.057 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:15.083 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:15.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:15.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:15.184 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:15.211 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:15.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:15.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:15.237 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:15 smithi029 ceph-mon[97787]: pgmap v1564: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 5641/267666 objects degraded (2.107%); 4422/267666 objects misplaced (1.652%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:15.237 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:15 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3772662291' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:39:15.237 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:15 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:39:15.241 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:15 smithi029 ceph-mon[93650]: pgmap v1564: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 5641/267666 objects degraded (2.107%); 4422/267666 objects misplaced (1.652%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:15.242 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:15 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3772662291' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:39:15.242 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:15 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:39:15.242 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:39:15.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:15.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:15.343 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:15.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:15.445 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:15.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:15 smithi087 ceph-mon[89014]: pgmap v1564: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 5641/267666 objects degraded (2.107%); 4422/267666 objects misplaced (1.652%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:15.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:15 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3772662291' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:39:15.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:15 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:39:15.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:15.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:15.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:15.585 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:15.616 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:15.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:15.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:15.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:15.757 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:15.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:15.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:15.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:15.908 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:15.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:15.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:16.010 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:16.038 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:16.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:16.139 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:16.168 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:16.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:16.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:16.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:16 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:39:16.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:16 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:39:16.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:16 smithi029 ceph-mon[93650]: osdmap e1154: 8 total, 8 up, 8 in 2024-04-12T11:39:16.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:16 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:39:16.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:16 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:39:16.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:16 smithi029 ceph-mon[97787]: osdmap e1154: 8 total, 8 up, 8 in 2024-04-12T11:39:16.270 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:16.296 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:16.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:16.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:16.398 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:16.425 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:16.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:16.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:16.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:16 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:39:16.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:16 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:39:16.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:16 smithi087 ceph-mon[89014]: osdmap e1154: 8 total, 8 up, 8 in 2024-04-12T11:39:16.527 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:16.554 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:16.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:16.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:16.656 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:16.685 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:16.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:16.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:16.787 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:16.835 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:16.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:16.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:16.937 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:16.978 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:16.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:16.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:17.079 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:17.106 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:17.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:17.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:17.208 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:17.235 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:17.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:17.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:17.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:17 smithi029 ceph-mon[93650]: pgmap v1566: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 5641/262848 objects degraded (2.146%); 4276/262848 objects misplaced (1.627%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:39:17.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:17 smithi029 ceph-mon[97787]: pgmap v1566: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 5641/262848 objects degraded (2.146%); 4276/262848 objects misplaced (1.627%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:39:17.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:17.383 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:17.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:17.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:17 smithi087 ceph-mon[89014]: pgmap v1566: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 5641/262848 objects degraded (2.146%); 4276/262848 objects misplaced (1.627%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:39:17.485 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:17.513 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:17.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:17.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:17.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:17.652 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:17.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:17.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:17.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:17.797 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:17.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:17.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:17.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:17.937 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:17.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:18.038 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:18.066 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:18.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:18.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:18.168 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:18.192 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:18.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:18.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:18.243 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:18.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:18.338 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:39:18.339 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:39:18.339 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5641/262848 objects degraded (2.146%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:18.339 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5641/262848 objects degraded (2.146%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:18.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:18.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:18.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:18.449 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:18.480 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:18.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:18.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:18.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:18.628 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:18.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:18.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:18.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5641/262848 objects degraded (2.146%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:18.729 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:18.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:18.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:18.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:18.861 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:18.887 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:18.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:18.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:18.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:19.021 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:39:19.034 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:19.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:19.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:19.137 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:19.163 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:19.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:19.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:19.264 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:19.327 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:19.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:19.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:19.429 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:19.455 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:19.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:19.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:19.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:19 smithi029 ceph-mon[93650]: pgmap v1567: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 5641/260121 objects degraded (2.169%); 4276/260121 objects misplaced (1.644%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:39:19.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:19 smithi029 ceph-mon[97787]: pgmap v1567: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 5641/260121 objects degraded (2.169%); 4276/260121 objects misplaced (1.644%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:39:19.557 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:19.585 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:19.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:19.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:19.601 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-04-12T11:39:19.687 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:19.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:19 smithi087 ceph-mon[89014]: pgmap v1567: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 5641/260121 objects degraded (2.169%); 4276/260121 objects misplaced (1.644%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:39:19.717 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:19.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:19.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:19.818 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:19.845 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:19.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:19.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:19.947 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:19.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:19.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:20.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:20.101 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:20.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:20.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:20.203 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:20.233 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:20.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:20.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:20.335 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:20.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:20.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:20.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:20.512 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:20.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:20.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:20.613 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:20.641 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:20.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:20.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:20.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:20 smithi087 ceph-mon[89014]: pgmap v1568: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5641/255237 objects degraded (2.210%); 4276/255237 objects misplaced (1.675%); 1.1 MiB/s, 16 objects/s recovering 2024-04-12T11:39:20.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:20.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:20 smithi029 ceph-mon[93650]: pgmap v1568: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5641/255237 objects degraded (2.210%); 4276/255237 objects misplaced (1.675%); 1.1 MiB/s, 16 objects/s recovering 2024-04-12T11:39:20.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:20 smithi029 ceph-mon[97787]: pgmap v1568: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5641/255237 objects degraded (2.210%); 4276/255237 objects misplaced (1.675%); 1.1 MiB/s, 16 objects/s recovering 2024-04-12T11:39:20.792 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:20.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:20.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:20.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:20.940 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:20.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:21.041 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:21.081 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:21.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:21.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:21.182 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:21.209 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:21.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:21.311 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:21.349 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:21.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:21.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:21.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:21.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:21.534 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:21.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:21.635 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:21.662 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:21.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:21.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:21.764 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:21.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:21.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:21.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:21.908 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:21.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:21.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:21.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:22.040 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:22.069 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:22.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:22.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:22.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:22.200 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:22.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:22.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:22.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:22.317 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:22.320 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:22.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:22.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:22.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2509420770' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T11:39:22.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T11:39:22.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[93650]: pgmap v1569: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 5641/251865 objects degraded (2.240%); 4123/251865 objects misplaced (1.637%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T11:39:22.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/148240376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:39:22.328 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:39:22.343 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.7 to 1 (10000) 2024-04-12T11:39:22.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:22.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:22.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:22.383 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:39:22.433 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:39:22.448 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:22.475 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:39:22.477 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:22.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:22.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:22.513 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:39:22.555 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:39:22.579 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:22.602 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:39:22.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:22.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:22.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2509420770' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T11:39:22.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T11:39:22.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[97787]: pgmap v1569: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 5641/251865 objects degraded (2.240%); 4123/251865 objects misplaced (1.637%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T11:39:22.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:22 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/148240376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:39:22.605 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:22.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:22.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:22.640 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:39:22.651 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:39:22.651 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:39:22.651 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:39:22.651 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:39:22.651 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:39:22.651 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:39:22.651 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:39:22.651 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:39:22.707 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:22.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:39:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:39:22.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:39:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:22 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2509420770' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T11:39:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:22 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T11:39:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:22 smithi087 ceph-mon[89014]: pgmap v1569: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 5641/251865 objects degraded (2.240%); 4123/251865 objects misplaced (1.637%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T11:39:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:22 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/148240376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:39:22.744 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:22.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:22.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:22.887 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:22.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:22.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:22.989 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:23.016 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:23.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:23.118 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:23.136 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-04-12T11:39:23.136 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:39:23.146 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:23.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:23.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:23.304 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:23.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:23.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:23.406 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:23.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:23 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-04-12T11:39:23.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:23 smithi029 ceph-mon[97787]: osdmap e1155: 8 total, 8 up, 8 in 2024-04-12T11:39:23.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5641/251865 objects degraded (2.240%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:23.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:23 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-04-12T11:39:23.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:23 smithi029 ceph-mon[93650]: osdmap e1155: 8 total, 8 up, 8 in 2024-04-12T11:39:23.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5641/251865 objects degraded (2.240%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:23.433 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:23.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:23.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:23.534 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:23.583 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:23.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:23.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:23.685 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:23.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:23 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-04-12T11:39:23.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:23 smithi087 ceph-mon[89014]: osdmap e1155: 8 total, 8 up, 8 in 2024-04-12T11:39:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5641/251865 objects degraded (2.240%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:39:23.725 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:23.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:23.827 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:23.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:23.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:23.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:24.023 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:24.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:24.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:24.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:24.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:24.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:24.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:24.291 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:24.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:24.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:24.392 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:24.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:24.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:24.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:24.521 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:24.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:24.549 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:24.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:24.649 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:24.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:24.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:24.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:24.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:24 smithi087 ceph-mon[89014]: osdmap e1156: 8 total, 8 up, 8 in 2024-04-12T11:39:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:24 smithi087 ceph-mon[89014]: pgmap v1572: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5641/245715 objects degraded (2.296%); 4123/245715 objects misplaced (1.678%); 1.2 MiB/s, 19 objects/s recovering 2024-04-12T11:39:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:24 smithi029 ceph-mon[97787]: osdmap e1156: 8 total, 8 up, 8 in 2024-04-12T11:39:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:24 smithi029 ceph-mon[97787]: pgmap v1572: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5641/245715 objects degraded (2.296%); 4123/245715 objects misplaced (1.678%); 1.2 MiB/s, 19 objects/s recovering 2024-04-12T11:39:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:24 smithi029 ceph-mon[93650]: osdmap e1156: 8 total, 8 up, 8 in 2024-04-12T11:39:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:24 smithi029 ceph-mon[93650]: pgmap v1572: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5641/245715 objects degraded (2.296%); 4123/245715 objects misplaced (1.678%); 1.2 MiB/s, 19 objects/s recovering 2024-04-12T11:39:24.780 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:24.803 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:24.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:24.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:24.904 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:24.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:24.932 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:24.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:25.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:25.062 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:25.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:25.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:25.164 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:25.187 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:25.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:25.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:39:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:39:24] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2024-04-12T11:39:25.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:25.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:25.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:25.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:25.446 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:25.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:25.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:25.548 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:25.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:25.572 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:25.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:25.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:25.710 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:25.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:25.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:25.812 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:25.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:25.839 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:25.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:25.940 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:25.966 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:25.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:25.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:26.068 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:26.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:26.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:26.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:26.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:26.223 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:26.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:26.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:26.325 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:26.353 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:26.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:26.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:26.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:26.478 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:26.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:26.580 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:26.607 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:26.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:26.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:26.747 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:26.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:26.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:26.849 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:26.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:26.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:26.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:26.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:27.000 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:27.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:27.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:27.102 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:27.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:27.126 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:27.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:26 smithi087 ceph-mon[89014]: pgmap v1573: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 5647/241284 objects degraded (2.340%); 3967/241284 objects misplaced (1.644%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:39:27.227 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:27.263 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:27.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:27.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:26 smithi029 ceph-mon[97787]: pgmap v1573: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 5647/241284 objects degraded (2.340%); 3967/241284 objects misplaced (1.644%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:39:27.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:26 smithi029 ceph-mon[93650]: pgmap v1573: 89 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 5647/241284 objects degraded (2.340%); 3967/241284 objects misplaced (1.644%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:39:27.365 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:27.409 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:27.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:27.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:27.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:27.552 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:27.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:27.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:27.654 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:27.680 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:27.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:27.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:27.804 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:27.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:27.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:27.906 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:27.929 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:27.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:28.031 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:28.055 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:28.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:28.157 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:28.180 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:28.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:28.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:28.282 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:28.309 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:28.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:28.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:28.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:28.434 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:28.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:28.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:28.536 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:28.565 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:28.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:28.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:28.667 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:28.707 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:28.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:28.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:28.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:28.844 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:28.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:28.945 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:28.984 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:28.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:28.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:29.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:29.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:29.125 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:29.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:28 smithi087 ceph-mon[89014]: pgmap v1574: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5647/239001 objects degraded (2.363%); 3967/239001 objects misplaced (1.660%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:39:29.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5647/239001 objects degraded (2.363%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:29.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:29.266 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:29.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:29.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:29.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:28 smithi029 ceph-mon[97787]: pgmap v1574: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5647/239001 objects degraded (2.363%); 3967/239001 objects misplaced (1.660%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:39:29.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5647/239001 objects degraded (2.363%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:29.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:28 smithi029 ceph-mon[93650]: pgmap v1574: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5647/239001 objects degraded (2.363%); 3967/239001 objects misplaced (1.660%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:39:29.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5647/239001 objects degraded (2.363%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:29.368 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:29.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:29.413 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:29.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:29.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:29.556 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:29.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:29.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:29.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:29.680 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:29.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:29.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:29.782 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:29.809 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:29.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:29.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:29.911 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:29.939 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:29.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:29.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:30.041 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:30.086 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:30.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:30.188 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:30.229 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:30.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:30.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:30.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:30.354 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:30.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:30.456 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:30.485 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:30.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:30.586 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:30.617 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:30.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:30.719 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:30.749 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:30.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:30.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:30.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:30.894 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:30.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:30.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:30.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:31.030 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:31.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:31.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:31.132 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:31.161 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:31.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:31.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:31.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:30 smithi087 ceph-mon[89014]: pgmap v1575: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 5647/236352 objects degraded (2.389%); 3967/236352 objects misplaced (1.678%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T11:39:31.262 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:31.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:30 smithi029 ceph-mon[93650]: pgmap v1575: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 5647/236352 objects degraded (2.389%); 3967/236352 objects misplaced (1.678%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T11:39:31.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:30 smithi029 ceph-mon[97787]: pgmap v1575: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 5647/236352 objects degraded (2.389%); 3967/236352 objects misplaced (1.678%); 1.2 MiB/s, 18 objects/s recovering 2024-04-12T11:39:31.292 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:31.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:31.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:31.394 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:31.422 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:31.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:31.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:31.523 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:31.559 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:31.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:31.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:31.660 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:31.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:31.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:31.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:31.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:31.828 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:31.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:31.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:31.930 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:31.957 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:31.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:32.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:32.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:32.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:32.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:32.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:32.229 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:32.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:32.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:32.330 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:32.367 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:32.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:32.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:32.469 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:32.506 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:32.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:32.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:32.607 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:32.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:32.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:32.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:39:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:39:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:39:32.750 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:32.788 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:32.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:32.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:32.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:32.927 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:32.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:32.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:32.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:32 smithi029 ceph-mon[97787]: pgmap v1576: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 5686/232983 objects degraded (2.441%); 3790/232983 objects misplaced (1.627%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:39:32.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:32 smithi029 ceph-mon[93650]: pgmap v1576: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 5686/232983 objects degraded (2.441%); 3790/232983 objects misplaced (1.627%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:39:33.029 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:33.057 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:33.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:33.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:33.158 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:33.188 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:33.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:33.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:32 smithi087 ceph-mon[89014]: pgmap v1576: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 5686/232983 objects degraded (2.441%); 3790/232983 objects misplaced (1.627%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:39:33.290 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:33.328 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:33.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:33.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:33.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:33.486 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:33.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:33.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:33.588 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:33.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:33.668 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:33.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:33.768 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:33.796 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:33.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:33.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:33.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:33.953 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:39:33.965 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:33.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:33.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5686/232983 objects degraded (2.441%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:33.996 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5686/232983 objects degraded (2.441%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:34.066 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:34.096 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:34.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:34.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:34.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:34.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5686/232983 objects degraded (2.441%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:34.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:34.248 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:34.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:34.349 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:34.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:34.389 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:34.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:34.490 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:34.517 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:34.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:34.619 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:34.648 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:34.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:34.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:34.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:34.789 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:34.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:34.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:34.891 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:34.931 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:34.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:34.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:35.033 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:35.080 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:35.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:35.181 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:34 smithi087 ceph-mon[89014]: pgmap v1577: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 5686/227559 objects degraded (2.499%); 3790/227559 objects misplaced (1.666%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:39:35.218 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:35.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:35.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:34 smithi029 ceph-mon[93650]: pgmap v1577: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 5686/227559 objects degraded (2.499%); 3790/227559 objects misplaced (1.666%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:39:35.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:34 smithi029 ceph-mon[97787]: pgmap v1577: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 5686/227559 objects degraded (2.499%); 3790/227559 objects misplaced (1.666%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:39:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:39:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:39:34] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.43.0" 2024-04-12T11:39:35.320 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:35.349 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:35.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:35.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:35.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:35.503 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:35.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:35.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:35.604 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:35.631 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:35.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:35.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:35.732 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:35.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:35.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:35.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:35.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:35.899 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:35.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:35.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:36.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:36.052 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:36.066 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:39:36.066 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:39:36.066 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:39:36.066 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:39:36.066 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:39:36.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:36.067 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:36.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:36.168 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:36.195 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:36.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:36.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:36.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:36.343 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:36.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:36.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:36.445 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:36.473 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:36.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:36.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:36.575 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:36.604 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:36.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:36.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:36.706 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:36.734 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:36.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:36.836 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:36.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:36.867 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:36.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:36.899 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:39:36.969 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:36.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-04-12T11:39:36.976 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:36 smithi029 ceph-mon[93650]: pgmap v1578: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5741/222807 objects degraded (2.577%); 3580/222807 objects misplaced (1.607%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T11:39:36.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:36 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/521285442' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:39:36.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:36 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:39:36.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:36.977 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:36 smithi029 ceph-mon[97787]: pgmap v1578: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5741/222807 objects degraded (2.577%); 3580/222807 objects misplaced (1.607%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T11:39:36.977 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:36 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/521285442' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:39:36.977 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:36 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:39:36.977 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:36.997 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:36.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:36.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:37.099 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:37.126 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:37.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:37.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:36 smithi087 ceph-mon[89014]: pgmap v1578: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5741/222807 objects degraded (2.577%); 3580/222807 objects misplaced (1.607%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T11:39:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:36 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/521285442' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:39:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:36 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:39:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:37.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:37.275 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:37.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:37.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:37.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:37.417 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:37.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:37.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:37.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:37.544 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:37.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:37.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:37.646 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:37.693 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:37.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:37.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:37.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:37.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:37.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:37.943 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:37.970 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:37.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:37.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:38.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:38.101 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:37 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:39:38.102 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:37 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:39:38.102 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:37 smithi029 ceph-mon[97787]: osdmap e1157: 8 total, 8 up, 8 in 2024-04-12T11:39:38.102 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:37 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5741/222807 objects degraded (2.577%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:38.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:37 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:39:38.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:37 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:39:38.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:37 smithi029 ceph-mon[93650]: osdmap e1157: 8 total, 8 up, 8 in 2024-04-12T11:39:38.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:37 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5741/222807 objects degraded (2.577%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:38.111 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:38.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:38.212 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:37 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:39:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:37 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:39:38.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:37 smithi087 ceph-mon[89014]: osdmap e1157: 8 total, 8 up, 8 in 2024-04-12T11:39:38.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:37 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5741/222807 objects degraded (2.577%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:38.241 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:38.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:38.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:38.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:38.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:38.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:38.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:38.487 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:38.516 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:38.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:38.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:38.618 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:38.644 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:38.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:38.746 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:38.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:38.774 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:38.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:38.874 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:38.905 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:38.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:38.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:39.007 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:39.035 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:39.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:39.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:39.068 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:39.092 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:38 smithi029 ceph-mon[93650]: pgmap v1580: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5741/220308 objects degraded (2.606%); 3580/220308 objects misplaced (1.625%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:39.092 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:38 smithi029 ceph-mon[97787]: pgmap v1580: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5741/220308 objects degraded (2.606%); 3580/220308 objects misplaced (1.625%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:39.137 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:39.164 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:39.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:39.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:38 smithi087 ceph-mon[89014]: pgmap v1580: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5741/220308 objects degraded (2.606%); 3580/220308 objects misplaced (1.625%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:39.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:39.296 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:39.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:39.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:39.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:39.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:39.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:39.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:39.540 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:39.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:39.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:39.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:39.669 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:39.698 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:39.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:39.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:39.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:39.839 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:39.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:39.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:39.981 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:39.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:39.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:40.083 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:40.110 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:40.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:40.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:40.212 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:40.251 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:40.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:40.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:40.372 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:40.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:40.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:40.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:40.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:40.566 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:40.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:40.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:40.668 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:40.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:40.726 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:40.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:40.827 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:40.853 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:40.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:40.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:40.983 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:40.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:41.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:41.094 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:39:41.128 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:41.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:41.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:41.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:40 smithi029 ceph-mon[97787]: pgmap v1581: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 5741/217101 objects degraded (2.644%); 3580/217101 objects misplaced (1.649%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:41.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:40 smithi029 ceph-mon[93650]: pgmap v1581: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 5741/217101 objects degraded (2.644%); 3580/217101 objects misplaced (1.649%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:40 smithi087 ceph-mon[89014]: pgmap v1581: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 5741/217101 objects degraded (2.644%); 3580/217101 objects misplaced (1.649%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:39:41.231 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:41.256 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:41.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:41.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:41.357 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:41.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:41.432 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:41.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:41.433 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:39:41.467 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:39:41.532 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:39:41.534 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:41.560 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:41.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:41.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:41.594 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:39:41.636 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:39:41.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:41.683 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:39:41.702 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:39:41.718 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:39:41.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:41.728 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:41.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:41.743 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:39:41.789 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:39:41.803 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:39:41.803 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:39:41.803 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:39:41.803 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:39:41.804 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:39:41.804 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:39:41.804 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:39:41.804 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:39:41.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:41.855 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:41.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:41.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:41.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:41.998 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:41.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:41.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:42.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:41 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1192323231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:39:42.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:41 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1192323231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:39:42.100 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:42.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:42.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:42.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:42.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:41 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1192323231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:39:42.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:42.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:42.302 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:42.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:42.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:42.447 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:42.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:42.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:42.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:42.591 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:42.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:42.614 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1581,"stamp":"2024-04-12T11:39:39.780349+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4741601258,"num_objects":72367,"num_object_clones":0,"num_object_copies":217101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5741,"num_objects_misplaced":3580,"num_objects_unfound":0,"num_objects_dirty":72367,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":142959,"num_write_kb":6882447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2045,"num_bytes_recovered":135138028,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":135275,"ondisk_log_size":135275,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16513796,"kb_used_data":14286780,"kb_used_omap":175,"kb_used_meta":2226640,"kb_avail":733447420,"statfs":{"total":767960285184,"available":751050158080,"internally_reserved":0,"allocated":14629662720,"data_stored":14605273213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":2280079529},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,6,3,12,4,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-420544512,"num_objects":-6417,"num_object_clones":0,"num_object_copies":-19251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":94,"num_objects_misplaced":-387,"num_objects_unfound":0,"num_objects_dirty":-6417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6417,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":293,"num_bytes_recovered":19202048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002806"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1664,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320393+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:39:23.320393+0000","last_peered":"2024-04-12T11:39:23.320393+0000","last_clean":"2024-04-12T11:39:23.320393+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:39:23.320393+0000","last_undegraded":"2024-04-12T11:39:23.320393+0000","last_fullsized":"2024-04-12T11:39:23.320393+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1672,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.324275+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:39:23.324275+0000","last_peered":"2024-04-12T11:39:23.324275+0000","last_clean":"2024-04-12T11:39:23.324275+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:39:23.324275+0000","last_undegraded":"2024-04-12T11:39:23.324275+0000","last_fullsized":"2024-04-12T11:39:23.324275+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1726,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910015+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:39:36.910015+0000","last_peered":"2024-04-12T11:39:36.910015+0000","last_clean":"2024-04-12T11:39:36.910015+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:39:36.910015+0000","last_undegraded":"2024-04-12T11:39:36.910015+0000","last_fullsized":"2024-04-12T11:39:36.910015+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1733,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908315+0000","last_change":"2024-04-12T11:38:21.933647+0000","last_active":"2024-04-12T11:39:36.908315+0000","last_peered":"2024-04-12T11:39:36.908315+0000","last_clean":"2024-04-12T11:39:36.908315+0000","last_became_active":"2024-04-12T11:38:21.933295+0000","last_became_peered":"2024-04-12T11:38:21.933295+0000","last_unstale":"2024-04-12T11:39:36.908315+0000","last_undegraded":"2024-04-12T11:39:36.908315+0000","last_fullsized":"2024-04-12T11:39:36.908315+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:37.402139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1680,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.324119+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:39:23.324119+0000","last_peered":"2024-04-12T11:39:23.324119+0000","last_clean":"2024-04-12T11:39:23.324119+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:39:23.324119+0000","last_undegraded":"2024-04-12T11:39:23.324119+0000","last_fullsized":"2024-04-12T11:39:23.324119+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1101,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.321912+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:39:23.321912+0000","last_peered":"2024-04-12T11:39:23.321912+0000","last_clean":"2024-04-12T11:39:23.321912+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:39:23.321912+0000","last_undegraded":"2024-04-12T11:39:23.321912+0000","last_fullsized":"2024-04-12T11:39:23.321912+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1717,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.907146+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:39:36.907146+0000","last_peered":"2024-04-12T11:39:36.907146+0000","last_clean":"2024-04-12T11:39:36.907146+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:39:36.907146+0000","last_undegraded":"2024-04-12T11:39:36.907146+0000","last_fullsized":"2024-04-12T11:39:36.907146+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1654,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.324267+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:39:23.324267+0000","last_peered":"2024-04-12T11:39:23.324267+0000","last_clean":"2024-04-12T11:39:23.324267+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:39:23.324267+0000","last_undegraded":"2024-04-12T11:39:23.324267+0000","last_fullsized":"2024-04-12T11:39:23.324267+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1107,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:32.320785+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:39:32.320785+0000","last_peered":"2024-04-12T11:39:32.320785+0000","last_clean":"2024-04-12T11:39:32.320785+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:39:32.320785+0000","last_undegraded":"2024-04-12T11:39:32.320785+0000","last_fullsized":"2024-04-12T11:39:32.320785+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1634,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.336966+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:39:23.336966+0000","last_peered":"2024-04-12T11:39:23.336966+0000","last_clean":"2024-04-12T11:39:23.336966+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:39:23.336966+0000","last_undegraded":"2024-04-12T11:39:23.336966+0000","last_fullsized":"2024-04-12T11:39:23.336966+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1116,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:32.441336+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:39:32.441336+0000","last_peered":"2024-04-12T11:39:32.441336+0000","last_clean":"2024-04-12T11:39:32.441336+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:39:32.441336+0000","last_undegraded":"2024-04-12T11:39:32.441336+0000","last_fullsized":"2024-04-12T11:39:32.441336+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1717,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910475+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:39:36.910475+0000","last_peered":"2024-04-12T11:39:36.910475+0000","last_clean":"2024-04-12T11:39:36.910475+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:39:36.910475+0000","last_undegraded":"2024-04-12T11:39:36.910475+0000","last_fullsized":"2024-04-12T11:39:36.910475+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1636,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320784+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:39:23.320784+0000","last_peered":"2024-04-12T11:39:23.320784+0000","last_clean":"2024-04-12T11:39:23.320784+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:39:23.320784+0000","last_undegraded":"2024-04-12T11:39:23.320784+0000","last_fullsized":"2024-04-12T11:39:23.320784+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1627,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320401+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:39:23.320401+0000","last_peered":"2024-04-12T11:39:23.320401+0000","last_clean":"2024-04-12T11:39:23.320401+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:39:23.320401+0000","last_undegraded":"2024-04-12T11:39:23.320401+0000","last_fullsized":"2024-04-12T11:39:23.320401+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1115,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337199+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:39:23.337199+0000","last_peered":"2024-04-12T11:39:23.337199+0000","last_clean":"2024-04-12T11:39:23.337199+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:39:23.337199+0000","last_undegraded":"2024-04-12T11:39:23.337199+0000","last_fullsized":"2024-04-12T11:39:23.337199+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1665,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.333593+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:39:23.333593+0000","last_peered":"2024-04-12T11:39:23.333593+0000","last_clean":"2024-04-12T11:39:23.333593+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:39:23.333593+0000","last_undegraded":"2024-04-12T11:39:23.333593+0000","last_fullsized":"2024-04-12T11:39:23.333593+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1718,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.906781+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:39:36.906781+0000","last_peered":"2024-04-12T11:39:36.906781+0000","last_clean":"2024-04-12T11:39:36.906781+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:39:36.906781+0000","last_undegraded":"2024-04-12T11:39:36.906781+0000","last_fullsized":"2024-04-12T11:39:36.906781+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1106,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:32.320748+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:39:32.320748+0000","last_peered":"2024-04-12T11:39:32.320748+0000","last_clean":"2024-04-12T11:39:32.320748+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:39:32.320748+0000","last_undegraded":"2024-04-12T11:39:32.320748+0000","last_fullsized":"2024-04-12T11:39:32.320748+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1098,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337154+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:39:23.337154+0000","last_peered":"2024-04-12T11:39:23.337154+0000","last_clean":"2024-04-12T11:39:23.337154+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:39:23.337154+0000","last_undegraded":"2024-04-12T11:39:23.337154+0000","last_fullsized":"2024-04-12T11:39:23.337154+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1719,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908223+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:39:36.908223+0000","last_peered":"2024-04-12T11:39:36.908223+0000","last_clean":"2024-04-12T11:39:36.908223+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:39:36.908223+0000","last_undegraded":"2024-04-12T11:39:36.908223+0000","last_fullsized":"2024-04-12T11:39:36.908223+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1109,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.321440+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:39:23.321440+0000","last_peered":"2024-04-12T11:39:23.321440+0000","last_clean":"2024-04-12T11:39:23.321440+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:39:23.321440+0000","last_undegraded":"2024-04-12T11:39:23.321440+0000","last_fullsized":"2024-04-12T11:39:23.321440+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1664,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337541+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:39:23.337541+0000","last_peered":"2024-04-12T11:39:23.337541+0000","last_clean":"2024-04-12T11:39:23.337541+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:39:23.337541+0000","last_undegraded":"2024-04-12T11:39:23.337541+0000","last_fullsized":"2024-04-12T11:39:23.337541+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1157'9061","reported_seq":9120,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:38.289014+0000","last_change":"2024-04-12T11:37:03.057199+0000","last_active":"2024-04-12T11:39:38.289014+0000","last_peered":"2024-04-12T11:39:38.289014+0000","last_clean":"2024-04-12T11:39:38.289014+0000","last_became_active":"2024-04-12T11:37:03.056776+0000","last_became_peered":"2024-04-12T11:37:03.056776+0000","last_unstale":"2024-04-12T11:39:38.289014+0000","last_undegraded":"2024-04-12T11:39:38.289014+0000","last_fullsized":"2024-04-12T11:39:38.289014+0000","mapping_epoch":1133,"log_start":"1139'1100","ondisk_log_start":"1139'1100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7961,"log_dups_size":0,"ondisk_log_size":7961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:16:29.272872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293535744,"num_objects":4479,"num_object_clones":0,"num_object_copies":13437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9061,"num_write_kb":433280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1114,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337739+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:39:23.337739+0000","last_peered":"2024-04-12T11:39:23.337739+0000","last_clean":"2024-04-12T11:39:23.337739+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:39:23.337739+0000","last_undegraded":"2024-04-12T11:39:23.337739+0000","last_fullsized":"2024-04-12T11:39:23.337739+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1092,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.328901+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:39:23.328901+0000","last_peered":"2024-04-12T11:39:23.328901+0000","last_clean":"2024-04-12T11:39:23.328901+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:39:23.328901+0000","last_undegraded":"2024-04-12T11:39:23.328901+0000","last_fullsized":"2024-04-12T11:39:23.328901+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1156'8875","reported_seq":8932,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:34.893124+0000","last_change":"2024-04-12T11:37:03.059071+0000","last_active":"2024-04-12T11:39:34.893124+0000","last_peered":"2024-04-12T11:39:34.893124+0000","last_clean":"2024-04-12T11:39:34.893124+0000","last_became_active":"2024-04-12T11:37:03.058769+0000","last_became_peered":"2024-04-12T11:37:03.058769+0000","last_unstale":"2024-04-12T11:39:34.893124+0000","last_undegraded":"2024-04-12T11:39:34.893124+0000","last_fullsized":"2024-04-12T11:39:34.893124+0000","mapping_epoch":1133,"log_start":"1140'1300","ondisk_log_start":"1140'1300","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7575,"log_dups_size":0,"ondisk_log_size":7575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:43:26.795096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307036160,"num_objects":4685,"num_object_clones":0,"num_object_copies":14055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8875,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1115,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337703+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:39:23.337703+0000","last_peered":"2024-04-12T11:39:23.337703+0000","last_clean":"2024-04-12T11:39:23.337703+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:39:23.337703+0000","last_undegraded":"2024-04-12T11:39:23.337703+0000","last_fullsized":"2024-04-12T11:39:23.337703+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1650,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.322747+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:39:23.322747+0000","last_peered":"2024-04-12T11:39:23.322747+0000","last_clean":"2024-04-12T11:39:23.322747+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:39:23.322747+0000","last_undegraded":"2024-04-12T11:39:23.322747+0000","last_fullsized":"2024-04-12T11:39:23.322747+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1157'8971","reported_seq":9030,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:38.092137+0000","last_change":"2024-04-12T11:37:03.059390+0000","last_active":"2024-04-12T11:39:38.092137+0000","last_peered":"2024-04-12T11:39:38.092137+0000","last_clean":"2024-04-12T11:39:38.092137+0000","last_became_active":"2024-04-12T11:37:03.058045+0000","last_became_peered":"2024-04-12T11:37:03.058045+0000","last_unstale":"2024-04-12T11:39:38.092137+0000","last_undegraded":"2024-04-12T11:39:38.092137+0000","last_fullsized":"2024-04-12T11:39:38.092137+0000","mapping_epoch":1133,"log_start":"1137'100","ondisk_log_start":"1137'100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8871,"log_dups_size":0,"ondisk_log_size":8871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:14:01.284893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294584320,"num_objects":4495,"num_object_clones":0,"num_object_copies":13485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8971,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1084,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337523+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:39:23.337523+0000","last_peered":"2024-04-12T11:39:23.337523+0000","last_clean":"2024-04-12T11:39:23.337523+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:39:23.337523+0000","last_undegraded":"2024-04-12T11:39:23.337523+0000","last_fullsized":"2024-04-12T11:39:23.337523+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1101,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.321524+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:39:23.321524+0000","last_peered":"2024-04-12T11:39:23.321524+0000","last_clean":"2024-04-12T11:39:23.321524+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:39:23.321524+0000","last_undegraded":"2024-04-12T11:39:23.321524+0000","last_fullsized":"2024-04-12T11:39:23.321524+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1156'8989","reported_seq":9046,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:34.829790+0000","last_change":"2024-04-12T11:37:03.054920+0000","last_active":"2024-04-12T11:39:34.829790+0000","last_peered":"2024-04-12T11:39:34.829790+0000","last_clean":"2024-04-12T11:39:34.829790+0000","last_became_active":"2024-04-12T11:37:03.054603+0000","last_became_peered":"2024-04-12T11:37:03.054603+0000","last_unstale":"2024-04-12T11:39:34.829790+0000","last_undegraded":"2024-04-12T11:39:34.829790+0000","last_fullsized":"2024-04-12T11:39:34.829790+0000","mapping_epoch":1133,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8989,"log_dups_size":0,"ondisk_log_size":8989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:34:45.149813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302710784,"num_objects":4619,"num_object_clones":0,"num_object_copies":13857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8989,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1107,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326601+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:39:23.326601+0000","last_peered":"2024-04-12T11:39:23.326601+0000","last_clean":"2024-04-12T11:39:23.326601+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:39:23.326601+0000","last_undegraded":"2024-04-12T11:39:23.326601+0000","last_fullsized":"2024-04-12T11:39:23.326601+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1728,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908408+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:39:36.908408+0000","last_peered":"2024-04-12T11:39:36.908408+0000","last_clean":"2024-04-12T11:39:36.908408+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:39:36.908408+0000","last_undegraded":"2024-04-12T11:39:36.908408+0000","last_fullsized":"2024-04-12T11:39:36.908408+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1157'9101","reported_seq":9160,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:38.120047+0000","last_change":"2024-04-12T11:37:03.063209+0000","last_active":"2024-04-12T11:39:38.120047+0000","last_peered":"2024-04-12T11:39:38.120047+0000","last_clean":"2024-04-12T11:39:38.120047+0000","last_became_active":"2024-04-12T11:37:03.062848+0000","last_became_peered":"2024-04-12T11:37:03.062848+0000","last_unstale":"2024-04-12T11:39:38.120047+0000","last_undegraded":"2024-04-12T11:39:38.120047+0000","last_fullsized":"2024-04-12T11:39:38.120047+0000","mapping_epoch":1133,"log_start":"1137'200","ondisk_log_start":"1137'200","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8901,"log_dups_size":0,"ondisk_log_size":8901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:55:26.299047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295632896,"num_objects":4511,"num_object_clones":0,"num_object_copies":13533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9101,"num_write_kb":435584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1658,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.324597+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:39:23.324597+0000","last_peered":"2024-04-12T11:39:23.324597+0000","last_clean":"2024-04-12T11:39:23.324597+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:39:23.324597+0000","last_undegraded":"2024-04-12T11:39:23.324597+0000","last_fullsized":"2024-04-12T11:39:23.324597+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1641,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320873+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:39:23.320873+0000","last_peered":"2024-04-12T11:39:23.320873+0000","last_clean":"2024-04-12T11:39:23.320873+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:39:23.320873+0000","last_undegraded":"2024-04-12T11:39:23.320873+0000","last_fullsized":"2024-04-12T11:39:23.320873+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1156'8828","reported_seq":8885,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:34.823098+0000","last_change":"2024-04-12T11:37:03.060739+0000","last_active":"2024-04-12T11:39:34.823098+0000","last_peered":"2024-04-12T11:39:34.823098+0000","last_clean":"2024-04-12T11:39:34.823098+0000","last_became_active":"2024-04-12T11:37:03.060414+0000","last_became_peered":"2024-04-12T11:37:03.060414+0000","last_unstale":"2024-04-12T11:39:34.823098+0000","last_undegraded":"2024-04-12T11:39:34.823098+0000","last_fullsized":"2024-04-12T11:39:34.823098+0000","mapping_epoch":1133,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8828,"log_dups_size":0,"ondisk_log_size":8828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:12:30.259488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294256640,"num_objects":4490,"num_object_clones":0,"num_object_copies":13470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8828,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":667,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320568+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:39:23.320568+0000","last_peered":"2024-04-12T11:39:23.320568+0000","last_clean":"2024-04-12T11:39:23.320568+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:39:23.320568+0000","last_undegraded":"2024-04-12T11:39:23.320568+0000","last_fullsized":"2024-04-12T11:39:23.320568+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1663,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320499+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:39:23.320499+0000","last_peered":"2024-04-12T11:39:23.320499+0000","last_clean":"2024-04-12T11:39:23.320499+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:39:23.320499+0000","last_undegraded":"2024-04-12T11:39:23.320499+0000","last_fullsized":"2024-04-12T11:39:23.320499+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1157'9094","reported_seq":9129,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:37.394650+0000","last_change":"2024-04-12T11:37:03.060364+0000","last_active":"2024-04-12T11:39:37.394650+0000","last_peered":"2024-04-12T11:39:37.394650+0000","last_clean":"2024-04-12T11:39:37.394650+0000","last_became_active":"2024-04-12T11:37:03.060146+0000","last_became_peered":"2024-04-12T11:37:03.060146+0000","last_unstale":"2024-04-12T11:39:37.394650+0000","last_undegraded":"2024-04-12T11:39:37.394650+0000","last_fullsized":"2024-04-12T11:39:37.394650+0000","mapping_epoch":1133,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9094,"log_dups_size":0,"ondisk_log_size":9094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:03.796242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295895064,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9094,"num_write_kb":435457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1649,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.322796+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:39:23.322796+0000","last_peered":"2024-04-12T11:39:23.322796+0000","last_clean":"2024-04-12T11:39:23.322796+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:39:23.322796+0000","last_undegraded":"2024-04-12T11:39:23.322796+0000","last_fullsized":"2024-04-12T11:39:23.322796+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1628,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320394+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:39:23.320394+0000","last_peered":"2024-04-12T11:39:23.320394+0000","last_clean":"2024-04-12T11:39:23.320394+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:39:23.320394+0000","last_undegraded":"2024-04-12T11:39:23.320394+0000","last_fullsized":"2024-04-12T11:39:23.320394+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1156'9032","reported_seq":9089,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:34.862370+0000","last_change":"2024-04-12T11:37:03.056864+0000","last_active":"2024-04-12T11:39:34.862370+0000","last_peered":"2024-04-12T11:39:34.862370+0000","last_clean":"2024-04-12T11:39:34.862370+0000","last_became_active":"2024-04-12T11:37:03.056523+0000","last_became_peered":"2024-04-12T11:37:03.056523+0000","last_unstale":"2024-04-12T11:39:34.862370+0000","last_undegraded":"2024-04-12T11:39:34.862370+0000","last_fullsized":"2024-04-12T11:39:34.862370+0000","mapping_epoch":1133,"log_start":"1140'1500","ondisk_log_start":"1140'1500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7532,"log_dups_size":0,"ondisk_log_size":7532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:57:58.750385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301072384,"num_objects":4594,"num_object_clones":0,"num_object_copies":13782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9032,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1666,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.324313+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:39:23.324313+0000","last_peered":"2024-04-12T11:39:23.324313+0000","last_clean":"2024-04-12T11:39:23.324313+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:39:23.324313+0000","last_undegraded":"2024-04-12T11:39:23.324313+0000","last_fullsized":"2024-04-12T11:39:23.324313+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1726,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.909990+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:39:36.909990+0000","last_peered":"2024-04-12T11:39:36.909990+0000","last_clean":"2024-04-12T11:39:36.909990+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:39:36.909990+0000","last_undegraded":"2024-04-12T11:39:36.909990+0000","last_fullsized":"2024-04-12T11:39:36.909990+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1156'8757","reported_seq":12886,"reported_epoch":1156,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-04-12T11:39:34.814337+0000","last_change":"2024-04-12T11:39:24.599690+0000","last_active":"2024-04-12T11:39:34.814337+0000","last_peered":"2024-04-12T11:39:34.814337+0000","last_clean":"2024-04-12T11:38:20.888121+0000","last_became_active":"2024-04-12T11:38:23.417421+0000","last_became_peered":"2024-04-12T11:38:23.417421+0000","last_unstale":"2024-04-12T11:39:34.814337+0000","last_undegraded":"2024-04-12T11:39:24.597597+0000","last_fullsized":"2024-04-12T11:38:23.247711+0000","mapping_epoch":1147,"log_start":"1138'800","ondisk_log_start":"1138'800","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7957,"log_dups_size":0,"ondisk_log_size":7957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294322176,"num_objects":4491,"num_object_clones":0,"num_object_copies":13473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":100,"num_objects_misplaced":3580,"num_objects_unfound":0,"num_objects_dirty":4491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8757,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2035,"num_bytes_recovered":133365760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":3580},{"shards":"4,6","objects":811}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1728,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.915621+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:39:36.915621+0000","last_peered":"2024-04-12T11:39:36.915621+0000","last_clean":"2024-04-12T11:39:36.915621+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:39:36.915621+0000","last_undegraded":"2024-04-12T11:39:36.915621+0000","last_fullsized":"2024-04-12T11:39:36.915621+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1123,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326865+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:39:23.326865+0000","last_peered":"2024-04-12T11:39:23.326865+0000","last_clean":"2024-04-12T11:39:23.326865+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:39:23.326865+0000","last_undegraded":"2024-04-12T11:39:23.326865+0000","last_fullsized":"2024-04-12T11:39:23.326865+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1719,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910329+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:39:36.910329+0000","last_peered":"2024-04-12T11:39:36.910329+0000","last_clean":"2024-04-12T11:39:36.910329+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:39:36.910329+0000","last_undegraded":"2024-04-12T11:39:36.910329+0000","last_fullsized":"2024-04-12T11:39:36.910329+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1157'8966","reported_seq":9000,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:37.377417+0000","last_change":"2024-04-12T11:37:03.060583+0000","last_active":"2024-04-12T11:39:37.377417+0000","last_peered":"2024-04-12T11:39:37.377417+0000","last_clean":"2024-04-12T11:39:37.377417+0000","last_became_active":"2024-04-12T11:37:03.060125+0000","last_became_peered":"2024-04-12T11:37:03.060125+0000","last_unstale":"2024-04-12T11:39:37.377417+0000","last_undegraded":"2024-04-12T11:39:37.377417+0000","last_fullsized":"2024-04-12T11:39:37.377417+0000","mapping_epoch":1133,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8966,"log_dups_size":0,"ondisk_log_size":8966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:35:30.859950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294912000,"num_objects":4500,"num_object_clones":0,"num_object_copies":13500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8966,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1113,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326627+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:39:23.326627+0000","last_peered":"2024-04-12T11:39:23.326627+0000","last_clean":"2024-04-12T11:39:23.326627+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:39:23.326627+0000","last_undegraded":"2024-04-12T11:39:23.326627+0000","last_fullsized":"2024-04-12T11:39:23.326627+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1649,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320285+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:39:23.320285+0000","last_peered":"2024-04-12T11:39:23.320285+0000","last_clean":"2024-04-12T11:39:23.320285+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:39:23.320285+0000","last_undegraded":"2024-04-12T11:39:23.320285+0000","last_fullsized":"2024-04-12T11:39:23.320285+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1728,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.915215+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:39:36.915215+0000","last_peered":"2024-04-12T11:39:36.915215+0000","last_clean":"2024-04-12T11:39:36.915215+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:39:36.915215+0000","last_undegraded":"2024-04-12T11:39:36.915215+0000","last_fullsized":"2024-04-12T11:39:36.915215+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1157'8857","reported_seq":8915,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:38.110079+0000","last_change":"2024-04-12T11:37:03.058470+0000","last_active":"2024-04-12T11:39:38.110079+0000","last_peered":"2024-04-12T11:39:38.110079+0000","last_clean":"2024-04-12T11:39:38.110079+0000","last_became_active":"2024-04-12T11:37:03.058149+0000","last_became_peered":"2024-04-12T11:37:03.058149+0000","last_unstale":"2024-04-12T11:39:38.110079+0000","last_undegraded":"2024-04-12T11:39:38.110079+0000","last_fullsized":"2024-04-12T11:39:38.110079+0000","mapping_epoch":1133,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8857,"log_dups_size":0,"ondisk_log_size":8857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:25:06.934877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287506432,"num_objects":4387,"num_object_clones":0,"num_object_copies":13161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8857,"num_write_kb":423808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1690,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.348377+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:39:23.348377+0000","last_peered":"2024-04-12T11:39:23.348377+0000","last_clean":"2024-04-12T11:39:23.348377+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:39:23.348377+0000","last_undegraded":"2024-04-12T11:39:23.348377+0000","last_fullsized":"2024-04-12T11:39:23.348377+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1719,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910150+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:39:36.910150+0000","last_peered":"2024-04-12T11:39:36.910150+0000","last_clean":"2024-04-12T11:39:36.910150+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:39:36.910150+0000","last_undegraded":"2024-04-12T11:39:36.910150+0000","last_fullsized":"2024-04-12T11:39:36.910150+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1722,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.911258+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:39:36.911258+0000","last_peered":"2024-04-12T11:39:36.911258+0000","last_clean":"2024-04-12T11:39:36.911258+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:39:36.911258+0000","last_undegraded":"2024-04-12T11:39:36.911258+0000","last_fullsized":"2024-04-12T11:39:36.911258+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1157'8902","reported_seq":8959,"reported_epoch":1157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:39:38.107484+0000","last_change":"2024-04-12T11:38:23.438040+0000","last_active":"2024-04-12T11:39:38.107484+0000","last_peered":"2024-04-12T11:39:38.107484+0000","last_clean":"2024-04-12T11:38:20.888095+0000","last_became_active":"2024-04-12T11:38:23.424268+0000","last_became_peered":"2024-04-12T11:38:23.424268+0000","last_unstale":"2024-04-12T11:39:38.107484+0000","last_undegraded":"2024-04-12T11:38:23.284099+0000","last_fullsized":"2024-04-12T11:38:23.245558+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8902,"log_dups_size":0,"ondisk_log_size":8902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289013760,"num_objects":4410,"num_object_clones":0,"num_object_copies":13230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8902,"num_write_kb":425984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":3395},{"shards":"0,7","objects":-1231}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1113,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326543+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:39:23.326543+0000","last_peered":"2024-04-12T11:39:23.326543+0000","last_clean":"2024-04-12T11:39:23.326543+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:39:23.326543+0000","last_undegraded":"2024-04-12T11:39:23.326543+0000","last_fullsized":"2024-04-12T11:39:23.326543+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1468,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:32.441271+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:39:32.441271+0000","last_peered":"2024-04-12T11:39:32.441271+0000","last_clean":"2024-04-12T11:39:32.441271+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:39:32.441271+0000","last_undegraded":"2024-04-12T11:39:32.441271+0000","last_fullsized":"2024-04-12T11:39:32.441271+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1633,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326553+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:39:23.326553+0000","last_peered":"2024-04-12T11:39:23.326553+0000","last_clean":"2024-04-12T11:39:23.326553+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:39:23.326553+0000","last_undegraded":"2024-04-12T11:39:23.326553+0000","last_fullsized":"2024-04-12T11:39:23.326553+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1101,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.321839+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:39:23.321839+0000","last_peered":"2024-04-12T11:39:23.321839+0000","last_clean":"2024-04-12T11:39:23.321839+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:39:23.321839+0000","last_undegraded":"2024-04-12T11:39:23.321839+0000","last_fullsized":"2024-04-12T11:39:23.321839+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1157'8729","reported_seq":8763,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:37.287940+0000","last_change":"2024-04-12T11:37:03.061698+0000","last_active":"2024-04-12T11:39:37.287940+0000","last_peered":"2024-04-12T11:39:37.287940+0000","last_clean":"2024-04-12T11:39:37.287940+0000","last_became_active":"2024-04-12T11:37:03.061469+0000","last_became_peered":"2024-04-12T11:37:03.061469+0000","last_unstale":"2024-04-12T11:39:37.287940+0000","last_undegraded":"2024-04-12T11:39:37.287940+0000","last_fullsized":"2024-04-12T11:39:37.287940+0000","mapping_epoch":1133,"log_start":"1137'100","ondisk_log_start":"1137'100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8629,"log_dups_size":0,"ondisk_log_size":8629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:09.906340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295370752,"num_objects":4507,"num_object_clones":0,"num_object_copies":13521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8729,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1120,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337067+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:39:23.337067+0000","last_peered":"2024-04-12T11:39:23.337067+0000","last_clean":"2024-04-12T11:39:23.337067+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:39:23.337067+0000","last_undegraded":"2024-04-12T11:39:23.337067+0000","last_fullsized":"2024-04-12T11:39:23.337067+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1729,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.907586+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:39:36.907586+0000","last_peered":"2024-04-12T11:39:36.907586+0000","last_clean":"2024-04-12T11:39:36.907586+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:39:36.907586+0000","last_undegraded":"2024-04-12T11:39:36.907586+0000","last_fullsized":"2024-04-12T11:39:36.907586+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1729,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908121+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:39:36.908121+0000","last_peered":"2024-04-12T11:39:36.908121+0000","last_clean":"2024-04-12T11:39:36.908121+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:39:36.908121+0000","last_undegraded":"2024-04-12T11:39:36.908121+0000","last_fullsized":"2024-04-12T11:39:36.908121+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"1157'8976","reported_seq":9010,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:37.297299+0000","last_change":"2024-04-12T11:37:03.060455+0000","last_active":"2024-04-12T11:39:37.297299+0000","last_peered":"2024-04-12T11:39:37.297299+0000","last_clean":"2024-04-12T11:39:37.297299+0000","last_became_active":"2024-04-12T11:37:03.060244+0000","last_became_peered":"2024-04-12T11:37:03.060244+0000","last_unstale":"2024-04-12T11:39:37.297299+0000","last_undegraded":"2024-04-12T11:39:37.297299+0000","last_fullsized":"2024-04-12T11:39:37.297299+0000","mapping_epoch":1133,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8976,"log_dups_size":0,"ondisk_log_size":8976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:47.787368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296091648,"num_objects":4518,"num_object_clones":0,"num_object_copies":13554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8976,"num_write_kb":431808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1120,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337548+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:39:23.337548+0000","last_peered":"2024-04-12T11:39:23.337548+0000","last_clean":"2024-04-12T11:39:23.337548+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:39:23.337548+0000","last_undegraded":"2024-04-12T11:39:23.337548+0000","last_fullsized":"2024-04-12T11:39:23.337548+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1719,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.915425+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:39:36.915425+0000","last_peered":"2024-04-12T11:39:36.915425+0000","last_clean":"2024-04-12T11:39:36.915425+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:39:36.915425+0000","last_undegraded":"2024-04-12T11:39:36.915425+0000","last_fullsized":"2024-04-12T11:39:36.915425+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1712,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908671+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:39:36.908671+0000","last_peered":"2024-04-12T11:39:36.908671+0000","last_clean":"2024-04-12T11:39:36.908671+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:39:36.908671+0000","last_undegraded":"2024-04-12T11:39:36.908671+0000","last_fullsized":"2024-04-12T11:39:36.908671+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1156'8610","reported_seq":8667,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:34.860817+0000","last_change":"2024-04-12T11:37:03.057084+0000","last_active":"2024-04-12T11:39:34.860817+0000","last_peered":"2024-04-12T11:39:34.860817+0000","last_clean":"2024-04-12T11:39:34.860817+0000","last_became_active":"2024-04-12T11:37:03.056772+0000","last_became_peered":"2024-04-12T11:37:03.056772+0000","last_unstale":"2024-04-12T11:39:34.860817+0000","last_undegraded":"2024-04-12T11:39:34.860817+0000","last_fullsized":"2024-04-12T11:39:34.860817+0000","mapping_epoch":1133,"log_start":"1140'1100","ondisk_log_start":"1140'1100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7510,"log_dups_size":0,"ondisk_log_size":7510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:02:41.845790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299761664,"num_objects":4574,"num_object_clones":0,"num_object_copies":13722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8610,"num_write_kb":421888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1649,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337423+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:39:23.337423+0000","last_peered":"2024-04-12T11:39:23.337423+0000","last_clean":"2024-04-12T11:39:23.337423+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:39:23.337423+0000","last_undegraded":"2024-04-12T11:39:23.337423+0000","last_fullsized":"2024-04-12T11:39:23.337423+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1719,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910224+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:39:36.910224+0000","last_peered":"2024-04-12T11:39:36.910224+0000","last_clean":"2024-04-12T11:39:36.910224+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:39:36.910224+0000","last_undegraded":"2024-04-12T11:39:36.910224+0000","last_fullsized":"2024-04-12T11:39:36.910224+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1647,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.324401+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:39:23.324401+0000","last_peered":"2024-04-12T11:39:23.324401+0000","last_clean":"2024-04-12T11:39:23.324401+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:39:23.324401+0000","last_undegraded":"2024-04-12T11:39:23.324401+0000","last_fullsized":"2024-04-12T11:39:23.324401+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1156'8911","reported_seq":8968,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:34.893310+0000","last_change":"2024-04-12T11:37:03.060299+0000","last_active":"2024-04-12T11:39:34.893310+0000","last_peered":"2024-04-12T11:39:34.893310+0000","last_clean":"2024-04-12T11:39:34.893310+0000","last_became_active":"2024-04-12T11:37:03.058897+0000","last_became_peered":"2024-04-12T11:37:03.058897+0000","last_unstale":"2024-04-12T11:39:34.893310+0000","last_undegraded":"2024-04-12T11:39:34.893310+0000","last_fullsized":"2024-04-12T11:39:34.893310+0000","mapping_epoch":1133,"log_start":"1140'1400","ondisk_log_start":"1140'1400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7511,"log_dups_size":0,"ondisk_log_size":7511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:45:37.844216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299302912,"num_objects":4567,"num_object_clones":0,"num_object_copies":13701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8911,"num_write_kb":431296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1125,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.350168+0000","last_change":"2024-04-12T11:39:23.349899+0000","last_active":"2024-04-12T11:39:23.350168+0000","last_peered":"2024-04-12T11:39:23.350168+0000","last_clean":"2024-04-12T11:39:23.350168+0000","last_became_active":"2024-04-12T11:39:23.349526+0000","last_became_peered":"2024-04-12T11:39:23.349526+0000","last_unstale":"2024-04-12T11:39:23.350168+0000","last_undegraded":"2024-04-12T11:39:23.350168+0000","last_fullsized":"2024-04-12T11:39:23.350168+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:36:47.115912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1732,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908402+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:39:36.908402+0000","last_peered":"2024-04-12T11:39:36.908402+0000","last_clean":"2024-04-12T11:39:36.908402+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:39:36.908402+0000","last_undegraded":"2024-04-12T11:39:36.908402+0000","last_fullsized":"2024-04-12T11:39:36.908402+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1636,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.322588+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:39:23.322588+0000","last_peered":"2024-04-12T11:39:23.322588+0000","last_clean":"2024-04-12T11:39:23.322588+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:39:23.322588+0000","last_undegraded":"2024-04-12T11:39:23.322588+0000","last_fullsized":"2024-04-12T11:39:23.322588+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1722,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908575+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:39:36.908575+0000","last_peered":"2024-04-12T11:39:36.908575+0000","last_clean":"2024-04-12T11:39:36.908575+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:39:36.908575+0000","last_undegraded":"2024-04-12T11:39:36.908575+0000","last_fullsized":"2024-04-12T11:39:36.908575+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":35,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.349518+0000","last_change":"2024-04-12T11:38:06.839832+0000","last_active":"2024-04-12T11:39:23.349518+0000","last_peered":"2024-04-12T11:39:23.349518+0000","last_clean":"2024-04-12T11:39:23.349518+0000","last_became_active":"2024-04-12T11:38:06.839460+0000","last_became_peered":"2024-04-12T11:38:06.839460+0000","last_unstale":"2024-04-12T11:39:23.349518+0000","last_undegraded":"2024-04-12T11:39:23.349518+0000","last_fullsized":"2024-04-12T11:39:23.349518+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:25:27.959933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1123,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326708+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:39:23.326708+0000","last_peered":"2024-04-12T11:39:23.326708+0000","last_clean":"2024-04-12T11:39:23.326708+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:39:23.326708+0000","last_undegraded":"2024-04-12T11:39:23.326708+0000","last_fullsized":"2024-04-12T11:39:23.326708+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1664,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320492+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:39:23.320492+0000","last_peered":"2024-04-12T11:39:23.320492+0000","last_clean":"2024-04-12T11:39:23.320492+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:39:23.320492+0000","last_undegraded":"2024-04-12T11:39:23.320492+0000","last_fullsized":"2024-04-12T11:39:23.320492+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1108,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326797+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:39:23.326797+0000","last_peered":"2024-04-12T11:39:23.326797+0000","last_clean":"2024-04-12T11:39:23.326797+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:39:23.326797+0000","last_undegraded":"2024-04-12T11:39:23.326797+0000","last_fullsized":"2024-04-12T11:39:23.326797+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1655,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320289+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:39:23.320289+0000","last_peered":"2024-04-12T11:39:23.320289+0000","last_clean":"2024-04-12T11:39:23.320289+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:39:23.320289+0000","last_undegraded":"2024-04-12T11:39:23.320289+0000","last_fullsized":"2024-04-12T11:39:23.320289+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1648,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.332361+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:39:23.332361+0000","last_peered":"2024-04-12T11:39:23.332361+0000","last_clean":"2024-04-12T11:39:23.332361+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:39:23.332361+0000","last_undegraded":"2024-04-12T11:39:23.332361+0000","last_fullsized":"2024-04-12T11:39:23.332361+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1658,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.336764+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:39:23.336764+0000","last_peered":"2024-04-12T11:39:23.336764+0000","last_clean":"2024-04-12T11:39:23.336764+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:39:23.336764+0000","last_undegraded":"2024-04-12T11:39:23.336764+0000","last_fullsized":"2024-04-12T11:39:23.336764+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1750,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.907261+0000","last_change":"2024-04-12T11:38:06.866508+0000","last_active":"2024-04-12T11:39:36.907261+0000","last_peered":"2024-04-12T11:39:36.907261+0000","last_clean":"2024-04-12T11:39:36.907261+0000","last_became_active":"2024-04-12T11:38:06.843752+0000","last_became_peered":"2024-04-12T11:38:06.843752+0000","last_unstale":"2024-04-12T11:39:36.907261+0000","last_undegraded":"2024-04-12T11:39:36.907261+0000","last_fullsized":"2024-04-12T11:39:36.907261+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:18:11.827424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1627,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.320085+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:39:23.320085+0000","last_peered":"2024-04-12T11:39:23.320085+0000","last_clean":"2024-04-12T11:39:23.320085+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:39:23.320085+0000","last_undegraded":"2024-04-12T11:39:23.320085+0000","last_fullsized":"2024-04-12T11:39:23.320085+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4741005336,"num_objects":72343,"num_object_clones":0,"num_object_copies":217029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5741,"num_objects_misplaced":3580,"num_objects_unfound":0,"num_objects_dirty":72343,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142659,"num_write_kb":6880001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2035,"num_bytes_recovered":133365760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14496968704,"data_stored":14496956488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":135059,"ondisk_log_size":135059,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234327,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1745796,"kb_used_data":1453880,"kb_used_omap":1,"kb_used_meta":291902,"kb_avail":91999356,"statfs":{"total":95995035648,"available":94207340544,"internally_reserved":0,"allocated":1488773120,"data_stored":1485707827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":298908107},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495964,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1655404,"kb_used_data":1453732,"kb_used_omap":1,"kb_used_meta":201662,"kb_avail":92089748,"statfs":{"total":95995035648,"available":94299901952,"internally_reserved":0,"allocated":1488621568,"data_stored":1485568200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":206502344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536274,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2329572,"kb_used_data":2085084,"kb_used_omap":30,"kb_used_meta":244321,"kb_avail":91415580,"statfs":{"total":95995035648,"available":93609553920,"internally_reserved":0,"allocated":2135126016,"data_stored":2132083915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":250185026},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190358,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2557684,"kb_used_data":2290984,"kb_used_omap":43,"kb_used_meta":266644,"kb_avail":91187468,"statfs":{"total":95995035648,"available":93375967232,"internally_reserved":0,"allocated":2345967616,"data_stored":2342914375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":273043499},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2,0,4,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849485,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2132892,"kb_used_data":1785748,"kb_used_omap":36,"kb_used_meta":347099,"kb_avail":91612260,"statfs":{"total":95995035648,"available":93810954240,"internally_reserved":0,"allocated":1828605952,"data_stored":1825564202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":355429556},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704838,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2585496,"kb_used_data":2210832,"kb_used_omap":17,"kb_used_meta":374638,"kb_avail":91159656,"statfs":{"total":95995035648,"available":93347487744,"internally_reserved":0,"allocated":2263891968,"data_stored":2260853788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":383629815},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018050,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1671276,"kb_used_data":1476708,"kb_used_omap":42,"kb_used_meta":194517,"kb_avail":92073876,"statfs":{"total":95995035648,"available":94283649024,"internally_reserved":0,"allocated":1512148992,"data_stored":1509095144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":199186232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757600,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1835676,"kb_used_data":1529812,"kb_used_omap":1,"kb_used_meta":305854,"kb_avail":91909476,"statfs":{"total":95995035648,"available":94115303424,"internally_reserved":0,"allocated":1566527488,"data_stored":1563485762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":313194950},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2333151232,"data_stored":2333147160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2124419072,"data_stored":2124415000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1805320192,"data_stored":1805320192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2240937984,"data_stored":2240937984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1497890816,"data_stored":1497890816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1551630336,"data_stored":1551630336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1468928000,"data_stored":1468923928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1474691072,"data_stored":1474691072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:39:42.619 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:39:42.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:39:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:39:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:39:42.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:42.749 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:42.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:42.850 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:42.877 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:42.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:42.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:42.979 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:43.007 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:43.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:43.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:43.154 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:43.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:43.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:42 smithi087 ceph-mon[89014]: from='client.44939 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:39:43.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:42 smithi087 ceph-mon[89014]: pgmap v1582: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5783/212616 objects degraded (2.720%); 3366/212616 objects misplaced (1.583%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:43.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:42 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5783/212616 objects degraded (2.720%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:43.256 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:43.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:42 smithi029 ceph-mon[97787]: from='client.44939 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:39:43.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:42 smithi029 ceph-mon[97787]: pgmap v1582: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5783/212616 objects degraded (2.720%); 3366/212616 objects misplaced (1.583%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:43.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:42 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5783/212616 objects degraded (2.720%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:42 smithi029 ceph-mon[93650]: from='client.44939 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:39:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:42 smithi029 ceph-mon[93650]: pgmap v1582: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 5783/212616 objects degraded (2.720%); 3366/212616 objects misplaced (1.583%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:42 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5783/212616 objects degraded (2.720%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:43.286 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:43.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:43.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:43.388 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:43.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:43.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:43.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:43.568 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:43.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:43.670 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:43.698 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:43.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:43.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:43.800 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:43.832 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:43.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:43.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:43.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:43.975 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:43.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:43.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:44.076 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:44.116 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:44.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:44.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:44.218 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:44.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:44.255 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:44.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:44.324 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:44.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:44.430 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:44.485 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:44.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:44.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:44.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:44.629 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:44.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:44.730 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:44.778 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:44.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:44.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:44.880 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:44.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:44.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:44.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:45.013 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:45.042 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:45.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:45.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:45.088 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:44 smithi029 ceph-mon[97787]: pgmap v1583: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 5783/207165 objects degraded (2.791%); 3366/207165 objects misplaced (1.625%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:45.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:44 smithi029 ceph-mon[93650]: pgmap v1583: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 5783/207165 objects degraded (2.791%); 3366/207165 objects misplaced (1.625%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:45.089 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:39:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:39:44] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-04-12T11:39:45.143 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:45.174 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:45.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:45.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:45.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:44 smithi087 ceph-mon[89014]: pgmap v1583: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 5783/207165 objects degraded (2.791%); 3366/207165 objects misplaced (1.625%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:45.276 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:45.305 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:45.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:45.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:45.407 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:45.440 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:45.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:45.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:45.542 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:45.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:45.575 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:45.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:45.676 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:45.705 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:45.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:45.807 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:45.835 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:45.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:45.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:45.860 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:39:45.937 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:45.965 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:45.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:45.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:46.067 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:46.094 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:46.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:46.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:46.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:46.225 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:46.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:46.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:46.327 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:46.359 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:46.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:46.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:46.461 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:46.488 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:46.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:46.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:46.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:46.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:46.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:46.732 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:46.758 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:46.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:46.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:46.860 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:46.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:46.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:46.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:46.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:47.014 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:47.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:47.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:46 smithi029 ceph-mon[97787]: pgmap v1584: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5806/202956 objects degraded (2.861%); 3177/202956 objects misplaced (1.565%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:39:47.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:46 smithi029 ceph-mon[93650]: pgmap v1584: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5806/202956 objects degraded (2.861%); 3177/202956 objects misplaced (1.565%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:39:47.037 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:39:47.037 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:39:47.116 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:47.146 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:47.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:47.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:47.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:46 smithi087 ceph-mon[89014]: pgmap v1584: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 5806/202956 objects degraded (2.861%); 3177/202956 objects misplaced (1.565%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:39:47.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:47.285 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:47.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:47.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:47.387 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:47.417 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:47.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:47.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:47.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:47.548 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:47.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:47.650 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:47.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:47.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:47.779 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:47.807 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:47.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:47.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:47.937 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:47.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:47.942 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-04-12T11:39:48.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:48.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:48.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:48.196 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:48.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:47 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5806/202956 objects degraded (2.861%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:48.232 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:48.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:48.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:48.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:47 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5806/202956 objects degraded (2.861%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:48.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:47 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5806/202956 objects degraded (2.861%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:48.333 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:48.424 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:48.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:48.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:48.526 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:48.554 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:48.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:48.640 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:39:48.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:48.656 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:39:48.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:48.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:48.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:48.841 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:48.871 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:48.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:48.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:48.973 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:49.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:49.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:49.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:48 smithi029 ceph-mon[97787]: pgmap v1585: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 5806/200367 objects degraded (2.898%); 3177/200367 objects misplaced (1.586%); 1.9 MiB/s, 31 objects/s recovering 2024-04-12T11:39:49.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:39:49.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:49.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:49.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:48 smithi029 ceph-mon[93650]: pgmap v1585: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 5806/200367 objects degraded (2.898%); 3177/200367 objects misplaced (1.586%); 1.9 MiB/s, 31 objects/s recovering 2024-04-12T11:39:49.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:39:49.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:49.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:49.103 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:49.136 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:49.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:49.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:49.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:49.266 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:49.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:49.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:49.368 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:49.425 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:49.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:49.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:49.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:48 smithi087 ceph-mon[89014]: pgmap v1585: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 5806/200367 objects degraded (2.898%); 3177/200367 objects misplaced (1.586%); 1.9 MiB/s, 31 objects/s recovering 2024-04-12T11:39:49.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:39:49.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:49.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:49.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:49.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:49.573 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:49.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:49.674 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:49.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:49.721 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:49.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:49.822 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:49.853 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:49.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:49.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:49.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:49.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:49.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:49.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:50.065 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:50.083 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1585,"stamp":"2024-04-12T11:39:47.784679+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4376041450,"num_objects":66789,"num_object_clones":0,"num_object_copies":200367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5806,"num_objects_misplaced":3177,"num_objects_unfound":0,"num_objects_dirty":66789,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":148537,"num_write_kb":6882447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2383,"num_bytes_recovered":157289196,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137253,"ondisk_log_size":137253,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15518456,"kb_used_data":13259320,"kb_used_omap":175,"kb_used_meta":2258704,"kb_avail":734442760,"statfs":{"total":767960285184,"available":752069386240,"internally_reserved":0,"allocated":13577543680,"data_stored":13553134898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":2312913065},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-490209280,"num_objects":-7480,"num_object_clones":0,"num_object_copies":-22440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":65,"num_objects_misplaced":-403,"num_objects_unfound":0,"num_objects_dirty":-7480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7480,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":338,"num_bytes_recovered":22151168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.900751"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1666,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.884543+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:39:36.884543+0000","last_peered":"2024-04-12T11:39:36.884543+0000","last_clean":"2024-04-12T11:39:36.884543+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:39:36.884543+0000","last_undegraded":"2024-04-12T11:39:36.884543+0000","last_fullsized":"2024-04-12T11:39:36.884543+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1674,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.906922+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:39:36.906922+0000","last_peered":"2024-04-12T11:39:36.906922+0000","last_clean":"2024-04-12T11:39:36.906922+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:39:36.906922+0000","last_undegraded":"2024-04-12T11:39:36.906922+0000","last_fullsized":"2024-04-12T11:39:36.906922+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1726,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910015+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:39:36.910015+0000","last_peered":"2024-04-12T11:39:36.910015+0000","last_clean":"2024-04-12T11:39:36.910015+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:39:36.910015+0000","last_undegraded":"2024-04-12T11:39:36.910015+0000","last_fullsized":"2024-04-12T11:39:36.910015+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1733,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908315+0000","last_change":"2024-04-12T11:38:21.933647+0000","last_active":"2024-04-12T11:39:36.908315+0000","last_peered":"2024-04-12T11:39:36.908315+0000","last_clean":"2024-04-12T11:39:36.908315+0000","last_became_active":"2024-04-12T11:38:21.933295+0000","last_became_peered":"2024-04-12T11:38:21.933295+0000","last_unstale":"2024-04-12T11:39:36.908315+0000","last_undegraded":"2024-04-12T11:39:36.908315+0000","last_fullsized":"2024-04-12T11:39:36.908315+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:37.402139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1682,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.898632+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:39:36.898632+0000","last_peered":"2024-04-12T11:39:36.898632+0000","last_clean":"2024-04-12T11:39:36.898632+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:39:36.898632+0000","last_undegraded":"2024-04-12T11:39:36.898632+0000","last_fullsized":"2024-04-12T11:39:36.898632+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1101,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.321912+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:39:23.321912+0000","last_peered":"2024-04-12T11:39:23.321912+0000","last_clean":"2024-04-12T11:39:23.321912+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:39:23.321912+0000","last_undegraded":"2024-04-12T11:39:23.321912+0000","last_fullsized":"2024-04-12T11:39:23.321912+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1717,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.907146+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:39:36.907146+0000","last_peered":"2024-04-12T11:39:36.907146+0000","last_clean":"2024-04-12T11:39:36.907146+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:39:36.907146+0000","last_undegraded":"2024-04-12T11:39:36.907146+0000","last_fullsized":"2024-04-12T11:39:36.907146+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1656,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.898720+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:39:36.898720+0000","last_peered":"2024-04-12T11:39:36.898720+0000","last_clean":"2024-04-12T11:39:36.898720+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:39:36.898720+0000","last_undegraded":"2024-04-12T11:39:36.898720+0000","last_fullsized":"2024-04-12T11:39:36.898720+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1107,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:32.320785+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:39:32.320785+0000","last_peered":"2024-04-12T11:39:32.320785+0000","last_clean":"2024-04-12T11:39:32.320785+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:39:32.320785+0000","last_undegraded":"2024-04-12T11:39:32.320785+0000","last_fullsized":"2024-04-12T11:39:32.320785+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1636,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.909228+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:39:36.909228+0000","last_peered":"2024-04-12T11:39:36.909228+0000","last_clean":"2024-04-12T11:39:36.909228+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:39:36.909228+0000","last_undegraded":"2024-04-12T11:39:36.909228+0000","last_fullsized":"2024-04-12T11:39:36.909228+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1116,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:32.441336+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:39:32.441336+0000","last_peered":"2024-04-12T11:39:32.441336+0000","last_clean":"2024-04-12T11:39:32.441336+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:39:32.441336+0000","last_undegraded":"2024-04-12T11:39:32.441336+0000","last_fullsized":"2024-04-12T11:39:32.441336+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1717,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910475+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:39:36.910475+0000","last_peered":"2024-04-12T11:39:36.910475+0000","last_clean":"2024-04-12T11:39:36.910475+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:39:36.910475+0000","last_undegraded":"2024-04-12T11:39:36.910475+0000","last_fullsized":"2024-04-12T11:39:36.910475+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1638,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.884715+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:39:36.884715+0000","last_peered":"2024-04-12T11:39:36.884715+0000","last_clean":"2024-04-12T11:39:36.884715+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:39:36.884715+0000","last_undegraded":"2024-04-12T11:39:36.884715+0000","last_fullsized":"2024-04-12T11:39:36.884715+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1629,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.885024+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:39:36.885024+0000","last_peered":"2024-04-12T11:39:36.885024+0000","last_clean":"2024-04-12T11:39:36.885024+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:39:36.885024+0000","last_undegraded":"2024-04-12T11:39:36.885024+0000","last_fullsized":"2024-04-12T11:39:36.885024+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1115,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337199+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:39:23.337199+0000","last_peered":"2024-04-12T11:39:23.337199+0000","last_clean":"2024-04-12T11:39:23.337199+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:39:23.337199+0000","last_undegraded":"2024-04-12T11:39:23.337199+0000","last_fullsized":"2024-04-12T11:39:23.337199+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1667,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.907235+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:39:36.907235+0000","last_peered":"2024-04-12T11:39:36.907235+0000","last_clean":"2024-04-12T11:39:36.907235+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:39:36.907235+0000","last_undegraded":"2024-04-12T11:39:36.907235+0000","last_fullsized":"2024-04-12T11:39:36.907235+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1718,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.906781+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:39:36.906781+0000","last_peered":"2024-04-12T11:39:36.906781+0000","last_clean":"2024-04-12T11:39:36.906781+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:39:36.906781+0000","last_undegraded":"2024-04-12T11:39:36.906781+0000","last_fullsized":"2024-04-12T11:39:36.906781+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1106,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:32.320748+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:39:32.320748+0000","last_peered":"2024-04-12T11:39:32.320748+0000","last_clean":"2024-04-12T11:39:32.320748+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:39:32.320748+0000","last_undegraded":"2024-04-12T11:39:32.320748+0000","last_fullsized":"2024-04-12T11:39:32.320748+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1098,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337154+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:39:23.337154+0000","last_peered":"2024-04-12T11:39:23.337154+0000","last_clean":"2024-04-12T11:39:23.337154+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:39:23.337154+0000","last_undegraded":"2024-04-12T11:39:23.337154+0000","last_fullsized":"2024-04-12T11:39:23.337154+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1719,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908223+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:39:36.908223+0000","last_peered":"2024-04-12T11:39:36.908223+0000","last_clean":"2024-04-12T11:39:36.908223+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:39:36.908223+0000","last_undegraded":"2024-04-12T11:39:36.908223+0000","last_fullsized":"2024-04-12T11:39:36.908223+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1109,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.321440+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:39:23.321440+0000","last_peered":"2024-04-12T11:39:23.321440+0000","last_clean":"2024-04-12T11:39:23.321440+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:39:23.321440+0000","last_undegraded":"2024-04-12T11:39:23.321440+0000","last_fullsized":"2024-04-12T11:39:23.321440+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1666,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.909321+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:39:36.909321+0000","last_peered":"2024-04-12T11:39:36.909321+0000","last_clean":"2024-04-12T11:39:36.909321+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:39:36.909321+0000","last_undegraded":"2024-04-12T11:39:36.909321+0000","last_fullsized":"2024-04-12T11:39:36.909321+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1157'9252","reported_seq":9311,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:43.362287+0000","last_change":"2024-04-12T11:37:03.057199+0000","last_active":"2024-04-12T11:39:43.362287+0000","last_peered":"2024-04-12T11:39:43.362287+0000","last_clean":"2024-04-12T11:39:43.362287+0000","last_became_active":"2024-04-12T11:37:03.056776+0000","last_became_peered":"2024-04-12T11:37:03.056776+0000","last_unstale":"2024-04-12T11:39:43.362287+0000","last_undegraded":"2024-04-12T11:39:43.362287+0000","last_fullsized":"2024-04-12T11:39:43.362287+0000","mapping_epoch":1133,"log_start":"1140'1300","ondisk_log_start":"1140'1300","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7952,"log_dups_size":0,"ondisk_log_size":7952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:16:29.272872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281018368,"num_objects":4288,"num_object_clones":0,"num_object_copies":12864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9252,"num_write_kb":433280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1114,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337739+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:39:23.337739+0000","last_peered":"2024-04-12T11:39:23.337739+0000","last_clean":"2024-04-12T11:39:23.337739+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:39:23.337739+0000","last_undegraded":"2024-04-12T11:39:23.337739+0000","last_fullsized":"2024-04-12T11:39:23.337739+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1092,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.328901+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:39:23.328901+0000","last_peered":"2024-04-12T11:39:23.328901+0000","last_clean":"2024-04-12T11:39:23.328901+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:39:23.328901+0000","last_undegraded":"2024-04-12T11:39:23.328901+0000","last_fullsized":"2024-04-12T11:39:23.328901+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1157'9303","reported_seq":9362,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:44.849835+0000","last_change":"2024-04-12T11:37:03.059071+0000","last_active":"2024-04-12T11:39:44.849835+0000","last_peered":"2024-04-12T11:39:44.849835+0000","last_clean":"2024-04-12T11:39:44.849835+0000","last_became_active":"2024-04-12T11:37:03.058769+0000","last_became_peered":"2024-04-12T11:37:03.058769+0000","last_unstale":"2024-04-12T11:39:44.849835+0000","last_undegraded":"2024-04-12T11:39:44.849835+0000","last_fullsized":"2024-04-12T11:39:44.849835+0000","mapping_epoch":1133,"log_start":"1140'1800","ondisk_log_start":"1140'1800","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7503,"log_dups_size":0,"ondisk_log_size":7503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:43:26.795096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278986752,"num_objects":4257,"num_object_clones":0,"num_object_copies":12771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9303,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1115,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337703+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:39:23.337703+0000","last_peered":"2024-04-12T11:39:23.337703+0000","last_clean":"2024-04-12T11:39:23.337703+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:39:23.337703+0000","last_undegraded":"2024-04-12T11:39:23.337703+0000","last_fullsized":"2024-04-12T11:39:23.337703+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1652,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.887035+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:39:36.887035+0000","last_peered":"2024-04-12T11:39:36.887035+0000","last_clean":"2024-04-12T11:39:36.887035+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:39:36.887035+0000","last_undegraded":"2024-04-12T11:39:36.887035+0000","last_fullsized":"2024-04-12T11:39:36.887035+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1157'9189","reported_seq":9248,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:43.147215+0000","last_change":"2024-04-12T11:37:03.059390+0000","last_active":"2024-04-12T11:39:43.147215+0000","last_peered":"2024-04-12T11:39:43.147215+0000","last_clean":"2024-04-12T11:39:43.147215+0000","last_became_active":"2024-04-12T11:37:03.058045+0000","last_became_peered":"2024-04-12T11:37:03.058045+0000","last_unstale":"2024-04-12T11:39:43.147215+0000","last_undegraded":"2024-04-12T11:39:43.147215+0000","last_fullsized":"2024-04-12T11:39:43.147215+0000","mapping_epoch":1133,"log_start":"1137'300","ondisk_log_start":"1137'300","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8889,"log_dups_size":0,"ondisk_log_size":8889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:14:01.284893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280297472,"num_objects":4277,"num_object_clones":0,"num_object_copies":12831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9189,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1084,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337523+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:39:23.337523+0000","last_peered":"2024-04-12T11:39:23.337523+0000","last_clean":"2024-04-12T11:39:23.337523+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:39:23.337523+0000","last_undegraded":"2024-04-12T11:39:23.337523+0000","last_fullsized":"2024-04-12T11:39:23.337523+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1101,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.321524+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:39:23.321524+0000","last_peered":"2024-04-12T11:39:23.321524+0000","last_clean":"2024-04-12T11:39:23.321524+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:39:23.321524+0000","last_undegraded":"2024-04-12T11:39:23.321524+0000","last_fullsized":"2024-04-12T11:39:23.321524+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1157'9397","reported_seq":9456,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:44.855395+0000","last_change":"2024-04-12T11:37:03.054920+0000","last_active":"2024-04-12T11:39:44.855395+0000","last_peered":"2024-04-12T11:39:44.855395+0000","last_clean":"2024-04-12T11:39:44.855395+0000","last_became_active":"2024-04-12T11:37:03.054603+0000","last_became_peered":"2024-04-12T11:37:03.054603+0000","last_unstale":"2024-04-12T11:39:44.855395+0000","last_undegraded":"2024-04-12T11:39:44.855395+0000","last_fullsized":"2024-04-12T11:39:44.855395+0000","mapping_epoch":1133,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9397,"log_dups_size":0,"ondisk_log_size":9397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:34:45.149813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275972096,"num_objects":4211,"num_object_clones":0,"num_object_copies":12633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9397,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1107,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326601+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:39:23.326601+0000","last_peered":"2024-04-12T11:39:23.326601+0000","last_clean":"2024-04-12T11:39:23.326601+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:39:23.326601+0000","last_undegraded":"2024-04-12T11:39:23.326601+0000","last_fullsized":"2024-04-12T11:39:23.326601+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1728,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908408+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:39:36.908408+0000","last_peered":"2024-04-12T11:39:36.908408+0000","last_clean":"2024-04-12T11:39:36.908408+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:39:36.908408+0000","last_undegraded":"2024-04-12T11:39:36.908408+0000","last_fullsized":"2024-04-12T11:39:36.908408+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1157'9289","reported_seq":9348,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:43.101959+0000","last_change":"2024-04-12T11:37:03.063209+0000","last_active":"2024-04-12T11:39:43.101959+0000","last_peered":"2024-04-12T11:39:43.101959+0000","last_clean":"2024-04-12T11:39:43.101959+0000","last_became_active":"2024-04-12T11:37:03.062848+0000","last_became_peered":"2024-04-12T11:37:03.062848+0000","last_unstale":"2024-04-12T11:39:43.101959+0000","last_undegraded":"2024-04-12T11:39:43.101959+0000","last_fullsized":"2024-04-12T11:39:43.101959+0000","mapping_epoch":1133,"log_start":"1137'400","ondisk_log_start":"1137'400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8889,"log_dups_size":0,"ondisk_log_size":8889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:55:26.299047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283312128,"num_objects":4323,"num_object_clones":0,"num_object_copies":12969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9289,"num_write_kb":435584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1660,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.909865+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:39:36.909865+0000","last_peered":"2024-04-12T11:39:36.909865+0000","last_clean":"2024-04-12T11:39:36.909865+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:39:36.909865+0000","last_undegraded":"2024-04-12T11:39:36.909865+0000","last_fullsized":"2024-04-12T11:39:36.909865+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1643,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.884813+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:39:36.884813+0000","last_peered":"2024-04-12T11:39:36.884813+0000","last_clean":"2024-04-12T11:39:36.884813+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:39:36.884813+0000","last_undegraded":"2024-04-12T11:39:36.884813+0000","last_fullsized":"2024-04-12T11:39:36.884813+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1157'9194","reported_seq":9253,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:44.821655+0000","last_change":"2024-04-12T11:37:03.060739+0000","last_active":"2024-04-12T11:39:44.821655+0000","last_peered":"2024-04-12T11:39:44.821655+0000","last_clean":"2024-04-12T11:39:44.821655+0000","last_became_active":"2024-04-12T11:37:03.060414+0000","last_became_peered":"2024-04-12T11:37:03.060414+0000","last_unstale":"2024-04-12T11:39:44.821655+0000","last_undegraded":"2024-04-12T11:39:44.821655+0000","last_fullsized":"2024-04-12T11:39:44.821655+0000","mapping_epoch":1133,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9194,"log_dups_size":0,"ondisk_log_size":9194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:12:30.259488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270270464,"num_objects":4124,"num_object_clones":0,"num_object_copies":12372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9194,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":669,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.884444+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:39:36.884444+0000","last_peered":"2024-04-12T11:39:36.884444+0000","last_clean":"2024-04-12T11:39:36.884444+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:39:36.884444+0000","last_undegraded":"2024-04-12T11:39:36.884444+0000","last_fullsized":"2024-04-12T11:39:36.884444+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1665,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.884458+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:39:36.884458+0000","last_peered":"2024-04-12T11:39:36.884458+0000","last_clean":"2024-04-12T11:39:36.884458+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:39:36.884458+0000","last_undegraded":"2024-04-12T11:39:36.884458+0000","last_fullsized":"2024-04-12T11:39:36.884458+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1157'9529","reported_seq":9564,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:47.363149+0000","last_change":"2024-04-12T11:37:03.060364+0000","last_active":"2024-04-12T11:39:47.363149+0000","last_peered":"2024-04-12T11:39:47.363149+0000","last_clean":"2024-04-12T11:39:47.363149+0000","last_became_active":"2024-04-12T11:37:03.060146+0000","last_became_peered":"2024-04-12T11:37:03.060146+0000","last_unstale":"2024-04-12T11:39:47.363149+0000","last_undegraded":"2024-04-12T11:39:47.363149+0000","last_fullsized":"2024-04-12T11:39:47.363149+0000","mapping_epoch":1133,"log_start":"1137'100","ondisk_log_start":"1137'100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9429,"log_dups_size":0,"ondisk_log_size":9429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:03.796242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267386904,"num_objects":4081,"num_object_clones":0,"num_object_copies":12243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4081,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9529,"num_write_kb":435457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1651,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.887089+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:39:36.887089+0000","last_peered":"2024-04-12T11:39:36.887089+0000","last_clean":"2024-04-12T11:39:36.887089+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:39:36.887089+0000","last_undegraded":"2024-04-12T11:39:36.887089+0000","last_fullsized":"2024-04-12T11:39:36.887089+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1630,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.884634+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:39:36.884634+0000","last_peered":"2024-04-12T11:39:36.884634+0000","last_clean":"2024-04-12T11:39:36.884634+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:39:36.884634+0000","last_undegraded":"2024-04-12T11:39:36.884634+0000","last_fullsized":"2024-04-12T11:39:36.884634+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1157'9460","reported_seq":9519,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:44.898542+0000","last_change":"2024-04-12T11:37:03.056864+0000","last_active":"2024-04-12T11:39:44.898542+0000","last_peered":"2024-04-12T11:39:44.898542+0000","last_clean":"2024-04-12T11:39:44.898542+0000","last_became_active":"2024-04-12T11:37:03.056523+0000","last_became_peered":"2024-04-12T11:37:03.056523+0000","last_unstale":"2024-04-12T11:39:44.898542+0000","last_undegraded":"2024-04-12T11:39:44.898542+0000","last_fullsized":"2024-04-12T11:39:44.898542+0000","mapping_epoch":1133,"log_start":"1140'1900","ondisk_log_start":"1140'1900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7560,"log_dups_size":0,"ondisk_log_size":7560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:57:58.750385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273022976,"num_objects":4166,"num_object_clones":0,"num_object_copies":12498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9460,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1668,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.898793+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:39:36.898793+0000","last_peered":"2024-04-12T11:39:36.898793+0000","last_clean":"2024-04-12T11:39:36.898793+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:39:36.898793+0000","last_undegraded":"2024-04-12T11:39:36.898793+0000","last_fullsized":"2024-04-12T11:39:36.898793+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1726,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.909990+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:39:36.909990+0000","last_peered":"2024-04-12T11:39:36.909990+0000","last_clean":"2024-04-12T11:39:36.909990+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:39:36.909990+0000","last_undegraded":"2024-04-12T11:39:36.909990+0000","last_fullsized":"2024-04-12T11:39:36.909990+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1157'9161","reported_seq":13968,"reported_epoch":1157,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-04-12T11:39:44.841824+0000","last_change":"2024-04-12T11:39:24.599690+0000","last_active":"2024-04-12T11:39:44.841824+0000","last_peered":"2024-04-12T11:39:44.841824+0000","last_clean":"2024-04-12T11:38:20.888121+0000","last_became_active":"2024-04-12T11:38:23.417421+0000","last_became_peered":"2024-04-12T11:38:23.417421+0000","last_unstale":"2024-04-12T11:39:44.841824+0000","last_undegraded":"2024-04-12T11:39:24.597597+0000","last_fullsized":"2024-04-12T11:38:23.247711+0000","mapping_epoch":1147,"log_start":"1140'1200","ondisk_log_start":"1140'1200","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7961,"log_dups_size":0,"ondisk_log_size":7961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267845632,"num_objects":4087,"num_object_clones":0,"num_object_copies":12261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":165,"num_objects_misplaced":3177,"num_objects_unfound":0,"num_objects_dirty":4087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9161,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2373,"num_bytes_recovered":155516928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":3177},{"shards":"4,6","objects":745}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1728,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.915621+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:39:36.915621+0000","last_peered":"2024-04-12T11:39:36.915621+0000","last_clean":"2024-04-12T11:39:36.915621+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:39:36.915621+0000","last_undegraded":"2024-04-12T11:39:36.915621+0000","last_fullsized":"2024-04-12T11:39:36.915621+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1123,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326865+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:39:23.326865+0000","last_peered":"2024-04-12T11:39:23.326865+0000","last_clean":"2024-04-12T11:39:23.326865+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:39:23.326865+0000","last_undegraded":"2024-04-12T11:39:23.326865+0000","last_fullsized":"2024-04-12T11:39:23.326865+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1719,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910329+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:39:36.910329+0000","last_peered":"2024-04-12T11:39:36.910329+0000","last_clean":"2024-04-12T11:39:36.910329+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:39:36.910329+0000","last_undegraded":"2024-04-12T11:39:36.910329+0000","last_fullsized":"2024-04-12T11:39:36.910329+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1157'9384","reported_seq":9418,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:47.386915+0000","last_change":"2024-04-12T11:37:03.060583+0000","last_active":"2024-04-12T11:39:47.386915+0000","last_peered":"2024-04-12T11:39:47.386915+0000","last_clean":"2024-04-12T11:39:47.386915+0000","last_became_active":"2024-04-12T11:37:03.060125+0000","last_became_peered":"2024-04-12T11:37:03.060125+0000","last_unstale":"2024-04-12T11:39:47.386915+0000","last_undegraded":"2024-04-12T11:39:47.386915+0000","last_fullsized":"2024-04-12T11:39:47.386915+0000","mapping_epoch":1133,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9384,"log_dups_size":0,"ondisk_log_size":9384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:35:30.859950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267517952,"num_objects":4082,"num_object_clones":0,"num_object_copies":12246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9384,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1113,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326627+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:39:23.326627+0000","last_peered":"2024-04-12T11:39:23.326627+0000","last_clean":"2024-04-12T11:39:23.326627+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:39:23.326627+0000","last_undegraded":"2024-04-12T11:39:23.326627+0000","last_fullsized":"2024-04-12T11:39:23.326627+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1651,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.884885+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:39:36.884885+0000","last_peered":"2024-04-12T11:39:36.884885+0000","last_clean":"2024-04-12T11:39:36.884885+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:39:36.884885+0000","last_undegraded":"2024-04-12T11:39:36.884885+0000","last_fullsized":"2024-04-12T11:39:36.884885+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1728,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.915215+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:39:36.915215+0000","last_peered":"2024-04-12T11:39:36.915215+0000","last_clean":"2024-04-12T11:39:36.915215+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:39:36.915215+0000","last_undegraded":"2024-04-12T11:39:36.915215+0000","last_fullsized":"2024-04-12T11:39:36.915215+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1157'9064","reported_seq":9122,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:43.157674+0000","last_change":"2024-04-12T11:37:03.058470+0000","last_active":"2024-04-12T11:39:43.157674+0000","last_peered":"2024-04-12T11:39:43.157674+0000","last_clean":"2024-04-12T11:39:43.157674+0000","last_became_active":"2024-04-12T11:37:03.058149+0000","last_became_peered":"2024-04-12T11:37:03.058149+0000","last_unstale":"2024-04-12T11:39:43.157674+0000","last_undegraded":"2024-04-12T11:39:43.157674+0000","last_fullsized":"2024-04-12T11:39:43.157674+0000","mapping_epoch":1133,"log_start":"1137'200","ondisk_log_start":"1137'200","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8864,"log_dups_size":0,"ondisk_log_size":8864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:25:06.934877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273940480,"num_objects":4180,"num_object_clones":0,"num_object_copies":12540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9064,"num_write_kb":423808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1692,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.900894+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:39:36.900894+0000","last_peered":"2024-04-12T11:39:36.900894+0000","last_clean":"2024-04-12T11:39:36.900894+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:39:36.900894+0000","last_undegraded":"2024-04-12T11:39:36.900894+0000","last_fullsized":"2024-04-12T11:39:36.900894+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1719,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910150+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:39:36.910150+0000","last_peered":"2024-04-12T11:39:36.910150+0000","last_clean":"2024-04-12T11:39:36.910150+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:39:36.910150+0000","last_undegraded":"2024-04-12T11:39:36.910150+0000","last_fullsized":"2024-04-12T11:39:36.910150+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1722,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.911258+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:39:36.911258+0000","last_peered":"2024-04-12T11:39:36.911258+0000","last_clean":"2024-04-12T11:39:36.911258+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:39:36.911258+0000","last_undegraded":"2024-04-12T11:39:36.911258+0000","last_fullsized":"2024-04-12T11:39:36.911258+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1157'9098","reported_seq":9160,"reported_epoch":1157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:39:43.063477+0000","last_change":"2024-04-12T11:38:23.438040+0000","last_active":"2024-04-12T11:39:43.063477+0000","last_peered":"2024-04-12T11:39:43.063477+0000","last_clean":"2024-04-12T11:38:20.888095+0000","last_became_active":"2024-04-12T11:38:23.424268+0000","last_became_peered":"2024-04-12T11:38:23.424268+0000","last_unstale":"2024-04-12T11:39:43.063477+0000","last_undegraded":"2024-04-12T11:38:23.284099+0000","last_fullsized":"2024-04-12T11:38:23.245558+0000","mapping_epoch":1147,"log_start":"1137'200","ondisk_log_start":"1137'200","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8898,"log_dups_size":0,"ondisk_log_size":8898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276168704,"num_objects":4214,"num_object_clones":0,"num_object_copies":12642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9098,"num_write_kb":425984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":3199},{"shards":"0,7","objects":-1427}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1113,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326543+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:39:23.326543+0000","last_peered":"2024-04-12T11:39:23.326543+0000","last_clean":"2024-04-12T11:39:23.326543+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:39:23.326543+0000","last_undegraded":"2024-04-12T11:39:23.326543+0000","last_fullsized":"2024-04-12T11:39:23.326543+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1468,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:32.441271+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:39:32.441271+0000","last_peered":"2024-04-12T11:39:32.441271+0000","last_clean":"2024-04-12T11:39:32.441271+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:39:32.441271+0000","last_undegraded":"2024-04-12T11:39:32.441271+0000","last_fullsized":"2024-04-12T11:39:32.441271+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1635,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.906756+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:39:36.906756+0000","last_peered":"2024-04-12T11:39:36.906756+0000","last_clean":"2024-04-12T11:39:36.906756+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:39:36.906756+0000","last_undegraded":"2024-04-12T11:39:36.906756+0000","last_fullsized":"2024-04-12T11:39:36.906756+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1101,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.321839+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:39:23.321839+0000","last_peered":"2024-04-12T11:39:23.321839+0000","last_clean":"2024-04-12T11:39:23.321839+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:39:23.321839+0000","last_undegraded":"2024-04-12T11:39:23.321839+0000","last_fullsized":"2024-04-12T11:39:23.321839+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1157'9165","reported_seq":9199,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:47.367796+0000","last_change":"2024-04-12T11:37:03.061698+0000","last_active":"2024-04-12T11:39:47.367796+0000","last_peered":"2024-04-12T11:39:47.367796+0000","last_clean":"2024-04-12T11:39:47.367796+0000","last_became_active":"2024-04-12T11:37:03.061469+0000","last_became_peered":"2024-04-12T11:37:03.061469+0000","last_unstale":"2024-04-12T11:39:47.367796+0000","last_undegraded":"2024-04-12T11:39:47.367796+0000","last_fullsized":"2024-04-12T11:39:47.367796+0000","mapping_epoch":1133,"log_start":"1138'500","ondisk_log_start":"1138'500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8665,"log_dups_size":0,"ondisk_log_size":8665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:09.906340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266797056,"num_objects":4071,"num_object_clones":0,"num_object_copies":12213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9165,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1120,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337067+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:39:23.337067+0000","last_peered":"2024-04-12T11:39:23.337067+0000","last_clean":"2024-04-12T11:39:23.337067+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:39:23.337067+0000","last_undegraded":"2024-04-12T11:39:23.337067+0000","last_fullsized":"2024-04-12T11:39:23.337067+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1729,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.907586+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:39:36.907586+0000","last_peered":"2024-04-12T11:39:36.907586+0000","last_clean":"2024-04-12T11:39:36.907586+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:39:36.907586+0000","last_undegraded":"2024-04-12T11:39:36.907586+0000","last_fullsized":"2024-04-12T11:39:36.907586+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1729,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908121+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:39:36.908121+0000","last_peered":"2024-04-12T11:39:36.908121+0000","last_clean":"2024-04-12T11:39:36.908121+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:39:36.908121+0000","last_undegraded":"2024-04-12T11:39:36.908121+0000","last_fullsized":"2024-04-12T11:39:36.908121+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"1157'9367","reported_seq":9401,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:47.215708+0000","last_change":"2024-04-12T11:37:03.060455+0000","last_active":"2024-04-12T11:39:47.215708+0000","last_peered":"2024-04-12T11:39:47.215708+0000","last_clean":"2024-04-12T11:39:47.215708+0000","last_became_active":"2024-04-12T11:37:03.060244+0000","last_became_peered":"2024-04-12T11:37:03.060244+0000","last_unstale":"2024-04-12T11:39:47.215708+0000","last_undegraded":"2024-04-12T11:39:47.215708+0000","last_fullsized":"2024-04-12T11:39:47.215708+0000","mapping_epoch":1133,"log_start":"0'0","ondisk_log_start":"0'0","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9367,"log_dups_size":0,"ondisk_log_size":9367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:47.787368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270467072,"num_objects":4127,"num_object_clones":0,"num_object_copies":12381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9367,"num_write_kb":431808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1120,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.337548+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:39:23.337548+0000","last_peered":"2024-04-12T11:39:23.337548+0000","last_clean":"2024-04-12T11:39:23.337548+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:39:23.337548+0000","last_undegraded":"2024-04-12T11:39:23.337548+0000","last_fullsized":"2024-04-12T11:39:23.337548+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1719,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.915425+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:39:36.915425+0000","last_peered":"2024-04-12T11:39:36.915425+0000","last_clean":"2024-04-12T11:39:36.915425+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:39:36.915425+0000","last_undegraded":"2024-04-12T11:39:36.915425+0000","last_fullsized":"2024-04-12T11:39:36.915425+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1712,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908671+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:39:36.908671+0000","last_peered":"2024-04-12T11:39:36.908671+0000","last_clean":"2024-04-12T11:39:36.908671+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:39:36.908671+0000","last_undegraded":"2024-04-12T11:39:36.908671+0000","last_fullsized":"2024-04-12T11:39:36.908671+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1157'9050","reported_seq":9109,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:44.944371+0000","last_change":"2024-04-12T11:37:03.057084+0000","last_active":"2024-04-12T11:39:44.944371+0000","last_peered":"2024-04-12T11:39:44.944371+0000","last_clean":"2024-04-12T11:39:44.944371+0000","last_became_active":"2024-04-12T11:37:03.056772+0000","last_became_peered":"2024-04-12T11:37:03.056772+0000","last_unstale":"2024-04-12T11:39:44.944371+0000","last_undegraded":"2024-04-12T11:39:44.944371+0000","last_fullsized":"2024-04-12T11:39:44.944371+0000","mapping_epoch":1133,"log_start":"1140'1500","ondisk_log_start":"1140'1500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7550,"log_dups_size":0,"ondisk_log_size":7550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:02:41.845790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270925824,"num_objects":4134,"num_object_clones":0,"num_object_copies":12402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9050,"num_write_kb":421888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1651,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.909201+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:39:36.909201+0000","last_peered":"2024-04-12T11:39:36.909201+0000","last_clean":"2024-04-12T11:39:36.909201+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:39:36.909201+0000","last_undegraded":"2024-04-12T11:39:36.909201+0000","last_fullsized":"2024-04-12T11:39:36.909201+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1719,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910224+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:39:36.910224+0000","last_peered":"2024-04-12T11:39:36.910224+0000","last_clean":"2024-04-12T11:39:36.910224+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:39:36.910224+0000","last_undegraded":"2024-04-12T11:39:36.910224+0000","last_fullsized":"2024-04-12T11:39:36.910224+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1649,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.898869+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:39:36.898869+0000","last_peered":"2024-04-12T11:39:36.898869+0000","last_clean":"2024-04-12T11:39:36.898869+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:39:36.898869+0000","last_undegraded":"2024-04-12T11:39:36.898869+0000","last_fullsized":"2024-04-12T11:39:36.898869+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1157'9335","reported_seq":9394,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:44.908960+0000","last_change":"2024-04-12T11:37:03.060299+0000","last_active":"2024-04-12T11:39:44.908960+0000","last_peered":"2024-04-12T11:39:44.908960+0000","last_clean":"2024-04-12T11:39:44.908960+0000","last_became_active":"2024-04-12T11:37:03.058897+0000","last_became_peered":"2024-04-12T11:37:03.058897+0000","last_unstale":"2024-04-12T11:39:44.908960+0000","last_undegraded":"2024-04-12T11:39:44.908960+0000","last_fullsized":"2024-04-12T11:39:44.908960+0000","mapping_epoch":1133,"log_start":"1140'1800","ondisk_log_start":"1140'1800","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7535,"log_dups_size":0,"ondisk_log_size":7535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:45:37.844216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271515648,"num_objects":4143,"num_object_clones":0,"num_object_copies":12429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9335,"num_write_kb":431296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1125,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.350168+0000","last_change":"2024-04-12T11:39:23.349899+0000","last_active":"2024-04-12T11:39:23.350168+0000","last_peered":"2024-04-12T11:39:23.350168+0000","last_clean":"2024-04-12T11:39:23.350168+0000","last_became_active":"2024-04-12T11:39:23.349526+0000","last_became_peered":"2024-04-12T11:39:23.349526+0000","last_unstale":"2024-04-12T11:39:23.350168+0000","last_undegraded":"2024-04-12T11:39:23.350168+0000","last_fullsized":"2024-04-12T11:39:23.350168+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:36:47.115912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1732,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908402+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:39:36.908402+0000","last_peered":"2024-04-12T11:39:36.908402+0000","last_clean":"2024-04-12T11:39:36.908402+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:39:36.908402+0000","last_undegraded":"2024-04-12T11:39:36.908402+0000","last_fullsized":"2024-04-12T11:39:36.908402+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1638,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.886867+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:39:36.886867+0000","last_peered":"2024-04-12T11:39:36.886867+0000","last_clean":"2024-04-12T11:39:36.886867+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:39:36.886867+0000","last_undegraded":"2024-04-12T11:39:36.886867+0000","last_fullsized":"2024-04-12T11:39:36.886867+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1722,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.908575+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:39:36.908575+0000","last_peered":"2024-04-12T11:39:36.908575+0000","last_clean":"2024-04-12T11:39:36.908575+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:39:36.908575+0000","last_undegraded":"2024-04-12T11:39:36.908575+0000","last_fullsized":"2024-04-12T11:39:36.908575+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":37,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.901156+0000","last_change":"2024-04-12T11:38:06.839832+0000","last_active":"2024-04-12T11:39:36.901156+0000","last_peered":"2024-04-12T11:39:36.901156+0000","last_clean":"2024-04-12T11:39:36.901156+0000","last_became_active":"2024-04-12T11:38:06.839460+0000","last_became_peered":"2024-04-12T11:38:06.839460+0000","last_unstale":"2024-04-12T11:39:36.901156+0000","last_undegraded":"2024-04-12T11:39:36.901156+0000","last_fullsized":"2024-04-12T11:39:36.901156+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:25:27.959933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1123,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326708+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:39:23.326708+0000","last_peered":"2024-04-12T11:39:23.326708+0000","last_clean":"2024-04-12T11:39:23.326708+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:39:23.326708+0000","last_undegraded":"2024-04-12T11:39:23.326708+0000","last_fullsized":"2024-04-12T11:39:23.326708+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1666,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.885156+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:39:36.885156+0000","last_peered":"2024-04-12T11:39:36.885156+0000","last_clean":"2024-04-12T11:39:36.885156+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:39:36.885156+0000","last_undegraded":"2024-04-12T11:39:36.885156+0000","last_fullsized":"2024-04-12T11:39:36.885156+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1108,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-04-12T11:39:23.326797+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:39:23.326797+0000","last_peered":"2024-04-12T11:39:23.326797+0000","last_clean":"2024-04-12T11:39:23.326797+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:39:23.326797+0000","last_undegraded":"2024-04-12T11:39:23.326797+0000","last_fullsized":"2024-04-12T11:39:23.326797+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1657,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.884349+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:39:36.884349+0000","last_peered":"2024-04-12T11:39:36.884349+0000","last_clean":"2024-04-12T11:39:36.884349+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:39:36.884349+0000","last_undegraded":"2024-04-12T11:39:36.884349+0000","last_fullsized":"2024-04-12T11:39:36.884349+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1650,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.910095+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:39:36.910095+0000","last_peered":"2024-04-12T11:39:36.910095+0000","last_clean":"2024-04-12T11:39:36.910095+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:39:36.910095+0000","last_undegraded":"2024-04-12T11:39:36.910095+0000","last_fullsized":"2024-04-12T11:39:36.910095+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1660,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.909058+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:39:36.909058+0000","last_peered":"2024-04-12T11:39:36.909058+0000","last_clean":"2024-04-12T11:39:36.909058+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:39:36.909058+0000","last_undegraded":"2024-04-12T11:39:36.909058+0000","last_fullsized":"2024-04-12T11:39:36.909058+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1750,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.907261+0000","last_change":"2024-04-12T11:38:06.866508+0000","last_active":"2024-04-12T11:39:36.907261+0000","last_peered":"2024-04-12T11:39:36.907261+0000","last_clean":"2024-04-12T11:39:36.907261+0000","last_became_active":"2024-04-12T11:38:06.843752+0000","last_became_peered":"2024-04-12T11:38:06.843752+0000","last_unstale":"2024-04-12T11:39:36.907261+0000","last_undegraded":"2024-04-12T11:39:36.907261+0000","last_fullsized":"2024-04-12T11:39:36.907261+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:18:11.827424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1629,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.884184+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:39:36.884184+0000","last_peered":"2024-04-12T11:39:36.884184+0000","last_clean":"2024-04-12T11:39:36.884184+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:39:36.884184+0000","last_undegraded":"2024-04-12T11:39:36.884184+0000","last_fullsized":"2024-04-12T11:39:36.884184+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4375445528,"num_objects":66765,"num_object_clones":0,"num_object_copies":200295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5806,"num_objects_misplaced":3177,"num_objects_unfound":0,"num_objects_dirty":66765,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":148237,"num_write_kb":6880001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2373,"num_bytes_recovered":155516928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13428797440,"data_stored":13428785224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137037,"ondisk_log_size":137037,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234329,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1621056,"kb_used_data":1319288,"kb_used_omap":1,"kb_used_meta":301758,"kb_avail":92124096,"statfs":{"total":95995035648,"available":94335074304,"internally_reserved":0,"allocated":1350950912,"data_stored":1347885619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":309000651},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495966,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511224,"kb_used_data":1317744,"kb_used_omap":1,"kb_used_meta":193470,"kb_avail":92233928,"statfs":{"total":95995035648,"available":94447542272,"internally_reserved":0,"allocated":1349369856,"data_stored":1346312633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":198113736},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536275,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2270692,"kb_used_data":2022172,"kb_used_omap":30,"kb_used_meta":248481,"kb_avail":91474460,"statfs":{"total":95995035648,"available":93669847040,"internally_reserved":0,"allocated":2070704128,"data_stored":2067662027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":254444866},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190359,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2458672,"kb_used_data":2187624,"kb_used_omap":43,"kb_used_meta":270804,"kb_avail":91286480,"statfs":{"total":95995035648,"available":93477355520,"internally_reserved":0,"allocated":2240126976,"data_stored":2237073735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":277303339},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849487,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1964264,"kb_used_data":1625440,"kb_used_omap":36,"kb_used_meta":338779,"kb_avail":91780888,"statfs":{"total":95995035648,"available":93983629312,"internally_reserved":0,"allocated":1664450560,"data_stored":1661404955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":346909876},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704840,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2412644,"kb_used_data":2024092,"kb_used_omap":17,"kb_used_meta":388526,"kb_avail":91332508,"statfs":{"total":95995035648,"available":93524488192,"internally_reserved":0,"allocated":2072670208,"data_stored":2069628173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":397851127},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018052,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1542328,"kb_used_data":1339440,"kb_used_omap":42,"kb_used_meta":202837,"kb_avail":92202824,"statfs":{"total":95995035648,"available":94415691776,"internally_reserved":0,"allocated":1371586560,"data_stored":1368528857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":207705912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757602,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1737576,"kb_used_data":1423520,"kb_used_omap":1,"kb_used_meta":314046,"kb_avail":92007576,"statfs":{"total":95995035648,"available":94215757824,"internally_reserved":0,"allocated":1457684480,"data_stored":1454638899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":321583558},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2226655232,"data_stored":2226651160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2045448192,"data_stored":2045444120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1642332160,"data_stored":1642332160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2048393216,"data_stored":2048393216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1360527360,"data_stored":1360527360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1442775040,"data_stored":1442775040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1329401856,"data_stored":1329397784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1333264384,"data_stored":1333264384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:39:50.087 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph status --format=json 2024-04-12T11:39:50.088 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:50.113 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:50.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:50.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:50.214 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:50.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:50.250 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:50.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:50.306 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:50 smithi029 ceph-mon[93650]: from='client.35034 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:39:50.306 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:50 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:50.306 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:50 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:50.306 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:50 smithi029 ceph-mon[97787]: from='client.35034 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:39:50.306 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:50 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:50.307 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:50 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:50.351 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:50.458 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:50.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:50.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:50 smithi087 ceph-mon[89014]: from='client.35034 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:39:50.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:50 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:50.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:50 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:50.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:50.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:50.611 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:50.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:50.712 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:50.737 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:50.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:50.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:50.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:50.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:50.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:50.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:51.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:51.025 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:51.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:51.111 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:39:51.126 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:51.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:51.171 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:51.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:51 smithi029 ceph-mon[97787]: pgmap v1586: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 5806/197004 objects degraded (2.947%); 3177/197004 objects misplaced (1.613%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T11:39:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:51 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1447880233' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:39:51.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:51 smithi029 ceph-mon[93650]: pgmap v1586: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 5806/197004 objects degraded (2.947%); 3177/197004 objects misplaced (1.613%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T11:39:51.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:51 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1447880233' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:39:51.271 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:51.316 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:51.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:51.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:51.372 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:51.418 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:51.449 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:51.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:51.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:51 smithi087 ceph-mon[89014]: pgmap v1586: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 5806/197004 objects degraded (2.947%); 3177/197004 objects misplaced (1.613%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T11:39:51.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:51 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1447880233' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:39:51.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:51.589 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:51.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:51.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:51.730 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:51.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:51.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:51.832 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:51.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:51.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:51.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:51.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:52.041 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:52.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:52.144 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:52.172 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:52.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:52.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:52.250 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:52 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:39:52.250 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:52 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1447880233' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:39:52.251 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:52 smithi029 ceph-mon[97787]: osdmap e1158: 8 total, 8 up, 8 in 2024-04-12T11:39:52.251 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:52.251 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:52 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:39:52.251 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:52 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1447880233' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:39:52.251 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:52 smithi029 ceph-mon[93650]: osdmap e1158: 8 total, 8 up, 8 in 2024-04-12T11:39:52.251 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:52.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:52.314 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:52.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:52.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:52.416 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:52.445 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:52.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:52.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:52.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:39:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:39:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:39:52.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:52 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:39:52.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:52 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1447880233' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:39:52.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:52 smithi087 ceph-mon[89014]: osdmap e1158: 8 total, 8 up, 8 in 2024-04-12T11:39:52.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:39:52.547 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:52.584 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:52.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:52.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:52.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:52.742 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:52.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:52.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:52.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:52.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:52.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:52.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:52.983 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:53.014 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:53.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:53.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:53.029 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:39:53.116 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:53.145 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:53.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:53.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:53.247 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:53.294 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:53.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:53.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:53.428 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:53.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:53.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:53 smithi087 ceph-mon[89014]: pgmap v1588: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5865/192603 objects degraded (3.045%); 2960/192603 objects misplaced (1.537%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:53.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5865/192603 objects degraded (3.045%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:53.483 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:53 smithi029 ceph-mon[93650]: pgmap v1588: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5865/192603 objects degraded (3.045%); 2960/192603 objects misplaced (1.537%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:53.483 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5865/192603 objects degraded (3.045%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:53.483 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:53 smithi029 ceph-mon[97787]: pgmap v1588: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5865/192603 objects degraded (3.045%); 2960/192603 objects misplaced (1.537%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:53.483 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5865/192603 objects degraded (3.045%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:53.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:53.574 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:53.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:53.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:53.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:53.720 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:53.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:53.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:53.821 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:53.851 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:53.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:53.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:53.954 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:39:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:54.001 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:54.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:39:54.102 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:54.152 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:54.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:54.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:54.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:54.300 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:54.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:54.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:54.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:54.443 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:54.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:54.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:54.546 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:54.574 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:54.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:54.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:54.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:54.713 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:54.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:54.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:54.728 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:39:54.815 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:54.847 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:54.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:54.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:54.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:54.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:54.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:54.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:55.085 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:55.117 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:55.117 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:39:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:39:54] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2024-04-12T11:39:55.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:55.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:55.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:55.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:55.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:55.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:55.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:55.384 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:55.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:55.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:55 smithi087 ceph-mon[89014]: pgmap v1589: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5865/187245 objects degraded (3.132%); 2960/187245 objects misplaced (1.581%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:55.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:55.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:55.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:55.512 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:55.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:55.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:55.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:55 smithi029 ceph-mon[93650]: pgmap v1589: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5865/187245 objects degraded (3.132%); 2960/187245 objects misplaced (1.581%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:55.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:55.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:55.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:55 smithi029 ceph-mon[97787]: pgmap v1589: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5865/187245 objects degraded (3.132%); 2960/187245 objects misplaced (1.581%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:39:55.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:55.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:39:55.613 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:55.641 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:55.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:55.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:55.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:55.781 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:55.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:55.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:55.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:55.915 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:55.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:55.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:56.017 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:56.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:56.040 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:56.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:56.141 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:56.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:56.166 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:56.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:56.267 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:56.314 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:56.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:56.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:56.416 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:56.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:56.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:56.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:56.564 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:56.592 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:56.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:56.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:56.742 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:56.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:56.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:56.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:56.894 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:56.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:56.995 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:57.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:57.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:57.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:57.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:57.150 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:57.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:57.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:57.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:57 smithi029 ceph-mon[97787]: pgmap v1590: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5915/182862 objects degraded (3.235%); 2737/182862 objects misplaced (1.497%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:39:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:57 smithi029 ceph-mon[93650]: pgmap v1590: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5915/182862 objects degraded (3.235%); 2737/182862 objects misplaced (1.497%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:39:57.279 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:57.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:57.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:57.381 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:57.409 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:57.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:57.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:57.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:57 smithi087 ceph-mon[89014]: pgmap v1590: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5915/182862 objects degraded (3.235%); 2737/182862 objects misplaced (1.497%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:39:57.510 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:57.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:57.548 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:57.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:57.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:57.687 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:57.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:57.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:57.789 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:57.818 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:57.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:57.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:57.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:57.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:57.959 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:57.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:58.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:58.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:58.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:58.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:58.194 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:58.231 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:58.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:58.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:58.333 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:39:58.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:58.363 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:58.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:39:58.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:39:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5915/182862 objects degraded (3.235%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:58.487 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:58.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:58.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:39:58.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5915/182862 objects degraded (3.235%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5915/182862 objects degraded (3.235%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:39:58.590 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:58.623 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:58.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:58.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:58.725 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:58.753 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:58.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:58.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:58.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:58.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:58.893 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:58.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:58.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:59.025 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:59.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:59.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:59.126 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:39:59.164 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:59.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:59.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:39:59.266 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:39:59.297 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:59.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:59.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:39:59.399 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:39:59.423 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:59.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:59.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:39:59.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:39:59 smithi087 ceph-mon[89014]: pgmap v1591: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5915/180495 objects degraded (3.277%); 2737/180495 objects misplaced (1.516%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:39:59.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:39:59 smithi029 ceph-mon[97787]: pgmap v1591: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5915/180495 objects degraded (3.277%); 2737/180495 objects misplaced (1.516%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:39:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:39:59 smithi029 ceph-mon[93650]: pgmap v1591: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5915/180495 objects degraded (3.277%); 2737/180495 objects misplaced (1.516%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:39:59.526 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:59.556 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:59.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:59.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:59.658 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:39:59.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:39:59.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:59.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:39:59.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:59.818 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:59.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:39:59.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:39:59.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:39:59.959 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:39:59.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:00.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:00.098 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:00.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:00.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:00.200 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:00.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:00.231 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:00.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:00.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:00.360 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:00.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:00.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:00.462 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:00.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:00 smithi087 ceph-mon[89014]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 5915/180495 objects degraded (3.277%), 2 pgs degraded, 2 pgs undersized 2024-04-12T11:40:00.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:00 smithi087 ceph-mon[89014]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-04-12T11:40:00.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:00 smithi087 ceph-mon[89014]: [WRN] PG_DEGRADED: Degraded data redundancy: 5915/180495 objects degraded (3.277%), 2 pgs degraded, 2 pgs undersized 2024-04-12T11:40:00.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:00 smithi087 ceph-mon[89014]: pg 7.1 is stuck undersized for 94s, current state active+recovering+undersized+degraded+remapped, last acting [4,6] 2024-04-12T11:40:00.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:00 smithi087 ceph-mon[89014]: pg 7.6 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-04-12T11:40:00.492 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:00.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:00.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:00.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:00 smithi029 ceph-mon[97787]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 5915/180495 objects degraded (3.277%), 2 pgs degraded, 2 pgs undersized 2024-04-12T11:40:00.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:00 smithi029 ceph-mon[97787]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-04-12T11:40:00.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:00 smithi029 ceph-mon[97787]: [WRN] PG_DEGRADED: Degraded data redundancy: 5915/180495 objects degraded (3.277%), 2 pgs degraded, 2 pgs undersized 2024-04-12T11:40:00.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:00 smithi029 ceph-mon[97787]: pg 7.1 is stuck undersized for 94s, current state active+recovering+undersized+degraded+remapped, last acting [4,6] 2024-04-12T11:40:00.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:00 smithi029 ceph-mon[97787]: pg 7.6 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-04-12T11:40:00.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:00 smithi029 ceph-mon[93650]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 5915/180495 objects degraded (3.277%), 2 pgs degraded, 2 pgs undersized 2024-04-12T11:40:00.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:00 smithi029 ceph-mon[93650]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-04-12T11:40:00.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:00 smithi029 ceph-mon[93650]: [WRN] PG_DEGRADED: Degraded data redundancy: 5915/180495 objects degraded (3.277%), 2 pgs degraded, 2 pgs undersized 2024-04-12T11:40:00.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:00 smithi029 ceph-mon[93650]: pg 7.1 is stuck undersized for 94s, current state active+recovering+undersized+degraded+remapped, last acting [4,6] 2024-04-12T11:40:00.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:00 smithi029 ceph-mon[93650]: pg 7.6 is stuck undersized for 94s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-04-12T11:40:00.594 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:00.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:00.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:00.724 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:00.766 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:00.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:00.867 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:00.898 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:00.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:00.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:01.000 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:01.028 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:01.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:01.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:01.130 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:01.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:01.154 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:01.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:01.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:01.294 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:01.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:01.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:01.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:01.424 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:01.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:01.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:01.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:01 smithi087 ceph-mon[89014]: pgmap v1592: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5915/177723 objects degraded (3.328%); 2737/177723 objects misplaced (1.540%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:40:01.509 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:01 smithi029 ceph-mon[97787]: pgmap v1592: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5915/177723 objects degraded (3.328%); 2737/177723 objects misplaced (1.540%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:40:01.510 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:01 smithi029 ceph-mon[93650]: pgmap v1592: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5915/177723 objects degraded (3.328%); 2737/177723 objects misplaced (1.540%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:40:01.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:01.576 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:01.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:01.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:01.678 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:01.707 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:01.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:01.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:01.809 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:01.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:01.839 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:01.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:01.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:01.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:01.977 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:01.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:02.078 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:02.107 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:02.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:02.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:02.208 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:02.236 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:02.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:02.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:02.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:02.385 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:02.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:02.487 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:02.515 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:02.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:02.616 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:02.642 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:02.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:40:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:40:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:40:02.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:02.773 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:02.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:02.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:02.875 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:02.913 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:02.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:02.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:03.015 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:03.041 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:03.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:03.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:03.142 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:03.166 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:03.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:03.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:03.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:03.291 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:03.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:03.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:03.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:03.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:03.416 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:03.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:03.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:03 smithi029 ceph-mon[97787]: pgmap v1593: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5935/174054 objects degraded (3.410%); 2561/174054 objects misplaced (1.471%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:40:03.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:03 smithi029 ceph-mon[93650]: pgmap v1593: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5935/174054 objects degraded (3.410%); 2561/174054 objects misplaced (1.471%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:40:03.442 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5935/174054 objects degraded (3.410%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:03.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:03 smithi087 ceph-mon[89014]: pgmap v1593: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5935/174054 objects degraded (3.410%); 2561/174054 objects misplaced (1.471%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:40:03.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5935/174054 objects degraded (3.410%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:03.518 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:03.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:03.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:03.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:03.648 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:03.680 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:03.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:03.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5935/174054 objects degraded (3.410%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:03.782 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:03.811 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:03.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:03.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:03.914 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:03.937 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:03.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:03.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:04.039 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:04.063 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:04.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:04.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:04.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:04.193 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:04.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:04.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:04.294 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:04.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:04.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:04.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:04.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:04.449 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:04.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:04.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:04.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:04.584 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:04.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:04.685 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:04.709 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:04.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:04.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:04.811 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:04.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:04.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:04.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:04.978 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:05.007 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:05.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:05.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:05.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:05.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:05.154 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:05.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:05.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:05 smithi029 ceph-mon[97787]: pgmap v1594: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5935/168717 objects degraded (3.518%); 2561/168717 objects misplaced (1.518%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:40:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:05 smithi029 ceph-mon[93650]: pgmap v1594: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5935/168717 objects degraded (3.518%); 2561/168717 objects misplaced (1.518%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:40:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:40:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:40:04] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2024-04-12T11:40:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:05.295 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:05.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:05.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:05.420 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:05.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:05.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:05.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:05 smithi087 ceph-mon[89014]: pgmap v1594: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5935/168717 objects degraded (3.518%); 2561/168717 objects misplaced (1.518%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:40:05.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:05.595 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:05.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:05.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:05.626 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:40:05.674 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:40:05.697 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:05.719 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:40:05.757 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:05.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:05.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:05.761 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:40:05.799 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:40:05.841 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:40:05.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:05.882 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:40:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:05.920 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:05.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:05.932 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:40:05.950 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:40:05.950 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:40:05.950 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:40:05.950 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:40:05.950 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:40:05.950 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:40:05.950 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:40:05.950 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:40:06.022 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:06.080 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:06.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:06.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:06.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:06.228 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:06.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:06.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:06.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1923793128' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:06.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1923793128' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:06.330 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:06.358 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:06.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:06.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:06.460 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:06.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1923793128' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:06.488 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:06.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:06.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:06.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:06.639 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:06.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:06.741 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:06.769 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:06.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:06.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:06.871 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:06.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:06.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:06.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:06.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:07.058 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:07.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:07.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:07.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:07.206 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:07.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:07.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:07.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:07.339 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:07.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:07.439 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:07.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:07 smithi087 ceph-mon[89014]: pgmap v1595: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 5951/164970 objects degraded (3.607%); 2349/164970 objects misplaced (1.424%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T11:40:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:07.477 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:07.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:07.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:07.494 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:07 smithi029 ceph-mon[93650]: pgmap v1595: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 5951/164970 objects degraded (3.607%); 2349/164970 objects misplaced (1.424%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T11:40:07.494 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:07.495 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:07 smithi029 ceph-mon[97787]: pgmap v1595: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 5951/164970 objects degraded (3.607%); 2349/164970 objects misplaced (1.424%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T11:40:07.495 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:07.579 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:07.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:07.622 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:07.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.44843.0:162374 7.7 e1158/1133 7:eb49909b:::benchmark_data_smithi029_161157_object54871:head v 1158'10120, mlcod=1158'10118)", 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:40:07.687068+0000", 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.022925033000000001, 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022955939000000002, 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:40:07.712 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.687068+0000", 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.687068+0000", 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:07.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.687069+0000", 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1500000000000005e-07 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.687074+0000", 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.045e-06 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.687082+0000", 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4789999999999996e-06 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.687088+0000", 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9050000000000002e-06 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.690624+0000", 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035360679999999999 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.690646+0000", 2024-04-12T11:40:07.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2055000000000001e-05 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.44843.0:162376 7.7 e1158/1133 7:e35e7dbe:::benchmark_data_smithi029_161157_object54873:head v 1158'10121, mlcod=1158'10118)", 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:40:07.691039+0000", 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.018954636, 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019055776999999999, 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:07.716 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.691039+0000", 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.691039+0000", 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.691039+0000", 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1600000000000001e-07 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.691045+0000", 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7819999999999999e-06 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.691057+0000", 2024-04-12T11:40:07.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2081e-05 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.691064+0000", 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3779999999999998e-06 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.695810+0000", 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047461420000000001 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:40:07.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.695836+0000", 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6282999999999999e-05 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.44843.0:162379 7.7 e1158/1133 7:e7f21543:::benchmark_data_smithi029_161157_object54876:head v 1158'10122, mlcod=1158'10118)", 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:40:07.694767+0000", 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.015226737000000001, 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015401957000000001, 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:07.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.694767+0000", 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.694767+0000", 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.694768+0000", 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1489999999999999e-06 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.694776+0000", 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1419999999999993e-06 2024-04-12T11:40:07.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.694787+0000", 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1299e-05 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.694795+0000", 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.306e-06 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705272+0000", 2024-04-12T11:40:07.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010477947 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705322+0000", 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9839000000000002e-05 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.44843.0:162378 7.2 7:433e2a6c:::benchmark_data_smithi029_161157_object54875:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1158)", 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:40:07.698719+0000", 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.011274671999999999, 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011513664, 2024-04-12T11:40:07.723 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.44843", 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 162378 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.698719+0000", 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:07.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.698719+0000", 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.698720+0000", 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.756e-06 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.698728+0000", 2024-04-12T11:40:07.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1169999999999998e-06 2024-04-12T11:40:07.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:07.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.698747+0000", 2024-04-12T11:40:07.726 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:07.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8978e-05 2024-04-12T11:40:07.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:07.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.698761+0000", 2024-04-12T11:40:07.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4518999999999999e-05 2024-04-12T11:40:07.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.698838+0000", 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7144000000000005e-05 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.698998+0000", 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015943500000000001 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.699166+0000", 2024-04-12T11:40:07.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016846999999999999 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.703216+0000", 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040498319999999997 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709660+0000", 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006443893 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.44843.0:162381 7.3 7:c921ac5c:::benchmark_data_smithi029_161157_object54878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1158)", 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:40:07.705565+0000", 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0044282590000000004, 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004756751, 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.44843", 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 162381 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.730 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705565+0000", 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705565+0000", 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:07.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705567+0000", 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.782e-06 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705573+0000", 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5740000000000001e-06 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705589+0000", 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5209e-05 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:07.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705604+0000", 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5486999999999999e-05 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705678+0000", 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3841e-05 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705823+0000", 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000144847 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.705969+0000", 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014602500000000001 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(7.1 1158/1147 [PushOp(7:88ef1106:::benchmark_data_smithi029_161157_object55925:head, version: 1141'3476, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:88ef1106:::benchmark_data_smithi029_161157_object55925:head@1141'3476, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:40:07.709292+0000", 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.00070115300000000004, 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011283210000000001, 2024-04-12T11:40:07.734 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709292+0000", 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709292+0000", 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709293+0000", 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5099999999999998e-07 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709343+0000", 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9804e-05 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709368+0000", 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5503e-05 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709386+0000", 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8002000000000001e-05 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709764+0000", 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000377272 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709777+0000", 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3047000000000001e-05 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.44843.0:162385 7.7 e1158/1133)", 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:40:07.709436+0000", 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.00055769700000000001, 2024-04-12T11:40:07.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001041583, 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709436+0000", 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:07.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709436+0000", 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709436+0000", 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3900000000000005e-07 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709442+0000", 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.694e-06 2024-04-12T11:40:07.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709453+0000", 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0995999999999999e-05 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:40:07.709459+0000", 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0789999999999997e-06 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:07.741 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2024-04-12T11:40:07.742 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 7 2024-04-12T11:40:07.742 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:40:07.769 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:07.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:07.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:07.871 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:07.898 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:07.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:07.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:08.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:08.006 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:40:08.006 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5951/164970 objects degraded (3.607%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1188,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1158,"num_osds":8,"num_up_osds":8,"osd_up_since":1712921015,"num_in_osds":8,"osd_in_since":1712920262,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":54990,"data_bytes":3602782186,"bytes_used":13723852800,"bytes_avail":754236432384,"bytes_total":767960285184,"degraded_objects":5951,"degraded_total":164970,"degraded_ratio":0.03607322543492756,"misplaced_objects":2349,"misplaced_total":164970,"misplaced_ratio":0.014238952536824879,"recovering_objects_per_sec":43,"recovering_bytes_per_sec":2866291,"recovering_keys_per_sec":0,"num_objects_recovered":525,"num_bytes_recovered":34406400,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":618},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.scbuux","status":"up:active","gid":14661}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":25,"modified":"2024-04-12T11:37:57.741061+0000","services":{}},"progress_events":{"53786749-146c-4436-89ac-aaaa1350cbf3":{"message":"Upgrade to 19.0.0-2874-g2cee1857 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"d4a8c707-9e6b-4992-81dd-3fa476e5d44b":{"message":"Global Recovery Event (60s)\n [===========================.] (remaining: 2s)","progress":0.96629214286804199,"add_to_ceph_s":true}}} 2024-04-12T11:40:08.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:08.038 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:08.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:08.139 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:08.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:08.167 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:08.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:08.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:08.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/164970 objects degraded (3.607%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:08.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:08 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3554879404' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-12T11:40:08.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/164970 objects degraded (3.607%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:08.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:08 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3554879404' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-12T11:40:08.296 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:08.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:08.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:08.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:08.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:08.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:08.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:08.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/164970 objects degraded (3.607%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:08.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:08 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3554879404' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-12T11:40:08.541 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:08.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:08.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:08.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:08.669 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:08.706 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:08.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:08.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:08.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:08.834 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:08.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:08.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:08.936 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:08.965 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:08.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:08.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:09.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:09.099 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:09.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:09.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:09.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:09 smithi029 ceph-mon[93650]: pgmap v1596: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 5951/162744 objects degraded (3.657%); 2349/162744 objects misplaced (1.443%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:40:09.200 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:09.235 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:09.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:09.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:09.337 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:09.368 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:09.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:09.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:09.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:09 smithi087 ceph-mon[89014]: pgmap v1596: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 5951/162744 objects degraded (3.657%); 2349/162744 objects misplaced (1.443%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:40:09.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:09.514 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:09.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:09.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:09.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:09 smithi029 ceph-mon[97787]: pgmap v1596: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 5951/162744 objects degraded (3.657%); 2349/162744 objects misplaced (1.443%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:40:09.616 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:09.644 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:09.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:09.661 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-04-12T11:40:09.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:40:09.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:09.769 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:09.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:09.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:09.871 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:09.949 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:09.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:10.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:10.081 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:10.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:10.182 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:10.208 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:10.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:10.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:10.311 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:10.334 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:10.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:10.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:10.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:10.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-04-12T11:40:10.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:10.507 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:10.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:10.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:40:10.607 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:10.608 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:10.642 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:10.703 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:10.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:10.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:10.805 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:10.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:10.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:10.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:11.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:11.053 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:11.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:11.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:11.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:11.202 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:11.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:11.303 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:11.332 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:11.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:11.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:11.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:11 smithi029 ceph-mon[93650]: pgmap v1597: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5951/159936 objects degraded (3.721%); 2349/159936 objects misplaced (1.469%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:40:11.382 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:11 smithi029 ceph-mon[97787]: pgmap v1597: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5951/159936 objects degraded (3.721%); 2349/159936 objects misplaced (1.469%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:40:11.434 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:11.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:11 smithi087 ceph-mon[89014]: pgmap v1597: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 5951/159936 objects degraded (3.721%); 2349/159936 objects misplaced (1.469%); 1.8 MiB/s, 29 objects/s recovering 2024-04-12T11:40:11.471 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:11.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:11.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:11.573 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:11.611 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:11.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:11.713 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:11.742 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:11.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:11.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:11.843 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:11.884 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:11.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:11.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:11.986 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:12.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:12.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:12.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:12.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:12.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:12.182 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:12.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:12.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:12.323 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:12.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:12.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:12.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:12.459 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:12.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:12.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:12.560 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:12.589 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:12.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:12.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:12.691 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:40:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:40:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:40:12.730 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:12.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:12.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:12.832 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:12.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:12.876 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:12.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:12.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:13.008 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:13.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:13.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:13.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:13.140 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:13.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:13.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:13.242 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:13.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:13 smithi029 ceph-mon[97787]: pgmap v1598: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5951/155562 objects degraded (3.825%); 2152/155562 objects misplaced (1.383%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T11:40:13.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/155562 objects degraded (3.825%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:13.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:13 smithi029 ceph-mon[93650]: pgmap v1598: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5951/155562 objects degraded (3.825%); 2152/155562 objects misplaced (1.383%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T11:40:13.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/155562 objects degraded (3.825%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:13.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:13.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:13.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:13.370 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:13.395 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:13.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:13.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:13.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:13 smithi087 ceph-mon[89014]: pgmap v1598: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 5951/155562 objects degraded (3.825%); 2152/155562 objects misplaced (1.383%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T11:40:13.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/155562 objects degraded (3.825%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:13.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:13.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:13.584 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:13.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:13.685 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:13.738 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:13.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:13.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:13.840 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:13.869 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:13.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:13.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:13.971 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:13.999 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:14.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:14.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:14.059 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:14.101 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:14.176 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:14.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:14.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:14.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:14.322 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:14.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:14.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:14.424 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:14.451 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:14.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:14.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:14.553 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:14.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:14.592 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:14.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:14.692 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:14.740 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:14.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:14.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:14.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:14.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:14.892 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:14.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:14.993 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:15.084 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:15.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:15.185 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:15.189 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:40:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:40:14] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.43.0" 2024-04-12T11:40:15.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:15.232 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:15.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:15.333 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:15.356 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:40:15.378 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:40:15.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:15.379 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:15.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:15.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:15 smithi087 ceph-mon[89014]: pgmap v1599: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5951/149934 objects degraded (3.969%); 2152/149934 objects misplaced (1.435%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:40:15.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:15.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:15 smithi029 ceph-mon[97787]: pgmap v1599: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5951/149934 objects degraded (3.969%); 2152/149934 objects misplaced (1.435%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:40:15.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:15 smithi029 ceph-mon[93650]: pgmap v1599: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5951/149934 objects degraded (3.969%); 2152/149934 objects misplaced (1.435%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:40:15.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:15.525 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:15.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:15.626 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:15.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:15.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:15.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:15.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:15.778 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:15.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:15.880 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:15.906 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:15.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:16.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:16.048 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:16.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:16.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:16.150 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:16.175 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:16.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:16.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:16.226 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:40:16.278 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:16.305 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:16.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:16.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:16.407 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:16.433 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:16.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:16.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:16.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:16 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/632337677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:40:16.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:16 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:40:16.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:16 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/632337677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:40:16.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:16 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:40:16.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:16 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/632337677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:40:16.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:16 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:40:16.535 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:16.587 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:16.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:16.689 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:16.717 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:16.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:16.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:16.858 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:16.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:16.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:16.960 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:17.003 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:17.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:17.105 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:17.133 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:17.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:17.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:17.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:17.263 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:17.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:17.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:17.365 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:17.391 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1599,"stamp":"2024-04-12T11:40:13.793339+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3274315754,"num_objects":49978,"num_object_clones":0,"num_object_copies":149934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5951,"num_objects_misplaced":2152,"num_objects_unfound":0,"num_objects_dirty":49978,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":165348,"num_write_kb":6882447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3263,"num_bytes_recovered":214960876,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139264,"ondisk_log_size":139264,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12553240,"kb_used_data":10199704,"kb_used_omap":175,"kb_used_meta":2353040,"kb_avail":737407976,"statfs":{"total":767960285184,"available":755105767424,"internally_reserved":0,"allocated":10444496896,"data_stored":10420057274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":2409513129},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,2,9,6,3,7,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-526909440,"num_objects":-8040,"num_object_clones":0,"num_object_copies":-24120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":-409,"num_objects_unfound":0,"num_objects_dirty":-8040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8040,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":393,"num_bytes_recovered":25755648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004427"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1668,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.101327+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:39:51.101327+0000","last_peered":"2024-04-12T11:39:51.101327+0000","last_clean":"2024-04-12T11:39:51.101327+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:39:51.101327+0000","last_undegraded":"2024-04-12T11:39:51.101327+0000","last_fullsized":"2024-04-12T11:39:51.101327+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1676,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107121+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:39:51.107121+0000","last_peered":"2024-04-12T11:39:51.107121+0000","last_clean":"2024-04-12T11:39:51.107121+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:39:51.107121+0000","last_undegraded":"2024-04-12T11:39:51.107121+0000","last_fullsized":"2024-04-12T11:39:51.107121+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1728,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.143138+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:39:51.143138+0000","last_peered":"2024-04-12T11:39:51.143138+0000","last_clean":"2024-04-12T11:39:51.143138+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:39:51.143138+0000","last_undegraded":"2024-04-12T11:39:51.143138+0000","last_fullsized":"2024-04-12T11:39:51.143138+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1736,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.113860+0000","last_change":"2024-04-12T11:38:21.933647+0000","last_active":"2024-04-12T11:39:51.113860+0000","last_peered":"2024-04-12T11:39:51.113860+0000","last_clean":"2024-04-12T11:39:51.113860+0000","last_became_active":"2024-04-12T11:38:21.933295+0000","last_became_peered":"2024-04-12T11:38:21.933295+0000","last_unstale":"2024-04-12T11:39:51.113860+0000","last_undegraded":"2024-04-12T11:39:51.113860+0000","last_fullsized":"2024-04-12T11:39:51.113860+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:37.402139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1685,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107943+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:39:51.107943+0000","last_peered":"2024-04-12T11:39:51.107943+0000","last_clean":"2024-04-12T11:39:51.107943+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:39:51.107943+0000","last_undegraded":"2024-04-12T11:39:51.107943+0000","last_fullsized":"2024-04-12T11:39:51.107943+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1103,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.114656+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:39:51.114656+0000","last_peered":"2024-04-12T11:39:51.114656+0000","last_clean":"2024-04-12T11:39:51.114656+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:39:51.114656+0000","last_undegraded":"2024-04-12T11:39:51.114656+0000","last_fullsized":"2024-04-12T11:39:51.114656+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1719,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.136751+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:39:51.136751+0000","last_peered":"2024-04-12T11:39:51.136751+0000","last_clean":"2024-04-12T11:39:51.136751+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:39:51.136751+0000","last_undegraded":"2024-04-12T11:39:51.136751+0000","last_fullsized":"2024-04-12T11:39:51.136751+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1658,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107988+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:39:51.107988+0000","last_peered":"2024-04-12T11:39:51.107988+0000","last_clean":"2024-04-12T11:39:51.107988+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:39:51.107988+0000","last_undegraded":"2024-04-12T11:39:51.107988+0000","last_fullsized":"2024-04-12T11:39:51.107988+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1108,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.912845+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:39:36.912845+0000","last_peered":"2024-04-12T11:39:36.912845+0000","last_clean":"2024-04-12T11:39:36.912845+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:39:36.912845+0000","last_undegraded":"2024-04-12T11:39:36.912845+0000","last_fullsized":"2024-04-12T11:39:36.912845+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1638,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107179+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:39:51.107179+0000","last_peered":"2024-04-12T11:39:51.107179+0000","last_clean":"2024-04-12T11:39:51.107179+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:39:51.107179+0000","last_undegraded":"2024-04-12T11:39:51.107179+0000","last_fullsized":"2024-04-12T11:39:51.107179+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1117,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.904204+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:39:36.904204+0000","last_peered":"2024-04-12T11:39:36.904204+0000","last_clean":"2024-04-12T11:39:36.904204+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:39:36.904204+0000","last_undegraded":"2024-04-12T11:39:36.904204+0000","last_fullsized":"2024-04-12T11:39:36.904204+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1719,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.125490+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:39:51.125490+0000","last_peered":"2024-04-12T11:39:51.125490+0000","last_clean":"2024-04-12T11:39:51.125490+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:39:51.125490+0000","last_undegraded":"2024-04-12T11:39:51.125490+0000","last_fullsized":"2024-04-12T11:39:51.125490+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1641,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.102432+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:39:51.102432+0000","last_peered":"2024-04-12T11:39:51.102432+0000","last_clean":"2024-04-12T11:39:51.102432+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:39:51.102432+0000","last_undegraded":"2024-04-12T11:39:51.102432+0000","last_fullsized":"2024-04-12T11:39:51.102432+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1632,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.101825+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:39:51.101825+0000","last_peered":"2024-04-12T11:39:51.101825+0000","last_clean":"2024-04-12T11:39:51.101825+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:39:51.101825+0000","last_undegraded":"2024-04-12T11:39:51.101825+0000","last_fullsized":"2024-04-12T11:39:51.101825+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1116,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.903789+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:39:36.903789+0000","last_peered":"2024-04-12T11:39:36.903789+0000","last_clean":"2024-04-12T11:39:36.903789+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:39:36.903789+0000","last_undegraded":"2024-04-12T11:39:36.903789+0000","last_fullsized":"2024-04-12T11:39:36.903789+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1669,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.108399+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:39:51.108399+0000","last_peered":"2024-04-12T11:39:51.108399+0000","last_clean":"2024-04-12T11:39:51.108399+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:39:51.108399+0000","last_undegraded":"2024-04-12T11:39:51.108399+0000","last_fullsized":"2024-04-12T11:39:51.108399+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1720,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.110485+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:39:51.110485+0000","last_peered":"2024-04-12T11:39:51.110485+0000","last_clean":"2024-04-12T11:39:51.110485+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:39:51.110485+0000","last_undegraded":"2024-04-12T11:39:51.110485+0000","last_fullsized":"2024-04-12T11:39:51.110485+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1107,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.912395+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:39:36.912395+0000","last_peered":"2024-04-12T11:39:36.912395+0000","last_clean":"2024-04-12T11:39:36.912395+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:39:36.912395+0000","last_undegraded":"2024-04-12T11:39:36.912395+0000","last_fullsized":"2024-04-12T11:39:36.912395+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1099,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.903722+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:39:36.903722+0000","last_peered":"2024-04-12T11:39:36.903722+0000","last_clean":"2024-04-12T11:39:36.903722+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:39:36.903722+0000","last_undegraded":"2024-04-12T11:39:36.903722+0000","last_fullsized":"2024-04-12T11:39:36.903722+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1721,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.113761+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:39:51.113761+0000","last_peered":"2024-04-12T11:39:51.113761+0000","last_clean":"2024-04-12T11:39:51.113761+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:39:51.113761+0000","last_undegraded":"2024-04-12T11:39:51.113761+0000","last_fullsized":"2024-04-12T11:39:51.113761+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1110,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.892999+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:39:36.892999+0000","last_peered":"2024-04-12T11:39:36.892999+0000","last_clean":"2024-04-12T11:39:36.892999+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:39:36.892999+0000","last_undegraded":"2024-04-12T11:39:36.892999+0000","last_fullsized":"2024-04-12T11:39:36.892999+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1669,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.114686+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:39:51.114686+0000","last_peered":"2024-04-12T11:39:51.114686+0000","last_clean":"2024-04-12T11:39:51.114686+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:39:51.114686+0000","last_undegraded":"2024-04-12T11:39:51.114686+0000","last_fullsized":"2024-04-12T11:39:51.114686+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1158'10507","reported_seq":10568,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:13.392652+0000","last_change":"2024-04-12T11:37:03.057199+0000","last_active":"2024-04-12T11:40:13.392652+0000","last_peered":"2024-04-12T11:40:13.392652+0000","last_clean":"2024-04-12T11:40:13.392652+0000","last_became_active":"2024-04-12T11:37:03.056776+0000","last_became_peered":"2024-04-12T11:37:03.056776+0000","last_unstale":"2024-04-12T11:40:13.392652+0000","last_undegraded":"2024-04-12T11:40:13.392652+0000","last_fullsized":"2024-04-12T11:40:13.392652+0000","mapping_epoch":1133,"log_start":"1140'2600","ondisk_log_start":"1140'2600","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:16:29.272872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198770688,"num_objects":3033,"num_object_clones":0,"num_object_copies":9099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10507,"num_write_kb":433280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1116,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.103860+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:39:51.103860+0000","last_peered":"2024-04-12T11:39:51.103860+0000","last_clean":"2024-04-12T11:39:51.103860+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:39:51.103860+0000","last_undegraded":"2024-04-12T11:39:51.103860+0000","last_fullsized":"2024-04-12T11:39:51.103860+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1093,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.912169+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:39:36.912169+0000","last_peered":"2024-04-12T11:39:36.912169+0000","last_clean":"2024-04-12T11:39:36.912169+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:39:36.912169+0000","last_undegraded":"2024-04-12T11:39:36.912169+0000","last_fullsized":"2024-04-12T11:39:36.912169+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1158'10286","reported_seq":10347,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:10.059044+0000","last_change":"2024-04-12T11:37:03.059071+0000","last_active":"2024-04-12T11:40:10.059044+0000","last_peered":"2024-04-12T11:40:10.059044+0000","last_clean":"2024-04-12T11:40:10.059044+0000","last_became_active":"2024-04-12T11:37:03.058769+0000","last_became_peered":"2024-04-12T11:37:03.058769+0000","last_unstale":"2024-04-12T11:40:10.059044+0000","last_undegraded":"2024-04-12T11:40:10.059044+0000","last_fullsized":"2024-04-12T11:40:10.059044+0000","mapping_epoch":1133,"log_start":"1140'2700","ondisk_log_start":"1140'2700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7586,"log_dups_size":0,"ondisk_log_size":7586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:43:26.795096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214564864,"num_objects":3274,"num_object_clones":0,"num_object_copies":9822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10286,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1117,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.103541+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:39:51.103541+0000","last_peered":"2024-04-12T11:39:51.103541+0000","last_clean":"2024-04-12T11:39:51.103541+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:39:51.103541+0000","last_undegraded":"2024-04-12T11:39:51.103541+0000","last_fullsized":"2024-04-12T11:39:51.103541+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1654,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.105230+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:39:51.105230+0000","last_peered":"2024-04-12T11:39:51.105230+0000","last_clean":"2024-04-12T11:39:51.105230+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:39:51.105230+0000","last_undegraded":"2024-04-12T11:39:51.105230+0000","last_fullsized":"2024-04-12T11:39:51.105230+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1158'10429","reported_seq":10490,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:13.522766+0000","last_change":"2024-04-12T11:37:03.059390+0000","last_active":"2024-04-12T11:40:13.522766+0000","last_peered":"2024-04-12T11:40:13.522766+0000","last_clean":"2024-04-12T11:40:13.522766+0000","last_became_active":"2024-04-12T11:37:03.058045+0000","last_became_peered":"2024-04-12T11:37:03.058045+0000","last_unstale":"2024-04-12T11:40:13.522766+0000","last_undegraded":"2024-04-12T11:40:13.522766+0000","last_fullsized":"2024-04-12T11:40:13.522766+0000","mapping_epoch":1133,"log_start":"1140'1600","ondisk_log_start":"1140'1600","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8829,"log_dups_size":0,"ondisk_log_size":8829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:14:01.284893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199032832,"num_objects":3037,"num_object_clones":0,"num_object_copies":9111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10429,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1086,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107861+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:39:51.107861+0000","last_peered":"2024-04-12T11:39:51.107861+0000","last_clean":"2024-04-12T11:39:51.107861+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:39:51.107861+0000","last_undegraded":"2024-04-12T11:39:51.107861+0000","last_fullsized":"2024-04-12T11:39:51.107861+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1102,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.893088+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:39:36.893088+0000","last_peered":"2024-04-12T11:39:36.893088+0000","last_clean":"2024-04-12T11:39:36.893088+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:39:36.893088+0000","last_undegraded":"2024-04-12T11:39:36.893088+0000","last_fullsized":"2024-04-12T11:39:36.893088+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1158'10398","reported_seq":10459,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:09.860871+0000","last_change":"2024-04-12T11:37:03.054920+0000","last_active":"2024-04-12T11:40:09.860871+0000","last_peered":"2024-04-12T11:40:09.860871+0000","last_clean":"2024-04-12T11:40:09.860871+0000","last_became_active":"2024-04-12T11:37:03.054603+0000","last_became_peered":"2024-04-12T11:37:03.054603+0000","last_unstale":"2024-04-12T11:40:09.860871+0000","last_undegraded":"2024-04-12T11:40:09.860871+0000","last_fullsized":"2024-04-12T11:40:09.860871+0000","mapping_epoch":1133,"log_start":"1137'300","ondisk_log_start":"1137'300","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10098,"log_dups_size":0,"ondisk_log_size":10098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:34:45.149813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210370560,"num_objects":3210,"num_object_clones":0,"num_object_copies":9630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10398,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1109,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.131797+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:39:51.131797+0000","last_peered":"2024-04-12T11:39:51.131797+0000","last_clean":"2024-04-12T11:39:51.131797+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:39:51.131797+0000","last_undegraded":"2024-04-12T11:39:51.131797+0000","last_fullsized":"2024-04-12T11:39:51.131797+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1731,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.110756+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:39:51.110756+0000","last_peered":"2024-04-12T11:39:51.110756+0000","last_clean":"2024-04-12T11:39:51.110756+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:39:51.110756+0000","last_undegraded":"2024-04-12T11:39:51.110756+0000","last_fullsized":"2024-04-12T11:39:51.110756+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1158'10527","reported_seq":10588,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:13.475272+0000","last_change":"2024-04-12T11:37:03.063209+0000","last_active":"2024-04-12T11:40:13.475272+0000","last_peered":"2024-04-12T11:40:13.475272+0000","last_clean":"2024-04-12T11:40:13.475272+0000","last_became_active":"2024-04-12T11:37:03.062848+0000","last_became_peered":"2024-04-12T11:37:03.062848+0000","last_unstale":"2024-04-12T11:40:13.475272+0000","last_undegraded":"2024-04-12T11:40:13.475272+0000","last_fullsized":"2024-04-12T11:40:13.475272+0000","mapping_epoch":1133,"log_start":"1140'1700","ondisk_log_start":"1140'1700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8827,"log_dups_size":0,"ondisk_log_size":8827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:55:26.299047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202178560,"num_objects":3085,"num_object_clones":0,"num_object_copies":9255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10527,"num_write_kb":435584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1662,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107551+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:39:51.107551+0000","last_peered":"2024-04-12T11:39:51.107551+0000","last_clean":"2024-04-12T11:39:51.107551+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:39:51.107551+0000","last_undegraded":"2024-04-12T11:39:51.107551+0000","last_fullsized":"2024-04-12T11:39:51.107551+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1645,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.102478+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:39:51.102478+0000","last_peered":"2024-04-12T11:39:51.102478+0000","last_clean":"2024-04-12T11:39:51.102478+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:39:51.102478+0000","last_undegraded":"2024-04-12T11:39:51.102478+0000","last_fullsized":"2024-04-12T11:39:51.102478+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1158'10157","reported_seq":10218,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:09.915564+0000","last_change":"2024-04-12T11:37:03.060739+0000","last_active":"2024-04-12T11:40:09.915564+0000","last_peered":"2024-04-12T11:40:09.915564+0000","last_clean":"2024-04-12T11:40:09.915564+0000","last_became_active":"2024-04-12T11:37:03.060414+0000","last_became_peered":"2024-04-12T11:37:03.060414+0000","last_unstale":"2024-04-12T11:40:09.915564+0000","last_undegraded":"2024-04-12T11:40:09.915564+0000","last_fullsized":"2024-04-12T11:40:09.915564+0000","mapping_epoch":1133,"log_start":"1137'100","ondisk_log_start":"1137'100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10057,"log_dups_size":0,"ondisk_log_size":10057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:12:30.259488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207159296,"num_objects":3161,"num_object_clones":0,"num_object_copies":9483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10157,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":671,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.101245+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:39:51.101245+0000","last_peered":"2024-04-12T11:39:51.101245+0000","last_clean":"2024-04-12T11:39:51.101245+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:39:51.101245+0000","last_undegraded":"2024-04-12T11:39:51.101245+0000","last_fullsized":"2024-04-12T11:39:51.101245+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1667,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.101895+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:39:51.101895+0000","last_peered":"2024-04-12T11:39:51.101895+0000","last_clean":"2024-04-12T11:39:51.101895+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:39:51.101895+0000","last_undegraded":"2024-04-12T11:39:51.101895+0000","last_fullsized":"2024-04-12T11:39:51.101895+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1158'10499","reported_seq":10534,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:12.359674+0000","last_change":"2024-04-12T11:37:03.060364+0000","last_active":"2024-04-12T11:40:12.359674+0000","last_peered":"2024-04-12T11:40:12.359674+0000","last_clean":"2024-04-12T11:40:12.359674+0000","last_became_active":"2024-04-12T11:37:03.060146+0000","last_became_peered":"2024-04-12T11:37:03.060146+0000","last_unstale":"2024-04-12T11:40:12.359674+0000","last_undegraded":"2024-04-12T11:40:12.359674+0000","last_fullsized":"2024-04-12T11:40:12.359674+0000","mapping_epoch":1133,"log_start":"1139'1100","ondisk_log_start":"1139'1100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9399,"log_dups_size":0,"ondisk_log_size":9399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:03.796242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203816984,"num_objects":3111,"num_object_clones":0,"num_object_copies":9333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3111,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10499,"num_write_kb":435457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1653,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.105313+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:39:51.105313+0000","last_peered":"2024-04-12T11:39:51.105313+0000","last_clean":"2024-04-12T11:39:51.105313+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:39:51.105313+0000","last_undegraded":"2024-04-12T11:39:51.105313+0000","last_fullsized":"2024-04-12T11:39:51.105313+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1632,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.102423+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:39:51.102423+0000","last_peered":"2024-04-12T11:39:51.102423+0000","last_clean":"2024-04-12T11:39:51.102423+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:39:51.102423+0000","last_undegraded":"2024-04-12T11:39:51.102423+0000","last_fullsized":"2024-04-12T11:39:51.102423+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1158'10445","reported_seq":10506,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:10.078566+0000","last_change":"2024-04-12T11:37:03.056864+0000","last_active":"2024-04-12T11:40:10.078566+0000","last_peered":"2024-04-12T11:40:10.078566+0000","last_clean":"2024-04-12T11:40:10.078566+0000","last_became_active":"2024-04-12T11:37:03.056523+0000","last_became_peered":"2024-04-12T11:37:03.056523+0000","last_unstale":"2024-04-12T11:40:10.078566+0000","last_undegraded":"2024-04-12T11:40:10.078566+0000","last_fullsized":"2024-04-12T11:40:10.078566+0000","mapping_epoch":1133,"log_start":"1140'2900","ondisk_log_start":"1140'2900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7545,"log_dups_size":0,"ondisk_log_size":7545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:57:58.750385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208470016,"num_objects":3181,"num_object_clones":0,"num_object_copies":9543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10445,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1670,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.108064+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:39:51.108064+0000","last_peered":"2024-04-12T11:39:51.108064+0000","last_clean":"2024-04-12T11:39:51.108064+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:39:51.108064+0000","last_undegraded":"2024-04-12T11:39:51.108064+0000","last_fullsized":"2024-04-12T11:39:51.108064+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1728,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.124240+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:39:51.124240+0000","last_peered":"2024-04-12T11:39:51.124240+0000","last_clean":"2024-04-12T11:39:51.124240+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:39:51.124240+0000","last_undegraded":"2024-04-12T11:39:51.124240+0000","last_fullsized":"2024-04-12T11:39:51.124240+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1158'10124","reported_seq":16694,"reported_epoch":1158,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-04-12T11:40:09.857820+0000","last_change":"2024-04-12T11:39:24.599690+0000","last_active":"2024-04-12T11:40:09.857820+0000","last_peered":"2024-04-12T11:40:09.857820+0000","last_clean":"2024-04-12T11:38:20.888121+0000","last_became_active":"2024-04-12T11:38:23.417421+0000","last_became_peered":"2024-04-12T11:38:23.417421+0000","last_unstale":"2024-04-12T11:40:09.857820+0000","last_undegraded":"2024-04-12T11:39:24.597597+0000","last_fullsized":"2024-04-12T11:38:23.247711+0000","mapping_epoch":1147,"log_start":"1140'2200","ondisk_log_start":"1140'2200","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7924,"log_dups_size":0,"ondisk_log_size":7924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204734464,"num_objects":3124,"num_object_clones":0,"num_object_copies":9372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":310,"num_objects_misplaced":2152,"num_objects_unfound":0,"num_objects_dirty":3124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10124,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3253,"num_bytes_recovered":213188608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":2152},{"shards":"4,6","objects":662}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1730,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.124723+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:39:51.124723+0000","last_peered":"2024-04-12T11:39:51.124723+0000","last_clean":"2024-04-12T11:39:51.124723+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:39:51.124723+0000","last_undegraded":"2024-04-12T11:39:51.124723+0000","last_fullsized":"2024-04-12T11:39:51.124723+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1124,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.912474+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:39:36.912474+0000","last_peered":"2024-04-12T11:39:36.912474+0000","last_clean":"2024-04-12T11:39:36.912474+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:39:36.912474+0000","last_undegraded":"2024-04-12T11:39:36.912474+0000","last_fullsized":"2024-04-12T11:39:36.912474+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1721,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.125326+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:39:51.125326+0000","last_peered":"2024-04-12T11:39:51.125326+0000","last_clean":"2024-04-12T11:39:51.125326+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:39:51.125326+0000","last_undegraded":"2024-04-12T11:39:51.125326+0000","last_fullsized":"2024-04-12T11:39:51.125326+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1158'10307","reported_seq":10342,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:12.355740+0000","last_change":"2024-04-12T11:37:03.060583+0000","last_active":"2024-04-12T11:40:12.355740+0000","last_peered":"2024-04-12T11:40:12.355740+0000","last_clean":"2024-04-12T11:40:12.355740+0000","last_became_active":"2024-04-12T11:37:03.060125+0000","last_became_peered":"2024-04-12T11:37:03.060125+0000","last_unstale":"2024-04-12T11:40:12.355740+0000","last_undegraded":"2024-04-12T11:40:12.355740+0000","last_fullsized":"2024-04-12T11:40:12.355740+0000","mapping_epoch":1133,"log_start":"1138'900","ondisk_log_start":"1138'900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9407,"log_dups_size":0,"ondisk_log_size":9407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:35:30.859950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207028224,"num_objects":3159,"num_object_clones":0,"num_object_copies":9477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10307,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1114,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.912326+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:39:36.912326+0000","last_peered":"2024-04-12T11:39:36.912326+0000","last_clean":"2024-04-12T11:39:36.912326+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:39:36.912326+0000","last_undegraded":"2024-04-12T11:39:36.912326+0000","last_fullsized":"2024-04-12T11:39:36.912326+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1653,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.101634+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:39:51.101634+0000","last_peered":"2024-04-12T11:39:51.101634+0000","last_clean":"2024-04-12T11:39:51.101634+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:39:51.101634+0000","last_undegraded":"2024-04-12T11:39:51.101634+0000","last_fullsized":"2024-04-12T11:39:51.101634+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1730,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.124459+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:39:51.124459+0000","last_peered":"2024-04-12T11:39:51.124459+0000","last_clean":"2024-04-12T11:39:51.124459+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:39:51.124459+0000","last_undegraded":"2024-04-12T11:39:51.124459+0000","last_fullsized":"2024-04-12T11:39:51.124459+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1158'10229","reported_seq":10290,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:13.470490+0000","last_change":"2024-04-12T11:37:03.058470+0000","last_active":"2024-04-12T11:40:13.470490+0000","last_peered":"2024-04-12T11:40:13.470490+0000","last_clean":"2024-04-12T11:40:13.470490+0000","last_became_active":"2024-04-12T11:37:03.058149+0000","last_became_peered":"2024-04-12T11:37:03.058149+0000","last_unstale":"2024-04-12T11:40:13.470490+0000","last_undegraded":"2024-04-12T11:40:13.470490+0000","last_fullsized":"2024-04-12T11:40:13.470490+0000","mapping_epoch":1133,"log_start":"1140'1400","ondisk_log_start":"1140'1400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8829,"log_dups_size":0,"ondisk_log_size":8829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:25:06.934877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197591040,"num_objects":3015,"num_object_clones":0,"num_object_copies":9045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10229,"num_write_kb":423808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1694,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.114253+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:39:51.114253+0000","last_peered":"2024-04-12T11:39:51.114253+0000","last_clean":"2024-04-12T11:39:51.114253+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:39:51.114253+0000","last_undegraded":"2024-04-12T11:39:51.114253+0000","last_fullsized":"2024-04-12T11:39:51.114253+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1721,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.124423+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:39:51.124423+0000","last_peered":"2024-04-12T11:39:51.124423+0000","last_clean":"2024-04-12T11:39:51.124423+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:39:51.124423+0000","last_undegraded":"2024-04-12T11:39:51.124423+0000","last_fullsized":"2024-04-12T11:39:51.124423+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1724,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.124274+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:39:51.124274+0000","last_peered":"2024-04-12T11:39:51.124274+0000","last_clean":"2024-04-12T11:39:51.124274+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:39:51.124274+0000","last_undegraded":"2024-04-12T11:39:51.124274+0000","last_fullsized":"2024-04-12T11:39:51.124274+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1158'10282","reported_seq":10346,"reported_epoch":1158,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:40:13.507697+0000","last_change":"2024-04-12T11:38:23.438040+0000","last_active":"2024-04-12T11:40:13.507697+0000","last_peered":"2024-04-12T11:40:13.507697+0000","last_clean":"2024-04-12T11:38:20.888095+0000","last_became_active":"2024-04-12T11:38:23.424268+0000","last_became_peered":"2024-04-12T11:38:23.424268+0000","last_unstale":"2024-04-12T11:40:13.507697+0000","last_undegraded":"2024-04-12T11:38:23.284099+0000","last_fullsized":"2024-04-12T11:38:23.245558+0000","mapping_epoch":1147,"log_start":"1140'1400","ondisk_log_start":"1140'1400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8882,"log_dups_size":0,"ondisk_log_size":8882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198574080,"num_objects":3030,"num_object_clones":0,"num_object_copies":9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10282,"num_write_kb":425984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":2015},{"shards":"0,7","objects":-2611}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1114,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.912520+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:39:36.912520+0000","last_peered":"2024-04-12T11:39:36.912520+0000","last_clean":"2024-04-12T11:39:36.912520+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:39:36.912520+0000","last_undegraded":"2024-04-12T11:39:36.912520+0000","last_fullsized":"2024-04-12T11:39:36.912520+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1469,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.906742+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:39:36.906742+0000","last_peered":"2024-04-12T11:39:36.906742+0000","last_clean":"2024-04-12T11:39:36.906742+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:39:36.906742+0000","last_undegraded":"2024-04-12T11:39:36.906742+0000","last_fullsized":"2024-04-12T11:39:36.906742+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1638,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.114513+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:39:51.114513+0000","last_peered":"2024-04-12T11:39:51.114513+0000","last_clean":"2024-04-12T11:39:51.114513+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:39:51.114513+0000","last_undegraded":"2024-04-12T11:39:51.114513+0000","last_fullsized":"2024-04-12T11:39:51.114513+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1103,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.110326+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:39:51.110326+0000","last_peered":"2024-04-12T11:39:51.110326+0000","last_clean":"2024-04-12T11:39:51.110326+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:39:51.110326+0000","last_undegraded":"2024-04-12T11:39:51.110326+0000","last_fullsized":"2024-04-12T11:39:51.110326+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1158'10166","reported_seq":10201,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:12.355397+0000","last_change":"2024-04-12T11:37:03.061698+0000","last_active":"2024-04-12T11:40:12.355397+0000","last_peered":"2024-04-12T11:40:12.355397+0000","last_clean":"2024-04-12T11:40:12.355397+0000","last_became_active":"2024-04-12T11:37:03.061469+0000","last_became_peered":"2024-04-12T11:37:03.061469+0000","last_unstale":"2024-04-12T11:40:12.355397+0000","last_undegraded":"2024-04-12T11:40:12.355397+0000","last_fullsized":"2024-04-12T11:40:12.355397+0000","mapping_epoch":1133,"log_start":"1140'1500","ondisk_log_start":"1140'1500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8666,"log_dups_size":0,"ondisk_log_size":8666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:09.906340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201195520,"num_objects":3070,"num_object_clones":0,"num_object_copies":9210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10166,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1122,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107289+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:39:51.107289+0000","last_peered":"2024-04-12T11:39:51.107289+0000","last_clean":"2024-04-12T11:39:51.107289+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:39:51.107289+0000","last_undegraded":"2024-04-12T11:39:51.107289+0000","last_fullsized":"2024-04-12T11:39:51.107289+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1731,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.124690+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:39:51.124690+0000","last_peered":"2024-04-12T11:39:51.124690+0000","last_clean":"2024-04-12T11:39:51.124690+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:39:51.124690+0000","last_undegraded":"2024-04-12T11:39:51.124690+0000","last_fullsized":"2024-04-12T11:39:51.124690+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1731,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.113678+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:39:51.113678+0000","last_peered":"2024-04-12T11:39:51.113678+0000","last_clean":"2024-04-12T11:39:51.113678+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:39:51.113678+0000","last_undegraded":"2024-04-12T11:39:51.113678+0000","last_fullsized":"2024-04-12T11:39:51.113678+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"1158'10344","reported_seq":10379,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:12.294085+0000","last_change":"2024-04-12T11:37:03.060455+0000","last_active":"2024-04-12T11:40:12.294085+0000","last_peered":"2024-04-12T11:40:12.294085+0000","last_clean":"2024-04-12T11:40:12.294085+0000","last_became_active":"2024-04-12T11:37:03.060244+0000","last_became_peered":"2024-04-12T11:37:03.060244+0000","last_unstale":"2024-04-12T11:40:12.294085+0000","last_undegraded":"2024-04-12T11:40:12.294085+0000","last_fullsized":"2024-04-12T11:40:12.294085+0000","mapping_epoch":1133,"log_start":"1137'300","ondisk_log_start":"1137'300","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10044,"log_dups_size":0,"ondisk_log_size":10044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:47.787368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206438400,"num_objects":3150,"num_object_clones":0,"num_object_copies":9450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10344,"num_write_kb":431808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1121,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.904153+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:39:36.904153+0000","last_peered":"2024-04-12T11:39:36.904153+0000","last_clean":"2024-04-12T11:39:36.904153+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:39:36.904153+0000","last_undegraded":"2024-04-12T11:39:36.904153+0000","last_fullsized":"2024-04-12T11:39:36.904153+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1721,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.124641+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:39:51.124641+0000","last_peered":"2024-04-12T11:39:51.124641+0000","last_clean":"2024-04-12T11:39:51.124641+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:39:51.124641+0000","last_undegraded":"2024-04-12T11:39:51.124641+0000","last_fullsized":"2024-04-12T11:39:51.124641+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1714,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.110972+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:39:51.110972+0000","last_peered":"2024-04-12T11:39:51.110972+0000","last_clean":"2024-04-12T11:39:51.110972+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:39:51.110972+0000","last_undegraded":"2024-04-12T11:39:51.110972+0000","last_fullsized":"2024-04-12T11:39:51.110972+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1158'10007","reported_seq":10068,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:10.023251+0000","last_change":"2024-04-12T11:37:03.057084+0000","last_active":"2024-04-12T11:40:10.023251+0000","last_peered":"2024-04-12T11:40:10.023251+0000","last_clean":"2024-04-12T11:40:10.023251+0000","last_became_active":"2024-04-12T11:37:03.056772+0000","last_became_peered":"2024-04-12T11:37:03.056772+0000","last_unstale":"2024-04-12T11:40:10.023251+0000","last_undegraded":"2024-04-12T11:40:10.023251+0000","last_fullsized":"2024-04-12T11:40:10.023251+0000","mapping_epoch":1133,"log_start":"1140'2500","ondisk_log_start":"1140'2500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7507,"log_dups_size":0,"ondisk_log_size":7507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:02:41.845790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208207872,"num_objects":3177,"num_object_clones":0,"num_object_copies":9531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10007,"num_write_kb":421888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1653,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.116879+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:39:51.116879+0000","last_peered":"2024-04-12T11:39:51.116879+0000","last_clean":"2024-04-12T11:39:51.116879+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:39:51.116879+0000","last_undegraded":"2024-04-12T11:39:51.116879+0000","last_fullsized":"2024-04-12T11:39:51.116879+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1721,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.124018+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:39:51.124018+0000","last_peered":"2024-04-12T11:39:51.124018+0000","last_clean":"2024-04-12T11:39:51.124018+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:39:51.124018+0000","last_undegraded":"2024-04-12T11:39:51.124018+0000","last_fullsized":"2024-04-12T11:39:51.124018+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1651,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.108152+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:39:51.108152+0000","last_peered":"2024-04-12T11:39:51.108152+0000","last_clean":"2024-04-12T11:39:51.108152+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:39:51.108152+0000","last_undegraded":"2024-04-12T11:39:51.108152+0000","last_fullsized":"2024-04-12T11:39:51.108152+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1158'10341","reported_seq":10402,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:10.048322+0000","last_change":"2024-04-12T11:37:03.060299+0000","last_active":"2024-04-12T11:40:10.048322+0000","last_peered":"2024-04-12T11:40:10.048322+0000","last_clean":"2024-04-12T11:40:10.048322+0000","last_became_active":"2024-04-12T11:37:03.058897+0000","last_became_peered":"2024-04-12T11:37:03.058897+0000","last_unstale":"2024-04-12T11:40:10.048322+0000","last_undegraded":"2024-04-12T11:40:10.048322+0000","last_fullsized":"2024-04-12T11:40:10.048322+0000","mapping_epoch":1133,"log_start":"1140'2800","ondisk_log_start":"1140'2800","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:45:37.844216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205586432,"num_objects":3137,"num_object_clones":0,"num_object_copies":9411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10341,"num_write_kb":431296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1127,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.114540+0000","last_change":"2024-04-12T11:39:23.349899+0000","last_active":"2024-04-12T11:39:51.114540+0000","last_peered":"2024-04-12T11:39:51.114540+0000","last_clean":"2024-04-12T11:39:51.114540+0000","last_became_active":"2024-04-12T11:39:23.349526+0000","last_became_peered":"2024-04-12T11:39:23.349526+0000","last_unstale":"2024-04-12T11:39:51.114540+0000","last_undegraded":"2024-04-12T11:39:51.114540+0000","last_fullsized":"2024-04-12T11:39:51.114540+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:36:47.115912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1734,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.113902+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:39:51.113902+0000","last_peered":"2024-04-12T11:39:51.113902+0000","last_clean":"2024-04-12T11:39:51.113902+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:39:51.113902+0000","last_undegraded":"2024-04-12T11:39:51.113902+0000","last_fullsized":"2024-04-12T11:39:51.113902+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1640,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.104953+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:39:51.104953+0000","last_peered":"2024-04-12T11:39:51.104953+0000","last_clean":"2024-04-12T11:39:51.104953+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:39:51.104953+0000","last_undegraded":"2024-04-12T11:39:51.104953+0000","last_fullsized":"2024-04-12T11:39:51.104953+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1724,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.110871+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:39:51.110871+0000","last_peered":"2024-04-12T11:39:51.110871+0000","last_clean":"2024-04-12T11:39:51.110871+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:39:51.110871+0000","last_undegraded":"2024-04-12T11:39:51.110871+0000","last_fullsized":"2024-04-12T11:39:51.110871+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":39,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.113768+0000","last_change":"2024-04-12T11:38:06.839832+0000","last_active":"2024-04-12T11:39:51.113768+0000","last_peered":"2024-04-12T11:39:51.113768+0000","last_clean":"2024-04-12T11:39:51.113768+0000","last_became_active":"2024-04-12T11:38:06.839460+0000","last_became_peered":"2024-04-12T11:38:06.839460+0000","last_unstale":"2024-04-12T11:39:51.113768+0000","last_undegraded":"2024-04-12T11:39:51.113768+0000","last_fullsized":"2024-04-12T11:39:51.113768+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:25:27.959933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1124,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.912521+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:39:36.912521+0000","last_peered":"2024-04-12T11:39:36.912521+0000","last_clean":"2024-04-12T11:39:36.912521+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:39:36.912521+0000","last_undegraded":"2024-04-12T11:39:36.912521+0000","last_fullsized":"2024-04-12T11:39:36.912521+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1668,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.101917+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:39:51.101917+0000","last_peered":"2024-04-12T11:39:51.101917+0000","last_clean":"2024-04-12T11:39:51.101917+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:39:51.101917+0000","last_undegraded":"2024-04-12T11:39:51.101917+0000","last_fullsized":"2024-04-12T11:39:51.101917+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1109,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-04-12T11:39:36.912947+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:39:36.912947+0000","last_peered":"2024-04-12T11:39:36.912947+0000","last_clean":"2024-04-12T11:39:36.912947+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:39:36.912947+0000","last_undegraded":"2024-04-12T11:39:36.912947+0000","last_fullsized":"2024-04-12T11:39:36.912947+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1659,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.101152+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:39:51.101152+0000","last_peered":"2024-04-12T11:39:51.101152+0000","last_clean":"2024-04-12T11:39:51.101152+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:39:51.101152+0000","last_undegraded":"2024-04-12T11:39:51.101152+0000","last_fullsized":"2024-04-12T11:39:51.101152+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1652,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107391+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:39:51.107391+0000","last_peered":"2024-04-12T11:39:51.107391+0000","last_clean":"2024-04-12T11:39:51.107391+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:39:51.107391+0000","last_undegraded":"2024-04-12T11:39:51.107391+0000","last_fullsized":"2024-04-12T11:39:51.107391+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1663,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107048+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:39:51.107048+0000","last_peered":"2024-04-12T11:39:51.107048+0000","last_clean":"2024-04-12T11:39:51.107048+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:39:51.107048+0000","last_undegraded":"2024-04-12T11:39:51.107048+0000","last_fullsized":"2024-04-12T11:39:51.107048+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1752,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.136865+0000","last_change":"2024-04-12T11:38:06.866508+0000","last_active":"2024-04-12T11:39:51.136865+0000","last_peered":"2024-04-12T11:39:51.136865+0000","last_clean":"2024-04-12T11:39:51.136865+0000","last_became_active":"2024-04-12T11:38:06.843752+0000","last_became_peered":"2024-04-12T11:38:06.843752+0000","last_unstale":"2024-04-12T11:39:51.136865+0000","last_undegraded":"2024-04-12T11:39:51.136865+0000","last_fullsized":"2024-04-12T11:39:51.136865+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:18:11.827424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1631,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.106176+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:39:51.106176+0000","last_peered":"2024-04-12T11:39:51.106176+0000","last_clean":"2024-04-12T11:39:51.106176+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:39:51.106176+0000","last_undegraded":"2024-04-12T11:39:51.106176+0000","last_fullsized":"2024-04-12T11:39:51.106176+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3273719832,"num_objects":49954,"num_object_clones":0,"num_object_copies":149862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5951,"num_objects_misplaced":2152,"num_objects_unfound":0,"num_objects_dirty":49954,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165048,"num_write_kb":6880001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3253,"num_bytes_recovered":213188608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10285297664,"data_stored":10285285448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139048,"ondisk_log_size":139048,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234334,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335180,"kb_used_data":1011268,"kb_used_omap":1,"kb_used_meta":323902,"kb_avail":92409972,"statfs":{"total":95995035648,"available":94627811328,"internally_reserved":0,"allocated":1035538432,"data_stored":1032469284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":331676107},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495971,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1218948,"kb_used_data":1003324,"kb_used_omap":1,"kb_used_meta":215614,"kb_avail":92526204,"statfs":{"total":95995035648,"available":94746832896,"internally_reserved":0,"allocated":1027403776,"data_stored":1024342698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":220789192},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536281,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1807728,"kb_used_data":1545192,"kb_used_omap":30,"kb_used_meta":262369,"kb_avail":91937424,"statfs":{"total":95995035648,"available":94143922176,"internally_reserved":0,"allocated":1582276608,"data_stored":1579230652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":268666178},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190365,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1896508,"kb_used_data":1589556,"kb_used_omap":43,"kb_used_meta":306900,"kb_avail":91848644,"statfs":{"total":95995035648,"available":94053011456,"internally_reserved":0,"allocated":1627705344,"data_stored":1624648248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":314265643},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,7,1,0,2,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849492,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1586420,"kb_used_data":1243436,"kb_used_omap":36,"kb_used_meta":342939,"kb_avail":92158732,"statfs":{"total":95995035648,"available":94370541568,"internally_reserved":0,"allocated":1273278464,"data_stored":1270229004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":351169716},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704845,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1976240,"kb_used_data":1587560,"kb_used_omap":17,"kb_used_meta":388526,"kb_avail":91768912,"statfs":{"total":95995035648,"available":93971365888,"internally_reserved":0,"allocated":1625661440,"data_stored":1622615550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":397851127},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018057,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1243908,"kb_used_data":1041020,"kb_used_omap":42,"kb_used_meta":202837,"kb_avail":92501244,"statfs":{"total":95995035648,"available":94721273856,"internally_reserved":0,"allocated":1066004480,"data_stored":1062942922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":207705912},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757607,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1488308,"kb_used_data":1178348,"kb_used_omap":1,"kb_used_meta":309950,"kb_avail":92256844,"statfs":{"total":95995035648,"available":94471008256,"internally_reserved":0,"allocated":1206628352,"data_stored":1203578916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":317389254},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1596395520,"data_stored":1596391448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1572474880,"data_stored":1572470808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1254490112,"data_stored":1254490112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1603403776,"data_stored":1603403776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1042612224,"data_stored":1042612224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1187250176,"data_stored":1187250176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1013387264,"data_stored":1013383192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1015283712,"data_stored":1015283712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:40:17.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:17.396 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:17.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:17.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:17 smithi087 ceph-mon[89014]: from='client.44948 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:40:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:17 smithi087 ceph-mon[89014]: pgmap v1600: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 5951/145524 objects degraded (4.089%); 1959/145524 objects misplaced (1.346%); 3.1 MiB/s, 48 objects/s recovering 2024-04-12T11:40:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:17 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:40:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:17 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:40:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:17 smithi087 ceph-mon[89014]: osdmap e1159: 8 total, 8 up, 8 in 2024-04-12T11:40:17.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:17.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:17 smithi029 ceph-mon[97787]: from='client.44948 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:40:17.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:17 smithi029 ceph-mon[97787]: pgmap v1600: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 5951/145524 objects degraded (4.089%); 1959/145524 objects misplaced (1.346%); 3.1 MiB/s, 48 objects/s recovering 2024-04-12T11:40:17.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:17 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:40:17.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:17 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:40:17.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:17 smithi029 ceph-mon[97787]: osdmap e1159: 8 total, 8 up, 8 in 2024-04-12T11:40:17.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:17 smithi029 ceph-mon[93650]: from='client.44948 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:40:17.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:17 smithi029 ceph-mon[93650]: pgmap v1600: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 5951/145524 objects degraded (4.089%); 1959/145524 objects misplaced (1.346%); 3.1 MiB/s, 48 objects/s recovering 2024-04-12T11:40:17.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:17 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:40:17.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:17 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:40:17.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:17 smithi029 ceph-mon[93650]: osdmap e1159: 8 total, 8 up, 8 in 2024-04-12T11:40:17.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:17.540 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:17.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:17.632 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:17.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:17.666 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:17.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:17.768 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:17.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:17.797 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:17.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:17.897 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:17.923 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:17.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:17.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:18.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:18.062 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:18.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:18.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:18.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:18.221 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:18.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:18.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:18.323 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:18.374 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:18.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:18.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:18.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:18.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/145524 objects degraded (4.089%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:18.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/145524 objects degraded (4.089%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:18.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:18.521 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:18.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:18.622 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:18.649 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:18.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:18.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/145524 objects degraded (4.089%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:18.751 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:18.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:18.779 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:18.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:18.880 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:18.921 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:18.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:19.023 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:19.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:19.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:19.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:19.153 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:19.182 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:19.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:19.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:19.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:19.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:19.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:19.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:19.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:19.471 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:19.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:19.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:19.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:19 smithi029 ceph-mon[97787]: pgmap v1602: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 5951/143169 objects degraded (4.157%); 1959/143169 objects misplaced (1.368%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:40:19.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:19 smithi029 ceph-mon[93650]: pgmap v1602: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 5951/143169 objects degraded (4.157%); 1959/143169 objects misplaced (1.368%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:40:19.573 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:19.604 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:19.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:19.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:19.705 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:19.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:19 smithi087 ceph-mon[89014]: pgmap v1602: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 5951/143169 objects degraded (4.157%); 1959/143169 objects misplaced (1.368%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:40:19.732 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:19.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:19.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:19.833 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:19.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:19.860 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:19.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:19.961 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:20.007 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:20.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:20.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:20.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:20.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:20.167 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:20.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:20.211 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:40:20.268 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:20.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:20.325 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:20.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:20.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:40:20.426 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:20.556 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:20.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:20.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:20.597 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:40:20.597 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:40:20.597 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:40:20.597 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:40:20.634 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:40:20.662 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:20.678 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:40:20.690 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:20.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:20.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:20.716 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:40:20.770 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:40:20.783 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:40:20.783 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:40:20.783 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:40:20.783 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:40:20.783 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:40:20.783 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:40:20.783 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:40:20.784 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:40:20.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:20.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:20.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:20.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:20 smithi029 ceph-mon[93650]: pgmap v1603: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 5951/140106 objects degraded (4.247%); 1959/140106 objects misplaced (1.398%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:40:20.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:20 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3961279501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:20.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:20.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:20.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:20 smithi029 ceph-mon[97787]: pgmap v1603: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 5951/140106 objects degraded (4.247%); 1959/140106 objects misplaced (1.398%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:40:20.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:20 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3961279501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:20.818 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:20.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:20.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:20.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:20 smithi087 ceph-mon[89014]: pgmap v1603: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 5951/140106 objects degraded (4.247%); 1959/140106 objects misplaced (1.398%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:40:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:20 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3961279501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:20.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:20.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:20.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:21.070 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:21.111 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:21.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:21.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:21.213 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:21.245 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:21.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:21.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:21.347 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:21.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:21.375 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:21.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:21.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:21.519 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:21.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:21.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:21.621 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:21.664 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:21.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:21.684 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:21.766 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:21.793 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:21.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:21.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:21.895 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:21.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:21.927 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:21.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:21.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:40:21.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:40:21.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:21.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:21.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:21.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:40:21.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:40:21.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:21.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:21.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:22.028 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:22.055 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:22.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:22.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:22.157 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:22.187 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:22.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:22.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:40:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:40:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:22.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:22.290 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:22.314 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:22.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:22.415 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:22.443 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:22.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:22.513 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:22.544 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:22.570 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:22.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:22.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:22.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:22.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:40:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:40:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:40:22.725 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:22.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:22.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:22.747 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:22 smithi029 ceph-mon[93650]: pgmap v1604: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 5951/135969 objects degraded (4.377%); 1759/135969 objects misplaced (1.294%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:22.827 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:22.857 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:22.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:22.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:22.958 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:22.986 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:22.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:22.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:23.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:22 smithi029 ceph-mon[97787]: pgmap v1604: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 5951/135969 objects degraded (4.377%); 1759/135969 objects misplaced (1.294%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:23.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:23.141 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:23.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:23.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:23.173 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:22 smithi087 ceph-mon[89014]: pgmap v1604: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 5951/135969 objects degraded (4.377%); 1759/135969 objects misplaced (1.294%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:23.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:23.294 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:23.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:23.337 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:40:23.337 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:40:23.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:23.422 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:23.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:23.523 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:23.560 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:23.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:23.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:23.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:23.709 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:23.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:23.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:23.810 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:23.843 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:23.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:23.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:23.945 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:23.990 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:23.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:24.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/135969 objects degraded (4.377%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:24.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/135969 objects degraded (4.377%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:24.091 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:24.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:24.134 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:24.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:24.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/135969 objects degraded (4.377%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:24.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:24.268 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:24.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:24.370 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:24.399 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:24.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:24.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:24.501 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:24.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:40:24.528 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:24.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:24.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:24.629 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:24.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-04-12T11:40:24.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:24.672 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:24.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:24.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:24.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:24.809 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:24.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:24.824 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:24 smithi029 ceph-mon[93650]: pgmap v1605: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 5951/130593 objects degraded (4.557%); 1759/130593 objects misplaced (1.347%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:24.824 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:24 smithi029 ceph-mon[97787]: pgmap v1605: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 5951/130593 objects degraded (4.557%); 1759/130593 objects misplaced (1.347%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:24.910 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:24.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:24.966 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:24.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:25.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:25.104 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:40:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:40:24] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-04-12T11:40:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:25.121 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:25.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:25.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:24 smithi087 ceph-mon[89014]: pgmap v1605: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 5951/130593 objects degraded (4.557%); 1759/130593 objects misplaced (1.347%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:25.223 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:25.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:25.253 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:25.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:25.354 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:25.397 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:25.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:25.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:25.499 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:25.529 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:25.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:25.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:25.630 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:25.655 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:25.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:25.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:25.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:25.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:25.811 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:25.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:25.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:25.951 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:25.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:25.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:26.052 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:26.079 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:26.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:26.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:26.181 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:26.217 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:26.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:26.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:26.319 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:26.361 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:26.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:26.462 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:26.488 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:26.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:26.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:26.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:26.614 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:26.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:26.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:26.716 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:26.745 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:26.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:26.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:26.847 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:26.874 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:26.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:26.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:26.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:27.013 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:27.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:27.077 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:26 smithi029 ceph-mon[97787]: pgmap v1606: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5951/126249 objects degraded (4.714%); 1567/126249 objects misplaced (1.241%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T11:40:27.078 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:26 smithi029 ceph-mon[93650]: pgmap v1606: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5951/126249 objects degraded (4.714%); 1567/126249 objects misplaced (1.241%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T11:40:27.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:27.167 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:27.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:27.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:26 smithi087 ceph-mon[89014]: pgmap v1606: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5951/126249 objects degraded (4.714%); 1567/126249 objects misplaced (1.241%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T11:40:27.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:27.286 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:27.292 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:27.324 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:27.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:27.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:27.470 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:27.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:27.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:27.572 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:27.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:27.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:27.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:27.724 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:27.748 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:27.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:27.760 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:40:27.761 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:40:27.850 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:27.882 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:27.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:27.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:27.984 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:28.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:28.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:28.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:28.112 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:28.140 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:28.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:28.241 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:28.274 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:28.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:28.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:28.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:28.399 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:28.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:28.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:28.501 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:28.528 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:28.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:28.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:28.630 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:28.661 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:28.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:28.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:28.763 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:28.786 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:28.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:28.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:28.888 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:28.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:28.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:28.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:29.014 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:29.040 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:29.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:29.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:29.142 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:29.170 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:29.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:29.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:29.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:28 smithi087 ceph-mon[89014]: from='client.35067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:40:29.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:28 smithi087 ceph-mon[89014]: pgmap v1607: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5951/123846 objects degraded (4.805%); 1567/123846 objects misplaced (1.265%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:40:29.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/123846 objects degraded (4.805%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:29.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:28 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3705007855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:40:29.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:28 smithi029 ceph-mon[97787]: from='client.35067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:40:29.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:28 smithi029 ceph-mon[97787]: pgmap v1607: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5951/123846 objects degraded (4.805%); 1567/123846 objects misplaced (1.265%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:40:29.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/123846 objects degraded (4.805%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:29.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:28 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3705007855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:40:29.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:28 smithi029 ceph-mon[93650]: from='client.35067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:40:29.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:28 smithi029 ceph-mon[93650]: pgmap v1607: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5951/123846 objects degraded (4.805%); 1567/123846 objects misplaced (1.265%); 2.1 MiB/s, 33 objects/s recovering 2024-04-12T11:40:29.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/123846 objects degraded (4.805%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:29.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:28 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3705007855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:40:29.272 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:29.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:29.317 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:29.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:29.418 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:29.441 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:40:29.445 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:29.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:29.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:29.546 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:29.569 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:29.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:29.671 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:29.694 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:29.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:29.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:29.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:29.832 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:29.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:29.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:29.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:29.979 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:29.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:29.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:30.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:30.118 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:30.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:30.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:29 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:40:30.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:29 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3705007855' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:40:30.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:29 smithi087 ceph-mon[89014]: osdmap e1160: 8 total, 8 up, 8 in 2024-04-12T11:40:30.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:30.264 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:30.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:30.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:30.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:29 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:40:30.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:29 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3705007855' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:40:30.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:29 smithi029 ceph-mon[93650]: osdmap e1160: 8 total, 8 up, 8 in 2024-04-12T11:40:30.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:29 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:40:30.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:29 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3705007855' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:40:30.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:29 smithi029 ceph-mon[97787]: osdmap e1160: 8 total, 8 up, 8 in 2024-04-12T11:40:30.366 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:30.394 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:30.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:30.495 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:30.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:30.526 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:30.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:30.627 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:30.657 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:30.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:30.759 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:30.803 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:30.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:30.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:30.905 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:30.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:30 smithi029 ceph-mon[93650]: pgmap v1609: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 5951/120891 objects degraded (4.923%); 1567/120891 objects misplaced (1.296%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T11:40:30.951 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:30.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:30.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:31.053 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:31.082 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:31.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:31.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:31.184 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:31.212 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:31.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:31.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:30 smithi087 ceph-mon[89014]: pgmap v1609: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 5951/120891 objects degraded (4.923%); 1567/120891 objects misplaced (1.296%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T11:40:31.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:30 smithi029 ceph-mon[97787]: pgmap v1609: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 5951/120891 objects degraded (4.923%); 1567/120891 objects misplaced (1.296%); 2.4 MiB/s, 39 objects/s recovering 2024-04-12T11:40:31.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:31.343 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:31.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:31.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:31.444 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:31.487 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:31.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:31.589 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:31.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:31.628 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:31.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:31.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:31.764 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:31.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:31.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:31.866 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:31.889 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:31.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:31.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:31.991 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:32.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:32.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:32.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:32.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:32.161 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:32.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:32.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:32.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:32.290 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:32.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:32.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:32.392 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:32.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:32.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:32.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:32.520 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:32.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:32.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:32.648 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:32.696 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:32.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:32.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:40:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:40:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:40:32.798 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:32.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:32.822 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:32.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:32.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:32.951 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:32.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:32.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:33.053 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:33.081 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:33.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:33.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:33.183 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:33.206 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:33.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:33.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:32 smithi087 ceph-mon[89014]: pgmap v1610: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 5951/117240 objects degraded (5.076%); 1363/117240 objects misplaced (1.163%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:33.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:32 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/117240 objects degraded (5.076%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:32 smithi029 ceph-mon[97787]: pgmap v1610: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 5951/117240 objects degraded (5.076%); 1363/117240 objects misplaced (1.163%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:32 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/117240 objects degraded (5.076%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:33.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:32 smithi029 ceph-mon[93650]: pgmap v1610: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 5951/117240 objects degraded (5.076%); 1363/117240 objects misplaced (1.163%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:33.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:32 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/117240 objects degraded (5.076%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:33.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:33.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:33.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:33.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:33.437 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:33.470 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:33.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:33.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:33.572 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:33.595 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:33.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:33.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:33.697 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:33.725 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:33.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:33.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:33.827 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:33.857 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:33.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:33.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:33.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:34.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:34.004 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:34.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:34.104 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:34.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:34.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:34.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:34.285 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:34.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:34.386 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:34.414 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:34.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:34.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:34.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:34.549 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:34.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:34.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:34.651 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:34.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:34.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:34.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:34.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:34.817 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:34.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:34.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:34.962 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:34.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:35.064 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:35.115 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:35.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:35.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:35.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:34 smithi087 ceph-mon[89014]: pgmap v1611: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5951/112401 objects degraded (5.294%); 1363/112401 objects misplaced (1.213%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:35.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:35.261 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:35.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:35.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:35.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:34 smithi029 ceph-mon[93650]: pgmap v1611: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5951/112401 objects degraded (5.294%); 1363/112401 objects misplaced (1.213%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:35.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:34 smithi029 ceph-mon[97787]: pgmap v1611: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5951/112401 objects degraded (5.294%); 1363/112401 objects misplaced (1.213%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:40:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:40:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:40:34] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-04-12T11:40:35.363 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:35.389 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:35.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:35.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:35.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:35.524 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:35.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:35.626 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:35.667 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:35.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:35.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:35.768 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:35.808 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:35.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:35.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:35.910 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:35.950 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:35.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:35.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:36.051 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:36.074 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:36.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:36.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:36.176 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:36.199 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:36.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:36.301 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:36.328 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:36.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:36.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:36.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:36.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:36.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:36.581 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:36.611 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:36.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:36.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:40:36.712 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:36.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:36.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:36.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:36.864 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:36.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:36.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:36.966 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:36.995 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:36.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:36.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:37.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:37.124 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:37.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:37.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:36 smithi087 ceph-mon[89014]: pgmap v1612: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5951/108042 objects degraded (5.508%); 1156/108042 objects misplaced (1.070%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T11:40:37.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:37.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:37.226 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:37.249 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:37.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:37.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:36 smithi029 ceph-mon[93650]: pgmap v1612: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5951/108042 objects degraded (5.508%); 1156/108042 objects misplaced (1.070%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T11:40:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:36 smithi029 ceph-mon[97787]: pgmap v1612: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 5951/108042 objects degraded (5.508%); 1156/108042 objects misplaced (1.070%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T11:40:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:40:37.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:37.351 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:37.377 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:37.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:37.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:37.479 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:37.505 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:37.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:37.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:37.606 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:37.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:37.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:37.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:37.735 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:37.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:37.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:37.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:37.873 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:37.915 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:37.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:37.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:38.017 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:38.050 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:38.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:38.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:38.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:38.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:38.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:38.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:38.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:37 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/108042 objects degraded (5.508%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:38.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:37 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/108042 objects degraded (5.508%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:38.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:37 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/108042 objects degraded (5.508%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:38.301 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:38.328 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:38.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:38.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:38.472 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:38.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:38.573 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:38.602 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:38.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:38.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:38.660 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1606,"stamp":"2024-04-12T11:40:25.800238+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2756909034,"num_objects":42083,"num_object_clones":0,"num_object_copies":126249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5951,"num_objects_misplaced":1567,"num_objects_unfound":0,"num_objects_dirty":42083,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":173243,"num_write_kb":6882447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3848,"num_bytes_recovered":253299436,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139259,"ondisk_log_size":139259,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11134904,"kb_used_data":8767544,"kb_used_omap":175,"kb_used_meta":2367120,"kb_avail":738826312,"statfs":{"total":767960285184,"available":756558143488,"internally_reserved":0,"allocated":8977965056,"data_stored":8953494594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":2423931049},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-421068800,"num_objects":-6425,"num_object_clones":0,"num_object_copies":-19275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-392,"num_objects_unfound":0,"num_objects_dirty":-6425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6425,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":392,"num_bytes_recovered":25690112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003035"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1670,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.208307+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:40:16.208307+0000","last_peered":"2024-04-12T11:40:16.208307+0000","last_clean":"2024-04-12T11:40:16.208307+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:40:16.208307+0000","last_undegraded":"2024-04-12T11:40:16.208307+0000","last_fullsized":"2024-04-12T11:40:16.208307+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1678,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.236562+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:40:16.236562+0000","last_peered":"2024-04-12T11:40:16.236562+0000","last_clean":"2024-04-12T11:40:16.236562+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:40:16.236562+0000","last_undegraded":"2024-04-12T11:40:16.236562+0000","last_fullsized":"2024-04-12T11:40:16.236562+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1730,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.214123+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:40:16.214123+0000","last_peered":"2024-04-12T11:40:16.214123+0000","last_clean":"2024-04-12T11:40:16.214123+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:40:16.214123+0000","last_undegraded":"2024-04-12T11:40:16.214123+0000","last_fullsized":"2024-04-12T11:40:16.214123+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1738,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.220436+0000","last_change":"2024-04-12T11:38:21.933647+0000","last_active":"2024-04-12T11:40:16.220436+0000","last_peered":"2024-04-12T11:40:16.220436+0000","last_clean":"2024-04-12T11:40:16.220436+0000","last_became_active":"2024-04-12T11:38:21.933295+0000","last_became_peered":"2024-04-12T11:38:21.933295+0000","last_unstale":"2024-04-12T11:40:16.220436+0000","last_undegraded":"2024-04-12T11:40:16.220436+0000","last_fullsized":"2024-04-12T11:40:16.220436+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:37.402139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1687,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.238183+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:40:16.238183+0000","last_peered":"2024-04-12T11:40:16.238183+0000","last_clean":"2024-04-12T11:40:16.238183+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:40:16.238183+0000","last_undegraded":"2024-04-12T11:40:16.238183+0000","last_fullsized":"2024-04-12T11:40:16.238183+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1104,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:02.406561+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:40:02.406561+0000","last_peered":"2024-04-12T11:40:02.406561+0000","last_clean":"2024-04-12T11:40:02.406561+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:40:02.406561+0000","last_undegraded":"2024-04-12T11:40:02.406561+0000","last_fullsized":"2024-04-12T11:40:02.406561+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1721,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.225526+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:40:16.225526+0000","last_peered":"2024-04-12T11:40:16.225526+0000","last_clean":"2024-04-12T11:40:16.225526+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:40:16.225526+0000","last_undegraded":"2024-04-12T11:40:16.225526+0000","last_fullsized":"2024-04-12T11:40:16.225526+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1660,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.238274+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:40:16.238274+0000","last_peered":"2024-04-12T11:40:16.238274+0000","last_clean":"2024-04-12T11:40:16.238274+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:40:16.238274+0000","last_undegraded":"2024-04-12T11:40:16.238274+0000","last_fullsized":"2024-04-12T11:40:16.238274+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1109,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.131964+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:39:51.131964+0000","last_peered":"2024-04-12T11:39:51.131964+0000","last_clean":"2024-04-12T11:39:51.131964+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:39:51.131964+0000","last_undegraded":"2024-04-12T11:39:51.131964+0000","last_fullsized":"2024-04-12T11:39:51.131964+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1640,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.237340+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:40:16.237340+0000","last_peered":"2024-04-12T11:40:16.237340+0000","last_clean":"2024-04-12T11:40:16.237340+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:40:16.237340+0000","last_undegraded":"2024-04-12T11:40:16.237340+0000","last_fullsized":"2024-04-12T11:40:16.237340+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1118,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107969+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:39:51.107969+0000","last_peered":"2024-04-12T11:39:51.107969+0000","last_clean":"2024-04-12T11:39:51.107969+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:39:51.107969+0000","last_undegraded":"2024-04-12T11:39:51.107969+0000","last_fullsized":"2024-04-12T11:39:51.107969+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1721,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.214506+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:40:16.214506+0000","last_peered":"2024-04-12T11:40:16.214506+0000","last_clean":"2024-04-12T11:40:16.214506+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:40:16.214506+0000","last_undegraded":"2024-04-12T11:40:16.214506+0000","last_fullsized":"2024-04-12T11:40:16.214506+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1643,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.208259+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:40:16.208259+0000","last_peered":"2024-04-12T11:40:16.208259+0000","last_clean":"2024-04-12T11:40:16.208259+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:40:16.208259+0000","last_undegraded":"2024-04-12T11:40:16.208259+0000","last_fullsized":"2024-04-12T11:40:16.208259+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1634,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.208200+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:40:16.208200+0000","last_peered":"2024-04-12T11:40:16.208200+0000","last_clean":"2024-04-12T11:40:16.208200+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:40:16.208200+0000","last_undegraded":"2024-04-12T11:40:16.208200+0000","last_fullsized":"2024-04-12T11:40:16.208200+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1117,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.103107+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:39:51.103107+0000","last_peered":"2024-04-12T11:39:51.103107+0000","last_clean":"2024-04-12T11:39:51.103107+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:39:51.103107+0000","last_undegraded":"2024-04-12T11:39:51.103107+0000","last_fullsized":"2024-04-12T11:39:51.103107+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1671,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.246421+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:40:16.246421+0000","last_peered":"2024-04-12T11:40:16.246421+0000","last_clean":"2024-04-12T11:40:16.246421+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:40:16.246421+0000","last_undegraded":"2024-04-12T11:40:16.246421+0000","last_fullsized":"2024-04-12T11:40:16.246421+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1722,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.213758+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:40:16.213758+0000","last_peered":"2024-04-12T11:40:16.213758+0000","last_clean":"2024-04-12T11:40:16.213758+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:40:16.213758+0000","last_undegraded":"2024-04-12T11:40:16.213758+0000","last_fullsized":"2024-04-12T11:40:16.213758+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1108,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.131792+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:39:51.131792+0000","last_peered":"2024-04-12T11:39:51.131792+0000","last_clean":"2024-04-12T11:39:51.131792+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:39:51.131792+0000","last_undegraded":"2024-04-12T11:39:51.131792+0000","last_fullsized":"2024-04-12T11:39:51.131792+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1100,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.103037+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:39:51.103037+0000","last_peered":"2024-04-12T11:39:51.103037+0000","last_clean":"2024-04-12T11:39:51.103037+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:39:51.103037+0000","last_undegraded":"2024-04-12T11:39:51.103037+0000","last_fullsized":"2024-04-12T11:39:51.103037+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1723,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.220353+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:40:16.220353+0000","last_peered":"2024-04-12T11:40:16.220353+0000","last_clean":"2024-04-12T11:40:16.220353+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:40:16.220353+0000","last_undegraded":"2024-04-12T11:40:16.220353+0000","last_fullsized":"2024-04-12T11:40:16.220353+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1111,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.110047+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:39:51.110047+0000","last_peered":"2024-04-12T11:39:51.110047+0000","last_clean":"2024-04-12T11:39:51.110047+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:39:51.110047+0000","last_undegraded":"2024-04-12T11:39:51.110047+0000","last_fullsized":"2024-04-12T11:39:51.110047+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1671,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.254255+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:40:16.254255+0000","last_peered":"2024-04-12T11:40:16.254255+0000","last_clean":"2024-04-12T11:40:16.254255+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:40:16.254255+0000","last_undegraded":"2024-04-12T11:40:16.254255+0000","last_fullsized":"2024-04-12T11:40:16.254255+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1159'10895","reported_seq":10958,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:23.330408+0000","last_change":"2024-04-12T11:37:03.057199+0000","last_active":"2024-04-12T11:40:23.330408+0000","last_peered":"2024-04-12T11:40:23.330408+0000","last_clean":"2024-04-12T11:40:23.330408+0000","last_became_active":"2024-04-12T11:37:03.056776+0000","last_became_peered":"2024-04-12T11:37:03.056776+0000","last_unstale":"2024-04-12T11:40:23.330408+0000","last_undegraded":"2024-04-12T11:40:23.330408+0000","last_fullsized":"2024-04-12T11:40:23.330408+0000","mapping_epoch":1133,"log_start":"1140'3000","ondisk_log_start":"1140'3000","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7895,"log_dups_size":0,"ondisk_log_size":7895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:16:29.272872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173342720,"num_objects":2645,"num_object_clones":0,"num_object_copies":7935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10895,"num_write_kb":433280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1117,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:02.455056+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:40:02.455056+0000","last_peered":"2024-04-12T11:40:02.455056+0000","last_clean":"2024-04-12T11:40:02.455056+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:40:02.455056+0000","last_undegraded":"2024-04-12T11:40:02.455056+0000","last_fullsized":"2024-04-12T11:40:02.455056+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1094,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.131681+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:39:51.131681+0000","last_peered":"2024-04-12T11:39:51.131681+0000","last_clean":"2024-04-12T11:39:51.131681+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:39:51.131681+0000","last_undegraded":"2024-04-12T11:39:51.131681+0000","last_fullsized":"2024-04-12T11:39:51.131681+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1159'10904","reported_seq":10967,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:25.116731+0000","last_change":"2024-04-12T11:37:03.059071+0000","last_active":"2024-04-12T11:40:25.116731+0000","last_peered":"2024-04-12T11:40:25.116731+0000","last_clean":"2024-04-12T11:40:25.116731+0000","last_became_active":"2024-04-12T11:37:03.058769+0000","last_became_peered":"2024-04-12T11:37:03.058769+0000","last_unstale":"2024-04-12T11:40:25.116731+0000","last_undegraded":"2024-04-12T11:40:25.116731+0000","last_fullsized":"2024-04-12T11:40:25.116731+0000","mapping_epoch":1133,"log_start":"1141'3400","ondisk_log_start":"1141'3400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7504,"log_dups_size":0,"ondisk_log_size":7504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:43:26.795096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174063616,"num_objects":2656,"num_object_clones":0,"num_object_copies":7968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10904,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1118,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:02.455036+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:40:02.455036+0000","last_peered":"2024-04-12T11:40:02.455036+0000","last_clean":"2024-04-12T11:40:02.455036+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:40:02.455036+0000","last_undegraded":"2024-04-12T11:40:02.455036+0000","last_fullsized":"2024-04-12T11:40:02.455036+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1656,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.210444+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:40:16.210444+0000","last_peered":"2024-04-12T11:40:16.210444+0000","last_clean":"2024-04-12T11:40:16.210444+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:40:16.210444+0000","last_undegraded":"2024-04-12T11:40:16.210444+0000","last_fullsized":"2024-04-12T11:40:16.210444+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1159'10835","reported_seq":10898,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:23.574157+0000","last_change":"2024-04-12T11:37:03.059390+0000","last_active":"2024-04-12T11:40:23.574157+0000","last_peered":"2024-04-12T11:40:23.574157+0000","last_clean":"2024-04-12T11:40:23.574157+0000","last_became_active":"2024-04-12T11:37:03.058045+0000","last_became_peered":"2024-04-12T11:37:03.058045+0000","last_unstale":"2024-04-12T11:40:23.574157+0000","last_undegraded":"2024-04-12T11:40:23.574157+0000","last_fullsized":"2024-04-12T11:40:23.574157+0000","mapping_epoch":1133,"log_start":"1140'2000","ondisk_log_start":"1140'2000","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8835,"log_dups_size":0,"ondisk_log_size":8835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:14:01.284893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172425216,"num_objects":2631,"num_object_clones":0,"num_object_copies":7893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10835,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1087,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:57.452213+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:39:57.452213+0000","last_peered":"2024-04-12T11:39:57.452213+0000","last_clean":"2024-04-12T11:39:57.452213+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:39:57.452213+0000","last_undegraded":"2024-04-12T11:39:57.452213+0000","last_fullsized":"2024-04-12T11:39:57.452213+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1103,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.110182+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:39:51.110182+0000","last_peered":"2024-04-12T11:39:51.110182+0000","last_clean":"2024-04-12T11:39:51.110182+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:39:51.110182+0000","last_undegraded":"2024-04-12T11:39:51.110182+0000","last_fullsized":"2024-04-12T11:39:51.110182+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1159'10977","reported_seq":11040,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:24.941968+0000","last_change":"2024-04-12T11:37:03.054920+0000","last_active":"2024-04-12T11:40:24.941968+0000","last_peered":"2024-04-12T11:40:24.941968+0000","last_clean":"2024-04-12T11:40:24.941968+0000","last_became_active":"2024-04-12T11:37:03.054603+0000","last_became_peered":"2024-04-12T11:37:03.054603+0000","last_unstale":"2024-04-12T11:40:24.941968+0000","last_undegraded":"2024-04-12T11:40:24.941968+0000","last_fullsized":"2024-04-12T11:40:24.941968+0000","mapping_epoch":1133,"log_start":"1138'900","ondisk_log_start":"1138'900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10077,"log_dups_size":0,"ondisk_log_size":10077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:34:45.149813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172425216,"num_objects":2631,"num_object_clones":0,"num_object_copies":7893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10977,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1110,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:57.332425+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:39:57.332425+0000","last_peered":"2024-04-12T11:39:57.332425+0000","last_clean":"2024-04-12T11:39:57.332425+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:39:57.332425+0000","last_undegraded":"2024-04-12T11:39:57.332425+0000","last_fullsized":"2024-04-12T11:39:57.332425+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1733,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.214007+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:40:16.214007+0000","last_peered":"2024-04-12T11:40:16.214007+0000","last_clean":"2024-04-12T11:40:16.214007+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:40:16.214007+0000","last_undegraded":"2024-04-12T11:40:16.214007+0000","last_fullsized":"2024-04-12T11:40:16.214007+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1159'10935","reported_seq":10994,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:23.600317+0000","last_change":"2024-04-12T11:37:03.063209+0000","last_active":"2024-04-12T11:40:23.600317+0000","last_peered":"2024-04-12T11:40:23.600317+0000","last_clean":"2024-04-12T11:40:23.600317+0000","last_became_active":"2024-04-12T11:37:03.062848+0000","last_became_peered":"2024-04-12T11:37:03.062848+0000","last_unstale":"2024-04-12T11:40:23.600317+0000","last_undegraded":"2024-04-12T11:40:23.600317+0000","last_fullsized":"2024-04-12T11:40:23.600317+0000","mapping_epoch":1133,"log_start":"1140'2100","ondisk_log_start":"1140'2100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8835,"log_dups_size":0,"ondisk_log_size":8835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:55:26.299047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175439872,"num_objects":2677,"num_object_clones":0,"num_object_copies":8031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10935,"num_write_kb":435584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1664,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.237493+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:40:16.237493+0000","last_peered":"2024-04-12T11:40:16.237493+0000","last_clean":"2024-04-12T11:40:16.237493+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:40:16.237493+0000","last_undegraded":"2024-04-12T11:40:16.237493+0000","last_fullsized":"2024-04-12T11:40:16.237493+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1647,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.208354+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:40:16.208354+0000","last_peered":"2024-04-12T11:40:16.208354+0000","last_clean":"2024-04-12T11:40:16.208354+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:40:16.208354+0000","last_undegraded":"2024-04-12T11:40:16.208354+0000","last_fullsized":"2024-04-12T11:40:16.208354+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1159'10765","reported_seq":10828,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:24.891799+0000","last_change":"2024-04-12T11:37:03.060739+0000","last_active":"2024-04-12T11:40:24.891799+0000","last_peered":"2024-04-12T11:40:24.891799+0000","last_clean":"2024-04-12T11:40:24.891799+0000","last_became_active":"2024-04-12T11:37:03.060414+0000","last_became_peered":"2024-04-12T11:37:03.060414+0000","last_unstale":"2024-04-12T11:40:24.891799+0000","last_undegraded":"2024-04-12T11:40:24.891799+0000","last_fullsized":"2024-04-12T11:40:24.891799+0000","mapping_epoch":1133,"log_start":"1138'700","ondisk_log_start":"1138'700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10065,"log_dups_size":0,"ondisk_log_size":10065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:12:30.259488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167313408,"num_objects":2553,"num_object_clones":0,"num_object_copies":7659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10765,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":673,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.208191+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:40:16.208191+0000","last_peered":"2024-04-12T11:40:16.208191+0000","last_clean":"2024-04-12T11:40:16.208191+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:40:16.208191+0000","last_undegraded":"2024-04-12T11:40:16.208191+0000","last_fullsized":"2024-04-12T11:40:16.208191+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1669,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.208393+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:40:16.208393+0000","last_peered":"2024-04-12T11:40:16.208393+0000","last_clean":"2024-04-12T11:40:16.208393+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:40:16.208393+0000","last_undegraded":"2024-04-12T11:40:16.208393+0000","last_fullsized":"2024-04-12T11:40:16.208393+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1159'10926","reported_seq":10963,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:22.410170+0000","last_change":"2024-04-12T11:37:03.060364+0000","last_active":"2024-04-12T11:40:22.410170+0000","last_peered":"2024-04-12T11:40:22.410170+0000","last_clean":"2024-04-12T11:40:22.410170+0000","last_became_active":"2024-04-12T11:37:03.060146+0000","last_became_peered":"2024-04-12T11:37:03.060146+0000","last_unstale":"2024-04-12T11:40:22.410170+0000","last_undegraded":"2024-04-12T11:40:22.410170+0000","last_fullsized":"2024-04-12T11:40:22.410170+0000","mapping_epoch":1133,"log_start":"1140'1500","ondisk_log_start":"1140'1500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9426,"log_dups_size":0,"ondisk_log_size":9426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:03.796242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175833112,"num_objects":2684,"num_object_clones":0,"num_object_copies":8052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2684,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10926,"num_write_kb":435457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1655,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.210526+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:40:16.210526+0000","last_peered":"2024-04-12T11:40:16.210526+0000","last_clean":"2024-04-12T11:40:16.210526+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:40:16.210526+0000","last_undegraded":"2024-04-12T11:40:16.210526+0000","last_fullsized":"2024-04-12T11:40:16.210526+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1634,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.208838+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:40:16.208838+0000","last_peered":"2024-04-12T11:40:16.208838+0000","last_clean":"2024-04-12T11:40:16.208838+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:40:16.208838+0000","last_undegraded":"2024-04-12T11:40:16.208838+0000","last_fullsized":"2024-04-12T11:40:16.208838+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1159'11064","reported_seq":11127,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:25.089664+0000","last_change":"2024-04-12T11:37:03.056864+0000","last_active":"2024-04-12T11:40:25.089664+0000","last_peered":"2024-04-12T11:40:25.089664+0000","last_clean":"2024-04-12T11:40:25.089664+0000","last_became_active":"2024-04-12T11:37:03.056523+0000","last_became_peered":"2024-04-12T11:37:03.056523+0000","last_unstale":"2024-04-12T11:40:25.089664+0000","last_undegraded":"2024-04-12T11:40:25.089664+0000","last_fullsized":"2024-04-12T11:40:25.089664+0000","mapping_epoch":1133,"log_start":"1141'3500","ondisk_log_start":"1141'3500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7564,"log_dups_size":0,"ondisk_log_size":7564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:57:58.750385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167903232,"num_objects":2562,"num_object_clones":0,"num_object_copies":7686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11064,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1672,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.238351+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:40:16.238351+0000","last_peered":"2024-04-12T11:40:16.238351+0000","last_clean":"2024-04-12T11:40:16.238351+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:40:16.238351+0000","last_undegraded":"2024-04-12T11:40:16.238351+0000","last_fullsized":"2024-04-12T11:40:16.238351+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1731,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.214132+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:40:16.214132+0000","last_peered":"2024-04-12T11:40:16.214132+0000","last_clean":"2024-04-12T11:40:16.214132+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:40:16.214132+0000","last_undegraded":"2024-04-12T11:40:16.214132+0000","last_fullsized":"2024-04-12T11:40:16.214132+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1159'10754","reported_seq":18496,"reported_epoch":1159,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-04-12T11:40:24.874693+0000","last_change":"2024-04-12T11:39:24.599690+0000","last_active":"2024-04-12T11:40:24.874693+0000","last_peered":"2024-04-12T11:40:24.874693+0000","last_clean":"2024-04-12T11:38:20.888121+0000","last_became_active":"2024-04-12T11:38:23.417421+0000","last_became_peered":"2024-04-12T11:38:23.417421+0000","last_unstale":"2024-04-12T11:40:24.874693+0000","last_undegraded":"2024-04-12T11:39:24.597597+0000","last_fullsized":"2024-04-12T11:38:23.247711+0000","mapping_epoch":1147,"log_start":"1140'2800","ondisk_log_start":"1140'2800","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7954,"log_dups_size":0,"ondisk_log_size":7954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163446784,"num_objects":2494,"num_object_clones":0,"num_object_copies":7482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":310,"num_objects_misplaced":1567,"num_objects_unfound":0,"num_objects_dirty":2494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10754,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3838,"num_bytes_recovered":251527168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":1567},{"shards":"4,6","objects":617}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1732,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.219448+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:40:16.219448+0000","last_peered":"2024-04-12T11:40:16.219448+0000","last_clean":"2024-04-12T11:40:16.219448+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:40:16.219448+0000","last_undegraded":"2024-04-12T11:40:16.219448+0000","last_fullsized":"2024-04-12T11:40:16.219448+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1126,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.215716+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:40:16.215716+0000","last_peered":"2024-04-12T11:40:16.215716+0000","last_clean":"2024-04-12T11:40:16.215716+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:40:16.215716+0000","last_undegraded":"2024-04-12T11:40:16.215716+0000","last_fullsized":"2024-04-12T11:40:16.215716+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1723,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.214387+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:40:16.214387+0000","last_peered":"2024-04-12T11:40:16.214387+0000","last_clean":"2024-04-12T11:40:16.214387+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:40:16.214387+0000","last_undegraded":"2024-04-12T11:40:16.214387+0000","last_fullsized":"2024-04-12T11:40:16.214387+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1159'10711","reported_seq":10747,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:22.367125+0000","last_change":"2024-04-12T11:37:03.060583+0000","last_active":"2024-04-12T11:40:22.367125+0000","last_peered":"2024-04-12T11:40:22.367125+0000","last_clean":"2024-04-12T11:40:22.367125+0000","last_became_active":"2024-04-12T11:37:03.060125+0000","last_became_peered":"2024-04-12T11:37:03.060125+0000","last_unstale":"2024-04-12T11:40:22.367125+0000","last_undegraded":"2024-04-12T11:40:22.367125+0000","last_fullsized":"2024-04-12T11:40:22.367125+0000","mapping_epoch":1133,"log_start":"1140'1300","ondisk_log_start":"1140'1300","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9411,"log_dups_size":0,"ondisk_log_size":9411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:35:30.859950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180551680,"num_objects":2755,"num_object_clones":0,"num_object_copies":8265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10711,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1115,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.131845+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:39:51.131845+0000","last_peered":"2024-04-12T11:39:51.131845+0000","last_clean":"2024-04-12T11:39:51.131845+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:39:51.131845+0000","last_undegraded":"2024-04-12T11:39:51.131845+0000","last_fullsized":"2024-04-12T11:39:51.131845+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1656,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.208675+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:40:16.208675+0000","last_peered":"2024-04-12T11:40:16.208675+0000","last_clean":"2024-04-12T11:40:16.208675+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:40:16.208675+0000","last_undegraded":"2024-04-12T11:40:16.208675+0000","last_fullsized":"2024-04-12T11:40:16.208675+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1732,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.224131+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:40:16.224131+0000","last_peered":"2024-04-12T11:40:16.224131+0000","last_clean":"2024-04-12T11:40:16.224131+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:40:16.224131+0000","last_undegraded":"2024-04-12T11:40:16.224131+0000","last_fullsized":"2024-04-12T11:40:16.224131+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1159'10654","reported_seq":10717,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:23.604175+0000","last_change":"2024-04-12T11:37:03.058470+0000","last_active":"2024-04-12T11:40:23.604175+0000","last_peered":"2024-04-12T11:40:23.604175+0000","last_clean":"2024-04-12T11:40:23.604175+0000","last_became_active":"2024-04-12T11:37:03.058149+0000","last_became_peered":"2024-04-12T11:37:03.058149+0000","last_unstale":"2024-04-12T11:40:23.604175+0000","last_undegraded":"2024-04-12T11:40:23.604175+0000","last_fullsized":"2024-04-12T11:40:23.604175+0000","mapping_epoch":1133,"log_start":"1140'1800","ondisk_log_start":"1140'1800","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8854,"log_dups_size":0,"ondisk_log_size":8854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:25:06.934877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169738240,"num_objects":2590,"num_object_clones":0,"num_object_copies":7770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10654,"num_write_kb":423808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1696,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.238675+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:40:16.238675+0000","last_peered":"2024-04-12T11:40:16.238675+0000","last_clean":"2024-04-12T11:40:16.238675+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:40:16.238675+0000","last_undegraded":"2024-04-12T11:40:16.238675+0000","last_fullsized":"2024-04-12T11:40:16.238675+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1724,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.213910+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:40:16.213910+0000","last_peered":"2024-04-12T11:40:16.213910+0000","last_clean":"2024-04-12T11:40:16.213910+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:40:16.213910+0000","last_undegraded":"2024-04-12T11:40:16.213910+0000","last_fullsized":"2024-04-12T11:40:16.213910+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1726,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.214191+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:40:16.214191+0000","last_peered":"2024-04-12T11:40:16.214191+0000","last_clean":"2024-04-12T11:40:16.214191+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:40:16.214191+0000","last_undegraded":"2024-04-12T11:40:16.214191+0000","last_fullsized":"2024-04-12T11:40:16.214191+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1159'10662","reported_seq":10728,"reported_epoch":1159,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:40:23.579402+0000","last_change":"2024-04-12T11:38:23.438040+0000","last_active":"2024-04-12T11:40:23.579402+0000","last_peered":"2024-04-12T11:40:23.579402+0000","last_clean":"2024-04-12T11:38:20.888095+0000","last_became_active":"2024-04-12T11:38:23.424268+0000","last_became_peered":"2024-04-12T11:38:23.424268+0000","last_unstale":"2024-04-12T11:40:23.579402+0000","last_undegraded":"2024-04-12T11:38:23.284099+0000","last_fullsized":"2024-04-12T11:38:23.245558+0000","mapping_epoch":1147,"log_start":"1140'1800","ondisk_log_start":"1140'1800","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8862,"log_dups_size":0,"ondisk_log_size":8862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173670400,"num_objects":2650,"num_object_clones":0,"num_object_copies":7950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10662,"num_write_kb":425984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":1635},{"shards":"0,7","objects":-2991}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1115,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.133857+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:39:51.133857+0000","last_peered":"2024-04-12T11:39:51.133857+0000","last_clean":"2024-04-12T11:39:51.133857+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:39:51.133857+0000","last_undegraded":"2024-04-12T11:39:51.133857+0000","last_fullsized":"2024-04-12T11:39:51.133857+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1470,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107366+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:39:51.107366+0000","last_peered":"2024-04-12T11:39:51.107366+0000","last_clean":"2024-04-12T11:39:51.107366+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:39:51.107366+0000","last_undegraded":"2024-04-12T11:39:51.107366+0000","last_fullsized":"2024-04-12T11:39:51.107366+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1640,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.236477+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:40:16.236477+0000","last_peered":"2024-04-12T11:40:16.236477+0000","last_clean":"2024-04-12T11:40:16.236477+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:40:16.236477+0000","last_undegraded":"2024-04-12T11:40:16.236477+0000","last_fullsized":"2024-04-12T11:40:16.236477+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1104,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:40:12.410094+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:40:12.410094+0000","last_peered":"2024-04-12T11:40:12.410094+0000","last_clean":"2024-04-12T11:40:12.410094+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:40:12.410094+0000","last_undegraded":"2024-04-12T11:40:12.410094+0000","last_fullsized":"2024-04-12T11:40:12.410094+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1159'10561","reported_seq":10597,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:22.423797+0000","last_change":"2024-04-12T11:37:03.061698+0000","last_active":"2024-04-12T11:40:22.423797+0000","last_peered":"2024-04-12T11:40:22.423797+0000","last_clean":"2024-04-12T11:40:22.423797+0000","last_became_active":"2024-04-12T11:37:03.061469+0000","last_became_peered":"2024-04-12T11:37:03.061469+0000","last_unstale":"2024-04-12T11:40:22.423797+0000","last_undegraded":"2024-04-12T11:40:22.423797+0000","last_fullsized":"2024-04-12T11:40:22.423797+0000","mapping_epoch":1133,"log_start":"1140'1900","ondisk_log_start":"1140'1900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8661,"log_dups_size":0,"ondisk_log_size":8661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:09.906340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175308800,"num_objects":2675,"num_object_clones":0,"num_object_copies":8025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10561,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1123,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:57.452169+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:39:57.452169+0000","last_peered":"2024-04-12T11:39:57.452169+0000","last_clean":"2024-04-12T11:39:57.452169+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:39:57.452169+0000","last_undegraded":"2024-04-12T11:39:57.452169+0000","last_fullsized":"2024-04-12T11:39:57.452169+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1733,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.222980+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:40:16.222980+0000","last_peered":"2024-04-12T11:40:16.222980+0000","last_clean":"2024-04-12T11:40:16.222980+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:40:16.222980+0000","last_undegraded":"2024-04-12T11:40:16.222980+0000","last_fullsized":"2024-04-12T11:40:16.222980+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1733,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.220262+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:40:16.220262+0000","last_peered":"2024-04-12T11:40:16.220262+0000","last_clean":"2024-04-12T11:40:16.220262+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:40:16.220262+0000","last_undegraded":"2024-04-12T11:40:16.220262+0000","last_fullsized":"2024-04-12T11:40:16.220262+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"1159'10709","reported_seq":10745,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:22.280475+0000","last_change":"2024-04-12T11:37:03.060455+0000","last_active":"2024-04-12T11:40:22.280475+0000","last_peered":"2024-04-12T11:40:22.280475+0000","last_clean":"2024-04-12T11:40:22.280475+0000","last_became_active":"2024-04-12T11:37:03.060244+0000","last_became_peered":"2024-04-12T11:37:03.060244+0000","last_unstale":"2024-04-12T11:40:22.280475+0000","last_undegraded":"2024-04-12T11:40:22.280475+0000","last_fullsized":"2024-04-12T11:40:22.280475+0000","mapping_epoch":1133,"log_start":"1138'700","ondisk_log_start":"1138'700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10009,"log_dups_size":0,"ondisk_log_size":10009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:47.787368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182517760,"num_objects":2785,"num_object_clones":0,"num_object_copies":8355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10709,"num_write_kb":431808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1122,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.107949+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:39:51.107949+0000","last_peered":"2024-04-12T11:39:51.107949+0000","last_clean":"2024-04-12T11:39:51.107949+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:39:51.107949+0000","last_undegraded":"2024-04-12T11:39:51.107949+0000","last_fullsized":"2024-04-12T11:39:51.107949+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1723,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.218437+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:40:16.218437+0000","last_peered":"2024-04-12T11:40:16.218437+0000","last_clean":"2024-04-12T11:40:16.218437+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:40:16.218437+0000","last_undegraded":"2024-04-12T11:40:16.218437+0000","last_fullsized":"2024-04-12T11:40:16.218437+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1716,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.214307+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:40:16.214307+0000","last_peered":"2024-04-12T11:40:16.214307+0000","last_clean":"2024-04-12T11:40:16.214307+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:40:16.214307+0000","last_undegraded":"2024-04-12T11:40:16.214307+0000","last_fullsized":"2024-04-12T11:40:16.214307+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1159'10652","reported_seq":10714,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:25.065649+0000","last_change":"2024-04-12T11:37:03.057084+0000","last_active":"2024-04-12T11:40:25.065649+0000","last_peered":"2024-04-12T11:40:25.065649+0000","last_clean":"2024-04-12T11:40:25.065649+0000","last_became_active":"2024-04-12T11:37:03.056772+0000","last_became_peered":"2024-04-12T11:37:03.056772+0000","last_unstale":"2024-04-12T11:40:25.065649+0000","last_undegraded":"2024-04-12T11:40:25.065649+0000","last_fullsized":"2024-04-12T11:40:25.065649+0000","mapping_epoch":1133,"log_start":"1141'3100","ondisk_log_start":"1141'3100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7552,"log_dups_size":0,"ondisk_log_size":7552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:02:41.845790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165937152,"num_objects":2532,"num_object_clones":0,"num_object_copies":7596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10652,"num_write_kb":421888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1655,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.253800+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:40:16.253800+0000","last_peered":"2024-04-12T11:40:16.253800+0000","last_clean":"2024-04-12T11:40:16.253800+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:40:16.253800+0000","last_undegraded":"2024-04-12T11:40:16.253800+0000","last_fullsized":"2024-04-12T11:40:16.253800+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1723,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.214446+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:40:16.214446+0000","last_peered":"2024-04-12T11:40:16.214446+0000","last_clean":"2024-04-12T11:40:16.214446+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:40:16.214446+0000","last_undegraded":"2024-04-12T11:40:16.214446+0000","last_fullsized":"2024-04-12T11:40:16.214446+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1653,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.238439+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:40:16.238439+0000","last_peered":"2024-04-12T11:40:16.238439+0000","last_clean":"2024-04-12T11:40:16.238439+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:40:16.238439+0000","last_undegraded":"2024-04-12T11:40:16.238439+0000","last_fullsized":"2024-04-12T11:40:16.238439+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1159'10939","reported_seq":11002,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:25.098802+0000","last_change":"2024-04-12T11:37:03.060299+0000","last_active":"2024-04-12T11:40:25.098802+0000","last_peered":"2024-04-12T11:40:25.098802+0000","last_clean":"2024-04-12T11:40:25.098802+0000","last_became_active":"2024-04-12T11:37:03.058897+0000","last_became_peered":"2024-04-12T11:37:03.058897+0000","last_unstale":"2024-04-12T11:40:25.098802+0000","last_undegraded":"2024-04-12T11:40:25.098802+0000","last_fullsized":"2024-04-12T11:40:25.098802+0000","mapping_epoch":1133,"log_start":"1141'3400","ondisk_log_start":"1141'3400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7539,"log_dups_size":0,"ondisk_log_size":7539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:45:37.844216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166395904,"num_objects":2539,"num_object_clones":0,"num_object_copies":7617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10939,"num_write_kb":431296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1128,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:57.406098+0000","last_change":"2024-04-12T11:39:23.349899+0000","last_active":"2024-04-12T11:39:57.406098+0000","last_peered":"2024-04-12T11:39:57.406098+0000","last_clean":"2024-04-12T11:39:57.406098+0000","last_became_active":"2024-04-12T11:39:23.349526+0000","last_became_peered":"2024-04-12T11:39:23.349526+0000","last_unstale":"2024-04-12T11:39:57.406098+0000","last_undegraded":"2024-04-12T11:39:57.406098+0000","last_fullsized":"2024-04-12T11:39:57.406098+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:36:47.115912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1736,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.220517+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:40:16.220517+0000","last_peered":"2024-04-12T11:40:16.220517+0000","last_clean":"2024-04-12T11:40:16.220517+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:40:16.220517+0000","last_undegraded":"2024-04-12T11:40:16.220517+0000","last_fullsized":"2024-04-12T11:40:16.220517+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1643,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.210099+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:40:16.210099+0000","last_peered":"2024-04-12T11:40:16.210099+0000","last_clean":"2024-04-12T11:40:16.210099+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:40:16.210099+0000","last_undegraded":"2024-04-12T11:40:16.210099+0000","last_fullsized":"2024-04-12T11:40:16.210099+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1726,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.214199+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:40:16.214199+0000","last_peered":"2024-04-12T11:40:16.214199+0000","last_clean":"2024-04-12T11:40:16.214199+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:40:16.214199+0000","last_undegraded":"2024-04-12T11:40:16.214199+0000","last_fullsized":"2024-04-12T11:40:16.214199+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":41,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.239101+0000","last_change":"2024-04-12T11:38:06.839832+0000","last_active":"2024-04-12T11:40:16.239101+0000","last_peered":"2024-04-12T11:40:16.239101+0000","last_clean":"2024-04-12T11:40:16.239101+0000","last_became_active":"2024-04-12T11:38:06.839460+0000","last_became_peered":"2024-04-12T11:38:06.839460+0000","last_unstale":"2024-04-12T11:40:16.239101+0000","last_undegraded":"2024-04-12T11:40:16.239101+0000","last_fullsized":"2024-04-12T11:40:16.239101+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:25:27.959933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1125,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.131936+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:39:51.131936+0000","last_peered":"2024-04-12T11:39:51.131936+0000","last_clean":"2024-04-12T11:39:51.131936+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:39:51.131936+0000","last_undegraded":"2024-04-12T11:39:51.131936+0000","last_fullsized":"2024-04-12T11:39:51.131936+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1670,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.208353+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:40:16.208353+0000","last_peered":"2024-04-12T11:40:16.208353+0000","last_clean":"2024-04-12T11:40:16.208353+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:40:16.208353+0000","last_undegraded":"2024-04-12T11:40:16.208353+0000","last_fullsized":"2024-04-12T11:40:16.208353+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1110,"reported_epoch":1158,"state":"active+clean","last_fresh":"2024-04-12T11:39:51.131961+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:39:51.131961+0000","last_peered":"2024-04-12T11:39:51.131961+0000","last_clean":"2024-04-12T11:39:51.131961+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:39:51.131961+0000","last_undegraded":"2024-04-12T11:39:51.131961+0000","last_fullsized":"2024-04-12T11:39:51.131961+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1661,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.208400+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:40:16.208400+0000","last_peered":"2024-04-12T11:40:16.208400+0000","last_clean":"2024-04-12T11:40:16.208400+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:40:16.208400+0000","last_undegraded":"2024-04-12T11:40:16.208400+0000","last_fullsized":"2024-04-12T11:40:16.208400+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1654,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.236859+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:40:16.236859+0000","last_peered":"2024-04-12T11:40:16.236859+0000","last_clean":"2024-04-12T11:40:16.236859+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:40:16.236859+0000","last_undegraded":"2024-04-12T11:40:16.236859+0000","last_fullsized":"2024-04-12T11:40:16.236859+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1665,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.237201+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:40:16.237201+0000","last_peered":"2024-04-12T11:40:16.237201+0000","last_clean":"2024-04-12T11:40:16.237201+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:40:16.237201+0000","last_undegraded":"2024-04-12T11:40:16.237201+0000","last_fullsized":"2024-04-12T11:40:16.237201+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1754,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.225617+0000","last_change":"2024-04-12T11:38:06.866508+0000","last_active":"2024-04-12T11:40:16.225617+0000","last_peered":"2024-04-12T11:40:16.225617+0000","last_clean":"2024-04-12T11:40:16.225617+0000","last_became_active":"2024-04-12T11:38:06.843752+0000","last_became_peered":"2024-04-12T11:38:06.843752+0000","last_unstale":"2024-04-12T11:40:16.225617+0000","last_undegraded":"2024-04-12T11:40:16.225617+0000","last_fullsized":"2024-04-12T11:40:16.225617+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:18:11.827424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1633,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.207947+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:40:16.207947+0000","last_peered":"2024-04-12T11:40:16.207947+0000","last_clean":"2024-04-12T11:40:16.207947+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:40:16.207947+0000","last_undegraded":"2024-04-12T11:40:16.207947+0000","last_fullsized":"2024-04-12T11:40:16.207947+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2756313112,"num_objects":42059,"num_object_clones":0,"num_object_copies":126177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5951,"num_objects_misplaced":1567,"num_objects_unfound":0,"num_objects_dirty":42059,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172943,"num_write_kb":6880001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3838,"num_bytes_recovered":251527168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8818733056,"data_stored":8818720840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139043,"ondisk_log_size":139043,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234336,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1212504,"kb_used_data":884496,"kb_used_omap":1,"kb_used_meta":327998,"kb_avail":92532648,"statfs":{"total":95995035648,"available":94753431552,"internally_reserved":0,"allocated":905723904,"data_stored":902650901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":335870411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495973,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1078736,"kb_used_data":871304,"kb_used_omap":1,"kb_used_meta":207422,"kb_avail":92666416,"statfs":{"total":95995035648,"available":94890409984,"internally_reserved":0,"allocated":892215296,"data_stored":889150363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":212400584},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536283,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1676220,"kb_used_data":1405492,"kb_used_omap":30,"kb_used_meta":270689,"kb_avail":92068932,"statfs":{"total":95995035648,"available":94278586368,"internally_reserved":0,"allocated":1439223808,"data_stored":1436173997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":277185858},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190367,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1683720,"kb_used_data":1380928,"kb_used_omap":43,"kb_used_meta":302740,"kb_avail":92061432,"statfs":{"total":95995035648,"available":94270906368,"internally_reserved":0,"allocated":1414070272,"data_stored":1411009321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":310005803},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849495,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1353920,"kb_used_data":1010936,"kb_used_omap":36,"kb_used_meta":342939,"kb_avail":92391232,"statfs":{"total":95995035648,"available":94608621568,"internally_reserved":0,"allocated":1035198464,"data_stored":1032145149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":351169716},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704848,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1705020,"kb_used_data":1312308,"kb_used_omap":17,"kb_used_meta":392686,"kb_avail":92040132,"statfs":{"total":95995035648,"available":94249095168,"internally_reserved":0,"allocated":1343803392,"data_stored":1340753647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":402110967},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018060,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049680,"kb_used_data":828744,"kb_used_omap":42,"kb_used_meta":220885,"kb_avail":92695472,"statfs":{"total":95995035648,"available":94920163328,"internally_reserved":0,"allocated":848633856,"data_stored":845568443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":226187064},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757609,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1375104,"kb_used_data":1073336,"kb_used_omap":1,"kb_used_meta":301758,"kb_avail":92370048,"statfs":{"total":95995035648,"available":94586929152,"internally_reserved":0,"allocated":1099096064,"data_stored":1096042773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":309000646},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1383141376,"data_stored":1383137304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1416105984,"data_stored":1416101912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1015414784,"data_stored":1015414784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1320288256,"data_stored":1320288256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":836501504,"data_stored":836501504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1084358656,"data_stored":1084358656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":878186496,"data_stored":878182424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":884736000,"data_stored":884736000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:40:38.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph status --format=json 2024-04-12T11:40:38.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:38.769 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:38.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:38.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:38.870 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:38.898 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:38.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:38.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:38.962 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:38 smithi029 ceph-mon[93650]: pgmap v1613: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5951/105582 objects degraded (5.636%); 1156/105582 objects misplaced (1.095%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T11:40:38.962 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:38 smithi029 ceph-mon[97787]: pgmap v1613: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5951/105582 objects degraded (5.636%); 1156/105582 objects misplaced (1.095%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T11:40:39.000 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:39.027 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:39.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:39.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:39.129 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:39.159 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:39.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:39.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:39.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:38 smithi087 ceph-mon[89014]: pgmap v1613: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5951/105582 objects degraded (5.636%); 1156/105582 objects misplaced (1.095%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T11:40:39.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:39.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:39.311 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:39.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:39.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:39.453 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:39.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:39.554 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:39.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:39.593 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:39.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:39.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:39.732 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:39.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:39.834 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:39.835 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:39.879 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:39.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:39.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:39.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:40.038 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:40.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:40.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:40.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:40.181 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:40.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:40.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:40.247 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:40.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:40.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:40.336 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:40.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:40.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:40.496 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:40.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:40.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:40.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:40.639 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:40.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:40.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:40.740 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:40.784 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:40.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:40.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:40.822 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:40:40.878 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:40:40.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:40.922 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:40:40.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:40.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:40.969 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:40:41.018 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:40:41.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:41.056 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:40:41.083 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:41.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:41.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:41.117 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:40:41.163 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:40:41.172 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:40:41.172 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:40:41.172 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:40:41.172 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:40:41.172 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:40:41.172 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:40:41.172 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:40:41.172 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:40:41.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:40 smithi087 ceph-mon[89014]: pgmap v1614: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 5951/102633 objects degraded (5.798%); 1156/102633 objects misplaced (1.126%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:40:41.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:40 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3271500587' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:41.226 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:41.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:41.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:41.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:40 smithi029 ceph-mon[97787]: pgmap v1614: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 5951/102633 objects degraded (5.798%); 1156/102633 objects misplaced (1.126%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:40:41.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:40 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3271500587' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:41.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:40 smithi029 ceph-mon[93650]: pgmap v1614: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 5951/102633 objects degraded (5.798%); 1156/102633 objects misplaced (1.126%); 2.4 MiB/s, 37 objects/s recovering 2024-04-12T11:40:41.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:40 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3271500587' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:41.327 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:41.355 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:41.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:41.457 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:41.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:41.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:41.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:41.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:41.627 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:41.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:41.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:41.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:41.767 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:41.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:41.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:41.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:41.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:41.915 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:41.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:42.016 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:42.045 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:42.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:42.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:42.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:42.192 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:42.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:42.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:42.294 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:42.320 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:42.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:42.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:42.324 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:40:42.324 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:40:42.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:40:42.324 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:40:42.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.324 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:65206 6.6 6:6325b4d4:::benchmark_data_smithi029_138569_object65205:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:40:42.324 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:41.890096+0000", 2024-04-12T11:40:42.324 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 420.43096830100001, 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.026141312, 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 65206 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.890096+0000", 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.890096+0000", 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.890097+0000", 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.486e-06 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.890125+0000", 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7586999999999999e-05 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.890135+0000", 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0231000000000001e-05 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.890149+0000", 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3747999999999999e-05 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.890274+0000", 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000124795 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2024-04-12T11:40:42.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.890280+0000", 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2430000000000002e-06 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.066309+0000", 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17602942399999999 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.066321+0000", 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1070000000000001e-05 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.916147+0000", 2024-04-12T11:40:42.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84982641599999997 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.916237+0000", 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0311999999999999e-05 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:65220 6.6 6:62df34e4:::benchmark_data_smithi029_138569_object65219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:42.094100+0000", 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 420.22696464199998, 2024-04-12T11:40:42.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83597521699999999, 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 65220 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.094100+0000", 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.094100+0000", 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.094101+0000", 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5e-06 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.094126+0000", 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5165e-05 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.094138+0000", 2024-04-12T11:40:42.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1574e-05 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.094159+0000", 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1486000000000001e-05 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.094220+0000", 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0296e-05 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.094228+0000", 2024-04-12T11:40:42.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8799999999999997e-06 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.921669+0000", 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.82744056899999996 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.921836+0000", 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016647400000000001 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.922012+0000", 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017607000000000001 2024-04-12T11:40:42.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.923583+0000", 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015716230000000001 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.929744+0000", 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061609560000000004 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.929962+0000", 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000217767 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.930021+0000", 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8810000000000001e-05 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.930075+0000", 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4046999999999999e-05 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75431 6.13 6.85725d33 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.031637+0000", 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.289427547, 2024-04-12T11:40:42.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.042905671, 2024-04-12T11:40:42.336 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.336 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.336 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.336 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.336 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.336 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.336 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75431 2024-04-12T11:40:42.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.336 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.031637+0000", 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.031637+0000", 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.031637+0000", 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1900000000000002e-07 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.031660+0000", 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2717e-05 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.031669+0000", 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5939999999999994e-06 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.031676+0000", 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2359999999999998e-06 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.032103+0000", 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000426826 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.032112+0000", 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2099999999999999e-06 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074529+0000", 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0424173640000001 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.339 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074542+0000", 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3105000000000001e-05 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75470 6.13 6.ee33f633 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.066182+0000", 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.25488247200002, 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.008395551, 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.340 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75470 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.066182+0000", 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.066182+0000", 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.066183+0000", 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.655e-06 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.066207+0000", 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3706999999999999e-05 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.066218+0000", 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0841e-05 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.342 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.066225+0000", 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5070000000000003e-06 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.066338+0000", 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000112057 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.066346+0000", 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.867e-06 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074569+0000", 2024-04-12T11:40:42.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.008222763 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074577+0000", 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1540000000000002e-06 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75499 6.13 6.b7d8f333 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.088435+0000", 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.23262919500002, 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98617297699999995, 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.344 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75499 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.088435+0000", 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.088435+0000", 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.088436+0000", 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8100000000000003e-07 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.088455+0000", 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9919999999999999e-05 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.088463+0000", 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1119999999999998e-06 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.088469+0000", 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.976e-06 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.093191+0000", 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047222000000000002 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.093200+0000", 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5859999999999994e-06 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074599+0000", 2024-04-12T11:40:42.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98139897799999998 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074608+0000", 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7239999999999998e-06 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75517 6.13 6.ac2ef93 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.103403+0000", 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.21766158299999, 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97129745300000003, 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.348 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75517 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.103403+0000", 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.103403+0000", 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.103404+0000", 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.699e-06 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.103426+0000", 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1316000000000001e-05 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.103438+0000", 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2685e-05 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.103447+0000", 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4309999999999994e-06 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.103678+0000", 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023079799999999999 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.103685+0000", 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5170000000000002e-06 2024-04-12T11:40:42.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074692+0000", 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97100678799999995 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074700+0000", 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2190000000000004e-06 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.352 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75525 6.13 6.e34608f3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.109102+0000", 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.21196252499999, 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96565104700000004, 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75525 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.353 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.109102+0000", 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.109102+0000", 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.109103+0000", 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5300000000000003e-07 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.109124+0000", 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1098e-05 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.109133+0000", 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4700000000000008e-06 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.109147+0000", 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3534e-05 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.109206+0000", 2024-04-12T11:40:42.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9107000000000002e-05 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.109214+0000", 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6729999999999998e-06 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074745+0000", 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96553041100000003 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074753+0000", 2024-04-12T11:40:42.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9009999999999994e-06 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75527 6.13 6.8149eed3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.110375+0000", 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.210689274, 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96440599900000001, 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.357 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75527 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110375+0000", 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110375+0000", 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110376+0000", 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2640000000000001e-06 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110400+0000", 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3926000000000001e-05 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110416+0000", 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5738000000000001e-05 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110425+0000", 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1730000000000001e-06 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110480+0000", 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4591e-05 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110487+0000", 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3150000000000003e-06 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074773+0000", 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96428641299999995 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074781+0000", 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5789999999999998e-06 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75529 6.13 6.4f09a293 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.110624+0000", 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.21043974399998, 2024-04-12T11:40:42.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96418390399999998, 2024-04-12T11:40:42.362 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.362 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.362 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.362 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.362 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.362 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.362 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75529 2024-04-12T11:40:42.362 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.362 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110624+0000", 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110624+0000", 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110625+0000", 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7700000000000003e-07 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.363 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110647+0000", 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1492999999999999e-05 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110660+0000", 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2714000000000001e-05 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110667+0000", 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8339999999999999e-06 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110718+0000", 2024-04-12T11:40:42.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0547999999999998e-05 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.110723+0000", 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2599999999999996e-06 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074801+0000", 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96407766500000003 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074808+0000", 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5129999999999999e-06 2024-04-12T11:40:42.365 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75540 6.13 6.ddb985f3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.121278+0000", 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.19978667700002, 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95366556099999999, 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.366 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75540 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.121278+0000", 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.121278+0000", 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.121278+0000", 2024-04-12T11:40:42.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7199999999999999e-07 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.121298+0000", 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9692000000000001e-05 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.121305+0000", 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3889999999999999e-06 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.121311+0000", 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.891e-06 2024-04-12T11:40:42.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.121410+0000", 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9153000000000004e-05 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.121425+0000", 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4630000000000001e-05 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074934+0000", 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95350955199999998 2024-04-12T11:40:42.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074943+0000", 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6819999999999992e-06 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75568 6.13 6.43945553 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.151173+0000", 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.16989142900002, 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92380001599999995, 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.370 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75568 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.151173+0000", 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.151173+0000", 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.151174+0000", 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4470000000000001e-06 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.151202+0000", 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7543999999999999e-05 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.151216+0000", 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4153e-05 2024-04-12T11:40:42.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.151225+0000", 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3009999999999995e-06 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.151294+0000", 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8880000000000005e-05 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.151303+0000", 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9379999999999998e-06 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.373 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074965+0000", 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92366221800000003 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074973+0000", 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5349999999999999e-06 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75649 6.13 6.5d805113 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.240349+0000", 2024-04-12T11:40:42.374 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.08071545899998, 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83465243700000002, 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75649 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.240349+0000", 2024-04-12T11:40:42.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.240349+0000", 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.240350+0000", 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7599999999999996e-07 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.240382+0000", 2024-04-12T11:40:42.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2502000000000002e-05 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.240391+0000", 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9309999999999997e-06 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.240398+0000", 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.63e-06 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.240455+0000", 2024-04-12T11:40:42.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6944999999999998e-05 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.240463+0000", 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3219999999999993e-06 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.074994+0000", 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83453087699999995 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075001+0000", 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4540000000000001e-06 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.378 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75681 6.13 6.60c7dcf3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.280099+0000", 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.04096516300001, 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.79492974000000005, 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75681 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.280099+0000", 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.280099+0000", 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.280100+0000", 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4190000000000001e-06 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.280123+0000", 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2308000000000001e-05 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.280135+0000", 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.207e-05 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.280152+0000", 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7139999999999999e-05 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.280219+0000", 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7150000000000006e-05 2024-04-12T11:40:42.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.280227+0000", 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2050000000000002e-06 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075021+0000", 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.79479404499999995 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075029+0000", 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4030000000000001e-06 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.382 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75683 6.13 6.9c125db3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.283454+0000", 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.037610648, 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.79160489999999994, 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75683 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.283454+0000", 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.283454+0000", 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.283455+0000", 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.793e-06 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.283478+0000", 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.26e-05 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.283496+0000", 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7912000000000001e-05 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.283505+0000", 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2490000000000007e-06 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.283572+0000", 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6548e-05 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.283581+0000", 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1719999999999996e-06 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075049+0000", 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.79146811500000003 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075058+0000", 2024-04-12T11:40:42.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5109999999999992e-06 2024-04-12T11:40:42.387 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.387 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.387 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75709 6.13 6.85a95673 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.320173+0000", 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 406.00089084000001, 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75491396099999997, 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75709 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.320173+0000", 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.320173+0000", 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.320175+0000", 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2729999999999999e-06 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.320193+0000", 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8838000000000002e-05 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.320204+0000", 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0560999999999999e-05 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.320211+0000", 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5490000000000003e-06 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.320288+0000", 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7231000000000006e-05 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.320298+0000", 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0044000000000001e-05 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075079+0000", 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75478143399999997 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075087+0000", 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0309999999999998e-06 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75729 6.13 6.78ae8253 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.346897+0000", 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 405.97416718800002, 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.728219379, 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.392 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75729 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.346897+0000", 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.346897+0000", 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.346898+0000", 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6300000000000004e-07 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.346920+0000", 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2473999999999999e-05 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.346931+0000", 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0658e-05 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.346938+0000", 2024-04-12T11:40:42.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4560000000000003e-06 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.346993+0000", 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4058999999999999e-05 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.347000+0000", 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9100000000000005e-06 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075109+0000", 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.72810858000000001 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075116+0000", 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.379e-06 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75802 6.13 6.55c60833 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1101)", 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:56.561032+0000", 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 405.76003240099999, 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51412040999999997, 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:40:42.396 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75802 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.561032+0000", 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.561032+0000", 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.561033+0000", 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3120000000000001e-06 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.561054+0000", 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0837999999999999e-05 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.561066+0000", 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1821000000000001e-05 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.561075+0000", 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0790000000000007e-06 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.561154+0000", 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8836000000000002e-05 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:56.561164+0000", 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0213e-05 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075138+0000", 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51397415700000004 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:57.075152+0000", 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4154e-05 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:100255 6.b 6:d2cf83c7:::benchmark_data_smithi029_138569_object100254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:40:42.400 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:34:27.599804+0000", 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 374.721260341, 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.471434462, 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 100255 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.599804+0000", 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.599804+0000", 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.599805+0000", 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3909999999999999e-06 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.599826+0000", 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1222000000000001e-05 2024-04-12T11:40:42.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.599839+0000", 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2105e-05 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.599847+0000", 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2029999999999992e-06 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.599908+0000", 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1022e-05 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.600030+0000", 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000121743 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.600172+0000", 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014215899999999999 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.604645+0000", 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044733230000000004 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.604680+0000", 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4625999999999997e-05 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.071138+0000", 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.46645821199999998 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.071189+0000", 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0696999999999999e-05 2024-04-12T11:40:42.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.071238+0000", 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9759000000000003e-05 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:100262 6.b 6:d257e746:::benchmark_data_smithi029_138569_object100261:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:34:27.605256+0000", 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 374.71580798700001, 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.47168871099999998, 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.406 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 100262 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.605256+0000", 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.605256+0000", 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.605257+0000", 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1179999999999999e-06 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.605278+0000", 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0506e-05 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.605287+0000", 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0850000000000003e-06 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.605294+0000", 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.832e-06 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.608694+0000", 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034002250000000002 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.608813+0000", 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118949 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2024-04-12T11:40:42.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.608954+0000", 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014082799999999999 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.071286+0000", 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.46233176399999998 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.071303+0000", 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7638999999999998e-05 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.076837+0000", 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055334970000000001 2024-04-12T11:40:42.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.076894+0000", 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6886999999999999e-05 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.076945+0000", 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1381000000000002e-05 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:100263 6.b 6:d3278a61:::benchmark_data_smithi029_138569_object100262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:34:27.607160+0000", 2024-04-12T11:40:42.411 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 374.71390471500001, 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.47088550699999998, 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 100263 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.607160+0000", 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.607160+0000", 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.607161+0000", 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.217e-06 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.607182+0000", 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1081999999999999e-05 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:40:42.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.607192+0000", 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8009999999999998e-06 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:27.607198+0000", 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6680000000000004e-06 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.064031+0000", 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.45683231699999999 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.064226+0000", 2024-04-12T11:40:42.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000195047 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.064633+0000", 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000407632 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.077621+0000", 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012987322000000001 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.077661+0000", 2024-04-12T11:40:42.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0138000000000001e-05 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.077701+0000", 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0311000000000001e-05 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.077999+0000", 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00029774899999999998 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:28.078045+0000", 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6223000000000001e-05 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.416 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.417 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.417 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:40:42.417 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:40:42.417 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:40:42.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:42.464 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:42.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:42.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:42.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:42.610 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:42.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:42.711 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:40:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:40:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:40:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:42.741 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:42.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:42.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:42.883 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:42.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:42.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:42.986 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:43.029 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:43.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:43.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:43.131 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:43.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:43.159 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:43.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:43.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:42 smithi029 ceph-mon[93650]: pgmap v1615: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5951/98439 objects degraded (6.045%); 903/98439 objects misplaced (0.917%); 3.5 MiB/s, 55 objects/s recovering 2024-04-12T11:40:43.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:42 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/98439 objects degraded (6.045%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:42 smithi029 ceph-mon[97787]: pgmap v1615: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5951/98439 objects degraded (6.045%); 903/98439 objects misplaced (0.917%); 3.5 MiB/s, 55 objects/s recovering 2024-04-12T11:40:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:42 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/98439 objects degraded (6.045%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:43.302 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:43.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:43.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:43.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:43.441 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:43.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:43.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:43.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:42 smithi087 ceph-mon[89014]: pgmap v1615: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5951/98439 objects degraded (6.045%); 903/98439 objects misplaced (0.917%); 3.5 MiB/s, 55 objects/s recovering 2024-04-12T11:40:43.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:42 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/98439 objects degraded (6.045%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:43.543 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:43.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:43.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:43.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:43.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-04-12T11:40:43.683 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:43.713 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:43.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:43.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:43.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:43.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:43.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:43.918 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:40:43.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:44.018 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:44.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:44.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:44.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:44.161 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:44.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:44.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:44.262 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:44.303 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:44.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:44.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:44.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:44.444 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:44.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:44.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:44.546 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:44.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:44.588 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:44.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:44.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:44.733 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:44.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:44.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:44.835 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:44.871 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:44.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:44.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:44.972 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:45.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:45.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:45.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:45.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:44 smithi029 ceph-mon[97787]: pgmap v1616: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 5951/92535 objects degraded (6.431%); 903/92535 objects misplaced (0.976%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:40:45.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:44 smithi029 ceph-mon[93650]: pgmap v1616: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 5951/92535 objects degraded (6.431%); 903/92535 objects misplaced (0.976%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:40:45.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:40:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:40:44] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2024-04-12T11:40:45.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:45.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:45.148 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:45.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:45.249 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:45.274 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:44 smithi087 ceph-mon[89014]: pgmap v1616: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 5951/92535 objects degraded (6.431%); 903/92535 objects misplaced (0.976%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:40:45.299 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:45.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:45.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:45.401 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:45.430 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:45.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:45.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:45.453 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:40:45.454 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5951/92535 objects degraded (6.431%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1225,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1160,"num_osds":8,"num_up_osds":8,"osd_up_since":1712921015,"num_in_osds":8,"osd_in_since":1712920262,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":30845,"data_bytes":2020415466,"bytes_used":9317998592,"bytes_avail":758642286592,"bytes_total":767960285184,"degraded_objects":5951,"degraded_total":92535,"degraded_ratio":0.064310801318420055,"misplaced_objects":903,"misplaced_total":92535,"misplaced_ratio":0.0097584697681958182,"recovering_objects_per_sec":38,"recovering_bytes_per_sec":2510907,"recovering_keys_per_sec":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":685},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.scbuux","status":"up:active","gid":14661}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":25,"modified":"2024-04-12T11:37:57.741061+0000","services":{}},"progress_events":{"53786749-146c-4436-89ac-aaaa1350cbf3":{"message":"Upgrade to 19.0.0-2874-g2cee1857 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"d4a8c707-9e6b-4992-81dd-3fa476e5d44b":{"message":"Global Recovery Event (60s)\n [===========================.] (remaining: 2s)","progress":0.96629214286804199,"add_to_ceph_s":true}}} 2024-04-12T11:40:45.531 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:45.571 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:45.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:45.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:45.672 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:45.701 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:45.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:45.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:45.713 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:40:45.713 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:40:45.713 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T11:40:45.713 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:40:45.803 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:45.833 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:45.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:45.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:45.981 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:45.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:45.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:46.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:46.125 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:46.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:46.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:46.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:46.243 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:45 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4222881333' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-12T11:40:46.253 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:46.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:46.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:46.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:45 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4222881333' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-12T11:40:46.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:45 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4222881333' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-12T11:40:46.275 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:40:46.355 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:46.390 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-04-12T11:40:46.426 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:46.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:46.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:46.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:46.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:46.581 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:46.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:46.657 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:40:46.682 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:46.724 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:46.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:46.826 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:46.845 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-04-12T11:40:46.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:40:46.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:46.886 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:46.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:46.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:47.057 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:47.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:47.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:46 smithi029 ceph-mon[93650]: pgmap v1617: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5951/88317 objects degraded (6.738%); 653/88317 objects misplaced (0.739%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:40:47.140 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:46 smithi029 ceph-mon[97787]: pgmap v1617: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5951/88317 objects degraded (6.738%); 653/88317 objects misplaced (0.739%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:40:47.157 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:47.188 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:47.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:47.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:47.290 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:47.305 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:46 smithi087 ceph-mon[89014]: pgmap v1617: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 5951/88317 objects degraded (6.738%); 653/88317 objects misplaced (0.739%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:40:47.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:47.337 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:47.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:47.438 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:47.495 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:47.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:47.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:47.559 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:47.566 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:47.597 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:47.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:47.650 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:47.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:47.751 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:47.797 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:47.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:47.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:47.899 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:47.925 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:47.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:47.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:48.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:48.034 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:40:48.080 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:48.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:48.094 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:47 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/88317 objects degraded (6.738%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:48.094 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:47 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/644202888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:40:48.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:48.220 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:48.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:48.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/88317 objects degraded (6.738%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:48.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:48 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/644202888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:40:48.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/88317 objects degraded (6.738%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:48.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:48 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/644202888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:40:48.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:48.362 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:48.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:48.464 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:48.494 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:48.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:48.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:48.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:48.630 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:48.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:48.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:48.732 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:48.745 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:40:48.745 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:40:48.745 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:40:48.745 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:40:48.745 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:40:48.767 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:48.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:48.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:48.869 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:48.901 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:48.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:48.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:49.002 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:49.047 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:49.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:49.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:49.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:49.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:49.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:49 smithi029 ceph-mon[97787]: pgmap v1618: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5951/85929 objects degraded (6.925%); 653/85929 objects misplaced (0.760%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T11:40:49.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:49 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:40:49.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:49 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/644202888' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:40:49.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:49 smithi029 ceph-mon[97787]: osdmap e1161: 8 total, 8 up, 8 in 2024-04-12T11:40:49.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:49 smithi029 ceph-mon[93650]: pgmap v1618: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5951/85929 objects degraded (6.925%); 653/85929 objects misplaced (0.760%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T11:40:49.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:49 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:40:49.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:49 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/644202888' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:40:49.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:49 smithi029 ceph-mon[93650]: osdmap e1161: 8 total, 8 up, 8 in 2024-04-12T11:40:49.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:49.309 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:49 smithi087 ceph-mon[89014]: pgmap v1618: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5951/85929 objects degraded (6.925%); 653/85929 objects misplaced (0.760%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T11:40:49.309 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:49 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:40:49.309 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:49 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/644202888' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:40:49.309 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:49 smithi087 ceph-mon[89014]: osdmap e1161: 8 total, 8 up, 8 in 2024-04-12T11:40:49.331 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:49.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:49.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:49.412 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-04-12T11:40:49.434 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:49.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:49.489 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:49.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:49.590 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:49.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:49.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:49.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:49.697 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:40:49.723 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:49.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:49.763 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:49.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:49.864 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:49.896 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:49.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:49.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:49.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:50.042 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:50.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:50.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:50.144 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:50.172 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:50.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:50.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:50.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:50.319 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:50.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:50.421 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:50.449 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:50.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:50.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:50.551 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:50.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:50.598 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:50.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:50.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:50.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:50.742 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:50.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:50.843 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:50.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:50.883 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:50.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:50.961 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:40:50.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:51.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:51.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:51.177 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:51.199 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:51 smithi087 ceph-mon[89014]: pgmap v1620: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5951/82956 objects degraded (7.174%); 653/82956 objects misplaced (0.787%); 3.1 MiB/s, 50 objects/s recovering 2024-04-12T11:40:51.208 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:51.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:51 smithi029 ceph-mon[97787]: pgmap v1620: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5951/82956 objects degraded (7.174%); 653/82956 objects misplaced (0.787%); 3.1 MiB/s, 50 objects/s recovering 2024-04-12T11:40:51.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:51 smithi029 ceph-mon[93650]: pgmap v1620: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5951/82956 objects degraded (7.174%); 653/82956 objects misplaced (0.787%); 3.1 MiB/s, 50 objects/s recovering 2024-04-12T11:40:51.310 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:51.338 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:51.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:51.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:51.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:51.501 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:51.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:51.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:51.521 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:40:51.521 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:40:51.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:40:51.521 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:40:51.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.521 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:65197 6.12 6:496fa9ff:::benchmark_data_smithi029_138569_object65196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:40:51.521 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:33:41.884945+0000", 2024-04-12T11:40:51.521 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 429.63244487899999, 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0453005319999999, 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 65197 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.522 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.884945+0000", 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.884945+0000", 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.884947+0000", 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.714e-06 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.884998+0000", 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0593000000000002e-05 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.885009+0000", 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1875000000000001e-05 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.885019+0000", 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4550000000000001e-06 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.944403+0000", 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.059384563000000001 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:41.944413+0000", 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.5899999999999997e-06 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.930158+0000", 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98574526900000004 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:33:42.930246+0000", 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7472999999999994e-05 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:82177 6.e 6:77b176da:::benchmark_data_smithi029_138569_object82176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:05.942896+0000", 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 405.574494039, 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.480965741, 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.526 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 82177 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942896+0000", 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942896+0000", 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942898+0000", 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4699999999999999e-06 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942945+0000", 2024-04-12T11:40:51.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7042999999999997e-05 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942959+0000", 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.454e-05 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.942967+0000", 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.909e-06 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.943018+0000", 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1025e-05 2024-04-12T11:40:51.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.943187+0000", 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000169271 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.943367+0000", 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018001099999999999 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.946093+0000", 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027254150000000001 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.240144+0000", 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29405152200000001 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.423798+0000", 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.183654027 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.423833+0000", 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5037999999999999e-05 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.423862+0000", 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.847e-05 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:82186 6.2 6:444ecc6a:::benchmark_data_smithi029_138569_object82185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:05.949311+0000", 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 405.56807888600002, 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.47286100399999997, 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.532 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 82186 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949311+0000", 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949311+0000", 2024-04-12T11:40:51.533 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.534 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.534 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949313+0000", 2024-04-12T11:40:51.534 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.711e-06 2024-04-12T11:40:51.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.534 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.535 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949362+0000", 2024-04-12T11:40:51.535 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8847999999999997e-05 2024-04-12T11:40:51.535 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.535 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.535 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.535 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949376+0000", 2024-04-12T11:40:51.535 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4411e-05 2024-04-12T11:40:51.535 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949385+0000", 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6989999999999992e-06 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949448+0000", 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3156000000000003e-05 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949618+0000", 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016943399999999999 2024-04-12T11:40:51.536 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.949790+0000", 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017233799999999999 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.951406+0000", 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001616159 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:05.951830+0000", 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00042385600000000002 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.422053+0000", 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.47022343300000002 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.422120+0000", 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6230999999999997e-05 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:06.422172+0000", 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2728000000000001e-05 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:86009 6.2 6:44ba1091:::benchmark_data_smithi029_138569_object86008:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1105)", 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:10.564705+0000", 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 400.95268518199998, 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.374530211, 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86009 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.539 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564705+0000", 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564705+0000", 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564708+0000", 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6290000000000002e-06 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564752+0000", 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4400000000000002e-05 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564782+0000", 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.0159e-05 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564798+0000", 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5739999999999998e-05 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.564859+0000", 2024-04-12T11:40:51.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1538000000000006e-05 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.565019+0000", 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000159075 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.565189+0000", 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017095899999999999 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.566932+0000", 2024-04-12T11:40:51.542 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001742221 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.568999+0000", 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0020677600000000001 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.939086+0000", 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.370086839 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.939167+0000", 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0799999999999999e-05 2024-04-12T11:40:51.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:10.939235+0000", 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8090999999999996e-05 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:100270 6.2 6:427fccc1:::benchmark_data_smithi029_138569_object100269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:27.623979+0000", 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 383.89341124200001, 2024-04-12T11:40:51.544 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.45260584799999998, 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 100270 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.623979+0000", 2024-04-12T11:40:51.545 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.623979+0000", 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.623981+0000", 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4509999999999999e-06 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624035+0000", 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3378999999999998e-05 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624060+0000", 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5372e-05 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624074+0000", 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3611000000000001e-05 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.547 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624139+0000", 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5037999999999996e-05 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624262+0000", 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012352500000000001 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.624361+0000", 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8776000000000004e-05 2024-04-12T11:40:51.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:27.625898+0000", 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0015374049999999999 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:28.068661+0000", 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.44276276199999998 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:28.076436+0000", 2024-04-12T11:40:51.549 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0077744729999999996 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:28.076524+0000", 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8150000000000001e-05 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:28.076585+0000", 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0906000000000003e-05 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.550 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:104581 6.e 6:7bc98b91:::benchmark_data_smithi029_138569_object104580:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:33.512510+0000", 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 378.00487967200002, 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.65829958200000005, 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 104581 2024-04-12T11:40:51.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512510+0000", 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512510+0000", 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.552 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512512+0000", 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.113e-06 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512558+0000", 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6186999999999997e-05 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512574+0000", 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6209e-05 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512584+0000", 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7079999999999992e-06 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512748+0000", 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016473099999999999 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.512895+0000", 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014678900000000001 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.513040+0000", 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014458400000000001 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.516481+0000", 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034407140000000001 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.516562+0000", 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1169999999999997e-05 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170688+0000", 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.65412592700000005 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170757+0000", 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9015000000000002e-05 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170810+0000", 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3434999999999999e-05 2024-04-12T11:40:51.556 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:104586 6.e 6:7eb2d00d:::benchmark_data_smithi029_138569_object104585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:33.664460+0000", 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 377.85292971799998, 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50644868300000001, 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:40:51.557 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 104586 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664460+0000", 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664460+0000", 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664462+0000", 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1969999999999999e-06 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664536+0000", 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4225000000000002e-05 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664556+0000", 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9970999999999999e-05 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664565+0000", 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1290000000000002e-06 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664627+0000", 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1921999999999994e-05 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664780+0000", 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000153327 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.560 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664948+0000", 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000167553 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.671103+0000", 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0061548640000000003 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.671158+0000", 2024-04-12T11:40:51.561 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5615999999999997e-05 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170846+0000", 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.49968758699999999 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170882+0000", 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6291000000000002e-05 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170909+0000", 2024-04-12T11:40:51.562 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.7001000000000001e-05 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34776.0:104587 6.e 6:714ec73f:::benchmark_data_smithi029_138569_object104586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:34:33.664586+0000", 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 377.85280406300001, 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50648105600000004, 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.563 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34776", 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 104587 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664586+0000", 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664586+0000", 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664586+0000", 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9200000000000002e-07 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664613+0000", 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6037000000000002e-05 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664619+0000", 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3849999999999999e-06 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.664623+0000", 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0260000000000001e-06 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.667984+0000", 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0033609600000000001 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.668098+0000", 2024-04-12T11:40:51.566 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011424799999999999 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.668314+0000", 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021553699999999999 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.671214+0000", 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0029004730000000002 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:33.671333+0000", 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000119077 2024-04-12T11:40:51.567 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.170958+0000", 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.49962431400000001 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.171014+0000", 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6456999999999998e-05 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:34:34.171067+0000", 2024-04-12T11:40:51.568 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3149999999999998e-05 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91829 7.1 7:88cc5a79:::benchmark_data_smithi029_161157_object91828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1145)", 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:20.898486+0000", 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 150.61890441, 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.525263109, 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.569 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91829 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898486+0000", 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898486+0000", 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.570 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898487+0000", 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.299e-06 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898522+0000", 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4456999999999999e-05 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898535+0000", 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3852999999999999e-05 2024-04-12T11:40:51.571 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898545+0000", 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6579999999999997e-06 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898595+0000", 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.011e-05 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898706+0000", 2024-04-12T11:40:51.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000110948 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,6", 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898838+0000", 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013242500000000001 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.901924+0000", 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030852610000000002 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.902226+0000", 2024-04-12T11:40:51.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00030198699999999998 2024-04-12T11:40:51.574 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.574 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.574 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.916736+0000", 2024-04-12T11:40:51.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014510232 2024-04-12T11:40:51.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.575 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.916752+0000", 2024-04-12T11:40:51.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6283999999999999e-05 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.933963+0000", 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0172107859999999 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.933980+0000", 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7091999999999998e-05 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.423651+0000", 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.489671127 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.423749+0000", 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7590000000000006e-05 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91848 7.1 7:8f4d9a6c:::benchmark_data_smithi029_161157_object91847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:20.920205+0000", 2024-04-12T11:40:51.577 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 150.597184844, 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5272139290000002, 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91848 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.578 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920205+0000", 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920205+0000", 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920207+0000", 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2619999999999999e-06 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.579 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920258+0000", 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1115000000000001e-05 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920270+0000", 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1823e-05 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920278+0000", 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2840000000000006e-06 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.580 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920329+0000", 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1069000000000002e-05 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920338+0000", 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4599999999999992e-06 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934016+0000", 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.013677768 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934026+0000", 2024-04-12T11:40:51.581 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7170000000000003e-06 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.425246+0000", 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.491220121 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.425362+0000", 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000116205 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.582 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.425481+0000", 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000119164 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432241+0000", 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0067593490000000004 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432262+0000", 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1274e-05 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.444301+0000", 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012039253999999999 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.444363+0000", 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2181999999999995e-05 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.447419+0000", 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030558820000000002 2024-04-12T11:40:51.584 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91849 7.1 7:837125d8:::benchmark_data_smithi029_161157_object91848:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:20.922774+0000", 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 150.59461641999999, 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5270688610000001, 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.585 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91849 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922774+0000", 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922774+0000", 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.586 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922775+0000", 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.125e-06 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922842+0000", 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7133999999999998e-05 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922855+0000", 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3383e-05 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922864+0000", 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7199999999999995e-06 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922925+0000", 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0645e-05 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922936+0000", 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0784999999999999e-05 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.588 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934056+0000", 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0111205409999999 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934065+0000", 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2140000000000002e-06 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.427765+0000", 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4936998859999999 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.427915+0000", 2024-04-12T11:40:51.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014997 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.428074+0000", 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015929400000000001 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432270+0000", 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0041953989999999998 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432297+0000", 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6922000000000001e-05 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.447493+0000", 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015195818 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.447562+0000", 2024-04-12T11:40:51.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9117999999999998e-05 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.449843+0000", 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002280907 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91900 7.1 7:8d6ac7c8:::benchmark_data_smithi029_161157_object91899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:20.979397+0000", 2024-04-12T11:40:51.592 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 150.53799334499999, 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.504759467, 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91900 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979397+0000", 2024-04-12T11:40:51.593 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979397+0000", 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979399+0000", 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.046e-06 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.594 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979464+0000", 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4839999999999996e-05 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979488+0000", 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3971000000000001e-05 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979499+0000", 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1527e-05 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.595 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979583+0000", 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3651000000000003e-05 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979599+0000", 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5883999999999999e-05 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934095+0000", 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.95449584399999998 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934104+0000", 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.935e-06 2024-04-12T11:40:51.596 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.430164+0000", 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.496060712 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.430308+0000", 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014382800000000001 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.430440+0000", 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013224299999999999 2024-04-12T11:40:51.597 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.434349+0000", 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0039086800000000003 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.436571+0000", 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002221771 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.481423+0000", 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.044852114999999998 2024-04-12T11:40:51.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.481466+0000", 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3538000000000002e-05 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.484156+0000", 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0026898820000000002 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.599 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91933 7.1 7:876a3b0c:::benchmark_data_smithi029_161157_object91932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:21.013528+0000", 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 150.50386167799999, 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4845660560000002, 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91933 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.600 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013528+0000", 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013528+0000", 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013530+0000", 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.333e-06 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.601 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013594+0000", 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3899999999999995e-05 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013605+0000", 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1736000000000001e-05 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013614+0000", 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1680000000000004e-06 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.602 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013667+0000", 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3594000000000002e-05 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013677+0000", 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3479999999999993e-06 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934131+0000", 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.92045452299999997 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.603 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934140+0000", 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.884e-06 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432328+0000", 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.498187623 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432421+0000", 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3357999999999996e-05 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.604 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432559+0000", 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000137911 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.436613+0000", 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0040539169999999998 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.438759+0000", 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0021465730000000001 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.495581+0000", 2024-04-12T11:40:51.605 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.056821266000000002 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.495624+0000", 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3303999999999997e-05 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.498095+0000", 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0024706179999999999 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91943 7.1 7:840e712a:::benchmark_data_smithi029_161157_object91942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:21.022248+0000", 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 150.49514167999999, 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4783233920000001, 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91943 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.607 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022248+0000", 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022248+0000", 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022250+0000", 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.172e-06 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.608 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022295+0000", 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5086999999999997e-05 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022306+0000", 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1361e-05 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022312+0000", 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1990000000000003e-06 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022355+0000", 2024-04-12T11:40:51.609 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3071000000000001e-05 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022364+0000", 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.208e-06 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934169+0000", 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.91180522799999997 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934178+0000", 2024-04-12T11:40:51.610 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7849999999999997e-06 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.434370+0000", 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.500192212 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.434464+0000", 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3703999999999996e-05 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.434573+0000", 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000109305 2024-04-12T11:40:51.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441132+0000", 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0065594959999999997 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441144+0000", 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1362999999999999e-05 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.498165+0000", 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.057021297999999998 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.498238+0000", 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2608999999999995e-05 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.500572+0000", 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023342940000000002 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.613 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91952 7.1 7:87628a27:::benchmark_data_smithi029_161157_object91951:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:21.031078+0000", 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 150.486311949, 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4732867770000002, 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91952 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.614 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031078+0000", 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031078+0000", 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031082+0000", 2024-04-12T11:40:51.615 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1400000000000002e-06 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031126+0000", 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4004000000000001e-05 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031143+0000", 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6473e-05 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031152+0000", 2024-04-12T11:40:51.616 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0589999999999992e-06 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031209+0000", 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6789000000000001e-05 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031228+0000", 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9326000000000001e-05 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934207+0000", 2024-04-12T11:40:51.617 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.90297881000000002 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934226+0000", 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.914e-05 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.436644+0000", 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5024184229999999 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.618 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.436749+0000", 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010474 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.436857+0000", 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000107469 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441201+0000", 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0043440850000000001 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.619 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.444157+0000", 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0029562410000000001 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.502529+0000", 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.058372119 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.502572+0000", 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3195999999999998e-05 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.620 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.504365+0000", 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001792763 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91957 7.1 7:814dd879:::benchmark_data_smithi029_161157_object91956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:21.039025+0000", 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 150.47836541699999, 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4672004350000001, 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.621 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91957 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039025+0000", 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039025+0000", 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039029+0000", 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2100000000000003e-06 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039074+0000", 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4944000000000002e-05 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039093+0000", 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9565e-05 2024-04-12T11:40:51.623 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039104+0000", 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0574000000000001e-05 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039168+0000", 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3652999999999995e-05 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039177+0000", 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0529999999999996e-06 2024-04-12T11:40:51.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934258+0000", 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.89508132299999998 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934267+0000", 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9949999999999994e-06 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.625 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.438793+0000", 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5045254809999999 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.438898+0000", 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010555700000000001 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.439007+0000", 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000108614 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.626 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.444234+0000", 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0052272839999999996 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.447434+0000", 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0032003829999999998 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.504437+0000", 2024-04-12T11:40:51.627 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.057002355999999997 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.504479+0000", 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2432000000000003e-05 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.506225+0000", 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001746011 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.628 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91980 7.1 7:8a33a954:::benchmark_data_smithi029_161157_object91979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:21.175961+0000", 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 150.34142872999999, 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3331854490000001, 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91980 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.629 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.175961+0000", 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.175961+0000", 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.175963+0000", 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.03e-06 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.630 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.176020+0000", 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6162999999999997e-05 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.176046+0000", 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5987e-05 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.176060+0000", 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3899e-05 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.631 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.176133+0000", 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3001000000000001e-05 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.176143+0000", 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0210999999999999e-05 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934296+0000", 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.75815370999999998 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.632 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934305+0000", 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2700000000000004e-06 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441249+0000", 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5069440429999998 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441417+0000", 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000168658 2024-04-12T11:40:51.633 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441604+0000", 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000186718 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.449895+0000", 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0082906639999999997 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.449907+0000", 2024-04-12T11:40:51.634 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2516e-05 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.506306+0000", 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.056398252000000003 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.506375+0000", 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8980999999999996e-05 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.509147+0000", 2024-04-12T11:40:51.635 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027723460000000002 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:92943 7.1 7:845f5e9a:::benchmark_data_smithi029_161157_object92942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:24.698637+0000", 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 146.81875291700001, 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.372957066, 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.636 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 92943 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.698637+0000", 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.698637+0000", 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.698639+0000", 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.812e-06 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.698681+0000", 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2237000000000002e-05 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.638 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.698694+0000", 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2434e-05 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.698702+0000", 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9789999999999993e-06 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.709281+0000", 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.010579045 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.639 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.640 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.709441+0000", 2024-04-12T11:40:51.640 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016007400000000001 2024-04-12T11:40:51.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.640 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.640 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.640 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:51.641 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.709632+0000", 2024-04-12T11:40:51.641 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000190745 2024-04-12T11:40:51.641 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.641 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.641 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.641 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.719659+0000", 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.010027697 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.719671+0000", 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2072999999999999e-05 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:25.068259+0000", 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34858811299999998 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.642 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:25.068301+0000", 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1287e-05 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:25.071594+0000", 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00329357 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.643 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:92949 7.1 7:81f56ab2:::benchmark_data_smithi029_161157_object92948:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-04-12T11:40:51.644 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:24.706908+0000", 2024-04-12T11:40:51.644 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 146.810481791, 2024-04-12T11:40:51.644 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.37498089200000001, 2024-04-12T11:40:51.644 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.644 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.644 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.644 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.644 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 92949 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.706908+0000", 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.646 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.706908+0000", 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.706909+0000", 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1500000000000005e-07 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.707010+0000", 2024-04-12T11:40:51.647 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000100957 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.707021+0000", 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.148e-05 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.707029+0000", 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4250000000000001e-06 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.648 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.725346+0000", 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.018317508 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.725489+0000", 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014226500000000001 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.725603+0000", 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000114294 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.649 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.734865+0000", 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0092626470000000006 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.734884+0000", 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8589000000000001e-05 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:25.077102+0000", 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34221759499999999 2024-04-12T11:40:51.650 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:25.077164+0000", 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2735000000000002e-05 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:25.081889+0000", 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0047248819999999997 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.651 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:92955 7.1 7:8dc50e35:::benchmark_data_smithi029_161157_object92954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:24.713206+0000", 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 146.804184612, 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.37101812200000001, 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 92955 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:40:51.652 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.713206+0000", 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.713206+0000", 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.713207+0000", 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.221e-06 2024-04-12T11:40:51.653 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.713258+0000", 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1348999999999999e-05 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.713271+0000", 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3315e-05 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:40:51.654 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.713278+0000", 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7129999999999999e-06 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.728131+0000", 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014852743999999999 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.728280+0000", 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014954500000000001 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:40:51.655 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.728434+0000", 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015404700000000001 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.737931+0000", 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0094970190000000006 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.737972+0000", 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0126999999999997e-05 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.656 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:25.081952+0000", 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34398051000000002 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:25.081992+0000", 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9647000000000002e-05 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:25.084224+0000", 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0022318849999999999 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.657 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.658 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.658 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:40:51.658 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:40:51.658 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:40:51.705 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:51.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:51.807 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:51.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:51.858 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:51.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:51.959 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:51.990 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:51.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:51.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:52.012 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:40:52.065 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:40:52.091 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:52.103 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:40:52.120 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:52.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:52.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:52.144 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:40:52.186 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:40:52.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:52 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1289206147' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:52.222 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:52.242 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:40:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:52 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1289206147' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:52.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:40:52.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:52 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1289206147' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:40:52.270 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:52.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:52.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:52.279 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:40:52.330 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:40:52.342 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:40:52.342 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:40:52.342 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:40:52.342 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:40:52.342 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:40:52.342 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:40:52.342 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:40:52.342 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:40:52.373 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:52.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:52.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:52.407 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-04-12T11:40:52.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:52.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:52.553 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:52.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:52.653 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:52.653 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:52.684 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:52.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:52.716 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:40:52.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:40:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:40:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:40:52.786 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:52.812 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:52.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:52.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:52.915 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:40:52.916 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:52.936 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:40:52.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:52.962 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:52.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:53.063 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:53.108 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:53.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:53.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:53.173 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:53 smithi087 ceph-mon[89014]: pgmap v1621: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5951/78741 objects degraded (7.558%); 422/78741 objects misplaced (0.536%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T11:40:53.173 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/78741 objects degraded (7.558%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:53.210 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:53.249 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:53.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:53.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:53.351 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:53.392 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:53.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:53.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:53 smithi029 ceph-mon[97787]: pgmap v1621: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5951/78741 objects degraded (7.558%); 422/78741 objects misplaced (0.536%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T11:40:53.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/78741 objects degraded (7.558%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:53.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:53 smithi029 ceph-mon[93650]: pgmap v1621: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 5951/78741 objects degraded (7.558%); 422/78741 objects misplaced (0.536%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T11:40:53.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/78741 objects degraded (7.558%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:53.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:53.545 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:53.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:53.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:53.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:53.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:53.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:53.824 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:53.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:53.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:53.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:53.963 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:53.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:53.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:54.065 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:54.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:54.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:54.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:54.197 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:54.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:54.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:54.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:54.289 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:54 smithi087 ceph-mon[89014]: from='client.54769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:40:54.346 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:54.390 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:54.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:54.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:54.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:54.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:54 smithi029 ceph-mon[93650]: from='client.54769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:40:54.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:54 smithi029 ceph-mon[97787]: from='client.54769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:40:54.540 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:54.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:54.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:54.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:54.673 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:54.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:54.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:54.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:54.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:54.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:54.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:54.916 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:54.953 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:54.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:54.968 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:40:54.968 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:40:55.055 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:55.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:55.103 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:55.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:55.204 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:55.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:55 smithi087 ceph-mon[89014]: pgmap v1622: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 5951/73023 objects degraded (8.149%); 422/73023 objects misplaced (0.578%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T11:40:55.231 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:55.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:55.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:55.248 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:55 smithi029 ceph-mon[93650]: pgmap v1622: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 5951/73023 objects degraded (8.149%); 422/73023 objects misplaced (0.578%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T11:40:55.248 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:40:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:40:54] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2024-04-12T11:40:55.248 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:55 smithi029 ceph-mon[97787]: pgmap v1622: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 5951/73023 objects degraded (8.149%); 422/73023 objects misplaced (0.578%); 3.0 MiB/s, 48 objects/s recovering 2024-04-12T11:40:55.333 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:55.361 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:55.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:55.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:55.463 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:55.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:40:55.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:55.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:55.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:40:55.592 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:55.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:55.619 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:55.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:55.629 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1621,"stamp":"2024-04-12T11:40:51.813423+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1719080938,"num_objects":26247,"num_object_clones":0,"num_object_copies":78741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5951,"num_objects_misplaced":422,"num_objects_unfound":0,"num_objects_dirty":26247,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":189079,"num_write_kb":6882447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4993,"num_bytes_recovered":328338156,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139395,"ondisk_log_size":139395,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8303428,"kb_used_data":5869052,"kb_used_omap":175,"kb_used_meta":2433872,"kb_avail":741657788,"statfs":{"total":767960285184,"available":759457574912,"internally_reserved":0,"allocated":6009909248,"data_stored":5985396381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":2492285097},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,6,4,8,8,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-430309376,"num_objects":-6566,"num_object_clones":0,"num_object_copies":-19698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-481,"num_objects_unfound":0,"num_objects_dirty":-6566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6566,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":481,"num_bytes_recovered":31522816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005932"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1674,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.017524+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:40:48.017524+0000","last_peered":"2024-04-12T11:40:48.017524+0000","last_clean":"2024-04-12T11:40:48.017524+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:40:48.017524+0000","last_undegraded":"2024-04-12T11:40:48.017524+0000","last_fullsized":"2024-04-12T11:40:48.017524+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1683,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.040168+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:40:48.040168+0000","last_peered":"2024-04-12T11:40:48.040168+0000","last_clean":"2024-04-12T11:40:48.040168+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:40:48.040168+0000","last_undegraded":"2024-04-12T11:40:48.040168+0000","last_fullsized":"2024-04-12T11:40:48.040168+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1734,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.032821+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:40:48.032821+0000","last_peered":"2024-04-12T11:40:48.032821+0000","last_clean":"2024-04-12T11:40:48.032821+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:40:48.032821+0000","last_undegraded":"2024-04-12T11:40:48.032821+0000","last_fullsized":"2024-04-12T11:40:48.032821+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1742,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.036984+0000","last_change":"2024-04-12T11:38:21.933647+0000","last_active":"2024-04-12T11:40:48.036984+0000","last_peered":"2024-04-12T11:40:48.036984+0000","last_clean":"2024-04-12T11:40:48.036984+0000","last_became_active":"2024-04-12T11:38:21.933295+0000","last_became_peered":"2024-04-12T11:38:21.933295+0000","last_unstale":"2024-04-12T11:40:48.036984+0000","last_undegraded":"2024-04-12T11:40:48.036984+0000","last_fullsized":"2024-04-12T11:40:48.036984+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:37.402139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1691,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.043366+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:40:48.043366+0000","last_peered":"2024-04-12T11:40:48.043366+0000","last_clean":"2024-04-12T11:40:48.043366+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:40:48.043366+0000","last_undegraded":"2024-04-12T11:40:48.043366+0000","last_fullsized":"2024-04-12T11:40:48.043366+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1105,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.223828+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:40:16.223828+0000","last_peered":"2024-04-12T11:40:16.223828+0000","last_clean":"2024-04-12T11:40:16.223828+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:40:16.223828+0000","last_undegraded":"2024-04-12T11:40:16.223828+0000","last_fullsized":"2024-04-12T11:40:16.223828+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1725,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.037389+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:40:48.037389+0000","last_peered":"2024-04-12T11:40:48.037389+0000","last_clean":"2024-04-12T11:40:48.037389+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:40:48.037389+0000","last_undegraded":"2024-04-12T11:40:48.037389+0000","last_fullsized":"2024-04-12T11:40:48.037389+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1664,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.043512+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:40:48.043512+0000","last_peered":"2024-04-12T11:40:48.043512+0000","last_clean":"2024-04-12T11:40:48.043512+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:40:48.043512+0000","last_undegraded":"2024-04-12T11:40:48.043512+0000","last_fullsized":"2024-04-12T11:40:48.043512+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1110,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.215794+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:40:16.215794+0000","last_peered":"2024-04-12T11:40:16.215794+0000","last_clean":"2024-04-12T11:40:16.215794+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:40:16.215794+0000","last_undegraded":"2024-04-12T11:40:16.215794+0000","last_fullsized":"2024-04-12T11:40:16.215794+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1644,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.040538+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:40:48.040538+0000","last_peered":"2024-04-12T11:40:48.040538+0000","last_clean":"2024-04-12T11:40:48.040538+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:40:48.040538+0000","last_undegraded":"2024-04-12T11:40:48.040538+0000","last_fullsized":"2024-04-12T11:40:48.040538+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1119,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.230182+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:40:16.230182+0000","last_peered":"2024-04-12T11:40:16.230182+0000","last_clean":"2024-04-12T11:40:16.230182+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:40:16.230182+0000","last_undegraded":"2024-04-12T11:40:16.230182+0000","last_fullsized":"2024-04-12T11:40:16.230182+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1725,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.032045+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:40:48.032045+0000","last_peered":"2024-04-12T11:40:48.032045+0000","last_clean":"2024-04-12T11:40:48.032045+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:40:48.032045+0000","last_undegraded":"2024-04-12T11:40:48.032045+0000","last_fullsized":"2024-04-12T11:40:48.032045+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1647,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.016935+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:40:48.016935+0000","last_peered":"2024-04-12T11:40:48.016935+0000","last_clean":"2024-04-12T11:40:48.016935+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:40:48.016935+0000","last_undegraded":"2024-04-12T11:40:48.016935+0000","last_fullsized":"2024-04-12T11:40:48.016935+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1638,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.017842+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:40:48.017842+0000","last_peered":"2024-04-12T11:40:48.017842+0000","last_clean":"2024-04-12T11:40:48.017842+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:40:48.017842+0000","last_undegraded":"2024-04-12T11:40:48.017842+0000","last_fullsized":"2024-04-12T11:40:48.017842+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1118,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.229959+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:40:16.229959+0000","last_peered":"2024-04-12T11:40:16.229959+0000","last_clean":"2024-04-12T11:40:16.229959+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:40:16.229959+0000","last_undegraded":"2024-04-12T11:40:16.229959+0000","last_fullsized":"2024-04-12T11:40:16.229959+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1675,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.042878+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:40:48.042878+0000","last_peered":"2024-04-12T11:40:48.042878+0000","last_clean":"2024-04-12T11:40:48.042878+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:40:48.042878+0000","last_undegraded":"2024-04-12T11:40:48.042878+0000","last_fullsized":"2024-04-12T11:40:48.042878+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1727,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.035515+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:40:48.035515+0000","last_peered":"2024-04-12T11:40:48.035515+0000","last_clean":"2024-04-12T11:40:48.035515+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:40:48.035515+0000","last_undegraded":"2024-04-12T11:40:48.035515+0000","last_fullsized":"2024-04-12T11:40:48.035515+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1109,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.215526+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:40:16.215526+0000","last_peered":"2024-04-12T11:40:16.215526+0000","last_clean":"2024-04-12T11:40:16.215526+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:40:16.215526+0000","last_undegraded":"2024-04-12T11:40:16.215526+0000","last_fullsized":"2024-04-12T11:40:16.215526+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1101,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.229848+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:40:16.229848+0000","last_peered":"2024-04-12T11:40:16.229848+0000","last_clean":"2024-04-12T11:40:16.229848+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:40:16.229848+0000","last_undegraded":"2024-04-12T11:40:16.229848+0000","last_fullsized":"2024-04-12T11:40:16.229848+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1727,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.036904+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:40:48.036904+0000","last_peered":"2024-04-12T11:40:48.036904+0000","last_clean":"2024-04-12T11:40:48.036904+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:40:48.036904+0000","last_undegraded":"2024-04-12T11:40:48.036904+0000","last_fullsized":"2024-04-12T11:40:48.036904+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1112,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.220540+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:40:16.220540+0000","last_peered":"2024-04-12T11:40:16.220540+0000","last_clean":"2024-04-12T11:40:16.220540+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:40:16.220540+0000","last_undegraded":"2024-04-12T11:40:16.220540+0000","last_fullsized":"2024-04-12T11:40:16.220540+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1675,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.041753+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:40:48.041753+0000","last_peered":"2024-04-12T11:40:48.041753+0000","last_clean":"2024-04-12T11:40:48.041753+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:40:48.041753+0000","last_undegraded":"2024-04-12T11:40:48.041753+0000","last_fullsized":"2024-04-12T11:40:48.041753+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1161'11858","reported_seq":11925,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.417211+0000","last_change":"2024-04-12T11:37:03.057199+0000","last_active":"2024-04-12T11:40:48.417211+0000","last_peered":"2024-04-12T11:40:48.417211+0000","last_clean":"2024-04-12T11:40:48.417211+0000","last_became_active":"2024-04-12T11:37:03.056776+0000","last_became_peered":"2024-04-12T11:37:03.056776+0000","last_unstale":"2024-04-12T11:40:48.417211+0000","last_undegraded":"2024-04-12T11:40:48.417211+0000","last_fullsized":"2024-04-12T11:40:48.417211+0000","mapping_epoch":1133,"log_start":"1141'3900","ondisk_log_start":"1141'3900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7958,"log_dups_size":0,"ondisk_log_size":7958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:16:29.272872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110231552,"num_objects":1682,"num_object_clones":0,"num_object_copies":5046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11858,"num_write_kb":433280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1118,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.230373+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:40:16.230373+0000","last_peered":"2024-04-12T11:40:16.230373+0000","last_clean":"2024-04-12T11:40:16.230373+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:40:16.230373+0000","last_undegraded":"2024-04-12T11:40:16.230373+0000","last_fullsized":"2024-04-12T11:40:16.230373+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1095,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.215317+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:40:16.215317+0000","last_peered":"2024-04-12T11:40:16.215317+0000","last_clean":"2024-04-12T11:40:16.215317+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:40:16.215317+0000","last_undegraded":"2024-04-12T11:40:16.215317+0000","last_fullsized":"2024-04-12T11:40:16.215317+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1161'11928","reported_seq":11995,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:50.275563+0000","last_change":"2024-04-12T11:37:03.059071+0000","last_active":"2024-04-12T11:40:50.275563+0000","last_peered":"2024-04-12T11:40:50.275563+0000","last_clean":"2024-04-12T11:40:50.275563+0000","last_became_active":"2024-04-12T11:37:03.058769+0000","last_became_peered":"2024-04-12T11:37:03.058769+0000","last_unstale":"2024-04-12T11:40:50.275563+0000","last_undegraded":"2024-04-12T11:40:50.275563+0000","last_fullsized":"2024-04-12T11:40:50.275563+0000","mapping_epoch":1133,"log_start":"1142'4400","ondisk_log_start":"1142'4400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7528,"log_dups_size":0,"ondisk_log_size":7528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:43:26.795096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106954752,"num_objects":1632,"num_object_clones":0,"num_object_copies":4896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11928,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1119,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.230286+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:40:16.230286+0000","last_peered":"2024-04-12T11:40:16.230286+0000","last_clean":"2024-04-12T11:40:16.230286+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:40:16.230286+0000","last_undegraded":"2024-04-12T11:40:16.230286+0000","last_fullsized":"2024-04-12T11:40:16.230286+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1660,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.015653+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:40:48.015653+0000","last_peered":"2024-04-12T11:40:48.015653+0000","last_clean":"2024-04-12T11:40:48.015653+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:40:48.015653+0000","last_undegraded":"2024-04-12T11:40:48.015653+0000","last_fullsized":"2024-04-12T11:40:48.015653+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1161'11844","reported_seq":11911,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.727602+0000","last_change":"2024-04-12T11:37:03.059390+0000","last_active":"2024-04-12T11:40:48.727602+0000","last_peered":"2024-04-12T11:40:48.727602+0000","last_clean":"2024-04-12T11:40:48.727602+0000","last_became_active":"2024-04-12T11:37:03.058045+0000","last_became_peered":"2024-04-12T11:37:03.058045+0000","last_unstale":"2024-04-12T11:40:48.727602+0000","last_undegraded":"2024-04-12T11:40:48.727602+0000","last_fullsized":"2024-04-12T11:40:48.727602+0000","mapping_epoch":1133,"log_start":"1141'3000","ondisk_log_start":"1141'3000","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8844,"log_dups_size":0,"ondisk_log_size":8844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:14:01.284893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106299392,"num_objects":1622,"num_object_clones":0,"num_object_copies":4866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11844,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1088,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.230103+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:40:16.230103+0000","last_peered":"2024-04-12T11:40:16.230103+0000","last_clean":"2024-04-12T11:40:16.230103+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:40:16.230103+0000","last_undegraded":"2024-04-12T11:40:16.230103+0000","last_fullsized":"2024-04-12T11:40:16.230103+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1104,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.220507+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:40:16.220507+0000","last_peered":"2024-04-12T11:40:16.220507+0000","last_clean":"2024-04-12T11:40:16.220507+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:40:16.220507+0000","last_undegraded":"2024-04-12T11:40:16.220507+0000","last_fullsized":"2024-04-12T11:40:16.220507+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1161'11967","reported_seq":12034,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:49.975652+0000","last_change":"2024-04-12T11:37:03.054920+0000","last_active":"2024-04-12T11:40:49.975652+0000","last_peered":"2024-04-12T11:40:49.975652+0000","last_clean":"2024-04-12T11:40:49.975652+0000","last_became_active":"2024-04-12T11:37:03.054603+0000","last_became_peered":"2024-04-12T11:37:03.054603+0000","last_unstale":"2024-04-12T11:40:49.975652+0000","last_undegraded":"2024-04-12T11:40:49.975652+0000","last_fullsized":"2024-04-12T11:40:49.975652+0000","mapping_epoch":1133,"log_start":"1140'1900","ondisk_log_start":"1140'1900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10067,"log_dups_size":0,"ondisk_log_size":10067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:34:45.149813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107544576,"num_objects":1641,"num_object_clones":0,"num_object_copies":4923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11967,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1111,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.215401+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:40:16.215401+0000","last_peered":"2024-04-12T11:40:16.215401+0000","last_clean":"2024-04-12T11:40:16.215401+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:40:16.215401+0000","last_undegraded":"2024-04-12T11:40:16.215401+0000","last_fullsized":"2024-04-12T11:40:16.215401+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1737,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.035728+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:40:48.035728+0000","last_peered":"2024-04-12T11:40:48.035728+0000","last_clean":"2024-04-12T11:40:48.035728+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:40:48.035728+0000","last_undegraded":"2024-04-12T11:40:48.035728+0000","last_fullsized":"2024-04-12T11:40:48.035728+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1161'11973","reported_seq":12039,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.723757+0000","last_change":"2024-04-12T11:37:03.063209+0000","last_active":"2024-04-12T11:40:48.723757+0000","last_peered":"2024-04-12T11:40:48.723757+0000","last_clean":"2024-04-12T11:40:48.723757+0000","last_became_active":"2024-04-12T11:37:03.062848+0000","last_became_peered":"2024-04-12T11:37:03.062848+0000","last_unstale":"2024-04-12T11:40:48.723757+0000","last_undegraded":"2024-04-12T11:40:48.723757+0000","last_fullsized":"2024-04-12T11:40:48.723757+0000","mapping_epoch":1133,"log_start":"1141'3100","ondisk_log_start":"1141'3100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8873,"log_dups_size":0,"ondisk_log_size":8873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:55:26.299047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107413504,"num_objects":1639,"num_object_clones":0,"num_object_copies":4917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11973,"num_write_kb":435584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1668,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.041635+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:40:48.041635+0000","last_peered":"2024-04-12T11:40:48.041635+0000","last_clean":"2024-04-12T11:40:48.041635+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:40:48.041635+0000","last_undegraded":"2024-04-12T11:40:48.041635+0000","last_fullsized":"2024-04-12T11:40:48.041635+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1651,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.017650+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:40:48.017650+0000","last_peered":"2024-04-12T11:40:48.017650+0000","last_clean":"2024-04-12T11:40:48.017650+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:40:48.017650+0000","last_undegraded":"2024-04-12T11:40:48.017650+0000","last_fullsized":"2024-04-12T11:40:48.017650+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1161'11745","reported_seq":11812,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:49.870696+0000","last_change":"2024-04-12T11:37:03.060739+0000","last_active":"2024-04-12T11:40:49.870696+0000","last_peered":"2024-04-12T11:40:49.870696+0000","last_clean":"2024-04-12T11:40:49.870696+0000","last_became_active":"2024-04-12T11:37:03.060414+0000","last_became_peered":"2024-04-12T11:37:03.060414+0000","last_unstale":"2024-04-12T11:40:49.870696+0000","last_undegraded":"2024-04-12T11:40:49.870696+0000","last_fullsized":"2024-04-12T11:40:49.870696+0000","mapping_epoch":1133,"log_start":"1140'1700","ondisk_log_start":"1140'1700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10045,"log_dups_size":0,"ondisk_log_size":10045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:12:30.259488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103088128,"num_objects":1573,"num_object_clones":0,"num_object_copies":4719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11745,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":677,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.017432+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:40:48.017432+0000","last_peered":"2024-04-12T11:40:48.017432+0000","last_clean":"2024-04-12T11:40:48.017432+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:40:48.017432+0000","last_undegraded":"2024-04-12T11:40:48.017432+0000","last_fullsized":"2024-04-12T11:40:48.017432+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1673,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.016730+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:40:48.016730+0000","last_peered":"2024-04-12T11:40:48.016730+0000","last_clean":"2024-04-12T11:40:48.016730+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:40:48.016730+0000","last_undegraded":"2024-04-12T11:40:48.016730+0000","last_fullsized":"2024-04-12T11:40:48.016730+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1160'11927","reported_seq":11965,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-04-12T11:40:47.314626+0000","last_change":"2024-04-12T11:37:03.060364+0000","last_active":"2024-04-12T11:40:47.314626+0000","last_peered":"2024-04-12T11:40:47.314626+0000","last_clean":"2024-04-12T11:40:47.314626+0000","last_became_active":"2024-04-12T11:37:03.060146+0000","last_became_peered":"2024-04-12T11:37:03.060146+0000","last_unstale":"2024-04-12T11:40:47.314626+0000","last_undegraded":"2024-04-12T11:40:47.314626+0000","last_fullsized":"2024-04-12T11:40:47.314626+0000","mapping_epoch":1133,"log_start":"1140'2500","ondisk_log_start":"1140'2500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9427,"log_dups_size":0,"ondisk_log_size":9427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:03.796242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110231576,"num_objects":1683,"num_object_clones":0,"num_object_copies":5049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1683,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11927,"num_write_kb":435457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1659,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.015731+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:40:48.015731+0000","last_peered":"2024-04-12T11:40:48.015731+0000","last_clean":"2024-04-12T11:40:48.015731+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:40:48.015731+0000","last_undegraded":"2024-04-12T11:40:48.015731+0000","last_fullsized":"2024-04-12T11:40:48.015731+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1638,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.017318+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:40:48.017318+0000","last_peered":"2024-04-12T11:40:48.017318+0000","last_clean":"2024-04-12T11:40:48.017318+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:40:48.017318+0000","last_undegraded":"2024-04-12T11:40:48.017318+0000","last_fullsized":"2024-04-12T11:40:48.017318+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1161'12013","reported_seq":12080,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:50.253234+0000","last_change":"2024-04-12T11:37:03.056864+0000","last_active":"2024-04-12T11:40:50.253234+0000","last_peered":"2024-04-12T11:40:50.253234+0000","last_clean":"2024-04-12T11:40:50.253234+0000","last_became_active":"2024-04-12T11:37:03.056523+0000","last_became_peered":"2024-04-12T11:37:03.056523+0000","last_unstale":"2024-04-12T11:40:50.253234+0000","last_undegraded":"2024-04-12T11:40:50.253234+0000","last_fullsized":"2024-04-12T11:40:50.253234+0000","mapping_epoch":1133,"log_start":"1142'4500","ondisk_log_start":"1142'4500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7513,"log_dups_size":0,"ondisk_log_size":7513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:57:58.750385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105709568,"num_objects":1613,"num_object_clones":0,"num_object_copies":4839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12013,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1677,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.043617+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:40:48.043617+0000","last_peered":"2024-04-12T11:40:48.043617+0000","last_clean":"2024-04-12T11:40:48.043617+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:40:48.043617+0000","last_undegraded":"2024-04-12T11:40:48.043617+0000","last_fullsized":"2024-04-12T11:40:48.043617+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1735,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.032302+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:40:48.032302+0000","last_peered":"2024-04-12T11:40:48.032302+0000","last_clean":"2024-04-12T11:40:48.032302+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:40:48.032302+0000","last_undegraded":"2024-04-12T11:40:48.032302+0000","last_fullsized":"2024-04-12T11:40:48.032302+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1161'11716","reported_seq":21752,"reported_epoch":1161,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-04-12T11:40:49.889569+0000","last_change":"2024-04-12T11:39:24.599690+0000","last_active":"2024-04-12T11:40:49.889569+0000","last_peered":"2024-04-12T11:40:49.889569+0000","last_clean":"2024-04-12T11:38:20.888121+0000","last_became_active":"2024-04-12T11:38:23.417421+0000","last_became_peered":"2024-04-12T11:38:23.417421+0000","last_unstale":"2024-04-12T11:40:49.889569+0000","last_undegraded":"2024-04-12T11:39:24.597597+0000","last_fullsized":"2024-04-12T11:38:23.247711+0000","mapping_epoch":1147,"log_start":"1141'3800","ondisk_log_start":"1141'3800","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7916,"log_dups_size":0,"ondisk_log_size":7916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100401152,"num_objects":1532,"num_object_clones":0,"num_object_copies":4596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":310,"num_objects_misplaced":422,"num_objects_unfound":0,"num_objects_dirty":1532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11716,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4983,"num_bytes_recovered":326565888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,6],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4,6","objects":422},{"shards":"4,6","objects":800}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1736,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.033441+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:40:48.033441+0000","last_peered":"2024-04-12T11:40:48.033441+0000","last_clean":"2024-04-12T11:40:48.033441+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:40:48.033441+0000","last_undegraded":"2024-04-12T11:40:48.033441+0000","last_fullsized":"2024-04-12T11:40:48.033441+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1127,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:17.337441+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:40:17.337441+0000","last_peered":"2024-04-12T11:40:17.337441+0000","last_clean":"2024-04-12T11:40:17.337441+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:40:17.337441+0000","last_undegraded":"2024-04-12T11:40:17.337441+0000","last_fullsized":"2024-04-12T11:40:17.337441+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1727,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.031880+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:40:48.031880+0000","last_peered":"2024-04-12T11:40:48.031880+0000","last_clean":"2024-04-12T11:40:48.031880+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:40:48.031880+0000","last_undegraded":"2024-04-12T11:40:48.031880+0000","last_fullsized":"2024-04-12T11:40:48.031880+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1160'11669","reported_seq":11706,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-04-12T11:40:47.384094+0000","last_change":"2024-04-12T11:37:03.060583+0000","last_active":"2024-04-12T11:40:47.384094+0000","last_peered":"2024-04-12T11:40:47.384094+0000","last_clean":"2024-04-12T11:40:47.384094+0000","last_became_active":"2024-04-12T11:37:03.060125+0000","last_became_peered":"2024-04-12T11:37:03.060125+0000","last_unstale":"2024-04-12T11:40:47.384094+0000","last_undegraded":"2024-04-12T11:40:47.384094+0000","last_fullsized":"2024-04-12T11:40:47.384094+0000","mapping_epoch":1133,"log_start":"1140'2200","ondisk_log_start":"1140'2200","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9469,"log_dups_size":0,"ondisk_log_size":9469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:35:30.859950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117768192,"num_objects":1797,"num_object_clones":0,"num_object_copies":5391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11669,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1116,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.215611+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:40:16.215611+0000","last_peered":"2024-04-12T11:40:16.215611+0000","last_clean":"2024-04-12T11:40:16.215611+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:40:16.215611+0000","last_undegraded":"2024-04-12T11:40:16.215611+0000","last_fullsized":"2024-04-12T11:40:16.215611+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1660,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.017688+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:40:48.017688+0000","last_peered":"2024-04-12T11:40:48.017688+0000","last_clean":"2024-04-12T11:40:48.017688+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:40:48.017688+0000","last_undegraded":"2024-04-12T11:40:48.017688+0000","last_fullsized":"2024-04-12T11:40:48.017688+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1736,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.033227+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:40:48.033227+0000","last_peered":"2024-04-12T11:40:48.033227+0000","last_clean":"2024-04-12T11:40:48.033227+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:40:48.033227+0000","last_undegraded":"2024-04-12T11:40:48.033227+0000","last_fullsized":"2024-04-12T11:40:48.033227+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1161'11616","reported_seq":11683,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.766224+0000","last_change":"2024-04-12T11:37:03.058470+0000","last_active":"2024-04-12T11:40:48.766224+0000","last_peered":"2024-04-12T11:40:48.766224+0000","last_clean":"2024-04-12T11:40:48.766224+0000","last_became_active":"2024-04-12T11:37:03.058149+0000","last_became_peered":"2024-04-12T11:37:03.058149+0000","last_unstale":"2024-04-12T11:40:48.766224+0000","last_undegraded":"2024-04-12T11:40:48.766224+0000","last_fullsized":"2024-04-12T11:40:48.766224+0000","mapping_epoch":1133,"log_start":"1140'2700","ondisk_log_start":"1140'2700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8916,"log_dups_size":0,"ondisk_log_size":8916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:25:06.934877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106692608,"num_objects":1628,"num_object_clones":0,"num_object_copies":4884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11616,"num_write_kb":423808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1700,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.031335+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:40:48.031335+0000","last_peered":"2024-04-12T11:40:48.031335+0000","last_clean":"2024-04-12T11:40:48.031335+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:40:48.031335+0000","last_undegraded":"2024-04-12T11:40:48.031335+0000","last_fullsized":"2024-04-12T11:40:48.031335+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1728,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.031880+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:40:48.031880+0000","last_peered":"2024-04-12T11:40:48.031880+0000","last_clean":"2024-04-12T11:40:48.031880+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:40:48.031880+0000","last_undegraded":"2024-04-12T11:40:48.031880+0000","last_fullsized":"2024-04-12T11:40:48.031880+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1730,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.032309+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:40:48.032309+0000","last_peered":"2024-04-12T11:40:48.032309+0000","last_clean":"2024-04-12T11:40:48.032309+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:40:48.032309+0000","last_undegraded":"2024-04-12T11:40:48.032309+0000","last_fullsized":"2024-04-12T11:40:48.032309+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1161'11643","reported_seq":11713,"reported_epoch":1161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:40:48.676176+0000","last_change":"2024-04-12T11:38:23.438040+0000","last_active":"2024-04-12T11:40:48.676176+0000","last_peered":"2024-04-12T11:40:48.676176+0000","last_clean":"2024-04-12T11:38:20.888095+0000","last_became_active":"2024-04-12T11:38:23.424268+0000","last_became_peered":"2024-04-12T11:38:23.424268+0000","last_unstale":"2024-04-12T11:40:48.676176+0000","last_undegraded":"2024-04-12T11:38:23.284099+0000","last_fullsized":"2024-04-12T11:38:23.245558+0000","mapping_epoch":1147,"log_start":"1140'2800","ondisk_log_start":"1140'2800","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8843,"log_dups_size":0,"ondisk_log_size":8843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109379584,"num_objects":1669,"num_object_clones":0,"num_object_copies":5007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11643,"num_write_kb":425984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5,7","objects":654},{"shards":"0,7","objects":-3972}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1116,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.216410+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:40:16.216410+0000","last_peered":"2024-04-12T11:40:16.216410+0000","last_clean":"2024-04-12T11:40:16.216410+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:40:16.216410+0000","last_undegraded":"2024-04-12T11:40:16.216410+0000","last_fullsized":"2024-04-12T11:40:16.216410+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1471,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.229829+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:40:16.229829+0000","last_peered":"2024-04-12T11:40:16.229829+0000","last_clean":"2024-04-12T11:40:16.229829+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:40:16.229829+0000","last_undegraded":"2024-04-12T11:40:16.229829+0000","last_fullsized":"2024-04-12T11:40:16.229829+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1644,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.040980+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:40:48.040980+0000","last_peered":"2024-04-12T11:40:48.040980+0000","last_clean":"2024-04-12T11:40:48.040980+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:40:48.040980+0000","last_undegraded":"2024-04-12T11:40:48.040980+0000","last_fullsized":"2024-04-12T11:40:48.040980+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1105,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.220589+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:40:16.220589+0000","last_peered":"2024-04-12T11:40:16.220589+0000","last_clean":"2024-04-12T11:40:16.220589+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:40:16.220589+0000","last_undegraded":"2024-04-12T11:40:16.220589+0000","last_fullsized":"2024-04-12T11:40:16.220589+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1160'11565","reported_seq":11602,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-04-12T11:40:47.450457+0000","last_change":"2024-04-12T11:37:03.061698+0000","last_active":"2024-04-12T11:40:47.450457+0000","last_peered":"2024-04-12T11:40:47.450457+0000","last_clean":"2024-04-12T11:40:47.450457+0000","last_became_active":"2024-04-12T11:37:03.061469+0000","last_became_peered":"2024-04-12T11:37:03.061469+0000","last_unstale":"2024-04-12T11:40:47.450457+0000","last_undegraded":"2024-04-12T11:40:47.450457+0000","last_fullsized":"2024-04-12T11:40:47.450457+0000","mapping_epoch":1133,"log_start":"1141'2900","ondisk_log_start":"1141'2900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8665,"log_dups_size":0,"ondisk_log_size":8665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:09.906340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109510656,"num_objects":1671,"num_object_clones":0,"num_object_copies":5013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11565,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1124,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.229764+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:40:16.229764+0000","last_peered":"2024-04-12T11:40:16.229764+0000","last_clean":"2024-04-12T11:40:16.229764+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:40:16.229764+0000","last_undegraded":"2024-04-12T11:40:16.229764+0000","last_fullsized":"2024-04-12T11:40:16.229764+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1738,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.036670+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:40:48.036670+0000","last_peered":"2024-04-12T11:40:48.036670+0000","last_clean":"2024-04-12T11:40:48.036670+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:40:48.036670+0000","last_undegraded":"2024-04-12T11:40:48.036670+0000","last_fullsized":"2024-04-12T11:40:48.036670+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1737,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.036811+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:40:48.036811+0000","last_peered":"2024-04-12T11:40:48.036811+0000","last_clean":"2024-04-12T11:40:48.036811+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:40:48.036811+0000","last_undegraded":"2024-04-12T11:40:48.036811+0000","last_fullsized":"2024-04-12T11:40:48.036811+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"1160'11750","reported_seq":11787,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-04-12T11:40:47.297128+0000","last_change":"2024-04-12T11:37:03.060455+0000","last_active":"2024-04-12T11:40:47.297128+0000","last_peered":"2024-04-12T11:40:47.297128+0000","last_clean":"2024-04-12T11:40:47.297128+0000","last_became_active":"2024-04-12T11:37:03.060244+0000","last_became_peered":"2024-04-12T11:37:03.060244+0000","last_unstale":"2024-04-12T11:40:47.297128+0000","last_undegraded":"2024-04-12T11:40:47.297128+0000","last_fullsized":"2024-04-12T11:40:47.297128+0000","mapping_epoch":1133,"log_start":"1140'1700","ondisk_log_start":"1140'1700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10050,"log_dups_size":0,"ondisk_log_size":10050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:47.787368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114294784,"num_objects":1744,"num_object_clones":0,"num_object_copies":5232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11750,"num_write_kb":431808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1123,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.230079+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:40:16.230079+0000","last_peered":"2024-04-12T11:40:16.230079+0000","last_clean":"2024-04-12T11:40:16.230079+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:40:16.230079+0000","last_undegraded":"2024-04-12T11:40:16.230079+0000","last_fullsized":"2024-04-12T11:40:16.230079+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1727,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.033397+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:40:48.033397+0000","last_peered":"2024-04-12T11:40:48.033397+0000","last_clean":"2024-04-12T11:40:48.033397+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:40:48.033397+0000","last_undegraded":"2024-04-12T11:40:48.033397+0000","last_fullsized":"2024-04-12T11:40:48.033397+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1721,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.035966+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:40:48.035966+0000","last_peered":"2024-04-12T11:40:48.035966+0000","last_clean":"2024-04-12T11:40:48.035966+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:40:48.035966+0000","last_undegraded":"2024-04-12T11:40:48.035966+0000","last_fullsized":"2024-04-12T11:40:48.035966+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1161'11647","reported_seq":11714,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:50.165804+0000","last_change":"2024-04-12T11:37:03.057084+0000","last_active":"2024-04-12T11:40:50.165804+0000","last_peered":"2024-04-12T11:40:50.165804+0000","last_clean":"2024-04-12T11:40:50.165804+0000","last_became_active":"2024-04-12T11:37:03.056772+0000","last_became_peered":"2024-04-12T11:37:03.056772+0000","last_unstale":"2024-04-12T11:40:50.165804+0000","last_undegraded":"2024-04-12T11:40:50.165804+0000","last_fullsized":"2024-04-12T11:40:50.165804+0000","mapping_epoch":1133,"log_start":"1142'4100","ondisk_log_start":"1142'4100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7547,"log_dups_size":0,"ondisk_log_size":7547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:02:41.845790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100728832,"num_objects":1537,"num_object_clones":0,"num_object_copies":4611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11647,"num_write_kb":421888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1660,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.041651+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:40:48.041651+0000","last_peered":"2024-04-12T11:40:48.041651+0000","last_clean":"2024-04-12T11:40:48.041651+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:40:48.041651+0000","last_undegraded":"2024-04-12T11:40:48.041651+0000","last_fullsized":"2024-04-12T11:40:48.041651+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1727,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.032133+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:40:48.032133+0000","last_peered":"2024-04-12T11:40:48.032133+0000","last_clean":"2024-04-12T11:40:48.032133+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:40:48.032133+0000","last_undegraded":"2024-04-12T11:40:48.032133+0000","last_fullsized":"2024-04-12T11:40:48.032133+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1657,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.043753+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:40:48.043753+0000","last_peered":"2024-04-12T11:40:48.043753+0000","last_clean":"2024-04-12T11:40:48.043753+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:40:48.043753+0000","last_undegraded":"2024-04-12T11:40:48.043753+0000","last_fullsized":"2024-04-12T11:40:48.043753+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1161'11918","reported_seq":11985,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:50.289878+0000","last_change":"2024-04-12T11:37:03.060299+0000","last_active":"2024-04-12T11:40:50.289878+0000","last_peered":"2024-04-12T11:40:50.289878+0000","last_clean":"2024-04-12T11:40:50.289878+0000","last_became_active":"2024-04-12T11:37:03.058897+0000","last_became_peered":"2024-04-12T11:37:03.058897+0000","last_unstale":"2024-04-12T11:40:50.289878+0000","last_undegraded":"2024-04-12T11:40:50.289878+0000","last_fullsized":"2024-04-12T11:40:50.289878+0000","mapping_epoch":1133,"log_start":"1142'4400","ondisk_log_start":"1142'4400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:45:37.844216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102236160,"num_objects":1560,"num_object_clones":0,"num_object_copies":4680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11918,"num_write_kb":431296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1129,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:16.223782+0000","last_change":"2024-04-12T11:39:23.349899+0000","last_active":"2024-04-12T11:40:16.223782+0000","last_peered":"2024-04-12T11:40:16.223782+0000","last_clean":"2024-04-12T11:40:16.223782+0000","last_became_active":"2024-04-12T11:39:23.349526+0000","last_became_peered":"2024-04-12T11:39:23.349526+0000","last_unstale":"2024-04-12T11:40:16.223782+0000","last_undegraded":"2024-04-12T11:40:16.223782+0000","last_fullsized":"2024-04-12T11:40:16.223782+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:36:47.115912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1740,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.037063+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:40:48.037063+0000","last_peered":"2024-04-12T11:40:48.037063+0000","last_clean":"2024-04-12T11:40:48.037063+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:40:48.037063+0000","last_undegraded":"2024-04-12T11:40:48.037063+0000","last_fullsized":"2024-04-12T11:40:48.037063+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1647,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.015403+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:40:48.015403+0000","last_peered":"2024-04-12T11:40:48.015403+0000","last_clean":"2024-04-12T11:40:48.015403+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:40:48.015403+0000","last_undegraded":"2024-04-12T11:40:48.015403+0000","last_fullsized":"2024-04-12T11:40:48.015403+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1730,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.035869+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:40:48.035869+0000","last_peered":"2024-04-12T11:40:48.035869+0000","last_clean":"2024-04-12T11:40:48.035869+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:40:48.035869+0000","last_undegraded":"2024-04-12T11:40:48.035869+0000","last_fullsized":"2024-04-12T11:40:48.035869+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":46,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.029201+0000","last_change":"2024-04-12T11:38:06.839832+0000","last_active":"2024-04-12T11:40:48.029201+0000","last_peered":"2024-04-12T11:40:48.029201+0000","last_clean":"2024-04-12T11:40:48.029201+0000","last_became_active":"2024-04-12T11:38:06.839460+0000","last_became_peered":"2024-04-12T11:38:06.839460+0000","last_unstale":"2024-04-12T11:40:48.029201+0000","last_undegraded":"2024-04-12T11:40:48.029201+0000","last_fullsized":"2024-04-12T11:40:48.029201+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:25:27.959933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1127,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-04-12T11:40:27.342391+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:40:27.342391+0000","last_peered":"2024-04-12T11:40:27.342391+0000","last_clean":"2024-04-12T11:40:27.342391+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:40:27.342391+0000","last_undegraded":"2024-04-12T11:40:27.342391+0000","last_fullsized":"2024-04-12T11:40:27.342391+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1674,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.017964+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:40:48.017964+0000","last_peered":"2024-04-12T11:40:48.017964+0000","last_clean":"2024-04-12T11:40:48.017964+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:40:48.017964+0000","last_undegraded":"2024-04-12T11:40:48.017964+0000","last_fullsized":"2024-04-12T11:40:48.017964+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1112,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-04-12T11:40:28.930473+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:40:28.930473+0000","last_peered":"2024-04-12T11:40:28.930473+0000","last_clean":"2024-04-12T11:40:28.930473+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:40:28.930473+0000","last_undegraded":"2024-04-12T11:40:28.930473+0000","last_fullsized":"2024-04-12T11:40:28.930473+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1665,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.015071+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:40:48.015071+0000","last_peered":"2024-04-12T11:40:48.015071+0000","last_clean":"2024-04-12T11:40:48.015071+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:40:48.015071+0000","last_undegraded":"2024-04-12T11:40:48.015071+0000","last_fullsized":"2024-04-12T11:40:48.015071+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1658,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.044111+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:40:48.044111+0000","last_peered":"2024-04-12T11:40:48.044111+0000","last_clean":"2024-04-12T11:40:48.044111+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:40:48.044111+0000","last_undegraded":"2024-04-12T11:40:48.044111+0000","last_fullsized":"2024-04-12T11:40:48.044111+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1669,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.040392+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:40:48.040392+0000","last_peered":"2024-04-12T11:40:48.040392+0000","last_clean":"2024-04-12T11:40:48.040392+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:40:48.040392+0000","last_undegraded":"2024-04-12T11:40:48.040392+0000","last_fullsized":"2024-04-12T11:40:48.040392+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1759,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.037475+0000","last_change":"2024-04-12T11:38:06.866508+0000","last_active":"2024-04-12T11:40:48.037475+0000","last_peered":"2024-04-12T11:40:48.037475+0000","last_clean":"2024-04-12T11:40:48.037475+0000","last_became_active":"2024-04-12T11:38:06.843752+0000","last_became_peered":"2024-04-12T11:38:06.843752+0000","last_unstale":"2024-04-12T11:40:48.037475+0000","last_undegraded":"2024-04-12T11:40:48.037475+0000","last_fullsized":"2024-04-12T11:40:48.037475+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:18:11.827424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1638,"reported_epoch":1161,"state":"active+clean","last_fresh":"2024-04-12T11:40:48.019679+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:40:48.019679+0000","last_peered":"2024-04-12T11:40:48.019679+0000","last_clean":"2024-04-12T11:40:48.019679+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:40:48.019679+0000","last_undegraded":"2024-04-12T11:40:48.019679+0000","last_fullsized":"2024-04-12T11:40:48.019679+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1718485016,"num_objects":26223,"num_object_clones":0,"num_object_copies":78669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5951,"num_objects_misplaced":422,"num_objects_unfound":0,"num_objects_dirty":26223,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":188779,"num_write_kb":6880001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4983,"num_bytes_recovered":326565888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5850017792,"data_stored":5850005576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139179,"ondisk_log_size":139179,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234341,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":880548,"kb_used_data":564828,"kb_used_omap":1,"kb_used_meta":315710,"kb_avail":92864604,"statfs":{"total":95995035648,"available":95093354496,"internally_reserved":0,"allocated":578383872,"data_stored":575307014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":323287499},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495978,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770716,"kb_used_data":559188,"kb_used_omap":1,"kb_used_meta":211518,"kb_avail":92974436,"statfs":{"total":95995035648,"available":95205822464,"internally_reserved":0,"allocated":572608512,"data_stored":569539724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":216594888},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536288,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1300744,"kb_used_data":1025792,"kb_used_omap":30,"kb_used_meta":274849,"kb_avail":92444408,"statfs":{"total":95995035648,"available":94663073792,"internally_reserved":0,"allocated":1050411008,"data_stored":1047357342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":281445698},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190372,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1185044,"kb_used_data":869580,"kb_used_omap":43,"kb_used_meta":315220,"kb_avail":92560108,"statfs":{"total":95995035648,"available":94781550592,"internally_reserved":0,"allocated":890449920,"data_stored":887385114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":322785323},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,2,4,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849500,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000028,"kb_used_data":634832,"kb_used_omap":36,"kb_used_meta":365147,"kb_avail":92745124,"statfs":{"total":95995035648,"available":94971006976,"internally_reserved":0,"allocated":650067968,"data_stored":647006943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":373910708},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704853,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1290968,"kb_used_data":880204,"kb_used_omap":17,"kb_used_meta":410734,"kb_avail":92454184,"statfs":{"total":95995035648,"available":94673084416,"internally_reserved":0,"allocated":901328896,"data_stored":898271441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":420592119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,1,4,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018065,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":753960,"kb_used_data":514976,"kb_used_omap":42,"kb_used_meta":238933,"kb_avail":92991192,"statfs":{"total":95995035648,"available":95222980608,"internally_reserved":0,"allocated":527335424,"data_stored":524262301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":244668216},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757614,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1121420,"kb_used_data":819652,"kb_used_omap":1,"kb_used_meta":301758,"kb_avail":92623732,"statfs":{"total":95995035648,"available":94846701568,"internally_reserved":0,"allocated":839323648,"data_stored":836266502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":309000646},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":863637504,"data_stored":863633432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1026166784,"data_stored":1026162712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":625344512,"data_stored":625344512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":873529344,"data_stored":873529344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":516358144,"data_stored":516358144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":822476800,"data_stored":822476800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":561516544,"data_stored":561512472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":560988160,"data_stored":560988160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:40:55.634 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-04-12T11:40:55.720 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:55.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:55.781 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:55.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:55.882 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:40:55.911 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:55.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:55.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:40:55.932 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:40:56.013 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:56.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:56.051 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:56.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:56.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:56.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:56.201 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:56.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:56.246 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:40:56.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:56.355 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:56.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:56.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:56.457 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:56.485 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:56.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:56.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:56.587 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:56.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:56.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:56.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:56.715 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:56.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:56.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:56.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:56.840 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:56.866 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:56.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:56.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:56.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:57.008 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:57.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:57.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:57.064 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:40:57.110 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:57.138 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:57.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:57.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:57.240 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:57.280 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:57.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:57.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:57.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:57.408 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:57 smithi087 ceph-mon[89014]: pgmap v1623: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5951/68652 objects degraded (8.668%); 160/68652 objects misplaced (0.233%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:40:57.408 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:57 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3732960054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:40:57.425 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:57.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:57.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:57.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:57 smithi029 ceph-mon[97787]: pgmap v1623: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5951/68652 objects degraded (8.668%); 160/68652 objects misplaced (0.233%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:40:57.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:57 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3732960054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:40:57.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:57 smithi029 ceph-mon[93650]: pgmap v1623: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 5951/68652 objects degraded (8.668%); 160/68652 objects misplaced (0.233%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:40:57.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:57 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3732960054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:40:57.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:57.551 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:57.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:57.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:57.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:57.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:57.692 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:57.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:57.793 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:57.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:57.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:57.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:57.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:57.987 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:57.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:57.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:58.049 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:40:58.049 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:40:58.090 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:40:58.120 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:58.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:58.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:40:58.222 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:40:58.260 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:58.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:58.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:40:58.362 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:58.375 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:58 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:40:58.375 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:58 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3732960054' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:40:58.375 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:58 smithi087 ceph-mon[89014]: osdmap e1162: 8 total, 8 up, 8 in 2024-04-12T11:40:58.375 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5951/68652 objects degraded (8.668%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:58.385 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:58.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:58.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:58.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:40:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:58 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:40:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:58 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3732960054' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:40:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:58 smithi029 ceph-mon[97787]: osdmap e1162: 8 total, 8 up, 8 in 2024-04-12T11:40:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5951/68652 objects degraded (8.668%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:58.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:58 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:40:58.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:58 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3732960054' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:40:58.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:58 smithi029 ceph-mon[93650]: osdmap e1162: 8 total, 8 up, 8 in 2024-04-12T11:40:58.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5951/68652 objects degraded (8.668%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:40:58.530 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:58.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:58.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:40:58.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:58.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:40:58.656 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:58.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:58.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:58.720 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-04-12T11:40:58.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:58.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:58.798 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:58.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:58.899 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:58.948 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:58.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:58.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:58.976 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:40:59.051 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:59.084 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:59.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:59.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:59.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:59.241 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:59.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:59.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:59.346 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:40:59 smithi087 ceph-mon[89014]: pgmap v1625: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 5951/66135 objects degraded (8.998%); 160/66135 objects misplaced (0.242%); 3.1 MiB/s, 50 objects/s recovering 2024-04-12T11:40:59.390 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:59.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:59.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:40:59.515 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:59.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:59.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:40:59.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:40:59 smithi029 ceph-mon[97787]: pgmap v1625: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 5951/66135 objects degraded (8.998%); 160/66135 objects misplaced (0.242%); 3.1 MiB/s, 50 objects/s recovering 2024-04-12T11:40:59.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:40:59 smithi029 ceph-mon[93650]: pgmap v1625: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 5951/66135 objects degraded (8.998%); 160/66135 objects misplaced (0.242%); 3.1 MiB/s, 50 objects/s recovering 2024-04-12T11:40:59.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:40:59.659 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:40:59.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:59.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:40:59.761 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:40:59.786 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:59.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:59.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:40:59.888 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:40:59.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:40:59.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:40:59.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:00.013 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:00.040 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:00.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:00.142 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:00.180 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:00.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:00.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:00.282 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:00.308 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:00.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:00.409 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:00.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:00.460 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:00.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:00.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:00.600 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:00.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:00.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:00.702 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:00.736 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:00.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:00.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:00.879 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:00.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:00.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:01.010 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:01.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:01.112 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:01.139 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:01.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:01.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:01.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:01.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:01.279 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:01.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:01.380 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:01.401 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:01 smithi087 ceph-mon[89014]: pgmap v1626: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 5951/63303 objects degraded (9.401%); 160/63303 objects misplaced (0.253%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:41:01.403 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:01.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:01.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:01.505 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:01.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:01 smithi029 ceph-mon[93650]: pgmap v1626: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 5951/63303 objects degraded (9.401%); 160/63303 objects misplaced (0.253%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:41:01.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:01 smithi029 ceph-mon[97787]: pgmap v1626: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 5951/63303 objects degraded (9.401%); 160/63303 objects misplaced (0.253%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:41:01.533 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:01.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:01.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:01.634 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:01.661 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:01.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:01.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:01.763 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:01.801 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:01.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:01.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:01.903 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:01.936 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:01.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:01.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:02.038 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:02.061 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:02.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:02.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:02.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:02.203 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:02.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:02.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:02.222 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:41:02.222 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:41:02.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:02.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:02.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:02.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:02.438 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:02.466 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:41:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:41:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:41:02.479 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:02.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:02.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:02.581 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:02.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:02.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:02.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:02.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:02.757 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:02.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:02.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:02.858 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:02.891 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:02.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:02.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:02.993 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:03.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:03.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:03.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:03.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:03.147 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:03.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:03.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:03.249 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:03.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:03.281 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:03.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:03.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:03.396 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:03 smithi087 ceph-mon[89014]: pgmap v1627: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 5804/59358 objects degraded (9.778%); 2.6 MiB/s, 56 objects/s recovering 2024-04-12T11:41:03.396 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5804/59358 objects degraded (9.778%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:41:03.396 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:03 smithi087 ceph-mon[89014]: osdmap e1163: 8 total, 8 up, 8 in 2024-04-12T11:41:03.414 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:03.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:03.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:03.424 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-04-12T11:41:03.516 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:03.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:03 smithi029 ceph-mon[97787]: pgmap v1627: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 5804/59358 objects degraded (9.778%); 2.6 MiB/s, 56 objects/s recovering 2024-04-12T11:41:03.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5804/59358 objects degraded (9.778%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:41:03.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:03 smithi029 ceph-mon[97787]: osdmap e1163: 8 total, 8 up, 8 in 2024-04-12T11:41:03.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:03 smithi029 ceph-mon[93650]: pgmap v1627: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 5804/59358 objects degraded (9.778%); 2.6 MiB/s, 56 objects/s recovering 2024-04-12T11:41:03.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5804/59358 objects degraded (9.778%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:41:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:03 smithi029 ceph-mon[93650]: osdmap e1163: 8 total, 8 up, 8 in 2024-04-12T11:41:03.541 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:03.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:03.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:03.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:03.654 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:41:03.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:03.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:03.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:03.768 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:03.791 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:03.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:03.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:03.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:03.918 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:03.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:03.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:04.020 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:04.047 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:04.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:04.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:04.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:04.188 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:04.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:04.290 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:04.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:04.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:04.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:04.419 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:04.463 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:04.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:04.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:04.565 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:04.593 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:04.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:04.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:04.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:04.735 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:04.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:04.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:04.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:04.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:04.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:04.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:04.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:05.006 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:05.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:05.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:05.108 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:05.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:05.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:05.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:05.165 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:04 smithi087 ceph-mon[89014]: pgmap v1629: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 5696/54669 objects degraded (10.419%); 65/54669 objects misplaced (0.119%); 1.6 MiB/s, 43 objects/s recovering 2024-04-12T11:41:05.165 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:04 smithi087 ceph-mon[89014]: osdmap e1164: 8 total, 8 up, 8 in 2024-04-12T11:41:05.236 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:04 smithi029 ceph-mon[97787]: pgmap v1629: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 5696/54669 objects degraded (10.419%); 65/54669 objects misplaced (0.119%); 1.6 MiB/s, 43 objects/s recovering 2024-04-12T11:41:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:05 smithi029 ceph-mon[97787]: osdmap e1164: 8 total, 8 up, 8 in 2024-04-12T11:41:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:04 smithi029 ceph-mon[93650]: pgmap v1629: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 5696/54669 objects degraded (10.419%); 65/54669 objects misplaced (0.119%); 1.6 MiB/s, 43 objects/s recovering 2024-04-12T11:41:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:05 smithi029 ceph-mon[93650]: osdmap e1164: 8 total, 8 up, 8 in 2024-04-12T11:41:05.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:41:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:41:04] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2024-04-12T11:41:05.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:05.281 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:05.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:05.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:05.420 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:05.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:05.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:05.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:05.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:05.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:05.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:05.585 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:41:05.585 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:41:05.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:05.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:05.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:05.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:05.793 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:05.836 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:05.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:05.937 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:05.974 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:05.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:05.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:06.075 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:06.101 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:06.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:06.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:06.203 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:06.219 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-04-12T11:41:06.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:06.359 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:06.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:06.460 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:06.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:06.489 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:06.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:06.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:06.625 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:41:06.635 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:06.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:06.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:06.737 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:06.767 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:06.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:06.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:06.869 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:06.898 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:06.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:06.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:07.000 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:07.013 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:41:07.024 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:07.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:07.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:07.036 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:06 smithi087 ceph-mon[89014]: pgmap v1631: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5533/50922 objects degraded (10.866%); 65/50922 objects misplaced (0.128%); 1.6 MiB/s, 64 objects/s recovering 2024-04-12T11:41:07.036 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:07.126 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:07.156 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:07.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:07.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:07.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:06 smithi029 ceph-mon[93650]: pgmap v1631: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5533/50922 objects degraded (10.866%); 65/50922 objects misplaced (0.128%); 1.6 MiB/s, 64 objects/s recovering 2024-04-12T11:41:07.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:07.166 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:06 smithi029 ceph-mon[97787]: pgmap v1631: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 5533/50922 objects degraded (10.866%); 65/50922 objects misplaced (0.128%); 1.6 MiB/s, 64 objects/s recovering 2024-04-12T11:41:07.166 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:07.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:07.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:07.302 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:07.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:07.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:07.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:07.452 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:07.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:07.553 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:07.579 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:07.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:07.681 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:07.705 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:07.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:07.807 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:07.829 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:41:07.833 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:07.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:07.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:07.877 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:41:07.916 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:41:07.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:07.973 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:07 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3820784559' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:41:07.973 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:07 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5533/50922 objects degraded (10.866%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:07.973 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:07 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3820784559' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:41:07.973 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:07 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5533/50922 objects degraded (10.866%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:07.976 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:41:07.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:07.987 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:07.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:08.031 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:41:08.068 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:41:08.089 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:08.113 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:41:08.129 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:08.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:08.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:08.149 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:41:08.156 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:41:08.156 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:41:08.156 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:41:08.156 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:41:08.156 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:41:08.157 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:41:08.157 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:41:08.157 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:41:08.207 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:07 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3820784559' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:41:08.207 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:07 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5533/50922 objects degraded (10.866%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:08.231 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:08.258 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:08.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:08.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:08.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:08.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:41:08.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:08.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:08.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:08.492 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:08.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:08.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:08.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:08.621 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:08.673 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:08.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:08.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:08.775 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:08.807 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:08.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:08.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:08.908 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:08.949 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:08.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:08.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:08.988 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:41:08.988 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:41:08.988 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:41:08.988 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:41:08.988 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:41:09.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:09.078 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:09.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:09.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:09.093 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:08 smithi029 ceph-mon[93650]: pgmap v1632: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1013 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5533/48684 objects degraded (11.365%); 65/48684 objects misplaced (0.134%); 1.6 MiB/s, 64 objects/s recovering 2024-04-12T11:41:09.093 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:08 smithi029 ceph-mon[97787]: pgmap v1632: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1013 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5533/48684 objects degraded (11.365%); 65/48684 objects misplaced (0.134%); 1.6 MiB/s, 64 objects/s recovering 2024-04-12T11:41:09.180 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:09.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:08 smithi087 ceph-mon[89014]: pgmap v1632: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 1013 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5533/48684 objects degraded (11.365%); 65/48684 objects misplaced (0.134%); 1.6 MiB/s, 64 objects/s recovering 2024-04-12T11:41:09.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:09.222 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:09.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:09.323 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:09.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:09.352 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:09.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:09.452 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:09.491 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:09.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:09.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:09.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:09.668 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:09.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:09.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:09.681 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:41:09.770 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:09.788 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-04-12T11:41:09.822 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:09.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:09.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:09.951 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:09.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:09.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:10.053 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:10.056 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:41:10.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:10.084 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:10.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:10.185 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:10.212 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:10.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:10.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:10.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:10.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:10.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:10.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:10.445 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:10.482 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:10.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:10.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:10.583 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:10.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:10.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:10.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:10.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:10.758 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:10.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:10.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:10.860 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:10.899 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:41:10.910 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:10.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:10.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:10.917 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:41:11.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:11.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:11.038 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:11.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:11.127 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:41:11.139 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:11.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:10 smithi029 ceph-mon[97787]: pgmap v1633: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 950 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 5516/45636 objects degraded (12.087%); 720 KiB/s, 35 objects/s recovering 2024-04-12T11:41:11.164 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:10 smithi029 ceph-mon[93650]: pgmap v1633: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 950 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 5516/45636 objects degraded (12.087%); 720 KiB/s, 35 objects/s recovering 2024-04-12T11:41:11.166 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:11.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:11.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:11.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:10 smithi087 ceph-mon[89014]: pgmap v1633: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 950 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 5516/45636 objects degraded (12.087%); 720 KiB/s, 35 objects/s recovering 2024-04-12T11:41:11.268 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:11.304 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:11.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:11.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:11.406 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:11.433 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:11.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:11.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:11.535 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:11.559 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:11.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:11.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:11.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:11.696 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:11.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:11.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:11.798 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:11.824 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:11.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:11.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:11.926 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:12585 7.7 7:e84009a8:::benchmark_data_smithi029_161157_object12584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1138)", 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:19.843855+0000", 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 232.08483973099999, 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.209079931, 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.934 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 12585 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.843855+0000", 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.935 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.843855+0000", 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.843854+0000", 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.843927+0000", 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2599000000000001e-05 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.936 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.843938+0000", 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1235e-05 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.843946+0000", 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6899999999999992e-06 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.844736+0000", 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00079023600000000004 2024-04-12T11:41:11.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.844838+0000", 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000102277 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.844950+0000", 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000112067 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.938 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.845441+0000", 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00049118199999999995 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.846978+0000", 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001536279 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.052861+0000", 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.20588293899999999 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:11.939 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.052906+0000", 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5688e-05 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.052935+0000", 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.8756e-05 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:12609 7.7 7:e77912c7:::benchmark_data_smithi029_161157_object12608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1138)", 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:19.851192+0000", 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 232.07750287499999, 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.207150639, 2024-04-12T11:41:11.940 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 12609 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.851192+0000", 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.941 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.851192+0000", 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.851191+0000", 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.851710+0000", 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00051894699999999996 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.942 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.851722+0000", 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1915e-05 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.851729+0000", 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5050000000000001e-06 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.851766+0000", 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6699000000000002e-05 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.943 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.851891+0000", 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000125518 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.852039+0000", 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000147262 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.852752+0000", 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00071391200000000003 2024-04-12T11:41:11.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:19.862099+0000", 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0093461229999999996 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.058264+0000", 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.19616587899999999 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.058309+0000", 2024-04-12T11:41:11.945 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4412000000000001e-05 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.058343+0000", 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3643000000000003e-05 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:13903 7.7 7:e8c19d2e:::benchmark_data_smithi029_161157_object13902:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1138)", 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:20.506224+0000", 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 231.42247085100001, 2024-04-12T11:41:11.946 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34592572399999999, 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 13903 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.506224+0000", 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.506224+0000", 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.506222+0000", 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.948 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.506269+0000", 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6273000000000002e-05 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.506280+0000", 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1296e-05 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.506288+0000", 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8429999999999993e-06 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.949 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.506326+0000", 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8445000000000002e-05 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.506423+0000", 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6812999999999996e-05 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.506515+0000", 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2007000000000002e-05 2024-04-12T11:41:11.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.507301+0000", 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00078562200000000003 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.846516+0000", 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.33921549699999998 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.852040+0000", 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0055236360000000002 2024-04-12T11:41:11.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.852107+0000", 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7195999999999998e-05 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:20.852150+0000", 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2580999999999998e-05 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.952 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:14503 7.d 7:b2a03582:::benchmark_data_smithi029_161157_object14502:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1138)", 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:21.063511+0000", 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 230.86518358699999, 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27278770000000002, 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 14503 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.953 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.063511+0000", 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.063511+0000", 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.063510+0000", 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.954 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.063561+0000", 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1128000000000002e-05 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.063571+0000", 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.5179999999999993e-06 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.063577+0000", 2024-04-12T11:41:11.955 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.613e-06 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.063608+0000", 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.0922e-05 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.063704+0000", 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.5592000000000002e-05 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1", 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.063806+0000", 2024-04-12T11:41:11.956 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010231899999999999 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.064525+0000", 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00071937800000000001 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.066263+0000", 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001738091 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.336169+0000", 2024-04-12T11:41:11.957 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.269905642 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.336238+0000", 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8956999999999997e-05 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.336299+0000", 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0849e-05 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.958 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:14519 7.d 7:b9b46c6b:::benchmark_data_smithi029_161157_object14518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1138)", 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:21.068208+0000", 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 230.860486482, 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.296369033, 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 14519 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.959 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.068208+0000", 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.068208+0000", 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.068207+0000", 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:41:11.960 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.961 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.961 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.961 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.068299+0000", 2024-04-12T11:41:11.961 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2273000000000002e-05 2024-04-12T11:41:11.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.961 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.068313+0000", 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4e-05 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.068322+0000", 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7849999999999997e-06 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.068358+0000", 2024-04-12T11:41:11.962 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6338999999999999e-05 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.068456+0000", 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7132000000000004e-05 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1", 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.068555+0000", 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9486000000000003e-05 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.963 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.069209+0000", 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00065362299999999999 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.337027+0000", 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.26781800700000002 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.364438+0000", 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.027411570999999999 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.964 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.364540+0000", 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000101549 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:21.364577+0000", 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.7460999999999999e-05 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.965 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:15731 7.d 7:b0df7cde:::benchmark_data_smithi029_161157_object15730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1138)", 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:22.100472+0000", 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 229.82822238, 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.221573137, 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 15731 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.966 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.100472+0000", 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.100472+0000", 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.100471+0000", 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.967 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.100540+0000", 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8816e-05 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.100551+0000", 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.095e-05 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.100559+0000", 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9650000000000008e-06 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.100613+0000", 2024-04-12T11:41:11.968 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3859000000000001e-05 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.100728+0000", 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000114831 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1", 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.100867+0000", 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013898900000000001 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.101775+0000", 2024-04-12T11:41:11.969 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00090797800000000004 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.104889+0000", 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003113775 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.321941+0000", 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.217052721 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.970 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.322002+0000", 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0430999999999998e-05 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.322046+0000", 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3778e-05 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.971 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:15733 7.d 7:b48d4403:::benchmark_data_smithi029_161157_object15732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1138)", 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:22.103241+0000", 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 229.825453928, 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.21890654900000001, 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 15733 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.972 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.103241+0000", 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.103241+0000", 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.103240+0000", 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.973 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.103294+0000", 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4440000000000001e-05 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.103306+0000", 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2437e-05 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.103321+0000", 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4802e-05 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.974 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.103355+0000", 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3439999999999998e-05 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.103448+0000", 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3424999999999994e-05 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1", 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.103539+0000", 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0648000000000004e-05 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.975 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.104347+0000", 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00080837300000000003 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.105584+0000", 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001236441 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.322076+0000", 2024-04-12T11:41:11.976 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.21649285200000001 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.322112+0000", 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5389000000000003e-05 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.322147+0000", 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5542e-05 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.977 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:15742 7.d 7:bc78ea03:::benchmark_data_smithi029_161157_object15741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1138)", 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:22.106415+0000", 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 229.82227955900001, 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.21581285, 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 15742 2024-04-12T11:41:11.978 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.106415+0000", 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.106415+0000", 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.106414+0000", 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:41:11.979 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.106470+0000", 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6357000000000003e-05 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.106481+0000", 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0961000000000001e-05 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.106487+0000", 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7610000000000004e-06 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.980 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.106513+0000", 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6282999999999999e-05 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.106594+0000", 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0678000000000002e-05 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1", 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.106680+0000", 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5614999999999999e-05 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.107252+0000", 2024-04-12T11:41:11.981 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00057187099999999995 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.107622+0000", 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00037082800000000002 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.322177+0000", 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.214554096 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.322203+0000", 2024-04-12T11:41:11.982 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6064999999999999e-05 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:22.322228+0000", 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5437999999999999e-05 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:23078 7.d 7:bd089a29:::benchmark_data_smithi029_161157_object23077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1140)", 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:37:25.841012+0000", 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 226.087683046, 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.25870395000000002, 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.983 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 23078 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.841012+0000", 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.841012+0000", 2024-04-12T11:41:11.984 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.841011+0000", 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.841071+0000", 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0374000000000002e-05 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.841080+0000", 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6430000000000001e-06 2024-04-12T11:41:11.985 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.841085+0000", 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0900000000000004e-06 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.841115+0000", 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.0069e-05 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.841223+0000", 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000108121 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.986 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.987 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1", 2024-04-12T11:41:11.987 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.841360+0000", 2024-04-12T11:41:11.987 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013723000000000001 2024-04-12T11:41:11.987 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.987 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.987 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:11.987 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.842344+0000", 2024-04-12T11:41:11.987 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00098332400000000005 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:25.846510+0000", 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0041664700000000002 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:26.099530+0000", 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.253020248 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:26.099668+0000", 2024-04-12T11:41:11.988 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000138238 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:37:26.099716+0000", 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7203999999999997e-05 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:92620 7.7 7:e8f3e6b0:::benchmark_data_smithi029_161157_object92619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:24.135611+0000", 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 167.793084128, 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.210545606, 2024-04-12T11:41:11.989 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 92620 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.135611+0000", 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.990 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.135611+0000", 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.135609+0000", 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.991 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.135653+0000", 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3795000000000003e-05 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.135667+0000", 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3783e-05 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.135675+0000", 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.943e-06 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.992 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.135722+0000", 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7577999999999997e-05 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.135874+0000", 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015140100000000001 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.136031+0000", 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015712699999999999 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.993 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.138928+0000", 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0028969899999999999 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.148819+0000", 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0098915200000000009 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.346087+0000", 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.197267373 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.994 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.346127+0000", 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0037999999999999e-05 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:24.346156+0000", 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.9660000000000001e-05 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:94363 7.7 7:e50094f3:::benchmark_data_smithi029_161157_object94362:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:27.428497+0000", 2024-04-12T11:41:11.995 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 164.50019741599999, 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.208201578, 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 94363 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.428497+0000", 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.996 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.428497+0000", 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.428496+0000", 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.428577+0000", 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0646e-05 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.997 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.428594+0000", 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7606999999999999e-05 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.428605+0000", 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.03e-05 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.428649+0000", 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4919999999999997e-05 2024-04-12T11:41:11.998 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.428747+0000", 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7868000000000006e-05 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.428845+0000", 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7300999999999996e-05 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.431370+0000", 2024-04-12T11:41:11.999 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0025256300000000001 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.624120+0000", 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.19275015500000001 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.636569+0000", 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012449042 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:12.000 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.636654+0000", 2024-04-12T11:41:12.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:12.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:12.002 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4436000000000003e-05 2024-04-12T11:41:12.002 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.002 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.002 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:12.002 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:27.636699+0000", 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5006000000000002e-05 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:115576 7.7 7:e48fa596:::benchmark_data_smithi029_161157_object8073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:56.354859+0000", 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 135.57383604399999, 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.25508292700000001, 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:12.003 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 115576 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.354859+0000", 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.354859+0000", 2024-04-12T11:41:12.004 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.354857+0000", 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.354866+0000", 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1509999999999993e-06 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.354887+0000", 2024-04-12T11:41:12.005 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1044e-05 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.354897+0000", 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0254999999999999e-05 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.354952+0000", 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4577999999999998e-05 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.006 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.355091+0000", 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000139052 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.355229+0000", 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000137467 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.358312+0000", 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030831470000000001 2024-04-12T11:41:12.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.580935+0000", 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.22262293699999999 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.609844+0000", 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.028909038000000001 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.008 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.609897+0000", 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3752000000000003e-05 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:56.609942+0000", 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4257e-05 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.009 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:117457 7.7 7:ec15890c:::benchmark_data_smithi029_161157_object9954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:59.430262+0000", 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 132.49843262100001, 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.21902869899999999, 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 117457 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.010 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.430262+0000", 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.430262+0000", 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.430260+0000", 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:41:12.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.430267+0000", 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7789999999999998e-06 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.430281+0000", 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3842000000000001e-05 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.430290+0000", 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0899999999999994e-06 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.012 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.430337+0000", 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7249000000000001e-05 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.430463+0000", 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012624399999999999 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.430544+0000", 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0301000000000002e-05 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.013 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.433522+0000", 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002978105 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.437977+0000", 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.004455634 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.649189+0000", 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.211211964 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.649250+0000", 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0507999999999997e-05 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:59.649291+0000", 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0878999999999999e-05 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.015 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:122426 7.7 7:e4e265c5:::benchmark_data_smithi029_161157_object14923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:39:06.403843+0000", 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 125.524851883, 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.21893985099999999, 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 122426 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:12.016 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.403843+0000", 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.403843+0000", 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.403842+0000", 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:12.017 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.403848+0000", 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1460000000000001e-06 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.403858+0000", 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0643e-05 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.403865+0000", 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0630000000000002e-06 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.403911+0000", 2024-04-12T11:41:12.018 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6390999999999997e-05 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.404051+0000", 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000140271 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.404173+0000", 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000121486 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:12.019 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.411398+0000", 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0072257509999999999 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.415888+0000", 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0044895569999999999 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.622712+0000", 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.20682356199999999 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.020 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.622753+0000", 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.155e-05 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.622783+0000", 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.9646e-05 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.021 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:122433 7.7 7:e437834c:::benchmark_data_smithi029_161157_object14930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:39:06.407099+0000", 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 125.521596143, 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.21581620300000001, 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 122433 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:12.022 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.407099+0000", 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.407099+0000", 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.407097+0000", 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.407105+0000", 2024-04-12T11:41:12.023 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4950000000000008e-06 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.407123+0000", 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7139999999999999e-05 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.407131+0000", 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8610000000000004e-06 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.407449+0000", 2024-04-12T11:41:12.024 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000317285 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.407553+0000", 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010444 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.407659+0000", 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000106205 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.411560+0000", 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003900302 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.416091+0000", 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.004531495 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.622827+0000", 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.20673607899999999 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.622875+0000", 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7389000000000003e-05 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.622915+0000", 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0238999999999999e-05 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.027 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:122442 7.7 7:ef28ed46:::benchmark_data_smithi029_161157_object14939:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:39:06.423461+0000", 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 125.505233859, 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.21459906000000001, 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 122442 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.423461+0000", 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.028 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.423461+0000", 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.423459+0000", 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.029 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.029 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:11 smithi087 ceph-mon[89014]: from='client.45005 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:41:12.030 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:12.030 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.423467+0000", 2024-04-12T11:41:12.030 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2290000000000003e-06 2024-04-12T11:41:12.030 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.030 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.030 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:12.030 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.423485+0000", 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7271000000000001e-05 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.423494+0000", 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3810000000000005e-06 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.423544+0000", 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9984000000000001e-05 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.031 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.423668+0000", 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000124153 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.423785+0000", 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000116674 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.426852+0000", 2024-04-12T11:41:12.032 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030667450000000001 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.427159+0000", 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00030720399999999999 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.637975+0000", 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.21081607899999999 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.638024+0000", 2024-04-12T11:41:12.033 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8708000000000002e-05 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:06.638060+0000", 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6368999999999997e-05 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:135812 7.7 7:ebbf9463:::benchmark_data_smithi029_161157_object28309:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1156)", 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:39:25.482246+0000", 2024-04-12T11:41:12.034 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 106.446449106, 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.20912581099999999, 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 135812 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.482246+0000", 2024-04-12T11:41:12.035 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.482246+0000", 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.482244+0000", 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.482251+0000", 2024-04-12T11:41:12.036 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0149999999999998e-06 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.482267+0000", 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5905000000000001e-05 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.482277+0000", 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9429999999999995e-06 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.482327+0000", 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0216000000000002e-05 2024-04-12T11:41:12.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.482453+0000", 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000125356 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.482566+0000", 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000113744 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.486131+0000", 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0035649990000000001 2024-04-12T11:41:12.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.495299+0000", 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0091675790000000004 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.691268+0000", 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.19596888300000001 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:12.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.691322+0000", 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.418e-05 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.691371+0000", 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9526e-05 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:135825 7.7 7:e9b4fd5b:::benchmark_data_smithi029_161157_object28322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1156)", 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:39:25.501910+0000", 2024-04-12T11:41:12.040 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 106.426785194, 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.212671156, 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 135825 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.501910+0000", 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.501910+0000", 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.501908+0000", 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:12.042 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.501915+0000", 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.481e-06 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.501929+0000", 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3607e-05 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.501939+0000", 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0555e-05 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.043 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.501992+0000", 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3063999999999999e-05 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.502131+0000", 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013862600000000001 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.502256+0000", 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012494000000000001 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.044 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.505851+0000", 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003594514 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.691659+0000", 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.18580829099999999 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.714488+0000", 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.022829571 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.045 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.714539+0000", 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0501999999999999e-05 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:25.714581+0000", 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1805999999999999e-05 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.046 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:136982 7.7 7:ef413786:::benchmark_data_smithi029_161157_object29479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1156)", 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:39:27.659149+0000", 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 104.269545826, 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.304027186, 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 136982 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:12.047 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.659149+0000", 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.659149+0000", 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.659148+0000", 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:12.048 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.659155+0000", 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1629999999999999e-06 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.659166+0000", 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.118e-05 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.659173+0000", 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.464e-06 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.659227+0000", 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3292999999999999e-05 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.659368+0000", 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000140915 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.659438+0000", 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0895999999999993e-05 2024-04-12T11:41:12.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.663485+0000", 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0040466929999999996 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.954092+0000", 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29060691100000002 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.963081+0000", 2024-04-12T11:41:12.051 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0089891139999999994 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.963134+0000", 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2607e-05 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.963176+0000", 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2382000000000002e-05 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.052 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:136983 7.d 7:b471558c:::benchmark_data_smithi029_161157_object29480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1156)", 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:39:27.663464+0000", 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 104.265230743, 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.299455574, 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 136983 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.053 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.663464+0000", 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.663464+0000", 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.663463+0000", 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.663470+0000", 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1489999999999997e-06 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.663481+0000", 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1615e-05 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.663491+0000", 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4790000000000002e-06 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.055 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.663543+0000", 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1629000000000003e-05 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.663697+0000", 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015453999999999999 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1", 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.663815+0000", 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000118261 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.056 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.668269+0000", 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0044538360000000001 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.668429+0000", 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015950099999999999 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.962839+0000", 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29441059600000002 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.057 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.962879+0000", 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9233999999999997e-05 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:39:27.962920+0000", 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1088999999999999e-05 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:12.058 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:41:12.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:12.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:12.157 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:12.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:12.204 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:11 smithi029 ceph-mon[97787]: from='client.45005 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:41:12.204 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:11 smithi029 ceph-mon[93650]: from='client.45005 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:41:12.258 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:12.287 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:12.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:12.388 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:12.414 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:12.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:12.516 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:12.543 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:12.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:12.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:12.559 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1633,"stamp":"2024-04-12T11:41:09.823153+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":995891178,"num_objects":15212,"num_object_clones":0,"num_object_copies":45636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5516,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15212,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":200114,"num_write_kb":6882447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5850,"num_bytes_recovered":361892588,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139430,"ondisk_log_size":139430,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6216936,"kb_used_data":3801816,"kb_used_omap":175,"kb_used_meta":2414608,"kb_avail":743744280,"statfs":{"total":767960285184,"available":761594142720,"internally_reserved":0,"allocated":3893059584,"data_stored":3868434754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":2472558761},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-299761664,"num_objects":-4574,"num_object_clones":0,"num_object_copies":-13722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4574,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":288,"num_bytes_recovered":5898240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002119"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1681,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901386+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:41:03.901386+0000","last_peered":"2024-04-12T11:41:03.901386+0000","last_clean":"2024-04-12T11:41:03.901386+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:41:03.901386+0000","last_undegraded":"2024-04-12T11:41:03.901386+0000","last_fullsized":"2024-04-12T11:41:03.901386+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1689,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903766+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:41:03.903766+0000","last_peered":"2024-04-12T11:41:03.903766+0000","last_clean":"2024-04-12T11:41:03.903766+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:41:03.903766+0000","last_undegraded":"2024-04-12T11:41:03.903766+0000","last_fullsized":"2024-04-12T11:41:03.903766+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1740,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.942659+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:41:03.942659+0000","last_peered":"2024-04-12T11:41:03.942659+0000","last_clean":"2024-04-12T11:41:03.942659+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:41:03.942659+0000","last_undegraded":"2024-04-12T11:41:03.942659+0000","last_fullsized":"2024-04-12T11:41:03.942659+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1748,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.905456+0000","last_change":"2024-04-12T11:38:21.933647+0000","last_active":"2024-04-12T11:41:03.905456+0000","last_peered":"2024-04-12T11:41:03.905456+0000","last_clean":"2024-04-12T11:41:03.905456+0000","last_became_active":"2024-04-12T11:38:21.933295+0000","last_became_peered":"2024-04-12T11:38:21.933295+0000","last_unstale":"2024-04-12T11:41:03.905456+0000","last_undegraded":"2024-04-12T11:41:03.905456+0000","last_fullsized":"2024-04-12T11:41:03.905456+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:37.402139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1697,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903198+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:41:03.903198+0000","last_peered":"2024-04-12T11:41:03.903198+0000","last_clean":"2024-04-12T11:41:03.903198+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:41:03.903198+0000","last_undegraded":"2024-04-12T11:41:03.903198+0000","last_fullsized":"2024-04-12T11:41:03.903198+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1109,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.906596+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:41:02.906596+0000","last_peered":"2024-04-12T11:41:02.906596+0000","last_clean":"2024-04-12T11:41:02.906596+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:41:02.906596+0000","last_undegraded":"2024-04-12T11:41:02.906596+0000","last_fullsized":"2024-04-12T11:41:02.906596+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1731,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901674+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:41:03.901674+0000","last_peered":"2024-04-12T11:41:03.901674+0000","last_clean":"2024-04-12T11:41:03.901674+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:41:03.901674+0000","last_undegraded":"2024-04-12T11:41:03.901674+0000","last_fullsized":"2024-04-12T11:41:03.901674+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1670,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903258+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:41:03.903258+0000","last_peered":"2024-04-12T11:41:03.903258+0000","last_clean":"2024-04-12T11:41:03.903258+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:41:03.903258+0000","last_undegraded":"2024-04-12T11:41:03.903258+0000","last_fullsized":"2024-04-12T11:41:03.903258+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1114,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.907872+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:41:02.907872+0000","last_peered":"2024-04-12T11:41:02.907872+0000","last_clean":"2024-04-12T11:41:02.907872+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:41:02.907872+0000","last_undegraded":"2024-04-12T11:41:02.907872+0000","last_fullsized":"2024-04-12T11:41:02.907872+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1650,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903745+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:41:03.903745+0000","last_peered":"2024-04-12T11:41:03.903745+0000","last_clean":"2024-04-12T11:41:03.903745+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:41:03.903745+0000","last_undegraded":"2024-04-12T11:41:03.903745+0000","last_fullsized":"2024-04-12T11:41:03.903745+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1123,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.913187+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:41:02.913187+0000","last_peered":"2024-04-12T11:41:02.913187+0000","last_clean":"2024-04-12T11:41:02.913187+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:41:02.913187+0000","last_undegraded":"2024-04-12T11:41:02.913187+0000","last_fullsized":"2024-04-12T11:41:02.913187+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1732,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.939302+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:41:03.939302+0000","last_peered":"2024-04-12T11:41:03.939302+0000","last_clean":"2024-04-12T11:41:03.939302+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:41:03.939302+0000","last_undegraded":"2024-04-12T11:41:03.939302+0000","last_fullsized":"2024-04-12T11:41:03.939302+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1653,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.902038+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:41:03.902038+0000","last_peered":"2024-04-12T11:41:03.902038+0000","last_clean":"2024-04-12T11:41:03.902038+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:41:03.902038+0000","last_undegraded":"2024-04-12T11:41:03.902038+0000","last_fullsized":"2024-04-12T11:41:03.902038+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1644,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901379+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:41:03.901379+0000","last_peered":"2024-04-12T11:41:03.901379+0000","last_clean":"2024-04-12T11:41:03.901379+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:41:03.901379+0000","last_undegraded":"2024-04-12T11:41:03.901379+0000","last_fullsized":"2024-04-12T11:41:03.901379+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1123,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.912809+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:41:02.912809+0000","last_peered":"2024-04-12T11:41:02.912809+0000","last_clean":"2024-04-12T11:41:02.912809+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:41:02.912809+0000","last_undegraded":"2024-04-12T11:41:02.912809+0000","last_fullsized":"2024-04-12T11:41:02.912809+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1681,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903879+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:41:03.903879+0000","last_peered":"2024-04-12T11:41:03.903879+0000","last_clean":"2024-04-12T11:41:03.903879+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:41:03.903879+0000","last_undegraded":"2024-04-12T11:41:03.903879+0000","last_fullsized":"2024-04-12T11:41:03.903879+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1733,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901218+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:41:03.901218+0000","last_peered":"2024-04-12T11:41:03.901218+0000","last_clean":"2024-04-12T11:41:03.901218+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:41:03.901218+0000","last_undegraded":"2024-04-12T11:41:03.901218+0000","last_fullsized":"2024-04-12T11:41:03.901218+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1113,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.903654+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:41:02.903654+0000","last_peered":"2024-04-12T11:41:02.903654+0000","last_clean":"2024-04-12T11:41:02.903654+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:41:02.903654+0000","last_undegraded":"2024-04-12T11:41:02.903654+0000","last_fullsized":"2024-04-12T11:41:02.903654+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1106,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.912652+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:41:02.912652+0000","last_peered":"2024-04-12T11:41:02.912652+0000","last_clean":"2024-04-12T11:41:02.912652+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:41:02.912652+0000","last_undegraded":"2024-04-12T11:41:02.912652+0000","last_fullsized":"2024-04-12T11:41:02.912652+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1733,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.905410+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:41:03.905410+0000","last_peered":"2024-04-12T11:41:03.905410+0000","last_clean":"2024-04-12T11:41:03.905410+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:41:03.905410+0000","last_undegraded":"2024-04-12T11:41:03.905410+0000","last_fullsized":"2024-04-12T11:41:03.905410+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1117,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.906149+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:41:02.906149+0000","last_peered":"2024-04-12T11:41:02.906149+0000","last_clean":"2024-04-12T11:41:02.906149+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:41:02.906149+0000","last_undegraded":"2024-04-12T11:41:02.906149+0000","last_fullsized":"2024-04-12T11:41:02.906149+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1681,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.904037+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:41:03.904037+0000","last_peered":"2024-04-12T11:41:03.904037+0000","last_clean":"2024-04-12T11:41:03.904037+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:41:03.904037+0000","last_undegraded":"2024-04-12T11:41:03.904037+0000","last_fullsized":"2024-04-12T11:41:03.904037+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1164'12660","reported_seq":12733,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:08.379376+0000","last_change":"2024-04-12T11:37:03.057199+0000","last_active":"2024-04-12T11:41:08.379376+0000","last_peered":"2024-04-12T11:41:08.379376+0000","last_clean":"2024-04-12T11:41:08.379376+0000","last_became_active":"2024-04-12T11:37:03.056776+0000","last_became_peered":"2024-04-12T11:37:03.056776+0000","last_unstale":"2024-04-12T11:41:08.379376+0000","last_undegraded":"2024-04-12T11:41:08.379376+0000","last_fullsized":"2024-04-12T11:41:08.379376+0000","mapping_epoch":1133,"log_start":"1142'4700","ondisk_log_start":"1142'4700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7960,"log_dups_size":0,"ondisk_log_size":7960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:16:29.272872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57671680,"num_objects":880,"num_object_clones":0,"num_object_copies":2640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12660,"num_write_kb":433280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1122,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.913005+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:41:02.913005+0000","last_peered":"2024-04-12T11:41:02.913005+0000","last_clean":"2024-04-12T11:41:02.913005+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:41:02.913005+0000","last_undegraded":"2024-04-12T11:41:02.913005+0000","last_fullsized":"2024-04-12T11:41:02.913005+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1099,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.903327+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:41:02.903327+0000","last_peered":"2024-04-12T11:41:02.903327+0000","last_clean":"2024-04-12T11:41:02.903327+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:41:02.903327+0000","last_undegraded":"2024-04-12T11:41:02.903327+0000","last_fullsized":"2024-04-12T11:41:02.903327+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1164'12521","reported_seq":12594,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:05.274208+0000","last_change":"2024-04-12T11:37:03.059071+0000","last_active":"2024-04-12T11:41:05.274208+0000","last_peered":"2024-04-12T11:41:05.274208+0000","last_clean":"2024-04-12T11:41:05.274208+0000","last_became_active":"2024-04-12T11:37:03.058769+0000","last_became_peered":"2024-04-12T11:37:03.058769+0000","last_unstale":"2024-04-12T11:41:05.274208+0000","last_undegraded":"2024-04-12T11:41:05.274208+0000","last_fullsized":"2024-04-12T11:41:05.274208+0000","mapping_epoch":1133,"log_start":"1144'5000","ondisk_log_start":"1144'5000","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7521,"log_dups_size":0,"ondisk_log_size":7521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:43:26.795096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68091904,"num_objects":1039,"num_object_clones":0,"num_object_copies":3117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12521,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1123,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.912906+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:41:02.912906+0000","last_peered":"2024-04-12T11:41:02.912906+0000","last_clean":"2024-04-12T11:41:02.912906+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:41:02.912906+0000","last_undegraded":"2024-04-12T11:41:02.912906+0000","last_fullsized":"2024-04-12T11:41:02.912906+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1667,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901658+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:41:03.901658+0000","last_peered":"2024-04-12T11:41:03.901658+0000","last_clean":"2024-04-12T11:41:03.901658+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:41:03.901658+0000","last_undegraded":"2024-04-12T11:41:03.901658+0000","last_fullsized":"2024-04-12T11:41:03.901658+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1164'12587","reported_seq":12660,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:08.825838+0000","last_change":"2024-04-12T11:37:03.059390+0000","last_active":"2024-04-12T11:41:08.825838+0000","last_peered":"2024-04-12T11:41:08.825838+0000","last_clean":"2024-04-12T11:41:08.825838+0000","last_became_active":"2024-04-12T11:37:03.058045+0000","last_became_peered":"2024-04-12T11:37:03.058045+0000","last_unstale":"2024-04-12T11:41:08.825838+0000","last_undegraded":"2024-04-12T11:41:08.825838+0000","last_fullsized":"2024-04-12T11:41:08.825838+0000","mapping_epoch":1133,"log_start":"1141'3700","ondisk_log_start":"1141'3700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8887,"log_dups_size":0,"ondisk_log_size":8887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:14:01.284893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57606144,"num_objects":879,"num_object_clones":0,"num_object_copies":2637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12587,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1092,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.913025+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:41:02.913025+0000","last_peered":"2024-04-12T11:41:02.913025+0000","last_clean":"2024-04-12T11:41:02.913025+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:41:02.913025+0000","last_undegraded":"2024-04-12T11:41:02.913025+0000","last_fullsized":"2024-04-12T11:41:02.913025+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1108,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.906324+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:41:02.906324+0000","last_peered":"2024-04-12T11:41:02.906324+0000","last_clean":"2024-04-12T11:41:02.906324+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:41:02.906324+0000","last_undegraded":"2024-04-12T11:41:02.906324+0000","last_fullsized":"2024-04-12T11:41:02.906324+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1164'12546","reported_seq":12619,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:04.917016+0000","last_change":"2024-04-12T11:37:03.054920+0000","last_active":"2024-04-12T11:41:04.917016+0000","last_peered":"2024-04-12T11:41:04.917016+0000","last_clean":"2024-04-12T11:41:04.917016+0000","last_became_active":"2024-04-12T11:37:03.054603+0000","last_became_peered":"2024-04-12T11:37:03.054603+0000","last_unstale":"2024-04-12T11:41:04.917016+0000","last_undegraded":"2024-04-12T11:41:04.917016+0000","last_fullsized":"2024-04-12T11:41:04.917016+0000","mapping_epoch":1133,"log_start":"1140'2500","ondisk_log_start":"1140'2500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10046,"log_dups_size":0,"ondisk_log_size":10046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:34:45.149813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69599232,"num_objects":1062,"num_object_clones":0,"num_object_copies":3186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12546,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1115,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.903561+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:41:02.903561+0000","last_peered":"2024-04-12T11:41:02.903561+0000","last_clean":"2024-04-12T11:41:02.903561+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:41:02.903561+0000","last_undegraded":"2024-04-12T11:41:02.903561+0000","last_fullsized":"2024-04-12T11:41:02.903561+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1743,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901113+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:41:03.901113+0000","last_peered":"2024-04-12T11:41:03.901113+0000","last_clean":"2024-04-12T11:41:03.901113+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:41:03.901113+0000","last_undegraded":"2024-04-12T11:41:03.901113+0000","last_fullsized":"2024-04-12T11:41:03.901113+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1164'12737","reported_seq":12810,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:08.824347+0000","last_change":"2024-04-12T11:37:03.063209+0000","last_active":"2024-04-12T11:41:08.824347+0000","last_peered":"2024-04-12T11:41:08.824347+0000","last_clean":"2024-04-12T11:41:08.824347+0000","last_became_active":"2024-04-12T11:37:03.062848+0000","last_became_peered":"2024-04-12T11:37:03.062848+0000","last_unstale":"2024-04-12T11:41:08.824347+0000","last_undegraded":"2024-04-12T11:41:08.824347+0000","last_fullsized":"2024-04-12T11:41:08.824347+0000","mapping_epoch":1133,"log_start":"1141'3900","ondisk_log_start":"1141'3900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8837,"log_dups_size":0,"ondisk_log_size":8837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:55:26.299047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57344000,"num_objects":875,"num_object_clones":0,"num_object_copies":2625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12737,"num_write_kb":435584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1674,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903531+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:41:03.903531+0000","last_peered":"2024-04-12T11:41:03.903531+0000","last_clean":"2024-04-12T11:41:03.903531+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:41:03.903531+0000","last_undegraded":"2024-04-12T11:41:03.903531+0000","last_fullsized":"2024-04-12T11:41:03.903531+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1657,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.902097+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:41:03.902097+0000","last_peered":"2024-04-12T11:41:03.902097+0000","last_clean":"2024-04-12T11:41:03.902097+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:41:03.902097+0000","last_undegraded":"2024-04-12T11:41:03.902097+0000","last_fullsized":"2024-04-12T11:41:03.902097+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1164'12306","reported_seq":12379,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:04.896303+0000","last_change":"2024-04-12T11:37:03.060739+0000","last_active":"2024-04-12T11:41:04.896303+0000","last_peered":"2024-04-12T11:41:04.896303+0000","last_clean":"2024-04-12T11:41:04.896303+0000","last_became_active":"2024-04-12T11:37:03.060414+0000","last_became_peered":"2024-04-12T11:37:03.060414+0000","last_unstale":"2024-04-12T11:41:04.896303+0000","last_undegraded":"2024-04-12T11:41:04.896303+0000","last_fullsized":"2024-04-12T11:41:04.896303+0000","mapping_epoch":1133,"log_start":"1140'2300","ondisk_log_start":"1140'2300","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10006,"log_dups_size":0,"ondisk_log_size":10006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:12:30.259488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66322432,"num_objects":1012,"num_object_clones":0,"num_object_copies":3036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12306,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":683,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901277+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:41:03.901277+0000","last_peered":"2024-04-12T11:41:03.901277+0000","last_clean":"2024-04-12T11:41:03.901277+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:41:03.901277+0000","last_undegraded":"2024-04-12T11:41:03.901277+0000","last_fullsized":"2024-04-12T11:41:03.901277+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1679,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901658+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:41:03.901658+0000","last_peered":"2024-04-12T11:41:03.901658+0000","last_clean":"2024-04-12T11:41:03.901658+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:41:03.901658+0000","last_undegraded":"2024-04-12T11:41:03.901658+0000","last_fullsized":"2024-04-12T11:41:03.901658+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1164'12724","reported_seq":12766,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:07.374405+0000","last_change":"2024-04-12T11:37:03.060364+0000","last_active":"2024-04-12T11:41:07.374405+0000","last_peered":"2024-04-12T11:41:07.374405+0000","last_clean":"2024-04-12T11:41:07.374405+0000","last_became_active":"2024-04-12T11:37:03.060146+0000","last_became_peered":"2024-04-12T11:37:03.060146+0000","last_unstale":"2024-04-12T11:41:07.374405+0000","last_undegraded":"2024-04-12T11:41:07.374405+0000","last_fullsized":"2024-04-12T11:41:07.374405+0000","mapping_epoch":1133,"log_start":"1141'3300","ondisk_log_start":"1141'3300","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9424,"log_dups_size":0,"ondisk_log_size":9424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:03.796242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57999384,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12724,"num_write_kb":435457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1666,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901754+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:41:03.901754+0000","last_peered":"2024-04-12T11:41:03.901754+0000","last_clean":"2024-04-12T11:41:03.901754+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:41:03.901754+0000","last_undegraded":"2024-04-12T11:41:03.901754+0000","last_fullsized":"2024-04-12T11:41:03.901754+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1644,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901934+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:41:03.901934+0000","last_peered":"2024-04-12T11:41:03.901934+0000","last_clean":"2024-04-12T11:41:03.901934+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:41:03.901934+0000","last_undegraded":"2024-04-12T11:41:03.901934+0000","last_fullsized":"2024-04-12T11:41:03.901934+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1164'12619","reported_seq":12692,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:05.293959+0000","last_change":"2024-04-12T11:37:03.056864+0000","last_active":"2024-04-12T11:41:05.293959+0000","last_peered":"2024-04-12T11:41:05.293959+0000","last_clean":"2024-04-12T11:41:05.293959+0000","last_became_active":"2024-04-12T11:37:03.056523+0000","last_became_peered":"2024-04-12T11:37:03.056523+0000","last_unstale":"2024-04-12T11:41:05.293959+0000","last_undegraded":"2024-04-12T11:41:05.293959+0000","last_fullsized":"2024-04-12T11:41:05.293959+0000","mapping_epoch":1133,"log_start":"1144'5100","ondisk_log_start":"1144'5100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7519,"log_dups_size":0,"ondisk_log_size":7519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:57:58.750385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65994752,"num_objects":1007,"num_object_clones":0,"num_object_copies":3021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12619,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1683,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903330+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:41:03.903330+0000","last_peered":"2024-04-12T11:41:03.903330+0000","last_clean":"2024-04-12T11:41:03.903330+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:41:03.903330+0000","last_undegraded":"2024-04-12T11:41:03.903330+0000","last_fullsized":"2024-04-12T11:41:03.903330+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1741,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.939728+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:41:03.939728+0000","last_peered":"2024-04-12T11:41:03.939728+0000","last_clean":"2024-04-12T11:41:03.939728+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:41:03.939728+0000","last_undegraded":"2024-04-12T11:41:03.939728+0000","last_fullsized":"2024-04-12T11:41:03.939728+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1164'12291","reported_seq":23804,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:04.840428+0000","last_change":"2024-04-12T11:41:03.934075+0000","last_active":"2024-04-12T11:41:04.840428+0000","last_peered":"2024-04-12T11:41:04.840428+0000","last_clean":"2024-04-12T11:41:04.840428+0000","last_became_active":"2024-04-12T11:41:03.904470+0000","last_became_peered":"2024-04-12T11:41:03.904470+0000","last_unstale":"2024-04-12T11:41:04.840428+0000","last_undegraded":"2024-04-12T11:41:04.840428+0000","last_fullsized":"2024-04-12T11:41:04.840428+0000","mapping_epoch":1163,"log_start":"1142'4300","ondisk_log_start":"1142'4300","created":1133,"last_epoch_clean":1164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7991,"log_dups_size":0,"ondisk_log_size":7991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:15:39.102459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62717952,"num_objects":957,"num_object_clones":0,"num_object_copies":2871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12291,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5715,"num_bytes_recovered":354222080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1743,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.939568+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:41:03.939568+0000","last_peered":"2024-04-12T11:41:03.939568+0000","last_clean":"2024-04-12T11:41:03.939568+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:41:03.939568+0000","last_undegraded":"2024-04-12T11:41:03.939568+0000","last_fullsized":"2024-04-12T11:41:03.939568+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1131,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.903728+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:41:02.903728+0000","last_peered":"2024-04-12T11:41:02.903728+0000","last_clean":"2024-04-12T11:41:02.903728+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:41:02.903728+0000","last_undegraded":"2024-04-12T11:41:02.903728+0000","last_fullsized":"2024-04-12T11:41:02.903728+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1734,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.939115+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:41:03.939115+0000","last_peered":"2024-04-12T11:41:03.939115+0000","last_clean":"2024-04-12T11:41:03.939115+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:41:03.939115+0000","last_undegraded":"2024-04-12T11:41:03.939115+0000","last_fullsized":"2024-04-12T11:41:03.939115+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1164'12481","reported_seq":12522,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:07.382277+0000","last_change":"2024-04-12T11:37:03.060583+0000","last_active":"2024-04-12T11:41:07.382277+0000","last_peered":"2024-04-12T11:41:07.382277+0000","last_clean":"2024-04-12T11:41:07.382277+0000","last_became_active":"2024-04-12T11:37:03.060125+0000","last_became_peered":"2024-04-12T11:37:03.060125+0000","last_unstale":"2024-04-12T11:41:07.382277+0000","last_undegraded":"2024-04-12T11:41:07.382277+0000","last_fullsized":"2024-04-12T11:41:07.382277+0000","mapping_epoch":1133,"log_start":"1141'3100","ondisk_log_start":"1141'3100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9381,"log_dups_size":0,"ondisk_log_size":9381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:35:30.859950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64552960,"num_objects":985,"num_object_clones":0,"num_object_copies":2955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12481,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1121,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.900874+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:41:03.900874+0000","last_peered":"2024-04-12T11:41:03.900874+0000","last_clean":"2024-04-12T11:41:03.900874+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:41:03.900874+0000","last_undegraded":"2024-04-12T11:41:03.900874+0000","last_fullsized":"2024-04-12T11:41:03.900874+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1666,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901301+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:41:03.901301+0000","last_peered":"2024-04-12T11:41:03.901301+0000","last_clean":"2024-04-12T11:41:03.901301+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:41:03.901301+0000","last_undegraded":"2024-04-12T11:41:03.901301+0000","last_fullsized":"2024-04-12T11:41:03.901301+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1743,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.939401+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:41:03.939401+0000","last_peered":"2024-04-12T11:41:03.939401+0000","last_clean":"2024-04-12T11:41:03.939401+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:41:03.939401+0000","last_undegraded":"2024-04-12T11:41:03.939401+0000","last_fullsized":"2024-04-12T11:41:03.939401+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1164'12413","reported_seq":12486,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:08.811742+0000","last_change":"2024-04-12T11:37:03.058470+0000","last_active":"2024-04-12T11:41:08.811742+0000","last_peered":"2024-04-12T11:41:08.811742+0000","last_clean":"2024-04-12T11:41:08.811742+0000","last_became_active":"2024-04-12T11:37:03.058149+0000","last_became_peered":"2024-04-12T11:37:03.058149+0000","last_unstale":"2024-04-12T11:41:08.811742+0000","last_undegraded":"2024-04-12T11:41:08.811742+0000","last_fullsized":"2024-04-12T11:41:08.811742+0000","mapping_epoch":1133,"log_start":"1141'3500","ondisk_log_start":"1141'3500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8913,"log_dups_size":0,"ondisk_log_size":8913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:25:06.934877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12413,"num_write_kb":423808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1706,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.950678+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:41:03.950678+0000","last_peered":"2024-04-12T11:41:03.950678+0000","last_clean":"2024-04-12T11:41:03.950678+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:41:03.950678+0000","last_undegraded":"2024-04-12T11:41:03.950678+0000","last_fullsized":"2024-04-12T11:41:03.950678+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1734,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.939170+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:41:03.939170+0000","last_peered":"2024-04-12T11:41:03.939170+0000","last_clean":"2024-04-12T11:41:03.939170+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:41:03.939170+0000","last_undegraded":"2024-04-12T11:41:03.939170+0000","last_fullsized":"2024-04-12T11:41:03.939170+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1736,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.939820+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:41:03.939820+0000","last_peered":"2024-04-12T11:41:03.939820+0000","last_clean":"2024-04-12T11:41:03.939820+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:41:03.939820+0000","last_undegraded":"2024-04-12T11:41:03.939820+0000","last_fullsized":"2024-04-12T11:41:03.939820+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1164'12409","reported_seq":12735,"reported_epoch":1164,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-04-12T11:41:08.808622+0000","last_change":"2024-04-12T11:41:02.486992+0000","last_active":"2024-04-12T11:41:08.808622+0000","last_peered":"2024-04-12T11:41:08.808622+0000","last_clean":"2024-04-12T11:38:20.888095+0000","last_became_active":"2024-04-12T11:38:23.424268+0000","last_became_peered":"2024-04-12T11:38:23.424268+0000","last_unstale":"2024-04-12T11:41:08.808622+0000","last_undegraded":"2024-04-12T11:38:23.284099+0000","last_fullsized":"2024-04-12T11:38:23.245558+0000","mapping_epoch":1147,"log_start":"1141'3500","ondisk_log_start":"1141'3500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8909,"log_dups_size":0,"ondisk_log_size":8909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59179008,"num_objects":903,"num_object_clones":0,"num_object_copies":2709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5516,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12409,"num_write_kb":425984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":125,"num_bytes_recovered":5898240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,7","objects":-4613}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1121,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.900807+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:41:03.900807+0000","last_peered":"2024-04-12T11:41:03.900807+0000","last_clean":"2024-04-12T11:41:03.900807+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:41:03.900807+0000","last_undegraded":"2024-04-12T11:41:03.900807+0000","last_fullsized":"2024-04-12T11:41:03.900807+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1475,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.912553+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:41:02.912553+0000","last_peered":"2024-04-12T11:41:02.912553+0000","last_clean":"2024-04-12T11:41:02.912553+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:41:02.912553+0000","last_undegraded":"2024-04-12T11:41:02.912553+0000","last_fullsized":"2024-04-12T11:41:02.912553+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1650,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903730+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:41:03.903730+0000","last_peered":"2024-04-12T11:41:03.903730+0000","last_clean":"2024-04-12T11:41:03.903730+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:41:03.903730+0000","last_undegraded":"2024-04-12T11:41:03.903730+0000","last_fullsized":"2024-04-12T11:41:03.903730+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1109,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.906499+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:41:02.906499+0000","last_peered":"2024-04-12T11:41:02.906499+0000","last_clean":"2024-04-12T11:41:02.906499+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:41:02.906499+0000","last_undegraded":"2024-04-12T11:41:02.906499+0000","last_fullsized":"2024-04-12T11:41:02.906499+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1164'12323","reported_seq":12364,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:07.436753+0000","last_change":"2024-04-12T11:37:03.061698+0000","last_active":"2024-04-12T11:41:07.436753+0000","last_peered":"2024-04-12T11:41:07.436753+0000","last_clean":"2024-04-12T11:41:07.436753+0000","last_became_active":"2024-04-12T11:37:03.061469+0000","last_became_peered":"2024-04-12T11:37:03.061469+0000","last_unstale":"2024-04-12T11:41:07.436753+0000","last_undegraded":"2024-04-12T11:41:07.436753+0000","last_fullsized":"2024-04-12T11:41:07.436753+0000","mapping_epoch":1133,"log_start":"1141'3700","ondisk_log_start":"1141'3700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8623,"log_dups_size":0,"ondisk_log_size":8623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:09.906340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59834368,"num_objects":913,"num_object_clones":0,"num_object_copies":2739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12323,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1128,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.912503+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:41:02.912503+0000","last_peered":"2024-04-12T11:41:02.912503+0000","last_clean":"2024-04-12T11:41:02.912503+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:41:02.912503+0000","last_undegraded":"2024-04-12T11:41:02.912503+0000","last_fullsized":"2024-04-12T11:41:02.912503+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1744,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901320+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:41:03.901320+0000","last_peered":"2024-04-12T11:41:03.901320+0000","last_clean":"2024-04-12T11:41:03.901320+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:41:03.901320+0000","last_undegraded":"2024-04-12T11:41:03.901320+0000","last_fullsized":"2024-04-12T11:41:03.901320+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1743,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.905355+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:41:03.905355+0000","last_peered":"2024-04-12T11:41:03.905355+0000","last_clean":"2024-04-12T11:41:03.905355+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:41:03.905355+0000","last_undegraded":"2024-04-12T11:41:03.905355+0000","last_fullsized":"2024-04-12T11:41:03.905355+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"1164'12525","reported_seq":12566,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:07.326159+0000","last_change":"2024-04-12T11:37:03.060455+0000","last_active":"2024-04-12T11:41:07.326159+0000","last_peered":"2024-04-12T11:41:07.326159+0000","last_clean":"2024-04-12T11:41:07.326159+0000","last_became_active":"2024-04-12T11:37:03.060244+0000","last_became_peered":"2024-04-12T11:37:03.060244+0000","last_unstale":"2024-04-12T11:41:07.326159+0000","last_undegraded":"2024-04-12T11:41:07.326159+0000","last_fullsized":"2024-04-12T11:41:07.326159+0000","mapping_epoch":1133,"log_start":"1140'2500","ondisk_log_start":"1140'2500","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10025,"log_dups_size":0,"ondisk_log_size":10025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:47.787368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63504384,"num_objects":969,"num_object_clones":0,"num_object_copies":2907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12525,"num_write_kb":431808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1127,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.913107+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:41:02.913107+0000","last_peered":"2024-04-12T11:41:02.913107+0000","last_clean":"2024-04-12T11:41:02.913107+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:41:02.913107+0000","last_undegraded":"2024-04-12T11:41:02.913107+0000","last_fullsized":"2024-04-12T11:41:02.913107+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1734,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.939558+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:41:03.939558+0000","last_peered":"2024-04-12T11:41:03.939558+0000","last_clean":"2024-04-12T11:41:03.939558+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:41:03.939558+0000","last_undegraded":"2024-04-12T11:41:03.939558+0000","last_fullsized":"2024-04-12T11:41:03.939558+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1727,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901257+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:41:03.901257+0000","last_peered":"2024-04-12T11:41:03.901257+0000","last_clean":"2024-04-12T11:41:03.901257+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:41:03.901257+0000","last_undegraded":"2024-04-12T11:41:03.901257+0000","last_fullsized":"2024-04-12T11:41:03.901257+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1164'12199","reported_seq":12272,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:05.413382+0000","last_change":"2024-04-12T11:37:03.057084+0000","last_active":"2024-04-12T11:41:05.413382+0000","last_peered":"2024-04-12T11:41:05.413382+0000","last_clean":"2024-04-12T11:41:05.413382+0000","last_became_active":"2024-04-12T11:37:03.056772+0000","last_became_peered":"2024-04-12T11:37:03.056772+0000","last_unstale":"2024-04-12T11:41:05.413382+0000","last_undegraded":"2024-04-12T11:41:05.413382+0000","last_fullsized":"2024-04-12T11:41:05.413382+0000","mapping_epoch":1133,"log_start":"1142'4600","ondisk_log_start":"1142'4600","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7599,"log_dups_size":0,"ondisk_log_size":7599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:02:41.845790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64552960,"num_objects":985,"num_object_clones":0,"num_object_copies":2955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12199,"num_write_kb":421888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1666,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.904024+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:41:03.904024+0000","last_peered":"2024-04-12T11:41:03.904024+0000","last_clean":"2024-04-12T11:41:03.904024+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:41:03.904024+0000","last_undegraded":"2024-04-12T11:41:03.904024+0000","last_fullsized":"2024-04-12T11:41:03.904024+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1733,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.939693+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:41:03.939693+0000","last_peered":"2024-04-12T11:41:03.939693+0000","last_clean":"2024-04-12T11:41:03.939693+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:41:03.939693+0000","last_undegraded":"2024-04-12T11:41:03.939693+0000","last_fullsized":"2024-04-12T11:41:03.939693+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1663,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903359+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:41:03.903359+0000","last_peered":"2024-04-12T11:41:03.903359+0000","last_clean":"2024-04-12T11:41:03.903359+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:41:03.903359+0000","last_undegraded":"2024-04-12T11:41:03.903359+0000","last_fullsized":"2024-04-12T11:41:03.903359+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1164'12473","reported_seq":12546,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:05.284413+0000","last_change":"2024-04-12T11:37:03.060299+0000","last_active":"2024-04-12T11:41:05.284413+0000","last_peered":"2024-04-12T11:41:05.284413+0000","last_clean":"2024-04-12T11:41:05.284413+0000","last_became_active":"2024-04-12T11:37:03.058897+0000","last_became_peered":"2024-04-12T11:37:03.058897+0000","last_unstale":"2024-04-12T11:41:05.284413+0000","last_undegraded":"2024-04-12T11:41:05.284413+0000","last_fullsized":"2024-04-12T11:41:05.284413+0000","mapping_epoch":1133,"log_start":"1144'4900","ondisk_log_start":"1144'4900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7573,"log_dups_size":0,"ondisk_log_size":7573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:45:37.844216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65863680,"num_objects":1005,"num_object_clones":0,"num_object_copies":3015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12473,"num_write_kb":431296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1133,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.906533+0000","last_change":"2024-04-12T11:39:23.349899+0000","last_active":"2024-04-12T11:41:02.906533+0000","last_peered":"2024-04-12T11:41:02.906533+0000","last_clean":"2024-04-12T11:41:02.906533+0000","last_became_active":"2024-04-12T11:39:23.349526+0000","last_became_peered":"2024-04-12T11:39:23.349526+0000","last_unstale":"2024-04-12T11:41:02.906533+0000","last_undegraded":"2024-04-12T11:41:02.906533+0000","last_fullsized":"2024-04-12T11:41:02.906533+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:36:47.115912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1746,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.905508+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:41:03.905508+0000","last_peered":"2024-04-12T11:41:03.905508+0000","last_clean":"2024-04-12T11:41:03.905508+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:41:03.905508+0000","last_undegraded":"2024-04-12T11:41:03.905508+0000","last_fullsized":"2024-04-12T11:41:03.905508+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1653,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901392+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:41:03.901392+0000","last_peered":"2024-04-12T11:41:03.901392+0000","last_clean":"2024-04-12T11:41:03.901392+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:41:03.901392+0000","last_undegraded":"2024-04-12T11:41:03.901392+0000","last_fullsized":"2024-04-12T11:41:03.901392+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1736,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901233+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:41:03.901233+0000","last_peered":"2024-04-12T11:41:03.901233+0000","last_clean":"2024-04-12T11:41:03.901233+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:41:03.901233+0000","last_undegraded":"2024-04-12T11:41:03.901233+0000","last_fullsized":"2024-04-12T11:41:03.901233+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":52,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.950840+0000","last_change":"2024-04-12T11:38:06.839832+0000","last_active":"2024-04-12T11:41:03.950840+0000","last_peered":"2024-04-12T11:41:03.950840+0000","last_clean":"2024-04-12T11:41:03.950840+0000","last_became_active":"2024-04-12T11:38:06.839460+0000","last_became_peered":"2024-04-12T11:38:06.839460+0000","last_unstale":"2024-04-12T11:41:03.950840+0000","last_undegraded":"2024-04-12T11:41:03.950840+0000","last_fullsized":"2024-04-12T11:41:03.950840+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:25:27.959933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1131,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.903705+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:41:02.903705+0000","last_peered":"2024-04-12T11:41:02.903705+0000","last_clean":"2024-04-12T11:41:02.903705+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:41:02.903705+0000","last_undegraded":"2024-04-12T11:41:02.903705+0000","last_fullsized":"2024-04-12T11:41:02.903705+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1681,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901539+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:41:03.901539+0000","last_peered":"2024-04-12T11:41:03.901539+0000","last_clean":"2024-04-12T11:41:03.901539+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:41:03.901539+0000","last_undegraded":"2024-04-12T11:41:03.901539+0000","last_fullsized":"2024-04-12T11:41:03.901539+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1116,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-04-12T11:41:02.903814+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:41:02.903814+0000","last_peered":"2024-04-12T11:41:02.903814+0000","last_clean":"2024-04-12T11:41:02.903814+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:41:02.903814+0000","last_undegraded":"2024-04-12T11:41:02.903814+0000","last_fullsized":"2024-04-12T11:41:02.903814+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1672,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901279+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:41:03.901279+0000","last_peered":"2024-04-12T11:41:03.901279+0000","last_clean":"2024-04-12T11:41:03.901279+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:41:03.901279+0000","last_undegraded":"2024-04-12T11:41:03.901279+0000","last_fullsized":"2024-04-12T11:41:03.901279+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1664,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903847+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:41:03.903847+0000","last_peered":"2024-04-12T11:41:03.903847+0000","last_clean":"2024-04-12T11:41:03.903847+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:41:03.903847+0000","last_undegraded":"2024-04-12T11:41:03.903847+0000","last_fullsized":"2024-04-12T11:41:03.903847+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1675,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.903602+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:41:03.903602+0000","last_peered":"2024-04-12T11:41:03.903602+0000","last_clean":"2024-04-12T11:41:03.903602+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:41:03.903602+0000","last_undegraded":"2024-04-12T11:41:03.903602+0000","last_fullsized":"2024-04-12T11:41:03.903602+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1765,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901700+0000","last_change":"2024-04-12T11:38:06.866508+0000","last_active":"2024-04-12T11:41:03.901700+0000","last_peered":"2024-04-12T11:41:03.901700+0000","last_clean":"2024-04-12T11:41:03.901700+0000","last_became_active":"2024-04-12T11:38:06.843752+0000","last_became_peered":"2024-04-12T11:38:06.843752+0000","last_unstale":"2024-04-12T11:41:03.901700+0000","last_undegraded":"2024-04-12T11:41:03.901700+0000","last_fullsized":"2024-04-12T11:41:03.901700+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:18:11.827424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1644,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-04-12T11:41:03.901205+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:41:03.901205+0000","last_peered":"2024-04-12T11:41:03.901205+0000","last_clean":"2024-04-12T11:41:03.901205+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:41:03.901205+0000","last_undegraded":"2024-04-12T11:41:03.901205+0000","last_fullsized":"2024-04-12T11:41:03.901205+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":995295256,"num_objects":15188,"num_object_clones":0,"num_object_copies":45564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5516,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15188,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":199814,"num_write_kb":6880001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5840,"num_bytes_recovered":360120320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3721801728,"data_stored":3721789512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139214,"ondisk_log_size":139214,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234345,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":634772,"kb_used_data":318988,"kb_used_omap":1,"kb_used_meta":315710,"kb_avail":93110380,"statfs":{"total":95995035648,"available":95345029120,"internally_reserved":0,"allocated":326643712,"data_stored":323551413,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":323287499},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495982,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":513676,"kb_used_data":306244,"kb_used_omap":1,"kb_used_meta":207422,"kb_avail":93231476,"statfs":{"total":95995035648,"available":95469031424,"internally_reserved":0,"allocated":313593856,"data_stored":310509627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":212400584},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536292,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":978936,"kb_used_data":716464,"kb_used_omap":30,"kb_used_meta":262369,"kb_avail":92766216,"statfs":{"total":95995035648,"available":94992605184,"internally_reserved":0,"allocated":733659136,"data_stored":730590029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":268666178},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190376,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":774980,"kb_used_data":472124,"kb_used_omap":43,"kb_used_meta":302740,"kb_avail":92970172,"statfs":{"total":95995035648,"available":95201456128,"internally_reserved":0,"allocated":483454976,"data_stored":480374729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":310005803},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849503,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":773312,"kb_used_data":412276,"kb_used_omap":36,"kb_used_meta":360987,"kb_avail":92971840,"statfs":{"total":95995035648,"available":95203164160,"internally_reserved":0,"allocated":422170624,"data_stored":419098013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":369650868},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704856,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1023548,"kb_used_data":608560,"kb_used_omap":17,"kb_used_meta":414894,"kb_avail":92721604,"statfs":{"total":95995035648,"available":94946922496,"internally_reserved":0,"allocated":623165440,"data_stored":620096399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":424851959},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018068,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599632,"kb_used_data":346756,"kb_used_omap":42,"kb_used_meta":252821,"kb_avail":93145520,"statfs":{"total":95995035648,"available":95381012480,"internally_reserved":0,"allocated":355078144,"data_stored":351993435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":258889528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757618,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":918080,"kb_used_data":620404,"kb_used_omap":1,"kb_used_meta":297662,"kb_avail":92827072,"statfs":{"total":95995035648,"available":95054921728,"internally_reserved":0,"allocated":635293696,"data_stored":632221109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":304806342},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":454037504,"data_stored":454033432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":710414336,"data_stored":710410264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":404357120,"data_stored":404357120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":598081536,"data_stored":598081536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":331808768,"data_stored":331808768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":616693760,"data_stored":616693760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":298848256,"data_stored":298844184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":307560448,"data_stored":307560448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:41:12.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph status --format=json 2024-04-12T11:41:12.645 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:12.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:12.710 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:12.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:41:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:41:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:41:12.811 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:12.840 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:41:12.864 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:12.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:12.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:12.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:12.972 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-04-12T11:41:13.017 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:13.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:13.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:13.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:13.166 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:12 smithi029 ceph-mon[97787]: pgmap v1634: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 860 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 5516/41319 objects degraded (13.350%); 645 KiB/s, 32 objects/s recovering 2024-04-12T11:41:13.166 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:12 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5516/41319 objects degraded (13.350%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:13.167 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:12 smithi029 ceph-mon[93650]: pgmap v1634: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 860 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 5516/41319 objects degraded (13.350%); 645 KiB/s, 32 objects/s recovering 2024-04-12T11:41:13.167 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:12 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5516/41319 objects degraded (13.350%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:13.169 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:13.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:13.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:12 smithi087 ceph-mon[89014]: pgmap v1634: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 860 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 5516/41319 objects degraded (13.350%); 645 KiB/s, 32 objects/s recovering 2024-04-12T11:41:13.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:12 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5516/41319 objects degraded (13.350%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:13.243 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:41:13.271 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:13.298 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:13.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:13.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:13.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:13.450 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:13.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:13.551 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:13.596 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:13.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:13.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:13.698 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:13.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:13.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:13.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:13.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:13.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:13.884 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:13.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:13.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:13 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2842354046' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:41:13.975 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:41:13.985 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:41:13.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:14.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:14.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:14.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:14.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:13 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2842354046' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:41:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:13 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2842354046' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:41:14.115 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:14.145 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:14.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:14.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:14.246 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:14.272 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:14.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:14.374 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:14.402 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:14.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:14.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:14.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:14.545 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:14.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:14.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:14.647 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:14.674 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:14.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:14.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:14.776 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:14.844 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:14.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:14.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:14.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:14.997 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:14.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:14.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:14 smithi029 ceph-mon[97787]: pgmap v1635: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 796 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 5516/38274 objects degraded (14.412%); 301 KiB/s, 24 objects/s recovering 2024-04-12T11:41:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:14 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:41:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:14 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2842354046' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:41:15.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:14 smithi029 ceph-mon[97787]: osdmap e1165: 8 total, 8 up, 8 in 2024-04-12T11:41:15.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:14 smithi029 ceph-mon[93650]: pgmap v1635: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 796 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 5516/38274 objects degraded (14.412%); 301 KiB/s, 24 objects/s recovering 2024-04-12T11:41:15.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:14 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:41:15.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:14 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2842354046' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:41:15.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:14 smithi029 ceph-mon[93650]: osdmap e1165: 8 total, 8 up, 8 in 2024-04-12T11:41:15.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:41:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:41:14] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2024-04-12T11:41:15.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:15.119 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:14 smithi087 ceph-mon[89014]: pgmap v1635: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 796 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 5516/38274 objects degraded (14.412%); 301 KiB/s, 24 objects/s recovering 2024-04-12T11:41:15.119 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:14 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:41:15.119 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:14 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2842354046' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:41:15.119 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:14 smithi087 ceph-mon[89014]: osdmap e1165: 8 total, 8 up, 8 in 2024-04-12T11:41:15.119 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.44843.0:205184 7.6 e1165/1147 7:607789d6:::benchmark_data_smithi029_161157_object97681:head v 1165'12683, mlcod=1150'6786)", 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:41:15.115221+0000", 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.0030918949999999999, 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0031162400000000002, 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:41:15.120 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:41:15.115221+0000", 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:41:15.115221+0000", 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:41:15.115220+0000", 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:15.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:41:15.115228+0000", 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0499999999999992e-06 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:41:15.115244+0000", 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6215999999999999e-05 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:41:15.115252+0000", 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8609999999999998e-06 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:15.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:41:15.115297+0000", 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5500000000000001e-05 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:41:15.115321+0000", 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3266000000000001e-05 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 1 2024-04-12T11:41:15.123 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:41:15.139 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:15.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:15.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:15.240 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:15.276 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:15.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:15.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:15.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:15.424 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:15.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:15.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:15.525 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:15.556 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:15.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:15.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:15.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:15.695 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:15.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:15.790 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-04-12T11:41:15.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:15.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:15.819 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:15.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:15.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:15.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:15.959 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:15.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:16.045 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:41:16.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:16.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:16.100 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:16.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:16.201 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:16.230 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:16.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:16.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:16.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:16.379 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:16.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:16.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:16.480 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:16.513 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:16.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:16.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:16.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:16.655 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:16.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:16.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:16.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:16.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:16.805 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:16.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:16.906 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:16.950 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:16.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:16.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:17.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:17.080 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:17.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:17.182 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:17.207 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:17.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:17.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:16 smithi087 ceph-mon[89014]: pgmap v1637: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 658 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5441/31632 objects degraded (17.201%); 301 KiB/s, 15 objects/s recovering 2024-04-12T11:41:17.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:16 smithi029 ceph-mon[97787]: pgmap v1637: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 658 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5441/31632 objects degraded (17.201%); 301 KiB/s, 15 objects/s recovering 2024-04-12T11:41:17.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:16 smithi029 ceph-mon[93650]: pgmap v1637: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 658 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 5441/31632 objects degraded (17.201%); 301 KiB/s, 15 objects/s recovering 2024-04-12T11:41:17.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:41:17.309 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:17.377 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:17.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:17.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:17.479 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:17.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:17.551 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:17.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:17.590 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:41:17.644 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:41:17.651 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:17.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:17.692 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:17.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:17.694 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:41:17.746 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:41:17.793 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:17.804 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:41:17.847 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:41:17.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:17.858 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:17.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:17.892 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:41:17.901 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:41:17.901 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:41:17.901 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:41:17.901 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:41:17.902 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:41:17.933 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:41:17.951 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:41:17.951 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:41:17.951 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:41:17.951 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:41:17.951 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:41:17.951 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:41:17.951 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:41:17.951 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:41:17.959 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:17.984 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:17.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:17.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:18.086 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:18.113 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:18.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:18.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:18.215 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:18.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:17 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3997295042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:41:18.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:17 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5441/31632 objects degraded (17.201%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:18.243 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:18.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:18.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:18.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:17 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3997295042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:41:18.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:17 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5441/31632 objects degraded (17.201%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:18.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:17 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3997295042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:41:18.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:17 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5441/31632 objects degraded (17.201%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:18.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:18.385 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:18.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:18.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:18.487 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:18.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:18.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:18.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:18.557 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-04-12T11:41:18.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:18.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:18.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:18.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:41:18.710 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5441/29265 objects degraded (18.592%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1258,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1165,"num_osds":8,"num_up_osds":8,"osd_up_since":1712921015,"num_in_osds":8,"osd_in_since":1712920262,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":9755,"data_bytes":638261226,"bytes_used":5301841920,"bytes_avail":762658443264,"bytes_total":767960285184,"degraded_objects":5441,"degraded_total":29265,"degraded_ratio":0.18592174953015547,"recovering_objects_per_sec":15,"recovering_bytes_per_sec":307799,"recovering_keys_per_sec":0,"num_objects_recovered":157,"num_bytes_recovered":3080192,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":646},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.scbuux","status":"up:active","gid":14661}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":25,"modified":"2024-04-12T11:37:57.741061+0000","services":{}},"progress_events":{"53786749-146c-4436-89ac-aaaa1350cbf3":{"message":"Upgrade to 19.0.0-2874-g2cee1857 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"d4a8c707-9e6b-4992-81dd-3fa476e5d44b":{"message":"Global Recovery Event (2m)\n [===========================.] (remaining: 1s)","progress":0.98876404762268066,"add_to_ceph_s":true}}} 2024-04-12T11:41:18.793 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:18.817 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:41:18.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:18.837 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:18.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:18.938 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:18.967 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:18.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:18.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:19.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:19.104 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:19.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:19.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:19.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:19.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:18 smithi087 ceph-mon[89014]: pgmap v1638: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 609 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 5441/29265 objects degraded (18.592%); 301 KiB/s, 15 objects/s recovering 2024-04-12T11:41:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:18 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1931056935' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-12T11:41:19.254 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:19.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:19.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:19.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:18 smithi029 ceph-mon[97787]: pgmap v1638: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 609 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 5441/29265 objects degraded (18.592%); 301 KiB/s, 15 objects/s recovering 2024-04-12T11:41:19.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:18 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1931056935' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-12T11:41:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:18 smithi029 ceph-mon[93650]: pgmap v1638: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 609 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 5441/29265 objects degraded (18.592%); 301 KiB/s, 15 objects/s recovering 2024-04-12T11:41:19.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:18 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1931056935' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-04-12T11:41:19.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:19.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:19.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:19.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:19.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:19.534 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:19.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:19.635 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-04-12T11:41:19.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:41:19.636 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:19.665 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:19.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:19.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:19.738 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:41:19.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:19.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:19.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:19.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:19.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:19.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:19.994 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:19.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:20.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:20.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:20.148 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:20.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:20.249 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:20.275 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:20.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:20.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:20.377 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:20.415 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:20.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:20.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:20.517 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:20.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:20.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:20.649 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:20.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:20.682 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:20.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:20.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:20.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:20.827 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:20.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:20.856 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:41:20.922 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:41:20.922 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:41:20.928 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:20.956 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:20.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:20.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:21.058 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:21.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:21.132 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:21.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:21.146 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:41:21.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:20 smithi087 ceph-mon[89014]: pgmap v1639: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 548 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 5371/26367 objects degraded (20.370%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:21.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:20 smithi029 ceph-mon[97787]: pgmap v1639: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 548 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 5371/26367 objects degraded (20.370%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:21.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:20 smithi029 ceph-mon[93650]: pgmap v1639: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 548 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 5371/26367 objects degraded (20.370%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:21.284 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:21.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:21.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:21.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:21.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:21.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:21.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:21.580 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:21.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:21.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-04-12T11:41:21.682 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:21.708 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:21.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:21.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:21.869 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:21.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:21.970 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:21.998 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:21.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:21.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:22.099 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:22.124 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:22.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:22.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:22.226 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:22.238 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:41:22.238 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:22.238 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:22.254 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:22.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:22.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:22.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:41:22.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:22.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:22.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:41:22.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:22.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:22.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:22.381 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:22.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:22.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:22.483 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:22.509 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:22.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:22.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:22.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:22.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:22.655 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:22.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:41:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:41:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:41:22.756 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:22.790 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:22.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:22.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:22.893 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:22.920 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:22.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:22.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:23.022 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:23.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:23.046 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:23.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:23.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:23.192 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:23.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:23.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:23.293 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:23.320 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:23.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:23.422 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:23.454 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:23.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:23.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:23 smithi029 ceph-mon[97787]: pgmap v1640: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 462 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5371/22221 objects degraded (24.171%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:23.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5371/22221 objects degraded (24.171%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:23 smithi029 ceph-mon[93650]: pgmap v1640: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 462 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5371/22221 objects degraded (24.171%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5371/22221 objects degraded (24.171%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:23.555 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:23.595 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:23.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:23.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:23.697 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:23.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:23 smithi087 ceph-mon[89014]: pgmap v1640: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 462 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5371/22221 objects degraded (24.171%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:23.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5371/22221 objects degraded (24.171%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:23.727 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:23.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:23.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:23.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:23.864 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:23.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:23.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:24.005 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:24.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:24.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:24.108 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:24.140 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:24.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:24.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:24.242 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:24.278 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:24.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:24.379 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:24.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:24.422 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:24.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:24.523 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:24.552 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:24.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:24.654 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:24.679 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:24.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:24.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:24.781 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:24.809 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:24.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:24.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:24.911 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:24.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:24.942 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:24.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:25.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:25.079 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:25.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:25.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:25.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:25.204 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:25.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:41:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:41:24] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2024-04-12T11:41:25.306 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:25.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:25.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:25.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:25.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:25.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:25 smithi087 ceph-mon[89014]: pgmap v1641: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 399 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5371/19185 objects degraded (27.996%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:25.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:25.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:25.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:25.581 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:25.610 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:25.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:25.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:25.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:25.735 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:25.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:25.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:25 smithi029 ceph-mon[93650]: pgmap v1641: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 399 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5371/19185 objects degraded (27.996%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:25.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:25 smithi029 ceph-mon[97787]: pgmap v1641: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 399 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 5371/19185 objects degraded (27.996%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:25.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:25.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:25.861 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:25.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:25.961 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:25.988 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:25.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:25.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:26.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:26.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:26.178 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:26.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:26.279 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:26.306 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:26.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:26.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:26.408 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:26.431 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:26.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:26.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:26.533 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:26.565 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:26.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:26.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:26.667 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:26.701 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:26.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:26.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:26.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:26.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:26.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:26.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:27.036 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:27.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:27.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:27.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:27.166 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:27.200 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:27.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:27.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:27.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:27.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:27.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:27.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:27.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:27.492 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:27.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:27.594 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:27.618 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:27.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:27.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:27 smithi087 ceph-mon[89014]: pgmap v1642: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 264 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 5291/12717 objects degraded (41.606%); 0 B/s, 12 objects/s recovering 2024-04-12T11:41:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:27 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:27 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:27.720 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:27.745 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:27.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:27.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:27 smithi029 ceph-mon[97787]: pgmap v1642: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 264 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 5291/12717 objects degraded (41.606%); 0 B/s, 12 objects/s recovering 2024-04-12T11:41:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:27 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:27 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:27 smithi029 ceph-mon[93650]: pgmap v1642: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 264 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 5291/12717 objects degraded (41.606%); 0 B/s, 12 objects/s recovering 2024-04-12T11:41:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:27 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:27 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:27.847 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:27.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:27.880 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:27.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:27.981 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:28.004 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:28.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:28.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:28.106 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:28.136 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:28.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:28.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:28.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:28.275 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:28.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:28.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:28.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:28.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:28.400 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:28.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:28.501 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:28.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:28.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:28.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:28.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:28.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:28.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:28.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5291/12717 objects degraded (41.606%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:28.706 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5291/12717 objects degraded (41.606%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:28.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5291/12717 objects degraded (41.606%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:28.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:28.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:28.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:28.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:28.931 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:28.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:28.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:29.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:29.102 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:29.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:29.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:29.204 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:29.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:29.230 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:29.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:29.331 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:29.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:29.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:29.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:29.459 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:29.492 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:29.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:29.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:29.594 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:29.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:29.632 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:29.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:29.682 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:41:29.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:29 smithi029 ceph-mon[97787]: pgmap v1643: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 216 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 5291/10425 objects degraded (50.753%); 0 B/s, 12 objects/s recovering 2024-04-12T11:41:29.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:29 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/675053938' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:41:29.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:29 smithi029 ceph-mon[93650]: pgmap v1643: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 216 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 5291/10425 objects degraded (50.753%); 0 B/s, 12 objects/s recovering 2024-04-12T11:41:29.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:29 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/675053938' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:41:29.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:29 smithi087 ceph-mon[89014]: pgmap v1643: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 216 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 5291/10425 objects degraded (50.753%); 0 B/s, 12 objects/s recovering 2024-04-12T11:41:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:29 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/675053938' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:41:29.733 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:29.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:29.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:29.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:29.861 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:29.889 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:29.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:29.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:29.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:30.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:30.028 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:30.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:30.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:30.169 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:30.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:30.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:30.272 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:30.307 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:30.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:30.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:30.408 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:30.438 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:30.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:30.540 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:30.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:30.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:30.600 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:30 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:41:30.600 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:30 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/675053938' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:41:30.600 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:30 smithi029 ceph-mon[97787]: osdmap e1166: 8 total, 8 up, 8 in 2024-04-12T11:41:30.600 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:30 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:41:30.600 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:30 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/675053938' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:41:30.600 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:30 smithi029 ceph-mon[93650]: osdmap e1166: 8 total, 8 up, 8 in 2024-04-12T11:41:30.669 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:30.706 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:30.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:30.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:30 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:41:30.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:30 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/675053938' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:41:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:30 smithi087 ceph-mon[89014]: osdmap e1166: 8 total, 8 up, 8 in 2024-04-12T11:41:30.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:30.836 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:30.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:30.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:30.938 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:30.978 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:30.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:30.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:31.080 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:31.096 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 107500 objects 2024-04-12T11:41:31.096 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :165.575 2024-04-12T11:41:31.103 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:31.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:31.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:31.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:31.228 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:31.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:31.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:31.330 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:31.354 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:31.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:31.456 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:31.480 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:31.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:31.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:31.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:31.605 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:31.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:31.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:31.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:31 smithi087 ceph-mon[89014]: pgmap v1645: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 159 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5228/7686 objects degraded (68.020%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:31.731 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:31.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:31.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:31 smithi029 ceph-mon[93650]: pgmap v1645: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 159 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5228/7686 objects degraded (68.020%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:31.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:31 smithi029 ceph-mon[97787]: pgmap v1645: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 159 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 5228/7686 objects degraded (68.020%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:31.833 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:31.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:31.857 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:31.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:31.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:31.982 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:31.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:31.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:32.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:32.109 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:32.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:32.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:32.210 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:32.238 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:32.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:32.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:32.340 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:32.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:32.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:32.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:32.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:32.489 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:32.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:32.593 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:32.617 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:32.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:32.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:32 smithi087 ceph-mon[89014]: pgmap v1646: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 82 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5228/3963 objects degraded (131.920%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:32.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:41:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:41:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:41:32.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:32.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:32.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:32.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:32 smithi029 ceph-mon[97787]: pgmap v1646: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 82 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5228/3963 objects degraded (131.920%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:32 smithi029 ceph-mon[93650]: pgmap v1646: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 82 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5228/3963 objects degraded (131.920%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:32.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:32.869 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:32.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:32.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:32.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:41:32.971 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:32.994 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:32.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:32.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:33.096 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:33.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:33.121 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:33.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:33.221 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:33.245 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:33.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:33.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:33.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:33.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:33.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:33.496 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:33.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:33.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:33.598 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:33.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:33.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:33.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5228/3963 objects degraded (131.920%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:33.724 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:33.748 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:33.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:33.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5228/3963 objects degraded (131.920%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5228/3963 objects degraded (131.920%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:33.850 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:33.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:33.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:33.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:33.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:34.001 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:34.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:34.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:34.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:34.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:34.129 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:34.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:34.229 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:34.257 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:34.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:34.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:34.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:34.382 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:34.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:34.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:34.484 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:34.512 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:34.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:34.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:34.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:34.637 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:34.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:34.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:34.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:34 smithi087 ceph-mon[89014]: pgmap v1647: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 39 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 5228/1896 objects degraded (275.738%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:34.741 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:34.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:34 smithi029 ceph-mon[97787]: pgmap v1647: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 39 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 5228/1896 objects degraded (275.738%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:34.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:34 smithi029 ceph-mon[93650]: pgmap v1647: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 39 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 5228/1896 objects degraded (275.738%); 0 B/s, 14 objects/s recovering 2024-04-12T11:41:34.769 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:34.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:34.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:34.870 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:34.894 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:34.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:34.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:34.996 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:35.020 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:35.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:35.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:35.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:35.151 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:35.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:35.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:41:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:41:34] "GET /metrics HTTP/1.1" 200 36434 "" "Prometheus/2.43.0" 2024-04-12T11:41:35.279 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:35.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:35.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:35.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:35.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:35.405 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:35.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:35.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:35.530 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:35.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:35.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:35.631 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:35.646 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:41:35.662 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:35.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:35.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:35.764 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:35.791 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:35.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:35.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:35.893 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:35.917 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:35.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:35.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:36.018 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:36.046 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:36.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:36.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:36.148 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:36.175 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:36.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:36.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:36.277 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:36.304 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:36.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:36.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:36.430 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:36.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:36.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:36.532 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:36.560 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:36.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:36.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:36.662 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:36.691 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:36.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:36.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:36.817 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:36.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:36.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:36.868 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-04-12T11:41:36.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-04-12T11:41:36.919 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:36.946 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:36.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:36.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:37.048 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:37.061 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:41:37.075 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:37.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:37.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:37.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:37.202 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:37.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:37.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:37.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:37.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:37.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:37.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:37.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:37.455 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:37.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:37.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:37.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:37 smithi029 ceph-mon[93650]: pgmap v1648: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 421 op/s; 4915/72 objects degraded (6826.389%); 0 B/s, 37 objects/s recovering 2024-04-12T11:41:37.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:37.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:37.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:37 smithi029 ceph-mon[97787]: pgmap v1648: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 421 op/s; 4915/72 objects degraded (6826.389%); 0 B/s, 37 objects/s recovering 2024-04-12T11:41:37.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:37.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:37.556 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:37.581 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:37.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:37.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:37.682 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:37.706 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:37.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:37.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:37 smithi087 ceph-mon[89014]: pgmap v1648: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 421 op/s; 4915/72 objects degraded (6826.389%); 0 B/s, 37 objects/s recovering 2024-04-12T11:41:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:41:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:37.810 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:37.838 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:37.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:37.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:37.939 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:37.966 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:37.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:37.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:38.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:38.091 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:38.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:38.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:38.192 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:38.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:38.221 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:38.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:38.321 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:38.348 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:38.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:38.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:38.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:38.473 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:38.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:38.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:38.477 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 4915/72 objects degraded (6826.389%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:38.478 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 4915/72 objects degraded (6826.389%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:38.575 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:38.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:38.603 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:38.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:38.687 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:41:38.687 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:41:38.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:38.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 4915/72 objects degraded (6826.389%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:38.730 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:38.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:38.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:38.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:38.857 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:41:38.858 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:38.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:38.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:38.861 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:41:38.961 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:38.989 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:38.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:38.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:39.090 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:39.117 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:39.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:39.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:39.219 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:39.246 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:39.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:39.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:39.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:39.372 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:39.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:39.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:39.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:39.478 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:41:39.499 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:41:39.502 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:39.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:39.520 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:41:39.541 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:41:39.565 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:41:39.585 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:41:39.586 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:39 smithi029 ceph-mon[97787]: pgmap v1649: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 4915/72 objects degraded (6826.389%); 0 B/s, 37 objects/s recovering 2024-04-12T11:41:39.586 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:39 smithi029 ceph-mon[93650]: pgmap v1649: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 4915/72 objects degraded (6826.389%); 0 B/s, 37 objects/s recovering 2024-04-12T11:41:39.603 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:39.607 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:41:39.630 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:39.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:39.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:39.635 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:41:39.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:41:39.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:41:39.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:41:39.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:41:39.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:41:39.637 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:41:39.637 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:41:39.637 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:41:39.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:39 smithi087 ceph-mon[89014]: pgmap v1649: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 4915/72 objects degraded (6826.389%); 0 B/s, 37 objects/s recovering 2024-04-12T11:41:39.732 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:39.758 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:39.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:39.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:39.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:39.884 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:39.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:39.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:39.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:40.009 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:40.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:40.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:40.138 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:40.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:40.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:40.240 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:40.267 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:40.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:40.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:40.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:40.396 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:40.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:40.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:40.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:40.521 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:40.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:40.623 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:40.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:40 smithi029 ceph-mon[97787]: from='client.45032 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:41:40.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:40 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3453227778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:41:40.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:40 smithi029 ceph-mon[93650]: from='client.45032 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:41:40.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:40 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3453227778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:41:40.651 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:40.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:40.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:40.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:40 smithi087 ceph-mon[89014]: from='client.45032 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:41:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:40 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3453227778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:41:40.752 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:40.779 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:40.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:40.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:40.881 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:40.908 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:40.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:40.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:41.010 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:41.037 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:41.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:41.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:41.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:41.165 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:41.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:41.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:41.267 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:41.291 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:41.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:41.393 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:41.418 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:41.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:41.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:41.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:41.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:41.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:41.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:41.648 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:41.673 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:41.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:41.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:41 smithi087 ceph-mon[89014]: pgmap v1650: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 4451/72 objects degraded (6181.944%); 0 B/s, 73 objects/s recovering 2024-04-12T11:41:41.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:41 smithi029 ceph-mon[93650]: pgmap v1650: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 4451/72 objects degraded (6181.944%); 0 B/s, 73 objects/s recovering 2024-04-12T11:41:41.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:41 smithi029 ceph-mon[97787]: pgmap v1650: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 4451/72 objects degraded (6181.944%); 0 B/s, 73 objects/s recovering 2024-04-12T11:41:41.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:41.801 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:41.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:41.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:41.903 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:41.928 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:41.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:41.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:42.029 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:42.057 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:42.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:42.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:42.183 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:42.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:42.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:42.285 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:42.313 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:42.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:42.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:42.414 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:42.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:42.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:42.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:42.501 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGRecoveryDeleteReply(7.6 e1166,1147 7:6a377295:::benchmark_data_smithi029_161157_object23403:head,1154'8091)", 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:41:42.496829+0000", 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0013809359999999999, 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014092079999999999, 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:41:42.496829+0000", 2024-04-12T11:41:42.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:41:42.496829+0000", 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:41:42.496830+0000", 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9199999999999999e-07 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:41:42.496835+0000", 2024-04-12T11:41:42.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8860000000000003e-06 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:41:42.496848+0000", 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.279e-05 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:41:42.496855+0000", 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3549999999999999e-06 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:41:42.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:41:42.496915+0000", 2024-04-12T11:41:42.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9295000000000001e-05 2024-04-12T11:41:42.505 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:41:42.505 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:41:42.505 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:41:42.505 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:41:42.505 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2024-04-12T11:41:42.505 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 1 2024-04-12T11:41:42.505 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:41:42.541 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:42.564 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:42.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:42.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:42.666 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:42.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:42.694 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:42.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:41:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:41:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:41:42.795 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:42.822 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:42.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:42.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:42.924 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:42.951 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:42.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:42.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:43.053 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:43.079 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:43.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:43.181 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:43.205 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:43.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:43.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:43.306 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:43.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:43.333 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:43.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:43.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:43.457 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:43.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:43.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:43.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-04-12T11:41:43.559 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:43.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:43.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:43.667 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:43 smithi029 ceph-mon[97787]: pgmap v1651: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 4451/72 objects degraded (6181.944%); 0 B/s, 64 objects/s recovering 2024-04-12T11:41:43.667 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 4451/72 objects degraded (6181.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:43.667 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:43 smithi029 ceph-mon[93650]: pgmap v1651: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 4451/72 objects degraded (6181.944%); 0 B/s, 64 objects/s recovering 2024-04-12T11:41:43.667 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 4451/72 objects degraded (6181.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:43.688 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:43.714 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:43.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:43.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:43 smithi087 ceph-mon[89014]: pgmap v1651: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 211 op/s; 4451/72 objects degraded (6181.944%); 0 B/s, 64 objects/s recovering 2024-04-12T11:41:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 4451/72 objects degraded (6181.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:43.816 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:43.843 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:43.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:43.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:43.945 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:43.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:43.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:43.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:44.074 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:44.091 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1649,"stamp":"2024-04-12T11:41:37.837087+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":215302,"num_write_kb":6882447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6451,"num_bytes_recovered":361892588,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139518,"ondisk_log_size":139518,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3269968,"kb_used_data":747276,"kb_used_omap":175,"kb_used_meta":2522448,"kb_avail":746691248,"statfs":{"total":767960285184,"available":764611837952,"internally_reserved":0,"allocated":765210624,"data_stored":740535866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":2582986921},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-226099224,"num_objects":-3451,"num_object_clones":0,"num_object_copies":-10353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-376,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3451,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":376,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002367"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1685,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.337912+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:41:29.337912+0000","last_peered":"2024-04-12T11:41:29.337912+0000","last_clean":"2024-04-12T11:41:29.337912+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:41:29.337912+0000","last_undegraded":"2024-04-12T11:41:29.337912+0000","last_fullsized":"2024-04-12T11:41:29.337912+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1693,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.348882+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:41:29.348882+0000","last_peered":"2024-04-12T11:41:29.348882+0000","last_clean":"2024-04-12T11:41:29.348882+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:41:29.348882+0000","last_undegraded":"2024-04-12T11:41:29.348882+0000","last_fullsized":"2024-04-12T11:41:29.348882+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1744,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.362639+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:41:29.362639+0000","last_peered":"2024-04-12T11:41:29.362639+0000","last_clean":"2024-04-12T11:41:29.362639+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:41:29.362639+0000","last_undegraded":"2024-04-12T11:41:29.362639+0000","last_fullsized":"2024-04-12T11:41:29.362639+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1752,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.377062+0000","last_change":"2024-04-12T11:38:21.933647+0000","last_active":"2024-04-12T11:41:29.377062+0000","last_peered":"2024-04-12T11:41:29.377062+0000","last_clean":"2024-04-12T11:41:29.377062+0000","last_became_active":"2024-04-12T11:38:21.933295+0000","last_became_peered":"2024-04-12T11:38:21.933295+0000","last_unstale":"2024-04-12T11:41:29.377062+0000","last_undegraded":"2024-04-12T11:41:29.377062+0000","last_fullsized":"2024-04-12T11:41:29.377062+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:37.402139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1701,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.348762+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:41:29.348762+0000","last_peered":"2024-04-12T11:41:29.348762+0000","last_clean":"2024-04-12T11:41:29.348762+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:41:29.348762+0000","last_undegraded":"2024-04-12T11:41:29.348762+0000","last_fullsized":"2024-04-12T11:41:29.348762+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1111,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.971660+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:41:13.971660+0000","last_peered":"2024-04-12T11:41:13.971660+0000","last_clean":"2024-04-12T11:41:13.971660+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:41:13.971660+0000","last_undegraded":"2024-04-12T11:41:13.971660+0000","last_fullsized":"2024-04-12T11:41:13.971660+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1736,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.357177+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:41:29.357177+0000","last_peered":"2024-04-12T11:41:29.357177+0000","last_clean":"2024-04-12T11:41:29.357177+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:41:29.357177+0000","last_undegraded":"2024-04-12T11:41:29.357177+0000","last_fullsized":"2024-04-12T11:41:29.357177+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1674,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.348857+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:41:29.348857+0000","last_peered":"2024-04-12T11:41:29.348857+0000","last_clean":"2024-04-12T11:41:29.348857+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:41:29.348857+0000","last_undegraded":"2024-04-12T11:41:29.348857+0000","last_fullsized":"2024-04-12T11:41:29.348857+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1116,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.976538+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:41:13.976538+0000","last_peered":"2024-04-12T11:41:13.976538+0000","last_clean":"2024-04-12T11:41:13.976538+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:41:13.976538+0000","last_undegraded":"2024-04-12T11:41:13.976538+0000","last_fullsized":"2024-04-12T11:41:13.976538+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1654,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.348474+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:41:29.348474+0000","last_peered":"2024-04-12T11:41:29.348474+0000","last_clean":"2024-04-12T11:41:29.348474+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:41:29.348474+0000","last_undegraded":"2024-04-12T11:41:29.348474+0000","last_fullsized":"2024-04-12T11:41:29.348474+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1125,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.963628+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:41:13.963628+0000","last_peered":"2024-04-12T11:41:13.963628+0000","last_clean":"2024-04-12T11:41:13.963628+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:41:13.963628+0000","last_undegraded":"2024-04-12T11:41:13.963628+0000","last_fullsized":"2024-04-12T11:41:13.963628+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1736,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.362996+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:41:29.362996+0000","last_peered":"2024-04-12T11:41:29.362996+0000","last_clean":"2024-04-12T11:41:29.362996+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:41:29.362996+0000","last_undegraded":"2024-04-12T11:41:29.362996+0000","last_fullsized":"2024-04-12T11:41:29.362996+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1657,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.337997+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:41:29.337997+0000","last_peered":"2024-04-12T11:41:29.337997+0000","last_clean":"2024-04-12T11:41:29.337997+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:41:29.337997+0000","last_undegraded":"2024-04-12T11:41:29.337997+0000","last_fullsized":"2024-04-12T11:41:29.337997+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1648,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.338187+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:41:29.338187+0000","last_peered":"2024-04-12T11:41:29.338187+0000","last_clean":"2024-04-12T11:41:29.338187+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:41:29.338187+0000","last_undegraded":"2024-04-12T11:41:29.338187+0000","last_fullsized":"2024-04-12T11:41:29.338187+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1125,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.963638+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:41:13.963638+0000","last_peered":"2024-04-12T11:41:13.963638+0000","last_clean":"2024-04-12T11:41:13.963638+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:41:13.963638+0000","last_undegraded":"2024-04-12T11:41:13.963638+0000","last_fullsized":"2024-04-12T11:41:13.963638+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1685,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.348437+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:41:29.348437+0000","last_peered":"2024-04-12T11:41:29.348437+0000","last_clean":"2024-04-12T11:41:29.348437+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:41:29.348437+0000","last_undegraded":"2024-04-12T11:41:29.348437+0000","last_fullsized":"2024-04-12T11:41:29.348437+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1737,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.357702+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:41:29.357702+0000","last_peered":"2024-04-12T11:41:29.357702+0000","last_clean":"2024-04-12T11:41:29.357702+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:41:29.357702+0000","last_undegraded":"2024-04-12T11:41:29.357702+0000","last_fullsized":"2024-04-12T11:41:29.357702+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1115,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.973316+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:41:13.973316+0000","last_peered":"2024-04-12T11:41:13.973316+0000","last_clean":"2024-04-12T11:41:13.973316+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:41:13.973316+0000","last_undegraded":"2024-04-12T11:41:13.973316+0000","last_fullsized":"2024-04-12T11:41:13.973316+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1108,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.963307+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:41:13.963307+0000","last_peered":"2024-04-12T11:41:13.963307+0000","last_clean":"2024-04-12T11:41:13.963307+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:41:13.963307+0000","last_undegraded":"2024-04-12T11:41:13.963307+0000","last_fullsized":"2024-04-12T11:41:13.963307+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1738,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.376975+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:41:29.376975+0000","last_peered":"2024-04-12T11:41:29.376975+0000","last_clean":"2024-04-12T11:41:29.376975+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:41:29.376975+0000","last_undegraded":"2024-04-12T11:41:29.376975+0000","last_fullsized":"2024-04-12T11:41:29.376975+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1119,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.971030+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:41:13.971030+0000","last_peered":"2024-04-12T11:41:13.971030+0000","last_clean":"2024-04-12T11:41:13.971030+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:41:13.971030+0000","last_undegraded":"2024-04-12T11:41:13.971030+0000","last_fullsized":"2024-04-12T11:41:13.971030+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1685,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.359250+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:41:29.359250+0000","last_peered":"2024-04-12T11:41:29.359250+0000","last_clean":"2024-04-12T11:41:29.359250+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:41:29.359250+0000","last_undegraded":"2024-04-12T11:41:29.359250+0000","last_fullsized":"2024-04-12T11:41:29.359250+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"1166'13540","reported_seq":13618,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.084297+0000","last_change":"2024-04-12T11:37:03.057199+0000","last_active":"2024-04-12T11:41:31.084297+0000","last_peered":"2024-04-12T11:41:31.084297+0000","last_clean":"2024-04-12T11:41:31.084297+0000","last_became_active":"2024-04-12T11:37:03.056776+0000","last_became_peered":"2024-04-12T11:37:03.056776+0000","last_unstale":"2024-04-12T11:41:31.084297+0000","last_undegraded":"2024-04-12T11:41:31.084297+0000","last_fullsized":"2024-04-12T11:41:31.084297+0000","mapping_epoch":1133,"log_start":"1145'5600","ondisk_log_start":"1145'5600","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7940,"log_dups_size":0,"ondisk_log_size":7940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:16:29.272872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13540,"num_write_kb":433280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1124,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.964265+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:41:13.964265+0000","last_peered":"2024-04-12T11:41:13.964265+0000","last_clean":"2024-04-12T11:41:13.964265+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:41:13.964265+0000","last_undegraded":"2024-04-12T11:41:13.964265+0000","last_fullsized":"2024-04-12T11:41:13.964265+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1102,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.972799+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:41:13.972799+0000","last_peered":"2024-04-12T11:41:13.972799+0000","last_clean":"2024-04-12T11:41:13.972799+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:41:13.972799+0000","last_undegraded":"2024-04-12T11:41:13.972799+0000","last_fullsized":"2024-04-12T11:41:13.972799+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1166'13560","reported_seq":13638,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.065053+0000","last_change":"2024-04-12T11:37:03.059071+0000","last_active":"2024-04-12T11:41:31.065053+0000","last_peered":"2024-04-12T11:41:31.065053+0000","last_clean":"2024-04-12T11:41:31.065053+0000","last_became_active":"2024-04-12T11:37:03.058769+0000","last_became_peered":"2024-04-12T11:37:03.058769+0000","last_unstale":"2024-04-12T11:41:31.065053+0000","last_undegraded":"2024-04-12T11:41:31.065053+0000","last_fullsized":"2024-04-12T11:41:31.065053+0000","mapping_epoch":1133,"log_start":"1148'6000","ondisk_log_start":"1148'6000","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7560,"log_dups_size":0,"ondisk_log_size":7560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:43:26.795096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13560,"num_write_kb":433920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1125,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.963627+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:41:13.963627+0000","last_peered":"2024-04-12T11:41:13.963627+0000","last_clean":"2024-04-12T11:41:13.963627+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:41:13.963627+0000","last_undegraded":"2024-04-12T11:41:13.963627+0000","last_fullsized":"2024-04-12T11:41:13.963627+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1671,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.337922+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:41:29.337922+0000","last_peered":"2024-04-12T11:41:29.337922+0000","last_clean":"2024-04-12T11:41:29.337922+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:41:29.337922+0000","last_undegraded":"2024-04-12T11:41:29.337922+0000","last_fullsized":"2024-04-12T11:41:29.337922+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1166'13466","reported_seq":13544,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.046598+0000","last_change":"2024-04-12T11:37:03.059390+0000","last_active":"2024-04-12T11:41:31.046598+0000","last_peered":"2024-04-12T11:41:31.046598+0000","last_clean":"2024-04-12T11:41:31.046598+0000","last_became_active":"2024-04-12T11:37:03.058045+0000","last_became_peered":"2024-04-12T11:37:03.058045+0000","last_unstale":"2024-04-12T11:41:31.046598+0000","last_undegraded":"2024-04-12T11:41:31.046598+0000","last_fullsized":"2024-04-12T11:41:31.046598+0000","mapping_epoch":1133,"log_start":"1142'4600","ondisk_log_start":"1142'4600","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8866,"log_dups_size":0,"ondisk_log_size":8866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:14:01.284893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13466,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1094,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.963535+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:41:13.963535+0000","last_peered":"2024-04-12T11:41:13.963535+0000","last_clean":"2024-04-12T11:41:13.963535+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:41:13.963535+0000","last_undegraded":"2024-04-12T11:41:13.963535+0000","last_fullsized":"2024-04-12T11:41:13.963535+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1111,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.971200+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:41:13.971200+0000","last_peered":"2024-04-12T11:41:13.971200+0000","last_clean":"2024-04-12T11:41:13.971200+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:41:13.971200+0000","last_undegraded":"2024-04-12T11:41:13.971200+0000","last_fullsized":"2024-04-12T11:41:13.971200+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1166'13608","reported_seq":13686,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.086032+0000","last_change":"2024-04-12T11:37:03.054920+0000","last_active":"2024-04-12T11:41:31.086032+0000","last_peered":"2024-04-12T11:41:31.086032+0000","last_clean":"2024-04-12T11:41:31.086032+0000","last_became_active":"2024-04-12T11:37:03.054603+0000","last_became_peered":"2024-04-12T11:37:03.054603+0000","last_unstale":"2024-04-12T11:41:31.086032+0000","last_undegraded":"2024-04-12T11:41:31.086032+0000","last_fullsized":"2024-04-12T11:41:31.086032+0000","mapping_epoch":1133,"log_start":"1141'3600","ondisk_log_start":"1141'3600","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:34:45.149813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13608,"num_write_kb":435456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1117,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.972992+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:41:13.972992+0000","last_peered":"2024-04-12T11:41:13.972992+0000","last_clean":"2024-04-12T11:41:13.972992+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:41:13.972992+0000","last_undegraded":"2024-04-12T11:41:13.972992+0000","last_fullsized":"2024-04-12T11:41:13.972992+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1747,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.357338+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:41:29.357338+0000","last_peered":"2024-04-12T11:41:29.357338+0000","last_clean":"2024-04-12T11:41:29.357338+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:41:29.357338+0000","last_undegraded":"2024-04-12T11:41:29.357338+0000","last_fullsized":"2024-04-12T11:41:29.357338+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1166'13612","reported_seq":13690,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.055442+0000","last_change":"2024-04-12T11:37:03.063209+0000","last_active":"2024-04-12T11:41:31.055442+0000","last_peered":"2024-04-12T11:41:31.055442+0000","last_clean":"2024-04-12T11:41:31.055442+0000","last_became_active":"2024-04-12T11:37:03.062848+0000","last_became_peered":"2024-04-12T11:37:03.062848+0000","last_unstale":"2024-04-12T11:41:31.055442+0000","last_undegraded":"2024-04-12T11:41:31.055442+0000","last_fullsized":"2024-04-12T11:41:31.055442+0000","mapping_epoch":1133,"log_start":"1142'4700","ondisk_log_start":"1142'4700","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8912,"log_dups_size":0,"ondisk_log_size":8912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:55:26.299047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13612,"num_write_kb":435584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1679,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.348234+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:41:29.348234+0000","last_peered":"2024-04-12T11:41:29.348234+0000","last_clean":"2024-04-12T11:41:29.348234+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:41:29.348234+0000","last_undegraded":"2024-04-12T11:41:29.348234+0000","last_fullsized":"2024-04-12T11:41:29.348234+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1662,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.338137+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:41:29.338137+0000","last_peered":"2024-04-12T11:41:29.338137+0000","last_clean":"2024-04-12T11:41:29.338137+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:41:29.338137+0000","last_undegraded":"2024-04-12T11:41:29.338137+0000","last_fullsized":"2024-04-12T11:41:29.338137+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1166'13318","reported_seq":13396,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.087760+0000","last_change":"2024-04-12T11:37:03.060739+0000","last_active":"2024-04-12T11:41:31.087760+0000","last_peered":"2024-04-12T11:41:31.087760+0000","last_clean":"2024-04-12T11:41:31.087760+0000","last_became_active":"2024-04-12T11:37:03.060414+0000","last_became_peered":"2024-04-12T11:37:03.060414+0000","last_unstale":"2024-04-12T11:41:31.087760+0000","last_undegraded":"2024-04-12T11:41:31.087760+0000","last_fullsized":"2024-04-12T11:41:31.087760+0000","mapping_epoch":1133,"log_start":"1141'3300","ondisk_log_start":"1141'3300","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10018,"log_dups_size":0,"ondisk_log_size":10018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:12:30.259488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13318,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":688,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.337858+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:41:29.337858+0000","last_peered":"2024-04-12T11:41:29.337858+0000","last_clean":"2024-04-12T11:41:29.337858+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:41:29.337858+0000","last_undegraded":"2024-04-12T11:41:29.337858+0000","last_fullsized":"2024-04-12T11:41:29.337858+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1684,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.338237+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:41:29.338237+0000","last_peered":"2024-04-12T11:41:29.338237+0000","last_clean":"2024-04-12T11:41:29.338237+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:41:29.338237+0000","last_undegraded":"2024-04-12T11:41:29.338237+0000","last_fullsized":"2024-04-12T11:41:29.338237+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1166'13610","reported_seq":13655,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.101554+0000","last_change":"2024-04-12T11:37:03.060364+0000","last_active":"2024-04-12T11:41:31.101554+0000","last_peered":"2024-04-12T11:41:31.101554+0000","last_clean":"2024-04-12T11:41:31.101554+0000","last_became_active":"2024-04-12T11:37:03.060146+0000","last_became_peered":"2024-04-12T11:37:03.060146+0000","last_unstale":"2024-04-12T11:41:31.101554+0000","last_undegraded":"2024-04-12T11:41:31.101554+0000","last_fullsized":"2024-04-12T11:41:31.101554+0000","mapping_epoch":1133,"log_start":"1142'4200","ondisk_log_start":"1142'4200","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9410,"log_dups_size":0,"ondisk_log_size":9410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:03.796242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13610,"num_write_kb":435457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1670,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.337950+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:41:29.337950+0000","last_peered":"2024-04-12T11:41:29.337950+0000","last_clean":"2024-04-12T11:41:29.337950+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:41:29.337950+0000","last_undegraded":"2024-04-12T11:41:29.337950+0000","last_fullsized":"2024-04-12T11:41:29.337950+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1649,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.338519+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:41:29.338519+0000","last_peered":"2024-04-12T11:41:29.338519+0000","last_clean":"2024-04-12T11:41:29.338519+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:41:29.338519+0000","last_undegraded":"2024-04-12T11:41:29.338519+0000","last_fullsized":"2024-04-12T11:41:29.338519+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1166'13626","reported_seq":13704,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.079126+0000","last_change":"2024-04-12T11:37:03.056864+0000","last_active":"2024-04-12T11:41:31.079126+0000","last_peered":"2024-04-12T11:41:31.079126+0000","last_clean":"2024-04-12T11:41:31.079126+0000","last_became_active":"2024-04-12T11:37:03.056523+0000","last_became_peered":"2024-04-12T11:37:03.056523+0000","last_unstale":"2024-04-12T11:41:31.079126+0000","last_undegraded":"2024-04-12T11:41:31.079126+0000","last_fullsized":"2024-04-12T11:41:31.079126+0000","mapping_epoch":1133,"log_start":"1148'6100","ondisk_log_start":"1148'6100","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7526,"log_dups_size":0,"ondisk_log_size":7526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:57:58.750385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13626,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1687,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.348938+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:41:29.348938+0000","last_peered":"2024-04-12T11:41:29.348938+0000","last_clean":"2024-04-12T11:41:29.348938+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:41:29.348938+0000","last_undegraded":"2024-04-12T11:41:29.348938+0000","last_fullsized":"2024-04-12T11:41:29.348938+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1745,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.361529+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:41:29.361529+0000","last_peered":"2024-04-12T11:41:29.361529+0000","last_clean":"2024-04-12T11:41:29.361529+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:41:29.361529+0000","last_undegraded":"2024-04-12T11:41:29.361529+0000","last_fullsized":"2024-04-12T11:41:29.361529+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1166'13248","reported_seq":24766,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.063524+0000","last_change":"2024-04-12T11:41:03.934075+0000","last_active":"2024-04-12T11:41:31.063524+0000","last_peered":"2024-04-12T11:41:31.063524+0000","last_clean":"2024-04-12T11:41:31.063524+0000","last_became_active":"2024-04-12T11:41:03.904470+0000","last_became_peered":"2024-04-12T11:41:03.904470+0000","last_unstale":"2024-04-12T11:41:31.063524+0000","last_undegraded":"2024-04-12T11:41:31.063524+0000","last_fullsized":"2024-04-12T11:41:31.063524+0000","mapping_epoch":1163,"log_start":"1144'5300","ondisk_log_start":"1144'5300","created":1133,"last_epoch_clean":1164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7948,"log_dups_size":0,"ondisk_log_size":7948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:15:39.102459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13248,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5715,"num_bytes_recovered":354222080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1747,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.373567+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:41:29.373567+0000","last_peered":"2024-04-12T11:41:29.373567+0000","last_clean":"2024-04-12T11:41:29.373567+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:41:29.373567+0000","last_undegraded":"2024-04-12T11:41:29.373567+0000","last_fullsized":"2024-04-12T11:41:29.373567+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1133,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.973415+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:41:13.973415+0000","last_peered":"2024-04-12T11:41:13.973415+0000","last_clean":"2024-04-12T11:41:13.973415+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:41:13.973415+0000","last_undegraded":"2024-04-12T11:41:13.973415+0000","last_fullsized":"2024-04-12T11:41:13.973415+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1738,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.362851+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:41:29.362851+0000","last_peered":"2024-04-12T11:41:29.362851+0000","last_clean":"2024-04-12T11:41:29.362851+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:41:29.362851+0000","last_undegraded":"2024-04-12T11:41:29.362851+0000","last_fullsized":"2024-04-12T11:41:29.362851+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1166'13466","reported_seq":13510,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.034428+0000","last_change":"2024-04-12T11:37:03.060583+0000","last_active":"2024-04-12T11:41:31.034428+0000","last_peered":"2024-04-12T11:41:31.034428+0000","last_clean":"2024-04-12T11:41:31.034428+0000","last_became_active":"2024-04-12T11:37:03.060125+0000","last_became_peered":"2024-04-12T11:37:03.060125+0000","last_unstale":"2024-04-12T11:41:31.034428+0000","last_undegraded":"2024-04-12T11:41:31.034428+0000","last_fullsized":"2024-04-12T11:41:31.034428+0000","mapping_epoch":1133,"log_start":"1142'4000","ondisk_log_start":"1142'4000","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":9466,"log_dups_size":0,"ondisk_log_size":9466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:35:30.859950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13466,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1123,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.973441+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:41:13.973441+0000","last_peered":"2024-04-12T11:41:13.973441+0000","last_clean":"2024-04-12T11:41:13.973441+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:41:13.973441+0000","last_undegraded":"2024-04-12T11:41:13.973441+0000","last_fullsized":"2024-04-12T11:41:13.973441+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1670,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.338118+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:41:29.338118+0000","last_peered":"2024-04-12T11:41:29.338118+0000","last_clean":"2024-04-12T11:41:29.338118+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:41:29.338118+0000","last_undegraded":"2024-04-12T11:41:29.338118+0000","last_fullsized":"2024-04-12T11:41:29.338118+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1747,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.373226+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:41:29.373226+0000","last_peered":"2024-04-12T11:41:29.373226+0000","last_clean":"2024-04-12T11:41:29.373226+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:41:29.373226+0000","last_undegraded":"2024-04-12T11:41:29.373226+0000","last_fullsized":"2024-04-12T11:41:29.373226+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1166'13244","reported_seq":13322,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.032407+0000","last_change":"2024-04-12T11:37:03.058470+0000","last_active":"2024-04-12T11:41:31.032407+0000","last_peered":"2024-04-12T11:41:31.032407+0000","last_clean":"2024-04-12T11:41:31.032407+0000","last_became_active":"2024-04-12T11:37:03.058149+0000","last_became_peered":"2024-04-12T11:37:03.058149+0000","last_unstale":"2024-04-12T11:41:31.032407+0000","last_undegraded":"2024-04-12T11:41:31.032407+0000","last_fullsized":"2024-04-12T11:41:31.032407+0000","mapping_epoch":1133,"log_start":"1142'4400","ondisk_log_start":"1142'4400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8844,"log_dups_size":0,"ondisk_log_size":8844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:25:06.934877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13244,"num_write_kb":423808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1711,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.345896+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:41:29.345896+0000","last_peered":"2024-04-12T11:41:29.345896+0000","last_clean":"2024-04-12T11:41:29.345896+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:41:29.345896+0000","last_undegraded":"2024-04-12T11:41:29.345896+0000","last_fullsized":"2024-04-12T11:41:29.345896+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1738,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.361992+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:41:29.361992+0000","last_peered":"2024-04-12T11:41:29.361992+0000","last_clean":"2024-04-12T11:41:29.361992+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:41:29.361992+0000","last_undegraded":"2024-04-12T11:41:29.361992+0000","last_fullsized":"2024-04-12T11:41:29.361992+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1740,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.362563+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:41:29.362563+0000","last_peered":"2024-04-12T11:41:29.362563+0000","last_clean":"2024-04-12T11:41:29.362563+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:41:29.362563+0000","last_undegraded":"2024-04-12T11:41:29.362563+0000","last_fullsized":"2024-04-12T11:41:29.362563+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1166'13312","reported_seq":14846,"reported_epoch":1166,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-04-12T11:41:34.103858+0000","last_change":"2024-04-12T11:41:02.486992+0000","last_active":"2024-04-12T11:41:34.103858+0000","last_peered":"2024-04-12T11:41:34.103858+0000","last_clean":"2024-04-12T11:38:20.888095+0000","last_became_active":"2024-04-12T11:38:23.424268+0000","last_became_peered":"2024-04-12T11:38:23.424268+0000","last_unstale":"2024-04-12T11:41:34.103858+0000","last_undegraded":"2024-04-12T11:38:23.284099+0000","last_fullsized":"2024-04-12T11:38:23.245558+0000","mapping_epoch":1147,"log_start":"1142'4400","ondisk_log_start":"1142'4400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8912,"log_dups_size":0,"ondisk_log_size":8912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13312,"num_write_kb":425984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":726,"num_bytes_recovered":5898240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,7","objects":-4915}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1123,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.972881+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:41:13.972881+0000","last_peered":"2024-04-12T11:41:13.972881+0000","last_clean":"2024-04-12T11:41:13.972881+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:41:13.972881+0000","last_undegraded":"2024-04-12T11:41:13.972881+0000","last_fullsized":"2024-04-12T11:41:13.972881+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1478,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.369009+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:41:29.369009+0000","last_peered":"2024-04-12T11:41:29.369009+0000","last_clean":"2024-04-12T11:41:29.369009+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:41:29.369009+0000","last_undegraded":"2024-04-12T11:41:29.369009+0000","last_fullsized":"2024-04-12T11:41:29.369009+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1654,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.349678+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:41:29.349678+0000","last_peered":"2024-04-12T11:41:29.349678+0000","last_clean":"2024-04-12T11:41:29.349678+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:41:29.349678+0000","last_undegraded":"2024-04-12T11:41:29.349678+0000","last_fullsized":"2024-04-12T11:41:29.349678+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1111,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.971578+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:41:13.971578+0000","last_peered":"2024-04-12T11:41:13.971578+0000","last_clean":"2024-04-12T11:41:13.971578+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:41:13.971578+0000","last_undegraded":"2024-04-12T11:41:13.971578+0000","last_fullsized":"2024-04-12T11:41:13.971578+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1166'13236","reported_seq":13280,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.084129+0000","last_change":"2024-04-12T11:37:03.061698+0000","last_active":"2024-04-12T11:41:31.084129+0000","last_peered":"2024-04-12T11:41:31.084129+0000","last_clean":"2024-04-12T11:41:31.084129+0000","last_became_active":"2024-04-12T11:37:03.061469+0000","last_became_peered":"2024-04-12T11:37:03.061469+0000","last_unstale":"2024-04-12T11:41:31.084129+0000","last_undegraded":"2024-04-12T11:41:31.084129+0000","last_fullsized":"2024-04-12T11:41:31.084129+0000","mapping_epoch":1133,"log_start":"1142'4600","ondisk_log_start":"1142'4600","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":8636,"log_dups_size":0,"ondisk_log_size":8636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:09.906340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13236,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1130,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.963081+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:41:13.963081+0000","last_peered":"2024-04-12T11:41:13.963081+0000","last_clean":"2024-04-12T11:41:13.963081+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:41:13.963081+0000","last_undegraded":"2024-04-12T11:41:13.963081+0000","last_fullsized":"2024-04-12T11:41:13.963081+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1748,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.357680+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:41:29.357680+0000","last_peered":"2024-04-12T11:41:29.357680+0000","last_clean":"2024-04-12T11:41:29.357680+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:41:29.357680+0000","last_undegraded":"2024-04-12T11:41:29.357680+0000","last_fullsized":"2024-04-12T11:41:29.357680+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1748,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.376873+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:41:29.376873+0000","last_peered":"2024-04-12T11:41:29.376873+0000","last_clean":"2024-04-12T11:41:29.376873+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:41:29.376873+0000","last_undegraded":"2024-04-12T11:41:29.376873+0000","last_fullsized":"2024-04-12T11:41:29.376873+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"1166'13494","reported_seq":13538,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:30.954329+0000","last_change":"2024-04-12T11:37:03.060455+0000","last_active":"2024-04-12T11:41:30.954329+0000","last_peered":"2024-04-12T11:41:30.954329+0000","last_clean":"2024-04-12T11:41:30.954329+0000","last_became_active":"2024-04-12T11:37:03.060244+0000","last_became_peered":"2024-04-12T11:37:03.060244+0000","last_unstale":"2024-04-12T11:41:30.954329+0000","last_undegraded":"2024-04-12T11:41:30.954329+0000","last_fullsized":"2024-04-12T11:41:30.954329+0000","mapping_epoch":1133,"log_start":"1141'3400","ondisk_log_start":"1141'3400","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":10094,"log_dups_size":0,"ondisk_log_size":10094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:47.787368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13494,"num_write_kb":431808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1130,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.963576+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:41:13.963576+0000","last_peered":"2024-04-12T11:41:13.963576+0000","last_clean":"2024-04-12T11:41:13.963576+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:41:13.963576+0000","last_undegraded":"2024-04-12T11:41:13.963576+0000","last_fullsized":"2024-04-12T11:41:13.963576+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1738,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.373481+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:41:29.373481+0000","last_peered":"2024-04-12T11:41:29.373481+0000","last_clean":"2024-04-12T11:41:29.373481+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:41:29.373481+0000","last_undegraded":"2024-04-12T11:41:29.373481+0000","last_fullsized":"2024-04-12T11:41:29.373481+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1731,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.357619+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:41:29.357619+0000","last_peered":"2024-04-12T11:41:29.357619+0000","last_clean":"2024-04-12T11:41:29.357619+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:41:29.357619+0000","last_undegraded":"2024-04-12T11:41:29.357619+0000","last_fullsized":"2024-04-12T11:41:29.357619+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1166'13184","reported_seq":13262,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.063867+0000","last_change":"2024-04-12T11:37:03.057084+0000","last_active":"2024-04-12T11:41:31.063867+0000","last_peered":"2024-04-12T11:41:31.063867+0000","last_clean":"2024-04-12T11:41:31.063867+0000","last_became_active":"2024-04-12T11:37:03.056772+0000","last_became_peered":"2024-04-12T11:37:03.056772+0000","last_unstale":"2024-04-12T11:41:31.063867+0000","last_undegraded":"2024-04-12T11:41:31.063867+0000","last_fullsized":"2024-04-12T11:41:31.063867+0000","mapping_epoch":1133,"log_start":"1145'5600","ondisk_log_start":"1145'5600","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7584,"log_dups_size":0,"ondisk_log_size":7584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:02:41.845790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13184,"num_write_kb":421888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1670,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.359134+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:41:29.359134+0000","last_peered":"2024-04-12T11:41:29.359134+0000","last_clean":"2024-04-12T11:41:29.359134+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:41:29.359134+0000","last_undegraded":"2024-04-12T11:41:29.359134+0000","last_fullsized":"2024-04-12T11:41:29.359134+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1737,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.361367+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:41:29.361367+0000","last_peered":"2024-04-12T11:41:29.361367+0000","last_clean":"2024-04-12T11:41:29.361367+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:41:29.361367+0000","last_undegraded":"2024-04-12T11:41:29.361367+0000","last_fullsized":"2024-04-12T11:41:29.361367+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1668,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.349021+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:41:29.349021+0000","last_peered":"2024-04-12T11:41:29.349021+0000","last_clean":"2024-04-12T11:41:29.349021+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:41:29.349021+0000","last_undegraded":"2024-04-12T11:41:29.349021+0000","last_fullsized":"2024-04-12T11:41:29.349021+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1166'13478","reported_seq":13556,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:31.031112+0000","last_change":"2024-04-12T11:37:03.060299+0000","last_active":"2024-04-12T11:41:31.031112+0000","last_peered":"2024-04-12T11:41:31.031112+0000","last_clean":"2024-04-12T11:41:31.031112+0000","last_became_active":"2024-04-12T11:37:03.058897+0000","last_became_peered":"2024-04-12T11:37:03.058897+0000","last_unstale":"2024-04-12T11:41:31.031112+0000","last_undegraded":"2024-04-12T11:41:31.031112+0000","last_fullsized":"2024-04-12T11:41:31.031112+0000","mapping_epoch":1133,"log_start":"1148'5900","ondisk_log_start":"1148'5900","created":1133,"last_epoch_clean":1134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:37:02.037842+0000","last_clean_scrub_stamp":"2024-04-12T11:37:02.037842+0000","objects_scrubbed":0,"log_size":7578,"log_dups_size":0,"ondisk_log_size":7578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:45:37.844216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13478,"num_write_kb":431296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1135,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.971359+0000","last_change":"2024-04-12T11:39:23.349899+0000","last_active":"2024-04-12T11:41:13.971359+0000","last_peered":"2024-04-12T11:41:13.971359+0000","last_clean":"2024-04-12T11:41:13.971359+0000","last_became_active":"2024-04-12T11:39:23.349526+0000","last_became_peered":"2024-04-12T11:39:23.349526+0000","last_unstale":"2024-04-12T11:41:13.971359+0000","last_undegraded":"2024-04-12T11:41:13.971359+0000","last_fullsized":"2024-04-12T11:41:13.971359+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:36:47.115912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1751,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.377810+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:41:29.377810+0000","last_peered":"2024-04-12T11:41:29.377810+0000","last_clean":"2024-04-12T11:41:29.377810+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:41:29.377810+0000","last_undegraded":"2024-04-12T11:41:29.377810+0000","last_fullsized":"2024-04-12T11:41:29.377810+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1657,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.337666+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:41:29.337666+0000","last_peered":"2024-04-12T11:41:29.337666+0000","last_clean":"2024-04-12T11:41:29.337666+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:41:29.337666+0000","last_undegraded":"2024-04-12T11:41:29.337666+0000","last_fullsized":"2024-04-12T11:41:29.337666+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1740,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.357509+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:41:29.357509+0000","last_peered":"2024-04-12T11:41:29.357509+0000","last_clean":"2024-04-12T11:41:29.357509+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:41:29.357509+0000","last_undegraded":"2024-04-12T11:41:29.357509+0000","last_fullsized":"2024-04-12T11:41:29.357509+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":56,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.346222+0000","last_change":"2024-04-12T11:38:06.839832+0000","last_active":"2024-04-12T11:41:29.346222+0000","last_peered":"2024-04-12T11:41:29.346222+0000","last_clean":"2024-04-12T11:41:29.346222+0000","last_became_active":"2024-04-12T11:38:06.839460+0000","last_became_peered":"2024-04-12T11:38:06.839460+0000","last_unstale":"2024-04-12T11:41:29.346222+0000","last_undegraded":"2024-04-12T11:41:29.346222+0000","last_fullsized":"2024-04-12T11:41:29.346222+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:25:27.959933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1133,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.973155+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:41:13.973155+0000","last_peered":"2024-04-12T11:41:13.973155+0000","last_clean":"2024-04-12T11:41:13.973155+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:41:13.973155+0000","last_undegraded":"2024-04-12T11:41:13.973155+0000","last_fullsized":"2024-04-12T11:41:13.973155+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1685,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.338501+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:41:29.338501+0000","last_peered":"2024-04-12T11:41:29.338501+0000","last_clean":"2024-04-12T11:41:29.338501+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:41:29.338501+0000","last_undegraded":"2024-04-12T11:41:29.338501+0000","last_fullsized":"2024-04-12T11:41:29.338501+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1118,"reported_epoch":1165,"state":"active+clean","last_fresh":"2024-04-12T11:41:13.973514+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:41:13.973514+0000","last_peered":"2024-04-12T11:41:13.973514+0000","last_clean":"2024-04-12T11:41:13.973514+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:41:13.973514+0000","last_undegraded":"2024-04-12T11:41:13.973514+0000","last_fullsized":"2024-04-12T11:41:13.973514+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1676,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.337718+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:41:29.337718+0000","last_peered":"2024-04-12T11:41:29.337718+0000","last_clean":"2024-04-12T11:41:29.337718+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:41:29.337718+0000","last_undegraded":"2024-04-12T11:41:29.337718+0000","last_fullsized":"2024-04-12T11:41:29.337718+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1669,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.364862+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:41:29.364862+0000","last_peered":"2024-04-12T11:41:29.364862+0000","last_clean":"2024-04-12T11:41:29.364862+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:41:29.364862+0000","last_undegraded":"2024-04-12T11:41:29.364862+0000","last_fullsized":"2024-04-12T11:41:29.364862+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1679,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.348334+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:41:29.348334+0000","last_peered":"2024-04-12T11:41:29.348334+0000","last_clean":"2024-04-12T11:41:29.348334+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:41:29.348334+0000","last_undegraded":"2024-04-12T11:41:29.348334+0000","last_fullsized":"2024-04-12T11:41:29.348334+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1769,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.357288+0000","last_change":"2024-04-12T11:38:06.866508+0000","last_active":"2024-04-12T11:41:29.357288+0000","last_peered":"2024-04-12T11:41:29.357288+0000","last_clean":"2024-04-12T11:41:29.357288+0000","last_became_active":"2024-04-12T11:38:06.843752+0000","last_became_peered":"2024-04-12T11:38:06.843752+0000","last_unstale":"2024-04-12T11:41:29.357288+0000","last_undegraded":"2024-04-12T11:41:29.357288+0000","last_fullsized":"2024-04-12T11:41:29.357288+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:18:11.827424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1648,"reported_epoch":1166,"state":"active+clean","last_fresh":"2024-04-12T11:41:29.337581+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:41:29.337581+0000","last_peered":"2024-04-12T11:41:29.337581+0000","last_clean":"2024-04-12T11:41:29.337581+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:41:29.337581+0000","last_undegraded":"2024-04-12T11:41:29.337581+0000","last_fullsized":"2024-04-12T11:41:29.337581+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":215002,"num_write_kb":6880001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6441,"num_bytes_recovered":360120320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":685309952,"data_stored":685309952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139302,"ondisk_log_size":139302,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234351,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":329708,"kb_used_data":9892,"kb_used_omap":1,"kb_used_meta":319806,"kb_avail":93415444,"statfs":{"total":95995035648,"available":95657414656,"internally_reserved":0,"allocated":10129408,"data_stored":7029341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":327481803},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495988,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":239460,"kb_used_data":9880,"kb_used_omap":1,"kb_used_meta":229566,"kb_avail":93505692,"statfs":{"total":95995035648,"available":95749828608,"internally_reserved":0,"allocated":10117120,"data_stored":7029195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":235076040},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536297,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":585996,"kb_used_data":319428,"kb_used_omap":30,"kb_used_meta":266529,"kb_avail":93159156,"statfs":{"total":95995035648,"available":95394975744,"internally_reserved":0,"allocated":327094272,"data_stored":324021469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":272926018},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190381,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":339032,"kb_used_data":9872,"kb_used_omap":43,"kb_used_meta":329108,"kb_avail":93406120,"statfs":{"total":95995035648,"available":95647866880,"internally_reserved":0,"allocated":10108928,"data_stored":7024985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":337006635},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849509,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":378644,"kb_used_data":9292,"kb_used_omap":36,"kb_used_meta":369307,"kb_avail":93366508,"statfs":{"total":95995035648,"available":95607304192,"internally_reserved":0,"allocated":9515008,"data_stored":6434629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":378170548},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704862,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":446416,"kb_used_data":9288,"kb_used_omap":17,"kb_used_meta":437102,"kb_avail":93298736,"statfs":{"total":95995035648,"available":95537905664,"internally_reserved":0,"allocated":9510912,"data_stored":6434103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":447592951},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018074,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":266340,"kb_used_data":9308,"kb_used_omap":42,"kb_used_meta":256981,"kb_avail":93478812,"statfs":{"total":95995035648,"available":95722303488,"internally_reserved":0,"allocated":9531392,"data_stored":6438915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":263149368},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757624,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":684372,"kb_used_data":370316,"kb_used_omap":1,"kb_used_meta":314046,"kb_avail":93060780,"statfs":{"total":95995035648,"available":95294238720,"internally_reserved":0,"allocated":379203584,"data_stored":376123229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":321583558},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":315621376,"data_stored":315621376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":369688576,"data_stored":369688576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:41:44.098 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:44.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:44.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:44.200 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:44.225 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:44.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:44.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:44.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:44.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:44.353 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:44.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:44.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:44.477 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:44.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:44.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:44.579 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:44.607 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:44.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:44.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:44.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:44.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:44.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:44.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:44.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:44.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:44.862 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:44.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:44.962 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:44.983 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:41:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:41:44] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2024-04-12T11:41:44.990 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:44.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:44.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:45.091 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:45.119 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:45.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:45.221 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:45.246 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:45.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:45.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:45.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:45.371 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:45.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:45.472 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:45.499 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:45.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:45.601 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:45.626 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:45.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:45.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:45.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:45 smithi087 ceph-mon[89014]: pgmap v1652: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 4451/72 objects degraded (6181.944%); 0 B/s, 64 objects/s recovering 2024-04-12T11:41:45.728 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:45.753 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:45.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:45.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:45 smithi029 ceph-mon[97787]: pgmap v1652: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 4451/72 objects degraded (6181.944%); 0 B/s, 64 objects/s recovering 2024-04-12T11:41:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:45 smithi029 ceph-mon[93650]: pgmap v1652: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 4451/72 objects degraded (6181.944%); 0 B/s, 64 objects/s recovering 2024-04-12T11:41:45.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:45.878 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:45.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:45.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:45.980 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:46.008 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:46.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:46.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:46.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:46.132 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:46.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:46.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:46.234 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:46.262 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:46.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:46.363 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:46.389 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:46.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:46.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:46.490 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:46.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:46.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:46.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:46.620 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:46.643 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:46.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:46.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:46.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:46 smithi087 ceph-mon[89014]: pgmap v1653: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 50 op/s; 3947/72 objects degraded (5481.944%); 0 B/s, 106 objects/s recovering 2024-04-12T11:41:46.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:41:46.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:46.760 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:46 smithi029 ceph-mon[97787]: pgmap v1653: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 50 op/s; 3947/72 objects degraded (5481.944%); 0 B/s, 106 objects/s recovering 2024-04-12T11:41:46.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:46 smithi029 ceph-mon[93650]: pgmap v1653: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 50 op/s; 3947/72 objects degraded (5481.944%); 0 B/s, 106 objects/s recovering 2024-04-12T11:41:46.774 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:46.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:46.875 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:46.899 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:46.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:46.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:47.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:47.024 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:47.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:47.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:47.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:41:47.126 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:47.153 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:47.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:47.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:47.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:47.278 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:47.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:47.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:47.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:47.405 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:47.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:47.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:47.530 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:47.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:47.631 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:47.659 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:47.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:47.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:47.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:47.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:47.787 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:47.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:47.888 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:47.912 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:47.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:47.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:48.014 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:48.042 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:48.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:48.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:48.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:48.167 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:48.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:48.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:48.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:48.292 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:48.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:48.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:48.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:48.417 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:48.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:48.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:48.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:48.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:48.548 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:48.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:48.648 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:48.672 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:48.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:48.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:48.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:48.798 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:48.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:48.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:48.900 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:48.923 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:48.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:48.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:49.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:49.051 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:49.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:49.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:49.153 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:49.181 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:49.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:49.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:49.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:48 smithi087 ceph-mon[89014]: pgmap v1654: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3947/72 objects degraded (5481.944%); 0 B/s, 80 objects/s recovering 2024-04-12T11:41:49.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 3947/72 objects degraded (5481.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:48 smithi029 ceph-mon[93650]: pgmap v1654: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3947/72 objects degraded (5481.944%); 0 B/s, 80 objects/s recovering 2024-04-12T11:41:49.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 3947/72 objects degraded (5481.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:48 smithi029 ceph-mon[97787]: pgmap v1654: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3947/72 objects degraded (5481.944%); 0 B/s, 80 objects/s recovering 2024-04-12T11:41:49.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 3947/72 objects degraded (5481.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:49.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:49.306 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:49.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:49.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:49.408 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:49.436 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:49.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:49.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:49.537 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:49.565 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:49.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:49.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:49.667 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:49.690 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:49.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:49.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:49.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:49.816 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:49.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:49.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:49.917 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:49.945 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:49.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:49.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:50.047 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:50.071 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:50.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:50.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:50.200 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:50.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:50.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:50.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:50.325 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:50.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:50.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:50.427 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:50.455 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:50.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:50.556 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:50.580 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:50.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:50.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:50.682 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:50.705 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:50.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:50.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:50.807 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:50.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:50.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:50.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:50.932 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:50.960 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:50.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:50.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:51.062 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:51.089 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:51.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:51.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:51.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:51.214 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:51.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:51.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:51.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:50 smithi087 ceph-mon[89014]: pgmap v1655: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3401/72 objects degraded (4723.611%); 0 B/s, 126 objects/s recovering 2024-04-12T11:41:51.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:50 smithi029 ceph-mon[93650]: pgmap v1655: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3401/72 objects degraded (4723.611%); 0 B/s, 126 objects/s recovering 2024-04-12T11:41:51.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:50 smithi029 ceph-mon[97787]: pgmap v1655: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3401/72 objects degraded (4723.611%); 0 B/s, 126 objects/s recovering 2024-04-12T11:41:51.317 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:51.345 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:51.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:51.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:51.446 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:51.474 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:51.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:51.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:51.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:51.601 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:51.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:51.703 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:51.727 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:51.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:51.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:51.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:51.856 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:51.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:51.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:51.958 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:51.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:51.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:51.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:52.088 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:52.115 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:52.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:52.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:52.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:52.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:52.241 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:52.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:52.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:52.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:52.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:41:52.343 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:52.370 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:52.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:52.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:52.472 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:52.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:52.501 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:52.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:52.602 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:52.629 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:52.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:52.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:41:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:41:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:41:52.731 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:52.758 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:52.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:52.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:52.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:52.884 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:52.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:52.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:52.986 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:53.009 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:53.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:53.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:53.111 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:53.134 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:53.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:53.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:53.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:52 smithi087 ceph-mon[89014]: pgmap v1656: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3401/72 objects degraded (4723.611%); 0 B/s, 87 objects/s recovering 2024-04-12T11:41:53.236 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:53.261 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:53.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:53.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:53.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:52 smithi029 ceph-mon[93650]: pgmap v1656: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3401/72 objects degraded (4723.611%); 0 B/s, 87 objects/s recovering 2024-04-12T11:41:53.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:52 smithi029 ceph-mon[97787]: pgmap v1656: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3401/72 objects degraded (4723.611%); 0 B/s, 87 objects/s recovering 2024-04-12T11:41:53.363 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:53.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:53.391 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:53.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:53.492 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:53.519 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:53.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:53.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:53.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:53.647 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:53.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:53.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:53.749 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:53.775 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:53.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:53.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:53.876 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:53.902 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:53.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:53.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:54.004 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:54.028 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:54.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:54.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:54.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:54.154 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:54.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:54.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:54.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 3401/72 objects degraded (4723.611%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:54.256 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:54.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 3401/72 objects degraded (4723.611%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:54.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 3401/72 objects degraded (4723.611%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:54.284 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:54.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:54.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:54.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:54.410 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:54.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:54.511 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:54.539 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:54.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:54.640 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:54.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:54.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:54.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:54.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:54.790 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:54.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:54.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:54.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:54.916 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:54.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:55.018 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:55.047 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:55.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:55.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:55.149 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:55.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:54 smithi029 ceph-mon[97787]: pgmap v1657: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3401/72 objects degraded (4723.611%); 0 B/s, 87 objects/s recovering 2024-04-12T11:41:55.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:54 smithi029 ceph-mon[93650]: pgmap v1657: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3401/72 objects degraded (4723.611%); 0 B/s, 87 objects/s recovering 2024-04-12T11:41:55.177 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:41:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:41:54] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.43.0" 2024-04-12T11:41:55.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:55.178 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:55.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:55.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:54 smithi087 ceph-mon[89014]: pgmap v1657: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3401/72 objects degraded (4723.611%); 0 B/s, 87 objects/s recovering 2024-04-12T11:41:55.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:55.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:55.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:55.404 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:55.432 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:55.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:55.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:55.534 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:55.561 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:55.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:55.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:55.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:55.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:55.689 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:55.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:55.790 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:55.817 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:55.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:55.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:55.919 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:55.947 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:55.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:55.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:56.049 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:56.077 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:56.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:56.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:56.178 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:56.202 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:56.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:56.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:56.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:56.331 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:56.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:56.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:56.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:56.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:56.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:56.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:56.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:41:56.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:56.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:56.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:41:56.683 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:56.710 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:56.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:56.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:56.811 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:56.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:56.836 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:56.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:56.936 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:56.965 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:56.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:56.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:57.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:57.090 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:57.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:57.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:57.192 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:57.216 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:57.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:57.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:57.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:56 smithi087 ceph-mon[89014]: pgmap v1658: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2796/72 objects degraded (3883.333%); 0 B/s, 137 objects/s recovering 2024-04-12T11:41:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:56 smithi029 ceph-mon[93650]: pgmap v1658: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2796/72 objects degraded (3883.333%); 0 B/s, 137 objects/s recovering 2024-04-12T11:41:57.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:56 smithi029 ceph-mon[97787]: pgmap v1658: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2796/72 objects degraded (3883.333%); 0 B/s, 137 objects/s recovering 2024-04-12T11:41:57.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:57.342 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:57.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:57.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:57.444 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:57.474 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:57.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:57.575 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:57.601 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:57.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:57.702 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:57.730 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:57.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:57.831 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:57.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:57.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:57.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:57.961 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:57.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:57.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:57.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:58.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:58.115 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:58.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:58.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:58.216 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:41:58.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:57 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 2796/72 objects degraded (3883.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:58.242 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:58.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:58.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:41:58.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:57 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 2796/72 objects degraded (3883.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:58.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:57 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 2796/72 objects degraded (3883.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:41:58.344 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:41:58.370 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:58.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:58.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:41:58.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:41:58.496 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:58.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:41:58.598 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:58.626 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:58.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:58.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:58.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:58.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:58.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:58.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:58.856 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:58.884 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:58.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:58.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:58.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:59.012 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:59.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:59.114 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:59.140 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:59.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:59.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:41:58 smithi087 ceph-mon[89014]: pgmap v1659: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2796/72 objects degraded (3883.333%); 0 B/s, 95 objects/s recovering 2024-04-12T11:41:59.242 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:59.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:41:58 smithi029 ceph-mon[97787]: pgmap v1659: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2796/72 objects degraded (3883.333%); 0 B/s, 95 objects/s recovering 2024-04-12T11:41:59.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:41:58 smithi029 ceph-mon[93650]: pgmap v1659: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2796/72 objects degraded (3883.333%); 0 B/s, 95 objects/s recovering 2024-04-12T11:41:59.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:59.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:59.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:59.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:59.394 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:59.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:59.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:41:59.496 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:41:59.524 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:59.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:41:59.626 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:41:59.649 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:59.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:59.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:41:59.751 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:41:59.778 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:41:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:59.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:41:59.879 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:41:59.905 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:41:59.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:41:59.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:00.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:00.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:00.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:00.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:00.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:00.158 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:00.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:00.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:00.259 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:00.287 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:00.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:00.388 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:00.411 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:00.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:00.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:00.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:00.536 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:00.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:00.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:00.638 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:00.661 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:00.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:00.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:00.763 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:00.788 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:00.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:00.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:00.889 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:00.917 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:00.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:00.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:01.019 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:01.042 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:01.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:01.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:01.144 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:01.171 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:01.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:01.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:01.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:00 smithi087 ceph-mon[89014]: pgmap v1660: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2102/72 objects degraded (2919.444%); 0 B/s, 153 objects/s recovering 2024-04-12T11:42:01.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:00 smithi029 ceph-mon[97787]: pgmap v1660: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2102/72 objects degraded (2919.444%); 0 B/s, 153 objects/s recovering 2024-04-12T11:42:01.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:00 smithi029 ceph-mon[93650]: pgmap v1660: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2102/72 objects degraded (2919.444%); 0 B/s, 153 objects/s recovering 2024-04-12T11:42:01.273 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:01.296 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:01.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:01.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:01.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:01.423 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:01.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:01.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:01.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:01.549 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:01.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:01.651 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:01.678 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:01.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:01.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:01.780 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:01.808 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:01.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:01.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:01.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:01.937 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:01.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:01.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:02.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:02.064 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:02.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:02.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:02.165 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:02.193 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:02.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:02.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:02.294 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:02.320 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:02.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:02.423 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:02.447 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:02.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:02.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:02.573 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:02.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:02.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:02.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:02.699 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:02.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:02.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:42:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:42:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:42:02.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:02.827 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:02.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:02.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:02.929 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:02.953 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:02.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:02.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:03.055 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:03.083 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:03.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:03.185 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:03.213 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:03.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:03.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:02 smithi087 ceph-mon[89014]: pgmap v1661: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2102/72 objects degraded (2919.444%); 0 B/s, 108 objects/s recovering 2024-04-12T11:42:03.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:02 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 2102/72 objects degraded (2919.444%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:42:03.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:02 smithi029 ceph-mon[97787]: pgmap v1661: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2102/72 objects degraded (2919.444%); 0 B/s, 108 objects/s recovering 2024-04-12T11:42:03.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:02 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 2102/72 objects degraded (2919.444%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:42:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:02 smithi029 ceph-mon[93650]: pgmap v1661: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2102/72 objects degraded (2919.444%); 0 B/s, 108 objects/s recovering 2024-04-12T11:42:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:02 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 2102/72 objects degraded (2919.444%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:42:03.315 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:03.343 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:03.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:03.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:03.445 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:03.472 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:03.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:03.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:03.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:03.611 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:03.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:03.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:03.713 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:03.737 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:03.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:03.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:03.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:03.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:03.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:03.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:03.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:03.992 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:03.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:03.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:04.094 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:04.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:04.123 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:04.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:04.224 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:04.248 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:04.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:04.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:04.349 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:04.377 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:04.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:04.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:04.479 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:04.506 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:04.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:04.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:04.608 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:04.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:04.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:04.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:04.733 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:04.760 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:04.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:04.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:04.862 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:04.889 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:04.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:04.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:04.968 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:42:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:42:04] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.43.0" 2024-04-12T11:42:04.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:05.017 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:05.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:05.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:05.118 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:05.146 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:05.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:05.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:05.248 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:05.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:04 smithi029 ceph-mon[93650]: pgmap v1662: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2102/72 objects degraded (2919.444%); 0 B/s, 108 objects/s recovering 2024-04-12T11:42:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:04 smithi029 ceph-mon[97787]: pgmap v1662: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2102/72 objects degraded (2919.444%); 0 B/s, 108 objects/s recovering 2024-04-12T11:42:05.275 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:05.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:05.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:05.377 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:05.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:05.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:05.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:05.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:04 smithi087 ceph-mon[89014]: pgmap v1662: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2102/72 objects degraded (2919.444%); 0 B/s, 108 objects/s recovering 2024-04-12T11:42:05.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:05.531 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:05.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:05.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:05.633 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:05.660 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:05.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:05.762 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:05.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:05.791 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:05.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:05.892 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:05.919 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:05.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:05.978 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:42:06.021 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:06.047 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:06.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:06.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:06.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:06.175 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:06.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:06.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:06.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:05 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1271495923' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:42:06.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:05 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1271495923' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:42:06.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:06.303 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:06.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:06.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:06.405 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:06.428 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:06.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:06.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:06.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:05 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1271495923' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:42:06.530 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:06.558 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:06.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:06.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:06.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:06.687 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:06.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:06.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:06.788 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:06.816 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:06.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:06.917 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:06.944 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:06.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:06.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:07.046 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:07.070 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:07.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:07.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:07.171 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:07.199 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:07.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:07.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:07.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:06 smithi029 ceph-mon[97787]: pgmap v1663: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1175/72 objects degraded (1631.944%); 0 B/s, 185 objects/s recovering 2024-04-12T11:42:07.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:06 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:42:07.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1271495923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:42:07.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:06 smithi029 ceph-mon[97787]: osdmap e1167: 8 total, 8 up, 8 in 2024-04-12T11:42:07.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:07.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:06 smithi029 ceph-mon[93650]: pgmap v1663: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1175/72 objects degraded (1631.944%); 0 B/s, 185 objects/s recovering 2024-04-12T11:42:07.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:06 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:42:07.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1271495923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:42:07.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:06 smithi029 ceph-mon[93650]: osdmap e1167: 8 total, 8 up, 8 in 2024-04-12T11:42:07.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:07.301 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:07.327 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:07.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:07.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:07.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:07.454 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:07.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:07.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:06 smithi087 ceph-mon[89014]: pgmap v1663: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1175/72 objects degraded (1631.944%); 0 B/s, 185 objects/s recovering 2024-04-12T11:42:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:06 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:42:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1271495923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:42:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:06 smithi087 ceph-mon[89014]: osdmap e1167: 8 total, 8 up, 8 in 2024-04-12T11:42:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:07.557 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:07.580 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:07.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:07.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:07.682 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:07.709 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:07.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:07.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:07.811 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:07.838 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:07.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:07.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:07.940 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:07.967 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:07.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:07.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:08.068 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:08.094 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:08.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:08.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:08.196 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:08.221 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:08.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:08.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:08.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:07 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 1175/72 objects degraded (1631.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:42:08.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:07 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 1175/72 objects degraded (1631.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:42:08.323 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:08.347 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:08.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:08.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:08.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:08.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:07 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 1175/72 objects degraded (1631.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:42:08.473 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:08.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:08.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:08.597 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:08.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:08.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:08.699 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:08.725 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:08.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:08.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:08.827 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:08.852 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:08.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:08.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:08.954 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:08.979 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:08.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:08.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:09.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:09.105 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:09.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:09.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:09.207 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:09.233 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:09.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:09.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:09.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:08 smithi029 ceph-mon[93650]: pgmap v1665: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1175/72 objects degraded (1631.944%); 0 B/s, 162 objects/s recovering 2024-04-12T11:42:09.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:08 smithi029 ceph-mon[97787]: pgmap v1665: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1175/72 objects degraded (1631.944%); 0 B/s, 162 objects/s recovering 2024-04-12T11:42:09.336 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:09.361 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:09.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:09.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:09.462 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:09.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:08 smithi087 ceph-mon[89014]: pgmap v1665: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1175/72 objects degraded (1631.944%); 0 B/s, 162 objects/s recovering 2024-04-12T11:42:09.487 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:09.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:09.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:09.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:09.614 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:09.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:09.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:09.715 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:09.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:09.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:09.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:09.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:09.871 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:09.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:09.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:09.973 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:10.000 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:10.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:10.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:10.102 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:10.127 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:10.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:10.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:10.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:10.252 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:10.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:10.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:09 smithi029 ceph-mon[93650]: osdmap e1168: 8 total, 8 up, 8 in 2024-04-12T11:42:10.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:09 smithi029 ceph-mon[97787]: osdmap e1168: 8 total, 8 up, 8 in 2024-04-12T11:42:10.354 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:10.381 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:10.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:10.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:10.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:09 smithi087 ceph-mon[89014]: osdmap e1168: 8 total, 8 up, 8 in 2024-04-12T11:42:10.483 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:10.511 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:10.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:10.612 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:10.638 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:10.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:10.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:10.740 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:10.762 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:10.762 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:10.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:10.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:10.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:10.899 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:10.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:10.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:11.001 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:10 smithi029 ceph-mon[93650]: pgmap v1667: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 262 objects/s recovering 2024-04-12T11:42:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:10 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1175/72 objects degraded (1631.944%), 1 pg degraded, 1 pg undersized) 2024-04-12T11:42:11.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:10 smithi029 ceph-mon[93650]: osdmap e1169: 8 total, 8 up, 8 in 2024-04-12T11:42:11.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:10 smithi029 ceph-mon[97787]: pgmap v1667: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 262 objects/s recovering 2024-04-12T11:42:11.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:10 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1175/72 objects degraded (1631.944%), 1 pg degraded, 1 pg undersized) 2024-04-12T11:42:11.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:10 smithi029 ceph-mon[97787]: osdmap e1169: 8 total, 8 up, 8 in 2024-04-12T11:42:11.028 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:11.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:11.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:11.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:11.154 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:11.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:11.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:11.256 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:11.280 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:11.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:11.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:11.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:11.406 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:11.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:11.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:11.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:10 smithi087 ceph-mon[89014]: pgmap v1667: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 262 objects/s recovering 2024-04-12T11:42:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:10 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1175/72 objects degraded (1631.944%), 1 pg degraded, 1 pg undersized) 2024-04-12T11:42:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:10 smithi087 ceph-mon[89014]: osdmap e1169: 8 total, 8 up, 8 in 2024-04-12T11:42:11.508 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:11.536 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:11.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:11.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:11.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:11.663 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:11.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:11.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:11.765 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:11.793 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:11.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:11.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:11.895 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:11.922 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:11.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:11.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:12.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:12.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:12.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:12.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:12.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:12.180 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:12.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:12.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:12.282 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:12.308 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:12.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:12.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:12.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:12.435 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:12.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:12.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:12.537 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:12.562 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:12.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:12.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:12.664 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:12.692 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:12.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:12.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:12.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:42:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:42:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:42:12.794 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:12.819 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:12.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:12.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:12.838 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:12.921 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:12.948 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:12.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:12.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:13.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:13.074 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:13.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:13.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:13.176 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:13.204 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:13.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:13.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:13.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:12 smithi029 ceph-mon[93650]: pgmap v1669: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 195 objects/s recovering 2024-04-12T11:42:13.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:12 smithi029 ceph-mon[97787]: pgmap v1669: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 195 objects/s recovering 2024-04-12T11:42:13.306 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:13.330 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:13.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:13.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:13.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:13.460 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:13.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:13.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:13.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:12 smithi087 ceph-mon[89014]: pgmap v1669: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 195 objects/s recovering 2024-04-12T11:42:13.562 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:13.586 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:13.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:13.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:13.688 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:13.716 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:13.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:13.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:13.818 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:13.845 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:13.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:13.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:13.947 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:13.976 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:13.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:14.013 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_2' removed 2024-04-12T11:42:14.078 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:14.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:14.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:14.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:14.231 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:14.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:14.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:14.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:14 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2209839527' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:42:14.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:14 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:42:14.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:14 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2209839527' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:42:14.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:14 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:42:14.333 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:14.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:14.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:14.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:14.459 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:14.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:14 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2209839527' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:42:14.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:14 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:42:14.482 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:14.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:14.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:42:14.583 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:14.606 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:14.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:14.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:14.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:14.731 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:14.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:14.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:14.833 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:14.857 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:14.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:14.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:14.982 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:14.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:15.005 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:42:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:42:14] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.43.0" 2024-04-12T11:42:15.084 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:15.107 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:15.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:15.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:15.209 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:15.232 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:15.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:15.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:15 smithi029 ceph-mon[93650]: pgmap v1670: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 149 objects/s recovering 2024-04-12T11:42:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:15 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:42:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:15 smithi029 ceph-mon[93650]: osdmap e1170: 8 total, 8 up, 8 in 2024-04-12T11:42:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:15 smithi029 ceph-mon[97787]: pgmap v1670: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 149 objects/s recovering 2024-04-12T11:42:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:15 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:42:15.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:15 smithi029 ceph-mon[97787]: osdmap e1170: 8 total, 8 up, 8 in 2024-04-12T11:42:15.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:15.358 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:15.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:15.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:15.460 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:15.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:15 smithi087 ceph-mon[89014]: pgmap v1670: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 149 objects/s recovering 2024-04-12T11:42:15.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:15 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:42:15.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:15 smithi087 ceph-mon[89014]: osdmap e1170: 8 total, 8 up, 8 in 2024-04-12T11:42:15.483 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:15.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:15.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:15.573 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-12T11:42:15.573 INFO:tasks.radosbench:Beginning radosbench... 2024-04-12T11:42:15.574 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-04-12T11:42:15.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-04-12T11:42:15.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:15.640 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:15.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:15.742 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:15.765 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:15.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:15.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:15.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:15.891 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:15.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:15.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:15.993 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:16.018 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:16.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:16.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:16.120 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:16.143 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:16.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:16.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:16.245 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:16.270 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:16.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:16.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:16.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:16.394 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:16.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:16.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:16.496 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:16.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:16.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:16.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:16.622 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:16.646 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:16.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:16.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:16.747 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:16.771 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:16.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:16.873 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:16.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:16.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:16.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:16.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:17.024 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:17.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:17.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:17.126 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:17.150 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:17.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:17.252 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:17.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:17 smithi029 ceph-mon[97787]: pgmap v1672: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:17.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:17 smithi029 ceph-mon[93650]: pgmap v1672: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:17.276 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:17.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:17.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:17.323 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:42:17.324 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:42:17.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:17.401 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:17.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:17 smithi087 ceph-mon[89014]: pgmap v1672: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:17.503 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:17.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:17.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:17.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:17.651 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:17.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:17.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:17.753 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:17.781 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:17.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:17.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:17.882 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:17.979 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:17.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:18.027 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:18 smithi029 ceph-mon[93650]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:42:18.027 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:18 smithi029 ceph-mon[97787]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:42:18.080 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:18.106 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:18.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:18.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:18.208 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:18.233 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:18.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:18.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:18.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:18.361 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:18.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:18.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:18.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:18.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:18 smithi087 ceph-mon[89014]: from='client.45056 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:42:18.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:18.489 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:18.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:18.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:18.617 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:18.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:18.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:18.719 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:18.746 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:18.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:18.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:18.849 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:18.876 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:18.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:18.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:18.978 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:19.005 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:19.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:19.107 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:19.134 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:19.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:19.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:19.236 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:19.262 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:19.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:19.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:19 smithi029 ceph-mon[93650]: pgmap v1673: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:19.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:19 smithi029 ceph-mon[97787]: pgmap v1673: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:19.364 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:19.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:19.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:19.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:19.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:19 smithi087 ceph-mon[89014]: pgmap v1673: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:19.491 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:19.517 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:19.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:19.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:19.618 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:19.644 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:19.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:19.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:19.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:19.774 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:19.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:19.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:19.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:19.903 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:19.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:19.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:20.004 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:20.033 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:20.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:20.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:20.135 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:20.162 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:20.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:20.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:20.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:20.291 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:20.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:20.392 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:20.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:20.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:20.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:20.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:20.548 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:20.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:20.649 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:20.676 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:20.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:20.778 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:20.806 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:20.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:20.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:20.908 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:20.935 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:20.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:21.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:21.064 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:21.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:21.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:21.165 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:21.193 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:21.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:21.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:21 smithi029 ceph-mon[93650]: pgmap v1674: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:21 smithi029 ceph-mon[97787]: pgmap v1674: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:21.294 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:21.322 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:21.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:21.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:21.423 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:21.451 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:21.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:21.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:21.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:21 smithi087 ceph-mon[89014]: pgmap v1674: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:21.552 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:21.580 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:21.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:21.682 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:21.709 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:21.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:21.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:21.838 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:21.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:21.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:21.940 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:21.968 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:21.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:21.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:22.070 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:22.097 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:22.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:22.198 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:22.226 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:22.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:22.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:22.328 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:22.356 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:22.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:22.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:22.458 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:22.485 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:22.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:22.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:22.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:22.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:22.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:42:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:42:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:42:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:22.718 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:22.745 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:22.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:22.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:22.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:22.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:22.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:22.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:22.874 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:22.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:22.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:22.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:23.004 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:23.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:23.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:23.106 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:23.133 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:23.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:23.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:23.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:23.262 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:23.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:23.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:23.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:23.390 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:23.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:23.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:23.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:23.517 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:23.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:23.618 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:23.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:23.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:23.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:23 smithi087 ceph-mon[89014]: pgmap v1675: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:23.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:23.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:23 smithi029 ceph-mon[93650]: pgmap v1675: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:23.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:23 smithi029 ceph-mon[97787]: pgmap v1675: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:23.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:23.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:23.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:23.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:23.897 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:23.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:23.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:23.998 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:24.027 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:24.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:24.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:24.157 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:24.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:24.259 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:24.284 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:24.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:24.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:24.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:24.415 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:24.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:24.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:24.517 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:24.543 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:24.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:24.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:24.547 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1672,"stamp":"2024-04-12T11:42:15.846456+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":1772268,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":258,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3164796,"kb_used_data":667036,"kb_used_omap":175,"kb_used_meta":2497488,"kb_avail":746796420,"statfs":{"total":767960285184,"available":764719534080,"internally_reserved":0,"allocated":683044864,"data_stored":658252719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":2557427881},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":57,"apply_latency_ms":57,"commit_latency_ns":57000000,"apply_latency_ns":57000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1175,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-215002,"num_write_kb":-6880001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-10181,"num_bytes_recovered":-360120320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.863835"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1693,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.010865+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:42:14.010865+0000","last_peered":"2024-04-12T11:42:14.010865+0000","last_clean":"2024-04-12T11:42:14.010865+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:42:14.010865+0000","last_undegraded":"2024-04-12T11:42:14.010865+0000","last_fullsized":"2024-04-12T11:42:14.010865+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1701,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.386336+0000","last_change":"2024-04-12T11:23:36.644665+0000","last_active":"2024-04-12T11:42:14.386336+0000","last_peered":"2024-04-12T11:42:14.386336+0000","last_clean":"2024-04-12T11:42:14.386336+0000","last_became_active":"2024-04-12T11:23:36.644219+0000","last_became_peered":"2024-04-12T11:23:36.644219+0000","last_unstale":"2024-04-12T11:42:14.386336+0000","last_undegraded":"2024-04-12T11:42:14.386336+0000","last_fullsized":"2024-04-12T11:42:14.386336+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:39:14.248298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1751,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.997687+0000","last_change":"2024-04-12T11:23:36.644969+0000","last_active":"2024-04-12T11:42:09.997687+0000","last_peered":"2024-04-12T11:42:09.997687+0000","last_clean":"2024-04-12T11:42:09.997687+0000","last_became_active":"2024-04-12T11:23:36.644618+0000","last_became_peered":"2024-04-12T11:23:36.644618+0000","last_unstale":"2024-04-12T11:42:09.997687+0000","last_undegraded":"2024-04-12T11:42:09.997687+0000","last_fullsized":"2024-04-12T11:42:09.997687+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:43:42.205593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1761,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.007331+0000","last_change":"2024-04-12T11:38:21.933647+0000","last_active":"2024-04-12T11:42:14.007331+0000","last_peered":"2024-04-12T11:42:14.007331+0000","last_clean":"2024-04-12T11:42:14.007331+0000","last_became_active":"2024-04-12T11:38:21.933295+0000","last_became_peered":"2024-04-12T11:38:21.933295+0000","last_unstale":"2024-04-12T11:42:14.007331+0000","last_undegraded":"2024-04-12T11:42:14.007331+0000","last_fullsized":"2024-04-12T11:42:14.007331+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:37.402139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1709,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.383608+0000","last_change":"2024-04-12T11:23:15.599189+0000","last_active":"2024-04-12T11:42:14.383608+0000","last_peered":"2024-04-12T11:42:14.383608+0000","last_clean":"2024-04-12T11:42:14.383608+0000","last_became_active":"2024-04-12T11:23:15.597111+0000","last_became_peered":"2024-04-12T11:23:15.597111+0000","last_unstale":"2024-04-12T11:42:14.383608+0000","last_undegraded":"2024-04-12T11:42:14.383608+0000","last_fullsized":"2024-04-12T11:42:14.383608+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:07.613791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1115,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.978614+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:42:09.978614+0000","last_peered":"2024-04-12T11:42:09.978614+0000","last_clean":"2024-04-12T11:42:09.978614+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:42:09.978614+0000","last_undegraded":"2024-04-12T11:42:09.978614+0000","last_fullsized":"2024-04-12T11:42:09.978614+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1744,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.007630+0000","last_change":"2024-04-12T11:23:15.600908+0000","last_active":"2024-04-12T11:42:14.007630+0000","last_peered":"2024-04-12T11:42:14.007630+0000","last_clean":"2024-04-12T11:42:14.007630+0000","last_became_active":"2024-04-12T11:23:15.600527+0000","last_became_peered":"2024-04-12T11:23:15.600527+0000","last_unstale":"2024-04-12T11:42:14.007630+0000","last_undegraded":"2024-04-12T11:42:14.007630+0000","last_fullsized":"2024-04-12T11:42:14.007630+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:50.447131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1683,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.383720+0000","last_change":"2024-04-12T11:23:15.599492+0000","last_active":"2024-04-12T11:42:14.383720+0000","last_peered":"2024-04-12T11:42:14.383720+0000","last_clean":"2024-04-12T11:42:14.383720+0000","last_became_active":"2024-04-12T11:23:15.597606+0000","last_became_peered":"2024-04-12T11:23:15.597606+0000","last_unstale":"2024-04-12T11:42:14.383720+0000","last_undegraded":"2024-04-12T11:42:14.383720+0000","last_fullsized":"2024-04-12T11:42:14.383720+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:40.116815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1120,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.995491+0000","last_change":"2024-04-12T11:22:19.619248+0000","last_active":"2024-04-12T11:42:08.995491+0000","last_peered":"2024-04-12T11:42:08.995491+0000","last_clean":"2024-04-12T11:42:08.995491+0000","last_became_active":"2024-04-12T11:22:19.618902+0000","last_became_peered":"2024-04-12T11:22:19.618902+0000","last_unstale":"2024-04-12T11:42:08.995491+0000","last_undegraded":"2024-04-12T11:42:08.995491+0000","last_fullsized":"2024-04-12T11:42:08.995491+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:20:18.648249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1663,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.384455+0000","last_change":"2024-04-12T11:23:15.597038+0000","last_active":"2024-04-12T11:42:14.384455+0000","last_peered":"2024-04-12T11:42:14.384455+0000","last_clean":"2024-04-12T11:42:14.384455+0000","last_became_active":"2024-04-12T11:23:15.592679+0000","last_became_peered":"2024-04-12T11:23:15.592679+0000","last_unstale":"2024-04-12T11:42:14.384455+0000","last_undegraded":"2024-04-12T11:42:14.384455+0000","last_fullsized":"2024-04-12T11:42:14.384455+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:49:27.110377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":1129,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.995799+0000","last_change":"2024-04-12T11:23:36.645834+0000","last_active":"2024-04-12T11:42:08.995799+0000","last_peered":"2024-04-12T11:42:08.995799+0000","last_clean":"2024-04-12T11:42:08.995799+0000","last_became_active":"2024-04-12T11:23:36.644994+0000","last_became_peered":"2024-04-12T11:23:36.644994+0000","last_unstale":"2024-04-12T11:42:08.995799+0000","last_undegraded":"2024-04-12T11:42:08.995799+0000","last_fullsized":"2024-04-12T11:42:08.995799+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:40:01.809427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1742,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.997915+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:42:09.997915+0000","last_peered":"2024-04-12T11:42:09.997915+0000","last_clean":"2024-04-12T11:42:09.997915+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:42:09.997915+0000","last_undegraded":"2024-04-12T11:42:09.997915+0000","last_fullsized":"2024-04-12T11:42:09.997915+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1666,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.010728+0000","last_change":"2024-04-12T11:23:36.643725+0000","last_active":"2024-04-12T11:42:14.010728+0000","last_peered":"2024-04-12T11:42:14.010728+0000","last_clean":"2024-04-12T11:42:14.010728+0000","last_became_active":"2024-04-12T11:23:36.643357+0000","last_became_peered":"2024-04-12T11:23:36.643357+0000","last_unstale":"2024-04-12T11:42:14.010728+0000","last_undegraded":"2024-04-12T11:42:14.010728+0000","last_fullsized":"2024-04-12T11:42:14.010728+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:10:43.331201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1657,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.012500+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:42:14.012500+0000","last_peered":"2024-04-12T11:42:14.012500+0000","last_clean":"2024-04-12T11:42:14.012500+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:42:14.012500+0000","last_undegraded":"2024-04-12T11:42:14.012500+0000","last_fullsized":"2024-04-12T11:42:14.012500+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":1128,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.994878+0000","last_change":"2024-04-12T11:23:36.647568+0000","last_active":"2024-04-12T11:42:08.994878+0000","last_peered":"2024-04-12T11:42:08.994878+0000","last_clean":"2024-04-12T11:42:08.994878+0000","last_became_active":"2024-04-12T11:23:36.647239+0000","last_became_peered":"2024-04-12T11:23:36.647239+0000","last_unstale":"2024-04-12T11:42:08.994878+0000","last_undegraded":"2024-04-12T11:42:08.994878+0000","last_fullsized":"2024-04-12T11:42:08.994878+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:26.727832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1694,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.385538+0000","last_change":"2024-04-12T11:23:15.592299+0000","last_active":"2024-04-12T11:42:14.385538+0000","last_peered":"2024-04-12T11:42:14.385538+0000","last_clean":"2024-04-12T11:42:14.385538+0000","last_became_active":"2024-04-12T11:23:15.590548+0000","last_became_peered":"2024-04-12T11:23:15.590548+0000","last_unstale":"2024-04-12T11:42:14.385538+0000","last_undegraded":"2024-04-12T11:42:14.385538+0000","last_fullsized":"2024-04-12T11:42:14.385538+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:48:23.442999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1745,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.005967+0000","last_change":"2024-04-12T11:23:15.594554+0000","last_active":"2024-04-12T11:42:14.005967+0000","last_peered":"2024-04-12T11:42:14.005967+0000","last_clean":"2024-04-12T11:42:14.005967+0000","last_became_active":"2024-04-12T11:23:15.594035+0000","last_became_peered":"2024-04-12T11:23:15.594035+0000","last_unstale":"2024-04-12T11:42:14.005967+0000","last_undegraded":"2024-04-12T11:42:14.005967+0000","last_fullsized":"2024-04-12T11:42:14.005967+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:21.984310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1119,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.994904+0000","last_change":"2024-04-12T11:23:15.595055+0000","last_active":"2024-04-12T11:42:08.994904+0000","last_peered":"2024-04-12T11:42:08.994904+0000","last_clean":"2024-04-12T11:42:08.994904+0000","last_became_active":"2024-04-12T11:23:15.594660+0000","last_became_peered":"2024-04-12T11:23:15.594660+0000","last_unstale":"2024-04-12T11:42:08.994904+0000","last_undegraded":"2024-04-12T11:42:08.994904+0000","last_fullsized":"2024-04-12T11:42:08.994904+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:50:10.614915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1111,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.994721+0000","last_change":"2024-04-12T11:23:36.648712+0000","last_active":"2024-04-12T11:42:08.994721+0000","last_peered":"2024-04-12T11:42:08.994721+0000","last_clean":"2024-04-12T11:42:08.994721+0000","last_became_active":"2024-04-12T11:23:36.648057+0000","last_became_peered":"2024-04-12T11:23:36.648057+0000","last_unstale":"2024-04-12T11:42:08.994721+0000","last_undegraded":"2024-04-12T11:42:08.994721+0000","last_fullsized":"2024-04-12T11:42:08.994721+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:35:30.149231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1746,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.007238+0000","last_change":"2024-04-12T11:23:15.593640+0000","last_active":"2024-04-12T11:42:14.007238+0000","last_peered":"2024-04-12T11:42:14.007238+0000","last_clean":"2024-04-12T11:42:14.007238+0000","last_became_active":"2024-04-12T11:23:15.593323+0000","last_became_peered":"2024-04-12T11:23:15.593323+0000","last_unstale":"2024-04-12T11:42:14.007238+0000","last_undegraded":"2024-04-12T11:42:14.007238+0000","last_fullsized":"2024-04-12T11:42:14.007238+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:09:08.935595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1122,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.978041+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:42:09.978041+0000","last_peered":"2024-04-12T11:42:09.978041+0000","last_clean":"2024-04-12T11:42:09.978041+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:42:09.978041+0000","last_undegraded":"2024-04-12T11:42:09.978041+0000","last_fullsized":"2024-04-12T11:42:09.978041+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1693,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.387539+0000","last_change":"2024-04-12T11:23:15.588498+0000","last_active":"2024-04-12T11:42:14.387539+0000","last_peered":"2024-04-12T11:42:14.387539+0000","last_clean":"2024-04-12T11:42:14.387539+0000","last_became_active":"2024-04-12T11:23:15.587229+0000","last_became_peered":"2024-04-12T11:23:15.587229+0000","last_unstale":"2024-04-12T11:42:14.387539+0000","last_undegraded":"2024-04-12T11:42:14.387539+0000","last_fullsized":"2024-04-12T11:42:14.387539+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:01.340688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":1127,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.995808+0000","last_change":"2024-04-12T11:23:36.646690+0000","last_active":"2024-04-12T11:42:08.995808+0000","last_peered":"2024-04-12T11:42:08.995808+0000","last_clean":"2024-04-12T11:42:08.995808+0000","last_became_active":"2024-04-12T11:23:36.646359+0000","last_became_peered":"2024-04-12T11:23:36.646359+0000","last_unstale":"2024-04-12T11:42:08.995808+0000","last_undegraded":"2024-04-12T11:42:08.995808+0000","last_fullsized":"2024-04-12T11:42:08.995808+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:21:12.076104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1105,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.994454+0000","last_change":"2024-04-12T11:12:59.164554+0000","last_active":"2024-04-12T11:42:08.994454+0000","last_peered":"2024-04-12T11:42:08.994454+0000","last_clean":"2024-04-12T11:42:08.994454+0000","last_became_active":"2024-04-12T11:12:59.164285+0000","last_became_peered":"2024-04-12T11:12:59.164285+0000","last_unstale":"2024-04-12T11:42:08.994454+0000","last_undegraded":"2024-04-12T11:42:08.994454+0000","last_fullsized":"2024-04-12T11:42:08.994454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:28:48.788046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":1128,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.995699+0000","last_change":"2024-04-12T11:23:36.639309+0000","last_active":"2024-04-12T11:42:08.995699+0000","last_peered":"2024-04-12T11:42:08.995699+0000","last_clean":"2024-04-12T11:42:08.995699+0000","last_became_active":"2024-04-12T11:23:36.638970+0000","last_became_peered":"2024-04-12T11:23:36.638970+0000","last_unstale":"2024-04-12T11:42:08.995699+0000","last_undegraded":"2024-04-12T11:42:08.995699+0000","last_fullsized":"2024-04-12T11:42:08.995699+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:06.906867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1679,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.011708+0000","last_change":"2024-04-12T11:23:36.649703+0000","last_active":"2024-04-12T11:42:14.011708+0000","last_peered":"2024-04-12T11:42:14.011708+0000","last_clean":"2024-04-12T11:42:14.011708+0000","last_became_active":"2024-04-12T11:23:36.646669+0000","last_became_peered":"2024-04-12T11:23:36.646669+0000","last_unstale":"2024-04-12T11:42:14.011708+0000","last_undegraded":"2024-04-12T11:42:14.011708+0000","last_fullsized":"2024-04-12T11:42:14.011708+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:15:08.633836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":1098,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:10.003711+0000","last_change":"2024-04-12T11:12:57.764278+0000","last_active":"2024-04-12T11:42:10.003711+0000","last_peered":"2024-04-12T11:42:10.003711+0000","last_clean":"2024-04-12T11:42:10.003711+0000","last_became_active":"2024-04-12T11:12:57.763998+0000","last_became_peered":"2024-04-12T11:12:57.763998+0000","last_unstale":"2024-04-12T11:42:10.003711+0000","last_undegraded":"2024-04-12T11:42:10.003711+0000","last_fullsized":"2024-04-12T11:42:10.003711+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:52:23.131785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1114,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.978183+0000","last_change":"2024-04-12T11:22:19.616769+0000","last_active":"2024-04-12T11:42:09.978183+0000","last_peered":"2024-04-12T11:42:09.978183+0000","last_clean":"2024-04-12T11:42:09.978183+0000","last_became_active":"2024-04-12T11:22:19.616362+0000","last_became_peered":"2024-04-12T11:22:19.616362+0000","last_unstale":"2024-04-12T11:42:09.978183+0000","last_undegraded":"2024-04-12T11:42:09.978183+0000","last_fullsized":"2024-04-12T11:42:09.978183+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:08:46.329827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1121,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:10.520816+0000","last_change":"2024-04-12T11:23:15.598107+0000","last_active":"2024-04-12T11:42:10.520816+0000","last_peered":"2024-04-12T11:42:10.520816+0000","last_clean":"2024-04-12T11:42:10.520816+0000","last_became_active":"2024-04-12T11:23:15.597746+0000","last_became_peered":"2024-04-12T11:23:15.597746+0000","last_unstale":"2024-04-12T11:42:10.520816+0000","last_undegraded":"2024-04-12T11:42:10.520816+0000","last_fullsized":"2024-04-12T11:42:10.520816+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:28.627026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1756,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.006138+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:42:14.006138+0000","last_peered":"2024-04-12T11:42:14.006138+0000","last_clean":"2024-04-12T11:42:14.006138+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:42:14.006138+0000","last_undegraded":"2024-04-12T11:42:14.006138+0000","last_fullsized":"2024-04-12T11:42:14.006138+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":1687,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.384642+0000","last_change":"2024-04-12T11:23:36.644752+0000","last_active":"2024-04-12T11:42:14.384642+0000","last_peered":"2024-04-12T11:42:14.384642+0000","last_clean":"2024-04-12T11:42:14.384642+0000","last_became_active":"2024-04-12T11:23:36.644320+0000","last_became_peered":"2024-04-12T11:23:36.644320+0000","last_unstale":"2024-04-12T11:42:14.384642+0000","last_undegraded":"2024-04-12T11:42:14.384642+0000","last_fullsized":"2024-04-12T11:42:14.384642+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:59:48.015336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1670,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.010792+0000","last_change":"2024-04-12T11:23:36.639178+0000","last_active":"2024-04-12T11:42:14.010792+0000","last_peered":"2024-04-12T11:42:14.010792+0000","last_clean":"2024-04-12T11:42:14.010792+0000","last_became_active":"2024-04-12T11:23:36.638718+0000","last_became_peered":"2024-04-12T11:23:36.638718+0000","last_unstale":"2024-04-12T11:42:14.010792+0000","last_undegraded":"2024-04-12T11:42:14.010792+0000","last_fullsized":"2024-04-12T11:42:14.010792+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:43.838570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":696,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.010758+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:42:14.010758+0000","last_peered":"2024-04-12T11:42:14.010758+0000","last_clean":"2024-04-12T11:42:14.010758+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:42:14.010758+0000","last_undegraded":"2024-04-12T11:42:14.010758+0000","last_fullsized":"2024-04-12T11:42:14.010758+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1692,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.010901+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:42:14.010901+0000","last_peered":"2024-04-12T11:42:14.010901+0000","last_clean":"2024-04-12T11:42:14.010901+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:42:14.010901+0000","last_undegraded":"2024-04-12T11:42:14.010901+0000","last_fullsized":"2024-04-12T11:42:14.010901+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1678,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.011793+0000","last_change":"2024-04-12T11:23:36.646966+0000","last_active":"2024-04-12T11:42:14.011793+0000","last_peered":"2024-04-12T11:42:14.011793+0000","last_clean":"2024-04-12T11:42:14.011793+0000","last_became_active":"2024-04-12T11:23:36.643599+0000","last_became_peered":"2024-04-12T11:23:36.643599+0000","last_unstale":"2024-04-12T11:42:14.011793+0000","last_undegraded":"2024-04-12T11:42:14.011793+0000","last_fullsized":"2024-04-12T11:42:14.011793+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:01.690700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1657,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.011662+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:42:14.011662+0000","last_peered":"2024-04-12T11:42:14.011662+0000","last_clean":"2024-04-12T11:42:14.011662+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:42:14.011662+0000","last_undegraded":"2024-04-12T11:42:14.011662+0000","last_fullsized":"2024-04-12T11:42:14.011662+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1695,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.383806+0000","last_change":"2024-04-12T11:23:15.597021+0000","last_active":"2024-04-12T11:42:14.383806+0000","last_peered":"2024-04-12T11:42:14.383806+0000","last_clean":"2024-04-12T11:42:14.383806+0000","last_became_active":"2024-04-12T11:23:15.590325+0000","last_became_peered":"2024-04-12T11:23:15.590325+0000","last_unstale":"2024-04-12T11:42:14.383806+0000","last_undegraded":"2024-04-12T11:42:14.383806+0000","last_fullsized":"2024-04-12T11:42:14.383806+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:39:07.042173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1751,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.999316+0000","last_change":"2024-04-12T11:23:36.645172+0000","last_active":"2024-04-12T11:42:09.999316+0000","last_peered":"2024-04-12T11:42:09.999316+0000","last_clean":"2024-04-12T11:42:09.999316+0000","last_became_active":"2024-04-12T11:23:36.644760+0000","last_became_peered":"2024-04-12T11:23:36.644760+0000","last_unstale":"2024-04-12T11:42:09.999316+0000","last_undegraded":"2024-04-12T11:42:09.999316+0000","last_fullsized":"2024-04-12T11:42:09.999316+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:29.500637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1753,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.998806+0000","last_change":"2024-04-12T11:22:19.612726+0000","last_active":"2024-04-12T11:42:09.998806+0000","last_peered":"2024-04-12T11:42:09.998806+0000","last_clean":"2024-04-12T11:42:09.998806+0000","last_became_active":"2024-04-12T11:22:19.610938+0000","last_became_peered":"2024-04-12T11:22:19.610938+0000","last_unstale":"2024-04-12T11:42:09.998806+0000","last_undegraded":"2024-04-12T11:42:09.998806+0000","last_fullsized":"2024-04-12T11:42:09.998806+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.516449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1136,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.994987+0000","last_change":"2024-04-12T11:23:15.594635+0000","last_active":"2024-04-12T11:42:08.994987+0000","last_peered":"2024-04-12T11:42:08.994987+0000","last_clean":"2024-04-12T11:42:08.994987+0000","last_became_active":"2024-04-12T11:23:15.594322+0000","last_became_peered":"2024-04-12T11:23:15.594322+0000","last_unstale":"2024-04-12T11:42:08.994987+0000","last_undegraded":"2024-04-12T11:42:08.994987+0000","last_fullsized":"2024-04-12T11:42:08.994987+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:51:32.346318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1744,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.997705+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:42:09.997705+0000","last_peered":"2024-04-12T11:42:09.997705+0000","last_clean":"2024-04-12T11:42:09.997705+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:42:09.997705+0000","last_undegraded":"2024-04-12T11:42:09.997705+0000","last_fullsized":"2024-04-12T11:42:09.997705+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1126,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.994758+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:42:08.994758+0000","last_peered":"2024-04-12T11:42:08.994758+0000","last_clean":"2024-04-12T11:42:08.994758+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:42:08.994758+0000","last_undegraded":"2024-04-12T11:42:08.994758+0000","last_fullsized":"2024-04-12T11:42:08.994758+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1678,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.012270+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:42:14.012270+0000","last_peered":"2024-04-12T11:42:14.012270+0000","last_clean":"2024-04-12T11:42:14.012270+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:42:14.012270+0000","last_undegraded":"2024-04-12T11:42:14.012270+0000","last_fullsized":"2024-04-12T11:42:14.012270+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1753,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.998460+0000","last_change":"2024-04-12T11:23:15.594015+0000","last_active":"2024-04-12T11:42:09.998460+0000","last_peered":"2024-04-12T11:42:09.998460+0000","last_clean":"2024-04-12T11:42:09.998460+0000","last_became_active":"2024-04-12T11:23:15.593720+0000","last_became_peered":"2024-04-12T11:23:15.593720+0000","last_unstale":"2024-04-12T11:42:09.998460+0000","last_undegraded":"2024-04-12T11:42:09.998460+0000","last_fullsized":"2024-04-12T11:42:09.998460+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:44.717607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1719,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.380678+0000","last_change":"2024-04-12T11:23:15.594498+0000","last_active":"2024-04-12T11:42:14.380678+0000","last_peered":"2024-04-12T11:42:14.380678+0000","last_clean":"2024-04-12T11:42:14.380678+0000","last_became_active":"2024-04-12T11:23:15.594174+0000","last_became_peered":"2024-04-12T11:23:15.594174+0000","last_unstale":"2024-04-12T11:42:14.380678+0000","last_undegraded":"2024-04-12T11:42:14.380678+0000","last_fullsized":"2024-04-12T11:42:14.380678+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:21:15.116600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1744,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.998007+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:42:09.998007+0000","last_peered":"2024-04-12T11:42:09.998007+0000","last_clean":"2024-04-12T11:42:09.998007+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:42:09.998007+0000","last_undegraded":"2024-04-12T11:42:09.998007+0000","last_fullsized":"2024-04-12T11:42:09.998007+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1747,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.999905+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:42:09.999905+0000","last_peered":"2024-04-12T11:42:09.999905+0000","last_clean":"2024-04-12T11:42:09.999905+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:42:09.999905+0000","last_undegraded":"2024-04-12T11:42:09.999905+0000","last_fullsized":"2024-04-12T11:42:09.999905+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1126,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.995453+0000","last_change":"2024-04-12T11:22:40.581078+0000","last_active":"2024-04-12T11:42:08.995453+0000","last_peered":"2024-04-12T11:42:08.995453+0000","last_clean":"2024-04-12T11:42:08.995453+0000","last_became_active":"2024-04-12T11:22:40.580697+0000","last_became_peered":"2024-04-12T11:22:40.580697+0000","last_unstale":"2024-04-12T11:42:08.995453+0000","last_undegraded":"2024-04-12T11:42:08.995453+0000","last_fullsized":"2024-04-12T11:42:08.995453+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:41.163650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1481,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.995051+0000","last_change":"2024-04-12T11:21:59.590728+0000","last_active":"2024-04-12T11:42:08.995051+0000","last_peered":"2024-04-12T11:42:08.995051+0000","last_clean":"2024-04-12T11:42:08.995051+0000","last_became_active":"2024-04-12T11:21:59.590304+0000","last_became_peered":"2024-04-12T11:21:59.590304+0000","last_unstale":"2024-04-12T11:42:08.995051+0000","last_undegraded":"2024-04-12T11:42:08.995051+0000","last_fullsized":"2024-04-12T11:42:08.995051+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:22:36.107552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1663,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.396568+0000","last_change":"2024-04-12T11:23:15.590483+0000","last_active":"2024-04-12T11:42:14.396568+0000","last_peered":"2024-04-12T11:42:14.396568+0000","last_clean":"2024-04-12T11:42:14.396568+0000","last_became_active":"2024-04-12T11:23:15.590110+0000","last_became_peered":"2024-04-12T11:23:15.590110+0000","last_unstale":"2024-04-12T11:42:14.396568+0000","last_undegraded":"2024-04-12T11:42:14.396568+0000","last_fullsized":"2024-04-12T11:42:14.396568+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:02:41.034598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1114,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.978524+0000","last_change":"2024-04-12T11:23:36.644604+0000","last_active":"2024-04-12T11:42:09.978524+0000","last_peered":"2024-04-12T11:42:09.978524+0000","last_clean":"2024-04-12T11:42:09.978524+0000","last_became_active":"2024-04-12T11:23:36.644338+0000","last_became_peered":"2024-04-12T11:23:36.644338+0000","last_unstale":"2024-04-12T11:42:09.978524+0000","last_undegraded":"2024-04-12T11:42:09.978524+0000","last_fullsized":"2024-04-12T11:42:09.978524+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:16:41.628473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1134,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.994966+0000","last_change":"2024-04-12T11:22:19.608960+0000","last_active":"2024-04-12T11:42:08.994966+0000","last_peered":"2024-04-12T11:42:08.994966+0000","last_clean":"2024-04-12T11:42:08.994966+0000","last_became_active":"2024-04-12T11:22:19.608621+0000","last_became_peered":"2024-04-12T11:22:19.608621+0000","last_unstale":"2024-04-12T11:42:08.994966+0000","last_undegraded":"2024-04-12T11:42:08.994966+0000","last_fullsized":"2024-04-12T11:42:08.994966+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:54.166602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1756,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.005931+0000","last_change":"2024-04-12T11:23:36.651570+0000","last_active":"2024-04-12T11:42:14.005931+0000","last_peered":"2024-04-12T11:42:14.005931+0000","last_clean":"2024-04-12T11:42:14.005931+0000","last_became_active":"2024-04-12T11:23:36.651210+0000","last_became_peered":"2024-04-12T11:23:36.651210+0000","last_unstale":"2024-04-12T11:42:14.005931+0000","last_undegraded":"2024-04-12T11:42:14.005931+0000","last_fullsized":"2024-04-12T11:42:14.005931+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:32:34.509788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1756,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.007007+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:42:14.007007+0000","last_peered":"2024-04-12T11:42:14.007007+0000","last_clean":"2024-04-12T11:42:14.007007+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:42:14.007007+0000","last_undegraded":"2024-04-12T11:42:14.007007+0000","last_fullsized":"2024-04-12T11:42:14.007007+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1133,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.995584+0000","last_change":"2024-04-12T11:23:36.645476+0000","last_active":"2024-04-12T11:42:08.995584+0000","last_peered":"2024-04-12T11:42:08.995584+0000","last_clean":"2024-04-12T11:42:08.995584+0000","last_became_active":"2024-04-12T11:23:36.644543+0000","last_became_peered":"2024-04-12T11:23:36.644543+0000","last_unstale":"2024-04-12T11:42:08.995584+0000","last_undegraded":"2024-04-12T11:42:08.995584+0000","last_fullsized":"2024-04-12T11:42:08.995584+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:57:43.803364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1744,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.998705+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:42:09.998705+0000","last_peered":"2024-04-12T11:42:09.998705+0000","last_clean":"2024-04-12T11:42:09.998705+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:42:09.998705+0000","last_undegraded":"2024-04-12T11:42:09.998705+0000","last_fullsized":"2024-04-12T11:42:09.998705+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1739,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.006911+0000","last_change":"2024-04-12T11:21:59.597229+0000","last_active":"2024-04-12T11:42:14.006911+0000","last_peered":"2024-04-12T11:42:14.006911+0000","last_clean":"2024-04-12T11:42:14.006911+0000","last_became_active":"2024-04-12T11:21:59.594262+0000","last_became_peered":"2024-04-12T11:21:59.594262+0000","last_unstale":"2024-04-12T11:42:14.006911+0000","last_undegraded":"2024-04-12T11:42:14.006911+0000","last_fullsized":"2024-04-12T11:42:14.006911+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:33:40.293910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":1678,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.387446+0000","last_change":"2024-04-12T11:23:15.592662+0000","last_active":"2024-04-12T11:42:14.387446+0000","last_peered":"2024-04-12T11:42:14.387446+0000","last_clean":"2024-04-12T11:42:14.387446+0000","last_became_active":"2024-04-12T11:23:15.590551+0000","last_became_peered":"2024-04-12T11:23:15.590551+0000","last_unstale":"2024-04-12T11:42:14.387446+0000","last_undegraded":"2024-04-12T11:42:14.387446+0000","last_fullsized":"2024-04-12T11:42:14.387446+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:28.880381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1744,"reported_epoch":1169,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.998540+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:42:09.998540+0000","last_peered":"2024-04-12T11:42:09.998540+0000","last_clean":"2024-04-12T11:42:09.998540+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:42:09.998540+0000","last_undegraded":"2024-04-12T11:42:09.998540+0000","last_fullsized":"2024-04-12T11:42:09.998540+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1676,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.383892+0000","last_change":"2024-04-12T11:23:15.596669+0000","last_active":"2024-04-12T11:42:14.383892+0000","last_peered":"2024-04-12T11:42:14.383892+0000","last_clean":"2024-04-12T11:42:14.383892+0000","last_became_active":"2024-04-12T11:23:15.589622+0000","last_became_peered":"2024-04-12T11:23:15.589622+0000","last_unstale":"2024-04-12T11:42:14.383892+0000","last_undegraded":"2024-04-12T11:42:14.383892+0000","last_fullsized":"2024-04-12T11:42:14.383892+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:28.694749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1139,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:09.978549+0000","last_change":"2024-04-12T11:39:23.349899+0000","last_active":"2024-04-12T11:42:09.978549+0000","last_peered":"2024-04-12T11:42:09.978549+0000","last_clean":"2024-04-12T11:42:09.978549+0000","last_became_active":"2024-04-12T11:39:23.349526+0000","last_became_peered":"2024-04-12T11:39:23.349526+0000","last_unstale":"2024-04-12T11:42:09.978549+0000","last_undegraded":"2024-04-12T11:42:09.978549+0000","last_fullsized":"2024-04-12T11:42:09.978549+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:36:47.115912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1759,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.007419+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:42:14.007419+0000","last_peered":"2024-04-12T11:42:14.007419+0000","last_clean":"2024-04-12T11:42:14.007419+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:42:14.007419+0000","last_undegraded":"2024-04-12T11:42:14.007419+0000","last_fullsized":"2024-04-12T11:42:14.007419+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1665,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.011437+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:42:14.011437+0000","last_peered":"2024-04-12T11:42:14.011437+0000","last_clean":"2024-04-12T11:42:14.011437+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:42:14.011437+0000","last_undegraded":"2024-04-12T11:42:14.011437+0000","last_fullsized":"2024-04-12T11:42:14.011437+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1749,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.006218+0000","last_change":"2024-04-12T11:23:36.646738+0000","last_active":"2024-04-12T11:42:14.006218+0000","last_peered":"2024-04-12T11:42:14.006218+0000","last_clean":"2024-04-12T11:42:14.006218+0000","last_became_active":"2024-04-12T11:23:36.646389+0000","last_became_peered":"2024-04-12T11:23:36.646389+0000","last_unstale":"2024-04-12T11:42:14.006218+0000","last_undegraded":"2024-04-12T11:42:14.006218+0000","last_fullsized":"2024-04-12T11:42:14.006218+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:32.954517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":64,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.380817+0000","last_change":"2024-04-12T11:38:06.839832+0000","last_active":"2024-04-12T11:42:14.380817+0000","last_peered":"2024-04-12T11:42:14.380817+0000","last_clean":"2024-04-12T11:42:14.380817+0000","last_became_active":"2024-04-12T11:38:06.839460+0000","last_became_peered":"2024-04-12T11:38:06.839460+0000","last_unstale":"2024-04-12T11:42:14.380817+0000","last_undegraded":"2024-04-12T11:42:14.380817+0000","last_fullsized":"2024-04-12T11:42:14.380817+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:25:27.959933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1136,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.994748+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:42:08.994748+0000","last_peered":"2024-04-12T11:42:08.994748+0000","last_clean":"2024-04-12T11:42:08.994748+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:42:08.994748+0000","last_undegraded":"2024-04-12T11:42:08.994748+0000","last_fullsized":"2024-04-12T11:42:08.994748+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1693,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.012632+0000","last_change":"2024-04-12T11:23:36.654127+0000","last_active":"2024-04-12T11:42:14.012632+0000","last_peered":"2024-04-12T11:42:14.012632+0000","last_clean":"2024-04-12T11:42:14.012632+0000","last_became_active":"2024-04-12T11:23:36.652933+0000","last_became_peered":"2024-04-12T11:23:36.652933+0000","last_unstale":"2024-04-12T11:42:14.012632+0000","last_undegraded":"2024-04-12T11:42:14.012632+0000","last_fullsized":"2024-04-12T11:42:14.012632+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:47.345869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1121,"reported_epoch":1168,"state":"active+clean","last_fresh":"2024-04-12T11:42:08.995064+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:42:08.995064+0000","last_peered":"2024-04-12T11:42:08.995064+0000","last_clean":"2024-04-12T11:42:08.995064+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:42:08.995064+0000","last_undegraded":"2024-04-12T11:42:08.995064+0000","last_fullsized":"2024-04-12T11:42:08.995064+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1684,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.010653+0000","last_change":"2024-04-12T11:23:36.647247+0000","last_active":"2024-04-12T11:42:14.010653+0000","last_peered":"2024-04-12T11:42:14.010653+0000","last_clean":"2024-04-12T11:42:14.010653+0000","last_became_active":"2024-04-12T11:23:36.646952+0000","last_became_peered":"2024-04-12T11:23:36.646952+0000","last_unstale":"2024-04-12T11:42:14.010653+0000","last_undegraded":"2024-04-12T11:42:14.010653+0000","last_fullsized":"2024-04-12T11:42:14.010653+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:59:49.423861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":1677,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.393729+0000","last_change":"2024-04-12T11:23:15.597694+0000","last_active":"2024-04-12T11:42:14.393729+0000","last_peered":"2024-04-12T11:42:14.393729+0000","last_clean":"2024-04-12T11:42:14.393729+0000","last_became_active":"2024-04-12T11:23:15.597284+0000","last_became_peered":"2024-04-12T11:23:15.597284+0000","last_unstale":"2024-04-12T11:42:14.393729+0000","last_undegraded":"2024-04-12T11:42:14.393729+0000","last_fullsized":"2024-04-12T11:42:14.393729+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:56:25.443330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1687,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.384311+0000","last_change":"2024-04-12T11:23:15.599354+0000","last_active":"2024-04-12T11:42:14.384311+0000","last_peered":"2024-04-12T11:42:14.384311+0000","last_clean":"2024-04-12T11:42:14.384311+0000","last_became_active":"2024-04-12T11:23:15.597594+0000","last_became_peered":"2024-04-12T11:23:15.597594+0000","last_unstale":"2024-04-12T11:42:14.384311+0000","last_undegraded":"2024-04-12T11:42:14.384311+0000","last_fullsized":"2024-04-12T11:42:14.384311+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:30:41.097804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1777,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.008741+0000","last_change":"2024-04-12T11:38:06.866508+0000","last_active":"2024-04-12T11:42:14.008741+0000","last_peered":"2024-04-12T11:42:14.008741+0000","last_clean":"2024-04-12T11:42:14.008741+0000","last_became_active":"2024-04-12T11:38:06.843752+0000","last_became_peered":"2024-04-12T11:38:06.843752+0000","last_unstale":"2024-04-12T11:42:14.008741+0000","last_undegraded":"2024-04-12T11:42:14.008741+0000","last_fullsized":"2024-04-12T11:42:14.008741+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:18:11.827424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1656,"reported_epoch":1170,"state":"active+clean","last_fresh":"2024-04-12T11:42:14.010345+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:42:14.010345+0000","last_peered":"2024-04-12T11:42:14.010345+0000","last_clean":"2024-04-12T11:42:14.010345+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:42:14.010345+0000","last_undegraded":"2024-04-12T11:42:14.010345+0000","last_fullsized":"2024-04-12T11:42:14.010345+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234358,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":329748,"kb_used_data":9928,"kb_used_omap":1,"kb_used_meta":319806,"kb_avail":93415404,"statfs":{"total":95995035648,"available":95657373696,"internally_reserved":0,"allocated":10166272,"data_stored":7054532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":327481803},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495995,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":239496,"kb_used_data":9916,"kb_used_omap":1,"kb_used_meta":229566,"kb_avail":93505656,"statfs":{"total":95995035648,"available":95749791744,"internally_reserved":0,"allocated":10153984,"data_stored":7054386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":235076040},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536305,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":509236,"kb_used_data":242664,"kb_used_omap":30,"kb_used_meta":266529,"kb_avail":93235916,"statfs":{"total":95995035648,"available":95473577984,"internally_reserved":0,"allocated":248487936,"data_stored":245403460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":272926018},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190389,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":339072,"kb_used_data":9908,"kb_used_omap":43,"kb_used_meta":329108,"kb_avail":93406080,"statfs":{"total":95995035648,"available":95647825920,"internally_reserved":0,"allocated":10145792,"data_stored":7050176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":337006635},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849517,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362060,"kb_used_data":9344,"kb_used_omap":36,"kb_used_meta":352667,"kb_avail":93383092,"statfs":{"total":95995035648,"available":95624286208,"internally_reserved":0,"allocated":9568256,"data_stored":6468203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":361131188},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":18,"apply_latency_ms":18,"commit_latency_ns":18000000,"apply_latency_ns":18000000},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704870,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":438152,"kb_used_data":9340,"kb_used_omap":17,"kb_used_meta":428782,"kb_avail":93307000,"statfs":{"total":95995035648,"available":95546368000,"internally_reserved":0,"allocated":9564160,"data_stored":6467677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":439073271},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":20,"apply_latency_ms":20,"commit_latency_ns":20000000,"apply_latency_ns":20000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018082,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":266396,"kb_used_data":9360,"kb_used_omap":42,"kb_used_meta":256981,"kb_avail":93478756,"statfs":{"total":95995035648,"available":95722246144,"internally_reserved":0,"allocated":9584640,"data_stored":6472489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":263149368},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757631,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":680636,"kb_used_data":366576,"kb_used_omap":1,"kb_used_meta":314046,"kb_avail":93064516,"statfs":{"total":95995035648,"available":95298064384,"internally_reserved":0,"allocated":375373824,"data_stored":372281796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":321583558},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:42:24.550 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-04-12T11:42:24.550 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:42:24.551 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-04-12T11:42:24.551 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-04-12T11:42:24.551 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-04-12T11:42:24.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-04-12T11:42:24.644 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:24.668 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:24.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:24.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:24.770 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:24.794 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:24.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:24.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:24.798 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:42:24.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:24.920 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:24.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:24.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:25.022 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:25.046 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:25.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:25.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:25.140 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:42:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:42:24] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.43.0" 2024-04-12T11:42:25.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:25.172 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:25.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:25.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:25.274 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:25.283 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:42:25.290 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:42:25.298 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:25.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:25.310 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:42:25.331 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:42:25.351 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:42:25.371 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:42:25.391 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:42:25.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:25 smithi029 ceph-mon[97787]: pgmap v1676: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:25.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:25 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2217359143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:42:25.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:25 smithi029 ceph-mon[93650]: pgmap v1676: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:25.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:25 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2217359143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:42:25.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:25.410 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:42:25.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:42:25.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:42:25.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:42:25.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:42:25.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:42:25.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:42:25.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:42:25.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:42:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:25.425 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:25.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:25.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:25 smithi087 ceph-mon[89014]: pgmap v1676: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:25.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:25 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2217359143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:42:25.526 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:25.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:25.551 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:25.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:25.652 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:25.675 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:25.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:25.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:25.777 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:25.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:25.802 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:25.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:25.902 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:25.926 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:25.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:25.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:26.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:26.050 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:26.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:26.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:26.152 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:26.176 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:26.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:26.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:26.278 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:26.303 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:26.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:26.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:26.404 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:26.428 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:26.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:26.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:26.530 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:26.531 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:42:26.531 INFO:teuthology.orchestra.run.smithi087.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-04-12T11:42:26.531 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:42:26.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:26.555 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:26.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:26.656 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:26.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:26.681 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:26.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:26.782 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:26.806 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:26.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:26.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:26.908 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:26.932 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:26.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:27.034 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:27.058 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:27.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:27.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:27.160 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:27.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:27.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:27.219 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:42:27.219 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:42:27.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:27.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:27.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:27.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:27.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:27.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:27.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:27.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:27.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:27 smithi029 ceph-mon[93650]: pgmap v1677: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:27.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:27 smithi029 ceph-mon[97787]: pgmap v1677: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:27.540 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:27.564 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:27.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:27.664 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:27.687 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:27.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:27.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:27.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:27 smithi087 ceph-mon[89014]: pgmap v1677: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:27.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:27.813 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:27.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:27.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:27.915 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:27.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:27.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:27.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:28.040 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:28.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:28.064 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:28.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:28.165 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:28.188 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:28.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:28.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:28.290 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:28.313 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:28.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:28.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:28.415 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:28.438 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:28.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:28.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:28.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:28.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:28.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:28.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-04-12T11:42:28.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:28.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:28.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:28.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:28.793 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:28.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:28.817 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:28.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:28.918 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:28.942 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:28.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:28.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:29.015 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:29.015 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:29.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:29.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:29.071 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:29.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:29.172 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:29.195 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:29.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:29.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:29.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:29.320 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:29.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:29.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:29.421 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:29.445 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:29.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:29.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:29.547 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:29.570 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:29.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:29.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:29 smithi029 ceph-mon[97787]: pgmap v1678: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:29.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:29 smithi029 ceph-mon[93650]: pgmap v1678: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:29.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:29.695 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:29.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:29.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:29.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:42:29.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:29 smithi087 ceph-mon[89014]: pgmap v1678: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:29.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:29.820 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:29.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:29.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:29.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:29.948 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:29.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:29.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:30.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:30.076 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:30.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:30.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:30.178 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:30.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:30.204 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:30.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:30.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:30.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:30.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:30.431 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:30.457 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:30.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:30.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:30.481 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:30.559 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:30.583 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:30.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:30.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:30.684 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:30.707 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:30.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:30.809 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:30.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:30.837 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:30.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:30.938 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:30.965 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:30.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:30.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:31.067 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:31.094 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:31.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:31.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:31.196 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:31.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:31.220 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:31.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:31.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:31.345 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:31.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:31.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:31.447 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:31.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:31.476 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:31.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:31.577 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:31.603 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:31.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:31.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:31.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:31 smithi087 ceph-mon[89014]: pgmap v1679: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:31.729 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:31.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:31.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:31.757 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:31 smithi029 ceph-mon[97787]: pgmap v1679: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:31.757 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:31 smithi029 ceph-mon[93650]: pgmap v1679: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:31.832 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:31.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:31.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:31.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:31.961 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:31.989 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:31.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:31.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:32.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:32.115 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:32.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:32.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:32.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:32.241 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:32.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:32.343 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:32.371 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:32.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:32.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:32.387 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:42:32.387 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:42:32.387 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-04-12T11:42:32.387 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2024-04-12T11:42:32.387 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 1 1 2024-04-12T11:42:32.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:32.496 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:32.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:32.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:32.621 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:32.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:32.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:42:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:42:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:42:32.723 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:32.751 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:32.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:32.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:32.852 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:32.880 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:32.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:32.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:32.982 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:33.008 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:33.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:33.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:33.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:33.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:33.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:33.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:33.263 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:33.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:33.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:33.364 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:33.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:33.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:33.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:33.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:33.518 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:33.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:33.619 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:33.643 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:33.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:33.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:33 smithi087 ceph-mon[89014]: pgmap v1680: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:33.745 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:33 smithi029 ceph-mon[93650]: pgmap v1680: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:33.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:33 smithi029 ceph-mon[97787]: pgmap v1680: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:33.770 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:33.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:33.871 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:33.898 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:33.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:33.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:34.000 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:34.025 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:34.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:34.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:34.108 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:34.121 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:42:34.121 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:42:34.127 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:34.153 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:34.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:34.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:34.255 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:34.283 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:34.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:34.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:34.327 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:42:34.385 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:34.412 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:34.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:34.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:34.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:34.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:34 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2638845300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:42:34.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:34 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:42:34.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:34 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2638845300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:42:34.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:34 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:42:34.537 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:34.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:34.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:34.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:34.663 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:34.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:34.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:34.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:34 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2638845300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:42:34.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:34 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:42:34.764 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:34.792 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:34.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:34.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:34.894 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:34.922 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:34.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:34.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:35.023 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:35.049 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:35.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:35.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:35.175 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:35.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:35.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:35.186 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:42:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:42:34] "GET /metrics HTTP/1.1" 200 34094 "" "Prometheus/2.43.0" 2024-04-12T11:42:35.208 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-04-12T11:42:35.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:35.316 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:35.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:35.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:35.418 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:35.446 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:35.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:35.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:35.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:35 smithi029 ceph-mon[93650]: pgmap v1681: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:35.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:35 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:42:35.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:35 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:42:35.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:35 smithi029 ceph-mon[93650]: osdmap e1171: 8 total, 8 up, 8 in 2024-04-12T11:42:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:35 smithi029 ceph-mon[97787]: pgmap v1681: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:35 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:42:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:35 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:42:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:35 smithi029 ceph-mon[97787]: osdmap e1171: 8 total, 8 up, 8 in 2024-04-12T11:42:35.547 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:35.574 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:35.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:35.676 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:35.702 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:35.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:35.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:35 smithi087 ceph-mon[89014]: pgmap v1681: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:35.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:35 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:42:35.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:35 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:42:35.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:35 smithi087 ceph-mon[89014]: osdmap e1171: 8 total, 8 up, 8 in 2024-04-12T11:42:35.804 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:35.830 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:35.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:35.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:35.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:35.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:35.958 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:35.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:36.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:36.084 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:36.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:36.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:36.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:36.209 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:36.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:36.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:36.311 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:36.335 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:36.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:36.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:36.436 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:36.463 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:36.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:36.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:36.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:36.590 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:36.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:36.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:36.691 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:36.715 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:36.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:36.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:36.816 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:36.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:36.845 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:36.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:36.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:36.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:36.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[93650]: pgmap v1683: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:36.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:42:36.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:42:36.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:36.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:36.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4228608675' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-12T11:42:36.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-12T11:42:36.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:36.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:36.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[97787]: pgmap v1683: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:36.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:42:36.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:42:36.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:36.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:36.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4228608675' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-12T11:42:36.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:36 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-12T11:42:36.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:36.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:36 smithi087 ceph-mon[89014]: pgmap v1683: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:42:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:42:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:36 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4228608675' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-12T11:42:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:36 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-12T11:42:36.970 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:36.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:36.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:37.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:37.099 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:37.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:37.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:37.201 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:37.210 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:37.223 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_3' created 2024-04-12T11:42:37.236 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:37.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:37.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:37.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:37.366 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:37.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:37.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:37.468 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:37.495 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:37.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:37.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:37.596 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:37.620 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:37.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:37.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:37.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:37.746 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:37.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:37.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:37.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:37.872 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:37.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:37.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:37.974 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:37.984 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:38.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:38.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:38.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:38.103 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:38.131 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:38.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:38.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:38.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:38.257 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:38.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:38.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:38.337 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:38 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-12T11:42:38.337 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:38 smithi029 ceph-mon[93650]: osdmap e1172: 8 total, 8 up, 8 in 2024-04-12T11:42:38.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:38 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-12T11:42:38.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:38 smithi029 ceph-mon[97787]: osdmap e1172: 8 total, 8 up, 8 in 2024-04-12T11:42:38.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:38.384 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:38.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:38.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:38.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:38 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-12T11:42:38.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:38 smithi087 ceph-mon[89014]: osdmap e1172: 8 total, 8 up, 8 in 2024-04-12T11:42:38.508 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:38.532 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:38.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:38.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:38.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:38.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:38.658 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:38.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:38.759 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:38.784 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:38.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:38.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-04-12T11:42:38.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:38.911 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:38.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:38.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:42:39.013 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:39.036 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:39.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:39.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:39.165 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:39.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:39.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:39.267 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:39.290 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:39.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:39.392 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:39.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:39.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:39.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:39.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:39 smithi087 ceph-mon[89014]: pgmap v1685: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:39.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:39 smithi087 ceph-mon[89014]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:42:39.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:39 smithi087 ceph-mon[89014]: osdmap e1173: 8 total, 8 up, 8 in 2024-04-12T11:42:39.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:39 smithi029 ceph-mon[93650]: pgmap v1685: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:39.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:39 smithi029 ceph-mon[93650]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:42:39.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:39 smithi029 ceph-mon[93650]: osdmap e1173: 8 total, 8 up, 8 in 2024-04-12T11:42:39.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:39 smithi029 ceph-mon[97787]: pgmap v1685: 89 pgs: 5 creating+peering, 11 unknown, 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:39.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:39 smithi029 ceph-mon[97787]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:42:39.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:39 smithi029 ceph-mon[97787]: osdmap e1173: 8 total, 8 up, 8 in 2024-04-12T11:42:39.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:39.545 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:39.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:39.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:39.647 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:39.674 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:39.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:39.776 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:39.800 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:39.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:39.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:39.929 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:39.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:40.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:40.054 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:40.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:40.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:40.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:40.179 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:40.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:40.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:40.280 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:40.308 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:40.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:40.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:40.410 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:40.438 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:40.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:40.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:40.470 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:40 smithi029 ceph-mon[97787]: osdmap e1174: 8 total, 8 up, 8 in 2024-04-12T11:42:40.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:40 smithi029 ceph-mon[93650]: osdmap e1174: 8 total, 8 up, 8 in 2024-04-12T11:42:40.540 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:40.568 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:40.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:40.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:40.669 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:40.693 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:40.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:40.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:40.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:40 smithi087 ceph-mon[89014]: osdmap e1174: 8 total, 8 up, 8 in 2024-04-12T11:42:40.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:40.818 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:40.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:40.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:40.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:40.942 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:40.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:40.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:41.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:41.067 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:41.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:41.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:41.192 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:41.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:41.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:41.294 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:41.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:41.318 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:41.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:41.419 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:41.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:41.443 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:41.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:41.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:41 smithi029 ceph-mon[93650]: pgmap v1688: 89 pgs: 5 creating+peering, 4 unknown, 80 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:41.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:41 smithi029 ceph-mon[97787]: pgmap v1688: 89 pgs: 5 creating+peering, 4 unknown, 80 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:41.544 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:41.572 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:41.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:41.673 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:41.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:41.701 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:41.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:41.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:41 smithi087 ceph-mon[89014]: pgmap v1688: 89 pgs: 5 creating+peering, 4 unknown, 80 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:41.802 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:41.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:41.831 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:41.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:41.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:41.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:41.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:41.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:42.056 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:42.079 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:42.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:42.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:42.181 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:42.209 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:42.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:42.311 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:42.338 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:42.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:42.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:42.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:42.464 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:42.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:42.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:42.504 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:42:42.524 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:42:42.543 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:42:42.563 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:42:42.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:42.583 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:42:42.589 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:42.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:42.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:42.603 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:42:42.623 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:42:42.642 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:42:42.644 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:42:42.644 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:42:42.644 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:42:42.644 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:42:42.644 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:42:42.644 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:42:42.644 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:42:42.644 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:42:42.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:42.713 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:42.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:42.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:42.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:42:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:42:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:42:42.815 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:42.843 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:42.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:42.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:42.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:42.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:42.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:43.070 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:43.094 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:43.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:43.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:43.195 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:42:43.195 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:42:43.195 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2024-04-12T11:42:43.195 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:42:43.196 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:43.224 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:43.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:43.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:43.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:43.349 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:43.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:43.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:43.450 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:43.478 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:43.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:43.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:43.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:43 smithi029 ceph-mon[97787]: pgmap v1689: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:43.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:43 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1208537170' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:42:43.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:43 smithi029 ceph-mon[93650]: pgmap v1689: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:43.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:43 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1208537170' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:42:43.580 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:43.607 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:43.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:43.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:43.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:43.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:43 smithi087 ceph-mon[89014]: pgmap v1689: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:43 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1208537170' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:42:43.733 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:43.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:43.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:43.834 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:43.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-04-12T11:42:43.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:43.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:43.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:43.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:43.987 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:43.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:44.089 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:44.112 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:44.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:44.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:44.214 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:44.243 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:44.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:44.344 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:44.371 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:44.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:44.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:44.473 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:44.501 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:44.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:44.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:44.597 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:44.598 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:44.602 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:44.629 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:44.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:44.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:44.730 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:44.753 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:44.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:44.855 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:44.883 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:44.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:44.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:44.974 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:42:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:42:44] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.43.0" 2024-04-12T11:42:44.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:45.009 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:45.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:45.111 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:45.134 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:45.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:45.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:45.236 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:45.238 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-04-12T11:42:45.260 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:45.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:45.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:45.362 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:45.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:45.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:45.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:45.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:45.516 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:45.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:45.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:45.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:45 smithi029 ceph-mon[93650]: pgmap v1690: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:45.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:45 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2747787474' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:42:45.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:45 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:42:45.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:45 smithi029 ceph-mon[97787]: pgmap v1690: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:45.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:45 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2747787474' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:42:45.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:45 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:42:45.617 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:45.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:45.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:45 smithi087 ceph-mon[89014]: pgmap v1690: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:45 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2747787474' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:42:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:45 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T11:42:45.747 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:45.774 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:45.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:45.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:45.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:45.901 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:45.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:45.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:46.002 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:46.030 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:46.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:46.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:46.156 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:46.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:46.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:46.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:46.281 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:46.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:46.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:46.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:42:46.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:46.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:46 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-04-12T11:42:46.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:46 smithi029 ceph-mon[97787]: osdmap e1175: 8 total, 8 up, 8 in 2024-04-12T11:42:46.394 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:46 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-04-12T11:42:46.394 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:46 smithi029 ceph-mon[93650]: osdmap e1175: 8 total, 8 up, 8 in 2024-04-12T11:42:46.417 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:46.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:46.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:46.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:46.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:46.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:46.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:46.649 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:46.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:46.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:46.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:46.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:46 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-04-12T11:42:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:46 smithi087 ceph-mon[89014]: osdmap e1175: 8 total, 8 up, 8 in 2024-04-12T11:42:46.778 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:46.806 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:46.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:46.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:46.908 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:46.932 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:46.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:46.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:47.034 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:47.057 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:47.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:47.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:47.158 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:47.186 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:47.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:47.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:47.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:47.312 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:47.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:47.414 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:47.442 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:47.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:47.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:47.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:47 smithi029 ceph-mon[93650]: pgmap v1692: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:47.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:47 smithi029 ceph-mon[97787]: pgmap v1692: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:47.544 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:47.548 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:47 smithi087 ceph-mon[89014]: pgmap v1692: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:47.572 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:47.572 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:47.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:47.573 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:47.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:47.674 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:47.702 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:47.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:47.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:47.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:47.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:47.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:47.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:47.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:47.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:47.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:47.966 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:42:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=cleanup t=2024-04-12T11:42:47.547302694Z level=info msg="Completed cleanup jobs" duration=13.639194ms 2024-04-12T11:42:48.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:48.081 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:48.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:48.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:48.207 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:48.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:48.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:48.308 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:48.331 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:48.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:48.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:48.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:48.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:48.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:48.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:48.557 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:48.585 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:48.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:48.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:48.687 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:48.714 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:48.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:48.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:48.816 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:48.842 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:48.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:48.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:48.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:48.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:48.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:48.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:49.070 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:49.098 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:49.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:49.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:49.199 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:49.224 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:49.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:49.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:49.325 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:49.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:49.354 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:49.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:49.356 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:49 smithi029 ceph-mon[97787]: pgmap v1693: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:49.356 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:49 smithi029 ceph-mon[93650]: pgmap v1693: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:49.456 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:49.483 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:49.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:49.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:49.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:49.608 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:49.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:49.710 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:49.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:49 smithi087 ceph-mon[89014]: pgmap v1693: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:49.733 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:49.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:49.835 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:49.862 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:49.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:49.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:49.963 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:49.991 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:49.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:49.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:50.093 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:50.121 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:50.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:50.223 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:50.248 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:50.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:50.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:50.256 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-04-12T11:42:50.256 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:42:50.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:50.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:50.374 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:50.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:50.475 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:50.503 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:50.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:50.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:50 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2522817794' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:42:50.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:50 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:42:50.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:50 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1013011925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:42:50.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:50 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:42:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:50 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2522817794' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:42:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:50 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:42:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:50 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1013011925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:42:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:50 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:42:50.604 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:50.632 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:50.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:50.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:50 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2522817794' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:42:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:50 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:42:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:50 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1013011925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:42:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:50 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:42:50.734 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:50.761 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:50.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:50.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:50.801 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:42:50.820 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:42:50.840 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:42:50.859 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:42:50.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:50.880 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:42:50.886 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:50.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:50.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:50.900 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:42:50.919 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:42:50.938 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:42:50.940 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:42:50.940 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:42:50.940 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:42:50.940 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:42:50.940 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:42:50.940 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:42:50.940 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:42:50.941 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:42:50.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:51.016 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:51.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:51.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:51.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:51.142 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:51.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:51.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:51.244 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:51.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:51.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:51.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:51.373 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:51.396 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:51.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:51.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:51.498 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:51.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[93650]: pgmap v1694: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:51.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:42:51.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:42:51.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:42:51.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[93650]: osdmap e1176: 8 total, 8 up, 8 in 2024-04-12T11:42:51.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1127212165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:42:51.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[97787]: pgmap v1694: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:51.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:42:51.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:42:51.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:42:51.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[97787]: osdmap e1176: 8 total, 8 up, 8 in 2024-04-12T11:42:51.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:51 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1127212165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:42:51.526 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:51.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:51.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:51.631 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:51.632 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:42:51.632 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:42:51.632 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:42:51.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd out 3 2024-04-12T11:42:51.660 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:51.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:51.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:51.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:51 smithi087 ceph-mon[89014]: pgmap v1694: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:51.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:51 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:42:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:51 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:42:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:51 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:42:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:51 smithi087 ceph-mon[89014]: osdmap e1176: 8 total, 8 up, 8 in 2024-04-12T11:42:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:51 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1127212165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:42:51.761 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:51.789 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:51.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:51.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:51.891 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:51.918 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:51.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:51.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:52.020 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:52.044 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:52.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:52.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:52.146 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:52.174 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:52.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:52.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:52.244 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:42:52.244 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:42:52.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:52.300 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:52.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:52.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:52.401 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:52.429 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:52.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:52.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:52.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:52.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:52.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:52.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:52.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:52.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:52.531 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:52.557 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:52.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:52.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:52.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:42:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:42:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:42:52.618 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:52.618 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:42:52.618 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:42:52.660 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:52.684 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:52.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:52.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:52.809 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:52.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:52.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:52.910 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:52.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:52.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:52.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:53.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:53.066 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:53.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:53.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:53.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:53.191 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:53.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:53.293 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:53.317 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:53.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:53.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:53.419 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:53.442 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:53.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:53.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:53.544 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:53.567 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:53.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:53.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:53.669 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:53.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:53.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:53.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:53 smithi087 ceph-mon[89014]: pgmap v1696: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:53 smithi087 ceph-mon[89014]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:42:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:53 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:42:53.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:53 smithi029 ceph-mon[93650]: pgmap v1696: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:53.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:53 smithi029 ceph-mon[93650]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:42:53.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:53 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:42:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:53 smithi029 ceph-mon[97787]: pgmap v1696: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:53 smithi029 ceph-mon[97787]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:42:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:53 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:42:53.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:53.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:53.820 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:53.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:53.921 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:53.944 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:53.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:53.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:53.951 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-04-12T11:42:53.954 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-12T11:42:53.954 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-12T11:42:54.024 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2024-04-12T11:42:54.024 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-12T11:42:54.024 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_194418 2024-04-12T11:42:54.024 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:42:54.024 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2024-04-12T11:42:54.046 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:54.095 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:54.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:54.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:54.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:54.232 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:54.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:54.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:54.334 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:54.372 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:54.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:54.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:54.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:54.510 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:54.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:54.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:54.548 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:54.575 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-04-12T11:42:54.612 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:54.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:54.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:54.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:54.747 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:54.782 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:54.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:54.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:54.884 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:54.916 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:54.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:55.018 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:55.024 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3194 3178 198.601 198.625 0.00139511 0.00440281 2024-04-12T11:42:55.041 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:55.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:55.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:55.143 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:55.179 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:55.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:42:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:42:54] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.43.0" 2024-04-12T11:42:55.280 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:42:55.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:55.317 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:55.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:42:55.416 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:42:55.418 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:55.454 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:55.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:55.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:55.557 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:55.597 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:55.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:55.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:55 smithi029 ceph-mon[97787]: pgmap v1697: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:55.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:55 smithi029 ceph-mon[93650]: pgmap v1697: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:55.699 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:55 smithi087 ceph-mon[89014]: pgmap v1697: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T11:42:55.731 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:55.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:55.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:55.833 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:55.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:42:55.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:55.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:55.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:55.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:56.019 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:56.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:56.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:56.024 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 5690 5674 177.283 156 0.00404257 0.00562687 2024-04-12T11:42:56.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:42:56.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:56.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:42:56.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:56.300 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:56.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:56.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:56.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:56.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:56.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:56.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:56.540 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:56.563 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:56.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:56.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:56.705 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:56.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:56.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:56.806 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:56.841 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:56.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:56.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:56.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:56.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:56.984 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:56.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:57.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 8385 8369 174.329 168.438 0.00564575 0.00572193 2024-04-12T11:42:57.073 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:57.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:57.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:57.151 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:57.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:57.253 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:57.286 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:57.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:57.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:57.389 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:57.416 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:57.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:57.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:57.488 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:57 smithi029 ceph-mon[97787]: pgmap v1698: 89 pgs: 89 active+clean; 71 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s wr, 112 op/s 2024-04-12T11:42:57.488 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:57 smithi029 ceph-mon[93650]: pgmap v1698: 89 pgs: 89 active+clean; 71 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s wr, 112 op/s 2024-04-12T11:42:57.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:57.560 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:57.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:57.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:57.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:42:57.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:57.706 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:57.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:42:57.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:57 smithi087 ceph-mon[89014]: pgmap v1698: 89 pgs: 89 active+clean; 71 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s wr, 112 op/s 2024-04-12T11:42:57.721 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:42:57.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:57.840 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:57.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:57.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:57.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:57.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:57.975 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:57.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:58.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 11057 11041 172.492 167 0.00403301 0.00578863 2024-04-12T11:42:58.076 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:58.114 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:58.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:58.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:58.216 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:58.258 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:58.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:58.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:58.336 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.3. 2024-04-12T11:42:58.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:58.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:58.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:58.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:58.487 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:58.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:58.511 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:58.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:58 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2977325795' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2024-04-12T11:42:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:58 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2024-04-12T11:42:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:58 smithi029 ceph-mon[97787]: Client client.admin marked osd.3 out, while it was still marked up 2024-04-12T11:42:58.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:58 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T11:42:58.279+0000 7fae24212700 -1 mon.a@0(leader).osd e1176 definitely_dead 0 2024-04-12T11:42:58.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:58 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2977325795' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2024-04-12T11:42:58.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:58 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2024-04-12T11:42:58.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:58 smithi029 ceph-mon[93650]: Client client.admin marked osd.3 out, while it was still marked up 2024-04-12T11:42:58.612 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:58.636 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:58.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:58 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2977325795' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2024-04-12T11:42:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:58 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2024-04-12T11:42:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:58 smithi087 ceph-mon[89014]: Client client.admin marked osd.3 out, while it was still marked up 2024-04-12T11:42:58.738 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:58.762 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:58.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:58.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:58.864 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:42:58.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:58.890 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:58.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:42:58.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:59.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:59.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:59.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:59.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 11914 11898 148.705 53.5625 0.00160258 0.00579563 2024-04-12T11:42:59.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:59.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:59.143 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:59.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:59.244 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:59.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:59.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:59.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:59.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:59.396 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:59.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:42:59.484 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:59 smithi029 ceph-mon[97787]: pgmap v1699: 89 pgs: 89 active+clean; 255 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s wr, 406 op/s 2024-04-12T11:42:59.484 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:59 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-04-12T11:42:59.485 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:42:59 smithi029 ceph-mon[97787]: osdmap e1177: 8 total, 8 up, 7 in 2024-04-12T11:42:59.485 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:59 smithi029 ceph-mon[93650]: pgmap v1699: 89 pgs: 89 active+clean; 255 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s wr, 406 op/s 2024-04-12T11:42:59.485 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:59 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-04-12T11:42:59.485 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:42:59 smithi029 ceph-mon[93650]: osdmap e1177: 8 total, 8 up, 7 in 2024-04-12T11:42:59.498 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:42:59.521 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:59.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:59.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:42:59.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:42:59.647 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:59.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:59.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:42:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:59 smithi087 ceph-mon[89014]: pgmap v1699: 89 pgs: 89 active+clean; 255 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s wr, 406 op/s 2024-04-12T11:42:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:59 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-04-12T11:42:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:42:59 smithi087 ceph-mon[89014]: osdmap e1177: 8 total, 8 up, 7 in 2024-04-12T11:42:59.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:42:59.773 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:42:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:59.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:42:59.874 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:42:59.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:42:59.900 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:42:59.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:00.001 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:00.025 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:00.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 11914 11898 123.921 0 - 0.00579563 2024-04-12T11:43:00.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:00.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:00.127 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:00.153 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:00.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:00.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:00.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:00.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:00.281 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:00.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:00.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:00.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:00.424 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:00.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:00.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:00 smithi029 ceph-mon[97787]: osdmap e1178: 8 total, 8 up, 7 in 2024-04-12T11:43:00.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:00 smithi029 ceph-mon[97787]: osdmap e1179: 8 total, 8 up, 7 in 2024-04-12T11:43:00.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:00 smithi029 ceph-mon[93650]: osdmap e1178: 8 total, 8 up, 7 in 2024-04-12T11:43:00.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:00 smithi029 ceph-mon[93650]: osdmap e1179: 8 total, 8 up, 7 in 2024-04-12T11:43:00.525 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:00.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:00.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:00.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:00.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:00.706 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:00.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:00.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:00.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:00 smithi087 ceph-mon[89014]: osdmap e1178: 8 total, 8 up, 7 in 2024-04-12T11:43:00.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:00 smithi087 ceph-mon[89014]: osdmap e1179: 8 total, 8 up, 7 in 2024-04-12T11:43:00.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:00.845 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:00.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:00.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:00.947 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:00.983 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:00.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:00.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:01.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 13619 13603 121.44 53.2812 0.00502082 0.0082275 2024-04-12T11:43:01.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:01.125 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:01.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:01.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:01.150 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:43:01.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:01.262 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:01.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:01.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:01.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:01.381 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:01 smithi029 ceph-mon[93650]: pgmap v1702: 89 pgs: 15 peering, 74 active+clean; 570 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T11:43:01.381 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:01 smithi029 ceph-mon[93650]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-04-12T11:43:01.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:01 smithi029 ceph-mon[97787]: pgmap v1702: 89 pgs: 15 peering, 74 active+clean; 570 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T11:43:01.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:01 smithi029 ceph-mon[97787]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-04-12T11:43:01.407 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:01.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:01.509 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:01.532 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:01.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:01.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:01.634 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:01.670 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:01.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:01.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:01 smithi087 ceph-mon[89014]: pgmap v1702: 89 pgs: 15 peering, 74 active+clean; 570 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T11:43:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:01 smithi087 ceph-mon[89014]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-04-12T11:43:01.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:01.795 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:01.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:01.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:01.897 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:01.920 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:01.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:01.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:02.022 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:02.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 15655 15639 122.164 127.25 0.00592933 0.00817822 2024-04-12T11:43:02.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:02.059 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:02.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:02.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:02.201 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:02.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:02.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:02.303 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:02.339 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:02.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:02.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:02.466 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:02.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:02.567 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:02.602 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:02.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:02.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:02.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:02.716 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:43:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:43:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:43:02.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:02.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:02.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:02.847 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:02.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:02.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:02.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:02.986 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:03.009 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:03.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:03.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:03.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 17616 17600 122.207 122.562 0.00489045 0.00807616 2024-04-12T11:43:03.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:03.151 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:03.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:03.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:03.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:03.285 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:03.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:03.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:03.387 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:03.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:03.411 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:03.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:03.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:03.552 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:03.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:03.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:03.654 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:03.690 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:03.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:03.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:03.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:03 smithi087 ceph-mon[89014]: pgmap v1704: 89 pgs: 2 remapped+peering, 15 peering, 72 active+clean; 665 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-12T11:43:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:03 smithi029 ceph-mon[97787]: pgmap v1704: 89 pgs: 2 remapped+peering, 15 peering, 72 active+clean; 665 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-12T11:43:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:03 smithi029 ceph-mon[93650]: pgmap v1704: 89 pgs: 2 remapped+peering, 15 peering, 72 active+clean; 665 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-12T11:43:03.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:03.818 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:03.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:03.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:03.920 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:03.950 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:03.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:03.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:04.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 19238 19222 120.123 101.375 0.00356381 0.00831846 2024-04-12T11:43:04.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:04.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:04.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:04.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:04.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:04.233 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:04.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:04.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:04.335 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:04.370 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:04.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:04.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:04.472 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:04.512 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:04.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:04.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:04.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:04 smithi029 ceph-mon[93650]: pgmap v1705: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 8 peering, 76 active+clean; 959 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s; 1482/46065 objects degraded (3.217%); 391/46065 objects misplaced (0.849%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:43:04.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:04 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 1482/46065 objects degraded (3.217%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:43:04.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:04 smithi029 ceph-mon[97787]: pgmap v1705: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 8 peering, 76 active+clean; 959 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s; 1482/46065 objects degraded (3.217%); 391/46065 objects misplaced (0.849%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:43:04.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:04 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 1482/46065 objects degraded (3.217%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:43:04.665 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:04.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:04.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:04 smithi087 ceph-mon[89014]: pgmap v1705: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 8 peering, 76 active+clean; 959 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s; 1482/46065 objects degraded (3.217%); 391/46065 objects misplaced (0.849%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:43:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:04 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 1482/46065 objects degraded (3.217%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:43:04.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:04.804 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:04.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:04.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:04.906 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:04.930 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:04.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:04.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:05.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 21018 21002 119.315 111.25 0.00201702 0.00826517 2024-04-12T11:43:05.032 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:05.061 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:05.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:05.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:05.163 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:05.201 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:05.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:05.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:43:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:43:04] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.43.0" 2024-04-12T11:43:05.302 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:05.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:05.326 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:05.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:05.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:05.464 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:05.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:05.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:05.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:05.595 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:05.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:05.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:05.697 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:05.722 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:05.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:05.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:05.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:05.847 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:05.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:05.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:05.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:05.973 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:05.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:05.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:06.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 21897 21881 113.95 54.9375 0.0020536 0.00838417 2024-04-12T11:43:06.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:06.098 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:06.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:06.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:06.171 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:43:06.171 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:43:06.171 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.8770157381312551 2024-04-12T11:43:06.171 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 2 0.8770157381312551 2024-04-12T11:43:06.200 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:06.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:06.240 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:06.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:06.341 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:06.365 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:06.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:06.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:06.467 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:06.491 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:06.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:06.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:06.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:06 smithi029 ceph-mon[93650]: osdmap e1180: 8 total, 8 up, 7 in 2024-04-12T11:43:06.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:06 smithi029 ceph-mon[93650]: pgmap v1707: 89 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 2269/55233 objects degraded (4.108%); 773/55233 objects misplaced (1.400%); 13 MiB/s, 215 objects/s recovering 2024-04-12T11:43:06.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:06.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:06.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:06 smithi029 ceph-mon[97787]: osdmap e1180: 8 total, 8 up, 7 in 2024-04-12T11:43:06.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:06 smithi029 ceph-mon[97787]: pgmap v1707: 89 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 2269/55233 objects degraded (4.108%); 773/55233 objects misplaced (1.400%); 13 MiB/s, 215 objects/s recovering 2024-04-12T11:43:06.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:06.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:06.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:06.616 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:06.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:06 smithi087 ceph-mon[89014]: osdmap e1180: 8 total, 8 up, 7 in 2024-04-12T11:43:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:06 smithi087 ceph-mon[89014]: pgmap v1707: 89 pgs: 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 2269/55233 objects degraded (4.108%); 773/55233 objects misplaced (1.400%); 13 MiB/s, 215 objects/s recovering 2024-04-12T11:43:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:06.718 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:06.742 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:06.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:06.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:06.844 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:06.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:06.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:06.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:06.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:07.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:07.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:07.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 22039 22023 105.867 8.875 0.00162855 0.00929865 2024-04-12T11:43:07.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:07.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:07.128 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:07.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:07.229 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:07.253 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:07.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:07.355 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:07.380 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:07.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:07.482 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:07.517 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:07.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:07.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:07.619 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:07.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:07.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:07.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:07 smithi087 ceph-mon[89014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-12T11:43:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:07 smithi087 ceph-mon[89014]: osdmap e1181: 8 total, 8 up, 7 in 2024-04-12T11:43:07.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:07.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:07 smithi029 ceph-mon[97787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-12T11:43:07.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:07 smithi029 ceph-mon[97787]: osdmap e1181: 8 total, 8 up, 7 in 2024-04-12T11:43:07.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:07 smithi029 ceph-mon[93650]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-12T11:43:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:07 smithi029 ceph-mon[93650]: osdmap e1181: 8 total, 8 up, 7 in 2024-04-12T11:43:07.773 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:07.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:07.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:07.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:07.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:07.902 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:07.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:08.003 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:08.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 22297 22281 99.4567 16.125 0.00462522 0.00969089 2024-04-12T11:43:08.028 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:08.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:08.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:08.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:08.153 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:08.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:08.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:08.254 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:08.280 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:08.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:08.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:08.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:08.408 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:08.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:08.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:08.510 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:08.548 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:08.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:08.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:08.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:08.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:08.689 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:08.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:08.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:08 smithi087 ceph-mon[89014]: osdmap e1182: 8 total, 8 up, 7 in 2024-04-12T11:43:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:08 smithi087 ceph-mon[89014]: pgmap v1710: 89 pgs: 1 active+undersized+remapped, 4 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s; 787/61473 objects degraded (1.280%); 1470/61473 objects misplaced (2.391%); 23 MiB/s, 365 objects/s recovering 2024-04-12T11:43:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:08 smithi087 ceph-mon[89014]: osdmap e1183: 8 total, 8 up, 7 in 2024-04-12T11:43:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:08 smithi029 ceph-mon[97787]: osdmap e1182: 8 total, 8 up, 7 in 2024-04-12T11:43:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:08 smithi029 ceph-mon[97787]: pgmap v1710: 89 pgs: 1 active+undersized+remapped, 4 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s; 787/61473 objects degraded (1.280%); 1470/61473 objects misplaced (2.391%); 23 MiB/s, 365 objects/s recovering 2024-04-12T11:43:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:08 smithi029 ceph-mon[97787]: osdmap e1183: 8 total, 8 up, 7 in 2024-04-12T11:43:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:08 smithi029 ceph-mon[93650]: osdmap e1182: 8 total, 8 up, 7 in 2024-04-12T11:43:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:08 smithi029 ceph-mon[93650]: pgmap v1710: 89 pgs: 1 active+undersized+remapped, 4 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 102 MiB/s wr, 1.64k op/s; 787/61473 objects degraded (1.280%); 1470/61473 objects misplaced (2.391%); 23 MiB/s, 365 objects/s recovering 2024-04-12T11:43:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:08 smithi029 ceph-mon[93650]: osdmap e1183: 8 total, 8 up, 7 in 2024-04-12T11:43:08.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:08.814 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:08.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:08.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:08.916 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:08.939 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:08.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:08.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:09.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 23362 23346 97.2634 66.5625 0.00831058 0.0102753 2024-04-12T11:43:09.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:09.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:09.081 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:09.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:09.182 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:09.224 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:09.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:09.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:09.350 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:09.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:09.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:09.452 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:09.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:09.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:09.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:09.591 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:09.627 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:09.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:09.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:09.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:09.755 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:09.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:09.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:09.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:09.896 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:09.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:09.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:09.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:10.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 25105 25089 97.9925 108.938 0.00861081 0.0102 2024-04-12T11:43:10.032 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:10.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:10.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:10.134 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:10.172 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:10.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:10.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:10.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:09 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 787/66855 objects degraded (1.177%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:43:10.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:09 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 787/66855 objects degraded (1.177%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:43:10.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:09 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 787/66855 objects degraded (1.177%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:43:10.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:10.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:10.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:10.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:10.323 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:43:10.369 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:43:10.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:10.415 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:43:10.452 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:10.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:10.458 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:43:10.506 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:43:10.550 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:43:10.554 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:10.561 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:43:10.561 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:65204 6.14 6:286e0599:::benchmark_data_smithi029_138569_object65203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:41.885340+0000", 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 568.67086578399994, 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0719870739999999, 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.562 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 65204 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885340+0000", 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885340+0000", 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885342+0000", 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4050000000000001e-06 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885364+0000", 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1936000000000001e-05 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885375+0000", 2024-04-12T11:43:10.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1408e-05 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885383+0000", 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9100000000000005e-06 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.979352+0000", 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.093969072000000001 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.979361+0000", 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9930000000000001e-06 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.957246+0000", 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97788512699999997 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.957327+0000", 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1223000000000005e-05 2024-04-12T11:43:10.566 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:65208 6.4 6:2236e19f:::benchmark_data_smithi029_138569_object65207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:41.885443+0000", 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 568.67076331099997, 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0581920730000001, 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.567 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 65208 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885443+0000", 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885443+0000", 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885443+0000", 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5300000000000004e-07 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885463+0000", 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9667999999999999e-05 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885471+0000", 2024-04-12T11:43:10.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5510000000000002e-06 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885476+0000", 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7899999999999996e-06 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.979414+0000", 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.093937309999999996 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.979423+0000", 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8400000000000001e-06 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.928913+0000", 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.949490104 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.929440+0000", 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00052702399999999996 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.943565+0000", 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014125464000000001 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.943635+0000", 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9769000000000002e-05 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:65212 6.11 6:8cc39d1f:::benchmark_data_smithi029_138569_object65211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:41.885484+0000", 2024-04-12T11:43:10.572 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 568.67072173500003, 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.056727341, 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 65212 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885484+0000", 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885484+0000", 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885485+0000", 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0699999999999997e-07 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885505+0000", 2024-04-12T11:43:10.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9922999999999999e-05 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885513+0000", 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1780000000000003e-06 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885518+0000", 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1880000000000001e-06 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.961991+0000", 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076472793999999997 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.962001+0000", 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8300000000000008e-06 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.942086+0000", 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98008479400000004 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.942212+0000", 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012612699999999999 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:65214 6.14 6:2b870084:::benchmark_data_smithi029_138569_object65213:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1093)", 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:41.885526+0000", 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 568.67067993600006, 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.084749467, 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.577 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 65214 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885526+0000", 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885526+0000", 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885527+0000", 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7599999999999997e-07 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885547+0000", 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0581000000000002e-05 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885555+0000", 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4710000000000001e-06 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.885560+0000", 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6969999999999998e-06 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.979446+0000", 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.093885315999999996 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:41.979453+0000", 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0090000000000001e-06 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.957354+0000", 2024-04-12T11:43:10.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97790074199999999 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.957494+0000", 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014065400000000001 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5", 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.957654+0000", 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015977499999999999 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.962348+0000", 2024-04-12T11:43:10.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046944159999999999 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.963735+0000", 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001386408 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.970157+0000", 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006422273 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.970221+0000", 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3867000000000004e-05 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:42.970276+0000", 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4781999999999998e-05 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75885 6.14 6:29f2ac6f:::benchmark_data_smithi029_138569_object75884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.991984+0000", 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56422249800005, 2024-04-12T11:43:10.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4136453849999999, 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75885 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.991984+0000", 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.991984+0000", 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.991986+0000", 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.897e-06 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992024+0000", 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8583e-05 2024-04-12T11:43:10.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992037+0000", 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2804e-05 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992047+0000", 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0088999999999999e-05 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993506+0000", 2024-04-12T11:43:10.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001458637 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993516+0000", 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0689e-05 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.398756+0000", 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4052396200000001 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.398922+0000", 2024-04-12T11:43:10.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016576199999999999 2024-04-12T11:43:10.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.399049+0000", 2024-04-12T11:43:10.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012729199999999999 2024-04-12T11:43:10.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.404218+0000", 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051686340000000001 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.404238+0000", 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0415e-05 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.404283+0000", 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4499000000000002e-05 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.404351+0000", 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7917999999999996e-05 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.404413+0000", 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2735000000000002e-05 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.405629+0000", 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001215811 2024-04-12T11:43:10.590 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75902 6.14 6:2aa45392:::benchmark_data_smithi029_138569_object75901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992164+0000", 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.564042266, 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.414696634, 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75902 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992164+0000", 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992164+0000", 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992165+0000", 2024-04-12T11:43:10.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7499999999999999e-07 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992188+0000", 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3244999999999999e-05 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992197+0000", 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4220000000000006e-06 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992209+0000", 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1966e-05 2024-04-12T11:43:10.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993548+0000", 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013380499999999999 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993555+0000", 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4290000000000004e-06 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.400820+0000", 2024-04-12T11:43:10.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.407265177 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.400998+0000", 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000177918 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.401201+0000", 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020347099999999999 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.405647+0000", 2024-04-12T11:43:10.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044459060000000003 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.405700+0000", 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3056000000000002e-05 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.405785+0000", 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4727000000000004e-05 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.405859+0000", 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4107e-05 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.405927+0000", 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7541999999999998e-05 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.406861+0000", 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00093374300000000001 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.597 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:75910 6.14 6:28a7acc5:::benchmark_data_smithi029_138569_object75909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992220+0000", 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56398597099997, 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.415772203, 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75910 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.598 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992220+0000", 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992220+0000", 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992221+0000", 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.06e-07 2024-04-12T11:43:10.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992273+0000", 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2272999999999999e-05 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992284+0000", 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0669000000000001e-05 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992290+0000", 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5169999999999996e-06 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993577+0000", 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012866780000000001 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993583+0000", 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4710000000000004e-06 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.402568+0000", 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4089841860000001 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.402733+0000", 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016501299999999999 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.402930+0000", 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000197169 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.405806+0000", 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028765779999999999 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.406934+0000", 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011280789999999999 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.407001+0000", 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6517e-05 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.407063+0000", 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2314e-05 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.407136+0000", 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2878000000000003e-05 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.407992+0000", 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00085635500000000005 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76065 6.14 6:2f32fea1:::benchmark_data_smithi029_138569_object76064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992299+0000", 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56390725799997, 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.424749673, 2024-04-12T11:43:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76065 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992299+0000", 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992299+0000", 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992299+0000", 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6899999999999998e-07 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992348+0000", 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8956999999999999e-05 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992357+0000", 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6619999999999994e-06 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992364+0000", 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.81e-06 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993612+0000", 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001247827 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993619+0000", 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.153e-06 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.408054+0000", 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.41443484 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.408237+0000", 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018379099999999999 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.408419+0000", 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018123299999999999 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.415420+0000", 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0070015920000000001 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.415478+0000", 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7215e-05 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.415644+0000", 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016620799999999999 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.415695+0000", 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1733000000000001e-05 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.415809+0000", 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113416 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.417049+0000", 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012397669999999999 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76091 6.14 6:2f257ce7:::benchmark_data_smithi029_138569_object76090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992373+0000", 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56383357899995, 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.427079268, 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.611 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76091 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992373+0000", 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992373+0000", 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992373+0000", 2024-04-12T11:43:10.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4799999999999998e-07 2024-04-12T11:43:10.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.613 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:43:10.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992417+0000", 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4196000000000002e-05 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992425+0000", 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6320000000000008e-06 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992431+0000", 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0939999999999996e-06 2024-04-12T11:43:10.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993645+0000", 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001213436 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993651+0000", 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.28e-06 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.409805+0000", 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4161538650000001 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.409967+0000", 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016278700000000001 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.410150+0000", 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018267199999999999 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.415500+0000", 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053500739999999998 2024-04-12T11:43:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.415575+0000", 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5124999999999996e-05 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.417115+0000", 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015399789999999999 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.417213+0000", 2024-04-12T11:43:10.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7415000000000002e-05 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.417317+0000", 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0001042 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.419452+0000", 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002134965 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.618 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76102 6.14 6:2aeae6a8:::benchmark_data_smithi029_138569_object76101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992439+0000", 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56376723100004, 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4319838919999999, 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76102 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992439+0000", 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992439+0000", 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992439+0000", 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3000000000000001e-07 2024-04-12T11:43:10.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992482+0000", 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3069999999999999e-05 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992490+0000", 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4059999999999999e-06 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992496+0000", 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7640000000000002e-06 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993673+0000", 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011773969999999999 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993679+0000", 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3040000000000001e-06 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.412310+0000", 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4186308510000001 2024-04-12T11:43:10.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.412520+0000", 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000210068 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.412717+0000", 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019671000000000001 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.415509+0000", 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027923259999999999 2024-04-12T11:43:10.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.422680+0000", 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071710760000000002 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.422769+0000", 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8808999999999996e-05 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.422833+0000", 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3775999999999994e-05 2024-04-12T11:43:10.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.422888+0000", 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5198999999999998e-05 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.424423+0000", 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001534706 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.625 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76140 6.14 6:2eb86dd4:::benchmark_data_smithi029_138569_object76139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992503+0000", 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56370347200004, 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4333063509999999, 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.626 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:10.628 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:10.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:10.628 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.628 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76140 2024-04-12T11:43:10.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.628 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992503+0000", 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992503+0000", 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992503+0000", 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.08e-07 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992552+0000", 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8560999999999998e-05 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992559+0000", 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2520000000000002e-06 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992565+0000", 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9549999999999997e-06 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993701+0000", 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001136181 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993708+0000", 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8059999999999997e-06 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.413923+0000", 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.420215327 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.414073+0000", 2024-04-12T11:43:10.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000150185 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.414272+0000", 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019872299999999999 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.419474+0000", 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052017640000000002 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.424498+0000", 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050237210000000001 2024-04-12T11:43:10.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.424552+0000", 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4168e-05 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.424603+0000", 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0800999999999997e-05 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.424668+0000", 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5327000000000007e-05 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.425809+0000", 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001141172 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76161 6.14 6:2cf861ec:::benchmark_data_smithi029_138569_object76160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992572+0000", 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56363426099995, 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4410656660000001, 2024-04-12T11:43:10.634 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76161 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992572+0000", 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992572+0000", 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992572+0000", 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9999999999999998e-07 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992616+0000", 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3683999999999997e-05 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992623+0000", 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1430000000000001e-06 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992629+0000", 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.84e-06 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993777+0000", 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011479909999999999 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993785+0000", 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3930000000000008e-06 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.419509+0000", 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.425723426 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.419682+0000", 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000173139 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.419856+0000", 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017422100000000001 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.422789+0000", 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029328840000000002 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.428394+0000", 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056051510000000001 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.428444+0000", 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0235000000000003e-05 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.432368+0000", 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039239189999999997 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.432431+0000", 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2960999999999996e-05 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.433638+0000", 2024-04-12T11:43:10.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001206279 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76225 6.14 6:2eff9a74:::benchmark_data_smithi029_138569_object76224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992636+0000", 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56357007700001, 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4430566360000001, 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.641 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76225 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992636+0000", 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992636+0000", 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992637+0000", 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4499999999999997e-07 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992680+0000", 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3414000000000001e-05 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992687+0000", 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2860000000000002e-06 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992693+0000", 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.48e-06 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993809+0000", 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011160289999999999 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993816+0000", 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5490000000000001e-06 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.421177+0000", 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.427360835 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.421323+0000", 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014593200000000001 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.421514+0000", 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000191294 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.424445+0000", 2024-04-12T11:43:10.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029301959999999999 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.428486+0000", 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040418499999999996 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.428542+0000", 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5328999999999998e-05 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.433954+0000", 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054119470000000003 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.434016+0000", 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2336999999999996e-05 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.435693+0000", 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001676713 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76241 6.14 6:2fdafc17:::benchmark_data_smithi029_138569_object76240:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992700+0000", 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56350649499996, 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4498316849999999, 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76241 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992700+0000", 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992700+0000", 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992700+0000", 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1199999999999998e-07 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992742+0000", 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2237999999999998e-05 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992749+0000", 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0180000000000004e-06 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992755+0000", 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4940000000000003e-06 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993837+0000", 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001082548 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993850+0000", 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2858e-05 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.425950+0000", 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.432099725 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.426853+0000", 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00090250400000000004 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.427061+0000", 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000208457 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.433899+0000", 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0068379319999999997 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.437529+0000", 2024-04-12T11:43:10.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036300400000000002 2024-04-12T11:43:10.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.653 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:43:10.654 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:43:10.654 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:43:10.654 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:43:10.654 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:43:10.654 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:43:10.654 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:43:10.654 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:43:10.654 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:43:10.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.437633+0000", 2024-04-12T11:43:10.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010404999999999999 2024-04-12T11:43:10.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.441384+0000", 2024-04-12T11:43:10.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037512930000000002 2024-04-12T11:43:10.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.441446+0000", 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2121999999999999e-05 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.442531+0000", 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010849939999999999 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.656 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76326 6.14 6:2e1498dc:::benchmark_data_smithi029_138569_object76325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992762+0000", 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56344450799998, 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4509592579999999, 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76326 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992762+0000", 2024-04-12T11:43:10.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992762+0000", 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992762+0000", 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1399999999999997e-07 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992812+0000", 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9849000000000003e-05 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992821+0000", 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5709999999999998e-06 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992826+0000", 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8039999999999998e-06 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993876+0000", 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001049924 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993884+0000", 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5750000000000003e-06 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.428575+0000", 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.434690885 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.428712+0000", 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013728699999999999 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.428872+0000", 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000160025 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.437581+0000", 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0087087829999999995 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.437749+0000", 2024-04-12T11:43:10.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016868100000000001 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.437856+0000", 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000106439 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.442547+0000", 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046907449999999996 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.442603+0000", 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5887000000000002e-05 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.443721+0000", 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001118389 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76333 6.14 6:2f8272cd:::benchmark_data_smithi029_138569_object76332:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992835+0000", 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56337101500003, 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.45213714, 2024-04-12T11:43:10.663 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76333 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992835+0000", 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992835+0000", 2024-04-12T11:43:10.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992836+0000", 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9500000000000003e-07 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992881+0000", 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4848999999999997e-05 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992888+0000", 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7400000000000004e-06 2024-04-12T11:43:10.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992894+0000", 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9710000000000001e-06 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993905+0000", 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010110640000000001 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993914+0000", 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8429999999999999e-06 2024-04-12T11:43:10.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.430137+0000", 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.43622312 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.430312+0000", 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017426 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.430570+0000", 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00025830900000000002 2024-04-12T11:43:10.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.437805+0000", 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072353809999999999 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.437908+0000", 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103018 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.443733+0000", 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058247790000000004 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.443808+0000", 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4988999999999996e-05 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.443869+0000", 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0646999999999997e-05 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.444972+0000", 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011036749999999999 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76361 6.14 6:2918a1af:::benchmark_data_smithi029_138569_object76360:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992902+0000", 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56330413600006, 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4603250189999999, 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76361 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.670 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992902+0000", 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992902+0000", 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992902+0000", 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0999999999999999e-07 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992945+0000", 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2364000000000003e-05 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992952+0000", 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8650000000000003e-06 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992957+0000", 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4909999999999996e-06 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993938+0000", 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00098088699999999995 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993945+0000", 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.72e-06 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.435738+0000", 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4417930029999999 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.435882+0000", 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014380299999999999 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.436049+0000", 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0001673 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.441368+0000", 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053186149999999996 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.443740+0000", 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023726429999999998 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.446978+0000", 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032375889999999999 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.451859+0000", 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048815589999999997 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.451919+0000", 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9789e-05 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.453227+0000", 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013079809999999999 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.676 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76414 6.14 6:2ad136ca:::benchmark_data_smithi029_138569_object76413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:58.992964+0000", 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.56324204299995, 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4625362319999999, 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76414 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992964+0000", 2024-04-12T11:43:10.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992964+0000", 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.992965+0000", 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3599999999999999e-07 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993011+0000", 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6508999999999998e-05 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993018+0000", 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1749999999999999e-06 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993024+0000", 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7089999999999999e-06 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993968+0000", 2024-04-12T11:43:10.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00094405699999999999 2024-04-12T11:43:10.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:58.993975+0000", 2024-04-12T11:43:10.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9990000000000002e-06 2024-04-12T11:43:10.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.437950+0000", 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.44397497 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.438103+0000", 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015332799999999999 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.438277+0000", 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017385400000000001 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.443747+0000", 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054700579999999999 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.446895+0000", 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031474049999999998 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.447017+0000", 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012264399999999999 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.453276+0000", 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062588690000000002 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.453342+0000", 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6149000000000007e-05 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.455500+0000", 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021580700000000002 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.683 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76431 6.14 6:2dd520ad:::benchmark_data_smithi029_138569_object76430:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:59.024360+0000", 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.53184582899996, 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.432572263, 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76431 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.024360+0000", 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.024360+0000", 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.024362+0000", 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.407e-06 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.024382+0000", 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0122000000000001e-05 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.024393+0000", 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1163e-05 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.024400+0000", 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.443e-06 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.024461+0000", 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1155999999999995e-05 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.024471+0000", 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0618000000000001e-05 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.439840+0000", 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4153689890000001 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.439993+0000", 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015272300000000001 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.440179+0000", 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018598099999999999 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.446848+0000", 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066687910000000003 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.446938+0000", 2024-04-12T11:43:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0160000000000004e-05 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.447059+0000", 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000121128 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.455607+0000", 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0085479149999999997 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.455815+0000", 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020798499999999999 2024-04-12T11:43:10.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.456933+0000", 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011176820000000001 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34776.0:76444 6.14 6:2ef7f712:::benchmark_data_smithi029_138569_object76443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1104)", 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:33:59.057439+0000", 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 551.49876694600005, 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4052830949999999, 2024-04-12T11:43:10.690 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34776", 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3238615914", 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76444 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.057439+0000", 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:10.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.057439+0000", 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.057441+0000", 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.939e-06 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.057469+0000", 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7382e-05 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.057489+0000", 2024-04-12T11:43:10.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0647999999999999e-05 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.057502+0000", 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2830000000000001e-05 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.057560+0000", 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7878999999999999e-05 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:43:10.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:33:59.057569+0000", 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7509999999999997e-06 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.445018+0000", 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.387449377 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.445230+0000", 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000212158 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4,7", 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.445403+0000", 2024-04-12T11:43:10.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000172802 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.448688+0000", 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032844929999999999 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.450928+0000", 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022401779999999998 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.451043+0000", 2024-04-12T11:43:10.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114926 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.460275+0000", 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092321030000000002 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.460432+0000", 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015729900000000001 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:34:00.462722+0000", 2024-04-12T11:43:10.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022903300000000001 2024-04-12T11:43:10.697 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.697 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.697 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.697 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:10.697 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:10.697 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:43:10.729 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:10.764 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:10.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:10.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:10.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:10.907 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:10.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:11.008 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:11.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 27299 27283 100.294 137.125 0.00424376 0.00996034 2024-04-12T11:43:11.042 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:11.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:11.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:11.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:11.168 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:11.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:11.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:11.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:10 smithi087 ceph-mon[89014]: pgmap v1712: 89 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 54 MiB/s wr, 867 op/s; 787/66855 objects degraded (1.177%); 871/66855 objects misplaced (1.303%); 19 MiB/s, 310 objects/s recovering 2024-04-12T11:43:11.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4132216150' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:43:11.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:10 smithi029 ceph-mon[93650]: pgmap v1712: 89 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 54 MiB/s wr, 867 op/s; 787/66855 objects degraded (1.177%); 871/66855 objects misplaced (1.303%); 19 MiB/s, 310 objects/s recovering 2024-04-12T11:43:11.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4132216150' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:43:11.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:10 smithi029 ceph-mon[97787]: pgmap v1712: 89 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 54 MiB/s wr, 867 op/s; 787/66855 objects degraded (1.177%); 871/66855 objects misplaced (1.303%); 19 MiB/s, 310 objects/s recovering 2024-04-12T11:43:11.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4132216150' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:43:11.270 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:11.294 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:11.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:11.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:11.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:11.420 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:11.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:11.521 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:11.544 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:11.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:11.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:11.646 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:11.669 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:11.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:11.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:11.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:11.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:11.794 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:11.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:11.895 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:11.921 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:11.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:11.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:11 smithi029 ceph-mon[93650]: osdmap e1184: 8 total, 8 up, 7 in 2024-04-12T11:43:12.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:11 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:12.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:11 smithi029 ceph-mon[97787]: osdmap e1184: 8 total, 8 up, 7 in 2024-04-12T11:43:12.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:11 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:12.022 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:12.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 27551 27535 95.5968 15.75 0.0031815 0.0104505 2024-04-12T11:43:12.054 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:12.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:12.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:12.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:12.180 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:12.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:12.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:12.305 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:12.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:12.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:12.314 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:11 smithi087 ceph-mon[89014]: osdmap e1184: 8 total, 8 up, 7 in 2024-04-12T11:43:12.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:11 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:12.407 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:12.431 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:12.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:12.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:12.533 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:12.557 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:12.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:12.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:12.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:12.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:12.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:43:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:43:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:43:12.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:12.808 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:12.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:12.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:12.910 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:12.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:12.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:12.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:13.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 27700 27684 91.0556 9.3125 0.00988121 0.0109769 2024-04-12T11:43:13.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:13.074 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:13.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:13.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:13.175 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:13.208 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:13.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:13.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:13.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:12 smithi029 ceph-mon[97787]: pgmap v1714: 89 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 1.4 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s; 899/69099 objects misplaced (1.301%); 25 MiB/s, 392 objects/s recovering 2024-04-12T11:43:13.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:12 smithi029 ceph-mon[97787]: osdmap e1185: 8 total, 8 up, 7 in 2024-04-12T11:43:13.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:12 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 787/66855 objects degraded (1.177%), 1 pg degraded) 2024-04-12T11:43:13.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:12 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:43:13.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:12 smithi029 ceph-mon[93650]: pgmap v1714: 89 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 1.4 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s; 899/69099 objects misplaced (1.301%); 25 MiB/s, 392 objects/s recovering 2024-04-12T11:43:13.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:12 smithi029 ceph-mon[93650]: osdmap e1185: 8 total, 8 up, 7 in 2024-04-12T11:43:13.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:12 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 787/66855 objects degraded (1.177%), 1 pg degraded) 2024-04-12T11:43:13.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:12 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:43:13.310 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:13.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:13.350 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:13.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:13.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:13.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:12 smithi087 ceph-mon[89014]: pgmap v1714: 89 pgs: 1 active+undersized+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 1.4 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 53 MiB/s wr, 846 op/s; 899/69099 objects misplaced (1.301%); 25 MiB/s, 392 objects/s recovering 2024-04-12T11:43:13.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:12 smithi087 ceph-mon[89014]: osdmap e1185: 8 total, 8 up, 7 in 2024-04-12T11:43:13.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:12 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 787/66855 objects degraded (1.177%), 1 pg degraded) 2024-04-12T11:43:13.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:12 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:43:13.487 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:13.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:13.588 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:13.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:13.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:13.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:13.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:13.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:13.741 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:13.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:13.842 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:13.879 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:13.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:13.980 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:14.015 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:14.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:14.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:14.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:43:14.026766+0000 min lat: 0.00105127 max lat: 2.05917 avg lat: 0.0108875 2024-04-12T11:43:14.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:43:14.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 29397 29381 91.8055 106.062 0.00492016 0.0108875 2024-04-12T11:43:14.117 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:14.150 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:14.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:14.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:14.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:13 smithi029 ceph-mon[93650]: osdmap e1186: 8 total, 8 up, 7 in 2024-04-12T11:43:14.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:13 smithi029 ceph-mon[97787]: osdmap e1186: 8 total, 8 up, 7 in 2024-04-12T11:43:14.288 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:14.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:14.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:14.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:14.413 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:14.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:14.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:14.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:43:14.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:13 smithi087 ceph-mon[89014]: osdmap e1186: 8 total, 8 up, 7 in 2024-04-12T11:43:14.515 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:14.556 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:14.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:14.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:14.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:14.704 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:14.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:14.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:14.805 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:14.841 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:14.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:14.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:14.981 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:14.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:15.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 31399 31383 93.3915 125.125 0.00690412 0.0107023 2024-04-12T11:43:15.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:15.107 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:15.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:15.209 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:15.245 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:15.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:15.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:15.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:14 smithi029 ceph-mon[97787]: pgmap v1717: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.6 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 202/80112 objects misplaced (0.252%); 24 MiB/s, 376 objects/s recovering 2024-04-12T11:43:15.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:14 smithi029 ceph-mon[93650]: pgmap v1717: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.6 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 202/80112 objects misplaced (0.252%); 24 MiB/s, 376 objects/s recovering 2024-04-12T11:43:15.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:43:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:43:14] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.43.0" 2024-04-12T11:43:15.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:15.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:15.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:15.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:15.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:14 smithi087 ceph-mon[89014]: pgmap v1717: 89 pgs: 1 active+recovering+undersized+remapped, 88 active+clean; 1.6 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 202/80112 objects misplaced (0.252%); 24 MiB/s, 376 objects/s recovering 2024-04-12T11:43:15.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:15.511 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:15.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:15.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:15.613 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:15.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:15.650 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:15.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:15.751 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:15.787 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:15.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:15.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:15.889 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:15.925 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:15.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:15.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:16.026 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:16.027 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 33139 33123 94.0892 108.75 0.00664389 0.0106227 2024-04-12T11:43:16.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:16.066 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:16.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:16.167 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:16.191 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:16.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:16.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:16.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:16.330 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:16.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:16.431 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:16.467 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:16.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:16.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:16.569 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:16.605 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:16.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:16.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:16.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:16.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:16.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:16.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:16.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:16.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:16.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:16.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:16.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:17.021 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:17.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:17.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:17.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 35128 35112 95.4027 124.312 0.00629867 0.0104767 2024-04-12T11:43:17.123 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:17.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:17.163 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:17.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:17.194 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:43:17.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-04-12T11:43:17.263 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:17.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:16 smithi029 ceph-mon[93650]: pgmap v1718: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 17 MiB/s, 276 objects/s recovering 2024-04-12T11:43:17.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:16 smithi029 ceph-mon[97787]: pgmap v1718: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 17 MiB/s, 276 objects/s recovering 2024-04-12T11:43:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:17.311 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:17.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:17.412 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:17.448 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:17.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:17.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:17.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:16 smithi087 ceph-mon[89014]: pgmap v1718: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 17 MiB/s, 276 objects/s recovering 2024-04-12T11:43:17.550 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:17.588 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:17.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:17.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:17.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:17.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:17.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:17.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:17.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:17.887 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:17.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:17.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:18.027 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 37216 37200 96.8645 130.5 0.00343672 0.0103183 2024-04-12T11:43:18.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:18.028 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:18.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:18.129 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:18.167 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:18.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:18.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:18.269 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:18.307 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:18.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:18.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:18.409 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:18.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:18.450 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:18.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:18.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:18.589 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:18.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:18.691 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:18.727 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:18.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:18.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:18.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:18.866 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:18.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:18.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:19.005 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:19.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:19.028 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 39341 39325 98.2972 132.812 0.00616113 0.0101667 2024-04-12T11:43:19.108 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:19.144 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:19.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:19.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:19.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:19.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:19 smithi029 ceph-mon[97787]: pgmap v1719: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s; 8.2 MiB/s, 130 objects/s recovering 2024-04-12T11:43:19.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:19 smithi029 ceph-mon[93650]: pgmap v1719: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s; 8.2 MiB/s, 130 objects/s recovering 2024-04-12T11:43:19.280 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:43:19.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:19.291 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:19.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:19.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:19.433 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:19.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:19 smithi087 ceph-mon[89014]: pgmap v1719: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s; 8.2 MiB/s, 130 objects/s recovering 2024-04-12T11:43:19.533 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:19.571 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:19.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:19.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:19.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:19.705 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:19.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:19.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:19.843 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:19.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:19.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:19.945 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:19.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:19.982 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:19.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:20.028 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 41024 41008 98.5618 105.188 0.00232244 0.0101272 2024-04-12T11:43:20.083 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:20.121 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:20.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:20.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:20.223 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:20.260 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:20.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:20.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:20.362 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:20.385 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:20.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:20.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:20.487 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:20.522 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:20.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:20.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:20.624 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:20.658 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:20.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:20.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:20.760 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:20.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:20.784 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:20.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:20.885 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:20.923 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:20.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:21.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:21.028 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 42479 42463 98.2791 90.9375 0.00764569 0.0101699 2024-04-12T11:43:21.064 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:21.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:21.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:21.167 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:21.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:21.204 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:21.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:21 smithi029 ceph-mon[97787]: pgmap v1720: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s; 7.0 MiB/s, 112 objects/s recovering 2024-04-12T11:43:21.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:21 smithi029 ceph-mon[93650]: pgmap v1720: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s; 7.0 MiB/s, 112 objects/s recovering 2024-04-12T11:43:21.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:21.328 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:21.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:21.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:21.430 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:21.455 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:21.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:21.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:21 smithi087 ceph-mon[89014]: pgmap v1720: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s; 7.0 MiB/s, 112 objects/s recovering 2024-04-12T11:43:21.557 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:21.581 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:21.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:21.682 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:21.721 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:21.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:21.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:21.822 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:21.869 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:21.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:21.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:21.971 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:22.004 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:22.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:22.028 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 43786 43770 97.6863 81.6875 0.00938473 0.0102326 2024-04-12T11:43:22.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:22.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:22.145 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:22.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:22.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:22.287 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:22.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:22.323 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:43:22.390 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:22.427 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:22.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:22.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:22.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:22.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:22.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:22 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2288165904' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:43:22.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:22.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:22.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:22.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:22 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2288165904' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:43:22.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:22.528 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:22.564 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:22.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:22.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:22.666 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:22.704 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:22.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:22.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:22 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2288165904' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:43:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:43:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:43:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:43:22.806 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:22.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:22.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:22.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:22.945 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:22.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:22.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:22.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:23.028 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 45572 45556 98.1665 111.625 0.00236247 0.0101745 2024-04-12T11:43:23.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:23.117 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:23.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:23.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:23.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:23.255 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:23.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:23.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:23.357 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:23.392 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:23.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:23.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:23.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[97787]: pgmap v1721: 89 pgs: 89 active+clean; 2.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 5.7 MiB/s, 90 objects/s recovering 2024-04-12T11:43:23.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:43:23.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2288165904' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:43:23.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[97787]: osdmap e1187: 8 total, 8 up, 7 in 2024-04-12T11:43:23.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2024-04-12T11:43:23.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]: dispatch 2024-04-12T11:43:23.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]}]: dispatch 2024-04-12T11:43:23.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[93650]: pgmap v1721: 89 pgs: 89 active+clean; 2.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 5.7 MiB/s, 90 objects/s recovering 2024-04-12T11:43:23.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:43:23.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2288165904' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:43:23.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[93650]: osdmap e1187: 8 total, 8 up, 7 in 2024-04-12T11:43:23.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2024-04-12T11:43:23.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]: dispatch 2024-04-12T11:43:23.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:23 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]}]: dispatch 2024-04-12T11:43:23.493 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:23.531 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:23.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:23.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:23.633 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:23.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:23.670 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:23.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:23 smithi087 ceph-mon[89014]: pgmap v1721: 89 pgs: 89 active+clean; 2.3 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 5.7 MiB/s, 90 objects/s recovering 2024-04-12T11:43:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:23 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:43:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:23 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2288165904' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:43:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:23 smithi087 ceph-mon[89014]: osdmap e1187: 8 total, 8 up, 7 in 2024-04-12T11:43:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2024-04-12T11:43:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]: dispatch 2024-04-12T11:43:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:23 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]}]: dispatch 2024-04-12T11:43:23.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:23.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:23.812 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:23.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:23.814 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:43:23.913 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:23.960 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:23.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:23.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:24.028 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 47308 47292 98.5107 108.5 0.00289685 0.0101461 2024-04-12T11:43:24.062 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:24.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:24.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:24.197 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:24.238 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:24.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:24.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:24.340 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:24.376 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:24.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:24.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:24.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:24.516 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:24.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:24.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:24.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:24 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2024-04-12T11:43:24.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:24 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]': finished 2024-04-12T11:43:24.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:24 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]}]': finished 2024-04-12T11:43:24.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:24 smithi029 ceph-mon[97787]: osdmap e1188: 8 total, 8 up, 7 in 2024-04-12T11:43:24.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:24 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2024-04-12T11:43:24.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:24 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]': finished 2024-04-12T11:43:24.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:24 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]}]': finished 2024-04-12T11:43:24.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:24 smithi029 ceph-mon[93650]: osdmap e1188: 8 total, 8 up, 7 in 2024-04-12T11:43:24.618 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:24.657 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:24.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:24 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2024-04-12T11:43:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:24 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]': finished 2024-04-12T11:43:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:24 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 0]}]': finished 2024-04-12T11:43:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:24 smithi087 ceph-mon[89014]: osdmap e1188: 8 total, 8 up, 7 in 2024-04-12T11:43:24.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:24.794 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:24.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:24.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:24.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:24.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:24.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:24.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:25.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 49102 49086 98.9495 112.125 0.00777243 0.0101004 2024-04-12T11:43:25.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:25.080 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:25.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:25.181 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:25.186 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:43:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:43:24] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2024-04-12T11:43:25.216 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:25.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:25.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:25.318 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:25.339 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 0.877016 (e084) 2024-04-12T11:43:25.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:25.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:25.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:25.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:43:25.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:25 smithi029 ceph-mon[97787]: pgmap v1724: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:43:25.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:25 smithi029 ceph-mon[97787]: osdmap e1189: 8 total, 8 up, 7 in 2024-04-12T11:43:25.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:25 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/14491326' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.8770157381312551}]: dispatch 2024-04-12T11:43:25.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:25 smithi029 ceph-mon[93650]: pgmap v1724: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:43:25.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:25 smithi029 ceph-mon[93650]: osdmap e1189: 8 total, 8 up, 7 in 2024-04-12T11:43:25.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:25 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/14491326' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.8770157381312551}]: dispatch 2024-04-12T11:43:25.459 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:25.492 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:25.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:25.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:25.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:25.629 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:25.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:25 smithi087 ceph-mon[89014]: pgmap v1724: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:43:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:25 smithi087 ceph-mon[89014]: osdmap e1189: 8 total, 8 up, 7 in 2024-04-12T11:43:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:25 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/14491326' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.8770157381312551}]: dispatch 2024-04-12T11:43:25.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:25.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:25.775 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:25.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:25.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:25.921 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:25.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:25.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:26.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:26.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 50885 50869 99.3394 111.438 0.0124336 0.0100612 2024-04-12T11:43:26.055 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:26.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:26.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:26.158 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:26.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:26.194 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:26.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:26.295 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:26.332 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:26.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:26.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:26.396 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:43:26.396 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:43:26.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.396 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:51424 8.f 8:fa2642f5:::benchmark_data_smithi029_194418_object51423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-04-12T11:43:26.396 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:26.387883+0000", 2024-04-12T11:43:26.396 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0060260009999999996, 2024-04-12T11:43:26.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060637290000000003, 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 51424 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.387883+0000", 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.387883+0000", 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.387885+0000", 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0269999999999998e-06 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.387909+0000", 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4273999999999998e-05 2024-04-12T11:43:26.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.387924+0000", 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4829e-05 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.387934+0000", 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9289999999999993e-06 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.388250+0000", 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000316187 2024-04-12T11:43:26.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.388407+0000", 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000156964 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.388668+0000", 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000260418 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:51428 8.f 8:ff1018fe:::benchmark_data_smithi029_194418_object51427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-04-12T11:43:26.400 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:26.390601+0000", 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0033077739999999999, 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034422200000000002, 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 51428 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.390601+0000", 2024-04-12T11:43:26.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.390601+0000", 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.390603+0000", 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.542e-06 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.390628+0000", 2024-04-12T11:43:26.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5486e-05 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.390644+0000", 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6265000000000001e-05 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.390658+0000", 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.362e-05 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.390719+0000", 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0389000000000001e-05 2024-04-12T11:43:26.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.390857+0000", 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000138138 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.390992+0000", 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000135129 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:51424 8.f e1191/1181)", 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:26.391157+0000", 2024-04-12T11:43:26.404 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0027518479999999999, 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002959546, 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391157+0000", 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391157+0000", 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391158+0000", 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9800000000000003e-07 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391163+0000", 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5870000000000003e-06 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391172+0000", 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8349999999999999e-06 2024-04-12T11:43:26.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391179+0000", 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1060000000000002e-06 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:51430 8.a e1191/1172 8:589d1bfc:::benchmark_data_smithi029_194418_object51429:head v 1191'3211, mlcod=1191'3209)", 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:26.391342+0000", 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0025671499999999998, 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028335639999999998, 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:26.407 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391342+0000", 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391342+0000", 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391343+0000", 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0399999999999995e-07 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391367+0000", 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4175000000000001e-05 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391376+0000", 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6169999999999996e-06 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391383+0000", 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7819999999999996e-06 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391887+0000", 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00050357199999999998 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.391907+0000", 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0126e-05 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:51429 8.7 e1191/1181 8:ec312982:::benchmark_data_smithi029_194418_object51428:head v 1191'3186, mlcod=1191'3184)", 2024-04-12T11:43:26.410 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:26.393194+0000", 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.00071458000000000003, 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010426610000000001, 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393194+0000", 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393194+0000", 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393195+0000", 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9899999999999999e-07 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393220+0000", 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4607e-05 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393231+0000", 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0994e-05 2024-04-12T11:43:26.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393238+0000", 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5850000000000002e-06 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393570+0000", 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000331398 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393590+0000", 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0343000000000002e-05 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.413 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:51426 8.e e1191/1172 8:7fe88877:::benchmark_data_smithi029_194418_object51425:head v 1191'3072, mlcod=1191'3070)", 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:26.393460+0000", 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.000448923, 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000837939, 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393460+0000", 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393460+0000", 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393461+0000", 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7100000000000001e-07 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393487+0000", 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6040000000000001e-05 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393499+0000", 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2374999999999999e-05 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393506+0000", 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0729999999999999e-06 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:43:26.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393840+0000", 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033420500000000002 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:26.393853+0000", 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2711000000000001e-05 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 6 2024-04-12T11:43:26.417 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:43:26.434 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:26.475 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:26.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:26.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:26 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/14491326' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.8770157381312551}]': finished 2024-04-12T11:43:26.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:26 smithi029 ceph-mon[93650]: osdmap e1190: 8 total, 8 up, 7 in 2024-04-12T11:43:26.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:26 smithi029 ceph-mon[93650]: osdmap e1191: 8 total, 8 up, 7 in 2024-04-12T11:43:26.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:26 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/14491326' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.8770157381312551}]': finished 2024-04-12T11:43:26.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:26 smithi029 ceph-mon[97787]: osdmap e1190: 8 total, 8 up, 7 in 2024-04-12T11:43:26.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:26 smithi029 ceph-mon[97787]: osdmap e1191: 8 total, 8 up, 7 in 2024-04-12T11:43:26.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:26.617 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:26.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:26 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/14491326' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.8770157381312551}]': finished 2024-04-12T11:43:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:26 smithi087 ceph-mon[89014]: osdmap e1190: 8 total, 8 up, 7 in 2024-04-12T11:43:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:26 smithi087 ceph-mon[89014]: osdmap e1191: 8 total, 8 up, 7 in 2024-04-12T11:43:26.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:26.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:26.754 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:26.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:26.855 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:26.888 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:26.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:26.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:26.990 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:27.026 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:27.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:27.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:27.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 52378 52362 99.1565 93.3125 0.0106571 0.0100802 2024-04-12T11:43:27.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:27.170 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:27.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:27.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:27.272 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:27.309 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:27.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:27.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:27.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:27.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:27.445 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:27.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:27.546 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:27.578 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:27.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:27.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:27.680 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:27.706 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:27.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:27.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:27 smithi087 ceph-mon[89014]: pgmap v1727: 89 pgs: 1 peering, 88 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-12T11:43:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:27 smithi029 ceph-mon[97787]: pgmap v1727: 89 pgs: 1 peering, 88 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-12T11:43:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:27 smithi029 ceph-mon[93650]: pgmap v1727: 89 pgs: 1 peering, 88 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-04-12T11:43:27.808 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:27.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:27.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:27.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:27.967 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:27.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:27.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:28.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 53712 53696 98.692 83.375 0.00234017 0.0101029 2024-04-12T11:43:28.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:28.094 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:28.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:28.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:28.232 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:28.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:28.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:28.334 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:28.372 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:28.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:28.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:28.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:28.511 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:28.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:28.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:28.613 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:28.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:28.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:28.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:28 smithi087 ceph-mon[89014]: pgmap v1729: 89 pgs: 1 peering, 88 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-04-12T11:43:28.755 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:28.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:28 smithi029 ceph-mon[97787]: pgmap v1729: 89 pgs: 1 peering, 88 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-04-12T11:43:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:28 smithi029 ceph-mon[93650]: pgmap v1729: 89 pgs: 1 peering, 88 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-04-12T11:43:28.793 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:28.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:28.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:28.895 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:28.931 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:28.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:29.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 55089 55073 98.331 86.0625 0.00796264 0.0101651 2024-04-12T11:43:29.034 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:29.068 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:29.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:29.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:29.170 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:29.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:29.194 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:29.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:29.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:29.337 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:29.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:29.440 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:29.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:29.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:29.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:29.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:29.589 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:29.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:29.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:29.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:29.728 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:29.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:29.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:29.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:29.852 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:29.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:29.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:29.954 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:29.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:29.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:29.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:30.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 56033 56017 97.2384 59 0.0116945 0.0102779 2024-04-12T11:43:30.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:30.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:30.113 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:30.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:30.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:30.253 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:30.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:30.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:30.355 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:30.394 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:30.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:30.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:30.496 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:30.531 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:30.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:30.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:30.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:30.658 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:30.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:30.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:30.799 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:30.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:30.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:30.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:30.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:30.932 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:30.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:31.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 57274 57258 96.7065 77.5625 0.00557507 0.0103353 2024-04-12T11:43:31.033 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:31.069 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:31.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:31.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:31.171 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:31.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:31.208 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:31.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:31.216 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:30 smithi087 ceph-mon[89014]: pgmap v1730: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 96 B/s, 0 objects/s recovering 2024-04-12T11:43:31.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:30 smithi029 ceph-mon[97787]: pgmap v1730: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 96 B/s, 0 objects/s recovering 2024-04-12T11:43:31.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:30 smithi029 ceph-mon[93650]: pgmap v1730: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 100 MiB/s wr, 1.60k op/s; 96 B/s, 0 objects/s recovering 2024-04-12T11:43:31.309 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:31.332 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:31.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:31.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:31.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:31.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:31.471 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:31.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:31.572 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:31.608 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:31.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:31.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:31.710 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:31.740 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:31.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:31.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:31.879 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:31.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:31.981 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:32.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:32.009 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:32.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:32.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 58671 58655 96.459 87.3125 0.00300648 0.0103514 2024-04-12T11:43:32.110 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:32.142 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:32.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:32.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:32.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:32.277 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:32.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:32.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:32.379 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:32.403 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:32.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:32.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:32.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:32.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:32.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:32.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:32.674 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:32.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:32.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:43:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:43:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:43:32.776 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:32.816 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:32.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:32.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:32.917 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:32.955 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:32.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:33.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 15 59906 59891 95.9663 77.25 0.00820605 0.0104147 2024-04-12T11:43:33.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:33.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:33.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:33.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:33.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:33.219 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:32 smithi087 ceph-mon[89014]: pgmap v1731: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 77 B/s, 0 objects/s recovering 2024-04-12T11:43:33.231 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:33.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:33.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:32 smithi029 ceph-mon[97787]: pgmap v1731: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 77 B/s, 0 objects/s recovering 2024-04-12T11:43:33.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:32 smithi029 ceph-mon[93650]: pgmap v1731: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 77 B/s, 0 objects/s recovering 2024-04-12T11:43:33.333 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:33.368 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:33.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:33.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:33.470 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:33.508 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:33.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:33.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:33.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:33.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:33.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:33.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:33.750 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:33.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:33.787 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:33.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:33.889 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:33.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:33.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:34.014 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:34.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:43:34.030112+0000 min lat: 0.00105127 max lat: 2.05917 avg lat: 0.0104467 2024-04-12T11:43:34.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:43:34.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 61259 61243 95.6789 84.5 0.0171461 0.0104467 2024-04-12T11:43:34.050 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:34.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:34.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:34.152 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:34.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:34.176 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:34.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:34.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:34.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:34.309 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:34.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:34.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:34.440 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:34.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:34.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:34.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:34.578 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:34.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:34.679 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:34.705 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:34.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:35.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:35.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 62252 62236 94.8588 62.0625 0.00786741 0.010537 2024-04-12T11:43:35.101 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:35.138 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:35.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:35.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:35.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:34 smithi087 ceph-mon[89014]: pgmap v1732: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 68 B/s, 0 objects/s recovering 2024-04-12T11:43:35.240 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:35.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:34 smithi029 ceph-mon[97787]: pgmap v1732: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 68 B/s, 0 objects/s recovering 2024-04-12T11:43:35.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:34 smithi029 ceph-mon[93650]: pgmap v1732: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 68 B/s, 0 objects/s recovering 2024-04-12T11:43:35.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:43:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:43:34] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2024-04-12T11:43:35.277 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:35.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:35.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:35.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:35.413 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:35.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:35.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:35.515 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:35.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:35.546 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:35.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:35.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:35.684 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:35.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:35.786 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:35.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:35.825 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:35.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:35.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:35.966 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:35.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:36.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 63617 63601 94.6314 85.3125 0.0111733 0.0105624 2024-04-12T11:43:36.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:36.105 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:36.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:36.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:36.116 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:43:36.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-04-12T11:43:36.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:36.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:36.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:36.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:36.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:36.392 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:36.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:36.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:36.494 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:36.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:36.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:36.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:36.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:36.662 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:43:36.662 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:43:36.663 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 4, 5, 6, 7] 2024-04-12T11:43:36.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd out 7 2024-04-12T11:43:36.720 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:36.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:36.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:36.822 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:36.862 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:36.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:36.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:36.889 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:43:36.941 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:43:36.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:36.990 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:43:37.012 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:37.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:37.032 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 64706 64690 94.0134 68.0625 0.0243939 0.0106288 2024-04-12T11:43:37.033 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:43:37.088 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:43:37.114 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:37.131 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:43:37.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:37.161 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:37.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:37.181 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:43:37.229 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:43:37.240 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:43:37.240 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:43:37.240 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:43:37.240 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:43:37.240 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:43:37.240 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:43:37.240 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:43:37.240 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:43:37.262 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:37.297 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:37.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:37.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:37.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:37.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:37.440 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:37.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:37.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[97787]: pgmap v1733: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 58 B/s, 0 objects/s recovering 2024-04-12T11:43:37.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:37.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:37.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:43:37.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:37.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/526460619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:43:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[93650]: pgmap v1733: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 58 B/s, 0 objects/s recovering 2024-04-12T11:43:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:43:37.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:37.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:37 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/526460619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:43:37.541 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:37.578 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:37.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:37.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:37.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:37.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:37.719 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:37.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:37.720 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:37 smithi087 ceph-mon[89014]: pgmap v1733: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 58 B/s, 0 objects/s recovering 2024-04-12T11:43:37.720 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:37.720 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:37.720 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:43:37.720 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:37.721 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:37 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/526460619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:43:37.821 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:37.856 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:37.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:37.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:37.960 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:37.983 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:37.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:37.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:38.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 65961 65945 93.6592 78.4375 0.00666899 0.0106724 2024-04-12T11:43:38.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:38.116 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:38.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:38.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:38.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:38.253 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:38.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:38.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:38.354 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:38.379 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:38.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:38.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:38.481 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:38.511 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:38.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:38.613 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:38.643 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:38.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:38.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:38.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:38.769 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:38.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:38.870 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:38.895 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:38.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:38.996 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:39.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 66900 66884 92.882 58.6875 0.00664052 0.0107606 2024-04-12T11:43:39.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:39.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:39.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:39.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:39.168 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:39.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:39.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:39.270 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:39.294 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:39.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:39.395 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:39.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:39.449 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:39.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:39.550 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:39.584 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:39.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:39.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:39.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:39 smithi087 ceph-mon[89014]: pgmap v1734: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 50 B/s, 0 objects/s recovering 2024-04-12T11:43:39.727 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:39.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:39.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:39 smithi029 ceph-mon[93650]: pgmap v1734: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 50 B/s, 0 objects/s recovering 2024-04-12T11:43:39.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:39 smithi029 ceph-mon[97787]: pgmap v1734: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s; 50 B/s, 0 objects/s recovering 2024-04-12T11:43:39.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:39.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:39.872 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:39.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:39.974 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:40.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:40.005 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:40.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:40.031 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 67996 67980 92.3519 68.5 0.0036007 0.0108087 2024-04-12T11:43:40.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:40.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:40.149 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:40.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:40.250 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:40.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:40.291 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:40.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:40.392 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:40.420 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:40.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:40.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:40.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:40.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:40.568 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:40.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:40.670 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:40.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:40.718 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:40.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:40.819 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:40.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:40.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:40.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:40.945 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:40.974 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:40.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:41.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 68964 68948 91.6741 60.5 0.0059862 0.0108926 2024-04-12T11:43:41.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:41.114 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:41.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:41.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:41.218 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:41.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:41.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:41.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:41.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:41.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:41.396 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:41.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:41.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:41.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:41 smithi029 ceph-mon[93650]: pgmap v1735: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 48 B/s, 0 objects/s recovering 2024-04-12T11:43:41.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:41 smithi029 ceph-mon[97787]: pgmap v1735: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 48 B/s, 0 objects/s recovering 2024-04-12T11:43:41.521 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:41.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:41.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:41.623 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:41.624 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:41 smithi087 ceph-mon[89014]: pgmap v1735: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 48 B/s, 0 objects/s recovering 2024-04-12T11:43:41.668 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:41.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:41.770 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:41.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:41.800 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:41.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:41.901 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:41.939 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:41.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:42.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 70011 69995 91.1274 65.4375 0.00704694 0.0109681 2024-04-12T11:43:42.041 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:42.085 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:42.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:42.187 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:42.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:42.220 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:42.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:42.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:42.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:42.346 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:42.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:42.446 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:42.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:42.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:42.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:42.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:42.618 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:42.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:42.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:43:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:43:42.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:43:42.721 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:42.752 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:42.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:42.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:42.854 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:42.889 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:42.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:42.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:42.990 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:43.016 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:43.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:43.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:43.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 71068 71052 90.6157 66.0625 0.00722588 0.0110255 2024-04-12T11:43:43.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:43.142 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:43.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:43.214 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:43 smithi029 ceph-mon[97787]: pgmap v1736: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-12T11:43:43.214 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:43 smithi029 ceph-mon[93650]: pgmap v1736: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-12T11:43:43.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:43.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:43.268 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:43.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:43.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:43:43.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:43.411 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:43.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:43.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:43.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:43 smithi087 ceph-mon[89014]: pgmap v1736: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-12T11:43:43.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:43 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:43.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:43 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:43.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:43.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:43 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:43.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:43 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:43.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:43 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:43.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:43 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:43:43.557 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:43.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:43.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:43.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:43.694 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:43.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:43.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:43.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:43.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:43.832 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:43.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:43.933 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:43.966 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:43.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:43.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:44.032 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 72062 72046 90.0443 62.125 0.00824153 0.011101 2024-04-12T11:43:44.068 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:44.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:44.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:44.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:44.163 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:43:44.163 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:43:44.234 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:44.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:44.272 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:44.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:44.373 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:44.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:44.415 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:44.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:44.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:44.559 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:44.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:44.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:44.707 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:44.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:44.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:44.808 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:43:44.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:44.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:44.842 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:44.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:44.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:44.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:44.989 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:44.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:45.034 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 73300 73284 89.7952 77.375 0.0270409 0.011131 2024-04-12T11:43:45.040 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:43:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:43:44] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2024-04-12T11:43:45.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:45.140 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:45.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:45.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:45.242 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:45.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:45.277 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:45.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:45.312 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:45 smithi029 ceph-mon[97787]: pgmap v1737: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-04-12T11:43:45.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:45 smithi029 ceph-mon[93650]: pgmap v1737: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-04-12T11:43:45.378 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:45.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:45.413 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:45.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:45.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:45 smithi087 ceph-mon[89014]: pgmap v1737: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-04-12T11:43:45.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:45.562 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:45.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:45.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:45.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:45.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:45.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:45.803 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:45.843 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:45.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:45.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:45.945 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:45.988 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:45.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:45.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:46.032 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 74677 74661 89.7234 86.0625 0.00797929 0.011141 2024-04-12T11:43:46.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:46.114 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:43:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:46.125 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:46.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:46.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:46.265 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:46.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:46.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:46.367 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:46.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:46 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2562790487' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:43:46.373 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:46 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2562790487' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:43:46.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:46.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:46.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:46.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:46 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2562790487' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:43:46.507 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:46.543 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:46.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:46.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:46.645 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:46.682 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:46.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:46.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:46.784 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:46.817 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:46.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:46.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:46.954 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:46.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:46.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:47.032 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 76063 76047 89.6648 86.625 0.0070715 0.0111478 2024-04-12T11:43:47.056 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:47.092 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:47.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:47.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:47.194 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:47.232 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:47.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:47 smithi029 ceph-mon[97787]: pgmap v1738: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-12T11:43:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:47 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:43:47.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:47 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2562790487' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:43:47.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:47 smithi029 ceph-mon[97787]: osdmap e1192: 8 total, 8 up, 7 in 2024-04-12T11:43:47.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:47 smithi029 ceph-mon[93650]: pgmap v1738: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-12T11:43:47.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:47 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:43:47.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:47 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2562790487' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:43:47.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:47 smithi029 ceph-mon[93650]: osdmap e1192: 8 total, 8 up, 7 in 2024-04-12T11:43:47.333 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:47.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:47.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:47.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:47 smithi087 ceph-mon[89014]: pgmap v1738: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-12T11:43:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:47 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:43:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:47 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2562790487' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:43:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:47 smithi087 ceph-mon[89014]: osdmap e1192: 8 total, 8 up, 7 in 2024-04-12T11:43:47.472 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:47.507 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:47.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:47.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:47.609 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:47.643 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:47.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:47.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:47.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:47.783 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:47.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:47.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:47.885 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:47.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:47.929 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:47.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:48.030 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:48.032 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 77435 77419 89.5922 85.75 0.0123841 0.0111572 2024-04-12T11:43:48.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:48.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:48.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:48.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:48.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:48.206 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:48.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:48.307 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:48.344 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:48.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:48.446 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:48.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:48.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:48.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:48.585 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:48.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:48.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:48.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:48.760 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:48.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:48.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:48.886 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:48.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:48.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:49.022 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:49.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:49.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:49.032 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 78638 78622 89.3301 75.1875 0.00939679 0.011189 2024-04-12T11:43:49.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:49.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:49.165 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:49.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:49.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:49.295 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:49.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:49.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:49.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:49.433 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:49.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:49.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:49.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:49 smithi087 ceph-mon[89014]: pgmap v1740: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-12T11:43:49.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:49 smithi029 ceph-mon[97787]: pgmap v1740: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-12T11:43:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:49 smithi029 ceph-mon[93650]: pgmap v1740: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-12T11:43:49.535 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:49.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:49.576 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:49.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:49.677 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:49.715 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:49.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:49.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:49.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:49.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:49.853 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:49.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:49.954 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:49.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:49.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:50.033 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 79900 79884 89.1433 78.875 0.00476043 0.0112129 2024-04-12T11:43:50.090 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:50.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:43:50.130 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:50.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:50.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:50.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:50.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:50.289 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:50.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:50.390 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:50.426 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:50.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:50.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:50.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:50.570 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:50.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:50.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:50.672 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:50.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:50.705 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:50.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:50.806 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:50.846 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:50.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:50.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:50.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:50.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:50.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:50.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:51.031 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:43:51.032 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 81106 81090 88.9017 75.375 0.00737392 0.0112433 2024-04-12T11:43:51.068 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:43:51.090 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:51.105 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:43:51.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:51.136 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:51.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:51.139 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:43:51.183 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:43:51.219 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:43:51.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:51.265 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:43:51.273 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:51.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:51.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:51.304 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:43:51.305 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:51 smithi029 ceph-mon[97787]: pgmap v1741: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T11:43:51.305 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:51 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2193874356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:43:51.305 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:51 smithi029 ceph-mon[93650]: pgmap v1741: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T11:43:51.305 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:51 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2193874356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:43:51.318 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:43:51.318 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:43:51.318 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:43:51.318 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:43:51.318 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:43:51.318 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:43:51.318 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:43:51.318 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:43:51.375 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:51.411 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:51.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:51.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:51.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:51 smithi087 ceph-mon[89014]: pgmap v1741: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T11:43:51.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:51 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2193874356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:43:51.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:51.557 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:51.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:51.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:51.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:51.697 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:51.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:51.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:51.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:51.835 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:51.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:51.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:51.937 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:51.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:51.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:51.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:52.033 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 82397 82381 88.7599 80.6875 0.00951962 0.0112617 2024-04-12T11:43:52.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:52.111 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:52.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:52.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:52.212 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:52.247 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:52.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:52.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:52.349 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:52.382 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:52.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:52.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:52.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:52.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:43:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:43:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:43:52.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:52.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:52.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:43:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:52.525 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:52.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:52.626 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:52.665 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:52.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:52.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:52.766 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:52.805 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:52.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:52.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:52.907 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:52.945 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:52.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:52.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:53.033 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 83690 83674 88.6251 80.8125 0.0109041 0.0112785 2024-04-12T11:43:53.047 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:53.085 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:53.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:53.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:53.188 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:53.223 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:53.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:53.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:53.325 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:53.359 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:53.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:53.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:53.407 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:53 smithi029 ceph-mon[97787]: pgmap v1742: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-12T11:43:53.407 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:53 smithi029 ceph-mon[93650]: pgmap v1742: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-12T11:43:53.460 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:53.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:53 smithi087 ceph-mon[89014]: pgmap v1742: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-12T11:43:53.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:53.512 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:53.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:53.613 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:53.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:53.658 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:53.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:53.759 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:53.816 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:53.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:53.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:53.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:53.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:53.981 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:53.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:54.011 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:43:54.035 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:43:54.035 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:43:54.035 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:43:54.035 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:43:54.035 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:43:54.035 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:43:54.035695+0000 min lat: 0.00105127 max lat: 2.05917 avg lat: 0.0113181 2024-04-12T11:43:54.036 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:43:54.036 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 84810 84794 88.3107 70 0.0218967 0.0113181 2024-04-12T11:43:54.082 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:54.141 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:54.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:54.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:54.242 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:54.277 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:54.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:54.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:54.379 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:54.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T11:43:54.195+0000 7fae24212700 -1 mon.a@0(leader).osd e1192 definitely_dead 0 2024-04-12T11:43:54.423 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:54.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:54.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:54.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:54.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:54.563 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:54.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:54.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:54.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:54.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:54.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:54.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:54.840 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:54.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:54.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:54.949 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-04-12T11:43:54.980 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:54.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:54.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:55.035 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 86057 86041 88.1405 77.9375 0.0101822 0.0113403 2024-04-12T11:43:55.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:55.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:55.119 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:55.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:55.161 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.7. 2024-04-12T11:43:55.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:55 smithi029 ceph-mon[97787]: pgmap v1743: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-12T11:43:55.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:55 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3529224242' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2024-04-12T11:43:55.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:55 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2024-04-12T11:43:55.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:55 smithi029 ceph-mon[97787]: Client client.admin marked osd.7 out, while it was still marked up 2024-04-12T11:43:55.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:55 smithi029 ceph-mon[93650]: pgmap v1743: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-12T11:43:55.169 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:43:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:43:54] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2024-04-12T11:43:55.172 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:43:55.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:55.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:55.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:55.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:55.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:55.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:55.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:55.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:55.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:55 smithi087 ceph-mon[89014]: pgmap v1743: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-12T11:43:55.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:55 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3529224242' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2024-04-12T11:43:55.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:55 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2024-04-12T11:43:55.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:55 smithi087 ceph-mon[89014]: Client client.admin marked osd.7 out, while it was still marked up 2024-04-12T11:43:55.471 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:55.495 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:55.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:55.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:55.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:55 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3529224242' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2024-04-12T11:43:55.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:55 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2024-04-12T11:43:55.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:55 smithi029 ceph-mon[93650]: Client client.admin marked osd.7 out, while it was still marked up 2024-04-12T11:43:55.597 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:55.621 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:55.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:55.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:55.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:55.748 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:55.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:55.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:55.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:55.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:55.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:55.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:55.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:56.002 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:56.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:56.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:56.036 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 86213 86197 86.8762 9.75 0.00386862 0.011347 2024-04-12T11:43:56.104 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:56.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:56.133 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:56.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:56.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:56.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:56.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:56.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:56.296 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:56 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-04-12T11:43:56.296 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:56 smithi029 ceph-mon[93650]: osdmap e1193: 8 total, 8 up, 6 in 2024-04-12T11:43:56.296 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:56 smithi029 ceph-mon[93650]: osdmap e1194: 8 total, 8 up, 6 in 2024-04-12T11:43:56.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:56 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-04-12T11:43:56.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:56 smithi029 ceph-mon[97787]: osdmap e1193: 8 total, 8 up, 6 in 2024-04-12T11:43:56.297 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:56 smithi029 ceph-mon[97787]: osdmap e1194: 8 total, 8 up, 6 in 2024-04-12T11:43:56.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:56.383 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:56.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:56.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:56.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:56 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-04-12T11:43:56.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:56 smithi087 ceph-mon[89014]: osdmap e1193: 8 total, 8 up, 6 in 2024-04-12T11:43:56.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:56 smithi087 ceph-mon[89014]: osdmap e1194: 8 total, 8 up, 6 in 2024-04-12T11:43:56.485 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:56.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:56.510 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:56.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:56.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:56.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:56.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:56.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:56.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:56.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:56.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:56.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:56.871 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:56.894 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:56.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:56.996 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:57.019 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:57.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:57.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:57.035 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 86213 86197 85.4974 0 - 0.011347 2024-04-12T11:43:57.120 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:57.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:57.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:57.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:57.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:57.278 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:57.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:57.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:57.380 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:57.411 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:57.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:57.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:57.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:57 smithi087 ceph-mon[89014]: pgmap v1745: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T11:43:57.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:57 smithi087 ceph-mon[89014]: osdmap e1195: 8 total, 8 up, 6 in 2024-04-12T11:43:57.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:43:57.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:57 smithi029 ceph-mon[97787]: pgmap v1745: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T11:43:57.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:57 smithi029 ceph-mon[97787]: osdmap e1195: 8 total, 8 up, 6 in 2024-04-12T11:43:57.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:57 smithi029 ceph-mon[93650]: pgmap v1745: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T11:43:57.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:57 smithi029 ceph-mon[93650]: osdmap e1195: 8 total, 8 up, 6 in 2024-04-12T11:43:57.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:57.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:57.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:43:57.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:57.682 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:57.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:57.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:57.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:57.819 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:57.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:57.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:57.921 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:57.962 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:57.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:57.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:58.036 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 86713 86697 84.6498 15.625 0.00718647 0.0118073 2024-04-12T11:43:58.064 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:43:58.091 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:58.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:58.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:43:58.193 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:58.217 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:58.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:58.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:58.319 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:43:58.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:58.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:58.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:43:58.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:58.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:58.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:58.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:58.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:58 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 10825/256638 objects degraded (4.218%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:43:58.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:58 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 10825/256638 objects degraded (4.218%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:43:58.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:58.616 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:58.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:58 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 10825/256638 objects degraded (4.218%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:43:58.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:43:58.758 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:58.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:58.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:43:58.860 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:58.911 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:58.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:58.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:59.013 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:59.036 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 87396 87380 84.0042 42.6875 0.0494687 0.0118979 2024-04-12T11:43:59.051 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:59.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:59.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:59.153 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:59.189 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:59.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:59.291 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:59.329 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:59.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:59.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:59.431 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:43:59.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:59.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:59.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:43:59.491 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:43:59 smithi029 ceph-mon[93650]: pgmap v1748: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s; 10825/256638 objects degraded (4.218%); 5255/256638 objects misplaced (2.048%); 32 KiB/s, 0 objects/s recovering 2024-04-12T11:43:59.491 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:43:59 smithi029 ceph-mon[97787]: pgmap v1748: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s; 10825/256638 objects degraded (4.218%); 5255/256638 objects misplaced (2.048%); 32 KiB/s, 0 objects/s recovering 2024-04-12T11:43:59.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:43:59.600 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:59.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:59.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:43:59.702 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:43:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:43:59 smithi087 ceph-mon[89014]: pgmap v1748: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s; 10825/256638 objects degraded (4.218%); 5255/256638 objects misplaced (2.048%); 32 KiB/s, 0 objects/s recovering 2024-04-12T11:43:59.735 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:43:59.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:59.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:43:59.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:43:59.878 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:43:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:43:59.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:43:59.979 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:00.018 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:00.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:00.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:00.036 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 88312 88296 83.5988 57.25 0.0189332 0.0119564 2024-04-12T11:44:00.120 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:00.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:00.159 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:00.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:00.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:00.304 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:00.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:00.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:00.405 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:00.440 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:00.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:00.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:00.542 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:00.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:00.578 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:00.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:00.678 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:00.713 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:00.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:00.815 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:00.850 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:00.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:00.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:00.952 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:00.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:00.988 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:00.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:00.989 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:01.036 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 89181 89165 83.1613 54.3125 0.00596885 0.0120194 2024-04-12T11:44:01.089 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:01.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:01.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:01.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:01.178 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:01.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:01.271 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:01.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:01.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:01 smithi029 ceph-mon[97787]: pgmap v1749: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 48 MiB/s wr, 771 op/s; 16115/260019 objects degraded (6.198%); 10537/260019 objects misplaced (4.052%); 533 KiB/s, 8 objects/s recovering 2024-04-12T11:44:01.333 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:01 smithi029 ceph-mon[93650]: pgmap v1749: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 48 MiB/s wr, 771 op/s; 16115/260019 objects degraded (6.198%); 10537/260019 objects misplaced (4.052%); 533 KiB/s, 8 objects/s recovering 2024-04-12T11:44:01.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:01.412 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:01.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:01.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:01.514 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:01.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:01.553 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:01.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:01.654 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:01.694 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:01.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:01.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:01 smithi087 ceph-mon[89014]: pgmap v1749: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 48 MiB/s wr, 771 op/s; 16115/260019 objects degraded (6.198%); 10537/260019 objects misplaced (4.052%); 533 KiB/s, 8 objects/s recovering 2024-04-12T11:44:01.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:01.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:01.833 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:01.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:01.934 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:01.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:01.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:01.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:02.036 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 90088 90072 82.772 56.6875 0.0205319 0.0120762 2024-04-12T11:44:02.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:02.110 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:02.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:02.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:02.212 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:02.257 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:02.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:02.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:02.359 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:02.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:02.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:02.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:02.492 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:02.530 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:02.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:02.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:02.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:02.676 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:02.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:02.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:44:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:44:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:44:02.777 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:02.818 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:02.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:02.919 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:02.961 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:02.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:02.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:03.036 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 90910 90894 82.3169 51.375 0.05154 0.0121397 2024-04-12T11:44:03.063 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:03.118 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:03.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:03.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:03.220 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:03.239 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:44:03.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:03.261 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:03.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:03.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:03 smithi029 ceph-mon[97787]: pgmap v1750: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 16115/261300 objects degraded (6.167%); 10537/261300 objects misplaced (4.033%); 476 KiB/s, 1 keys/s, 8 objects/s recovering 2024-04-12T11:44:03.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:03 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/632828978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:44:03.337 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:03 smithi029 ceph-mon[93650]: pgmap v1750: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 16115/261300 objects degraded (6.167%); 10537/261300 objects misplaced (4.033%); 476 KiB/s, 1 keys/s, 8 objects/s recovering 2024-04-12T11:44:03.337 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:03 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/632828978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:44:03.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:03.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:03.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:03.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:03.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:03.538 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:03.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:03.640 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:03.678 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:03.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:03.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:03 smithi087 ceph-mon[89014]: pgmap v1750: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 16115/261300 objects degraded (6.167%); 10537/261300 objects misplaced (4.033%); 476 KiB/s, 1 keys/s, 8 objects/s recovering 2024-04-12T11:44:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:03 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/632828978' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:44:03.780 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:03.824 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:03.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:03.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:03.835 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6] out_osds: [3, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:44:03.835 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:44:03.835 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2024-04-12T11:44:03.836 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:44:03.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:03.933 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:44:03.933 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:44:03.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:44:03.933 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:44:03.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.933 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:86738 8.a e1195/1172 8:534adc23:::benchmark_data_smithi029_194418_object86737:head v 1195'5464, mlcod=1195'5462)", 2024-04-12T11:44:03.933 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:58.059454+0000", 2024-04-12T11:44:03.933 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 5.8683048439999999, 2024-04-12T11:44:03.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.062029055, 2024-04-12T11:44:03.933 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.934 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.934 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.934 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.059454+0000", 2024-04-12T11:44:03.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.059454+0000", 2024-04-12T11:44:03.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.059455+0000", 2024-04-12T11:44:03.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.82e-07 2024-04-12T11:44:03.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.059476+0000", 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1367999999999999e-05 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.059484+0000", 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4309999999999994e-06 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.059490+0000", 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8320000000000002e-06 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.059555+0000", 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4842e-05 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.059603+0000", 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7967999999999999e-05 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.062485+0000", 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028823339999999999 2024-04-12T11:44:03.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.121483+0000", 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.058997797999999997 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:86742 8.d e1195/1194 8:b3d64dcd:::benchmark_data_smithi029_194418_object86741:head v 1195'5449, mlcod=0'0)", 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:58.065552+0000", 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 5.8622066779999997, 2024-04-12T11:44:03.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.064224273999999998, 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.065552+0000", 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.065552+0000", 2024-04-12T11:44:03.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.065554+0000", 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0669999999999999e-06 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.065608+0000", 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3804999999999997e-05 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.121460+0000", 2024-04-12T11:44:03.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.055852245000000002 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.121472+0000", 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1268e-05 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.121535+0000", 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3548999999999998e-05 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.121565+0000", 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9671000000000001e-05 2024-04-12T11:44:03.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.129731+0000", 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0081664489999999992 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.129777+0000", 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5219999999999997e-05 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86752 8.f 8:fbf9aef9:::benchmark_data_smithi029_194418_object86751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1195)", 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:58.127752+0000", 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 5.8000066950000004, 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.094424965999999999, 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86752 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.943 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.127752+0000", 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.127752+0000", 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.127754+0000", 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0849999999999999e-06 2024-04-12T11:44:03.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.127785+0000", 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0428000000000001e-05 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.127799+0000", 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4408e-05 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.127810+0000", 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0738e-05 2024-04-12T11:44:03.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.127882+0000", 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2001000000000004e-05 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.128042+0000", 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016033700000000001 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:44:03.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.128206+0000", 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000163559 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.130910+0000", 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00270375 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.131552+0000", 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00064278299999999996 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.221964+0000", 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.090411619999999998 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.222114+0000", 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014998999999999999 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.222177+0000", 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3267000000000003e-05 2024-04-12T11:44:03.948 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.949 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.949 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:86971 8.7 e1195/1181 8:e990c674:::benchmark_data_smithi029_194418_object86970:head v 1195'5354, mlcod=1195'5351)", 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:58.455637+0000", 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 5.4721221800000004, 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14706006999999999, 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455637+0000", 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455637+0000", 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455638+0000", 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4300000000000001e-07 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455660+0000", 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2208000000000002e-05 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455671+0000", 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0711e-05 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455679+0000", 2024-04-12T11:44:03.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0029999999999994e-06 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455735+0000", 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5893000000000002e-05 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.594891+0000", 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13915641000000001 2024-04-12T11:44:03.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.602655+0000", 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077640230000000001 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.602697+0000", 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1878999999999997e-05 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.954 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:86965 8.d e1195/1194 8:bf06d268:::benchmark_data_smithi029_194418_object86964:head v 1195'5473, mlcod=0'0)", 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:58.455851+0000", 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 5.4719080780000002, 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.146416983, 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455851+0000", 2024-04-12T11:44:03.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455851+0000", 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455852+0000", 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0699999999999997e-07 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.455931+0000", 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9486000000000004e-05 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.596020+0000", 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14008893 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.596030+0000", 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7200000000000001e-06 2024-04-12T11:44:03.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.596092+0000", 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2025000000000003e-05 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.596122+0000", 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9932000000000001e-05 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.602206+0000", 2024-04-12T11:44:03.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060848899999999999 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.602268+0000", 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1393e-05 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(8.6 1195/1194 [PushOp(8:6ea5542b:::benchmark_data_smithi029_194418_object653:head, version: 1176'36, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:6ea5542b:::benchmark_data_smithi029_194418_object653:head@1176'36, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:58.456512+0000", 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 5.4712466820000003, 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.154377916, 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.959 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.456512+0000", 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.456512+0000", 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.456513+0000", 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1500000000000004e-07 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.456565+0000", 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2154000000000002e-05 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.594946+0000", 2024-04-12T11:44:03.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13838091499999999 2024-04-12T11:44:03.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.594975+0000", 2024-04-12T11:44:03.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8527e-05 2024-04-12T11:44:03.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.608341+0000", 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013365909 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.608356+0000", 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5707999999999999e-05 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.610890+0000", 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002533988 2024-04-12T11:44:03.963 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86983 8.f 8:f8ff8992:::benchmark_data_smithi029_194418_object86982:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1195)", 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:58.471102+0000", 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 5.4566567389999996, 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13859902499999999, 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:44:03.964 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86983 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.471102+0000", 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.471102+0000", 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.471104+0000", 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0109999999999999e-06 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.471132+0000", 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7390999999999999e-05 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.594881+0000", 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12374932 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.594904+0000", 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2877000000000001e-05 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.595520+0000", 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00061621199999999999 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.595716+0000", 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019625999999999999 2024-04-12T11:44:03.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.595892+0000", 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000175453 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.602646+0000", 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067541370000000003 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.604021+0000", 2024-04-12T11:44:03.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013747970000000001 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.609506+0000", 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054849199999999999 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.609651+0000", 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000145656 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:58.609701+0000", 2024-04-12T11:44:03.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9991000000000003e-05 2024-04-12T11:44:03.970 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.970 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.970 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.970 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:87342 8.6 e1195/1194 8:6e37586e:::benchmark_data_smithi029_194418_object87341:head v 1195'5410, mlcod=1176'39)", 2024-04-12T11:44:03.970 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:59.002243+0000", 2024-04-12T11:44:03.971 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 4.9255160130000002, 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.040638220000000003, 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.002243+0000", 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.002243+0000", 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.002244+0000", 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0160000000000001e-06 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.002269+0000", 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4922999999999998e-05 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.002286+0000", 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6708e-05 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.002294+0000", 2024-04-12T11:44:03.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9480000000000008e-06 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.011735+0000", 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0094414719999999994 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.011763+0000", 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7742999999999998e-05 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.042832+0000", 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031069670000000001 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.042881+0000", 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8739999999999998e-05 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:87351 8.6 e1195/1194 8:641b0f5d:::benchmark_data_smithi029_194418_object87350:head v 1195'5411, mlcod=1176'39)", 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:59.005996+0000", 2024-04-12T11:44:03.976 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 4.9217627679999998, 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036941517, 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.005996+0000", 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.005996+0000", 2024-04-12T11:44:03.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.005997+0000", 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1600000000000002e-07 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.006015+0000", 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8405000000000001e-05 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.006025+0000", 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9629999999999993e-06 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.006030+0000", 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9450000000000001e-06 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.020179+0000", 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014149515999999999 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.020206+0000", 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7092000000000001e-05 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.042914+0000", 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022707676999999999 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.042938+0000", 2024-04-12T11:44:03.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3603000000000001e-05 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:87366 8.e e1195/1172 8:7db0decb:::benchmark_data_smithi029_194418_object87365:head v 1195'5282, mlcod=1195'5279)", 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:59.009910+0000", 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 4.9178490669999997, 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033001279000000001, 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.981 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.009910+0000", 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.009910+0000", 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.009911+0000", 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5700000000000002e-07 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.009934+0000", 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3504000000000001e-05 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.009950+0000", 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6045e-05 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.009957+0000", 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1770000000000001e-06 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.019708+0000", 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0097510470000000005 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.019728+0000", 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9947000000000001e-05 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.042885+0000", 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023156988 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:59.042911+0000", 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5814000000000001e-05 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:89793 8.6 e1195/1194 8:693d9d87:::benchmark_data_smithi029_194418_object89792:head v 1195'5556, mlcod=1176'95)", 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:01.735783+0000", 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 2.1919759910000001, 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.037732750000000002, 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.735783+0000", 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.735783+0000", 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.735784+0000", 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.02e-06 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.735811+0000", 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6778e-05 2024-04-12T11:44:03.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.735834+0000", 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3482999999999999e-05 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.735842+0000", 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2320000000000001e-06 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.738582+0000", 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027394300000000002 2024-04-12T11:44:03.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.738607+0000", 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4807000000000002e-05 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.773421+0000", 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.034814602 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.773516+0000", 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4398e-05 2024-04-12T11:44:03.989 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:89799 8.6 e1195/1194 8:6cd6d1cc:::benchmark_data_smithi029_194418_object89798:head v 1195'5557, mlcod=1176'95)", 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:01.738475+0000", 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 2.1892837699999999, 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.035301641000000002, 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.738475+0000", 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.738475+0000", 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.738476+0000", 2024-04-12T11:44:03.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9099999999999991e-07 2024-04-12T11:44:03.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.738498+0000", 2024-04-12T11:44:03.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1872e-05 2024-04-12T11:44:03.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.738516+0000", 2024-04-12T11:44:03.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7666e-05 2024-04-12T11:44:03.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.738523+0000", 2024-04-12T11:44:03.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5129999999999999e-06 2024-04-12T11:44:03.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.745668+0000", 2024-04-12T11:44:03.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071448040000000003 2024-04-12T11:44:03.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.745737+0000", 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8668000000000001e-05 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.773520+0000", 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027782793 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:01.773777+0000", 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00025733400000000001 2024-04-12T11:44:03.995 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:90184 8.b e1195/1185 8:dba511c2:::benchmark_data_smithi029_194418_object90183:head v 1195'5749, mlcod=1195'5746)", 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:02.165978+0000", 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.761781458, 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.041876742000000002, 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:03.996 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.165978+0000", 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.165978+0000", 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:03.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.165978+0000", 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6399999999999998e-07 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.204483+0000", 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.038504983 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.204489+0000", 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5740000000000001e-06 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.204494+0000", 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.938e-06 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.204557+0000", 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2428000000000006e-05 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.204574+0000", 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7713000000000001e-05 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:03.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.207828+0000", 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003253733 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.207854+0000", 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6109e-05 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.000 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:90367 8.6 e1195/1194 8:6e7e303f:::benchmark_data_smithi029_194418_object90366:head v 1195'5587, mlcod=1176'110)", 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:02.386912+0000", 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.5408467429999999, 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.040380681000000002, 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.386912+0000", 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.386912+0000", 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.386913+0000", 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.58e-07 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:04.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.386934+0000", 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1282999999999999e-05 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.386942+0000", 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1659999999999994e-06 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.386948+0000", 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7030000000000003e-06 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:04.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.389629+0000", 2024-04-12T11:44:04.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026815540000000001 2024-04-12T11:44:04.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:04.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.389657+0000", 2024-04-12T11:44:04.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7427e-05 2024-04-12T11:44:04.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:04.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.427259+0000", 2024-04-12T11:44:04.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.037602292000000002 2024-04-12T11:44:04.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:04.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.427293+0000", 2024-04-12T11:44:04.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3797999999999997e-05 2024-04-12T11:44:04.005 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.005 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:90368 8.6 e1195/1194 8:6975069d:::benchmark_data_smithi029_194418_object90367:head v 1195'5588, mlcod=1176'110)", 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:02.390395+0000", 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.537363544, 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036935338999999998, 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:04.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.390395+0000", 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.390395+0000", 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:04.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.390396+0000", 2024-04-12T11:44:04.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9999999999999997e-07 2024-04-12T11:44:04.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:04.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.390421+0000", 2024-04-12T11:44:04.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5077000000000001e-05 2024-04-12T11:44:04.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.390437+0000", 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5775999999999999e-05 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.390444+0000", 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2339999999999997e-06 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.399564+0000", 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091201289999999994 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.399594+0000", 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9830000000000001e-05 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.427297+0000", 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027702351 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.427331+0000", 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4242000000000002e-05 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.010 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:90373 8.6 e1195/1194 8:6d1c9448:::benchmark_data_smithi029_194418_object90372:head v 1195'5589, mlcod=1176'110)", 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:02.394955+0000", 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.532804279, 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.035196352, 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:04.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.394955+0000", 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.394955+0000", 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.394956+0000", 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.113e-06 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.394982+0000", 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6296999999999999e-05 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.395000+0000", 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7489000000000001e-05 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.395008+0000", 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4719999999999995e-06 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.409501+0000", 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014492813 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.409526+0000", 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5435999999999998e-05 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.430105+0000", 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020578612999999999 2024-04-12T11:44:04.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.430151+0000", 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6118999999999997e-05 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:90518 8.f 8:f130932f:::benchmark_data_smithi029_194418_object90517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1195)", 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:02.557519+0000", 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.3702404449999999, 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033436360999999998, 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:04.015 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 90518 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557519+0000", 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:04.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557519+0000", 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557520+0000", 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.006e-06 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557546+0000", 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6205000000000001e-05 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:04.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557556+0000", 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7999999999999993e-06 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557563+0000", 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0570000000000004e-06 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.566168+0000", 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0086049560000000004 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.566296+0000", 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012832800000000001 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.568613+0000", 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002316621 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.577162+0000", 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.008549081 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.578110+0000", 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00094844099999999995 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.590835+0000", 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012725447000000001 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.590904+0000", 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8077000000000006e-05 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.590955+0000", 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1341999999999997e-05 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:90519 8.f 8:f5e8520c:::benchmark_data_smithi029_194418_object90518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1195)", 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:02.557576+0000", 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.37018346, 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.038960540000000002, 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:04.021 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 90519 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557576+0000", 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557576+0000", 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557576+0000", 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3499999999999996e-07 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557596+0000", 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9848999999999999e-05 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557603+0000", 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0679999999999999e-06 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.557609+0000", 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6609999999999997e-06 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.572614+0000", 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015005413 2024-04-12T11:44:04.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.572990+0000", 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000375533 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.573150+0000", 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016091400000000001 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.591008+0000", 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017857669 2024-04-12T11:44:04.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.591051+0000", 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2722000000000002e-05 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.596183+0000", 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051322019999999998 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.596382+0000", 2024-04-12T11:44:04.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000198641 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.596536+0000", 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015433299999999999 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:90544 8.6 e1195/1194 8:6911bfed:::benchmark_data_smithi029_194418_object90543:head v 1195'5599, mlcod=1176'113)", 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:02.602185+0000", 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.325573927, 2024-04-12T11:44:04.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.034819876, 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.602185+0000", 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.602185+0000", 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.602186+0000", 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9699999999999995e-07 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.602210+0000", 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3994000000000001e-05 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.602221+0000", 2024-04-12T11:44:04.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0900999999999999e-05 2024-04-12T11:44:04.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:04.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.602227+0000", 2024-04-12T11:44:04.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.516e-06 2024-04-12T11:44:04.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:04.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.603166+0000", 2024-04-12T11:44:04.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000938583 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.603191+0000", 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4765000000000001e-05 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.636895+0000", 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033704616 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:04.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:02.637005+0000", 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010980399999999999 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:91592 8.1 e1196/1172 8:861807b6:::benchmark_data_smithi029_194418_object91591:head v 1196'5748, mlcod=1196'5744)", 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:03.859168+0000", 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.068590877999999994, 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033587074000000001, 2024-04-12T11:44:04.032 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:03.859168+0000", 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:03.859168+0000", 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:44:04.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:03.859169+0000", 2024-04-12T11:44:04.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2139999999999999e-06 2024-04-12T11:44:04.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:44:04.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:03.859195+0000", 2024-04-12T11:44:04.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6188999999999999e-05 2024-04-12T11:44:04.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:03.859214+0000", 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8122999999999999e-05 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:03.859223+0000", 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9819999999999997e-06 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:03.860311+0000", 2024-04-12T11:44:04.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010879520000000001 2024-04-12T11:44:04.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:44:04.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:03.860377+0000", 2024-04-12T11:44:04.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6079000000000002e-05 2024-04-12T11:44:04.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:44:04.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:03.892568+0000", 2024-04-12T11:44:04.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.032191407999999998 2024-04-12T11:44:04.037 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:44:04.037 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:44:04.037 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:44:04.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:03.892755+0000", 2024-04-12T11:44:04.037 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:04.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:04.038 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018712700000000001 2024-04-12T11:44:04.038 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.038 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:04.038 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.038 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:44:04.038 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:44:04.039 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:44:04.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:04.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 91718 91702 81.8622 50.5 0.00899091 0.0122083 2024-04-12T11:44:04.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:04.179 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:04.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:04.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:04.281 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:04.309 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:04.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:04.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:04.411 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:04.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:04.450 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:04.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:04 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:44:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:04 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/632828978' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:44:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:04 smithi029 ceph-mon[97787]: osdmap e1196: 8 total, 8 up, 6 in 2024-04-12T11:44:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:04 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:44:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:04 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/632828978' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:44:04.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:04 smithi029 ceph-mon[93650]: osdmap e1196: 8 total, 8 up, 6 in 2024-04-12T11:44:04.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:04.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:04.589 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:04.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:04.690 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:04.716 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:04 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:44:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:04 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/632828978' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:44:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:04 smithi087 ceph-mon[89014]: osdmap e1196: 8 total, 8 up, 6 in 2024-04-12T11:44:04.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:04.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:04.819 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:04.857 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:04.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:04.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:04.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:04.997 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:04.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:04.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:05.037 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 92417 92401 81.3245 43.6875 0.0480129 0.0122878 2024-04-12T11:44:05.098 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:05.122 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:05.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:05.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:05.224 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:05.245 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:44:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:44:04] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2024-04-12T11:44:05.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:05.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:05.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:05.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:05.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:05.385 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:05.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:05.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:05.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:05 smithi029 ceph-mon[97787]: pgmap v1752: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 709 op/s; 16113/269397 objects degraded (5.981%); 10384/269397 objects misplaced (3.855%); 1.6 MiB/s, 1 keys/s, 27 objects/s recovering 2024-04-12T11:44:05.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:05 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16113/269397 objects degraded (5.981%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:05.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:05 smithi029 ceph-mon[93650]: pgmap v1752: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 709 op/s; 16113/269397 objects degraded (5.981%); 10384/269397 objects misplaced (3.855%); 1.6 MiB/s, 1 keys/s, 27 objects/s recovering 2024-04-12T11:44:05.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:05 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16113/269397 objects degraded (5.981%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:05.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:05.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:05.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:05.622 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:05.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:05.657 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:05.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:05 smithi087 ceph-mon[89014]: pgmap v1752: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 709 op/s; 16113/269397 objects degraded (5.981%); 10384/269397 objects misplaced (3.855%); 1.6 MiB/s, 1 keys/s, 27 objects/s recovering 2024-04-12T11:44:05.771 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:05 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16113/269397 objects degraded (5.981%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:05.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:05.810 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:05.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:05.911 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:05.947 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:05.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:05.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:06.037 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 93128 93112 80.8121 44.4375 0.0187168 0.0123695 2024-04-12T11:44:06.048 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:06.082 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:06.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:06.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:06.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:06.221 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:06.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:06.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:06.322 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:06.361 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:06.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:06.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:06.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:06.498 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:06.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:06.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:06.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:06.635 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:06.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:06.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:06.737 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:06.766 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:06.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:06.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:06.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:06.892 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:06.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:06.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:06.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:07.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:07.024 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:07.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:07.037 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 93709 93693 80.2026 36.3125 0.00886326 0.0124475 2024-04-12T11:44:07.125 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:07.161 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:07.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:07.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:07.263 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:07.304 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:07.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:07.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:07.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:07.429 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:07.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:07.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:07.531 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:07.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:07.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:07.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:07.668 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:07.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:07 smithi029 ceph-mon[97787]: pgmap v1753: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 803 op/s; 16113/273987 objects degraded (5.881%); 10287/273987 objects misplaced (3.755%); 2.2 MiB/s, 1 keys/s, 35 objects/s recovering 2024-04-12T11:44:07.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:07.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:07.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:07 smithi029 ceph-mon[93650]: pgmap v1753: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 803 op/s; 16113/273987 objects degraded (5.881%); 10287/273987 objects misplaced (3.755%); 2.2 MiB/s, 1 keys/s, 35 objects/s recovering 2024-04-12T11:44:07.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:07.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:07.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:07.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:07.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:07.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:07 smithi087 ceph-mon[89014]: pgmap v1753: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 803 op/s; 16113/273987 objects degraded (5.881%); 10287/273987 objects misplaced (3.755%); 2.2 MiB/s, 1 keys/s, 35 objects/s recovering 2024-04-12T11:44:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:07.795 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:07.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:07.830 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:07.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:07.931 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:07.963 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:07.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:07.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:08.037 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 94407 94391 79.7083 43.625 0.0311678 0.0125386 2024-04-12T11:44:08.065 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:08.105 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:08.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:08.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:08.207 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:08.246 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:08.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:08.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:08.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:08.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:08.385 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:08.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:08.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:08.531 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:08.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:08.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:08.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:08.670 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:08.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:08.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:08.772 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:08.807 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:08.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:08.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:08.910 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:08.943 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:08.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:08.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:09.038 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 95252 95236 79.3496 52.8125 0.017818 0.0125966 2024-04-12T11:44:09.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:09.089 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:09.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:09.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:09.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:09.242 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:09.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:09.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:09.343 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:09.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:09.375 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:09.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:09.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:09.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:09.512 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:09.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:09.613 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:09.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:09.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:09.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:09.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:09 smithi029 ceph-mon[97787]: pgmap v1754: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 710 op/s; 16113/277971 objects degraded (5.797%); 10091/277971 objects misplaced (3.630%); 3.1 MiB/s, 0 keys/s, 50 objects/s recovering 2024-04-12T11:44:09.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:09.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:09.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:09 smithi029 ceph-mon[93650]: pgmap v1754: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 710 op/s; 16113/277971 objects degraded (5.797%); 10091/277971 objects misplaced (3.630%); 3.1 MiB/s, 0 keys/s, 50 objects/s recovering 2024-04-12T11:44:09.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:09.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:09 smithi087 ceph-mon[89014]: pgmap v1754: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 710 op/s; 16113/277971 objects degraded (5.797%); 10091/277971 objects misplaced (3.630%); 3.1 MiB/s, 0 keys/s, 50 objects/s recovering 2024-04-12T11:44:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:09.748 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:09.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:09.778 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:09.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:09.879 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:09.913 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:09.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:10.014 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:10.037 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 96032 96016 78.947 48.75 0.00566084 0.0126589 2024-04-12T11:44:10.047 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:10.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:10.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:10.149 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:10.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:10.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:10.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:10.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:10.325 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:10.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:10.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:10.426 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:10.468 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:10.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:10.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:10.570 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:10.603 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:10.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:10.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:10.705 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:10 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16113/282690 objects degraded (5.700%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:10.743 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:10.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:10 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16113/282690 objects degraded (5.700%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:10 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16113/282690 objects degraded (5.700%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:10.844 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:10.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:10.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:10.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:11.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:11.020 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:11.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:11.038 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 96913 96897 78.6367 55.0625 0.0213085 0.012711 2024-04-12T11:44:11.122 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:11.160 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:11.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:11.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:11.300 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:11.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:11.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:11.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:11.440 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:11.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:11.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:11.542 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:11.576 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:11.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:11.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:11.678 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:11.714 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:11.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:11.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:11 smithi087 ceph-mon[89014]: pgmap v1755: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s; 16113/282690 objects degraded (5.700%); 9995/282690 objects misplaced (3.536%); 3.4 MiB/s, 0 keys/s, 55 objects/s recovering 2024-04-12T11:44:11.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:11 smithi029 ceph-mon[97787]: pgmap v1755: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s; 16113/282690 objects degraded (5.700%); 9995/282690 objects misplaced (3.536%); 3.4 MiB/s, 0 keys/s, 55 objects/s recovering 2024-04-12T11:44:11.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:11 smithi029 ceph-mon[93650]: pgmap v1755: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 47 MiB/s wr, 755 op/s; 16113/282690 objects degraded (5.700%); 9995/282690 objects misplaced (3.536%); 3.4 MiB/s, 0 keys/s, 55 objects/s recovering 2024-04-12T11:44:11.816 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:11.858 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:11.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:11.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-04-12T11:44:11.960 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:11.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:11.998 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:11.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:12.038 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 97728 97712 78.2812 50.9375 0.0332636 0.0127693 2024-04-12T11:44:12.099 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:12.138 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:12.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:12.240 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:12.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:12.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:12.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:12.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:12.409 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:12.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:12.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:12.459 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:12.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:12.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:12.584 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:12.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:12.585 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:44:12.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:12 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:44:12.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:12 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:44:12.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:12 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:12.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:12 smithi029 ceph-mon[93650]: pgmap v1756: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 49 MiB/s wr, 781 op/s; 16113/284745 objects degraded (5.659%); 9995/284745 objects misplaced (3.510%); 3.4 MiB/s, 54 objects/s recovering 2024-04-12T11:44:12.615 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:12 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:44:12.615 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:12 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:44:12.615 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:12 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:12.615 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:12 smithi029 ceph-mon[97787]: pgmap v1756: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 49 MiB/s wr, 781 op/s; 16113/284745 objects degraded (5.659%); 9995/284745 objects misplaced (3.510%); 3.4 MiB/s, 54 objects/s recovering 2024-04-12T11:44:12.622 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:44:12.663 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:44:12.685 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:12.699 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:44:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:44:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:44:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:44:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:12 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:44:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:12 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:44:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:12 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:12 smithi087 ceph-mon[89014]: pgmap v1756: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 49 MiB/s wr, 781 op/s; 16113/284745 objects degraded (5.659%); 9995/284745 objects misplaced (3.510%); 3.4 MiB/s, 54 objects/s recovering 2024-04-12T11:44:12.722 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:12.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:12.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:12.733 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:44:12.753 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:44:12.790 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:44:12.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:12.829 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:44:12.836 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:44:12.837 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:44:12.837 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:44:12.837 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:44:12.837 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:44:12.837 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:44:12.837 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:44:12.837 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:44:12.871 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:12.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:12.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:12.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:13.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:13.020 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:13.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:13.038 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 98450 98434 77.8615 45.125 0.00644399 0.0128374 2024-04-12T11:44:13.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:13.145 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:13.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:13.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:13.249 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:13.286 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:13.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:13.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:13.389 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:13.431 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:13.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:13.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:13.533 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:13.556 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:13.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:13.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:13.658 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:13.684 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:13.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:13.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:13 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2940030920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:44:13.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:13 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2940030920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:44:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:13 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2940030920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:44:13.787 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:13.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:44:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:13.830 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:13.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:13.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:13.986 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:13.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:14.038 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:44:14.038326+0000 min lat: 0.00105127 max lat: 2.39218 avg lat: 0.0129211 2024-04-12T11:44:14.038 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:44:14.038 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 99050 99034 77.357 37.5 0.0184432 0.0129211 2024-04-12T11:44:14.088 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:14.111 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:14.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:14.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:14.213 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:14.236 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:14.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:14.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:14.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:14.382 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:14.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:14.483 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:14.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:14 smithi029 ceph-mon[97787]: pgmap v1757: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 51 MiB/s wr, 814 op/s; 16113/292059 objects degraded (5.517%); 9831/292059 objects misplaced (3.366%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:44:14.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:14 smithi029 ceph-mon[93650]: pgmap v1757: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 51 MiB/s wr, 814 op/s; 16113/292059 objects degraded (5.517%); 9831/292059 objects misplaced (3.366%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:44:14.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:14.521 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:14.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:14.622 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:14.659 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:14.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:14.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:14 smithi087 ceph-mon[89014]: pgmap v1757: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 51 MiB/s wr, 814 op/s; 16113/292059 objects degraded (5.517%); 9831/292059 objects misplaced (3.366%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:44:14.761 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:14.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:14.790 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:14.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:14.891 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:14.926 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:14.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:14.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:15.028 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:15.039 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 99722 99706 76.9198 42 0.0150384 0.0129946 2024-04-12T11:44:15.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:15.063 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:15.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:15.164 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:15.188 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:15.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:15.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:44:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:44:14] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2024-04-12T11:44:15.290 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:15.322 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:15.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:15.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:15.424 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:15.468 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:15.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:15.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:15.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:15.607 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:15.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:15.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:15.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:15.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:15.740 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:15.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:15.841 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:15.879 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:15.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:15.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:16.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:16.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:16.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:16.039 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 100391 100375 76.4916 41.8125 0.00814436 0.0130665 2024-04-12T11:44:16.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:16.155 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:16.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:16.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:16.257 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:16.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:15 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16113/296307 objects degraded (5.438%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:16.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:15 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16113/296307 objects degraded (5.438%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:16.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:16.293 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:16.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:16.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:16.418 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:16.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:16.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:16.466 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:15 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16113/296307 objects degraded (5.438%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:16.519 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:16.554 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:16.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:16.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:16.656 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:16.690 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:16.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:16.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:16.837 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:16.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:16.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:16.939 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:16.975 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:16.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:16.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:17.039 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 101073 101057 76.0836 42.625 0.0405452 0.0131373 2024-04-12T11:44:17.077 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:17.112 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:17.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:17.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:17.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:17.247 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:17.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:17.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:17.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:16 smithi029 ceph-mon[97787]: pgmap v1758: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 16113/296307 objects degraded (5.438%); 9753/296307 objects misplaced (3.292%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T11:44:17.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:16 smithi029 ceph-mon[93650]: pgmap v1758: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 16113/296307 objects degraded (5.438%); 9753/296307 objects misplaced (3.292%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T11:44:17.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:17.383 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:17.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:16 smithi087 ceph-mon[89014]: pgmap v1758: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 16113/296307 objects degraded (5.438%); 9753/296307 objects misplaced (3.292%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T11:44:17.484 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:17.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:17.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:17.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:17.622 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:17.661 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:17.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:17.762 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:17.814 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:17.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:17.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:17.948 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:17.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:17.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:18.039 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 101894 101878 75.7887 51.3125 0.00922141 0.0131892 2024-04-12T11:44:18.050 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:18.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:18.090 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:18.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:18.191 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:18.215 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:18.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:18.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:18.317 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:18.352 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:18.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:18.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:18.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:18.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:18.490 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:18.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:18.591 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:18.625 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:18.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:18.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:18.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:18.751 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:18.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:18.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:18.852 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:18.887 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:18.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:18.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:18.989 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:19.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:19.025 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:19.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:19.039 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 102582 102566 75.4029 43 0.00489407 0.0132553 2024-04-12T11:44:19.126 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:19.160 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:19.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:19.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:19.250 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:18 smithi029 ceph-mon[97787]: pgmap v1759: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 46 MiB/s wr, 729 op/s; 16113/300249 objects degraded (5.367%); 9537/300249 objects misplaced (3.176%); 3.9 MiB/s, 62 objects/s recovering 2024-04-12T11:44:19.251 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:18 smithi029 ceph-mon[93650]: pgmap v1759: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 46 MiB/s wr, 729 op/s; 16113/300249 objects degraded (5.367%); 9537/300249 objects misplaced (3.176%); 3.9 MiB/s, 62 objects/s recovering 2024-04-12T11:44:19.262 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:19.295 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:19.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:19.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:19.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:19.435 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:19.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:19.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:18 smithi087 ceph-mon[89014]: pgmap v1759: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 46 MiB/s wr, 729 op/s; 16113/300249 objects degraded (5.367%); 9537/300249 objects misplaced (3.176%); 3.9 MiB/s, 62 objects/s recovering 2024-04-12T11:44:19.537 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:19.579 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:19.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:19.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:19.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:19.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:19.715 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:19.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:19.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:19.839 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:19.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:19.941 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:19.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:19.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:20.039 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 103254 103238 75.0145 42 0.0219151 0.0133248 2024-04-12T11:44:20.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:20.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:20.112 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:20.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:20.213 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:20.246 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:20.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:20.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:20.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:20.383 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:20.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:20.485 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:20.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:20.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:20.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:20.621 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:20.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:20.659 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:20.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:20.760 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:20.795 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:20.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:20.897 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:20.930 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:20.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:20.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:21.032 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:21.039 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 104086 104070 74.7499 52 0.0198677 0.0133717 2024-04-12T11:44:21.066 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:21.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:21.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:21.168 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:21.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:21.203 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:21.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:21.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:20 smithi029 ceph-mon[93650]: pgmap v1760: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 47 MiB/s wr, 746 op/s; 16113/304860 objects degraded (5.285%); 9434/304860 objects misplaced (3.095%); 3.4 MiB/s, 54 objects/s recovering 2024-04-12T11:44:21.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:20 smithi029 ceph-mon[97787]: pgmap v1760: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 47 MiB/s wr, 746 op/s; 16113/304860 objects degraded (5.285%); 9434/304860 objects misplaced (3.095%); 3.4 MiB/s, 54 objects/s recovering 2024-04-12T11:44:21.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:21.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:21.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:21.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:21.434 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:20 smithi087 ceph-mon[89014]: pgmap v1760: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 47 MiB/s wr, 746 op/s; 16113/304860 objects degraded (5.285%); 9434/304860 objects misplaced (3.095%); 3.4 MiB/s, 54 objects/s recovering 2024-04-12T11:44:21.468 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:21.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:21.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:21.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:21.595 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:21.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:21.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:21.697 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:21.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:21.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:21.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:21.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:21.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:21.852 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:21.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:21.954 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:21.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:21.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:21.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:22.039 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 104646 104630 74.2982 35 0.0853955 0.0134533 2024-04-12T11:44:22.092 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:22.131 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:22.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:22.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:22.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:22.268 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:22.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:22.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:22.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:22.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:22.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:22.370 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:22.408 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:22.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:22.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:22.510 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:22.548 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:22.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:22.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:22.688 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:22.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:22.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:44:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:44:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:44:22.779 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:22.791 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:22.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:22.825 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:22.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:22.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:22.963 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:22.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:22.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:23.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 105488 105472 74.0547 52.625 0.0218828 0.0134978 2024-04-12T11:44:23.065 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:23.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:23.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:23.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:23.239 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:23.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:23.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:22 smithi029 ceph-mon[97787]: pgmap v1761: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 42 MiB/s wr, 672 op/s; 16113/306912 objects degraded (5.250%); 9434/306912 objects misplaced (3.074%); 2.9 MiB/s, 46 objects/s recovering 2024-04-12T11:44:23.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:22 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16113/306912 objects degraded (5.250%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:23.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:22 smithi029 ceph-mon[93650]: pgmap v1761: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 42 MiB/s wr, 672 op/s; 16113/306912 objects degraded (5.250%); 9434/306912 objects misplaced (3.074%); 2.9 MiB/s, 46 objects/s recovering 2024-04-12T11:44:23.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:22 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16113/306912 objects degraded (5.250%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:23.341 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:23.376 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:23.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:23.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:23.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:22 smithi087 ceph-mon[89014]: pgmap v1761: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 42 MiB/s wr, 672 op/s; 16113/306912 objects degraded (5.250%); 9434/306912 objects misplaced (3.074%); 2.9 MiB/s, 46 objects/s recovering 2024-04-12T11:44:23.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:22 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16113/306912 objects degraded (5.250%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:23.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:23.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:23.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:23.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:23.627 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:23.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:23.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:23.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:23.784 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:23.819 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:23.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:23.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:23.920 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:23.959 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:23.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:23.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:24.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 90 2 106307 106305 73.8103 52.0625 0.0166097 0.0135428 2024-04-12T11:44:24.060 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:24.084 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:24.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:24.185 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:24.209 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:24.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:24.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:24.310 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:24.314 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:44:24.334 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:24.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:24.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:24.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:24.461 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:24.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:24.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:24.471 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:24.563 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:24.587 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:24.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:24.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:24.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:24.714 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:24.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:24.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:24.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:24.840 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:24.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:24.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:24.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:24.967 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:24.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:24.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:25.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0233 2024-04-12T11:44:25.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 106307 2024-04-12T11:44:25.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2024-04-12T11:44:25.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2024-04-12T11:44:25.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 73.8052 2024-04-12T11:44:25.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 38.3049 2024-04-12T11:44:25.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 198.625 2024-04-12T11:44:25.040 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2024-04-12T11:44:25.041 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1180 2024-04-12T11:44:25.041 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 612.881 2024-04-12T11:44:25.041 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3178 2024-04-12T11:44:25.041 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2024-04-12T11:44:25.041 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.013543 2024-04-12T11:44:25.041 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0483917 2024-04-12T11:44:25.041 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 2.39218 2024-04-12T11:44:25.041 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00105127 2024-04-12T11:44:25.056 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2024-04-12T11:44:25.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:25.105 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:25.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:25.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:25.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:24 smithi029 ceph-mon[97787]: pgmap v1762: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 51 MiB/s wr, 809 op/s; 16113/313884 objects degraded (5.133%); 9240/313884 objects misplaced (2.944%); 3.9 MiB/s, 62 objects/s recovering 2024-04-12T11:44:25.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:24 smithi029 ceph-mon[93650]: pgmap v1762: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 51 MiB/s wr, 809 op/s; 16113/313884 objects degraded (5.133%); 9240/313884 objects misplaced (2.944%); 3.9 MiB/s, 62 objects/s recovering 2024-04-12T11:44:25.198 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:44:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:44:24] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2024-04-12T11:44:25.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:25.255 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:25.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:25.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:25.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:25.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:25.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:25.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:25.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:24 smithi087 ceph-mon[89014]: pgmap v1762: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 51 MiB/s wr, 809 op/s; 16113/313884 objects degraded (5.133%); 9240/313884 objects misplaced (2.944%); 3.9 MiB/s, 62 objects/s recovering 2024-04-12T11:44:25.491 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:25.526 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:25.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:25.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:25.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:25.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:25.659 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:25.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:25.760 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:25.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:25.802 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:25.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:25.903 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:25.940 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:25.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:25.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:26.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:26.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:26.083 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:26.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:26.103 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:44:26.103 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:44:26.184 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:26.220 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:26.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:26.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:26.322 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:26.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:26.359 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:26.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:26.460 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:26.492 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:26.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:26.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:26.594 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:26.623 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:26.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:26.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:26.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:26.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:26.766 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:26.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:26.867 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:26.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:26.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:26.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:27.005 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:27.037 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:27.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:27.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:27.139 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:27.163 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:27.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:27.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:27.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:27 smithi029 ceph-mon[97787]: pgmap v1763: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 44 MiB/s wr, 705 op/s; 16113/317403 objects degraded (5.077%); 9108/317403 objects misplaced (2.870%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:27.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:27 smithi029 ceph-mon[93650]: pgmap v1763: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 44 MiB/s wr, 705 op/s; 16113/317403 objects degraded (5.077%); 9108/317403 objects misplaced (2.870%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:27.292 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:27.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:27.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:27.431 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:27.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:27.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:27.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:27 smithi087 ceph-mon[89014]: pgmap v1763: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 44 MiB/s wr, 705 op/s; 16113/317403 objects degraded (5.077%); 9108/317403 objects misplaced (2.870%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:27.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:27.559 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:27.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:27.660 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:27.694 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:27.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:27.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:27.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:27.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:27.833 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:27.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:27.934 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:27.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:27.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:27.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:28.072 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:28 smithi029 ceph-mon[93650]: from='client.35274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:44:28.073 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16113/317403 objects degraded (5.077%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:28.073 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:28 smithi029 ceph-mon[97787]: from='client.35274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:44:28.073 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16113/317403 objects degraded (5.077%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:28.074 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:28.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:28.109 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:28.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:28.210 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:28.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:28.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:28.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:28.345 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:28.383 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:28.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:28.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:28 smithi087 ceph-mon[89014]: from='client.35274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:44:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16113/317403 objects degraded (5.077%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:28.485 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:28.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:28.522 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:28.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:28.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:28.658 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:28.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:28.760 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:28.793 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:28.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:28.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:28.895 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:28.926 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:28.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:29.027 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:29.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:29.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:29.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:29.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:29.189 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:29.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:29.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:29.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:29 smithi029 ceph-mon[93650]: pgmap v1764: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 85 B/s rd, 39 MiB/s wr, 686 op/s; 16113/316689 objects degraded (5.088%); 8858/316689 objects misplaced (2.797%); 4.7 MiB/s, 74 objects/s recovering 2024-04-12T11:44:29.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:29 smithi029 ceph-mon[97787]: pgmap v1764: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 85 B/s rd, 39 MiB/s wr, 686 op/s; 16113/316689 objects degraded (5.088%); 8858/316689 objects misplaced (2.797%); 4.7 MiB/s, 74 objects/s recovering 2024-04-12T11:44:29.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:29.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:29.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:29.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:29.430 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:29.466 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:29 smithi087 ceph-mon[89014]: pgmap v1764: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 85 B/s rd, 39 MiB/s wr, 686 op/s; 16113/316689 objects degraded (5.088%); 8858/316689 objects misplaced (2.797%); 4.7 MiB/s, 74 objects/s recovering 2024-04-12T11:44:29.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:29.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:29.568 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:29.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:29.607 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:29.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:29.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:29.740 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:29.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:29.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:29.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:29.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:29.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:29.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:29.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:30.012 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:30.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:30.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:30.114 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:30.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:30.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:30.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:30.289 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:30.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:30.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:30.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:30.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:30.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:30.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:30.484 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:44:30.496 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:44:30.538 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:30.574 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:30.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:30.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:30.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:30.710 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:30.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:30.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:30.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:30.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:30.850 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:30.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:30.951 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:30.996 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:30.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:31.097 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:31.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:31.132 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:31.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:31.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:31.268 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:31.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:31.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:31.370 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:31.396 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:31.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:31.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:31 smithi087 ceph-mon[89014]: pgmap v1765: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 85 B/s rd, 33 MiB/s wr, 687 op/s; 16113/312975 objects degraded (5.148%); 8740/312975 objects misplaced (2.793%); 4.1 MiB/s, 66 objects/s recovering 2024-04-12T11:44:31.498 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:31.504 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:31 smithi029 ceph-mon[97787]: pgmap v1765: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 85 B/s rd, 33 MiB/s wr, 687 op/s; 16113/312975 objects degraded (5.148%); 8740/312975 objects misplaced (2.793%); 4.1 MiB/s, 66 objects/s recovering 2024-04-12T11:44:31.504 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:31 smithi029 ceph-mon[93650]: pgmap v1765: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 85 B/s rd, 33 MiB/s wr, 687 op/s; 16113/312975 objects degraded (5.148%); 8740/312975 objects misplaced (2.793%); 4.1 MiB/s, 66 objects/s recovering 2024-04-12T11:44:31.534 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:31.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:31.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:31.636 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:31.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:31.663 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:31.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:31.764 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:31.790 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:31.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:31.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:31.892 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:31.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:31.939 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:31.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:32.040 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:32.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:32.082 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:32.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:32.183 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:32.223 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:32.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:32.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:32.325 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:32.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:32.362 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:32.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:32.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-04-12T11:44:32.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:32.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:32.490 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:32.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:32.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:32.641 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:32.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:32.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:44:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:44:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:44:32.742 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:32.781 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:32.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:32.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:32.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:32.922 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:32.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:32.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:33.024 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:33.059 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:33.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:33.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:33.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:33.197 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:33.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:33.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:33 smithi029 ceph-mon[93650]: pgmap v1766: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 25 MiB/s wr, 615 op/s; 16113/310968 objects degraded (5.182%); 8740/310968 objects misplaced (2.811%); 3.6 MiB/s, 57 objects/s recovering 2024-04-12T11:44:33.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16113/310968 objects degraded (5.182%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:33.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:33 smithi029 ceph-mon[97787]: pgmap v1766: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 25 MiB/s wr, 615 op/s; 16113/310968 objects degraded (5.182%); 8740/310968 objects misplaced (2.811%); 3.6 MiB/s, 57 objects/s recovering 2024-04-12T11:44:33.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16113/310968 objects degraded (5.182%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:33.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:33.341 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:33.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:33.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:33.457 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1763,"stamp":"2024-04-12T11:44:25.881708+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6932797394,"num_objects":105801,"num_object_clones":0,"num_object_copies":317403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16113,"num_objects_misplaced":9108,"num_objects_unfound":0,"num_objects_dirty":105801,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":106093,"num_write_kb":6772686,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5949,"num_bytes_recovered":390013086,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101909,"ondisk_log_size":101909,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":21133296,"kb_used_data":18842688,"kb_used_omap":156,"kb_used_meta":2290403,"kb_avail":541337616,"statfs":{"total":575970213888,"available":554329718784,"internally_reserved":0,"allocated":19294912512,"data_stored":19275370548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":160537,"internal_metadata":2345372903},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":553648128,"num_objects":8448,"num_object_clones":0,"num_object_copies":25344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-723,"num_objects_unfound":0,"num_objects_dirty":8448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8464,"num_write_kb":541184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":723,"num_bytes_recovered":47382528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003672"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1746,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.237212+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:44:03.237212+0000","last_peered":"2024-04-12T11:44:03.237212+0000","last_clean":"2024-04-12T11:44:03.237212+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:44:03.237212+0000","last_undegraded":"2024-04-12T11:44:03.237212+0000","last_fullsized":"2024-04-12T11:44:03.237212+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":619,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.244127+0000","last_change":"2024-04-12T11:42:59.337220+0000","last_active":"2024-04-12T11:44:03.244127+0000","last_peered":"2024-04-12T11:44:03.244127+0000","last_clean":"2024-04-12T11:44:03.244127+0000","last_became_active":"2024-04-12T11:42:59.336833+0000","last_became_peered":"2024-04-12T11:42:59.336833+0000","last_unstale":"2024-04-12T11:44:03.244127+0000","last_undegraded":"2024-04-12T11:44:03.244127+0000","last_fullsized":"2024-04-12T11:44:03.244127+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:19:20.874750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1812,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.244134+0000","last_change":"2024-04-12T11:42:59.337181+0000","last_active":"2024-04-12T11:44:03.244134+0000","last_peered":"2024-04-12T11:44:03.244134+0000","last_clean":"2024-04-12T11:44:03.244134+0000","last_became_active":"2024-04-12T11:42:59.336830+0000","last_became_peered":"2024-04-12T11:42:59.336830+0000","last_unstale":"2024-04-12T11:44:03.244134+0000","last_undegraded":"2024-04-12T11:44:03.244134+0000","last_fullsized":"2024-04-12T11:44:03.244134+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:27:47.567234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1826,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.259106+0000","last_change":"2024-04-12T11:43:56.150606+0000","last_active":"2024-04-12T11:44:03.259106+0000","last_peered":"2024-04-12T11:44:03.259106+0000","last_clean":"2024-04-12T11:44:03.259106+0000","last_became_active":"2024-04-12T11:43:56.150052+0000","last_became_peered":"2024-04-12T11:43:56.150052+0000","last_unstale":"2024-04-12T11:44:03.259106+0000","last_undegraded":"2024-04-12T11:44:03.259106+0000","last_fullsized":"2024-04-12T11:44:03.259106+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:05:08.049346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":586,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.250750+0000","last_change":"2024-04-12T11:42:59.337290+0000","last_active":"2024-04-12T11:44:03.250750+0000","last_peered":"2024-04-12T11:44:03.250750+0000","last_clean":"2024-04-12T11:44:03.250750+0000","last_became_active":"2024-04-12T11:42:59.336952+0000","last_became_peered":"2024-04-12T11:42:59.336952+0000","last_unstale":"2024-04-12T11:44:03.250750+0000","last_undegraded":"2024-04-12T11:44:03.250750+0000","last_fullsized":"2024-04-12T11:44:03.250750+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:09:22.202715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1143,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.244586+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:44:03.244586+0000","last_peered":"2024-04-12T11:44:03.244586+0000","last_clean":"2024-04-12T11:44:03.244586+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:44:03.244586+0000","last_undegraded":"2024-04-12T11:44:03.244586+0000","last_fullsized":"2024-04-12T11:44:03.244586+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1803,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.253564+0000","last_change":"2024-04-12T11:42:59.339046+0000","last_active":"2024-04-12T11:44:03.253564+0000","last_peered":"2024-04-12T11:44:03.253564+0000","last_clean":"2024-04-12T11:44:03.253564+0000","last_became_active":"2024-04-12T11:42:59.338611+0000","last_became_peered":"2024-04-12T11:42:59.338611+0000","last_unstale":"2024-04-12T11:44:03.253564+0000","last_undegraded":"2024-04-12T11:44:03.253564+0000","last_fullsized":"2024-04-12T11:44:03.253564+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:49:40.315371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":570,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.245650+0000","last_change":"2024-04-12T11:42:59.333787+0000","last_active":"2024-04-12T11:44:03.245650+0000","last_peered":"2024-04-12T11:44:03.245650+0000","last_clean":"2024-04-12T11:44:03.245650+0000","last_became_active":"2024-04-12T11:42:59.333435+0000","last_became_peered":"2024-04-12T11:42:59.333435+0000","last_unstale":"2024-04-12T11:44:03.245650+0000","last_undegraded":"2024-04-12T11:44:03.245650+0000","last_fullsized":"2024-04-12T11:44:03.245650+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:08:22.197071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1157,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.236009+0000","last_change":"2024-04-12T11:43:56.151486+0000","last_active":"2024-04-12T11:44:03.236009+0000","last_peered":"2024-04-12T11:44:03.236009+0000","last_clean":"2024-04-12T11:44:03.236009+0000","last_became_active":"2024-04-12T11:43:56.150899+0000","last_became_peered":"2024-04-12T11:43:56.150899+0000","last_unstale":"2024-04-12T11:44:03.236009+0000","last_undegraded":"2024-04-12T11:44:03.236009+0000","last_fullsized":"2024-04-12T11:44:03.236009+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:12:54.212867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":507,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.518252+0000","last_change":"2024-04-12T11:43:56.151421+0000","last_active":"2024-04-12T11:43:57.518252+0000","last_peered":"2024-04-12T11:43:57.518252+0000","last_clean":"2024-04-12T11:43:57.518252+0000","last_became_active":"2024-04-12T11:43:56.150376+0000","last_became_peered":"2024-04-12T11:43:56.150376+0000","last_unstale":"2024-04-12T11:43:57.518252+0000","last_undegraded":"2024-04-12T11:43:57.518252+0000","last_fullsized":"2024-04-12T11:43:57.518252+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:39:43.515297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":34,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.237586+0000","last_change":"2024-04-12T11:43:56.182709+0000","last_active":"2024-04-12T11:44:03.237586+0000","last_peered":"2024-04-12T11:44:03.237586+0000","last_clean":"2024-04-12T11:44:03.237586+0000","last_became_active":"2024-04-12T11:43:56.148507+0000","last_became_peered":"2024-04-12T11:43:56.148507+0000","last_unstale":"2024-04-12T11:44:03.237586+0000","last_undegraded":"2024-04-12T11:44:03.237586+0000","last_fullsized":"2024-04-12T11:44:03.237586+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:34:45.396138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1797,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.247687+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:44:03.247687+0000","last_peered":"2024-04-12T11:44:03.247687+0000","last_clean":"2024-04-12T11:44:03.247687+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:44:03.247687+0000","last_undegraded":"2024-04-12T11:44:03.247687+0000","last_fullsized":"2024-04-12T11:44:03.247687+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1734,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.238515+0000","last_change":"2024-04-12T11:43:56.174527+0000","last_active":"2024-04-12T11:44:03.238515+0000","last_peered":"2024-04-12T11:44:03.238515+0000","last_clean":"2024-04-12T11:44:03.238515+0000","last_became_active":"2024-04-12T11:43:56.153682+0000","last_became_peered":"2024-04-12T11:43:56.153682+0000","last_unstale":"2024-04-12T11:44:03.238515+0000","last_undegraded":"2024-04-12T11:44:03.238515+0000","last_fullsized":"2024-04-12T11:44:03.238515+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:14.066902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1709,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.237276+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:44:03.237276+0000","last_peered":"2024-04-12T11:44:03.237276+0000","last_clean":"2024-04-12T11:44:03.237276+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:44:03.237276+0000","last_undegraded":"2024-04-12T11:44:03.237276+0000","last_fullsized":"2024-04-12T11:44:03.237276+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":34,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.237261+0000","last_change":"2024-04-12T11:43:56.186225+0000","last_active":"2024-04-12T11:44:03.237261+0000","last_peered":"2024-04-12T11:44:03.237261+0000","last_clean":"2024-04-12T11:44:03.237261+0000","last_became_active":"2024-04-12T11:43:56.155707+0000","last_became_peered":"2024-04-12T11:43:56.155707+0000","last_unstale":"2024-04-12T11:44:03.237261+0000","last_undegraded":"2024-04-12T11:44:03.237261+0000","last_fullsized":"2024-04-12T11:44:03.237261+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:08:41.431176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":576,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.243669+0000","last_change":"2024-04-12T11:43:56.153564+0000","last_active":"2024-04-12T11:44:03.243669+0000","last_peered":"2024-04-12T11:44:03.243669+0000","last_clean":"2024-04-12T11:44:03.243669+0000","last_became_active":"2024-04-12T11:43:56.153146+0000","last_became_peered":"2024-04-12T11:43:56.153146+0000","last_unstale":"2024-04-12T11:44:03.243669+0000","last_undegraded":"2024-04-12T11:44:03.243669+0000","last_fullsized":"2024-04-12T11:44:03.243669+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:40:52.881313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1810,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.252182+0000","last_change":"2024-04-12T11:43:56.148865+0000","last_active":"2024-04-12T11:44:03.252182+0000","last_peered":"2024-04-12T11:44:03.252182+0000","last_clean":"2024-04-12T11:44:03.252182+0000","last_became_active":"2024-04-12T11:43:56.148534+0000","last_became_peered":"2024-04-12T11:43:56.148534+0000","last_unstale":"2024-04-12T11:44:03.252182+0000","last_undegraded":"2024-04-12T11:44:03.252182+0000","last_fullsized":"2024-04-12T11:44:03.252182+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:12:50.185550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1165,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.242856+0000","last_change":"2024-04-12T11:43:56.155575+0000","last_active":"2024-04-12T11:44:03.242856+0000","last_peered":"2024-04-12T11:44:03.242856+0000","last_clean":"2024-04-12T11:44:03.242856+0000","last_became_active":"2024-04-12T11:43:56.154887+0000","last_became_peered":"2024-04-12T11:43:56.154887+0000","last_unstale":"2024-04-12T11:44:03.242856+0000","last_undegraded":"2024-04-12T11:44:03.242856+0000","last_fullsized":"2024-04-12T11:44:03.242856+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:54:30.766490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1148,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.245571+0000","last_change":"2024-04-12T11:43:56.147413+0000","last_active":"2024-04-12T11:44:03.245571+0000","last_peered":"2024-04-12T11:44:03.245571+0000","last_clean":"2024-04-12T11:44:03.245571+0000","last_became_active":"2024-04-12T11:43:56.146334+0000","last_became_peered":"2024-04-12T11:43:56.146334+0000","last_unstale":"2024-04-12T11:44:03.245571+0000","last_undegraded":"2024-04-12T11:44:03.245571+0000","last_fullsized":"2024-04-12T11:44:03.245571+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:09:38.929592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1811,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.259201+0000","last_change":"2024-04-12T11:43:56.148041+0000","last_active":"2024-04-12T11:44:03.259201+0000","last_peered":"2024-04-12T11:44:03.259201+0000","last_clean":"2024-04-12T11:44:03.259201+0000","last_became_active":"2024-04-12T11:43:56.147573+0000","last_became_peered":"2024-04-12T11:43:56.147573+0000","last_unstale":"2024-04-12T11:44:03.259201+0000","last_undegraded":"2024-04-12T11:44:03.259201+0000","last_fullsized":"2024-04-12T11:44:03.259201+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:08:42.586831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1150,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.141740+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:43:57.141740+0000","last_peered":"2024-04-12T11:43:57.141740+0000","last_clean":"2024-04-12T11:43:57.141740+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:43:57.141740+0000","last_undegraded":"2024-04-12T11:43:57.141740+0000","last_fullsized":"2024-04-12T11:43:57.141740+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":540,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.248289+0000","last_change":"2024-04-12T11:43:56.154309+0000","last_active":"2024-04-12T11:44:03.248289+0000","last_peered":"2024-04-12T11:44:03.248289+0000","last_clean":"2024-04-12T11:44:03.248289+0000","last_became_active":"2024-04-12T11:43:56.153734+0000","last_became_peered":"2024-04-12T11:43:56.153734+0000","last_unstale":"2024-04-12T11:44:03.248289+0000","last_undegraded":"2024-04-12T11:44:03.248289+0000","last_fullsized":"2024-04-12T11:44:03.248289+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:27:56.403751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.4","version":"1196'6812","reported_seq":6845,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:22.509039+0000","last_change":"2024-04-12T11:42:38.223256+0000","last_active":"2024-04-12T11:44:22.509039+0000","last_peered":"2024-04-12T11:44:22.509039+0000","last_clean":"2024-04-12T11:44:22.509039+0000","last_became_active":"2024-04-12T11:42:38.222777+0000","last_became_peered":"2024-04-12T11:42:38.222777+0000","last_unstale":"2024-04-12T11:44:22.509039+0000","last_undegraded":"2024-04-12T11:44:22.509039+0000","last_fullsized":"2024-04-12T11:44:22.509039+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6812,"log_dups_size":0,"ondisk_log_size":6812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:30.994347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446431232,"num_objects":6812,"num_object_clones":0,"num_object_copies":20436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6812,"num_write_kb":435968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":32,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.239032+0000","last_change":"2024-04-12T11:43:56.162609+0000","last_active":"2024-04-12T11:44:03.239032+0000","last_peered":"2024-04-12T11:44:03.239032+0000","last_clean":"2024-04-12T11:44:03.239032+0000","last_became_active":"2024-04-12T11:43:56.145127+0000","last_became_peered":"2024-04-12T11:43:56.145127+0000","last_unstale":"2024-04-12T11:44:03.239032+0000","last_undegraded":"2024-04-12T11:44:03.239032+0000","last_fullsized":"2024-04-12T11:44:03.239032+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:52:01.077341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1142,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.136984+0000","last_change":"2024-04-12T11:43:56.156727+0000","last_active":"2024-04-12T11:43:57.136984+0000","last_peered":"2024-04-12T11:43:57.136984+0000","last_clean":"2024-04-12T11:43:57.136984+0000","last_became_active":"2024-04-12T11:43:56.150002+0000","last_became_peered":"2024-04-12T11:43:56.150002+0000","last_unstale":"2024-04-12T11:43:57.136984+0000","last_undegraded":"2024-04-12T11:43:57.136984+0000","last_fullsized":"2024-04-12T11:43:57.136984+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:57:14.144397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"1196'6556","reported_seq":6597,"reported_epoch":1196,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:44:22.545768+0000","last_change":"2024-04-12T11:43:57.514718+0000","last_active":"2024-04-12T11:44:22.545768+0000","last_peered":"2024-04-12T11:44:22.545768+0000","last_clean":"2024-04-12T11:43:55.132835+0000","last_became_active":"2024-04-12T11:43:57.459665+0000","last_became_peered":"2024-04-12T11:43:57.459665+0000","last_unstale":"2024-04-12T11:44:22.545768+0000","last_undegraded":"2024-04-12T11:43:57.313864+0000","last_fullsized":"2024-04-12T11:43:57.141332+0000","mapping_epoch":1194,"log_start":"0'0","ondisk_log_start":"0'0","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:28:33.915771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429654016,"num_objects":6556,"num_object_clones":0,"num_object_copies":19668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5418,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6556,"num_write_kb":419584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6","objects":1138},{"shards":"4,6,7","objects":5418}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":32,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.239182+0000","last_change":"2024-04-12T11:43:56.165865+0000","last_active":"2024-04-12T11:44:03.239182+0000","last_peered":"2024-04-12T11:44:03.239182+0000","last_clean":"2024-04-12T11:44:03.239182+0000","last_became_active":"2024-04-12T11:43:56.148337+0000","last_became_peered":"2024-04-12T11:43:56.148337+0000","last_unstale":"2024-04-12T11:44:03.239182+0000","last_undegraded":"2024-04-12T11:44:03.239182+0000","last_fullsized":"2024-04-12T11:44:03.239182+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:33:40.815504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1738,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.236902+0000","last_change":"2024-04-12T11:42:59.339953+0000","last_active":"2024-04-12T11:44:03.236902+0000","last_peered":"2024-04-12T11:44:03.236902+0000","last_clean":"2024-04-12T11:44:03.236902+0000","last_became_active":"2024-04-12T11:42:59.339584+0000","last_became_peered":"2024-04-12T11:42:59.339584+0000","last_unstale":"2024-04-12T11:44:03.236902+0000","last_undegraded":"2024-04-12T11:44:03.236902+0000","last_fullsized":"2024-04-12T11:44:03.236902+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:52:18.645653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"1196'6574","reported_seq":7742,"reported_epoch":1196,"state":"active+recovering+undersized+remapped","last_fresh":"2024-04-12T11:44:24.475055+0000","last_change":"2024-04-12T11:43:57.370553+0000","last_active":"2024-04-12T11:44:24.475055+0000","last_peered":"2024-04-12T11:44:24.475055+0000","last_clean":"2024-04-12T11:43:55.131806+0000","last_became_active":"2024-04-12T11:43:57.319663+0000","last_became_peered":"2024-04-12T11:43:57.319663+0000","last_unstale":"2024-04-12T11:44:24.475055+0000","last_undegraded":"2024-04-12T11:44:24.475055+0000","last_fullsized":"2024-04-12T11:43:57.139728+0000","mapping_epoch":1194,"log_start":"0'0","ondisk_log_start":"0'0","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6574,"log_dups_size":0,"ondisk_log_size":6574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430833664,"num_objects":6574,"num_object_clones":0,"num_object_copies":19722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4776,"num_objects_unfound":0,"num_objects_dirty":6574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6574,"num_write_kb":420736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":553,"num_bytes_recovered":36241408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":1798},{"shards":"0,6,7","objects":4776}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":30,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.444091+0000","last_change":"2024-04-12T11:43:56.168807+0000","last_active":"2024-04-12T11:43:57.444091+0000","last_peered":"2024-04-12T11:43:57.444091+0000","last_clean":"2024-04-12T11:43:57.444091+0000","last_became_active":"2024-04-12T11:43:56.161055+0000","last_became_peered":"2024-04-12T11:43:56.161055+0000","last_unstale":"2024-04-12T11:43:57.444091+0000","last_undegraded":"2024-04-12T11:43:57.444091+0000","last_fullsized":"2024-04-12T11:43:57.444091+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:14:27.853099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1160,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.142497+0000","last_change":"2024-04-12T11:43:56.147442+0000","last_active":"2024-04-12T11:43:57.142497+0000","last_peered":"2024-04-12T11:43:57.142497+0000","last_clean":"2024-04-12T11:43:57.142497+0000","last_became_active":"2024-04-12T11:43:56.147064+0000","last_became_peered":"2024-04-12T11:43:56.147064+0000","last_unstale":"2024-04-12T11:43:57.142497+0000","last_undegraded":"2024-04-12T11:43:57.142497+0000","last_fullsized":"2024-04-12T11:43:57.142497+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:40:51.192073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1196'6553","reported_seq":8021,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:23.816070+0000","last_change":"2024-04-12T11:43:07.410348+0000","last_active":"2024-04-12T11:44:23.816070+0000","last_peered":"2024-04-12T11:44:23.816070+0000","last_clean":"2024-04-12T11:44:23.816070+0000","last_became_active":"2024-04-12T11:43:07.407264+0000","last_became_peered":"2024-04-12T11:43:07.407264+0000","last_unstale":"2024-04-12T11:44:23.816070+0000","last_undegraded":"2024-04-12T11:44:23.816070+0000","last_fullsized":"2024-04-12T11:44:23.816070+0000","mapping_epoch":1181,"log_start":"1179'700","ondisk_log_start":"1179'700","created":1172,"last_epoch_clean":1182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":5853,"log_dups_size":0,"ondisk_log_size":5853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:00:29.338924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429457408,"num_objects":6553,"num_object_clones":0,"num_object_copies":19659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6553,"num_write_kb":419392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":699,"num_bytes_recovered":45809664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1157,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.137255+0000","last_change":"2024-04-12T11:42:59.344910+0000","last_active":"2024-04-12T11:43:57.137255+0000","last_peered":"2024-04-12T11:43:57.137255+0000","last_clean":"2024-04-12T11:43:57.137255+0000","last_became_active":"2024-04-12T11:42:59.344561+0000","last_became_peered":"2024-04-12T11:42:59.344561+0000","last_unstale":"2024-04-12T11:43:57.137255+0000","last_undegraded":"2024-04-12T11:43:57.137255+0000","last_fullsized":"2024-04-12T11:43:57.137255+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:57:23.336391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1808,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.252437+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:44:03.252437+0000","last_peered":"2024-04-12T11:44:03.252437+0000","last_clean":"2024-04-12T11:44:03.252437+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:44:03.252437+0000","last_undegraded":"2024-04-12T11:44:03.252437+0000","last_fullsized":"2024-04-12T11:44:03.252437+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"1196'6499","reported_seq":8040,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:22.501625+0000","last_change":"2024-04-12T11:43:08.412243+0000","last_active":"2024-04-12T11:44:22.501625+0000","last_peered":"2024-04-12T11:44:22.501625+0000","last_clean":"2024-04-12T11:44:22.501625+0000","last_became_active":"2024-04-12T11:43:08.409396+0000","last_became_peered":"2024-04-12T11:43:08.409396+0000","last_unstale":"2024-04-12T11:44:22.501625+0000","last_undegraded":"2024-04-12T11:44:22.501625+0000","last_fullsized":"2024-04-12T11:44:22.501625+0000","mapping_epoch":1182,"log_start":"0'0","ondisk_log_start":"0'0","created":1172,"last_epoch_clean":1183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6499,"log_dups_size":0,"ondisk_log_size":6499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:28:06.154291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425918464,"num_objects":6499,"num_object_clones":0,"num_object_copies":19497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6499,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":744,"num_bytes_recovered":48758784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":88,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.252531+0000","last_change":"2024-04-12T11:42:59.353950+0000","last_active":"2024-04-12T11:44:03.252531+0000","last_peered":"2024-04-12T11:44:03.252531+0000","last_clean":"2024-04-12T11:44:03.252531+0000","last_became_active":"2024-04-12T11:42:59.338131+0000","last_became_peered":"2024-04-12T11:42:59.338131+0000","last_unstale":"2024-04-12T11:44:03.252531+0000","last_undegraded":"2024-04-12T11:44:03.252531+0000","last_fullsized":"2024-04-12T11:44:03.252531+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:14.399669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1729,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.239737+0000","last_change":"2024-04-12T11:42:59.338730+0000","last_active":"2024-04-12T11:44:03.239737+0000","last_peered":"2024-04-12T11:44:03.239737+0000","last_clean":"2024-04-12T11:44:03.239737+0000","last_became_active":"2024-04-12T11:42:59.337814+0000","last_became_peered":"2024-04-12T11:42:59.337814+0000","last_unstale":"2024-04-12T11:44:03.239737+0000","last_undegraded":"2024-04-12T11:44:03.239737+0000","last_fullsized":"2024-04-12T11:44:03.239737+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:57:01.279589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"1196'6618","reported_seq":6676,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:23.833016+0000","last_change":"2024-04-12T11:42:38.230043+0000","last_active":"2024-04-12T11:44:23.833016+0000","last_peered":"2024-04-12T11:44:23.833016+0000","last_clean":"2024-04-12T11:44:23.833016+0000","last_became_active":"2024-04-12T11:42:38.229766+0000","last_became_peered":"2024-04-12T11:42:38.229766+0000","last_unstale":"2024-04-12T11:44:23.833016+0000","last_undegraded":"2024-04-12T11:44:23.833016+0000","last_fullsized":"2024-04-12T11:44:23.833016+0000","mapping_epoch":1172,"log_start":"1179'800","ondisk_log_start":"1179'800","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":5818,"log_dups_size":0,"ondisk_log_size":5818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:53:59.675961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433717248,"num_objects":6618,"num_object_clones":0,"num_object_copies":19854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6618,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":749,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.237132+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:44:03.237132+0000","last_peered":"2024-04-12T11:44:03.237132+0000","last_clean":"2024-04-12T11:44:03.237132+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:44:03.237132+0000","last_undegraded":"2024-04-12T11:44:03.237132+0000","last_fullsized":"2024-04-12T11:44:03.237132+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1745,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.237325+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:44:03.237325+0000","last_peered":"2024-04-12T11:44:03.237325+0000","last_clean":"2024-04-12T11:44:03.237325+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:44:03.237325+0000","last_undegraded":"2024-04-12T11:44:03.237325+0000","last_fullsized":"2024-04-12T11:44:03.237325+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"1196'6626","reported_seq":6684,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:24.010565+0000","last_change":"2024-04-12T11:42:38.225355+0000","last_active":"2024-04-12T11:44:24.010565+0000","last_peered":"2024-04-12T11:44:24.010565+0000","last_clean":"2024-04-12T11:44:24.010565+0000","last_became_active":"2024-04-12T11:42:38.224970+0000","last_became_peered":"2024-04-12T11:42:38.224970+0000","last_unstale":"2024-04-12T11:44:24.010565+0000","last_undegraded":"2024-04-12T11:44:24.010565+0000","last_fullsized":"2024-04-12T11:44:24.010565+0000","mapping_epoch":1172,"log_start":"1176'300","ondisk_log_start":"1176'300","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6326,"log_dups_size":0,"ondisk_log_size":6326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:31:02.802515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":434241536,"num_objects":6626,"num_object_clones":0,"num_object_copies":19878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6626,"num_write_kb":424064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1737,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.236979+0000","last_change":"2024-04-12T11:43:56.145347+0000","last_active":"2024-04-12T11:44:03.236979+0000","last_peered":"2024-04-12T11:44:03.236979+0000","last_clean":"2024-04-12T11:44:03.236979+0000","last_became_active":"2024-04-12T11:43:56.144685+0000","last_became_peered":"2024-04-12T11:43:56.144685+0000","last_unstale":"2024-04-12T11:44:03.236979+0000","last_undegraded":"2024-04-12T11:44:03.236979+0000","last_fullsized":"2024-04-12T11:44:03.236979+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:32:49.360750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1710,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.239206+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:44:03.239206+0000","last_peered":"2024-04-12T11:44:03.239206+0000","last_clean":"2024-04-12T11:44:03.239206+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:44:03.239206+0000","last_undegraded":"2024-04-12T11:44:03.239206+0000","last_fullsized":"2024-04-12T11:44:03.239206+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"1196'6735","reported_seq":6792,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:23.824056+0000","last_change":"2024-04-12T11:42:38.229120+0000","last_active":"2024-04-12T11:44:23.824056+0000","last_peered":"2024-04-12T11:44:23.824056+0000","last_clean":"2024-04-12T11:44:23.824056+0000","last_became_active":"2024-04-12T11:42:38.228835+0000","last_became_peered":"2024-04-12T11:42:38.228835+0000","last_unstale":"2024-04-12T11:44:23.824056+0000","last_undegraded":"2024-04-12T11:44:23.824056+0000","last_fullsized":"2024-04-12T11:44:23.824056+0000","mapping_epoch":1172,"log_start":"1179'900","ondisk_log_start":"1179'900","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":5835,"log_dups_size":0,"ondisk_log_size":5835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:22:40.716473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":441384960,"num_objects":6735,"num_object_clones":0,"num_object_copies":20205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6735,"num_write_kb":431040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":577,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.247956+0000","last_change":"2024-04-12T11:43:56.159154+0000","last_active":"2024-04-12T11:44:03.247956+0000","last_peered":"2024-04-12T11:44:03.247956+0000","last_clean":"2024-04-12T11:44:03.247956+0000","last_became_active":"2024-04-12T11:43:56.158776+0000","last_became_peered":"2024-04-12T11:43:56.158776+0000","last_unstale":"2024-04-12T11:44:03.247956+0000","last_undegraded":"2024-04-12T11:44:03.247956+0000","last_fullsized":"2024-04-12T11:44:03.247956+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:16:48.105709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1818,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.248392+0000","last_change":"2024-04-12T11:43:56.148925+0000","last_active":"2024-04-12T11:44:03.248392+0000","last_peered":"2024-04-12T11:44:03.248392+0000","last_clean":"2024-04-12T11:44:03.248392+0000","last_became_active":"2024-04-12T11:43:56.148579+0000","last_became_peered":"2024-04-12T11:43:56.148579+0000","last_unstale":"2024-04-12T11:44:03.248392+0000","last_undegraded":"2024-04-12T11:44:03.248392+0000","last_fullsized":"2024-04-12T11:44:03.248392+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:24:26.609238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"1196'6462","reported_seq":6520,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:24.032385+0000","last_change":"2024-04-12T11:42:38.226043+0000","last_active":"2024-04-12T11:44:24.032385+0000","last_peered":"2024-04-12T11:44:24.032385+0000","last_clean":"2024-04-12T11:44:24.032385+0000","last_became_active":"2024-04-12T11:42:38.225220+0000","last_became_peered":"2024-04-12T11:42:38.225220+0000","last_unstale":"2024-04-12T11:44:24.032385+0000","last_undegraded":"2024-04-12T11:44:24.032385+0000","last_fullsized":"2024-04-12T11:44:24.032385+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6462,"log_dups_size":0,"ondisk_log_size":6462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:55:26.430554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423493632,"num_objects":6462,"num_object_clones":0,"num_object_copies":19386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6462,"num_write_kb":413568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1814,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.243932+0000","last_change":"2024-04-12T11:43:56.152821+0000","last_active":"2024-04-12T11:44:03.243932+0000","last_peered":"2024-04-12T11:44:03.243932+0000","last_clean":"2024-04-12T11:44:03.243932+0000","last_became_active":"2024-04-12T11:43:56.152289+0000","last_became_peered":"2024-04-12T11:43:56.152289+0000","last_unstale":"2024-04-12T11:44:03.243932+0000","last_undegraded":"2024-04-12T11:44:03.243932+0000","last_fullsized":"2024-04-12T11:44:03.243932+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:07:33.484071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1173,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.138236+0000","last_change":"2024-04-12T11:42:59.341071+0000","last_active":"2024-04-12T11:43:57.138236+0000","last_peered":"2024-04-12T11:43:57.138236+0000","last_clean":"2024-04-12T11:43:57.138236+0000","last_became_active":"2024-04-12T11:42:59.340434+0000","last_became_peered":"2024-04-12T11:42:59.340434+0000","last_unstale":"2024-04-12T11:43:57.138236+0000","last_undegraded":"2024-04-12T11:43:57.138236+0000","last_fullsized":"2024-04-12T11:43:57.138236+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:26:24.535122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1799,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.247600+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:44:03.247600+0000","last_peered":"2024-04-12T11:44:03.247600+0000","last_clean":"2024-04-12T11:44:03.247600+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:44:03.247600+0000","last_undegraded":"2024-04-12T11:44:03.247600+0000","last_fullsized":"2024-04-12T11:44:03.247600+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"1196'6418","reported_seq":9806,"reported_epoch":1196,"state":"active+recovering+undersized+remapped","last_fresh":"2024-04-12T11:44:22.522661+0000","last_change":"2024-04-12T11:43:57.375472+0000","last_active":"2024-04-12T11:44:22.522661+0000","last_peered":"2024-04-12T11:44:22.522661+0000","last_clean":"2024-04-12T11:43:55.127664+0000","last_became_active":"2024-04-12T11:43:57.335534+0000","last_became_peered":"2024-04-12T11:43:57.335534+0000","last_unstale":"2024-04-12T11:44:22.522661+0000","last_undegraded":"2024-04-12T11:44:22.522661+0000","last_fullsized":"2024-04-12T11:43:57.137634+0000","mapping_epoch":1194,"log_start":"0'0","ondisk_log_start":"0'0","created":1172,"last_epoch_clean":1185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6418,"log_dups_size":0,"ondisk_log_size":6418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:10:10.534296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420610048,"num_objects":6418,"num_object_clones":0,"num_object_copies":19254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4332,"num_objects_unfound":0,"num_objects_dirty":6418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6418,"num_write_kb":410752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1664,"num_bytes_recovered":109051904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0],"avail_no_missing":["5","0","7"],"object_location_counts":[{"shards":"0,5","objects":2086},{"shards":"0,5,7","objects":4332}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1154,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.137523+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:43:57.137523+0000","last_peered":"2024-04-12T11:43:57.137523+0000","last_clean":"2024-04-12T11:43:57.137523+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:43:57.137523+0000","last_undegraded":"2024-04-12T11:43:57.137523+0000","last_fullsized":"2024-04-12T11:43:57.137523+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1732,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.236832+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:44:03.236832+0000","last_peered":"2024-04-12T11:44:03.236832+0000","last_clean":"2024-04-12T11:44:03.236832+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:44:03.236832+0000","last_undegraded":"2024-04-12T11:44:03.236832+0000","last_fullsized":"2024-04-12T11:44:03.236832+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":25,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.252761+0000","last_change":"2024-04-12T11:43:24.350344+0000","last_active":"2024-04-12T11:44:03.252761+0000","last_peered":"2024-04-12T11:44:03.252761+0000","last_clean":"2024-04-12T11:44:03.252761+0000","last_became_active":"2024-04-12T11:43:24.350004+0000","last_became_peered":"2024-04-12T11:43:24.350004+0000","last_unstale":"2024-04-12T11:44:03.252761+0000","last_undegraded":"2024-04-12T11:44:03.252761+0000","last_fullsized":"2024-04-12T11:44:03.252761+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:41:34.660146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"1196'6781","reported_seq":8427,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:24.009625+0000","last_change":"2024-04-12T11:43:12.996056+0000","last_active":"2024-04-12T11:44:24.009625+0000","last_peered":"2024-04-12T11:44:24.009625+0000","last_clean":"2024-04-12T11:44:24.009625+0000","last_became_active":"2024-04-12T11:43:12.993500+0000","last_became_peered":"2024-04-12T11:43:12.993500+0000","last_unstale":"2024-04-12T11:44:24.009625+0000","last_undegraded":"2024-04-12T11:44:24.009625+0000","last_fullsized":"2024-04-12T11:44:24.009625+0000","mapping_epoch":1185,"log_start":"1176'500","ondisk_log_start":"1176'500","created":1172,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6281,"log_dups_size":0,"ondisk_log_size":6281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:30:40.297135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444399616,"num_objects":6781,"num_object_clones":0,"num_object_copies":20343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6781,"num_write_kb":433984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":788,"num_bytes_recovered":51642368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1778,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.245853+0000","last_change":"2024-04-12T11:42:59.339036+0000","last_active":"2024-04-12T11:44:03.245853+0000","last_peered":"2024-04-12T11:44:03.245853+0000","last_clean":"2024-04-12T11:44:03.245853+0000","last_became_active":"2024-04-12T11:42:59.338650+0000","last_became_peered":"2024-04-12T11:42:59.338650+0000","last_unstale":"2024-04-12T11:44:03.245853+0000","last_undegraded":"2024-04-12T11:44:03.245853+0000","last_fullsized":"2024-04-12T11:44:03.245853+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:12:21.069607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1800,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.247899+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:44:03.247899+0000","last_peered":"2024-04-12T11:44:03.247899+0000","last_clean":"2024-04-12T11:44:03.247899+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:44:03.247899+0000","last_undegraded":"2024-04-12T11:44:03.247899+0000","last_fullsized":"2024-04-12T11:44:03.247899+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1802,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.248293+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:44:03.248293+0000","last_peered":"2024-04-12T11:44:03.248293+0000","last_clean":"2024-04-12T11:44:03.248293+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:44:03.248293+0000","last_undegraded":"2024-04-12T11:44:03.248293+0000","last_fullsized":"2024-04-12T11:44:03.248293+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"1196'6602","reported_seq":6635,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:22.525491+0000","last_change":"2024-04-12T11:42:38.224309+0000","last_active":"2024-04-12T11:44:22.525491+0000","last_peered":"2024-04-12T11:44:22.525491+0000","last_clean":"2024-04-12T11:44:22.525491+0000","last_became_active":"2024-04-12T11:42:38.224042+0000","last_became_peered":"2024-04-12T11:42:38.224042+0000","last_unstale":"2024-04-12T11:44:22.525491+0000","last_undegraded":"2024-04-12T11:44:22.525491+0000","last_fullsized":"2024-04-12T11:44:22.525491+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6602,"log_dups_size":0,"ondisk_log_size":6602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:38:16.773159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432668672,"num_objects":6602,"num_object_clones":0,"num_object_copies":19806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6602,"num_write_kb":422528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1163,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.137147+0000","last_change":"2024-04-12T11:43:56.152680+0000","last_active":"2024-04-12T11:43:57.137147+0000","last_peered":"2024-04-12T11:43:57.137147+0000","last_clean":"2024-04-12T11:43:57.137147+0000","last_became_active":"2024-04-12T11:43:56.152265+0000","last_became_peered":"2024-04-12T11:43:56.152265+0000","last_unstale":"2024-04-12T11:43:57.137147+0000","last_undegraded":"2024-04-12T11:43:57.137147+0000","last_fullsized":"2024-04-12T11:43:57.137147+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:13:06.764237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":569,"reported_epoch":1196,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:44:03.250855+0000","last_change":"2024-04-12T11:43:57.342894+0000","last_active":"2024-04-12T11:44:03.250855+0000","last_peered":"2024-04-12T11:44:03.250855+0000","last_clean":"2024-04-12T11:17:23.256141+0000","last_became_active":"2024-04-12T11:43:57.342486+0000","last_became_peered":"2024-04-12T11:43:57.342486+0000","last_unstale":"2024-04-12T11:44:03.250855+0000","last_undegraded":"2024-04-12T11:43:57.139935+0000","last_fullsized":"2024-04-12T11:43:57.139080+0000","mapping_epoch":1194,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":520,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.137321+0000","last_change":"2024-04-12T11:42:59.339273+0000","last_active":"2024-04-12T11:43:57.137321+0000","last_peered":"2024-04-12T11:43:57.137321+0000","last_clean":"2024-04-12T11:43:57.137321+0000","last_became_active":"2024-04-12T11:42:59.336638+0000","last_became_peered":"2024-04-12T11:42:59.336638+0000","last_unstale":"2024-04-12T11:43:57.137321+0000","last_undegraded":"2024-04-12T11:43:57.137321+0000","last_fullsized":"2024-04-12T11:43:57.137321+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:39:56.003479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1152,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.243990+0000","last_change":"2024-04-12T11:43:56.158074+0000","last_active":"2024-04-12T11:44:03.243990+0000","last_peered":"2024-04-12T11:44:03.243990+0000","last_clean":"2024-04-12T11:44:03.243990+0000","last_became_active":"2024-04-12T11:43:56.157310+0000","last_became_peered":"2024-04-12T11:43:56.157310+0000","last_unstale":"2024-04-12T11:44:03.243990+0000","last_undegraded":"2024-04-12T11:44:03.243990+0000","last_fullsized":"2024-04-12T11:44:03.243990+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:57:31.307741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1196'6688","reported_seq":6746,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:23.788010+0000","last_change":"2024-04-12T11:42:38.227777+0000","last_active":"2024-04-12T11:44:23.788010+0000","last_peered":"2024-04-12T11:44:23.788010+0000","last_clean":"2024-04-12T11:44:23.788010+0000","last_became_active":"2024-04-12T11:42:38.227462+0000","last_became_peered":"2024-04-12T11:42:38.227462+0000","last_unstale":"2024-04-12T11:44:23.788010+0000","last_undegraded":"2024-04-12T11:44:23.788010+0000","last_fullsized":"2024-04-12T11:44:23.788010+0000","mapping_epoch":1172,"log_start":"1179'900","ondisk_log_start":"1179'900","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":5788,"log_dups_size":0,"ondisk_log_size":5788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:05.339212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438304768,"num_objects":6688,"num_object_clones":0,"num_object_copies":20064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6688,"num_write_kb":428032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":509,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.247756+0000","last_change":"2024-04-12T11:43:56.158427+0000","last_active":"2024-04-12T11:44:03.247756+0000","last_peered":"2024-04-12T11:44:03.247756+0000","last_clean":"2024-04-12T11:44:03.247756+0000","last_became_active":"2024-04-12T11:43:56.157935+0000","last_became_peered":"2024-04-12T11:43:56.157935+0000","last_unstale":"2024-04-12T11:44:03.247756+0000","last_undegraded":"2024-04-12T11:44:03.247756+0000","last_fullsized":"2024-04-12T11:44:03.247756+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:50:35.685850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1815,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.253378+0000","last_change":"2024-04-12T11:42:59.340232+0000","last_active":"2024-04-12T11:44:03.253378+0000","last_peered":"2024-04-12T11:44:03.253378+0000","last_clean":"2024-04-12T11:44:03.253378+0000","last_became_active":"2024-04-12T11:42:59.339803+0000","last_became_peered":"2024-04-12T11:42:59.339803+0000","last_unstale":"2024-04-12T11:44:03.253378+0000","last_undegraded":"2024-04-12T11:44:03.253378+0000","last_fullsized":"2024-04-12T11:44:03.253378+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:03:04.963920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1809,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.259318+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:44:03.259318+0000","last_peered":"2024-04-12T11:44:03.259318+0000","last_clean":"2024-04-12T11:44:03.259318+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:44:03.259318+0000","last_undegraded":"2024-04-12T11:44:03.259318+0000","last_fullsized":"2024-04-12T11:44:03.259318+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"1196'6686","reported_seq":8243,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:25.185899+0000","last_change":"2024-04-12T11:43:07.412509+0000","last_active":"2024-04-12T11:44:25.185899+0000","last_peered":"2024-04-12T11:44:25.185899+0000","last_clean":"2024-04-12T11:44:25.185899+0000","last_became_active":"2024-04-12T11:43:07.408341+0000","last_became_peered":"2024-04-12T11:43:07.408341+0000","last_unstale":"2024-04-12T11:44:25.185899+0000","last_undegraded":"2024-04-12T11:44:25.185899+0000","last_fullsized":"2024-04-12T11:44:25.185899+0000","mapping_epoch":1181,"log_start":"0'0","ondisk_log_start":"0'0","created":1172,"last_epoch_clean":1182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6686,"log_dups_size":0,"ondisk_log_size":6686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:46:11.970180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437125120,"num_objects":6670,"num_object_clones":0,"num_object_copies":20010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6686,"num_write_kb":427392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":745,"num_bytes_recovered":48824320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":583,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.253397+0000","last_change":"2024-04-12T11:43:56.157421+0000","last_active":"2024-04-12T11:44:03.253397+0000","last_peered":"2024-04-12T11:44:03.253397+0000","last_clean":"2024-04-12T11:44:03.253397+0000","last_became_active":"2024-04-12T11:43:56.157019+0000","last_became_peered":"2024-04-12T11:43:56.157019+0000","last_unstale":"2024-04-12T11:44:03.253397+0000","last_undegraded":"2024-04-12T11:44:03.253397+0000","last_fullsized":"2024-04-12T11:44:03.253397+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:54:07.044019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1799,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.243833+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:44:03.243833+0000","last_peered":"2024-04-12T11:44:03.243833+0000","last_clean":"2024-04-12T11:44:03.243833+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:44:03.243833+0000","last_undegraded":"2024-04-12T11:44:03.243833+0000","last_fullsized":"2024-04-12T11:44:03.243833+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1798,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.252741+0000","last_change":"2024-04-12T11:43:56.155443+0000","last_active":"2024-04-12T11:44:03.252741+0000","last_peered":"2024-04-12T11:44:03.252741+0000","last_clean":"2024-04-12T11:44:03.252741+0000","last_became_active":"2024-04-12T11:43:56.155047+0000","last_became_peered":"2024-04-12T11:43:56.155047+0000","last_unstale":"2024-04-12T11:44:03.252741+0000","last_undegraded":"2024-04-12T11:44:03.252741+0000","last_fullsized":"2024-04-12T11:44:03.252741+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:40:02.200764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"1196'6567","reported_seq":8098,"reported_epoch":1196,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:44:23.887845+0000","last_change":"2024-04-12T11:43:57.332982+0000","last_active":"2024-04-12T11:44:23.887845+0000","last_peered":"2024-04-12T11:44:23.887845+0000","last_clean":"2024-04-12T11:43:55.131021+0000","last_became_active":"2024-04-12T11:43:57.324070+0000","last_became_peered":"2024-04-12T11:43:57.324070+0000","last_unstale":"2024-04-12T11:44:23.887845+0000","last_undegraded":"2024-04-12T11:43:57.170018+0000","last_fullsized":"2024-04-12T11:43:57.138831+0000","mapping_epoch":1194,"log_start":"0'0","ondisk_log_start":"0'0","created":1172,"last_epoch_clean":1181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6567,"log_dups_size":0,"ondisk_log_size":6567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430374912,"num_objects":6567,"num_object_clones":0,"num_object_copies":19701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6567,"num_write_kb":420288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":730,"num_bytes_recovered":47841280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":1279},{"shards":"0,5,7","objects":5288}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":32,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.253750+0000","last_change":"2024-04-12T11:43:56.162844+0000","last_active":"2024-04-12T11:44:03.253750+0000","last_peered":"2024-04-12T11:44:03.253750+0000","last_clean":"2024-04-12T11:44:03.253750+0000","last_became_active":"2024-04-12T11:43:56.151211+0000","last_became_peered":"2024-04-12T11:43:56.151211+0000","last_unstale":"2024-04-12T11:44:03.253750+0000","last_undegraded":"2024-04-12T11:44:03.253750+0000","last_fullsized":"2024-04-12T11:44:03.253750+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:02:10.334942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1799,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.245145+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:44:03.245145+0000","last_peered":"2024-04-12T11:44:03.245145+0000","last_clean":"2024-04-12T11:44:03.245145+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:44:03.245145+0000","last_undegraded":"2024-04-12T11:44:03.245145+0000","last_fullsized":"2024-04-12T11:44:03.245145+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":521,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.250654+0000","last_change":"2024-04-12T11:43:56.151074+0000","last_active":"2024-04-12T11:44:03.250654+0000","last_peered":"2024-04-12T11:44:03.250654+0000","last_clean":"2024-04-12T11:44:03.250654+0000","last_became_active":"2024-04-12T11:43:56.150730+0000","last_became_peered":"2024-04-12T11:43:56.150730+0000","last_unstale":"2024-04-12T11:44:03.250654+0000","last_undegraded":"2024-04-12T11:44:03.250654+0000","last_fullsized":"2024-04-12T11:44:03.250654+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:37.858527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.d","version":"1196'6616","reported_seq":6657,"reported_epoch":1196,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:44:22.445024+0000","last_change":"2024-04-12T11:43:57.536882+0000","last_active":"2024-04-12T11:44:22.445024+0000","last_peered":"2024-04-12T11:44:22.445024+0000","last_clean":"2024-04-12T11:43:55.121092+0000","last_became_active":"2024-04-12T11:43:57.523925+0000","last_became_peered":"2024-04-12T11:43:57.523925+0000","last_unstale":"2024-04-12T11:44:22.445024+0000","last_undegraded":"2024-04-12T11:43:57.179930+0000","last_fullsized":"2024-04-12T11:43:57.137324+0000","mapping_epoch":1194,"log_start":"0'0","ondisk_log_start":"0'0","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6616,"log_dups_size":0,"ondisk_log_size":6616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:56:46.018305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433586176,"num_objects":6616,"num_object_clones":0,"num_object_copies":19848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5405,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6616,"num_write_kb":423424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5","objects":1211},{"shards":"2,5,7","objects":5405}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1176,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.244476+0000","last_change":"2024-04-12T11:43:56.152491+0000","last_active":"2024-04-12T11:44:03.244476+0000","last_peered":"2024-04-12T11:44:03.244476+0000","last_clean":"2024-04-12T11:44:03.244476+0000","last_became_active":"2024-04-12T11:43:56.150273+0000","last_became_peered":"2024-04-12T11:43:56.150273+0000","last_unstale":"2024-04-12T11:44:03.244476+0000","last_undegraded":"2024-04-12T11:44:03.244476+0000","last_fullsized":"2024-04-12T11:44:03.244476+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:40:52.389513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1812,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.258913+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:44:03.258913+0000","last_peered":"2024-04-12T11:44:03.258913+0000","last_clean":"2024-04-12T11:44:03.258913+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:44:03.258913+0000","last_undegraded":"2024-04-12T11:44:03.258913+0000","last_fullsized":"2024-04-12T11:44:03.258913+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1719,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.236755+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:44:03.236755+0000","last_peered":"2024-04-12T11:44:03.236755+0000","last_clean":"2024-04-12T11:44:03.236755+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:44:03.236755+0000","last_undegraded":"2024-04-12T11:44:03.236755+0000","last_fullsized":"2024-04-12T11:44:03.236755+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1807,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.252639+0000","last_change":"2024-04-12T11:43:56.149237+0000","last_active":"2024-04-12T11:44:03.252639+0000","last_peered":"2024-04-12T11:44:03.252639+0000","last_clean":"2024-04-12T11:44:03.252639+0000","last_became_active":"2024-04-12T11:43:56.148838+0000","last_became_peered":"2024-04-12T11:43:56.148838+0000","last_unstale":"2024-04-12T11:44:03.252639+0000","last_undegraded":"2024-04-12T11:44:03.252639+0000","last_fullsized":"2024-04-12T11:44:03.252639+0000","mapping_epoch":1193,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1194,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:03:17.622036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.137342+0000","last_change":"2024-04-12T11:43:26.346666+0000","last_active":"2024-04-12T11:43:57.137342+0000","last_peered":"2024-04-12T11:43:57.137342+0000","last_clean":"2024-04-12T11:43:57.137342+0000","last_became_active":"2024-04-12T11:43:26.346357+0000","last_became_peered":"2024-04-12T11:43:26.346357+0000","last_unstale":"2024-04-12T11:43:57.137342+0000","last_undegraded":"2024-04-12T11:43:57.137342+0000","last_fullsized":"2024-04-12T11:43:57.137342+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:18.945599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1164,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.137354+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:43:57.137354+0000","last_peered":"2024-04-12T11:43:57.137354+0000","last_clean":"2024-04-12T11:43:57.137354+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:43:57.137354+0000","last_undegraded":"2024-04-12T11:43:57.137354+0000","last_fullsized":"2024-04-12T11:43:57.137354+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1752,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.237369+0000","last_change":"2024-04-12T11:42:59.337935+0000","last_active":"2024-04-12T11:44:03.237369+0000","last_peered":"2024-04-12T11:44:03.237369+0000","last_clean":"2024-04-12T11:44:03.237369+0000","last_became_active":"2024-04-12T11:42:59.337568+0000","last_became_peered":"2024-04-12T11:42:59.337568+0000","last_unstale":"2024-04-12T11:44:03.237369+0000","last_undegraded":"2024-04-12T11:44:03.237369+0000","last_fullsized":"2024-04-12T11:44:03.237369+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:37:47.566124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1149,"reported_epoch":1195,"state":"active+clean","last_fresh":"2024-04-12T11:43:57.138345+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:43:57.138345+0000","last_peered":"2024-04-12T11:43:57.138345+0000","last_clean":"2024-04-12T11:43:57.138345+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:43:57.138345+0000","last_undegraded":"2024-04-12T11:43:57.138345+0000","last_fullsized":"2024-04-12T11:43:57.138345+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1743,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.237060+0000","last_change":"2024-04-12T11:42:59.339283+0000","last_active":"2024-04-12T11:44:03.237060+0000","last_peered":"2024-04-12T11:44:03.237060+0000","last_clean":"2024-04-12T11:44:03.237060+0000","last_became_active":"2024-04-12T11:42:59.338955+0000","last_became_peered":"2024-04-12T11:42:59.338955+0000","last_unstale":"2024-04-12T11:44:03.237060+0000","last_undegraded":"2024-04-12T11:44:03.237060+0000","last_fullsized":"2024-04-12T11:44:03.237060+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:26:21.796575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":48,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.250174+0000","last_change":"2024-04-12T11:42:59.339640+0000","last_active":"2024-04-12T11:44:03.250174+0000","last_peered":"2024-04-12T11:44:03.250174+0000","last_clean":"2024-04-12T11:44:03.250174+0000","last_became_active":"2024-04-12T11:42:59.339091+0000","last_became_peered":"2024-04-12T11:42:59.339091+0000","last_unstale":"2024-04-12T11:44:03.250174+0000","last_undegraded":"2024-04-12T11:44:03.250174+0000","last_fullsized":"2024-04-12T11:44:03.250174+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:20:23.178049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":575,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.245893+0000","last_change":"2024-04-12T11:42:59.339931+0000","last_active":"2024-04-12T11:44:03.245893+0000","last_peered":"2024-04-12T11:44:03.245893+0000","last_clean":"2024-04-12T11:44:03.245893+0000","last_became_active":"2024-04-12T11:42:59.339533+0000","last_became_peered":"2024-04-12T11:42:59.339533+0000","last_unstale":"2024-04-12T11:44:03.245893+0000","last_undegraded":"2024-04-12T11:44:03.245893+0000","last_fullsized":"2024-04-12T11:44:03.245893+0000","mapping_epoch":1177,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:49:22.558551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1842,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.253654+0000","last_change":"2024-04-12T11:43:26.370162+0000","last_active":"2024-04-12T11:44:03.253654+0000","last_peered":"2024-04-12T11:44:03.253654+0000","last_clean":"2024-04-12T11:44:03.253654+0000","last_became_active":"2024-04-12T11:43:26.352240+0000","last_became_peered":"2024-04-12T11:43:26.352240+0000","last_unstale":"2024-04-12T11:44:03.253654+0000","last_undegraded":"2024-04-12T11:44:03.253654+0000","last_fullsized":"2024-04-12T11:44:03.253654+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:34:09.364519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1709,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-04-12T11:44:03.236636+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:44:03.236636+0000","last_peered":"2024-04-12T11:44:03.236636+0000","last_clean":"2024-04-12T11:44:03.236636+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:44:03.236636+0000","last_undegraded":"2024-04-12T11:44:03.236636+0000","last_fullsized":"2024-04-12T11:44:03.236636+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":6718,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6932201472,"num_objects":105777,"num_object_clones":0,"num_object_copies":317331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16111,"num_objects_misplaced":9108,"num_objects_unfound":0,"num_objects_dirty":105777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":105793,"num_write_kb":6770240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5924,"num_bytes_recovered":388235264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21022969856,"data_stored":21022965784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101693,"ondisk_log_size":101693,"up":48,"acting":43,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234384,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496021,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2490508,"kb_used_data":2180548,"kb_used_omap":1,"kb_used_meta":309950,"kb_avail":91254644,"statfs":{"total":95995035648,"available":93444755456,"internally_reserved":0,"allocated":2232881152,"data_stored":2229612249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":317389256},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536331,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3308848,"kb_used_data":2970088,"kb_used_omap":30,"kb_used_meta":338721,"kb_avail":90436304,"statfs":{"total":95995035648,"available":92606775296,"internally_reserved":0,"allocated":3041370112,"data_stored":3038123839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":346850626},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190415,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3427260,"kb_used_data":3034228,"kb_used_omap":43,"kb_used_meta":392980,"kb_avail":90317892,"statfs":{"total":95995035648,"available":92485521408,"internally_reserved":0,"allocated":3107049472,"data_stored":3103792127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":402411563},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849543,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4018748,"kb_used_data":3545268,"kb_used_omap":36,"kb_used_meta":473435,"kb_avail":89726404,"statfs":{"total":95995035648,"available":91879837696,"internally_reserved":0,"allocated":3630354432,"data_stored":3627100639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":484797620},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":0,"seq":2534030704896,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018108,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3602248,"kb_used_data":3218880,"kb_used_omap":42,"kb_used_meta":383317,"kb_avail":90142904,"statfs":{"total":95995035648,"available":92306333696,"internally_reserved":0,"allocated":3296133120,"data_stored":3292867221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":392517432},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757657,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4285684,"kb_used_data":3893676,"kb_used_omap":1,"kb_used_meta":391998,"kb_avail":89459468,"statfs":{"total":95995035648,"available":91606495232,"internally_reserved":0,"allocated":3987124224,"data_stored":3983874473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":401406406},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3096510464,"data_stored":3096510464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3053912064,"data_stored":3053912064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3615887360,"data_stored":3615883288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":100007936,"data_stored":100007936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3291742208,"data_stored":3291742208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3884777472,"data_stored":3884777472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2230386688,"data_stored":2230386688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1749745664,"data_stored":1749745664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:44:33.462 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-04-12T11:44:33.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool set unique_pool_3 pg_num 26 2024-04-12T11:44:33.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:33 smithi087 ceph-mon[89014]: pgmap v1766: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 25 MiB/s wr, 615 op/s; 16113/310968 objects degraded (5.182%); 8740/310968 objects misplaced (2.811%); 3.6 MiB/s, 57 objects/s recovering 2024-04-12T11:44:33.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16113/310968 objects degraded (5.182%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:33.480 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:33.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:33.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:33.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:33.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:33.642 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:33.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:33.714 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:33.743 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:33.774 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:33.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:33.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:33.786 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:33.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:33.921 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:33.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:33.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:34.023 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:34.079 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:34.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:34.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:34.181 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:34.220 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:34.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:34.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:34.322 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:34.356 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:34.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:34.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:34.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:34.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:34.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:34.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:34.600 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:34.636 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:34.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:34.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:34.738 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:34.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:34.772 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:34.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:34.874 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:34.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:34.912 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:34.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:35.013 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:35.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:35.051 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:35.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:35.059 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:44:35.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:35.196 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:35.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:35.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:35 smithi029 ceph-mon[97787]: pgmap v1767: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 21 MiB/s wr, 759 op/s; 16113/303744 objects degraded (5.305%); 8543/303744 objects misplaced (2.813%); 4.6 MiB/s, 74 objects/s recovering 2024-04-12T11:44:35.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:35 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3239376974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:44:35.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:35 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:44:35.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:35 smithi029 ceph-mon[93650]: pgmap v1767: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 21 MiB/s wr, 759 op/s; 16113/303744 objects degraded (5.305%); 8543/303744 objects misplaced (2.813%); 4.6 MiB/s, 74 objects/s recovering 2024-04-12T11:44:35.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:35 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3239376974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:44:35.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:35 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:44:35.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:44:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:44:34] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2024-04-12T11:44:35.299 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:35.334 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:35.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:35.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:35.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:35.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:35 smithi087 ceph-mon[89014]: pgmap v1767: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 21 MiB/s wr, 759 op/s; 16113/303744 objects degraded (5.305%); 8543/303744 objects misplaced (2.813%); 4.6 MiB/s, 74 objects/s recovering 2024-04-12T11:44:35.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:35 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3239376974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:44:35.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:35 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:44:35.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:35.470 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:35.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:35.571 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:35.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:35.608 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:35.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:35.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:35.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:35.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:35.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:35.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:35.880 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:35.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:35.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:35.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:36.029 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:36.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:36.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:36.131 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:36.171 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:36.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:36.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:36 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:44:36.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:36 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:44:36.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:36 smithi029 ceph-mon[97787]: osdmap e1197: 8 total, 8 up, 6 in 2024-04-12T11:44:36.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:36 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:44:36.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:36 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:44:36.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:36 smithi029 ceph-mon[93650]: osdmap e1197: 8 total, 8 up, 6 in 2024-04-12T11:44:36.271 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:36.304 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:36.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:36.407 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:36.446 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:36.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:36.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:36.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:36 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:44:36.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:36 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:44:36.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:36 smithi087 ceph-mon[89014]: osdmap e1197: 8 total, 8 up, 6 in 2024-04-12T11:44:36.548 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:36.582 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:36.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:36.684 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:36.727 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:36.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:36.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:36.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:36.860 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:36.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:36.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:36.962 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:37.038 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:37.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:37.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:37.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:37.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:37.183 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:37.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:37.265 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:44:37.265 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:44:37.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:37.328 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:37.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:37.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:37.430 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:37.455 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:37.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:37.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:37.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:37.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:37.593 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:37.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:37.694 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:37 smithi087 ceph-mon[89014]: pgmap v1769: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 102 B/s rd, 3.3 MiB/s wr, 717 op/s; 16113/299019 objects degraded (5.389%); 8430/299019 objects misplaced (2.819%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:44:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:37.729 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:37.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:37.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:37 smithi029 ceph-mon[93650]: pgmap v1769: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 102 B/s rd, 3.3 MiB/s wr, 717 op/s; 16113/299019 objects degraded (5.389%); 8430/299019 objects misplaced (2.819%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:44:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:37 smithi029 ceph-mon[97787]: pgmap v1769: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 102 B/s rd, 3.3 MiB/s wr, 717 op/s; 16113/299019 objects degraded (5.389%); 8430/299019 objects misplaced (2.819%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T11:44:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:44:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:37.831 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:37.868 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:37.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:37.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:37.969 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:38.005 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:38.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:38.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:38.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:38.155 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:38.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:38.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:38.180 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:38.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:38.294 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:38.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:38.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:38.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:38.438 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:38.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:38.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16113/299019 objects degraded (5.389%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:38.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16113/299019 objects degraded (5.389%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:38.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:38.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:38.579 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:38.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:38.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:38.716 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16113/299019 objects degraded (5.389%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:38.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:38.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:38.732 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-04-12T11:44:38.819 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:38.852 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:38.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:38.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:38.954 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:38.992 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:38.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:38.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:39.094 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:39.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:39.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:39.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:39.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:39.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:39.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:39.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:39.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:39.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:39.412 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:39.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:39.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:39.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:39 smithi029 ceph-mon[93650]: pgmap v1770: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 288 KiB/s wr, 753 op/s; 16113/294333 objects degraded (5.474%); 8250/294333 objects misplaced (2.803%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:39.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:39 smithi029 ceph-mon[97787]: pgmap v1770: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 288 KiB/s wr, 753 op/s; 16113/294333 objects degraded (5.474%); 8250/294333 objects misplaced (2.803%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:39.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:39.559 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:39.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:39.632 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:44:39.661 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:39.670 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:44:39.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:39.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:39.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:39.708 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:44:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:39 smithi087 ceph-mon[89014]: pgmap v1770: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 288 KiB/s wr, 753 op/s; 16113/294333 objects degraded (5.474%); 8250/294333 objects misplaced (2.803%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:39.748 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:44:39.781 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:44:39.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:39.829 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:44:39.838 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:39.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:39.866 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:44:39.901 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:44:39.904 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:44:39.904 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:44:39.904 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:44:39.904 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:44:39.904 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:44:39.904 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:44:39.904 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:44:39.905 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:44:39.939 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:39.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:39.978 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:39.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:40.079 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:40.112 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:40.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:40.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:40.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:40.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:44:40.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:40.260 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:40.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:40.362 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:40.397 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:40.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:40.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:40.499 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:40.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:40 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2648604326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:44:40.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:40 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2648604326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:44:40.536 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:40.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:40.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:40.638 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:40.676 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:40.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:40.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:40 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2648604326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:44:40.779 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:40.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:40.818 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:40.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:40.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:40.943 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:40.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:40.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:41.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:41.080 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:41.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:41.181 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:41.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:41.217 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:41.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:41.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:41.389 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:41.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:41.491 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:41.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:41 smithi029 ceph-mon[93650]: pgmap v1771: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 16113/288843 objects degraded (5.578%); 8138/288843 objects misplaced (2.817%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:41.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:41 smithi029 ceph-mon[97787]: pgmap v1771: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 16113/288843 objects degraded (5.578%); 8138/288843 objects misplaced (2.817%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:41.525 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:41.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:41.627 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:41.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:41.667 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:41.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:41 smithi087 ceph-mon[89014]: pgmap v1771: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 16113/288843 objects degraded (5.578%); 8138/288843 objects misplaced (2.817%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:41.769 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:41.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:41.809 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:41.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:41.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:41.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:41.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:41.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:42.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:42.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:42.092 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:42.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:42.113 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:42.125 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:42.193 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:42.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:42.221 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:42.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:42.322 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:42.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:42.366 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:42.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:42.468 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:42.504 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:42.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:42.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:42.606 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:42.643 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:42.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:42.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:44:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:44:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:44:42.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:42.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:44:42.780 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:42.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:42.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:42.882 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:42.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:42.976 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:42.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:43.078 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:43.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:43.102 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:43.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:43.203 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:43.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:43.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:43.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:43.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:43.365 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:43.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:43.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:43.377 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 8 pg_num to 26 2024-04-12T11:44:43.467 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:43.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:43.502 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:43.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:43.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:43.637 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:43.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:43.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:43 smithi087 ceph-mon[89014]: pgmap v1772: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 16113/286458 objects degraded (5.625%); 8138/286458 objects misplaced (2.841%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:43 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4269930072' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:44:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16113/286458 objects degraded (5.625%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:43.739 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:43 smithi029 ceph-mon[93650]: pgmap v1772: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 16113/286458 objects degraded (5.625%); 8138/286458 objects misplaced (2.841%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:43 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4269930072' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:44:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16113/286458 objects degraded (5.625%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:43 smithi029 ceph-mon[97787]: pgmap v1772: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 16113/286458 objects degraded (5.625%); 8138/286458 objects misplaced (2.841%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:44:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:43 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4269930072' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:44:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16113/286458 objects degraded (5.625%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:43.773 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:43.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:43.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:43.876 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:43.910 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:43.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:43.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:44.012 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:44.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:44.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:44.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:44.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:44.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:44.189 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:44.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:44.290 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:44.320 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:44.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:44.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:44.422 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:44.488 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:44.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:44.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:44.637 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:44.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:44.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:44.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:44 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4269930072' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2024-04-12T11:44:44.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:44 smithi087 ceph-mon[89014]: osdmap e1198: 8 total, 8 up, 6 in 2024-04-12T11:44:44.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:44 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2024-04-12T11:44:44.739 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:44.762 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:44.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:44.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:44 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4269930072' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2024-04-12T11:44:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:44 smithi029 ceph-mon[97787]: osdmap e1198: 8 total, 8 up, 6 in 2024-04-12T11:44:44.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:44 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2024-04-12T11:44:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:44 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4269930072' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2024-04-12T11:44:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:44 smithi029 ceph-mon[93650]: osdmap e1198: 8 total, 8 up, 6 in 2024-04-12T11:44:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:44 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2024-04-12T11:44:44.865 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:44.889 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:44.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:44.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:44.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:45.016 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:45.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:45.118 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:45.141 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:45.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:45.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:45.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:45.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:45.269 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:45.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:45.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:44:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:44:44] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2024-04-12T11:44:45.370 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:45.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:45.401 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:45.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:45.502 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:45.531 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:45.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:45.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:45.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:45.661 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:45.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:45.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:45 smithi087 ceph-mon[89014]: pgmap v1774: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 793 op/s; 16113/278763 objects degraded (5.780%); 7970/278763 objects misplaced (2.859%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T11:44:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:45 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2024-04-12T11:44:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:45 smithi087 ceph-mon[89014]: osdmap e1199: 8 total, 8 up, 6 in 2024-04-12T11:44:45.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:45 smithi029 ceph-mon[93650]: pgmap v1774: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 793 op/s; 16113/278763 objects degraded (5.780%); 7970/278763 objects misplaced (2.859%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T11:44:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:45 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2024-04-12T11:44:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:45 smithi029 ceph-mon[93650]: osdmap e1199: 8 total, 8 up, 6 in 2024-04-12T11:44:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:45 smithi029 ceph-mon[97787]: pgmap v1774: 89 pgs: 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 793 op/s; 16113/278763 objects degraded (5.780%); 7970/278763 objects misplaced (2.859%); 3.3 MiB/s, 52 objects/s recovering 2024-04-12T11:44:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:45 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2024-04-12T11:44:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:45 smithi029 ceph-mon[97787]: osdmap e1199: 8 total, 8 up, 6 in 2024-04-12T11:44:45.789 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:45.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:45.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:45.914 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:45.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:45.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:46.016 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:46.040 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:46.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:46.142 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:46.166 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:46.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:46.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:46.269 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:46.293 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:46.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:46.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:46.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:46.431 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:46.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:46.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:46.533 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:46.578 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:46.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:46.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:46.681 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:46.713 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:46.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:46.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:46 smithi087 ceph-mon[89014]: osdmap e1200: 8 total, 8 up, 6 in 2024-04-12T11:44:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:46 smithi087 ceph-mon[89014]: pgmap v1777: 99 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 10 unknown, 1 peering, 1 remapped+peering; 5.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 774 op/s; 11768/274893 objects degraded (4.281%); 7881/274893 objects misplaced (2.867%); 2.8 MiB/s, 44 objects/s recovering 2024-04-12T11:44:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:46 smithi087 ceph-mon[89014]: 8.1 scrub starts 2024-04-12T11:44:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:46 smithi029 ceph-mon[93650]: osdmap e1200: 8 total, 8 up, 6 in 2024-04-12T11:44:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:46 smithi029 ceph-mon[93650]: pgmap v1777: 99 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 10 unknown, 1 peering, 1 remapped+peering; 5.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 774 op/s; 11768/274893 objects degraded (4.281%); 7881/274893 objects misplaced (2.867%); 2.8 MiB/s, 44 objects/s recovering 2024-04-12T11:44:46.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:46 smithi029 ceph-mon[93650]: 8.1 scrub starts 2024-04-12T11:44:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:46 smithi029 ceph-mon[97787]: osdmap e1200: 8 total, 8 up, 6 in 2024-04-12T11:44:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:46 smithi029 ceph-mon[97787]: pgmap v1777: 99 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 10 unknown, 1 peering, 1 remapped+peering; 5.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 774 op/s; 11768/274893 objects degraded (4.281%); 7881/274893 objects misplaced (2.867%); 2.8 MiB/s, 44 objects/s recovering 2024-04-12T11:44:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:46 smithi029 ceph-mon[97787]: 8.1 scrub starts 2024-04-12T11:44:46.814 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:46.861 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:46.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:46.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:46.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:46.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:46.999 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:47.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:47.100 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:47.137 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:47.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:47.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:47.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:47.270 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:47.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:47.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:47.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:47.407 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:47.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:47.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:47.509 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:47.539 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:47.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:47.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:47.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:47.674 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:47.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:47 smithi087 ceph-mon[89014]: osdmap e1201: 8 total, 8 up, 6 in 2024-04-12T11:44:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:47 smithi087 ceph-mon[89014]: 8.1 scrub ok 2024-04-12T11:44:47.758 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:47 smithi029 ceph-mon[97787]: osdmap e1201: 8 total, 8 up, 6 in 2024-04-12T11:44:47.758 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:47 smithi029 ceph-mon[97787]: 8.1 scrub ok 2024-04-12T11:44:47.758 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:47 smithi029 ceph-mon[93650]: osdmap e1201: 8 total, 8 up, 6 in 2024-04-12T11:44:47.758 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:47 smithi029 ceph-mon[93650]: 8.1 scrub ok 2024-04-12T11:44:47.776 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:47.806 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:47.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:47.907 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:47.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:47.942 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:47.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:48.043 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:48.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:48.077 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:48.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:48.178 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:48.212 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:48.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:48.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:48.314 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:48.354 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:48.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:48.455 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:48.494 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:48.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:48.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:48.596 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:48.637 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:48.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:48.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:48 smithi087 ceph-mon[89014]: pgmap v1779: 99 pgs: 4 active+recovery_wait+degraded, 83 active+clean, 2 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 unknown, 1 peering, 1 remapped+peering; 5.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 10643/271836 objects degraded (3.915%); 8689/271836 objects misplaced (3.196%); 5.8 MiB/s, 92 objects/s recovering 2024-04-12T11:44:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11768/274893 objects degraded (4.281%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:48 smithi087 ceph-mon[89014]: 8.11 scrub starts 2024-04-12T11:44:48.739 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:48 smithi029 ceph-mon[97787]: pgmap v1779: 99 pgs: 4 active+recovery_wait+degraded, 83 active+clean, 2 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 unknown, 1 peering, 1 remapped+peering; 5.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 10643/271836 objects degraded (3.915%); 8689/271836 objects misplaced (3.196%); 5.8 MiB/s, 92 objects/s recovering 2024-04-12T11:44:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11768/274893 objects degraded (4.281%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:48 smithi029 ceph-mon[97787]: 8.11 scrub starts 2024-04-12T11:44:48.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:48 smithi029 ceph-mon[93650]: pgmap v1779: 99 pgs: 4 active+recovery_wait+degraded, 83 active+clean, 2 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 unknown, 1 peering, 1 remapped+peering; 5.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 10643/271836 objects degraded (3.915%); 8689/271836 objects misplaced (3.196%); 5.8 MiB/s, 92 objects/s recovering 2024-04-12T11:44:48.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11768/274893 objects degraded (4.281%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:48.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:48 smithi029 ceph-mon[93650]: 8.11 scrub starts 2024-04-12T11:44:48.773 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:48.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:48.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:48.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:48.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:48.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:48.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:49.014 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:49.049 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:49.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:49.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:49.151 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:49.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:49.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:49.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:49.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:49.326 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:49.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:49.378 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:44:49.378 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:44:49.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:44:49.378 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [] 2024-04-12T11:44:49.379 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:44:49.428 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:49.463 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:49.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:49.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:49.534 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:49 smithi029 ceph-mon[97787]: 8.11 scrub ok 2024-04-12T11:44:49.534 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:49 smithi029 ceph-mon[93650]: 8.11 scrub ok 2024-04-12T11:44:49.565 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:49.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:49.603 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:49.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:49.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:49 smithi087 ceph-mon[89014]: 8.11 scrub ok 2024-04-12T11:44:49.757 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:49.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:49.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:49.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:49.905 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:49.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:50.007 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:50.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:50.042 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:50.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:50.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:50.178 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:50.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:50.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:50.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:50.292 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:44:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:50.343 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:50.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:50.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:50 smithi029 ceph-mon[97787]: pgmap v1780: 99 pgs: 6 active+recovery_wait+degraded, 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 unknown, 1 peering; 5.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 14906/267531 objects degraded (5.572%); 8454/267531 objects misplaced (3.160%); 7.5 MiB/s, 120 objects/s recovering 2024-04-12T11:44:50.444 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:50.481 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:50.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:50.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:50.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:50.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:50.624 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:50.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:50 smithi087 ceph-mon[89014]: pgmap v1780: 99 pgs: 6 active+recovery_wait+degraded, 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 unknown, 1 peering; 5.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 14906/267531 objects degraded (5.572%); 8454/267531 objects misplaced (3.160%); 7.5 MiB/s, 120 objects/s recovering 2024-04-12T11:44:50.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:50.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:50.765 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:50.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:50.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:50 smithi029 ceph-mon[93650]: pgmap v1780: 99 pgs: 6 active+recovery_wait+degraded, 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 unknown, 1 peering; 5.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 14906/267531 objects degraded (5.572%); 8454/267531 objects misplaced (3.160%); 7.5 MiB/s, 120 objects/s recovering 2024-04-12T11:44:50.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-04-12T11:44:50.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:50.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:50.911 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:50.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:51.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:51.049 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:51.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:51.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:51.141 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:44:51.151 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:51.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:51.188 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:51.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:51.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:51.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:51.330 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:51.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:51.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:51.438 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:44:51.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:51.471 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:51.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:51.535 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:51 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/940927362' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:44:51.536 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:51 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:44:51.536 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:51.571 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:51.609 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:51.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:51.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:51.711 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:51.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:51.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:51.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:51.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:51 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/940927362' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:44:51.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:51 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:44:51.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:51 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/940927362' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:44:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:51 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:44:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:44:51.846 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:51.884 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:51.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:51.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:51.986 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:52.012 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:52.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:52.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:52.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:52.152 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:52.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:52.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:52.254 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:52.288 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:52.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:52.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:52.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:52.422 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:52.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:52.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:52.524 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:52.560 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:52.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:52.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:52.591 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:44:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:44:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:44:52.591 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:52 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:44:52.592 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:52 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:44:52.592 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:52 smithi087 ceph-mon[89014]: osdmap e1202: 8 total, 8 up, 6 in 2024-04-12T11:44:52.592 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:52 smithi087 ceph-mon[89014]: pgmap v1782: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 14906/265344 objects degraded (5.618%); 8454/265344 objects misplaced (3.186%); 6.0 MiB/s, 96 objects/s recovering 2024-04-12T11:44:52.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:52.698 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:52.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:52.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:52 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:44:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:52 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:44:52.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:52 smithi029 ceph-mon[97787]: osdmap e1202: 8 total, 8 up, 6 in 2024-04-12T11:44:52.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:52 smithi029 ceph-mon[97787]: pgmap v1782: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 14906/265344 objects degraded (5.618%); 8454/265344 objects misplaced (3.186%); 6.0 MiB/s, 96 objects/s recovering 2024-04-12T11:44:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:52 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:44:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:52 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:44:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:52 smithi029 ceph-mon[93650]: osdmap e1202: 8 total, 8 up, 6 in 2024-04-12T11:44:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:52 smithi029 ceph-mon[93650]: pgmap v1782: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 14906/265344 objects degraded (5.618%); 8454/265344 objects misplaced (3.186%); 6.0 MiB/s, 96 objects/s recovering 2024-04-12T11:44:52.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:52.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:52.837 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:52.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:52.938 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:52.961 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:52.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:52.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:53.008 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:44:53.008 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:44:53.008 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.008 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:128194 8.b 8:d2f42fb0:::benchmark_data_smithi029_194418_object21884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:44:53.008 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:52.870300+0000", 2024-04-12T11:44:53.008 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.136253282, 2024-04-12T11:44:53.008 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.136287404, 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 128194 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.870300+0000", 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:44:53.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.870300+0000", 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.870301+0000", 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6139999999999999e-06 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.870308+0000", 2024-04-12T11:44:53.010 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6440000000000003e-06 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.870325+0000", 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7292999999999999e-05 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.870334+0000", 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2369999999999992e-06 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:44:53.011 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.870802+0000", 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000468844 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.871506+0000", 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00070334599999999998 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.871747+0000", 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00024110000000000001 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:53.012 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.878184+0000", 2024-04-12T11:44:53.013 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0064370809999999999 2024-04-12T11:44:53.013 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.013 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.013 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.879735+0000", 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001551085 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:128195 8.12 8:494b2706:::benchmark_data_smithi029_194418_object21885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:52.874242+0000", 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.132310919, 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.132405403, 2024-04-12T11:44:53.014 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:44:53.015 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:44:53.015 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:44:53.015 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:44:53.015 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:44:53.015 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:44:53.015 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 128195 2024-04-12T11:44:53.015 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.015 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:44:53.015 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.015 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.874242+0000", 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.874242+0000", 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.874244+0000", 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6080000000000001e-06 2024-04-12T11:44:53.016 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.017 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.017 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:44:53.017 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.874255+0000", 2024-04-12T11:44:53.017 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1285e-05 2024-04-12T11:44:53.017 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.017 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.018 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:44:53.018 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.874268+0000", 2024-04-12T11:44:53.018 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2666e-05 2024-04-12T11:44:53.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.018 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.018 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.874276+0000", 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1259999999999998e-06 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.874317+0000", 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1621e-05 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.874453+0000", 2024-04-12T11:44:53.019 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000135895 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,5", 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.874710+0000", 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00025625799999999998 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.876189+0000", 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001479171 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.876756+0000", 2024-04-12T11:44:53.020 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00056703700000000005 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:128201 8.12 8:49044845:::benchmark_data_smithi029_194418_object21891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:52.879905+0000", 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.12664811100000001, 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.12679407200000001, 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:44:53.021 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 128201 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.879905+0000", 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.879905+0000", 2024-04-12T11:44:53.022 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:44:53.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.023 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:44:53.023 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.879907+0000", 2024-04-12T11:44:53.023 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5230000000000001e-06 2024-04-12T11:44:53.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.023 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:44:53.023 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.879913+0000", 2024-04-12T11:44:53.023 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9369999999999998e-06 2024-04-12T11:44:53.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.024 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:44:53.024 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.879928+0000", 2024-04-12T11:44:53.024 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4413000000000001e-05 2024-04-12T11:44:53.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.024 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:44:53.024 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.879936+0000", 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4290000000000001e-06 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.879994+0000", 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7865999999999998e-05 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.880128+0000", 2024-04-12T11:44:53.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013405900000000001 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,5", 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.880407+0000", 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00027918499999999999 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.881681+0000", 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00127323 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:52.881991+0000", 2024-04-12T11:44:53.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00031069499999999999 2024-04-12T11:44:53.027 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:44:53.027 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:44:53.027 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:44:53.027 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:44:53.027 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2024-04-12T11:44:53.027 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 3 2024-04-12T11:44:53.027 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:44:53.062 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:53.103 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:53.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:53.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:53.204 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:53.242 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:53.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:53.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:53.344 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:53.368 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:53.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:53.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:53.470 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:53.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:53.504 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:53.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 14906/265344 objects degraded (5.618%), 9 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:53.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 14906/265344 objects degraded (5.618%), 9 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:53.605 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:53.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:53.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:53.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:53.679 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 14906/265344 objects degraded (5.618%), 9 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:53.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:53.740 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-04-12T11:44:53.764 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:53.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:53.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:53.866 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:53.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:53.915 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:53.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:54.016 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:54.022 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6] out_osds: [3, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:44:54.022 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:44:54.022 INFO:tasks.thrashosds.thrasher:Adding osd 3 2024-04-12T11:44:54.022 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd in 3 2024-04-12T11:44:54.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:54.061 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:54.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:54.125 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:44:54.162 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:54.189 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:54.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:54.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:54.291 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:54.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:54.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:54.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:54.437 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:54.479 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:54.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:54.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:54.554 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:54 smithi087 ceph-mon[89014]: pgmap v1783: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 14790/256404 objects degraded (5.768%); 8162/256404 objects misplaced (3.183%); 8.0 MiB/s, 129 objects/s recovering 2024-04-12T11:44:54.581 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:54.614 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:54.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:54.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:54.716 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:54.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:54.757 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:54.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:54.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:54 smithi029 ceph-mon[97787]: pgmap v1783: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 14790/256404 objects degraded (5.768%); 8162/256404 objects misplaced (3.183%); 8.0 MiB/s, 129 objects/s recovering 2024-04-12T11:44:54.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:54 smithi029 ceph-mon[93650]: pgmap v1783: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 14790/256404 objects degraded (5.768%); 8162/256404 objects misplaced (3.183%); 8.0 MiB/s, 129 objects/s recovering 2024-04-12T11:44:54.858 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:54.893 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:54.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:54.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:54.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:55.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:55.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:55.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:55.133 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:55.160 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:55.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:55.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:55.262 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:55.374 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:44:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:44:54] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2024-04-12T11:44:55.414 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:55.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:55.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:55.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:55.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:55.541 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:55.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:55.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:55.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:55.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:55.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:55.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:55.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:55.802 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:55.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:55.902 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:55.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:55.939 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:55.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:56.041 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:56.067 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:56.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:56.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:56.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:56.193 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:56.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:56.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:56.295 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:56.331 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:56.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:56.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:56.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:56.468 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:56.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:56.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:56.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:56.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:56.609 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:56.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:56.710 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:56.746 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:56.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:56.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:56.848 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:56.889 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:56.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:56.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:56.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:57.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:57.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:57.105 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:56 smithi087 ceph-mon[89014]: pgmap v1784: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 10 op/s; 14668/251592 objects degraded (5.830%); 7893/251592 objects misplaced (3.137%); 9.3 MiB/s, 149 objects/s recovering 2024-04-12T11:44:57.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:57.137 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:44:57.137 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:44:57.138 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:44:57.138 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:44:57.138 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:44:57.140 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:57.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:57.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:57.242 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:57.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:56 smithi029 ceph-mon[97787]: pgmap v1784: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 10 op/s; 14668/251592 objects degraded (5.830%); 7893/251592 objects misplaced (3.137%); 9.3 MiB/s, 149 objects/s recovering 2024-04-12T11:44:57.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:56 smithi029 ceph-mon[93650]: pgmap v1784: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 10 op/s; 14668/251592 objects degraded (5.830%); 7893/251592 objects misplaced (3.137%); 9.3 MiB/s, 149 objects/s recovering 2024-04-12T11:44:57.280 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:57.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:57.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:57.417 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:57.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:57.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:57.519 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:57.552 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:57.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:57.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:57.654 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:57.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:57.692 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:57.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:57.791 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-04-12T11:44:57.793 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:57.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:57.817 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:57.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:57.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:57.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:57.942 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:57.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:58.030 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:44:58.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:58.067 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:58.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:58.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:58.158 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:57 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 14668/251592 objects degraded (5.830%), 9 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:58.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:44:58.206 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:58.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:58.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:44:58.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:57 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 14668/251592 objects degraded (5.830%), 9 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:58.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:57 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 14668/251592 objects degraded (5.830%), 9 pgs degraded (PG_DEGRADED) 2024-04-12T11:44:58.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:58.346 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:58.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:58.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:58.448 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:58.482 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:58.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:58.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:58.584 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:58.608 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:58.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:58.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:58.710 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:44:58.735 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:58.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:58.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:44:58.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:58.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:58.874 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:58.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:58.975 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:59.011 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:59.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:59.112 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:44:59.137 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:59.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:59.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:44:59.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:44:58 smithi087 ceph-mon[89014]: pgmap v1785: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 53 op/s; 14508/247224 objects degraded (5.868%); 7647/247224 objects misplaced (3.093%); 9.4 MiB/s, 151 objects/s recovering 2024-04-12T11:44:59.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:44:59.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:59.265 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:59.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:44:59.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:44:58 smithi029 ceph-mon[93650]: pgmap v1785: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 53 op/s; 14508/247224 objects degraded (5.868%); 7647/247224 objects misplaced (3.093%); 9.4 MiB/s, 151 objects/s recovering 2024-04-12T11:44:59.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:44:58 smithi029 ceph-mon[97787]: pgmap v1785: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 53 op/s; 14508/247224 objects degraded (5.868%); 7647/247224 objects misplaced (3.093%); 9.4 MiB/s, 151 objects/s recovering 2024-04-12T11:44:59.366 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:44:59.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:59.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:59.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:44:59.507 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:44:59.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:44:59.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:59.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:44:59.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:44:59.687 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:59.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:59.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:44:59.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:59.821 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:59.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:59.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:44:59.923 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:44:59.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:44:59.947 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:44:59.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:00.048 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:00.093 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:00.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:00.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:00.194 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:00.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:00.234 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:00.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:00.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:00.375 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:00.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:00.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:00.446 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:45:00.446 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:45:00.446 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:45:00.446 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:45:00.446 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.446 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91829 7.1 7:88cc5a79:::benchmark_data_smithi029_161157_object91828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1145)", 2024-04-12T11:45:00.446 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:20.898486+0000", 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 399.54407853999999, 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.525263109, 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91829 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898486+0000", 2024-04-12T11:45:00.447 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898486+0000", 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898487+0000", 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.299e-06 2024-04-12T11:45:00.448 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898522+0000", 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4456999999999999e-05 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898535+0000", 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3852999999999999e-05 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898545+0000", 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6579999999999997e-06 2024-04-12T11:45:00.449 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898595+0000", 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.011e-05 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898706+0000", 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000110948 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,6", 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.898838+0000", 2024-04-12T11:45:00.450 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013242500000000001 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.901924+0000", 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030852610000000002 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.902226+0000", 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00030198699999999998 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.916736+0000", 2024-04-12T11:45:00.451 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014510232 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.916752+0000", 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6283999999999999e-05 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.933963+0000", 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0172107859999999 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.933980+0000", 2024-04-12T11:45:00.452 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7091999999999998e-05 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.423651+0000", 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.489671127 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.423749+0000", 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7590000000000006e-05 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.453 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91848 7.1 7:8f4d9a6c:::benchmark_data_smithi029_161157_object91847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:20.920205+0000", 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 399.52235897399999, 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5272139290000002, 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91848 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.454 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920205+0000", 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920205+0000", 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920207+0000", 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2619999999999999e-06 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.455 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920258+0000", 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1115000000000001e-05 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920270+0000", 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1823e-05 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920278+0000", 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2840000000000006e-06 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.456 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920329+0000", 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1069000000000002e-05 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.920338+0000", 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4599999999999992e-06 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934016+0000", 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.013677768 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.457 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934026+0000", 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7170000000000003e-06 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.425246+0000", 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.491220121 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.425362+0000", 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000116205 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:45:00.458 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.425481+0000", 2024-04-12T11:45:00.459 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000119164 2024-04-12T11:45:00.459 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.459 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.459 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.459 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432241+0000", 2024-04-12T11:45:00.459 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0067593490000000004 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432262+0000", 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1274e-05 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.444301+0000", 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012039253999999999 2024-04-12T11:45:00.460 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.444363+0000", 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2181999999999995e-05 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.447419+0000", 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030558820000000002 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.461 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91849 7.1 7:837125d8:::benchmark_data_smithi029_161157_object91848:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:20.922774+0000", 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 399.51979054999998, 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5270688610000001, 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91849 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.462 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922774+0000", 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922774+0000", 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922775+0000", 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.125e-06 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.463 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922842+0000", 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7133999999999998e-05 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922855+0000", 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3383e-05 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922864+0000", 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7199999999999995e-06 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.464 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922925+0000", 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0645e-05 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.922936+0000", 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0784999999999999e-05 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934056+0000", 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0111205409999999 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.465 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.466 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.466 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934065+0000", 2024-04-12T11:45:00.466 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2140000000000002e-06 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.427765+0000", 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4936998859999999 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.427915+0000", 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014997 2024-04-12T11:45:00.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.468 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.468 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:45:00.468 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.428074+0000", 2024-04-12T11:45:00.468 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015929400000000001 2024-04-12T11:45:00.468 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.468 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.468 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432270+0000", 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0041953989999999998 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432297+0000", 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6922000000000001e-05 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.469 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.447493+0000", 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015195818 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.447562+0000", 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9117999999999998e-05 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.449843+0000", 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002280907 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.470 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.471 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.471 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.471 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91900 7.1 7:8d6ac7c8:::benchmark_data_smithi029_161157_object91899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:45:00.471 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:20.979397+0000", 2024-04-12T11:45:00.471 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 399.46316747499998, 2024-04-12T11:45:00.471 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.504759467, 2024-04-12T11:45:00.471 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.471 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.472 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.472 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.472 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:45:00.472 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:45:00.472 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91900 2024-04-12T11:45:00.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.472 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.473 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.473 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979397+0000", 2024-04-12T11:45:00.473 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.473 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.474 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.474 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979397+0000", 2024-04-12T11:45:00.474 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.474 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.474 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.475 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.475 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979399+0000", 2024-04-12T11:45:00.475 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.046e-06 2024-04-12T11:45:00.475 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.475 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.475 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979464+0000", 2024-04-12T11:45:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4839999999999996e-05 2024-04-12T11:45:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979488+0000", 2024-04-12T11:45:00.476 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3971000000000001e-05 2024-04-12T11:45:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979499+0000", 2024-04-12T11:45:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1527e-05 2024-04-12T11:45:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.477 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.478 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.478 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979583+0000", 2024-04-12T11:45:00.478 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3651000000000003e-05 2024-04-12T11:45:00.478 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:20.979599+0000", 2024-04-12T11:45:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5883999999999999e-05 2024-04-12T11:45:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.479 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934095+0000", 2024-04-12T11:45:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.95449584399999998 2024-04-12T11:45:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.480 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934104+0000", 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.935e-06 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.430164+0000", 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.496060712 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.481 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.430308+0000", 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014382800000000001 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.430440+0000", 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013224299999999999 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.434349+0000", 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0039086800000000003 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.482 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.436571+0000", 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002221771 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.481423+0000", 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.044852114999999998 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.481466+0000", 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3538000000000002e-05 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.483 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.484156+0000", 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0026898820000000002 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91933 7.1 7:876a3b0c:::benchmark_data_smithi029_161157_object91932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:21.013528+0000", 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 399.42903580799998, 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4845660560000002, 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.484 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91933 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013528+0000", 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013528+0000", 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013530+0000", 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.333e-06 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013594+0000", 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3899999999999995e-05 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013605+0000", 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1736000000000001e-05 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013614+0000", 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1680000000000004e-06 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013667+0000", 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3594000000000002e-05 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.013677+0000", 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3479999999999993e-06 2024-04-12T11:45:00.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934131+0000", 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.92045452299999997 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934140+0000", 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.884e-06 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432328+0000", 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.498187623 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432421+0000", 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3357999999999996e-05 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.432559+0000", 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000137911 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.436613+0000", 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0040539169999999998 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.438759+0000", 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0021465730000000001 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.495581+0000", 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.056821266000000002 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.490 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.495624+0000", 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3303999999999997e-05 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.498095+0000", 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0024706179999999999 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91943 7.1 7:840e712a:::benchmark_data_smithi029_161157_object91942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:21.022248+0000", 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 399.42031580999998, 2024-04-12T11:45:00.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4783233920000001, 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91943 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022248+0000", 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022248+0000", 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022250+0000", 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.172e-06 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022295+0000", 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5086999999999997e-05 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022306+0000", 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1361e-05 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022312+0000", 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1990000000000003e-06 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022355+0000", 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3071000000000001e-05 2024-04-12T11:45:00.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.022364+0000", 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.208e-06 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934169+0000", 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.91180522799999997 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934178+0000", 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7849999999999997e-06 2024-04-12T11:45:00.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.434370+0000", 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.500192212 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.434464+0000", 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3703999999999996e-05 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.434573+0000", 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000109305 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.496 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441132+0000", 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0065594959999999997 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441144+0000", 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1362999999999999e-05 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.498165+0000", 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.057021297999999998 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.498238+0000", 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2608999999999995e-05 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.500572+0000", 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023342940000000002 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.498 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91952 7.1 7:87628a27:::benchmark_data_smithi029_161157_object91951:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:21.031078+0000", 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 399.41148607899999, 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4732867770000002, 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91952 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.499 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031078+0000", 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031078+0000", 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031082+0000", 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1400000000000002e-06 2024-04-12T11:45:00.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031126+0000", 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4004000000000001e-05 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031143+0000", 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6473e-05 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.501 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031152+0000", 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0589999999999992e-06 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031209+0000", 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6789000000000001e-05 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.031228+0000", 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9326000000000001e-05 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.502 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934207+0000", 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.90297881000000002 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934226+0000", 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.914e-05 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.436644+0000", 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5024184229999999 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.503 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.436749+0000", 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010474 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.436857+0000", 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000107469 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441201+0000", 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0043440850000000001 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.444157+0000", 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0029562410000000001 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.502529+0000", 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.058372119 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.502572+0000", 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3195999999999998e-05 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.504365+0000", 2024-04-12T11:45:00.506 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001792763 2024-04-12T11:45:00.506 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.506 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.506 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.506 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.506 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91957 7.1 7:814dd879:::benchmark_data_smithi029_161157_object91956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:45:00.506 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:21.039025+0000", 2024-04-12T11:45:00.506 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 399.40353954699998, 2024-04-12T11:45:00.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:00.508 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4672004350000001, 2024-04-12T11:45:00.508 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.508 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.508 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.508 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.508 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91957 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039025+0000", 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039025+0000", 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039029+0000", 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2100000000000003e-06 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039074+0000", 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4944000000000002e-05 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039093+0000", 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9565e-05 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.510 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039104+0000", 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0574000000000001e-05 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039168+0000", 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3652999999999995e-05 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.039177+0000", 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0529999999999996e-06 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934258+0000", 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.89508132299999998 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934267+0000", 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9949999999999994e-06 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.438793+0000", 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5045254809999999 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.438898+0000", 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010555700000000001 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.439007+0000", 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000108614 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.444234+0000", 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0052272839999999996 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.513 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.447434+0000", 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0032003829999999998 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.504437+0000", 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.057002355999999997 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.504479+0000", 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2432000000000003e-05 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.514 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.506225+0000", 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001746011 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.44843.0:91980 7.1 7:8a33a954:::benchmark_data_smithi029_161157_object91979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:38:21.175961+0000", 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 399.26660285999998, 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3331854490000001, 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.515 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.44843", 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 91980 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.175961+0000", 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.175961+0000", 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.175963+0000", 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.03e-06 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.176020+0000", 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6162999999999997e-05 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.176046+0000", 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5987e-05 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.176060+0000", 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3899e-05 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.176133+0000", 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3001000000000001e-05 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.176143+0000", 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0210999999999999e-05 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.518 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934296+0000", 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.75815370999999998 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:21.934305+0000", 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2700000000000004e-06 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441249+0000", 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5069440429999998 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.519 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441417+0000", 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000168658 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.441604+0000", 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000186718 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.449895+0000", 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0082906639999999997 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.449907+0000", 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2516e-05 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.506306+0000", 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.056398252000000003 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.506375+0000", 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8980999999999996e-05 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.521 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:38:23.509147+0000", 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027723460000000002 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27471 8.b 8:d02dd3f5:::benchmark_data_smithi029_194418_object27470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:11.988879+0000", 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.453685729, 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0098002610000001, 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.522 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27471 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.988879+0000", 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.988879+0000", 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.988881+0000", 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9879999999999999e-06 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.989541+0000", 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00066067199999999995 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.989563+0000", 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1852e-05 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.989575+0000", 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1559000000000001e-05 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.989689+0000", 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000114862 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.989704+0000", 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4188999999999999e-05 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.994809+0000", 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0051048359999999 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.994924+0000", 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000115919 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.995030+0000", 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000106 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.526 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998437+0000", 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003406737 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998469+0000", 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.1661000000000001e-05 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998610+0000", 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014149200000000001 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998645+0000", 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4501000000000001e-05 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998679+0000", 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3992999999999998e-05 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27494 8.b 8:d360160f:::benchmark_data_smithi029_194418_object27493:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-04-12T11:45:00.528 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:12.001199+0000", 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.44136546, 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.99766434500000001, 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27494 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.001199+0000", 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.001199+0000", 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.001200+0000", 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.09e-07 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.001245+0000", 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5467999999999999e-05 2024-04-12T11:45:00.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.001258+0000", 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.327e-05 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.001265+0000", 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4949999999999996e-06 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.001428+0000", 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016303499999999999 2024-04-12T11:45:00.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.001439+0000", 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1489e-05 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.996189+0000", 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.994749247 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.996278+0000", 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9079000000000006e-05 2024-04-12T11:45:00.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.996363+0000", 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5453999999999999e-05 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998733+0000", 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023696450000000001 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998750+0000", 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6789000000000001e-05 2024-04-12T11:45:00.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998778+0000", 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.8699000000000001e-05 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998827+0000", 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8433000000000002e-05 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.534 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998863+0000", 2024-04-12T11:45:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6433000000000002e-05 2024-04-12T11:45:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.535 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27497 8.b 8:d20392c7:::benchmark_data_smithi029_194418_object27496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:12.003369+0000", 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.43919544800001, 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.99562686700000003, 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27497 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.536 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.003369+0000", 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.003369+0000", 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.003370+0000", 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.513e-06 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.003407+0000", 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6390000000000002e-05 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.003420+0000", 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3375e-05 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.003430+0000", 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4530000000000008e-06 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.003510+0000", 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0615e-05 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.003523+0000", 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2435e-05 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.996900+0000", 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.99337783000000002 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.539 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.997024+0000", 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012369 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.997115+0000", 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1188999999999997e-05 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998757+0000", 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0016413319999999999 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998903+0000", 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014617300000000001 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998946+0000", 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2713999999999998e-05 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998974+0000", 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.8229e-05 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998996+0000", 2024-04-12T11:45:00.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1929e-05 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27505 8.b 8:d6d90a10:::benchmark_data_smithi029_194418_object27504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:12.006539+0000", 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.436025274, 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.993353284, 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.542 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27505 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.006539+0000", 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.006539+0000", 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.543 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.006541+0000", 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6649999999999999e-06 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.006652+0000", 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011136100000000001 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.006667+0000", 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4479e-05 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.544 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.006675+0000", 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4020000000000002e-06 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.006775+0000", 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9772999999999995e-05 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.006787+0000", 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1987000000000001e-05 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.545 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.997744+0000", 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.99095778499999998 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.997820+0000", 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5797000000000006e-05 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.997906+0000", 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5482000000000006e-05 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.998913+0000", 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00100731 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.999741+0000", 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00082812300000000005 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.999787+0000", 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6285000000000002e-05 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.999845+0000", 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7747000000000001e-05 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.999892+0000", 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7088e-05 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27517 8.b 8:d5179181:::benchmark_data_smithi029_194418_object27516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-04-12T11:45:00.548 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:12.010994+0000", 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.431570123, 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.994255887, 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27517 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.010994+0000", 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.010994+0000", 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.010996+0000", 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9570000000000001e-06 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.011043+0000", 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6468999999999998e-05 2024-04-12T11:45:00.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.011063+0000", 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0117000000000001e-05 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.011074+0000", 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1489e-05 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.011159+0000", 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4909999999999998e-05 2024-04-12T11:45:00.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.011172+0000", 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2792000000000001e-05 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.999043+0000", 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98787158900000005 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.999126+0000", 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2919999999999999e-05 2024-04-12T11:45:00.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.999210+0000", 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4056999999999998e-05 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.001920+0000", 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027091609999999999 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.002889+0000", 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00096893900000000004 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005144+0000", 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0022549530000000001 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005199+0000", 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5794000000000001e-05 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005250+0000", 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0739999999999999e-05 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.554 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27526 8.b 8:d4fb5e4b:::benchmark_data_smithi029_194418_object27525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:12.015634+0000", 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.426929857, 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98983976699999998, 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27526 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.555 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.015634+0000", 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.015634+0000", 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.015636+0000", 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3200000000000001e-06 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.015676+0000", 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9983e-05 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.015687+0000", 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1558e-05 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.015695+0000", 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5989999999999996e-06 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.557 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.015753+0000", 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8459999999999999e-05 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.015761+0000", 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5719999999999997e-06 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.000000+0000", 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98423914800000001 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.000132+0000", 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013177499999999999 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.000277+0000", 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014499399999999999 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.002896+0000", 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0026190279999999998 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005304+0000", 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0024084890000000002 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005350+0000", 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5822000000000003e-05 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005406+0000", 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5927000000000002e-05 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.560 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005474+0000", 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8091999999999998e-05 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27530 8.b 8:d9cf16f2:::benchmark_data_smithi029_194418_object27529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:12.016353+0000", 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.426211006, 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98934882800000001, 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.561 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27530 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.016353+0000", 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.016353+0000", 2024-04-12T11:45:00.562 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.016354+0000", 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1880000000000001e-06 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.016410+0000", 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5250000000000001e-05 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.016428+0000", 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7892000000000001e-05 2024-04-12T11:45:00.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.016437+0000", 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.5680000000000005e-06 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.016495+0000", 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7822999999999998e-05 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.016506+0000", 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0654e-05 2024-04-12T11:45:00.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.000997+0000", 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98449157399999998 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.001122+0000", 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012525500000000001 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.001233+0000", 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011061399999999999 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005104+0000", 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0038708380000000001 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005529+0000", 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00042476500000000002 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005586+0000", 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6910999999999998e-05 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.566 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005642+0000", 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6663000000000003e-05 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005702+0000", 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9833000000000001e-05 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27531 8.b 8:d8efd6b7:::benchmark_data_smithi029_194418_object27530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:12.017516+0000", 2024-04-12T11:45:00.567 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.42504806399999, 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98838330900000004, 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27531 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.017516+0000", 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.568 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.017516+0000", 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.017517+0000", 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.254e-06 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.017548+0000", 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.0899999999999999e-05 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.017563+0000", 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5088e-05 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.017572+0000", 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4509999999999992e-06 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.017645+0000", 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3165999999999997e-05 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.570 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.017657+0000", 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1601e-05 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.001959+0000", 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98430267500000002 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.002080+0000", 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000120696 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.571 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.002245+0000", 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016458900000000001 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005265+0000", 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030208129999999998 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005755+0000", 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00048961299999999996 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005801+0000", 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6081000000000001e-05 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005859+0000", 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7938e-05 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005900+0000", 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0444000000000001e-05 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.573 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27537 8.b 8:d5999b76:::benchmark_data_smithi029_194418_object27536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:12.018712+0000", 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.423852364, 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98735966500000005, 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27537 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.574 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.018712+0000", 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.018712+0000", 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.018713+0000", 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7600000000000006e-07 2024-04-12T11:45:00.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.018770+0000", 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6959999999999997e-05 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.018783+0000", 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3042e-05 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.018792+0000", 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.5869999999999999e-06 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.018846+0000", 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.304e-05 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:12.018852+0000", 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7479999999999996e-06 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.577 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.002948+0000", 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98409543899999996 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.003076+0000", 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012789999999999999 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,2", 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.003196+0000", 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000120494 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.578 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005271+0000", 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0020744079999999998 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005940+0000", 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00066992699999999996 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.005988+0000", 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7645000000000002e-05 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.006037+0000", 2024-04-12T11:45:00.579 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8690999999999998e-05 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:13.006072+0000", 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4808000000000003e-05 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121584 8.2 8:44060fd6:::benchmark_data_smithi029_194418_object15274:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.398991+0000", 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 16.043573240000001, 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0786393190000001, 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.580 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121584 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.398991+0000", 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.581 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.398991+0000", 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.398993+0000", 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1129999999999999e-06 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399004+0000", 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0386e-05 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399030+0000", 2024-04-12T11:45:00.582 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6458999999999999e-05 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399045+0000", 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4905e-05 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.527862+0000", 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.12881675000000001 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.527871+0000", 2024-04-12T11:45:00.583 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1859999999999998e-06 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.464862+0000", 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.93699132699999998 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for missing object", 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.464989+0000", 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000126616 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.477560+0000", 2024-04-12T11:45:00.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012571438000000001 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.477630+0000", 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0139e-05 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121593 8.12 8:4a564882:::benchmark_data_smithi029_194418_object15283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.417299+0000", 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 16.025265633, 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.095640062, 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T11:45:00.585 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121593 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.417299+0000", 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.417299+0000", 2024-04-12T11:45:00.586 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.417302+0000", 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5300000000000001e-06 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.417316+0000", 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3375e-05 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.417349+0000", 2024-04-12T11:45:00.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3185000000000002e-05 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.417364+0000", 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5313000000000001e-05 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.527622+0000", 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.110257955 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:00.588 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.527630+0000", 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6399999999999997e-06 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.485799+0000", 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.95816962999999999 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.485929+0000", 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000129498 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,5", 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.486183+0000", 2024-04-12T11:45:00.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00025395700000000002 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.487933+0000", 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00174983 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.490685+0000", 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027525829999999999 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.512826+0000", 2024-04-12T11:45:00.590 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.022140842000000001 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.512886+0000", 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9567000000000001e-05 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.512939+0000", 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3156999999999999e-05 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:00.591 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:45:00.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:00.628 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:00.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:00.729 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:00.764 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:00.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:00.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:00.866 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:00.905 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:00.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:00.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:01.007 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:01.045 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:01.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:01.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:01.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:01.186 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:01.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:01.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:01.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:00 smithi087 ceph-mon[89014]: pgmap v1786: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 14508/244686 objects degraded (5.929%); 7647/244686 objects misplaced (3.125%); 7.5 MiB/s, 120 objects/s recovering 2024-04-12T11:45:01.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:00 smithi029 ceph-mon[97787]: pgmap v1786: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 14508/244686 objects degraded (5.929%); 7647/244686 objects misplaced (3.125%); 7.5 MiB/s, 120 objects/s recovering 2024-04-12T11:45:01.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:00 smithi029 ceph-mon[93650]: pgmap v1786: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 14508/244686 objects degraded (5.929%); 7647/244686 objects misplaced (3.125%); 7.5 MiB/s, 120 objects/s recovering 2024-04-12T11:45:01.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:01.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:01.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:01.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:01.419 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:01.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:01.459 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:01.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:01.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:01.584 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:01.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:01.677 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-04-12T11:45:01.685 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:01.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:01.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:01.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:01.861 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:01.918 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:01.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:01.982 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:45:02.019 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:02.043 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:02.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:02.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:02.145 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:02.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:02.189 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:02.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:02.290 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:02.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:02.328 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:02.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:02.402 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:45:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:45:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:45:02.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:02.466 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:02.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:02.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:02.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:02.591 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:02.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:02.692 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:02.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:02.717 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:02.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:02.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:02.849 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:02.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:02.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:02.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:02.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:02.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:02.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:03.091 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:03.122 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:03.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:03.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:03.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:02 smithi087 ceph-mon[89014]: pgmap v1787: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 14405/240750 objects degraded (5.983%); 7439/240750 objects misplaced (3.090%); 9.0 MiB/s, 144 objects/s recovering 2024-04-12T11:45:03.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:02 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 14405/240750 objects degraded (5.983%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:03.223 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:03.256 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:03.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:03.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:03.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:02 smithi029 ceph-mon[97787]: pgmap v1787: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 14405/240750 objects degraded (5.983%); 7439/240750 objects misplaced (3.090%); 9.0 MiB/s, 144 objects/s recovering 2024-04-12T11:45:03.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:02 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 14405/240750 objects degraded (5.983%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:03.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:02 smithi029 ceph-mon[93650]: pgmap v1787: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 14405/240750 objects degraded (5.983%); 7439/240750 objects misplaced (3.090%); 9.0 MiB/s, 144 objects/s recovering 2024-04-12T11:45:03.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:02 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 14405/240750 objects degraded (5.983%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:03.357 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:03.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:03.394 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:03.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:03.497 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:03.536 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:03.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:03.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:03.638 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:03.662 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:03.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:03.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:03.764 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:03.801 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:03.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:03.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:03.903 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:03.916 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:45:03.916 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:45:03.916 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.916 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:136367 8.d 8:b582f1bf:::benchmark_data_smithi029_194418_object30057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:45:03.916 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:03.881117+0000", 2024-04-12T11:45:03.916 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.031812249000000001, 2024-04-12T11:45:03.916 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.031847435, 2024-04-12T11:45:03.916 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:03.916 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 136367 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.881117+0000", 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.881117+0000", 2024-04-12T11:45:03.917 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.881116+0000", 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.881121+0000", 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0509999999999996e-06 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:03.918 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.881137+0000", 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6518999999999999e-05 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.881144+0000", 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0269999999999998e-06 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.911009+0000", 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.029864866 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.919 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.911178+0000", 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000169183 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.911352+0000", 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017367199999999999 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.920 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.921 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:136393 8.c e1202/1194 8:394be258:::benchmark_data_smithi029_194418_object30083:head v 1202'8425, mlcod=1179'994)", 2024-04-12T11:45:03.921 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:03.911688+0000", 2024-04-12T11:45:03.921 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.0012407189999999999, 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0013579550000000001, 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.911688+0000", 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:03.922 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.911688+0000", 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.911687+0000", 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.911699+0000", 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.136e-05 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.923 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.911710+0000", 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0763e-05 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.911717+0000", 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2799999999999998e-06 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.912388+0000", 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00067120899999999998 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.924 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.912410+0000", 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1793999999999998e-05 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:136385 8.18 8:1f0afadf:::benchmark_data_smithi029_194418_object30075:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:03.898581+0000", 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.014347976, 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014533931, 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:03.925 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 136385 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898581+0000", 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898581+0000", 2024-04-12T11:45:03.926 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.927 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.927 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.927 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:03.927 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898579+0000", 2024-04-12T11:45:03.927 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:45:03.927 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.927 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.927 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:03.927 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898585+0000", 2024-04-12T11:45:03.928 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7130000000000002e-06 2024-04-12T11:45:03.928 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.928 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.928 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:03.928 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898594+0000", 2024-04-12T11:45:03.928 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.003e-06 2024-04-12T11:45:03.929 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.929 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.929 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:03.929 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898601+0000", 2024-04-12T11:45:03.929 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6789999999999999e-06 2024-04-12T11:45:03.929 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.930 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.930 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:03.930 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.900320+0000", 2024-04-12T11:45:03.930 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0017187319999999999 2024-04-12T11:45:03.930 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.930 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.930 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:45:03.930 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.901024+0000", 2024-04-12T11:45:03.930 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00070446699999999996 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.909763+0000", 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0087390339999999997 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.909854+0000", 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0506000000000004e-05 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.931 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:45:03.932 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.910339+0000", 2024-04-12T11:45:03.932 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00048574300000000001 2024-04-12T11:45:03.932 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.932 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:03.932 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.932 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.932 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.932 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:136386 8.4 8:20c270b8:::benchmark_data_smithi029_194418_object30076:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:45:03.932 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:03.898644+0000", 2024-04-12T11:45:03.932 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.014284797, 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.01455649, 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 136386 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898644+0000", 2024-04-12T11:45:03.933 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.934 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.934 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.934 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:03.934 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898644+0000", 2024-04-12T11:45:03.934 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.934 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.934 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.934 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:03.934 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898644+0000", 2024-04-12T11:45:03.935 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:03.935 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.935 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.935 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:03.935 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898647+0000", 2024-04-12T11:45:03.935 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6169999999999998e-06 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898656+0000", 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9199999999999993e-06 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898661+0000", 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.152e-06 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.936 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.937 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:03.937 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898699+0000", 2024-04-12T11:45:03.937 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.7895000000000002e-05 2024-04-12T11:45:03.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.938 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.938 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:03.938 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.898833+0000", 2024-04-12T11:45:03.938 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000133449 2024-04-12T11:45:03.938 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.938 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.939 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:45:03.939 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.899368+0000", 2024-04-12T11:45:03.939 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00053493699999999998 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.902213+0000", 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002844994 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.902288+0000", 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5389000000000006e-05 2024-04-12T11:45:03.940 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:136359 8.d 8:bbf21d39:::benchmark_data_smithi029_194418_object30049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:03.875769+0000", 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.037160216000000003, 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.037526302999999997, 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:03.941 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 136359 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.875769+0000", 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.942 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.875769+0000", 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.875767+0000", 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:03.943 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.875774+0000", 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2169999999999997e-06 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.875785+0000", 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0949e-05 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.875793+0000", 2024-04-12T11:45:03.944 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.328e-06 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.895867+0000", 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.020073756000000002 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.896041+0000", 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000174842 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.946 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:45:03.946 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.896207+0000", 2024-04-12T11:45:03.947 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016610099999999999 2024-04-12T11:45:03.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.947 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.947 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:03.947 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.899928+0000", 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0037208779999999999 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.907745+0000", 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0078170070000000008 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.948 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:136360 8.d 8:b61ecbb1:::benchmark_data_smithi029_194418_object30050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:03.876442+0000", 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.036487230000000002, 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.036962656000000003, 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 136360 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.876442+0000", 2024-04-12T11:45:03.949 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.876442+0000", 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.876441+0000", 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.876447+0000", 2024-04-12T11:45:03.950 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0260000000000004e-06 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.876460+0000", 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3698e-05 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.876468+0000", 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0390000000000005e-06 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.899949+0000", 2024-04-12T11:45:03.951 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.023480253 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.900088+0000", 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000139851 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.900232+0000", 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014395200000000001 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:03.952 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.907734+0000", 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.007501059 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:03.907751+0000", 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7034e-05 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 6 2024-04-12T11:45:03.953 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:45:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:03.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:03.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:04.056 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:04.079 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:04.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:04.181 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:04.217 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:04.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:04.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:04.319 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:04.352 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:04.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:04.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:04.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:04.478 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:04.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:04.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:04.580 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:04.611 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:04.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:04.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:04.712 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:04.714 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-04-12T11:45:04.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:04.795 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:04.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:04.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:04.926 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:04.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:05.028 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:05.084 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:05.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:05.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:05.117 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:45:05.186 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:05.211 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:05.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:05.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:05.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:04 smithi087 ceph-mon[89014]: pgmap v1788: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 14245/233691 objects degraded (6.096%); 7203/233691 objects misplaced (3.082%); 10 MiB/s, 159 objects/s recovering 2024-04-12T11:45:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:04 smithi029 ceph-mon[97787]: pgmap v1788: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 14245/233691 objects degraded (6.096%); 7203/233691 objects misplaced (3.082%); 10 MiB/s, 159 objects/s recovering 2024-04-12T11:45:05.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:04 smithi029 ceph-mon[93650]: pgmap v1788: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 14245/233691 objects degraded (6.096%); 7203/233691 objects misplaced (3.082%); 10 MiB/s, 159 objects/s recovering 2024-04-12T11:45:05.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:45:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:45:04] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2024-04-12T11:45:05.313 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:05.351 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:05.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:05.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:05.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:05.490 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:05.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:05.592 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:05.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:05.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:05.724 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:05.749 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:05.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:05.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:05.888 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:05.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:05.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:05.990 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:06.023 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:06.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:06.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:06.149 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:06.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:06.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:06.250 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:06.281 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:06.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:06.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:06.383 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:06.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:06.426 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:06.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:06.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:06.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:06.551 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:06.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:06.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:06.688 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:06.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:06.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:06.790 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:06.824 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:06.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:06.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:06.926 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:06.963 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:06.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:06.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:07.065 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:07.100 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:07.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:07.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:07.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:07.229 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:07.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:07.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:07.331 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:07.356 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:07.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:07.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:07.458 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:07.493 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:07.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:07.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:07.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:07.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:07 smithi029 ceph-mon[97787]: pgmap v1789: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 14160/229071 objects degraded (6.181%); 6955/229071 objects misplaced (3.036%); 9.6 MiB/s, 153 objects/s recovering 2024-04-12T11:45:07.604 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:45:07.604 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:07.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:07 smithi029 ceph-mon[93650]: pgmap v1789: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 14160/229071 objects degraded (6.181%); 6955/229071 objects misplaced (3.036%); 9.6 MiB/s, 153 objects/s recovering 2024-04-12T11:45:07.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:45:07.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:07.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:07.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:07.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:07.683 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:07 smithi087 ceph-mon[89014]: pgmap v1789: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 14160/229071 objects degraded (6.181%); 6955/229071 objects misplaced (3.036%); 9.6 MiB/s, 153 objects/s recovering 2024-04-12T11:45:07.683 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:45:07.683 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:07.735 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:07.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:07.759 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:07.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:07.860 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:07.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:07.890 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:07.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:07.974 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:45:07.974 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:45:07.974 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:45:07.974 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:45:07.974 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:45:07.991 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:08.025 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:08.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:08.127 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:08.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:08.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:08.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:08.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:08.292 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:08.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:08.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:08.394 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:08.429 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:08.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:08.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:08.530 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:08.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:08.558 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:08.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:08.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:08.688 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 14160/229071 objects degraded (6.181%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:08.697 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:08.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:08.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:08.726 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-04-12T11:45:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 14160/229071 objects degraded (6.181%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 14160/229071 objects degraded (6.181%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:08.799 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:08.855 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:08.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:08.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:08.957 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:08.997 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:08.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:08.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:09.069 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:45:09.100 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:09.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:09.147 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:09.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:09.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:09.285 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:09.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:09.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:09.387 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:09.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:09.414 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:09.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:09 smithi087 ceph-mon[89014]: pgmap v1790: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 740 op/s; 13791/224922 objects degraded (6.131%); 6705/224922 objects misplaced (2.981%); 11 MiB/s, 172 objects/s recovering 2024-04-12T11:45:09.515 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:09.543 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:09.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:09.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:09.649 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:09.686 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:09.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:09.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:09 smithi029 ceph-mon[97787]: pgmap v1790: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 740 op/s; 13791/224922 objects degraded (6.131%); 6705/224922 objects misplaced (2.981%); 11 MiB/s, 172 objects/s recovering 2024-04-12T11:45:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:09 smithi029 ceph-mon[93650]: pgmap v1790: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 740 op/s; 13791/224922 objects degraded (6.131%); 6705/224922 objects misplaced (2.981%); 11 MiB/s, 172 objects/s recovering 2024-04-12T11:45:09.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:09.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:09.827 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:09.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:09.928 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:09.967 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:09.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:09.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:10.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:10.104 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:10.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:10.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:10.206 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:10.243 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:10.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:10.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:10.345 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:10.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:10.391 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:10.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:10.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:10.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:10.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:10.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:10.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:10.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:10.652 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:10.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:10.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:10.785 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:10.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:10.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:10.886 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:10.932 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:10.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:10.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:11.022 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27186 8.8 8:17183391:::benchmark_data_smithi029_194418_object27185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1183)", 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:10.973857+0000", 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 120.045342025, 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0065482020000001, 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.023 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27186 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.973857+0000", 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.973857+0000", 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.973856+0000", 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.973921+0000", 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5730999999999998e-05 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.973931+0000", 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0023999999999999e-05 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.973939+0000", 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2529999999999998e-06 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.973987+0000", 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.871e-05 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.974119+0000", 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000132032 2024-04-12T11:45:11.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,7", 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.974804+0000", 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00068437800000000003 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.976879+0000", 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0020753049999999999 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.977788+0000", 2024-04-12T11:45:11.027 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00090863499999999998 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.983521+0000", 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0057333810000000001 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.983532+0000", 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0987e-05 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.028 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.980314+0000", 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.996781743 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.980405+0000", 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0970000000000005e-05 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27217 8.8 8:1519c0c4:::benchmark_data_smithi029_194418_object27216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1184)", 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:10.989748+0000", 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 120.029450986, 2024-04-12T11:45:11.029 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.99829194200000004, 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27217 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.989748+0000", 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.030 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.989748+0000", 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.989747+0000", 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.989816+0000", 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9358999999999998e-05 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.031 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.989825+0000", 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7250000000000003e-06 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.989832+0000", 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7159999999999997e-06 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.990005+0000", 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017312299999999999 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.990019+0000", 2024-04-12T11:45:11.032 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.473e-05 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.980855+0000", 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.99083562800000002 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.981015+0000", 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015970599999999999 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,7", 2024-04-12T11:45:11.033 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.981162+0000", 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014749000000000001 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.984895+0000", 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0037324670000000002 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.984937+0000", 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2089000000000003e-05 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.034 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.987950+0000", 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030132269999999998 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.988003+0000", 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2998999999999999e-05 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.988040+0000", 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6591000000000003e-05 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.035 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:27240 8.8 8:1c8a2744:::benchmark_data_smithi029_194418_object27239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1184)", 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:10.998007+0000", 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 120.02119145100001, 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.99035096199999995, 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 27240 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.036 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.998007+0000", 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.998007+0000", 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.998007+0000", 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.037 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.998034+0000", 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.7540000000000001e-05 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.998041+0000", 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3319999999999997e-06 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.998046+0000", 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.271e-06 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.038 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.998285+0000", 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00023952500000000001 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:10.998292+0000", 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7510000000000002e-06 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.983071+0000", 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98477845100000005 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.983165+0000", 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4429e-05 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,7", 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.983269+0000", 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010432799999999999 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.987899+0000", 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0046300309999999997 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.988226+0000", 2024-04-12T11:45:11.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00032645900000000003 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.988276+0000", 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0327000000000001e-05 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.988323+0000", 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6625999999999997e-05 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.041 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:11.988358+0000", 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5333000000000002e-05 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86177 8.8 8:1a5dda29:::benchmark_data_smithi029_194418_object86176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.147500+0000", 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 75.871698519999995, 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.2084737479999998, 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.042 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86177 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.147500+0000", 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.043 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.044 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.147500+0000", 2024-04-12T11:45:11.044 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.045 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.045 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.045 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.147498+0000", 2024-04-12T11:45:11.045 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999976 2024-04-12T11:45:11.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.045 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.147555+0000", 2024-04-12T11:45:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7263999999999999e-05 2024-04-12T11:45:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.147568+0000", 2024-04-12T11:45:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2858e-05 2024-04-12T11:45:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.147578+0000", 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6309999999999998e-06 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.154241+0000", 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0066634859999999997 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.154253+0000", 2024-04-12T11:45:11.047 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1970999999999999e-05 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141158+0000", 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98690443999999999 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141175+0000", 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6977999999999999e-05 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.343354+0000", 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.202179296 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.343529+0000", 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000174752 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:45:11.049 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.343644+0000", 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011539700000000001 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.349778+0000", 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0061339819999999996 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.349815+0000", 2024-04-12T11:45:11.050 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.7150000000000002e-05 2024-04-12T11:45:11.051 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:11.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.052 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.353069+0000", 2024-04-12T11:45:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0032534640000000002 2024-04-12T11:45:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.353121+0000", 2024-04-12T11:45:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2673000000000002e-05 2024-04-12T11:45:11.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.355974+0000", 2024-04-12T11:45:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0028526390000000001 2024-04-12T11:45:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86197 8.8 8:1fec4c19:::benchmark_data_smithi029_194418_object86196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:45:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.176971+0000", 2024-04-12T11:45:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 75.842227264000002, 2024-04-12T11:45:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1808644720000001, 2024-04-12T11:45:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.055 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86197 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.176971+0000", 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.176971+0000", 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.056 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.176970+0000", 2024-04-12T11:45:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:45:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.177023+0000", 2024-04-12T11:45:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3797e-05 2024-04-12T11:45:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.057 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.177040+0000", 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6534000000000001e-05 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.177049+0000", 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9090000000000006e-06 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.177093+0000", 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4441999999999999e-05 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.177102+0000", 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3000000000000002e-06 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141202+0000", 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.96410006000000004 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141212+0000", 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0696000000000001e-05 2024-04-12T11:45:11.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.349850+0000", 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.208637234 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.350006+0000", 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015651200000000001 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.350185+0000", 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000178398 2024-04-12T11:45:11.060 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.356115+0000", 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0059302690000000002 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.356127+0000", 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2486999999999999e-05 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.061 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.356152+0000", 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4717999999999999e-05 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.356195+0000", 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2840999999999999e-05 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.357836+0000", 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0016410159999999999 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.062 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86198 8.d 8:b25429fd:::benchmark_data_smithi029_194418_object86197:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.182981+0000", 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 75.836218095000007, 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3781462019999999, 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86198 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.063 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.182981+0000", 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.182981+0000", 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.182980+0000", 2024-04-12T11:45:11.064 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183030+0000", 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0064e-05 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183043+0000", 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3103e-05 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183049+0000", 2024-04-12T11:45:11.065 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6030000000000001e-06 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183091+0000", 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2048000000000001e-05 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183100+0000", 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9330000000000007e-06 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.066 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.152079+0000", 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.96897820199999996 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.152095+0000", 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6198e-05 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.533373+0000", 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.381278574 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.067 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.533532+0000", 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000158362 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.533723+0000", 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019160499999999999 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.540263+0000", 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0065393789999999997 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.068 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.543593+0000", 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003329958 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.557370+0000", 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.013777571000000001 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.557438+0000", 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7574e-05 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.069 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.561127+0000", 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003689014 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86200 8.d 8:ba4ef213:::benchmark_data_smithi029_194418_object86199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.183347+0000", 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 75.835852079999995, 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3881285370000001, 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.070 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86200 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183347+0000", 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183347+0000", 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.071 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183346+0000", 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183411+0000", 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5055000000000006e-05 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183419+0000", 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6399999999999997e-06 2024-04-12T11:45:11.072 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183424+0000", 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4729999999999999e-06 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183469+0000", 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5092999999999997e-05 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.073 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.183481+0000", 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1454000000000001e-05 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.156530+0000", 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.97304958600000002 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.156544+0000", 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3558e-05 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.074 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.540303+0000", 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3837593109999999 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.540444+0000", 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000140879 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.540612+0000", 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016814400000000001 2024-04-12T11:45:11.075 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.550188+0000", 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0095763549999999999 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.550235+0000", 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6319000000000002e-05 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.567603+0000", 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.017368614000000001 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.076 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.567698+0000", 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.5074999999999994e-05 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.571475+0000", 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0037767040000000001 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.077 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86202 8.8 8:1a9d72ca:::benchmark_data_smithi029_194418_object86201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.190678+0000", 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 75.828520772000005, 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.176912067, 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86202 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.078 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.190678+0000", 2024-04-12T11:45:11.079 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.079 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.079 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.079 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.079 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.190678+0000", 2024-04-12T11:45:11.079 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.079 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.079 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.079 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.079 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.190676+0000", 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999976 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.190750+0000", 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4395999999999997e-05 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.190792+0000", 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1999000000000002e-05 2024-04-12T11:45:11.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.190817+0000", 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4995999999999999e-05 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.190917+0000", 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000100041 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.081 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.190939+0000", 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.2272999999999998e-05 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141256+0000", 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.95031685700000001 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.082 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141267+0000", 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0505e-05 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.357884+0000", 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.216617609 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.358045+0000", 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016023400000000001 2024-04-12T11:45:11.083 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.358244+0000", 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019939099999999999 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.361347+0000", 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003103392 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.364419+0000", 2024-04-12T11:45:11.084 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003071776 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.364456+0000", 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6893000000000001e-05 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.364511+0000", 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5420999999999997e-05 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.085 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.367590+0000", 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030784879999999999 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86210 8.8 8:1ca30b57:::benchmark_data_smithi029_194418_object86209:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.201257+0000", 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 75.817941212999997, 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1772436810000002, 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.086 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86210 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.201257+0000", 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.087 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.201257+0000", 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.201257+0000", 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.088 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.201286+0000", 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.9692e-05 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.201295+0000", 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7960000000000001e-06 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.089 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.201300+0000", 2024-04-12T11:45:11.090 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2050000000000001e-06 2024-04-12T11:45:11.090 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.090 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.090 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.090 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.201347+0000", 2024-04-12T11:45:11.090 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6456999999999999e-05 2024-04-12T11:45:11.090 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.090 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.090 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.201358+0000", 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1659e-05 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141293+0000", 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.93993494499999997 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.091 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141303+0000", 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8859999999999999e-06 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.361389+0000", 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2200858530000001 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.361497+0000", 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000108435 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:45:11.092 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.361611+0000", 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011348100000000001 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.367611+0000", 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.006000371 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.367655+0000", 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3690999999999999e-05 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.093 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.375398+0000", 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0077430329999999999 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.375469+0000", 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1234000000000003e-05 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.378501+0000", 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030318400000000001 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.094 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86212 8.d 8:b0827921:::benchmark_data_smithi029_194418_object86211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.203245+0000", 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 75.815953313999998, 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3797950590000001, 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86212 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.095 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.203245+0000", 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.203245+0000", 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.203245+0000", 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.096 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.203313+0000", 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7787999999999995e-05 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.203321+0000", 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5210000000000002e-06 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.203326+0000", 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.972e-06 2024-04-12T11:45:11.097 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.203376+0000", 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9499000000000002e-05 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.203386+0000", 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0654e-05 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.156705+0000", 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.953318416 2024-04-12T11:45:11.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.156716+0000", 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.137e-05 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.543627+0000", 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3869114339999999 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.543803+0000", 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000175501 2024-04-12T11:45:11.099 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.543984+0000", 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018124800000000001 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.550243+0000", 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.006258614 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.557200+0000", 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0069574600000000004 2024-04-12T11:45:11.100 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.579581+0000", 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.022380595 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.579669+0000", 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7942000000000006e-05 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.583040+0000", 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003371621 2024-04-12T11:45:11.101 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86213 8.d 8:b8d11e0d:::benchmark_data_smithi029_194418_object86212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.206047+0000", 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 75.813151395000006, 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3806205500000002, 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.102 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86213 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.206047+0000", 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.206047+0000", 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.103 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.206047+0000", 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.206100+0000", 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3409000000000003e-05 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.206109+0000", 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6470000000000004e-06 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.104 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.206114+0000", 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7820000000000001e-06 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.206179+0000", 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5055999999999995e-05 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.206190+0000", 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1342000000000001e-05 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.105 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.156744+0000", 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.95055443299999998 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.156754+0000", 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3300000000000005e-06 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.546979+0000", 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3902250949999999 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.106 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.547153+0000", 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017396100000000001 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.547341+0000", 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018792 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.557140+0000", 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0097991859999999997 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.107 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.557255+0000", 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000115612 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.583140+0000", 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.025884698000000001 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.583241+0000", 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010070599999999999 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.586668+0000", 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034269349999999999 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121576 8.18 8:1d7d016d:::benchmark_data_smithi029_194418_object15266:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.399035+0000", 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 26.620164109000001, 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0988020039999999, 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:45:11.109 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121576 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399035+0000", 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399035+0000", 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399033+0000", 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399042+0000", 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4490000000000005e-06 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.111 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399065+0000", 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3108999999999998e-05 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399074+0000", 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2990000000000002e-06 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.670921+0000", 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.271846221 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.112 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.113 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.113 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671080+0000", 2024-04-12T11:45:11.113 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000159576 2024-04-12T11:45:11.113 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.113 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.113 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.113 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.497759+0000", 2024-04-12T11:45:11.113 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.826679198 2024-04-12T11:45:11.113 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.113 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.497837+0000", 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.7175000000000006e-05 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121578 8.10 8:084f0e2a:::benchmark_data_smithi029_194418_object15268:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.399081+0000", 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 26.620117203, 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.106838121, 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.114 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121578 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399081+0000", 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.115 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399081+0000", 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399081+0000", 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399084+0000", 2024-04-12T11:45:11.116 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.9809999999999999e-06 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399090+0000", 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.541e-06 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399094+0000", 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5940000000000004e-06 2024-04-12T11:45:11.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.670983+0000", 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27188868300000002 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.670992+0000", 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6059999999999992e-06 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.498931+0000", 2024-04-12T11:45:11.118 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.82793838600000003 2024-04-12T11:45:11.119 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.119 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.119 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for missing object", 2024-04-12T11:45:11.119 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499038+0000", 2024-04-12T11:45:11.119 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000106881 2024-04-12T11:45:11.119 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.119 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.119 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.119 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.505870+0000", 2024-04-12T11:45:11.119 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0068320639999999997 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.505920+0000", 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9781999999999999e-05 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.120 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121582 8.0 8:0510f3a3:::benchmark_data_smithi029_194418_object15272:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.399098+0000", 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 26.620101073000001, 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0998131179999999, 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121582 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399098+0000", 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399098+0000", 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399097+0000", 2024-04-12T11:45:11.123 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:11.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.123 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.123 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.123 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399099+0000", 2024-04-12T11:45:11.123 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.086e-06 2024-04-12T11:45:11.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.123 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.123 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.124 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399105+0000", 2024-04-12T11:45:11.124 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1619999999999999e-06 2024-04-12T11:45:11.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.124 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.124 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.125 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399109+0000", 2024-04-12T11:45:11.125 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3599999999999998e-06 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671018+0000", 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27190955900000002 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671030+0000", 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1755e-05 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.126 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.445095+0000", 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.77406526799999997 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for missing object", 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.445229+0000", 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000133368 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.498850+0000", 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.053620909000000001 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.498911+0000", 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0937000000000003e-05 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121583 8.8 8:12cc2f79:::benchmark_data_smithi029_194418_object15273:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.399112+0000", 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 26.62008651, 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0449477219999999, 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.128 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121583 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399112+0000", 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399112+0000", 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.129 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399112+0000", 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399114+0000", 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9700000000000002e-06 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399119+0000", 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2179999999999998e-06 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.130 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399123+0000", 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1420000000000003e-06 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671197+0000", 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27207400700000001 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671228+0000", 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.1222e-05 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.131 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.395226+0000", 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.72399789199999998 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.395242+0000", 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6022e-05 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.416023+0000", 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.020780601 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.132 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.416171+0000", 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000148016 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.416601+0000", 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00042996700000000001 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.439114+0000", 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.022512972999999999 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.133 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.439156+0000", 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1631999999999997e-05 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.442719+0000", 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0035630050000000002 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.442777+0000", 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8477999999999998e-05 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.134 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.444060+0000", 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0012828360000000001 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121585 8.14 8:28973092:::benchmark_data_smithi029_194418_object15275:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.399126+0000", 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 26.620072251, 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0818906880000001, 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.135 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121585 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399126+0000", 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399126+0000", 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399126+0000", 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399128+0000", 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.934e-06 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399133+0000", 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0159999999999999e-06 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.137 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399137+0000", 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.031e-06 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.593770+0000", 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.194633167 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.593786+0000", 2024-04-12T11:45:11.138 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5892999999999998e-05 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.480915+0000", 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.88712889100000003 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.481017+0000", 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010200999999999999 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.139 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121586 8.10 8:0aa62305:::benchmark_data_smithi029_194418_object15276:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.399141+0000", 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 26.620057790000001, 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.106455145, 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121586 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399141+0000", 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399141+0000", 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399141+0000", 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.141 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399142+0000", 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.796e-06 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399147+0000", 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0490000000000003e-06 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399152+0000", 2024-04-12T11:45:11.142 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0840000000000002e-06 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671108+0000", 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27195625499999998 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671114+0000", 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3090000000000001e-06 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.143 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499065+0000", 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.82795127000000002 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499162+0000", 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6533000000000006e-05 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499420+0000", 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00025791200000000001 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.144 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.501048+0000", 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0016276769999999999 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.501056+0000", 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7320000000000004e-06 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.505424+0000", 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0043680480000000002 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.145 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.505481+0000", 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6601000000000002e-05 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.505596+0000", 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011513799999999999 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121587 8.18 8:1c51c4e9:::benchmark_data_smithi029_194418_object15277:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:11.146 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.399155+0000", 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 26.620043991999999, 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1178041300000001, 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121587 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.147 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399155+0000", 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399155+0000", 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399154+0000", 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.148 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.149 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.149 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399156+0000", 2024-04-12T11:45:11.149 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7940000000000001e-06 2024-04-12T11:45:11.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.149 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.149 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.149 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399161+0000", 2024-04-12T11:45:11.149 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0019999999999997e-06 2024-04-12T11:45:11.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.149 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399165+0000", 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0099999999999997e-06 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671135+0000", 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27197022599999998 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671141+0000", 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6210000000000001e-06 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.150 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.498602+0000", 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.82746076800000001 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.498747+0000", 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014516799999999999 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499085+0000", 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00033804899999999999 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.502948+0000", 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0038633209999999999 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.505398+0000", 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0024498060000000001 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.515177+0000", 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0097784619999999999 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.515255+0000", 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8028999999999994e-05 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.516959+0000", 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0017041319999999999 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.153 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121591 8.0 8:06671471:::benchmark_data_smithi029_194418_object15281:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.399168+0000", 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 26.620030401000001, 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.099381768, 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121591 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.154 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399168+0000", 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399168+0000", 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.155 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399168+0000", 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399170+0000", 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7999999999999999e-06 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399175+0000", 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9660000000000004e-06 2024-04-12T11:45:11.156 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399179+0000", 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.092e-06 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671169+0000", 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27199008600000002 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.157 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.158 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.158 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.671184+0000", 2024-04-12T11:45:11.158 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.47e-05 2024-04-12T11:45:11.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.158 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.158 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.445256+0000", 2024-04-12T11:45:11.158 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.77407232000000004 2024-04-12T11:45:11.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.445377+0000", 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000121032 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.445786+0000", 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00040886699999999999 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.159 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.460827+0000", 2024-04-12T11:45:11.160 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015041407999999999 2024-04-12T11:45:11.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.160 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.160 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.462434+0000", 2024-04-12T11:45:11.160 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0016066450000000001 2024-04-12T11:45:11.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.160 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.160 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.498394+0000", 2024-04-12T11:45:11.161 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.035960559000000003 2024-04-12T11:45:11.161 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.161 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.161 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.161 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.498448+0000", 2024-04-12T11:45:11.161 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3266000000000001e-05 2024-04-12T11:45:11.161 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.161 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.161 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.498550+0000", 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000102277 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121592 8.4 8:200b2e79:::benchmark_data_smithi029_194418_object15282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.413256+0000", 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 26.605942275, 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0681655729999999, 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:11.162 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121592 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.413256+0000", 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.413256+0000", 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.163 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.413254+0000", 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999976 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.413267+0000", 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2945000000000001e-05 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.413293+0000", 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6508000000000001e-05 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.164 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.413307+0000", 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4141000000000001e-05 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.587248+0000", 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.173940493 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.587261+0000", 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2675e-05 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.165 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.426974+0000", 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.83971336200000002 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.427106+0000", 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013207200000000001 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.427446+0000", 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00034024199999999999 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.166 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.429148+0000", 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0017016760000000001 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.448275+0000", 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.019126871 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.481342+0000", 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.033067521000000002 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.167 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.481383+0000", 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0745999999999999e-05 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.481422+0000", 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8886e-05 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:11.168 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:45:11.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:11.170 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:11.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:11.271 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:11.308 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:11.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:11.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:11.410 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:11.457 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:11.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:11.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:11 smithi087 ceph-mon[89014]: pgmap v1791: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 13791/222231 objects degraded (6.206%); 6705/222231 objects misplaced (3.017%); 8.6 MiB/s, 138 objects/s recovering 2024-04-12T11:45:11.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:11.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:11.602 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:11.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:11.703 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:11.741 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:11.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:11.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:11.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:11 smithi029 ceph-mon[97787]: pgmap v1791: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 13791/222231 objects degraded (6.206%); 6705/222231 objects misplaced (3.017%); 8.6 MiB/s, 138 objects/s recovering 2024-04-12T11:45:11.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:11 smithi029 ceph-mon[93650]: pgmap v1791: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 13791/222231 objects degraded (6.206%); 6705/222231 objects misplaced (3.017%); 8.6 MiB/s, 138 objects/s recovering 2024-04-12T11:45:11.843 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:11.896 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:11.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:11.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:11.998 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:12.036 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:12.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:12.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:12.111 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:45:12.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:12.173 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:12.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:12.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:12.274 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:12.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-04-12T11:45:12.327 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:12.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:12.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:12.429 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:12.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:12.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:12.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:12.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:12 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:45:12.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:12 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:45:12.585 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:12.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:12.636 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:12.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:12.710 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:45:12.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:12 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:45:12.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:45:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:45:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:45:12.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:12.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:12.772 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:12.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:12.874 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:12.916 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:12.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:12.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:13.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:13.051 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:13.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:13.153 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:13.189 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:13.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:13.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:13.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:13.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:13.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:13.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:13.430 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:13.463 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:13.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:13.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:13.469 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:45:13.509 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:45:13.509 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:13 smithi029 ceph-mon[97787]: pgmap v1792: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 736 op/s; 13654/218148 objects degraded (6.259%); 6489/218148 objects misplaced (2.975%); 10 MiB/s, 167 objects/s recovering 2024-04-12T11:45:13.510 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13654/218148 objects degraded (6.259%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:13.510 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:13 smithi029 ceph-mon[93650]: pgmap v1792: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 736 op/s; 13654/218148 objects degraded (6.259%); 6489/218148 objects misplaced (2.975%); 10 MiB/s, 167 objects/s recovering 2024-04-12T11:45:13.510 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13654/218148 objects degraded (6.259%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:13.544 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:45:13.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:13.583 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:45:13.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:13.602 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:13.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:13.622 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:45:13.667 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:45:13.684 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:13 smithi087 ceph-mon[89014]: pgmap v1792: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 736 op/s; 13654/218148 objects degraded (6.259%); 6489/218148 objects misplaced (2.975%); 10 MiB/s, 167 objects/s recovering 2024-04-12T11:45:13.684 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13654/218148 objects degraded (6.259%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:13.694 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:45:13.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:13.726 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:45:13.731 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:45:13.731 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:45:13.731 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:45:13.731 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:45:13.731 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:45:13.731 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:45:13.732 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:45:13.732 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:45:13.746 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:13.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:13.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:13.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:13.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:13.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:13.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:45:13.983 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:14.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:14.020 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:14.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:14.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:14.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:14.158 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:14.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:14.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:14.297 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:14.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:14.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:14.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:14.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:14.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:14.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:14.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:14.575 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:14.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:14.677 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:14.681 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:14 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3201749762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:45:14.720 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:14.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:14.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:14 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3201749762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:45:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:14 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3201749762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:45:14.822 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:14.857 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:14.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:14.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:14.995 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:14.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:14.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:15.097 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:15.121 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:15.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:15.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:15.223 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:15.248 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:45:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:45:14] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2024-04-12T11:45:15.259 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:15.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:15.361 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:15.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:15.386 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:15.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:15.487 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:15.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:15 smithi029 ceph-mon[93650]: pgmap v1793: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 840 op/s; 13207/210498 objects degraded (6.274%); 6239/210498 objects misplaced (2.964%); 12 MiB/s, 199 objects/s recovering 2024-04-12T11:45:15.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:15 smithi029 ceph-mon[97787]: pgmap v1793: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 840 op/s; 13207/210498 objects degraded (6.274%); 6239/210498 objects misplaced (2.964%); 12 MiB/s, 199 objects/s recovering 2024-04-12T11:45:15.522 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:15.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:15.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:15.624 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:15.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:15.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:15.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:15 smithi087 ceph-mon[89014]: pgmap v1793: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 840 op/s; 13207/210498 objects degraded (6.274%); 6239/210498 objects misplaced (2.964%); 12 MiB/s, 199 objects/s recovering 2024-04-12T11:45:15.753 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:15.788 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:15.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:15.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:15.890 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:45:15.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:15.968 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:15.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:15.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:16.070 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:16.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:16.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:16.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:16.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:16.230 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:16.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:16.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:16.363 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:16.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:16.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:16.373 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.3. 2024-04-12T11:45:16.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:16 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/601418566' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:45:16.374 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:16 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:45:16.374 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:16 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T11:45:16.278+0000 7fae24212700 -1 mon.a@0(leader).osd e1202 definitely_dead 0 2024-04-12T11:45:16.374 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:16 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/601418566' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:45:16.374 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:16 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:45:16.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:16.489 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:16.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:16.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:16.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:16.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:16.616 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:16.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:16 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/601418566' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:45:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:16 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:45:16.717 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:16.744 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:16.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:16.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:16.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:16.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:16.870 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:16.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:16.971 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:16.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:16.997 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:16.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:17.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:17.121 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:17.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:17.223 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:17.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:17.248 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:17.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:17.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:45:17.349 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:17.380 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:17.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:17.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:17.483 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:17.506 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:17.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:17.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:17.574 INFO:tasks.thrashosds.thrasher:Added osd 3 2024-04-12T11:45:17.609 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:17.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:17.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:17.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:17 smithi087 ceph-mon[89014]: pgmap v1794: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 13058/205317 objects degraded (6.360%); 5975/205317 objects misplaced (2.910%); 13 MiB/s, 201 objects/s recovering 2024-04-12T11:45:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:17 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-04-12T11:45:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:17 smithi087 ceph-mon[89014]: osdmap e1203: 8 total, 8 up, 7 in 2024-04-12T11:45:17.735 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:17.755 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:17 smithi029 ceph-mon[97787]: pgmap v1794: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 13058/205317 objects degraded (6.360%); 5975/205317 objects misplaced (2.910%); 13 MiB/s, 201 objects/s recovering 2024-04-12T11:45:17.755 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:17 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-04-12T11:45:17.755 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:17 smithi029 ceph-mon[97787]: osdmap e1203: 8 total, 8 up, 7 in 2024-04-12T11:45:17.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:17 smithi029 ceph-mon[93650]: pgmap v1794: 99 pgs: 6 active+recovery_wait+degraded, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 787 op/s; 13058/205317 objects degraded (6.360%); 5975/205317 objects misplaced (2.910%); 13 MiB/s, 201 objects/s recovering 2024-04-12T11:45:17.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:17 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-04-12T11:45:17.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:17 smithi029 ceph-mon[93650]: osdmap e1203: 8 total, 8 up, 7 in 2024-04-12T11:45:17.758 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:17.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:17.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:17.860 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:17.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:17.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:17.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:17.990 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:18.014 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:18.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:18.116 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:18.140 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:18.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:18.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:18.242 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:18.268 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:18.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:18.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:18.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:18.415 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:18.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:18.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:18.517 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:18.541 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:18.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:18.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:18.643 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:18.668 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:18.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:18.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:18.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:18 smithi029 ceph-mon[97787]: osdmap e1204: 8 total, 8 up, 7 in 2024-04-12T11:45:18.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13058/205317 objects degraded (6.360%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:18.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:18 smithi029 ceph-mon[97787]: osdmap e1205: 8 total, 8 up, 7 in 2024-04-12T11:45:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:18 smithi029 ceph-mon[93650]: osdmap e1204: 8 total, 8 up, 7 in 2024-04-12T11:45:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13058/205317 objects degraded (6.360%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:18 smithi029 ceph-mon[93650]: osdmap e1205: 8 total, 8 up, 7 in 2024-04-12T11:45:18.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:18 smithi087 ceph-mon[89014]: osdmap e1204: 8 total, 8 up, 7 in 2024-04-12T11:45:18.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13058/205317 objects degraded (6.360%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:18.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:18 smithi087 ceph-mon[89014]: osdmap e1205: 8 total, 8 up, 7 in 2024-04-12T11:45:18.770 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:18.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:18.827 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:18.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:18.928 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:18.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:18.964 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:18.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:19.065 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:19.105 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:19.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:19.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:19.207 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:19.247 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:19.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:19.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:19.349 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:19.389 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:19.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:19.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:19.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:19.533 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:19.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:19.550 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:45:19.596 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:45:19.597 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:45:19.597 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:19 smithi087 ceph-mon[89014]: pgmap v1797: 99 pgs: 2 remapped+peering, 3 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 829 op/s; 10831/202323 objects degraded (5.353%); 4139/202323 objects misplaced (2.046%); 17 MiB/s, 278 objects/s recovering 2024-04-12T11:45:19.597 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:19 smithi087 ceph-mon[89014]: 8.8 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8213 2024-04-12T11:45:19.597 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:19 smithi087 ceph-mon[89014]: 8.10 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8250 2024-04-12T11:45:19.597 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:19 smithi087 ceph-mon[89014]: 8.b starting backfill to osd.3 from (0'0,0'0] MAX to 1202'9412 2024-04-12T11:45:19.597 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:19 smithi087 ceph-mon[89014]: 8.f starting backfill to osd.3 from (0'0,0'0] MAX to 1202'9261 2024-04-12T11:45:19.597 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:19 smithi087 ceph-mon[89014]: 8.0 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8264 2024-04-12T11:45:19.598 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:19 smithi087 ceph-mon[89014]: 8.c starting backfill to osd.3 from (0'0,0'0] MAX to 1202'9008 2024-04-12T11:45:19.598 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:19 smithi087 ceph-mon[89014]: 8.18 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8196 2024-04-12T11:45:19.598 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:19 smithi087 ceph-mon[89014]: 8.17 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8218 2024-04-12T11:45:19.598 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:19 smithi087 ceph-mon[89014]: 8.7 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8286 2024-04-12T11:45:19.634 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:19.666 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:19.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:19.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:19.712 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[93650]: pgmap v1797: 99 pgs: 2 remapped+peering, 3 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 829 op/s; 10831/202323 objects degraded (5.353%); 4139/202323 objects misplaced (2.046%); 17 MiB/s, 278 objects/s recovering 2024-04-12T11:45:19.712 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[93650]: 8.8 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8213 2024-04-12T11:45:19.712 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[93650]: 8.10 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8250 2024-04-12T11:45:19.712 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[93650]: 8.b starting backfill to osd.3 from (0'0,0'0] MAX to 1202'9412 2024-04-12T11:45:19.712 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[93650]: 8.f starting backfill to osd.3 from (0'0,0'0] MAX to 1202'9261 2024-04-12T11:45:19.712 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[93650]: 8.0 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8264 2024-04-12T11:45:19.712 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[93650]: 8.c starting backfill to osd.3 from (0'0,0'0] MAX to 1202'9008 2024-04-12T11:45:19.713 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[93650]: 8.18 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8196 2024-04-12T11:45:19.713 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[93650]: 8.17 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8218 2024-04-12T11:45:19.713 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[93650]: 8.7 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8286 2024-04-12T11:45:19.713 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[97787]: pgmap v1797: 99 pgs: 2 remapped+peering, 3 active+recovery_wait+degraded, 87 active+clean, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 829 op/s; 10831/202323 objects degraded (5.353%); 4139/202323 objects misplaced (2.046%); 17 MiB/s, 278 objects/s recovering 2024-04-12T11:45:19.713 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[97787]: 8.8 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8213 2024-04-12T11:45:19.713 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[97787]: 8.10 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8250 2024-04-12T11:45:19.713 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[97787]: 8.b starting backfill to osd.3 from (0'0,0'0] MAX to 1202'9412 2024-04-12T11:45:19.713 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[97787]: 8.f starting backfill to osd.3 from (0'0,0'0] MAX to 1202'9261 2024-04-12T11:45:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[97787]: 8.0 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8264 2024-04-12T11:45:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[97787]: 8.c starting backfill to osd.3 from (0'0,0'0] MAX to 1202'9008 2024-04-12T11:45:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[97787]: 8.18 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8196 2024-04-12T11:45:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[97787]: 8.17 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8218 2024-04-12T11:45:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:19 smithi029 ceph-mon[97787]: 8.7 starting backfill to osd.3 from (0'0,0'0] MAX to 1202'8286 2024-04-12T11:45:19.769 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:19.816 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:19.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:19.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:19.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:19.940 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:19.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:19.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:20.042 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:20.078 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:20.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:20.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:20.180 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:20.203 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:20.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:20.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:20.341 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:20.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:20.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:20.405 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:45:20.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:20.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:20.477 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:20.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:20.578 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:20.594 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:20 smithi087 ceph-mon[89014]: pgmap v1799: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 3 active+recovery_wait+degraded, 83 active+clean, 5 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 624 op/s; 10831/199251 objects degraded (5.436%); 20504/199251 objects misplaced (10.291%); 13 MiB/s, 205 objects/s recovering 2024-04-12T11:45:20.594 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:20 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1666817460' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:45:20.617 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:20.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:20.632 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-04-12T11:45:20.719 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:20.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:20.842 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:20.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:20.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:20 smithi029 ceph-mon[93650]: pgmap v1799: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 3 active+recovery_wait+degraded, 83 active+clean, 5 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 624 op/s; 10831/199251 objects degraded (5.436%); 20504/199251 objects misplaced (10.291%); 13 MiB/s, 205 objects/s recovering 2024-04-12T11:45:20.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:20 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1666817460' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:45:20.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:20 smithi029 ceph-mon[97787]: pgmap v1799: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 3 active+recovery_wait+degraded, 83 active+clean, 5 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 624 op/s; 10831/199251 objects degraded (5.436%); 20504/199251 objects misplaced (10.291%); 13 MiB/s, 205 objects/s recovering 2024-04-12T11:45:20.845 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:20 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1666817460' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:45:20.943 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:20.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:20.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:21.070 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:21.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:21.118 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:21.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:21.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:21.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:21.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:21.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:21.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:21.382 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:21.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:21.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:21.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:21.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:21.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:21.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:21.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:21.660 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:21.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:21 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:45:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:21 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1666817460' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:45:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:21 smithi087 ceph-mon[89014]: osdmap e1206: 8 total, 8 up, 7 in 2024-04-12T11:45:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:21.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:21 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:45:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:21 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1666817460' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:45:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:21 smithi029 ceph-mon[97787]: osdmap e1206: 8 total, 8 up, 7 in 2024-04-12T11:45:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:21 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:45:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:21 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1666817460' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:45:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:21 smithi029 ceph-mon[93650]: osdmap e1206: 8 total, 8 up, 7 in 2024-04-12T11:45:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:21.800 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:21.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:21.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:21.934 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:21.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:21.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:22.036 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:22.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:22.082 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:22.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:22.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:22.223 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:22.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:22.325 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:22.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:22.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:22.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:22.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:22.504 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:22.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:22.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:22 smithi029 ceph-mon[97787]: pgmap v1801: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 11686/197682 objects degraded (5.912%); 18698/197682 objects misplaced (9.459%); 15 MiB/s, 237 objects/s recovering 2024-04-12T11:45:22.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:22 smithi029 ceph-mon[93650]: pgmap v1801: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 11686/197682 objects degraded (5.912%); 18698/197682 objects misplaced (9.459%); 15 MiB/s, 237 objects/s recovering 2024-04-12T11:45:22.575 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 3] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:45:22.575 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:45:22.575 INFO:tasks.thrashosds.thrasher:Adding osd 7 2024-04-12T11:45:22.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd in 7 2024-04-12T11:45:22.605 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:22.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:22.644 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:22.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:45:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:45:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:45:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:22 smithi087 ceph-mon[89014]: pgmap v1801: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 11686/197682 objects degraded (5.912%); 18698/197682 objects misplaced (9.459%); 15 MiB/s, 237 objects/s recovering 2024-04-12T11:45:22.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:22.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:22.785 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:22.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:22.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:22.934 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:22.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:22.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:23.036 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:23.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:23.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:23.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:23.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:23.222 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:23.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:23.323 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:23.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:23.367 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:23.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:23.407 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:45:23.468 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:23.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:23.517 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:23.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:23.558 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11686/197682 objects degraded (5.912%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:23.618 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:23.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:23.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:23.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:23.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11686/197682 objects degraded (5.912%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11686/197682 objects degraded (5.912%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:23.794 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:23.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:23.895 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:23.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:23.923 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:23.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:24.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:24.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:24.064 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:24.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:24.167 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:24.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:24.203 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:24.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:24.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:24.335 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:24.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:24.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:24.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:24.478 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:24.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:24.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:24.580 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:24.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:24.617 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:24.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:24 smithi087 ceph-mon[89014]: pgmap v1802: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 14191/194901 objects degraded (7.281%); 19533/194901 objects misplaced (10.022%); 19 MiB/s, 303 objects/s recovering 2024-04-12T11:45:24.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:24.755 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:24.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:24 smithi029 ceph-mon[93650]: pgmap v1802: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 14191/194901 objects degraded (7.281%); 19533/194901 objects misplaced (10.022%); 19 MiB/s, 303 objects/s recovering 2024-04-12T11:45:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:24 smithi029 ceph-mon[97787]: pgmap v1802: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 14191/194901 objects degraded (7.281%); 19533/194901 objects misplaced (10.022%); 19 MiB/s, 303 objects/s recovering 2024-04-12T11:45:24.857 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:24.902 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:24.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:24.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:25.004 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:25.048 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:25.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:25.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:25.149 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:25.193 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:25.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:25.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:45:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:45:24] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2024-04-12T11:45:25.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:25.333 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:25.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:25.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:25.434 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:25.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:25.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:25.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:25.577 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:25.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:25.619 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:25.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:25.720 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:25.771 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:25.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:25.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:25.874 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:25.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:25.914 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:25.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:26.015 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:26.051 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:26.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:26.153 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:26.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:26.204 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:26.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:26.290 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:45:26.291 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:45:26.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:26.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:26.344 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:26.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:26.445 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:26.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:26.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:26.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:26.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:26.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:26.609 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:26.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:26.710 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:26.747 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:26.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:26.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:26.850 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:26.889 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:26.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:26.990 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:27.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:27.033 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:27.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:27.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:27.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:27.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:27.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:27.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:26 smithi087 ceph-mon[89014]: pgmap v1803: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 13865/187341 objects degraded (7.401%); 17670/187341 objects misplaced (9.432%); 19 MiB/s, 310 objects/s recovering 2024-04-12T11:45:27.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:27.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:26 smithi029 ceph-mon[97787]: pgmap v1803: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 13865/187341 objects degraded (7.401%); 17670/187341 objects misplaced (9.432%); 19 MiB/s, 310 objects/s recovering 2024-04-12T11:45:27.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:26 smithi029 ceph-mon[93650]: pgmap v1803: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 13865/187341 objects degraded (7.401%); 17670/187341 objects misplaced (9.432%); 19 MiB/s, 310 objects/s recovering 2024-04-12T11:45:27.302 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:27.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:27.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:27.359 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-04-12T11:45:27.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:27.428 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:27.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:27.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:27.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:27.553 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:27.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:27.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:27.703 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:27.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:27.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:27.805 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:27.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:27.837 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:27.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:27.938 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:27.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:27.963 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:27.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:28.064 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:28.106 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:28.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:28.175 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:45:28.207 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:28.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:27 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13865/187341 objects degraded (7.401%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:28.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:28.251 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:28.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:28.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:27 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13865/187341 objects degraded (7.401%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:28.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:27 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13865/187341 objects degraded (7.401%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:28.352 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:28.394 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:28.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:28.496 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:28.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:28.525 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:28.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:28.626 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:28.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:28.673 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:28.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:28.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:28.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:28.813 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:28.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:28.914 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:28.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:28.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:29.057 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:29.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:29.085 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:29.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:29.187 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:29.226 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:29.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:29.327 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:29.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:29.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:29.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:29.459 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:29 smithi087 ceph-mon[89014]: pgmap v1804: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 13540/184008 objects degraded (7.358%); 17397/184008 objects misplaced (9.454%); 20 MiB/s, 322 objects/s recovering 2024-04-12T11:45:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:29 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:45:29.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:29 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:45:29.488 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:29.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:29.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:29.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:29 smithi029 ceph-mon[97787]: pgmap v1804: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 13540/184008 objects degraded (7.358%); 17397/184008 objects misplaced (9.454%); 20 MiB/s, 322 objects/s recovering 2024-04-12T11:45:29.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:29 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:45:29.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:29 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:45:29.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:29 smithi029 ceph-mon[93650]: pgmap v1804: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 13540/184008 objects degraded (7.358%); 17397/184008 objects misplaced (9.454%); 20 MiB/s, 322 objects/s recovering 2024-04-12T11:45:29.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:29 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:45:29.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:29 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:45:29.590 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:29.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:29.616 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:29.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:29.717 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:29.758 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:29.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:29.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:29.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:29.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:29.893 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:29.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:29.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:30.038 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:30.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:30.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:30.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:30.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:30.184 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:30.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:30.257 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:45:30.257 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:45:30.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:30.323 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:30.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:30.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:30.425 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:30.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:30.460 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:30.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:30.561 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:30.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:30.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:30.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:30.688 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:30.728 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:30.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:30.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:30.830 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:30.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:30.870 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:30.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:30.958 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-04-12T11:45:30.970 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:31.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:31.040 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:31.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:31.141 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:31.181 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:31.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:31.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:31.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:31.292 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:45:31.338 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:31.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:31.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:31.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:31.471 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:45:31.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:31 smithi029 ceph-mon[93650]: pgmap v1805: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 13540/179397 objects degraded (7.548%); 15975/179397 objects misplaced (8.905%); 23 MiB/s, 363 objects/s recovering 2024-04-12T11:45:31.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:31 smithi029 ceph-mon[97787]: pgmap v1805: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 13540/179397 objects degraded (7.548%); 15975/179397 objects misplaced (8.905%); 23 MiB/s, 363 objects/s recovering 2024-04-12T11:45:31.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:31.534 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:31.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:31.607 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:31 smithi087 ceph-mon[89014]: pgmap v1805: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 13540/179397 objects degraded (7.548%); 15975/179397 objects misplaced (8.905%); 23 MiB/s, 363 objects/s recovering 2024-04-12T11:45:31.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:31.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:31.679 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:31.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:31.780 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:31.829 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:31.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:31.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:31.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:31.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:31.977 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:31.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:32.078 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:32.113 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:32.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:32.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:32.215 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:32.253 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:32.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:32.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:32.310 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:45:32.355 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:32.361 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:45:32.361 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:32 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/183088562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:45:32.362 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:32 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/183088562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:45:32.386 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:32.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:32.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:32.403 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:45:32.443 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:45:32.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:32 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/183088562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:45:32.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:45:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:45:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:45:32.482 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:45:32.488 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:32.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:32.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:32.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:32.522 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:45:32.559 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:45:32.603 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:45:32.616 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:45:32.616 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:45:32.616 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:45:32.616 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:45:32.616 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:45:32.616 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:45:32.616 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:45:32.616 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:45:32.622 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:32.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:32.664 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:32.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:32.764 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:32.808 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:32.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:32.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:32.954 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:32.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:32.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:33.055 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:33.097 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:33.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:33.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:33.189 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:45:33.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:33.205 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:45:33.206 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:45:33.206 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T11:45:33.206 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:45:33.288 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:33.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:33.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:33.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:33.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:33.446 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:33.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:33 smithi087 ceph-mon[89014]: pgmap v1806: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 13243/176925 objects degraded (7.485%); 15654/176925 objects misplaced (8.848%); 20 MiB/s, 327 objects/s recovering 2024-04-12T11:45:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13243/176925 objects degraded (7.485%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:33.548 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:33.577 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:33.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:33.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:33.679 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:33.713 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:33.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:33.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:33 smithi029 ceph-mon[93650]: pgmap v1806: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 13243/176925 objects degraded (7.485%); 15654/176925 objects misplaced (8.848%); 20 MiB/s, 327 objects/s recovering 2024-04-12T11:45:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13243/176925 objects degraded (7.485%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:33.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:33 smithi029 ceph-mon[97787]: pgmap v1806: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 13243/176925 objects degraded (7.485%); 15654/176925 objects misplaced (8.848%); 20 MiB/s, 327 objects/s recovering 2024-04-12T11:45:33.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13243/176925 objects degraded (7.485%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:33.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:33.854 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:33.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:33.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:33.956 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:33.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:33.980 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:33.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:34.080 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:34.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:34.117 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:34.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:34.218 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:34.250 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:34.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:34.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:34.384 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:34.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:34.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:34.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:34.524 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:34.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:34.626 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:34.654 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:34.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:34.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:34.756 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:34.794 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:34.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:34.897 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:34.936 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:34.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:34.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:35.038 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:35.055 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-04-12T11:45:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:35.064 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:35.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:35.165 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:35.189 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:35.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:35.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:45:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:45:34] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2024-04-12T11:45:35.291 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:35.311 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:45:35.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:35.334 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:35.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:35.435 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:35.476 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:35.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:35.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:35.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:35.605 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:35.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:35.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:35.608 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:35 smithi087 ceph-mon[89014]: pgmap v1807: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 12909/173541 objects degraded (7.439%); 15256/173541 objects misplaced (8.791%); 23 MiB/s, 374 objects/s recovering 2024-04-12T11:45:35.707 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:35.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:35.740 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:35.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:35 smithi029 ceph-mon[93650]: pgmap v1807: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 12909/173541 objects degraded (7.439%); 15256/173541 objects misplaced (8.791%); 23 MiB/s, 374 objects/s recovering 2024-04-12T11:45:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:35 smithi029 ceph-mon[97787]: pgmap v1807: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 12909/173541 objects degraded (7.439%); 15256/173541 objects misplaced (8.791%); 23 MiB/s, 374 objects/s recovering 2024-04-12T11:45:35.841 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:35.876 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:35.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:35.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:36.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:36.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:36.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:36.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:36.151 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:36.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:36.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:36.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:36.287 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:36.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:36.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:36.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:36.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:36.414 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:36.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:36.515 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:36.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:36.555 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:36.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:36.657 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:36.683 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:36.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:36.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:36.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:36.824 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:36.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:36.926 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:36.959 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:36.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:36.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:37.061 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:37.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:37.088 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:37.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:37.121 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:45:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:45:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:45:37.121 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:45:37.122 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:45:37.189 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:37.220 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:37.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:37.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:37.322 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:37.359 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:37.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:37.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:37.461 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:37.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:37 smithi087 ceph-mon[89014]: pgmap v1808: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 12568/165621 objects degraded (7.588%); 13512/165621 objects misplaced (8.158%); 26 MiB/s, 419 objects/s recovering 2024-04-12T11:45:37.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:37.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:37.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:37.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:37.592 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:37.628 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:37.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:37.730 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:37.753 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:37.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:37.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:37 smithi029 ceph-mon[97787]: pgmap v1808: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 12568/165621 objects degraded (7.588%); 13512/165621 objects misplaced (8.158%); 26 MiB/s, 419 objects/s recovering 2024-04-12T11:45:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:37.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:37 smithi029 ceph-mon[93650]: pgmap v1808: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 5 active+remapped+backfill_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 12568/165621 objects degraded (7.588%); 13512/165621 objects misplaced (8.158%); 26 MiB/s, 419 objects/s recovering 2024-04-12T11:45:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:37.804 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-04-12T11:45:37.855 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:37.883 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:37.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:37.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:37.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:38.011 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:38.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:38.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:38.027 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:45:38.113 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:38.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:38.151 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:38.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:38.252 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:38.293 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:38.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:38.395 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:38.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:38.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:38.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:38.430 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 12568/165621 objects degraded (7.588%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:38.521 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:38.546 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:38.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:38.647 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:38.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:38.672 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:38.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:38.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 12568/165621 objects degraded (7.588%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:38.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 12568/165621 objects degraded (7.588%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:38.773 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:38.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:38.808 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:38.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:38.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:38.951 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:38.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:39.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:39.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:39.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:39.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:39.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:39.211 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:39.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:39.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:39.313 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:39.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:39.342 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:39.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:39.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:39.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:39.467 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:39.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:39.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:39 smithi087 ceph-mon[89014]: pgmap v1809: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 10239/162087 objects degraded (6.317%); 15322/162087 objects misplaced (9.453%); 22 MiB/s, 373 objects/s recovering 2024-04-12T11:45:39.568 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:39.594 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:39.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:39.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:39.696 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:39.733 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:39.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:39.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:39.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:39 smithi029 ceph-mon[97787]: pgmap v1809: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 10239/162087 objects degraded (6.317%); 15322/162087 objects misplaced (9.453%); 22 MiB/s, 373 objects/s recovering 2024-04-12T11:45:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:39 smithi029 ceph-mon[93650]: pgmap v1809: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 10239/162087 objects degraded (6.317%); 15322/162087 objects misplaced (9.453%); 22 MiB/s, 373 objects/s recovering 2024-04-12T11:45:39.835 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:39.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:39.874 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:39.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:39.975 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:40.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:40.013 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:40.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:40.114 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:40.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:40.155 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:40.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:40.193 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:45:40.193 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:45:40.193 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:45:40.193 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:45:40.194 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.194 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:82349 8.5 e1192/1172 8:af40c32a:::benchmark_data_smithi029_194418_object82348:head v 1192'5180, mlcod=1192'5172)", 2024-04-12T11:45:40.194 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:52.000062+0000", 2024-04-12T11:45:40.194 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.19029098199999, 2024-04-12T11:45:40.194 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.027332301, 2024-04-12T11:45:40.194 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:40.194 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:40.195 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:40.195 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.195 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:40.195 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.000062+0000", 2024-04-12T11:45:40.195 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:40.195 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.196 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.196 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:40.196 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.000062+0000", 2024-04-12T11:45:40.196 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:40.196 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.196 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.196 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:40.197 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.000061+0000", 2024-04-12T11:45:40.197 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:45:40.197 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.197 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.197 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:40.198 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.000084+0000", 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.2515999999999999e-05 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.000095+0000", 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1479e-05 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.000102+0000", 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4379999999999998e-06 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.199 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.005959+0000", 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0058562390000000001 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.005980+0000", 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1166e-05 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.027323+0000", 2024-04-12T11:45:40.200 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.021343405999999999 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.027394+0000", 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1052999999999999e-05 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:82361 8.d e1192/1172 8:b14af178:::benchmark_data_smithi029_194418_object82360:head v 1192'5186, mlcod=1192'5184)", 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:52.006046+0000", 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 108.18430744699999, 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.021273144000000001, 2024-04-12T11:45:40.201 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.006046+0000", 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.006046+0000", 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:40.202 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.006045+0000", 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.006068+0000", 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3374e-05 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:40.203 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.006080+0000", 2024-04-12T11:45:40.204 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1695000000000001e-05 2024-04-12T11:45:40.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.204 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.204 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:40.204 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.006087+0000", 2024-04-12T11:45:40.204 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3409999999999997e-06 2024-04-12T11:45:40.204 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.204 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.204 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:40.204 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.006175+0000", 2024-04-12T11:45:40.205 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8255000000000001e-05 2024-04-12T11:45:40.205 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.205 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.205 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:40.205 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.006195+0000", 2024-04-12T11:45:40.205 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9874e-05 2024-04-12T11:45:40.205 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.206 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.206 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:40.206 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.027277+0000", 2024-04-12T11:45:40.206 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.021082209000000001 2024-04-12T11:45:40.206 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.207 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.207 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:40.207 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:52.027319+0000", 2024-04-12T11:45:40.207 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1309e-05 2024-04-12T11:45:40.207 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.207 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:40.208 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.208 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.208 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.208 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:85344 8.d e1192/1172 8:b831bbf8:::benchmark_data_smithi029_194418_object85343:head v 1192'5346, mlcod=1192'5344)", 2024-04-12T11:45:40.208 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:54.478750+0000", 2024-04-12T11:45:40.208 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 105.711602855, 2024-04-12T11:45:40.208 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.02096721, 2024-04-12T11:45:40.208 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:40.208 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:40.208 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.478750+0000", 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.478750+0000", 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.478749+0000", 2024-04-12T11:45:40.209 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.481446+0000", 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002697339 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.481460+0000", 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3918e-05 2024-04-12T11:45:40.210 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.481466+0000", 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.066e-06 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.481534+0000", 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8319999999999996e-05 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.481562+0000", 2024-04-12T11:45:40.211 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.7243e-05 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.499676+0000", 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.018114425 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.499717+0000", 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1267000000000003e-05 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.212 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:85345 8.5 e1192/1172 8:affca2e2:::benchmark_data_smithi029_194418_object85344:head v 1192'5359, mlcod=1192'5357)", 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:54.485211+0000", 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 105.70514169099999, 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.017213471000000001, 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.485211+0000", 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.213 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:40.214 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.485211+0000", 2024-04-12T11:45:40.214 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:40.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.214 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:40.214 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.485211+0000", 2024-04-12T11:45:40.214 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:40.214 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.214 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.214 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.485237+0000", 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6429999999999999e-05 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.485249+0000", 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2316e-05 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:40.215 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.485256+0000", 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2230000000000004e-06 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.486957+0000", 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001701431 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.486979+0000", 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1773e-05 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.216 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.502347+0000", 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015367875 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.502425+0000", 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8109999999999996e-05 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:85968 8.d e1192/1172 8:bde35d30:::benchmark_data_smithi029_194418_object85967:head v 1192'5394, mlcod=1192'5390)", 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:54.968534+0000", 2024-04-12T11:45:40.217 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 105.221818595, 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.017649128, 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.968534+0000", 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.968534+0000", 2024-04-12T11:45:40.218 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.968534+0000", 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.968557+0000", 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3153999999999999e-05 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:45:40.219 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.968564+0000", 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4289999999999998e-06 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.968568+0000", 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7099999999999998e-06 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.974917+0000", 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0063486580000000001 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.220 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.974963+0000", 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5800999999999997e-05 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.986085+0000", 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.011122251999999999 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:54.986184+0000", 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8441000000000002e-05 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:40.221 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.222 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:45:40.222 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:45:40.222 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:45:40.256 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:40.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:40.292 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:40.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:40.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:40.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:40.418 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:40.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:40.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:40.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:40.545 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:40.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:40.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:40.685 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:40.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:40.787 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:40.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:40.833 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:40.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:40.934 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:40.975 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-04-12T11:45:40.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:40.981 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:40.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:41.082 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:41.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:41.116 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:41.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:41.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:41.268 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:41.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:41.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:41.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:41.408 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:41.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:41.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:41.510 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:41.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:41.537 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:41.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:41.638 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:41.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:41.662 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:41.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:41 smithi087 ceph-mon[89014]: pgmap v1810: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 10239/158040 objects degraded (6.479%); 13914/158040 objects misplaced (8.804%); 23 MiB/s, 386 objects/s recovering 2024-04-12T11:45:41.763 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:41 smithi029 ceph-mon[97787]: pgmap v1810: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 10239/158040 objects degraded (6.479%); 13914/158040 objects misplaced (8.804%); 23 MiB/s, 386 objects/s recovering 2024-04-12T11:45:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:41 smithi029 ceph-mon[93650]: pgmap v1810: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 2 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 10239/158040 objects degraded (6.479%); 13914/158040 objects misplaced (8.804%); 23 MiB/s, 386 objects/s recovering 2024-04-12T11:45:41.802 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:41.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:41.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:41.904 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:41.941 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:41.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:41.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:42.044 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:42.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:42.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:42.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:42.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:42.205 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:42.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:42.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:42.307 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:42.343 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:42.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:42.444 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:42.476 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:42.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:42.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:42.578 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:42.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:42.602 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:42.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:42.703 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:45:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:45:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:45:42.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:42.728 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:42.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:42.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:42.863 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:42.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:42.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:42.965 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:43.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:43.000 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:43.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:43.102 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:43.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:43.139 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:43.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:43.240 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:43.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:43.267 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:43.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:43.368 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:43.408 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:43.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:43.509 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:43.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:43.542 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:43.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:43.643 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:43.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:43.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:43.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:43 smithi087 ceph-mon[89014]: pgmap v1811: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 9962/155718 objects degraded (6.397%); 13757/155718 objects misplaced (8.835%); 22 MiB/s, 372 objects/s recovering 2024-04-12T11:45:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9962/155718 objects degraded (6.397%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:43 smithi029 ceph-mon[97787]: pgmap v1811: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 9962/155718 objects degraded (6.397%); 13757/155718 objects misplaced (8.835%); 22 MiB/s, 372 objects/s recovering 2024-04-12T11:45:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9962/155718 objects degraded (6.397%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:43 smithi029 ceph-mon[93650]: pgmap v1811: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 1 active+recovering; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 9962/155718 objects degraded (6.397%); 13757/155718 objects misplaced (8.835%); 22 MiB/s, 372 objects/s recovering 2024-04-12T11:45:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9962/155718 objects degraded (6.397%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:43.777 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:43.814 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:43.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:43.916 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:43.971 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:43.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:43.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:44.073 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:44.104 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:44.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:44.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:44.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:44.231 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:44.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:44.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:44.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:44.363 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:44.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:44.464 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:44.501 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:44.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:44.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:44.602 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:44.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:44.627 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:44.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:44.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:44.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:44.754 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:44.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:44.856 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:44.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:44.882 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:44.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:44.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:45.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:45.023 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:45.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:45.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:45.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:45.168 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:45.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:45:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:45:44] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2024-04-12T11:45:45.269 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:45.294 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:45.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:45.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:45.421 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:45.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:45.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:45.523 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:45.546 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:45:45.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:45.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:45.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:45.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:45.711 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:45.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:45.716 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:45 smithi087 ceph-mon[89014]: pgmap v1812: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 85 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 9718/153153 objects degraded (6.345%); 13462/153153 objects misplaced (8.790%); 22 MiB/s, 389 objects/s recovering 2024-04-12T11:45:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:45 smithi029 ceph-mon[93650]: pgmap v1812: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 85 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 9718/153153 objects degraded (6.345%); 13462/153153 objects misplaced (8.790%); 22 MiB/s, 389 objects/s recovering 2024-04-12T11:45:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:45 smithi029 ceph-mon[97787]: pgmap v1812: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 85 active+clean, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 9718/153153 objects degraded (6.345%); 13462/153153 objects misplaced (8.790%); 22 MiB/s, 389 objects/s recovering 2024-04-12T11:45:45.813 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:45.861 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:45.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:45.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:45.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:45.991 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:45.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:45.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:46.093 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:46.117 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:46.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:46.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:46.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:46.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:46.268 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:46.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:46.369 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:46.403 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:46.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:46.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:46.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:46.537 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:46.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:46.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:46.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:46.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:46.664 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:46.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:46.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:46.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:46.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:46.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:46.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:46.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:46.952 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:46.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:47.053 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:47.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:47.088 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:47.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:47.190 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:47.214 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:47.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:47.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:47.317 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:47.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:47.352 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:47.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:47.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:47.493 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:47.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:47.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:47.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:47.630 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:47.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:47.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:47 smithi087 ceph-mon[89014]: pgmap v1813: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 9427/146256 objects degraded (6.446%); 12222/146256 objects misplaced (8.357%); 22 MiB/s, 398 objects/s recovering 2024-04-12T11:45:47.732 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:47.758 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:47.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:47.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:47.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:47 smithi029 ceph-mon[93650]: pgmap v1813: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 9427/146256 objects degraded (6.446%); 12222/146256 objects misplaced (8.357%); 22 MiB/s, 398 objects/s recovering 2024-04-12T11:45:47.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:47 smithi029 ceph-mon[97787]: pgmap v1813: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 85 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 757 op/s; 9427/146256 objects degraded (6.446%); 12222/146256 objects misplaced (8.357%); 22 MiB/s, 398 objects/s recovering 2024-04-12T11:45:47.860 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:47.951 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:47.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:48.052 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:48.221 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:48.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:48.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:48.323 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:48.359 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:48.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:48.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:48.461 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:48.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:48.486 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:48.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:48.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:48.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:48.612 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:48.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:48.713 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9427/146256 objects degraded (6.446%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:48.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:48.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:48.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:48.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9427/146256 objects degraded (6.446%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:48.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9427/146256 objects degraded (6.446%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:48.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:48.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:48.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:48.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:49.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:49.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:49.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:49.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:49.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:49.145 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:49.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:49.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:49.275 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:49.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:49.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:49.377 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:49.403 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:49.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:49.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:49.505 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:49 smithi029 ceph-mon[93650]: pgmap v1814: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 5 active+remapped+backfill_wait; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 7833/143268 objects degraded (5.467%); 12634/143268 objects misplaced (8.818%); 20 MiB/s, 357 objects/s recovering 2024-04-12T11:45:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:49 smithi029 ceph-mon[93650]: osdmap e1207: 8 total, 8 up, 7 in 2024-04-12T11:45:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:49 smithi029 ceph-mon[97787]: pgmap v1814: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 5 active+remapped+backfill_wait; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 7833/143268 objects degraded (5.467%); 12634/143268 objects misplaced (8.818%); 20 MiB/s, 357 objects/s recovering 2024-04-12T11:45:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:49 smithi029 ceph-mon[97787]: osdmap e1207: 8 total, 8 up, 7 in 2024-04-12T11:45:49.530 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:49.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:49.632 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:49.668 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:49.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:49.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:49 smithi087 ceph-mon[89014]: pgmap v1814: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 5 active+remapped+backfill_wait; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 7833/143268 objects degraded (5.467%); 12634/143268 objects misplaced (8.818%); 20 MiB/s, 357 objects/s recovering 2024-04-12T11:45:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:49 smithi087 ceph-mon[89014]: osdmap e1207: 8 total, 8 up, 7 in 2024-04-12T11:45:49.771 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:49.804 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:49.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:49.906 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:49.936 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:49.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:50.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:50.061 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:50.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:50.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:50.163 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:50.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:50.198 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:50.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:50.299 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:50.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:50.335 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:50.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:50.436 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:50.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:50.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:50.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:50.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:50.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:50.586 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:50.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:50.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:50.711 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:50.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:50.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:50 smithi087 ceph-mon[89014]: osdmap e1208: 8 total, 8 up, 7 in 2024-04-12T11:45:50.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:50 smithi029 ceph-mon[93650]: osdmap e1208: 8 total, 8 up, 7 in 2024-04-12T11:45:50.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:50 smithi029 ceph-mon[97787]: osdmap e1208: 8 total, 8 up, 7 in 2024-04-12T11:45:50.813 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:50.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:50.850 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:50.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:50.951 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:50.984 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:50.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:50.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:51.087 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:51.110 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:51.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:51.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:51.212 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:51.235 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:51.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:51.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:51.337 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:51.370 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:51.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:51.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:51.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:51.511 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:51.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:51.612 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:51.636 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:51.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:51.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:51.639 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:51 smithi029 ceph-mon[93650]: pgmap v1817: 99 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 7833/140334 objects degraded (5.582%); 11785/140334 objects misplaced (8.398%); 19 MiB/s, 344 objects/s recovering 2024-04-12T11:45:51.639 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:51.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:51 smithi029 ceph-mon[97787]: pgmap v1817: 99 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 7833/140334 objects degraded (5.582%); 11785/140334 objects misplaced (8.398%); 19 MiB/s, 344 objects/s recovering 2024-04-12T11:45:51.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:51 smithi087 ceph-mon[89014]: pgmap v1817: 99 pgs: 2 active+remapped+backfilling, 1 active+recovery_wait+degraded, 3 active+recovery_wait+degraded+remapped, 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 7833/140334 objects degraded (5.582%); 11785/140334 objects misplaced (8.398%); 19 MiB/s, 344 objects/s recovering 2024-04-12T11:45:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:45:51.738 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:51.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:51.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:51.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:51.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:51.924 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:51.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:51.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:52.026 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:52.068 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:52.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:52.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:52.170 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:52.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:52.194 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:52.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:52.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:52.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:52.335 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:52.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:52.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:52.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:52.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:52.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:52.564 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:52.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:52.589 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:52.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:52.617 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:52 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2771699294' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2024-04-12T11:45:52.617 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:52 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2024-04-12T11:45:52.617 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:45:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:45:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:45:52.689 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:52.714 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:52.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:52.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:52 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2771699294' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2024-04-12T11:45:52.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:52 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2024-04-12T11:45:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T11:45:52.153+0000 7fae24212700 -1 mon.a@0(leader).osd e1208 definitely_dead 0 2024-04-12T11:45:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:52 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2771699294' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2024-04-12T11:45:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:52 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2024-04-12T11:45:52.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:52.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:52.842 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:52.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:52.943 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:52.973 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:52.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:52.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:53.075 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:53.108 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:53.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:53.209 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:53.238 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:53.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:53.340 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:53.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:53.374 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:53.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:53.476 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:53.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:53.502 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:53.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:53.604 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:53.628 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.7. 2024-04-12T11:45:53.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:53.654 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:53.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:53.723 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:53 smithi087 ceph-mon[89014]: pgmap v1818: 99 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 3 active+recovery_wait+degraded+remapped, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 7555/137469 objects degraded (5.496%); 11421/137469 objects misplaced (8.308%); 20 MiB/s, 319 objects/s recovering 2024-04-12T11:45:53.723 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:53 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-04-12T11:45:53.723 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:53 smithi087 ceph-mon[89014]: osdmap e1209: 8 total, 8 up, 8 in 2024-04-12T11:45:53.723 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 7555/137469 objects degraded (5.496%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:53.723 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:53 smithi087 ceph-mon[89014]: osdmap e1210: 8 total, 8 up, 8 in 2024-04-12T11:45:53.755 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:53.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:53 smithi029 ceph-mon[93650]: pgmap v1818: 99 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 3 active+recovery_wait+degraded+remapped, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 7555/137469 objects degraded (5.496%); 11421/137469 objects misplaced (8.308%); 20 MiB/s, 319 objects/s recovering 2024-04-12T11:45:53.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:53 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-04-12T11:45:53.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:53 smithi029 ceph-mon[93650]: osdmap e1209: 8 total, 8 up, 8 in 2024-04-12T11:45:53.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 7555/137469 objects degraded (5.496%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:53.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:53 smithi029 ceph-mon[93650]: osdmap e1210: 8 total, 8 up, 8 in 2024-04-12T11:45:53.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:53 smithi029 ceph-mon[97787]: pgmap v1818: 99 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 3 active+recovery_wait+degraded+remapped, 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+remapped+backfill_wait; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 7555/137469 objects degraded (5.496%); 11421/137469 objects misplaced (8.308%); 20 MiB/s, 319 objects/s recovering 2024-04-12T11:45:53.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:53 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-04-12T11:45:53.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:53 smithi029 ceph-mon[97787]: osdmap e1209: 8 total, 8 up, 8 in 2024-04-12T11:45:53.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 7555/137469 objects degraded (5.496%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:45:53.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:53 smithi029 ceph-mon[97787]: osdmap e1210: 8 total, 8 up, 8 in 2024-04-12T11:45:54.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:54.014 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:54.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:54.115 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:54.143 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:45:54.143 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:45:54.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:54.164 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:54.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:54.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:54.308 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:54.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:54.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:54.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:54.447 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:54.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:54.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:54.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:54.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:54.684 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:54.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:54.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:54.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:54.744 INFO:tasks.thrashosds.thrasher:Added osd 7 2024-04-12T11:45:54.826 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:54.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:54.870 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:54.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:54.963 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:45:54] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2024-04-12T11:45:54.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:55.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:55.007 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:55.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:55.108 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:55.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:55.148 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:55.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:55.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:54 smithi087 ceph-mon[89014]: pgmap v1821: 99 pgs: 3 remapped+peering, 17 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 3657/135462 objects degraded (2.700%); 8095/135462 objects misplaced (5.976%); 20 MiB/s, 319 objects/s recovering 2024-04-12T11:45:55.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:54 smithi087 ceph-mon[89014]: osdmap e1211: 8 total, 8 up, 8 in 2024-04-12T11:45:55.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:54 smithi087 ceph-mon[89014]: 8.c starting backfill to osd.3 from (1191'4188,1208'10339] MIN to 1208'10340 2024-04-12T11:45:55.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:54 smithi087 ceph-mon[89014]: 8.8 continuing backfill to osd.3 from (1176'400,1208'8889] MIN to 1208'8889 2024-04-12T11:45:55.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:54 smithi087 ceph-mon[89014]: 8.15 starting backfill to osd.7 from (0'0,0'0] MAX to 1208'9063 2024-04-12T11:45:55.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:54 smithi087 ceph-mon[89014]: 8.16 starting backfill to osd.7 from (0'0,0'0] MAX to 1208'8965 2024-04-12T11:45:55.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:54 smithi087 ceph-mon[89014]: 8.18 continuing backfill to osd.3 from (1176'400,1208'8832] MIN to 1208'8832 2024-04-12T11:45:55.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:54 smithi087 ceph-mon[89014]: Health check failed: Reduced data availability: 3 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-04-12T11:45:55.249 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[93650]: pgmap v1821: 99 pgs: 3 remapped+peering, 17 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 3657/135462 objects degraded (2.700%); 8095/135462 objects misplaced (5.976%); 20 MiB/s, 319 objects/s recovering 2024-04-12T11:45:55.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[93650]: osdmap e1211: 8 total, 8 up, 8 in 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[93650]: 8.c starting backfill to osd.3 from (1191'4188,1208'10339] MIN to 1208'10340 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[93650]: 8.8 continuing backfill to osd.3 from (1176'400,1208'8889] MIN to 1208'8889 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[93650]: 8.15 starting backfill to osd.7 from (0'0,0'0] MAX to 1208'9063 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[93650]: 8.16 starting backfill to osd.7 from (0'0,0'0] MAX to 1208'8965 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[93650]: 8.18 continuing backfill to osd.3 from (1176'400,1208'8832] MIN to 1208'8832 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[93650]: Health check failed: Reduced data availability: 3 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[97787]: pgmap v1821: 99 pgs: 3 remapped+peering, 17 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 71 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 3657/135462 objects degraded (2.700%); 8095/135462 objects misplaced (5.976%); 20 MiB/s, 319 objects/s recovering 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[97787]: osdmap e1211: 8 total, 8 up, 8 in 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[97787]: 8.c starting backfill to osd.3 from (1191'4188,1208'10339] MIN to 1208'10340 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[97787]: 8.8 continuing backfill to osd.3 from (1176'400,1208'8889] MIN to 1208'8889 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[97787]: 8.15 starting backfill to osd.7 from (0'0,0'0] MAX to 1208'9063 2024-04-12T11:45:55.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[97787]: 8.16 starting backfill to osd.7 from (0'0,0'0] MAX to 1208'8965 2024-04-12T11:45:55.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[97787]: 8.18 continuing backfill to osd.3 from (1176'400,1208'8832] MIN to 1208'8832 2024-04-12T11:45:55.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:54 smithi029 ceph-mon[97787]: Health check failed: Reduced data availability: 3 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-04-12T11:45:55.281 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:55.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:55.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:55.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:55.423 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:55.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:55.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:55.560 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:55.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:55.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:55.662 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:55.697 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:55.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:55.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:55.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:55.838 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:55.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:55.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:55.941 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:55.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:55.976 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:55.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:56.077 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:56.129 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:56.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:56.231 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:56.279 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:56.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:56.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:56.416 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:56.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:56.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:56.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:56.554 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:56.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:56.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:56.656 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:56.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:56.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:56.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:56.794 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:56.834 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:56.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:56.935 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:56.974 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:56.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:57.076 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:57.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:57.118 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:57.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:57.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:56 smithi087 ceph-mon[89014]: pgmap v1823: 99 pgs: 3 remapped+peering, 17 peering, 70 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 5254/131469 objects degraded (3.996%); 12898/131469 objects misplaced (9.811%); 29 MiB/s, 464 objects/s recovering 2024-04-12T11:45:57.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:57.256 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:56 smithi029 ceph-mon[93650]: pgmap v1823: 99 pgs: 3 remapped+peering, 17 peering, 70 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 5254/131469 objects degraded (3.996%); 12898/131469 objects misplaced (9.811%); 29 MiB/s, 464 objects/s recovering 2024-04-12T11:45:57.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:56 smithi029 ceph-mon[97787]: pgmap v1823: 99 pgs: 3 remapped+peering, 17 peering, 70 active+clean, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 5254/131469 objects degraded (3.996%); 12898/131469 objects misplaced (9.811%); 29 MiB/s, 464 objects/s recovering 2024-04-12T11:45:57.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:57.257 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:57.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:57.358 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:57.383 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:57.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:57.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:57.485 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:57.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:57.509 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:57.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:57.610 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:57.643 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:57.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:57.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:57.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:57.779 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:57.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:57.882 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:45:57.883 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:45:57.883 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:45:57.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.884 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:171763 8.15 e1211/1210 8:ab37bdc5:::benchmark_data_smithi029_194418_object65453:head v 1211'9166, mlcod=1211'9162)", 2024-04-12T11:45:57.884 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:57.867973+0000", 2024-04-12T11:45:57.884 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.00077861499999999997, 2024-04-12T11:45:57.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00080150000000000002, 2024-04-12T11:45:57.884 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:45:57.884 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:45:57.884 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:45:57.884 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:45:57.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.867973+0000", 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.867973+0000", 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.867974+0000", 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0500000000000003e-07 2024-04-12T11:45:57.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.867979+0000", 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2510000000000002e-06 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.867990+0000", 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.047e-05 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.867995+0000", 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7919999999999998e-06 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.868047+0000", 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1257000000000001e-05 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.868071+0000", 2024-04-12T11:45:57.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4719000000000002e-05 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:171744 8.c 8:38289548:::benchmark_data_smithi029_194418_object65434:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1211)", 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:57.841406+0000", 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.027345464, 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027413963, 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:45:57.888 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 171744 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.841406+0000", 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:45:57.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.841406+0000", 2024-04-12T11:45:57.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:45:57.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.841408+0000", 2024-04-12T11:45:57.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.418e-06 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.841413+0000", 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5319999999999997e-06 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.841430+0000", 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6441000000000001e-05 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:45:57.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.841439+0000", 2024-04-12T11:45:57.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3209999999999994e-06 2024-04-12T11:45:57.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:45:57.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.841502+0000", 2024-04-12T11:45:57.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2929999999999995e-05 2024-04-12T11:45:57.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.841639+0000", 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013668600000000001 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5,7", 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.841775+0000", 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013671500000000001 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.845661+0000", 2024-04-12T11:45:57.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003885847 2024-04-12T11:45:57.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:45:57.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.845770+0000", 2024-04-12T11:45:57.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108857 2024-04-12T11:45:57.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.845926+0000", 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015543800000000001 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:171715 8.c 8:3e1bf5f8:::benchmark_data_smithi029_194418_object65405:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1211)", 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:57.811300+0000", 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.057451636, 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057579546000000002, 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:45:57.895 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 171715 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.811300+0000", 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:45:57.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.811300+0000", 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.811302+0000", 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8190000000000001e-06 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.811308+0000", 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9719999999999997e-06 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:45:57.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.811319+0000", 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1413000000000001e-05 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.811328+0000", 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1189999999999997e-06 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.811382+0000", 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4809999999999999e-05 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.811555+0000", 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000172848 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5,7", 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.811704+0000", 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014837699999999999 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.821902+0000", 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010198743 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.821925+0000", 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2365e-05 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.821961+0000", 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6424000000000002e-05 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:171722 8.c 8:3fb28a52:::benchmark_data_smithi029_194418_object65412:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1211)", 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:57.819084+0000", 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.049667683999999997, 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.049853175999999999, 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 171722 2024-04-12T11:45:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.819084+0000", 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.819084+0000", 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.819086+0000", 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.844e-06 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.819091+0000", 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1109999999999999e-06 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.819108+0000", 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7127999999999999e-05 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.819118+0000", 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7499999999999998e-06 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.821995+0000", 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028768790000000002 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.822130+0000", 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000134664 2024-04-12T11:45:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5,7", 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.822267+0000", 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013765199999999999 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.825854+0000", 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035866589999999999 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.827026+0000", 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011724909999999999 2024-04-12T11:45:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.827472+0000", 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000445246 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:171692 8.c 8:331a3f55:::benchmark_data_smithi029_194418_object65382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1211)", 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:57.786270+0000", 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.082481944000000001, 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.082740093000000001, 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:45:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 171692 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.786270+0000", 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.786270+0000", 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.786272+0000", 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7540000000000001e-06 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.786277+0000", 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6710000000000004e-06 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.786289+0000", 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2058999999999999e-05 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.786297+0000", 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9109999999999993e-06 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.786650+0000", 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035231900000000003 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.786828+0000", 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017838700000000001 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5,7", 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.786980+0000", 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015159599999999999 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.790834+0000", 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038547910000000002 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.793616+0000", 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027813450000000002 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.793810+0000", 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000193804 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:171704 8.c 8:3116ca6d:::benchmark_data_smithi029_194418_object65394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1211)", 2024-04-12T11:45:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:57.801382+0000", 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.067369898999999997, 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067684269000000005, 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 171704 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:45:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.801382+0000", 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.801382+0000", 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.801384+0000", 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7400000000000001e-06 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:45:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.801391+0000", 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1620000000000002e-06 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.801408+0000", 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7224e-05 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.801418+0000", 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.397e-06 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.801483+0000", 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5901999999999993e-05 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.801638+0000", 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015454800000000001 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5,7", 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.801863+0000", 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022458299999999999 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.810506+0000", 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0086434589999999992 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.810695+0000", 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000188925 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.821918+0000", 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011222793999999999 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:45:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:171734 8.c 8:39a1e81b:::benchmark_data_smithi029_194418_object65424:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1211)", 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:57.830412+0000", 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.038339844999999997, 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.038745675, 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 171734 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:45:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:45:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.830412+0000", 2024-04-12T11:45:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:45:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.830412+0000", 2024-04-12T11:45:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:45:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.830413+0000", 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.347e-06 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.830419+0000", 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4909999999999996e-06 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.830438+0000", 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9029999999999999e-05 2024-04-12T11:45:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:45:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.830447+0000", 2024-04-12T11:45:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0750000000000004e-06 2024-04-12T11:45:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:45:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.830510+0000", 2024-04-12T11:45:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3163000000000005e-05 2024-04-12T11:45:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.830656+0000", 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014542799999999999 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5,7", 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.830793+0000", 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013771200000000001 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.834682+0000", 2024-04-12T11:45:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038883350000000001 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.834706+0000", 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4481000000000002e-05 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:57.835637+0000", 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00093100500000000005 2024-04-12T11:45:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:45:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:45:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2024-04-12T11:45:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 7 2024-04-12T11:45:57.924 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:45:57.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:57.926 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:57.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:45:58.027 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:58.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:58.062 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:58.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:58.163 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:45:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:58.196 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:58.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:45:58.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:57 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5254/131469 objects degraded (3.996%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:45:58.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:57 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5254/131469 objects degraded (3.996%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:45:58.297 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:58.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:58.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:58.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:58.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:57 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5254/131469 objects degraded (3.996%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:45:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:58.470 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:58.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:58.571 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:58.608 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:58.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:58.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:58.710 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:45:58.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:58.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:58.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:45:58.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:45:58.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:58.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:45:58.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:45:59.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:59.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:59.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:45:59.107 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:45:59.116 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:59.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-04-12T11:45:59.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:59.155 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:59.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:59.246 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:58 smithi029 ceph-mon[93650]: pgmap v1824: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 10371/128472 objects degraded (8.073%); 20379/128472 objects misplaced (15.863%); 24 MiB/s, 384 objects/s recovering 2024-04-12T11:45:59.246 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:45:58 smithi029 ceph-mon[93650]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 7 pgs peering) 2024-04-12T11:45:59.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:58 smithi029 ceph-mon[97787]: pgmap v1824: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 10371/128472 objects degraded (8.073%); 20379/128472 objects misplaced (15.863%); 24 MiB/s, 384 objects/s recovering 2024-04-12T11:45:59.247 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:45:58 smithi029 ceph-mon[97787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 7 pgs peering) 2024-04-12T11:45:59.256 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:59.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:59.300 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:59.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:45:59.401 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:45:59.436 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:59.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:59.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:45:59.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:58 smithi087 ceph-mon[89014]: pgmap v1824: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 10371/128472 objects degraded (8.073%); 20379/128472 objects misplaced (15.863%); 24 MiB/s, 384 objects/s recovering 2024-04-12T11:45:59.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:45:58 smithi087 ceph-mon[89014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 7 pgs peering) 2024-04-12T11:45:59.538 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:59.573 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:59.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:59.675 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:59.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:59.711 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:59.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:59.745 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 3, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:45:59.745 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:45:59.746 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 4, 5, 6, 3, 7] 2024-04-12T11:45:59.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd out 3 2024-04-12T11:45:59.812 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:45:59.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:59.851 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:45:59.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:45:59.952 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:45:59.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:45:59.987 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:45:59.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:00.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:00.122 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:00.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:00.223 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:00.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:00.260 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:00.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:00.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:00.398 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:00.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:00.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:00.499 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:00.538 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:00.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:00.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:00.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:00.677 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:00.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:00.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:00.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:00.829 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:00.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:00.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:00.931 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:00.966 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:00.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:00.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:01.068 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:01.100 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:01.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:01.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:01.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:01.222 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:00 smithi029 ceph-mon[93650]: pgmap v1825: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 10371/123132 objects degraded (8.423%); 19657/123132 objects misplaced (15.964%); 19 MiB/s, 305 objects/s recovering 2024-04-12T11:46:01.222 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:00 smithi029 ceph-mon[97787]: pgmap v1825: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 10371/123132 objects degraded (8.423%); 19657/123132 objects misplaced (15.964%); 19 MiB/s, 305 objects/s recovering 2024-04-12T11:46:01.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:01.250 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:01.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:01.351 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:01.392 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:01.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:01.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:00 smithi087 ceph-mon[89014]: pgmap v1825: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 10371/123132 objects degraded (8.423%); 19657/123132 objects misplaced (15.964%); 19 MiB/s, 305 objects/s recovering 2024-04-12T11:46:01.493 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:01.531 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:01.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:01.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:01.633 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:01.670 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:01.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:01.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:01.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:01.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:01.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:01.849 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:46:01.869 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:46:01.910 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:46:01.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:01.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:01.954 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:01.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:01.954 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:46:01.989 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:46:02.008 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:46:02.029 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:46:02.055 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:02.069 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:46:02.094 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:02.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:02.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:02.111 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:46:02.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:46:02.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:46:02.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:46:02.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:46:02.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:46:02.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:46:02.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:46:02.131 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:46:02.196 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:02.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:01 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3848622093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:46:02.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:01 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1935478349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:02.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:01 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3848622093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:46:02.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:01 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1935478349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:02.226 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:02.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:02.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:02.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:01 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3848622093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:46:02.315 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:01 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1935478349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:02.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:02.365 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:02.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:02.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:02.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:02.503 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:02.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:02.604 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:02.633 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:02.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:02.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:46:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:46:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:46:02.735 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:02.773 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:02.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:02.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:02.875 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:02.906 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:02.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:03.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:03.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:03.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:03.154 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:03.191 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:03.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:03.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:03.222 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:03 smithi029 ceph-mon[93650]: pgmap v1826: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 10059/118593 objects degraded (8.482%); 19514/118593 objects misplaced (16.455%); 11 MiB/s, 1 keys/s, 170 objects/s recovering 2024-04-12T11:46:03.222 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:03 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:46:03.222 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:03 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3848622093' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:46:03.222 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:03 smithi029 ceph-mon[93650]: osdmap e1212: 8 total, 8 up, 8 in 2024-04-12T11:46:03.222 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 10059/118593 objects degraded (8.482%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:03.222 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:03 smithi029 ceph-mon[97787]: pgmap v1826: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 10059/118593 objects degraded (8.482%); 19514/118593 objects misplaced (16.455%); 11 MiB/s, 1 keys/s, 170 objects/s recovering 2024-04-12T11:46:03.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:03 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:46:03.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:03 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3848622093' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:46:03.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:03 smithi029 ceph-mon[97787]: osdmap e1212: 8 total, 8 up, 8 in 2024-04-12T11:46:03.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 10059/118593 objects degraded (8.482%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:03.293 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:03.331 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:03.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:03.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:03.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:03.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:03 smithi087 ceph-mon[89014]: pgmap v1826: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 10059/118593 objects degraded (8.482%); 19514/118593 objects misplaced (16.455%); 11 MiB/s, 1 keys/s, 170 objects/s recovering 2024-04-12T11:46:03.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:03 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:46:03.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:03 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3848622093' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:46:03.469 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:03 smithi087 ceph-mon[89014]: osdmap e1212: 8 total, 8 up, 8 in 2024-04-12T11:46:03.470 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 10059/118593 objects degraded (8.482%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:03.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:03.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:03.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:03.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:03.607 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:03.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:03.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:03.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:03.758 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:03.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:03.860 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:03.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:03.894 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:03.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:03.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:04.036 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:04.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:04.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:04.138 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:04.176 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:04.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:04.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:04.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:04.315 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:04.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:04.415 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:04.446 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:04.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:04.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:04.512 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:04.548 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:04.590 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:04.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:04.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:04.693 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:04.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:04.724 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:04.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:04.818 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:04.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:04.858 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:04.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:04.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:04.949 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:46:04.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:05.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:05.035 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:05.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:05.136 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:05.188 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:05.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:05.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:05.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:05 smithi029 ceph-mon[97787]: pgmap v1828: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9770/114114 objects degraded (8.562%); 19272/114114 objects misplaced (16.888%); 10 MiB/s, 1 keys/s, 162 objects/s recovering 2024-04-12T11:46:05.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:05 smithi029 ceph-mon[93650]: pgmap v1828: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9770/114114 objects degraded (8.562%); 19272/114114 objects misplaced (16.888%); 10 MiB/s, 1 keys/s, 162 objects/s recovering 2024-04-12T11:46:05.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:46:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:46:04] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2024-04-12T11:46:05.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:05.332 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:05.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:05.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:05.434 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:05 smithi087 ceph-mon[89014]: pgmap v1828: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9770/114114 objects degraded (8.562%); 19272/114114 objects misplaced (16.888%); 10 MiB/s, 1 keys/s, 162 objects/s recovering 2024-04-12T11:46:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:05.470 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:05.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:05.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:05.603 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:05.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:05.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:05.705 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:05.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:05.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:05.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:05.760 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:46:05.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:05.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:05.880 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:05.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:05.982 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:06.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:06.015 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:06.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:06.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:06.153 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:06.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:06.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:06.255 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:06.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:06.293 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:06.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:06.394 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:06.431 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:06.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:06.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:06.533 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:06.570 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:06.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:06.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:06.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:06.708 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:06.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:06.809 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:06.845 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:06.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:06.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:06.984 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:06.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:06.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:07.086 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:07.118 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:07.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:07.147 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:07 smithi029 ceph-mon[97787]: pgmap v1829: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 9460/104469 objects degraded (9.055%); 18449/104469 objects misplaced (17.660%); 1.5 MiB/s, 0 keys/s, 24 objects/s recovering 2024-04-12T11:46:07.148 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:07.148 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:07.148 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:07.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:07 smithi029 ceph-mon[93650]: pgmap v1829: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 9460/104469 objects degraded (9.055%); 18449/104469 objects misplaced (17.660%); 1.5 MiB/s, 0 keys/s, 24 objects/s recovering 2024-04-12T11:46:07.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:07.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:07.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:07.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:07.269 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:07.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:07.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:07.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:07.411 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:07.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:07.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:07 smithi087 ceph-mon[89014]: pgmap v1829: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 9460/104469 objects degraded (9.055%); 18449/104469 objects misplaced (17.660%); 1.5 MiB/s, 0 keys/s, 24 objects/s recovering 2024-04-12T11:46:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:07.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:07.576 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:07.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:07.677 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:07.719 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:07.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:07.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:07.821 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:07.862 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:07.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:07.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:07.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:07.997 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:07.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:08.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:08.157 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:08.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:08.258 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:08.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9460/104469 objects degraded (9.055%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:08.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9460/104469 objects degraded (9.055%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:08.292 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:08.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:08.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:08.394 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:08.429 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:08.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:08.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:08.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9460/104469 objects degraded (9.055%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:08.532 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:08.574 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:08.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:08.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:08.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:08.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:08.720 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:08.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:08.821 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:08.865 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:08.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:08.966 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:09.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:09.006 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:09.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:09.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T11:46:08.850+0000 7fae24212700 -1 mon.a@0(leader).osd e1212 definitely_dead 0 2024-04-12T11:46:09.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:09.131 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:09.132 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:09.146 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:09.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:09.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:09.233 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:46:09.234 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:46:09.248 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:09.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[97787]: pgmap v1830: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 9178/100158 objects degraded (9.164%); 18191/100158 objects misplaced (18.162%); 1.2 MiB/s, 19 objects/s recovering 2024-04-12T11:46:09.271 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:46:09.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:46:09.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:09.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2432302537' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2024-04-12T11:46:09.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[97787]: Client client.admin marked osd.3 out, while it was still marked up 2024-04-12T11:46:09.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[93650]: pgmap v1830: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 9178/100158 objects degraded (9.164%); 18191/100158 objects misplaced (18.162%); 1.2 MiB/s, 19 objects/s recovering 2024-04-12T11:46:09.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:46:09.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:46:09.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:09.273 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2432302537' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2024-04-12T11:46:09.273 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:09 smithi029 ceph-mon[93650]: Client client.admin marked osd.3 out, while it was still marked up 2024-04-12T11:46:09.287 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:09.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:09.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:09.381 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.3. 2024-04-12T11:46:09.388 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:09.429 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:09.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:09.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:09 smithi087 ceph-mon[89014]: pgmap v1830: 99 pgs: 1 active+recovering+undersized+remapped, 86 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 943 op/s; 9178/100158 objects degraded (9.164%); 18191/100158 objects misplaced (18.162%); 1.2 MiB/s, 19 objects/s recovering 2024-04-12T11:46:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:46:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:46:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2432302537' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2024-04-12T11:46:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:09 smithi087 ceph-mon[89014]: Client client.admin marked osd.3 out, while it was still marked up 2024-04-12T11:46:09.531 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:09.555 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:09.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:09.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:09.656 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:09.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:09.681 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:09.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:09.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:09.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:09.806 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:09.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:09.908 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:09.931 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:09.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:09.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:10.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:10.057 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:10.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:10.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:10.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:10.184 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:10.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:10.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:10.285 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:10.309 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:10.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:10.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:10.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:10.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:10.450 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:10.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:10.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:10.576 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:10.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:10.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:10.678 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:10.701 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:10.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:10.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2432302537' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-04-12T11:46:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:10 smithi087 ceph-mon[89014]: osdmap e1213: 8 total, 8 up, 7 in 2024-04-12T11:46:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2432302537' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-04-12T11:46:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:10 smithi029 ceph-mon[97787]: osdmap e1213: 8 total, 8 up, 7 in 2024-04-12T11:46:10.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2432302537' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-04-12T11:46:10.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:10 smithi029 ceph-mon[93650]: osdmap e1213: 8 total, 8 up, 7 in 2024-04-12T11:46:10.802 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:10.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:10.827 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:10.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:10.928 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:10.952 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:10.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:10.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:11.054 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:11.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:11.078 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:11.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:11.179 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:11.203 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:11.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:11.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:11.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:11.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:11.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:11.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:11.431 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:11.461 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:11.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:11.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:11.596 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:11.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:11.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:11.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:11 smithi087 ceph-mon[89014]: pgmap v1832: 99 pgs: 2 remapped+peering, 4 peering, 1 active+recovering+undersized+remapped, 83 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 9178/95085 objects degraded (9.652%); 15538/95085 objects misplaced (16.341%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T11:46:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:11 smithi087 ceph-mon[89014]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:46:11.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:11 smithi087 ceph-mon[89014]: osdmap e1214: 8 total, 8 up, 7 in 2024-04-12T11:46:11.734 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:11.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:11.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:11 smithi029 ceph-mon[97787]: pgmap v1832: 99 pgs: 2 remapped+peering, 4 peering, 1 active+recovering+undersized+remapped, 83 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 9178/95085 objects degraded (9.652%); 15538/95085 objects misplaced (16.341%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T11:46:11.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:11 smithi029 ceph-mon[97787]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:46:11.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:11 smithi029 ceph-mon[97787]: osdmap e1214: 8 total, 8 up, 7 in 2024-04-12T11:46:11.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:11 smithi029 ceph-mon[93650]: pgmap v1832: 99 pgs: 2 remapped+peering, 4 peering, 1 active+recovering+undersized+remapped, 83 active+clean, 5 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 9178/95085 objects degraded (9.652%); 15538/95085 objects misplaced (16.341%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T11:46:11.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:11 smithi029 ceph-mon[93650]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:46:11.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:11 smithi029 ceph-mon[93650]: osdmap e1214: 8 total, 8 up, 7 in 2024-04-12T11:46:11.837 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:11.878 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:11.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:11.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:11.979 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:12.016 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:12.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:12.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:12.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:12.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:12.151 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:12.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:12.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:12.288 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:12.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:12.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:12.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:12.425 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:12.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:12.526 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:12.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:12.560 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:12.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:12.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:12.690 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:12.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:12.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:46:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:46:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:46:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:12 smithi087 ceph-mon[89014]: 8.b starting backfill to osd.2 from (0'0,0'0] MAX to 1212'11676 2024-04-12T11:46:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:12 smithi087 ceph-mon[89014]: osdmap e1215: 8 total, 8 up, 7 in 2024-04-12T11:46:12.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:12 smithi029 ceph-mon[93650]: 8.b starting backfill to osd.2 from (0'0,0'0] MAX to 1212'11676 2024-04-12T11:46:12.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:12 smithi029 ceph-mon[93650]: osdmap e1215: 8 total, 8 up, 7 in 2024-04-12T11:46:12.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:12 smithi029 ceph-mon[97787]: 8.b starting backfill to osd.2 from (0'0,0'0] MAX to 1212'11676 2024-04-12T11:46:12.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:12 smithi029 ceph-mon[97787]: osdmap e1215: 8 total, 8 up, 7 in 2024-04-12T11:46:12.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:12.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:12.907 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:12.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:13.007 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:13.050 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:13.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:13.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:13.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:13.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:13.198 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:13.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:13.300 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:13.335 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:13.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:13.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:13.438 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:13.471 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:13.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:13.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:13.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:13 smithi029 ceph-mon[97787]: pgmap v1835: 99 pgs: 2 remapped+peering, 13 peering, 1 active+recovering+undersized+remapped, 75 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.9 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 7106/91863 objects degraded (7.735%); 10379/91863 objects misplaced (11.298%); 1.1 MiB/s, 16 objects/s recovering 2024-04-12T11:46:13.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 7106/91863 objects degraded (7.735%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:13.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:13 smithi029 ceph-mon[93650]: pgmap v1835: 99 pgs: 2 remapped+peering, 13 peering, 1 active+recovering+undersized+remapped, 75 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.9 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 7106/91863 objects degraded (7.735%); 10379/91863 objects misplaced (11.298%); 1.1 MiB/s, 16 objects/s recovering 2024-04-12T11:46:13.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 7106/91863 objects degraded (7.735%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:13.573 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:13.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:13.609 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:13.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:13.710 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:13 smithi087 ceph-mon[89014]: pgmap v1835: 99 pgs: 2 remapped+peering, 13 peering, 1 active+recovering+undersized+remapped, 75 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.9 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 700 op/s; 7106/91863 objects degraded (7.735%); 10379/91863 objects misplaced (11.298%); 1.1 MiB/s, 16 objects/s recovering 2024-04-12T11:46:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 7106/91863 objects degraded (7.735%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:13.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:13.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:13.856 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:13.899 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:13.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:13.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:14.001 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:14.032 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:14.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:14.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:14.134 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:14.170 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:14.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:14.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:14.273 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:14.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:14.297 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:14.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:14.398 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:14.430 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:14.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:14.531 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:14.561 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:14.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:14.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:14.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:14.694 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:14.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:14.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:14.821 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:14.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:14.923 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:14.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:14.958 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:14.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:15.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:15.087 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:15.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:15.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:15.189 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:15.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:15.219 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:15.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:46:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:46:14] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2024-04-12T11:46:15.320 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:15.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:15.350 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:15.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:15.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:15.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:15.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:15.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:15.585 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:15.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:15.629 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:15.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:15.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:15 smithi087 ceph-mon[89014]: pgmap v1836: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 13 peering, 77 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 10194/89103 objects degraded (11.441%); 5045/89103 objects misplaced (5.662%); 2.3 MiB/s, 37 objects/s recovering 2024-04-12T11:46:15.730 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:15.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:15.767 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:15.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:15.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:15 smithi029 ceph-mon[93650]: pgmap v1836: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 13 peering, 77 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 10194/89103 objects degraded (11.441%); 5045/89103 objects misplaced (5.662%); 2.3 MiB/s, 37 objects/s recovering 2024-04-12T11:46:15.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:15 smithi029 ceph-mon[97787]: pgmap v1836: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 13 peering, 77 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait; 1.8 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 10194/89103 objects degraded (11.441%); 5045/89103 objects misplaced (5.662%); 2.3 MiB/s, 37 objects/s recovering 2024-04-12T11:46:15.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:15.899 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:15.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:15.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:16.001 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:16.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:16.038 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:16.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:16.139 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:16.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:16.164 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:16.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:16.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:16.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:16.292 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:16.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:16.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:16.417 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:16.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:16.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:16.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:16.557 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:16.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:16.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:16.660 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:16.699 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:16.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:16.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:16 smithi087 ceph-mon[89014]: pgmap v1837: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 15245/83355 objects degraded (18.289%); 4179/83355 objects misplaced (5.013%); 2.1 MiB/s, 34 objects/s recovering 2024-04-12T11:46:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:16 smithi087 ceph-mon[89014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-04-12T11:46:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:16 smithi029 ceph-mon[93650]: pgmap v1837: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 15245/83355 objects degraded (18.289%); 4179/83355 objects misplaced (5.013%); 2.1 MiB/s, 34 objects/s recovering 2024-04-12T11:46:16.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:16 smithi029 ceph-mon[93650]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-04-12T11:46:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:16 smithi029 ceph-mon[97787]: pgmap v1837: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 15245/83355 objects degraded (18.289%); 4179/83355 objects misplaced (5.013%); 2.1 MiB/s, 34 objects/s recovering 2024-04-12T11:46:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:16 smithi029 ceph-mon[97787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-04-12T11:46:16.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:16.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:16.825 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:16.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:16.854 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:46:16.855 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:46:16.855 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-04-12T11:46:16.855 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-04-12T11:46:16.855 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-04-12T11:46:16.855 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-04-12T11:46:16.925 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:16.953 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:16.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:16.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:17.055 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:17.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:17.098 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:17.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:17.199 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:17.232 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:17.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:17.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:17.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:17.364 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:17.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:17.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:17.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:17.467 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:46:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:17.495 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:17.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:17.597 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:17.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:17.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:17.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:17.724 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:17.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:17.752 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:17.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:17.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:17.886 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:17.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:17.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:18.028 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:18.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:18.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:18.055 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-04-12T11:46:18.130 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:18.166 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:18.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:18.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:18.268 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:18.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15245/83355 objects degraded (18.289%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:18.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15245/83355 objects degraded (18.289%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:18.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:18.308 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:18.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:18.409 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:18.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:18.434 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:18.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:18.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15245/83355 objects degraded (18.289%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:18.535 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:18.559 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:18.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:18.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:18.661 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:18.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:18.686 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:18.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:18.787 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:18.821 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:18.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:18.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:18.926 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:18.961 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:18.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:18.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:19.063 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:19.100 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:19.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:19.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:19.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:19 smithi087 ceph-mon[89014]: pgmap v1838: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 15245/79572 objects degraded (19.159%); 3884/79572 objects misplaced (4.881%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:46:19.230 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:19.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:19.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:19.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:19 smithi029 ceph-mon[97787]: pgmap v1838: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 15245/79572 objects degraded (19.159%); 3884/79572 objects misplaced (4.881%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:46:19.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:19 smithi029 ceph-mon[93650]: pgmap v1838: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.6 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 15245/79572 objects degraded (19.159%); 3884/79572 objects misplaced (4.881%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:46:19.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:19.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:19.367 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:19.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:19.468 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:19.492 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:19.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:19.594 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:19.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:19.619 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:19.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:19.720 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:19.744 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:19.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:19.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:19.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:19.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:19.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:19.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:19.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:20.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:20.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:20.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:20.116 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:20.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:20.157 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:20.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:20.258 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:20.282 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:20.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:20.384 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:20.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:20.417 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:20.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:20.518 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:20.559 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:20.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:20.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:20.661 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:20.703 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:20.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:20.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:20.827 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:20.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:20.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:20.929 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:20.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:20.954 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:20.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:21.055 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:21.091 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:21.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:21.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:21.193 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:21.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:21.226 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:21.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:21.327 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:21.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:21.368 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:21.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:21 smithi087 ceph-mon[89014]: pgmap v1839: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 15245/77277 objects degraded (19.728%); 3884/77277 objects misplaced (5.026%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T11:46:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:21 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/371606639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:21.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:21.498 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:21.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:21 smithi029 ceph-mon[93650]: pgmap v1839: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 15245/77277 objects degraded (19.728%); 3884/77277 objects misplaced (5.026%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T11:46:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:21 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/371606639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:21 smithi029 ceph-mon[97787]: pgmap v1839: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 15245/77277 objects degraded (19.728%); 3884/77277 objects misplaced (5.026%); 2.7 MiB/s, 43 objects/s recovering 2024-04-12T11:46:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:21 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/371606639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:21.600 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:21.639 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:21.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:21.740 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:21.780 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:21.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:21.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:21.884 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:21.927 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:21.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:21.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:22.029 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:22.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:22.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:22.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:22.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:22.190 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:22.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:22.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:22.292 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:22.316 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:22.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:22.329 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:22.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:22.430 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:22.441 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:46:22.441 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:46:22.441 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:46:22.441 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:46:22.442 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:46:22.442 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:46:22.442 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:46:22.442 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:46:22.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:46:22.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:46:22.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:46:22.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:46:22.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:46:22.442 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:46:22.443 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:46:22.443 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:46:22.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:22.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:22.460 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:22.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:22.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:22.562 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:22.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:22.602 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:22.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:22.703 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:46:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:46:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:46:22.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:22.743 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:22.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:22.844 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:22.868 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:22.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:22.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:22.970 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:23.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:23.011 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:23.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:23.110 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:23 smithi087 ceph-mon[89014]: pgmap v1840: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 15245/74085 objects degraded (20.578%); 3620/74085 objects misplaced (4.886%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:46:23.111 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15245/74085 objects degraded (20.578%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:23.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:23.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:23.136 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:23.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:23.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:23.269 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:23.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:23.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:23.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:23.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:23.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:23.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:23.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:23.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:23 smithi029 ceph-mon[93650]: pgmap v1840: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 15245/74085 objects degraded (20.578%); 3620/74085 objects misplaced (4.886%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:46:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15245/74085 objects degraded (20.578%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:23.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:23 smithi029 ceph-mon[97787]: pgmap v1840: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 15245/74085 objects degraded (20.578%); 3620/74085 objects misplaced (4.886%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:46:23.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15245/74085 objects degraded (20.578%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:23.528 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:23.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:23.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:23.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:23.654 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:23.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:23.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:23.756 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:23.784 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:23.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:23.886 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:23.928 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:23.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:23.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:24.030 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:24.070 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:24.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:24.171 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:24.217 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:24.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:24.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:24.319 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:24.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:24.372 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:24.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:24.473 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:24.524 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:24.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:24.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:24.626 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:24.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:24.653 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:24.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:24.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:24.789 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:24.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:24.891 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:24.924 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:24.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:24.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:25.003 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:46:25.003 INFO:teuthology.orchestra.run.smithi087.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-04-12T11:46:25.003 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:46:25.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:25.076 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:25.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:25.177 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:25.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:25.213 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:25.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:25.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:25 smithi029 ceph-mon[97787]: pgmap v1841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 15245/70521 objects degraded (21.618%); 3339/70521 objects misplaced (4.735%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:46:25.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:25 smithi029 ceph-mon[93650]: pgmap v1841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 15245/70521 objects degraded (21.618%); 3339/70521 objects misplaced (4.735%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:46:25.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:46:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:46:24] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2024-04-12T11:46:25.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:25.352 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:25.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:25.453 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:25.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:25 smithi087 ceph-mon[89014]: pgmap v1841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 15245/70521 objects degraded (21.618%); 3339/70521 objects misplaced (4.735%); 3.2 MiB/s, 51 objects/s recovering 2024-04-12T11:46:25.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:25.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:25.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:25.592 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:25.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:25.631 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:25.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:25.732 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:25.764 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:25.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:25.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:25.866 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:25.891 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:25.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:25.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:25.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:26.030 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:26.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:26.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:26.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:26.164 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:26.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:26.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:26.266 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:26.307 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:26.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:26.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:26.409 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:26.443 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:26.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:26.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:26.544 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:26.583 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:26.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:26.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:26.685 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:26.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:26.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:26.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:26.840 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:26.884 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:26.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:26.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:26.986 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:27.023 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:27.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:27.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:27.125 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:27.157 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:27.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:27.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:27.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:27 smithi029 ceph-mon[93650]: pgmap v1842: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 15245/66144 objects degraded (23.048%); 3130/66144 objects misplaced (4.732%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:46:27.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:27 smithi029 ceph-mon[97787]: pgmap v1842: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 15245/66144 objects degraded (23.048%); 3130/66144 objects misplaced (4.732%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:46:27.259 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:27.298 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:27.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:27.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:27.389 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:27.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:27.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:27.467 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:27.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:27.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:27 smithi087 ceph-mon[89014]: pgmap v1842: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 15245/66144 objects degraded (23.048%); 3130/66144 objects misplaced (4.732%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:46:27.569 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:27.604 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:27.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:27.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:27.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:27.756 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:27.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:27.857 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:27.894 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:27.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:27.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:27.996 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:28.032 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:28.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:28.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:28.050 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:46:28.134 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:28.171 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:28.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:28.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:28 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/790361219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:28.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:28 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:28.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15245/66144 objects degraded (23.048%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:28.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:28 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/790361219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:28.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:28 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:28.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15245/66144 objects degraded (23.048%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:28.271 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:28.309 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:28.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:28.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:28.411 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:28.449 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:28.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:28 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/790361219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:28 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15245/66144 objects degraded (23.048%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:28.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:28.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:28.588 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:28.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:28.689 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:28.720 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:28.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:28.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:28.741 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:46:28.822 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:28.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:28.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:28.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:28.962 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:28.992 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:28.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:28.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:29.094 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:29.132 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:29.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:29.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:29.234 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:29.269 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:29.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:29.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:29.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:29.406 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:29.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:29.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:29 smithi087 ceph-mon[89014]: pgmap v1843: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 15167/60093 objects degraded (25.239%); 2742/60093 objects misplaced (4.563%); 3.3 MiB/s, 53 objects/s recovering 2024-04-12T11:46:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:29 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:46:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:29 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:46:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:29 smithi087 ceph-mon[89014]: osdmap e1216: 8 total, 8 up, 7 in 2024-04-12T11:46:29.508 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:29.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:29 smithi029 ceph-mon[97787]: pgmap v1843: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 15167/60093 objects degraded (25.239%); 2742/60093 objects misplaced (4.563%); 3.3 MiB/s, 53 objects/s recovering 2024-04-12T11:46:29.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:29 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:46:29.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:29 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:46:29.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:29 smithi029 ceph-mon[97787]: osdmap e1216: 8 total, 8 up, 7 in 2024-04-12T11:46:29.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:29 smithi029 ceph-mon[93650]: pgmap v1843: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 15167/60093 objects degraded (25.239%); 2742/60093 objects misplaced (4.563%); 3.3 MiB/s, 53 objects/s recovering 2024-04-12T11:46:29.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:29 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:46:29.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:29 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:46:29.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:29 smithi029 ceph-mon[93650]: osdmap e1216: 8 total, 8 up, 7 in 2024-04-12T11:46:29.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:29.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:29.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:29.643 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:29.648 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:29.682 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:29.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:29.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:29.784 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:29.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:29.821 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:29.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:29.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:29.957 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:29.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:30.059 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:30.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:30.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:30.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:30.196 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:30.232 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:30.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:30.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:30.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:30.372 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:30.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:30.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:30.473 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:30.504 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:46:30.504 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:46:30.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:46:30.504 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:46:30.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.504 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.44843.0:91871 7.6 7:6d8cb788:::benchmark_data_smithi029_161157_object91870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:46:30.504 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:38:20.949255+0000", 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 489.54908650099998, 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5011506839999997, 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.44843", 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91871 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.949255+0000", 2024-04-12T11:46:30.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.949255+0000", 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.949257+0000", 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8959999999999999e-06 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.949278+0000", 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1160000000000001e-05 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.949294+0000", 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6396e-05 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.949304+0000", 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1589999999999999e-06 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.954050+0000", 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047460469999999998 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.954062+0000", 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2223e-05 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929115+0000", 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97505345099999996 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929129+0000", 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3665999999999999e-05 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.434817+0000", 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.505688422 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.435000+0000", 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018226700000000001 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.435208+0000", 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020837399999999999 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.445555+0000", 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010347264 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.445642+0000", 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6311999999999996e-05 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.445727+0000", 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5779000000000007e-05 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.445803+0000", 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5933000000000006e-05 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.450406+0000", 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046023349999999999 2024-04-12T11:46:30.511 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.44843.0:91892 7.6 7:6d02590a:::benchmark_data_smithi029_161157_object91891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:38:20.973235+0000", 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 489.52510652799998, 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4909189239999998, 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.44843", 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:46:30.512 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91892 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973235+0000", 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973235+0000", 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973235+0000", 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6700000000000003e-07 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973257+0000", 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1047999999999999e-05 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973264+0000", 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9070000000000007e-06 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973272+0000", 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1760000000000004e-06 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973317+0000", 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5849000000000001e-05 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973326+0000", 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1710000000000002e-06 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929196+0000", 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95587051300000003 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929201+0000", 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6689999999999997e-06 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.450467+0000", 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5212666559999999 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.450633+0000", 2024-04-12T11:46:30.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016593599999999999 2024-04-12T11:46:30.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T11:46:30.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.450815+0000", 2024-04-12T11:46:30.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018203300000000001 2024-04-12T11:46:30.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.518 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.459346+0000", 2024-04-12T11:46:30.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0085301430000000004 2024-04-12T11:46:30.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.459361+0000", 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5739e-05 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.459403+0000", 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1282000000000002e-05 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.459466+0000", 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3597999999999997e-05 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.464154+0000", 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046876369999999997 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.44843.0:91893 7.6 7:6aa456f4:::benchmark_data_smithi029_161157_object91892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:38:20.973600+0000", 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 489.52474096399999, 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5405596699999999, 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.520 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.44843", 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91893 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973600+0000", 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973600+0000", 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973601+0000", 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3200000000000004e-07 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973623+0000", 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1214999999999999e-05 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973633+0000", 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0851e-05 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973642+0000", 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2570000000000007e-06 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973714+0000", 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2288999999999999e-05 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:20.973724+0000", 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7739999999999999e-06 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929214+0000", 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95549076099999997 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929219+0000", 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5050000000000001e-06 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.464204+0000", 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.534984844 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.464359+0000", 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000154881 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.464525+0000", 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016645399999999999 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.474535+0000", 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010009783 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.474628+0000", 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2744999999999994e-05 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.509151+0000", 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.034523826000000001 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.509209+0000", 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7351e-05 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.514160+0000", 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049513020000000003 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.44843.0:91938 7.6 7:68679bdd:::benchmark_data_smithi029_161157_object91937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:46:30.527 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:38:21.018038+0000", 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 489.48030375899998, 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5014712129999999, 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.44843", 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91938 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.018038+0000", 2024-04-12T11:46:30.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.018038+0000", 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.018039+0000", 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4169999999999999e-06 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.018061+0000", 2024-04-12T11:46:30.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2067999999999999e-05 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.018078+0000", 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6713000000000001e-05 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.018086+0000", 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4500000000000004e-06 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.018154+0000", 2024-04-12T11:46:30.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8088000000000003e-05 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.018165+0000", 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0742e-05 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929232+0000", 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.911066504 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929236+0000", 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2409999999999997e-06 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.467523+0000", 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.53828758 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.467668+0000", 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014460100000000001 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T11:46:30.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.467834+0000", 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000166367 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.474684+0000", 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0068499420000000004 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.479151+0000", 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044668620000000003 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.514218+0000", 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.035067095999999999 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.514281+0000", 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2589999999999995e-05 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.519509+0000", 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052279520000000001 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.534 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.44843.0:91944 7.6 7:6d3c97a3:::benchmark_data_smithi029_161157_object91943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:38:21.023180+0000", 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 489.47516121899997, 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5012275119999998, 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.44843", 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91944 2024-04-12T11:46:30.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.023180+0000", 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.023180+0000", 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.023181+0000", 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0049999999999999e-06 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.023208+0000", 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6452e-05 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.023218+0000", 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9329999999999996e-06 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.023225+0000", 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8329999999999994e-06 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.023295+0000", 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9338999999999995e-05 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.023308+0000", 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.278e-05 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929248+0000", 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90594089700000002 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929253+0000", 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4390000000000002e-06 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.471161+0000", 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5419084189999999 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.471345+0000", 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000183664 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.471538+0000", 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019279900000000001 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.474740+0000", 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003201855 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.479161+0000", 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044209239999999997 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.519569+0000", 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.040408334999999997 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.519623+0000", 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4109e-05 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.524408+0000", 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047847289999999997 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.44843.0:91949 7.6 7:6dea77b4:::benchmark_data_smithi029_161157_object91948:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1146)", 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:38:21.027280+0000", 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 489.47106154599999, 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5048516599999999, 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.542 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.44843", 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3312790213", 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91949 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.027280+0000", 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.027280+0000", 2024-04-12T11:46:30.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:30.545 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:30.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.027281+0000", 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.068e-06 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.027301+0000", 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9715e-05 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.027308+0000", 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4900000000000003e-06 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.027314+0000", 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1469999999999998e-06 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.027369+0000", 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4617000000000002e-05 2024-04-12T11:46:30.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.027376+0000", 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8329999999999996e-06 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929270+0000", 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90189450900000001 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:21.929278+0000", 2024-04-12T11:46:30.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7770000000000002e-06 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.474777+0000", 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5454992750000001 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.474954+0000", 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017619599999999999 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.475153+0000", 2024-04-12T11:46:30.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019994500000000001 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.479560+0000", 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004406887 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.479849+0000", 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00028822599999999999 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.527315+0000", 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.047466795999999999 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.527377+0000", 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1731999999999998e-05 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:38:23.532132+0000", 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047544470000000002 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86173 8.c 8:3a57f77c:::benchmark_data_smithi029_194418_object86172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:55.135336+0000", 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 155.36300534099999, 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1975276350000001, 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86173 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.552 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135336+0000", 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135336+0000", 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135338+0000", 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5659999999999999e-06 2024-04-12T11:46:30.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135360+0000", 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2385999999999999e-05 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135375+0000", 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5014000000000001e-05 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135382+0000", 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4290000000000004e-06 2024-04-12T11:46:30.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135436+0000", 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3847999999999997e-05 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135623+0000", 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018665899999999999 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135802+0000", 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017909800000000001 2024-04-12T11:46:30.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.144228+0000", 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084262569999999995 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.148863+0000", 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046350970000000003 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.161556+0000", 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012692075000000001 2024-04-12T11:46:30.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.161570+0000", 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4482e-05 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.138838+0000", 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97726798400000003 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.138854+0000", 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.611e-05 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.332789+0000", 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.19393528 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.332864+0000", 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4350000000000005e-05 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.558 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86183 8.c 8:359770e0:::benchmark_data_smithi029_194418_object86182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:55.164650+0000", 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 155.333691915, 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1898649269999999, 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86183 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164650+0000", 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164650+0000", 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164650+0000", 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8899999999999999e-07 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164675+0000", 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5018e-05 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164683+0000", 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0479999999999999e-06 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164690+0000", 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.494e-06 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164770+0000", 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0673999999999994e-05 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164783+0000", 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2289e-05 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.139700+0000", 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.974917755 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.139712+0000", 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1844e-05 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.336155+0000", 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1964429970000001 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.336322+0000", 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000167233 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5", 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.336494+0000", 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000171206 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.342386+0000", 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058923559999999996 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.342480+0000", 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3707000000000003e-05 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.351322+0000", 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0088422550000000003 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.351382+0000", 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0513999999999997e-05 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.354514+0000", 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031319479999999999 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.565 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86185 8.6 8:67dce7ce:::benchmark_data_smithi029_194418_object86184:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:55.167892+0000", 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 155.33044951799999, 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1736517800000001, 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86185 2024-04-12T11:46:30.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167892+0000", 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167892+0000", 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167894+0000", 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5850000000000001e-06 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167914+0000", 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0137999999999999e-05 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167924+0000", 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0271e-05 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167930+0000", 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5180000000000001e-06 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.173181+0000", 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005250083 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.173191+0000", 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0868e-05 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141604+0000", 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96841302900000004 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141617+0000", 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2167000000000001e-05 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.327809+0000", 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1861921360000001 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.327970+0000", 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000161464 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.328086+0000", 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115804 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.337011+0000", 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089251799999999996 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.337034+0000", 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3028e-05 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.337074+0000", 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9662000000000001e-05 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.337116+0000", 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1924e-05 2024-04-12T11:46:30.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.341544+0000", 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044279230000000003 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86186 8.6 8:62e71c40:::benchmark_data_smithi029_194418_object86185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:55.167943+0000", 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 155.33039876300001, 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.180328485, 2024-04-12T11:46:30.573 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86186 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167943+0000", 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167943+0000", 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167943+0000", 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9999999999999998e-07 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167960+0000", 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7036000000000001e-05 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167967+0000", 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6080000000000001e-06 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167972+0000", 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5910000000000003e-06 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.173221+0000", 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005248356 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.173229+0000", 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0139999999999998e-06 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141646+0000", 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96841718899999996 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141655+0000", 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4879999999999996e-06 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.334011+0000", 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.192355271 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.334193+0000", 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018225399999999999 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.334396+0000", 2024-04-12T11:46:30.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020332999999999999 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.341686+0000", 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072902039999999998 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.341752+0000", 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5209999999999994e-05 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.344686+0000", 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029342679999999999 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.344726+0000", 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0436999999999999e-05 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.348271+0000", 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035448290000000002 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86189 8.6 8:69abc3bd:::benchmark_data_smithi029_194418_object86188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:55.168015+0000", 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 155.330326259, 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.183327738, 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86189 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168015+0000", 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168015+0000", 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168016+0000", 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5999999999999999e-07 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168035+0000", 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9789999999999999e-05 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168042+0000", 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1809999999999998e-06 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168046+0000", 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9799999999999998e-06 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.181852+0000", 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013805784999999999 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.181863+0000", 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0898e-05 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141682+0000", 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95981845899999996 2024-04-12T11:46:30.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141691+0000", 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1929999999999999e-06 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.341802+0000", 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.200110867 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.341985+0000", 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018331000000000001 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.342199+0000", 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021373900000000001 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.348371+0000", 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061724989999999997 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.348382+0000", 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1092e-05 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.348412+0000", 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9864000000000001e-05 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.348452+0000", 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0018999999999997e-05 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.351343+0000", 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028907020000000002 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.588 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:121589 8.6 8:671b29b8:::benchmark_data_smithi029_194418_object15279:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:44.398957+0000", 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 106.099384165, 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0599646630000001, 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 121589 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.589 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398957+0000", 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398957+0000", 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398959+0000", 2024-04-12T11:46:30.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9599999999999999e-06 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398969+0000", 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6709999999999994e-06 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398985+0000", 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5784999999999999e-05 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398997+0000", 2024-04-12T11:46:30.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2522e-05 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.657919+0000", 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25892183699999999 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.657929+0000", 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5419999999999994e-06 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.391598+0000", 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73366960800000003 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.391612+0000", 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4151000000000001e-05 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.427659+0000", 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.036046614 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.427796+0000", 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013716499999999999 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.458097+0000", 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030300494000000001 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.458922+0000", 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00082531399999999995 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.594 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168251 8.c 8:3d814e3d:::benchmark_data_smithi029_194418_object61941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.323733+0000", 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 38.174608141, 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.010112119, 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168251 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.595 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323733+0000", 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323733+0000", 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323735+0000", 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4589999999999999e-06 2024-04-12T11:46:30.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323741+0000", 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0070000000000003e-06 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323757+0000", 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5869999999999999e-05 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323765+0000", 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0649999999999999e-06 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.360710+0000", 2024-04-12T11:46:30.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036944880999999999 2024-04-12T11:46:30.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2024-04-12T11:46:30.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.363394+0000", 2024-04-12T11:46:30.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002683983 2024-04-12T11:46:30.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.378917+0000", 2024-04-12T11:46:30.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015522948 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.379027+0000", 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011003899999999999 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4,5", 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.379202+0000", 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000175716 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394056+0000", 2024-04-12T11:46:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014854192 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394068+0000", 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1245e-05 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.967696+0000", 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.57362859499999996 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.967709+0000", 2024-04-12T11:46:30.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2986000000000001e-05 2024-04-12T11:46:30.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.314657+0000", 2024-04-12T11:46:30.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.346947624 2024-04-12T11:46:30.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2024-04-12T11:46:30.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.314840+0000", 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018301 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.333767+0000", 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018927557000000001 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.333845+0000", 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7941999999999993e-05 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168281 8.16 8:69d01196:::benchmark_data_smithi029_194418_object61971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.354640+0000", 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 38.143701798999999, 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8219508370000002, 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168281 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354640+0000", 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354640+0000", 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354641+0000", 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3090000000000001e-06 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354645+0000", 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4440000000000001e-06 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354656+0000", 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0263e-05 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354664+0000", 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9300000000000003e-06 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354959+0000", 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000295481 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.355057+0000", 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7807999999999996e-05 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.357030+0000", 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019734489999999999 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.364390+0000", 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073593649999999997 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388599+0000", 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024208994000000001 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388612+0000", 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3397000000000001e-05 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973262+0000", 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.58464973899999995 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973272+0000", 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0592e-05 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176509+0000", 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2032365409999999 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176590+0000", 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1525000000000003e-05 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168294 8.c 8:33ff478c:::benchmark_data_smithi029_194418_object61984:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.387000+0000", 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 38.111341039999999, 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9560116860000001, 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.612 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168294 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387000+0000", 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387000+0000", 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387002+0000", 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3769999999999999e-06 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387006+0000", 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.206e-06 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387016+0000", 2024-04-12T11:46:30.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6930000000000002e-06 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387023+0000", 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8180000000000004e-06 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394098+0000", 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0070740580000000003 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394105+0000", 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2269999999999996e-06 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.968399+0000", 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.57429384699999997 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.968408+0000", 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2350000000000005e-06 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.318138+0000", 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3497297449999999 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.318280+0000", 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000142356 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5,7", 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.318409+0000", 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000128713 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.330729+0000", 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012320516 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.330916+0000", 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018688 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.331041+0000", 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012471800000000001 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.341131+0000", 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010089964 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.341357+0000", 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000226734 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.343012+0000", 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016545990000000001 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168295 8.6 8:63357634:::benchmark_data_smithi029_194418_object61985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.387036+0000", 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 38.111305367999996, 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.799481667, 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168295 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387036+0000", 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387036+0000", 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387037+0000", 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1799999999999995e-07 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387039+0000", 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6240000000000002e-06 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387046+0000", 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.888e-06 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387053+0000", 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5780000000000004e-06 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388702+0000", 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001649347 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388711+0000", 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8720000000000008e-06 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973114+0000", 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.58440336500000001 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973125+0000", 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0274e-05 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.165152+0000", 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.192027036 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.165316+0000", 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000164561 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.165826+0000", 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000509495 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170021+0000", 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041956379999999998 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.173298+0000", 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003276251 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.185462+0000", 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012164071 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.185532+0000", 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0222999999999995e-05 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.186518+0000", 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00098592600000000003 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168309 8.16 8:68816e76:::benchmark_data_smithi029_194418_object61999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.404119+0000", 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 38.094222696000003, 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.786614266, 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168309 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404119+0000", 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404119+0000", 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404120+0000", 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.393e-06 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404126+0000", 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9630000000000003e-06 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404169+0000", 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2865999999999999e-05 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404179+0000", 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0002e-05 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404244+0000", 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4998000000000003e-05 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404255+0000", 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0845e-05 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973300+0000", 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56904551800000003 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973305+0000", 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8539999999999996e-06 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176619+0000", 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2033139099999999 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176694+0000", 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4870999999999995e-05 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6,7", 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.177067+0000", 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000373023 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.181264+0000", 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041970030000000004 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.183957+0000", 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026927610000000001 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.184058+0000", 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100765 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190488+0000", 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0064309559999999998 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190552+0000", 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3929000000000004e-05 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190733+0000", 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000180609 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168311 8.6 8:65642b03:::benchmark_data_smithi029_194418_object62001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.408072+0000", 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 38.090269708999998, 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.782363267, 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.634 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168311 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408072+0000", 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408072+0000", 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408073+0000", 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4950000000000001e-06 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408080+0000", 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6250000000000001e-06 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408097+0000", 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7031999999999999e-05 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408107+0000", 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8470000000000008e-06 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408190+0000", 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3578000000000005e-05 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408206+0000", 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5577e-05 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973171+0000", 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56496518699999998 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973182+0000", 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0550000000000001e-05 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170067+0000", 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1968853310000001 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170241+0000", 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017409500000000001 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170963+0000", 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00072238800000000002 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176537+0000", 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055731390000000004 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176553+0000", 2024-04-12T11:46:30.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6648999999999999e-05 2024-04-12T11:46:30.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.188471+0000", 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011917366 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.188530+0000", 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9120000000000003e-05 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190435+0000", 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001905288 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168312 8.6 8:679ba278:::benchmark_data_smithi029_194418_object62002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.409075+0000", 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 38.089266586999997, 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.782290435, 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168312 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409075+0000", 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409075+0000", 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409076+0000", 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2200000000000002e-07 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409080+0000", 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4240000000000003e-06 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409092+0000", 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1416e-05 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409100+0000", 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7260000000000008e-06 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409233+0000", 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013282999999999999 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409245+0000", 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2078e-05 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973213+0000", 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56396750699999998 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973222+0000", 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1900000000000001e-06 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.173473+0000", 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.200250587 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.173613+0000", 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000140218 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.174102+0000", 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00048906700000000004 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.178907+0000", 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048052770000000002 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.181255+0000", 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023480789999999999 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190540+0000", 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092848930000000007 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190596+0000", 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6042999999999999e-05 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.191365+0000", 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00076917800000000003 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168313 8.16 8:6d52a699:::benchmark_data_smithi029_194418_object62003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.410304+0000", 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 38.088037685000003, 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.780861611, 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168313 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:30.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410304+0000", 2024-04-12T11:46:30.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:30.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410304+0000", 2024-04-12T11:46:30.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:30.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410305+0000", 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7499999999999998e-07 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410310+0000", 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7209999999999999e-06 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410327+0000", 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6297999999999999e-05 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410336+0000", 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0359999999999995e-06 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410401+0000", 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5167999999999997e-05 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410413+0000", 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1912999999999999e-05 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973335+0000", 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56292207400000005 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973345+0000", 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.91e-06 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.178899+0000", 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2055542189999999 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:30.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.178983+0000", 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4301999999999993e-05 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6,7", 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.179305+0000", 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032191099999999997 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.181339+0000", 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020332520000000001 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.184110+0000", 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027713640000000001 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.184229+0000", 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011907499999999999 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190999+0000", 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067697510000000001 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.191053+0000", 2024-04-12T11:46:30.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3785000000000001e-05 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.191165+0000", 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011285699999999999 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:30.656 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:46:30.656 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:30.697 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:30.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:30.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:30.713 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:46:30.713 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:46:30.713 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2024-04-12T11:46:30.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:46:30.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:30.838 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:30.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:30.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:30.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:30.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:30.970 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:30.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:31.071 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:31.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:31.095 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:31.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:31.196 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:31.235 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:31.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:31.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:31.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:31.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:31.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:31.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:31.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:31 smithi087 ceph-mon[89014]: pgmap v1845: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 15167/57018 objects degraded (26.600%); 2742/57018 objects misplaced (4.809%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T11:46:31.502 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:31.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:31.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:31.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:31 smithi029 ceph-mon[93650]: pgmap v1845: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 15167/57018 objects degraded (26.600%); 2742/57018 objects misplaced (4.809%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T11:46:31.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:31 smithi029 ceph-mon[97787]: pgmap v1845: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 15167/57018 objects degraded (26.600%); 2742/57018 objects misplaced (4.809%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T11:46:31.603 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:31.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:46:31.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:31.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:31.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:31.735 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:31.771 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:31.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:31.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:31.873 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:31.903 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:31.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:31.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:32.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:32.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:32.036 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:32.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:32.137 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:32.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:32.169 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:32.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:32.271 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:32.319 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:32.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:32.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:32.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:32.463 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:32.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:32.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:32.565 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:32.601 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:32.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:32.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:32.703 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:46:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:46:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:46:32.742 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:32.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:32.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:32.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:32.878 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:32.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:32.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:32.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-04-12T11:46:32.980 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:33.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:33.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:33.120 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:33.159 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:33.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:33.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:33.260 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:33 smithi029 ceph-mon[97787]: pgmap v1846: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 14873/52167 objects degraded (28.510%); 2355/52167 objects misplaced (4.514%); 4.7 MiB/s, 75 objects/s recovering 2024-04-12T11:46:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 14873/52167 objects degraded (28.510%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:33.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:33 smithi029 ceph-mon[93650]: pgmap v1846: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 14873/52167 objects degraded (28.510%); 2355/52167 objects misplaced (4.514%); 4.7 MiB/s, 75 objects/s recovering 2024-04-12T11:46:33.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 14873/52167 objects degraded (28.510%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:33.304 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:33.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:33.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:33.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:33.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:33.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:33.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:33 smithi087 ceph-mon[89014]: pgmap v1846: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 14873/52167 objects degraded (28.510%); 2355/52167 objects misplaced (4.514%); 4.7 MiB/s, 75 objects/s recovering 2024-04-12T11:46:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 14873/52167 objects degraded (28.510%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:33.540 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:33.571 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:33.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:33.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:33.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:33.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:33.749 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:33.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:33.851 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:33.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:33.894 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:33.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:33.995 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:34.035 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:34.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:34.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:34.137 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:34.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:34.172 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:34.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:34.228 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:34.232 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:34.232 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:34.273 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:34.326 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:34.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:34.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:34.347 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:46:34.373 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:46:34.409 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:46:34.428 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:34.440 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:46:34.467 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:34.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:34.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:34.480 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:46:34.519 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:46:34.553 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:46:34.569 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:34.582 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:46:34.588 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:46:34.588 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:46:34.588 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:46:34.588 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:46:34.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:46:34.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:46:34.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:46:34.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:46:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:34.604 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:34.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:34.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:34.736 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:34.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:34.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:34.838 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:34.872 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:34.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:34.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:34.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:35.004 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:35.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:35.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:35.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:46:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:46:34] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2024-04-12T11:46:35.106 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:35.143 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:35.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:35.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:35.245 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:35.275 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:35.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:35.377 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:35.411 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:35.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:35.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:35.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:35 smithi087 ceph-mon[89014]: pgmap v1847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 984 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 14587/47280 objects degraded (30.852%); 2072/47280 objects misplaced (4.382%); 6.2 MiB/s, 99 objects/s recovering 2024-04-12T11:46:35.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:35 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3984997195' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:35.514 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:35.515 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:35 smithi029 ceph-mon[93650]: pgmap v1847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 984 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 14587/47280 objects degraded (30.852%); 2072/47280 objects misplaced (4.382%); 6.2 MiB/s, 99 objects/s recovering 2024-04-12T11:46:35.515 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:35 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3984997195' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:35.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:35 smithi029 ceph-mon[97787]: pgmap v1847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 984 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 14587/47280 objects degraded (30.852%); 2072/47280 objects misplaced (4.382%); 6.2 MiB/s, 99 objects/s recovering 2024-04-12T11:46:35.517 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:35 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3984997195' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:35.551 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:35.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:35.652 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:35.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:35.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:35.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:35.789 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:35.825 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:35.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:35.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:35.927 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:35.966 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:35.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:35.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:36.068 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:36.099 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:36.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:36.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:36.201 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:36.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:36.233 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:36.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:36.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:36.376 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:36.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:36.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:36.477 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:36.508 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:36.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:36.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:36.646 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:36.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:36.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:36.748 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:36.783 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:36.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:36.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:36.885 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:36.918 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:36.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:36.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:37.020 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:37.051 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:37.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:37.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:37.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:37.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:37.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:37.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:37.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:37.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:37.320 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:37.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:37.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:37.460 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:37.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:37.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:37.562 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:37.586 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:37 smithi029 ceph-mon[97787]: pgmap v1848: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 869 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 812 op/s; 14587/41754 objects degraded (34.936%); 1814/41754 objects misplaced (4.344%); 6.2 MiB/s, 99 objects/s recovering 2024-04-12T11:46:37.586 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:37.586 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:37.586 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:37 smithi029 ceph-mon[93650]: pgmap v1848: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 869 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 812 op/s; 14587/41754 objects degraded (34.936%); 1814/41754 objects misplaced (4.344%); 6.2 MiB/s, 99 objects/s recovering 2024-04-12T11:46:37.587 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:37.587 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:37.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:37.600 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:37.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:37.701 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:37 smithi087 ceph-mon[89014]: pgmap v1848: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 869 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 812 op/s; 14587/41754 objects degraded (34.936%); 1814/41754 objects misplaced (4.344%); 6.2 MiB/s, 99 objects/s recovering 2024-04-12T11:46:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:37.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:37.732 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:37.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:37.833 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:37.872 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:37.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:37.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:37.974 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:38.009 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:38.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:38.111 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:38.146 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:38.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:38.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:38.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:38.284 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:38.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:38.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:38.386 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:38.425 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:38.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:38.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:38.526 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:38.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:38.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:38.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:38.573 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 14587/41754 objects degraded (34.936%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:38.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 14587/41754 objects degraded (34.936%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:38.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:38.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:38.708 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:38.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 14587/41754 objects degraded (34.936%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:38.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:38.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:38.847 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:38.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:38.948 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:38.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:38.993 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:38.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:39.094 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:39.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:39.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:39.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:39.233 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:39.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:39.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:39.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:39.336 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:46:39.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:39 smithi029 ceph-mon[97787]: pgmap v1849: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 719 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 14044/34548 objects degraded (40.651%); 1520/34548 objects misplaced (4.400%); 8.1 MiB/s, 143 objects/s recovering 2024-04-12T11:46:39.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:39 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2671598764' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:46:39.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:39 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:46:39.370 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:39 smithi029 ceph-mon[93650]: pgmap v1849: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 719 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 14044/34548 objects degraded (40.651%); 1520/34548 objects misplaced (4.400%); 8.1 MiB/s, 143 objects/s recovering 2024-04-12T11:46:39.370 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:39 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2671598764' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:46:39.370 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:39 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:46:39.373 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:39.418 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:39.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:39.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:39.520 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:39.555 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:39.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:39.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:39.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:39.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:39.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:39.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:39 smithi087 ceph-mon[89014]: pgmap v1849: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 719 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 14044/34548 objects degraded (40.651%); 1520/34548 objects misplaced (4.400%); 8.1 MiB/s, 143 objects/s recovering 2024-04-12T11:46:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:39 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2671598764' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:46:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:39 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:46:39.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:39.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:39.822 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:39.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:39.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:39.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:39.962 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:39.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:40.063 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:40.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:40.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:40.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:40.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:40.230 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:40.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:40.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:40.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:40.367 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:40.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:40.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:40 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:46:40.394 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:40 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:46:40.394 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:40 smithi029 ceph-mon[97787]: osdmap e1217: 8 total, 8 up, 7 in 2024-04-12T11:46:40.394 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:40 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:46:40.394 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:40 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:46:40.394 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:40 smithi029 ceph-mon[93650]: osdmap e1217: 8 total, 8 up, 7 in 2024-04-12T11:46:40.469 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:40.508 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:40.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:40.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:46:40.609 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:40.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:40.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:40 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:46:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:40 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:46:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:40 smithi087 ceph-mon[89014]: osdmap e1217: 8 total, 8 up, 7 in 2024-04-12T11:46:40.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:40.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:40.778 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:40.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:40.879 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:40.918 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:40.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:40.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:41.020 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:41.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:41.059 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:41.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:41.065 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:46:41.065 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:46:41.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:41.195 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:41.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:41.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:41.297 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:41.334 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:41.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:41.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:41.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:41.476 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:41.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:41.578 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:41.621 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:41.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:41.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:41 smithi087 ceph-mon[89014]: pgmap v1851: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 652 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 14044/31338 objects degraded (44.815%); 1520/31338 objects misplaced (4.850%); 8.1 MiB/s, 143 objects/s recovering 2024-04-12T11:46:41.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:41.760 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:41.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:41.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:41 smithi029 ceph-mon[97787]: pgmap v1851: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 652 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 14044/31338 objects degraded (44.815%); 1520/31338 objects misplaced (4.850%); 8.1 MiB/s, 143 objects/s recovering 2024-04-12T11:46:41.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:41 smithi029 ceph-mon[93650]: pgmap v1851: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 652 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 855 op/s; 14044/31338 objects degraded (44.815%); 1520/31338 objects misplaced (4.850%); 8.1 MiB/s, 143 objects/s recovering 2024-04-12T11:46:41.862 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:41.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:41.900 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:41.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:42.001 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:42.033 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:42.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:42.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:42.135 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:42.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:42.176 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:42.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:42.259 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:46:42.277 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:42.313 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:42.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:42.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:42.414 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:42.448 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:42.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:42.549 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:42.592 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:42.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:42.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:42.694 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:46:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:46:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:46:42.728 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:42.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:42.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:42.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:42.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:42.866 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:42.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:42.967 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:42.992 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:42.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:42.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:43.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:43.118 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:43.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:43.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:43.252 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:43.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:43.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:43.354 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:43.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:43.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:43.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:43.487 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:43.524 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:43.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:43.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:43.625 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:43.670 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:43.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:43.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:43 smithi087 ceph-mon[89014]: pgmap v1852: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 549 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 13765/26412 objects degraded (52.116%); 1111/26412 objects misplaced (4.206%); 8.0 MiB/s, 142 objects/s recovering 2024-04-12T11:46:43.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13765/26412 objects degraded (52.116%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:43 smithi029 ceph-mon[93650]: pgmap v1852: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 549 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 13765/26412 objects degraded (52.116%); 1111/26412 objects misplaced (4.206%); 8.0 MiB/s, 142 objects/s recovering 2024-04-12T11:46:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13765/26412 objects degraded (52.116%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:43 smithi029 ceph-mon[97787]: pgmap v1852: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 549 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 858 op/s; 13765/26412 objects degraded (52.116%); 1111/26412 objects misplaced (4.206%); 8.0 MiB/s, 142 objects/s recovering 2024-04-12T11:46:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13765/26412 objects degraded (52.116%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:43.772 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:43.815 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:43.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:43.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:43.948 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:43.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:43.997 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-04-12T11:46:44.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:44.090 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:44.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:44.191 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:44.225 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:44.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:44.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:44.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:44.367 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:44.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:44.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:44.469 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:44.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:44.512 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:44.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:44.614 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:44.622 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:44.629 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:44.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:44.650 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:44.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:44.657 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:44.751 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:44.788 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:44.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:44.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:44.946 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:46:44.956 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:46:44.992 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:45.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:46:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:46:44] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2024-04-12T11:46:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:45.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:45.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:45.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:45.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:45.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:45.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:45.289 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:45.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:45.391 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:45.433 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:45.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:45.535 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:45.565 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:45.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:45.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:45.667 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:45.704 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:45.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:45 smithi087 ceph-mon[89014]: pgmap v1853: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 451 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 13516/21699 objects degraded (62.289%); 1007/21699 objects misplaced (4.641%); 6.8 MiB/s, 139 objects/s recovering 2024-04-12T11:46:45.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:45 smithi029 ceph-mon[93650]: pgmap v1853: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 451 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 13516/21699 objects degraded (62.289%); 1007/21699 objects misplaced (4.641%); 6.8 MiB/s, 139 objects/s recovering 2024-04-12T11:46:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:45 smithi029 ceph-mon[97787]: pgmap v1853: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 451 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 13516/21699 objects degraded (62.289%); 1007/21699 objects misplaced (4.641%); 6.8 MiB/s, 139 objects/s recovering 2024-04-12T11:46:45.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:45.838 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:45.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:45.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:45.940 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:45.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:45.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:45.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:46.084 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:46.113 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:46.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:46.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:46.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:46.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:46.247 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:46.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:46.348 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:46.371 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:46.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:46.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:46.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:46.505 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:46.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:46.607 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:46.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:46.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:46.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:46 smithi087 ceph-mon[89014]: from='client.45245 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:46:46.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:46.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:46 smithi029 ceph-mon[97787]: from='client.45245 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:46:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:46 smithi029 ceph-mon[93650]: from='client.45245 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:46:46.780 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:46.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:46.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:46.882 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:46.910 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:46.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:46.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:47.012 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:47.036 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:47.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:47.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:47.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:47.175 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:47.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:47.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:47.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:47.305 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:47.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:47.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:47.407 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:47.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:47.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:47.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:47.541 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:47.585 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:47.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:47.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:47.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:47 smithi087 ceph-mon[89014]: pgmap v1854: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 337 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 13516/16227 objects degraded (83.293%); 726/16227 objects misplaced (4.474%); 6.8 MiB/s, 139 objects/s recovering 2024-04-12T11:46:47.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:47.718 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:47.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:47 smithi029 ceph-mon[97787]: pgmap v1854: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 337 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 13516/16227 objects degraded (83.293%); 726/16227 objects misplaced (4.474%); 6.8 MiB/s, 139 objects/s recovering 2024-04-12T11:46:47.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:47 smithi029 ceph-mon[93650]: pgmap v1854: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 337 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 13516/16227 objects degraded (83.293%); 726/16227 objects misplaced (4.474%); 6.8 MiB/s, 139 objects/s recovering 2024-04-12T11:46:47.819 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:47.854 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:47.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:47.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:47.956 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:47.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:47.980 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:47.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:48.081 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:48.123 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:48.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:48.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:48.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:48.262 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:48.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:48.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:48.363 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:48.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:48.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:48.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:48.496 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:48.531 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:48.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:48.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:48.634 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:48.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:48.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13516/16227 objects degraded (83.293%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:48.768 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:48.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13516/16227 objects degraded (83.293%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:48.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13516/16227 objects degraded (83.293%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:48.802 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:48.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:48.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:48.904 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:48.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:48.943 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:48.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:49.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:49.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:49.077 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:49.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:49.178 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:49.211 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:49.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:49.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:49.313 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:49.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:49.356 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:49.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:49.457 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:49.486 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:49.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:49.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:49 smithi029 ceph-mon[93650]: pgmap v1855: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 195 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 13022/9414 objects degraded (138.326%); 498/9414 objects misplaced (5.290%); 6.4 MiB/s, 138 objects/s recovering 2024-04-12T11:46:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:49 smithi029 ceph-mon[97787]: pgmap v1855: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 195 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 13022/9414 objects degraded (138.326%); 498/9414 objects misplaced (5.290%); 6.4 MiB/s, 138 objects/s recovering 2024-04-12T11:46:49.588 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:49.626 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:49.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:49 smithi087 ceph-mon[89014]: pgmap v1855: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 195 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 840 op/s; 13022/9414 objects degraded (138.326%); 498/9414 objects misplaced (5.290%); 6.4 MiB/s, 138 objects/s recovering 2024-04-12T11:46:49.728 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:49.762 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:49.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:49.864 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:49.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:49.904 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:49.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:50.005 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:50.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:50.041 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:50.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:50.111 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 106307 objects 2024-04-12T11:46:50.111 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :145.054 2024-04-12T11:46:50.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:50.167 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:50.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:50.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:50.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:50.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:50.294 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:50.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:50.395 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:50.422 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:50.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:50.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:50.524 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:50.549 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:50.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:50.651 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:50.676 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:50.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:50.777 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:50.803 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:50.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:50.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:50.905 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:50.930 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:50.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:50.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:51.032 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:51.056 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:51.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:51.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:51.158 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:51.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:51.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:51.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:51.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:51.319 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:51.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:51.325 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1853,"stamp":"2024-04-12T11:46:43.930173+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":472979434,"num_objects":7233,"num_object_clones":0,"num_object_copies":21699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13516,"num_objects_misplaced":1007,"num_objects_unfound":0,"num_objects_dirty":7233,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":198043,"num_write_kb":6381839,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29955,"num_bytes_recovered":1907897866,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":223921,"ondisk_log_size":223921,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":305,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5975964,"kb_used_data":2754456,"kb_used_omap":158,"kb_used_meta":3221217,"kb_avail":650240100,"statfs":{"total":671965249536,"available":665845862400,"internally_reserved":0,"allocated":2820562944,"data_stored":2796668575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":162126,"internal_metadata":3298526898},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-558825472,"num_objects":-8527,"num_object_clones":0,"num_object_copies":-25581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1071,"num_objects_misplaced":-1065,"num_objects_unfound":0,"num_objects_dirty":-8527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8527,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1394,"num_bytes_recovered":71368728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003218"},"pg_stats":[{"pgid":"8.11","version":"1217'10087","reported_seq":10231,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.365243+0000","last_change":"2024-04-12T11:44:48.778110+0000","last_active":"2024-04-12T11:46:39.365243+0000","last_peered":"2024-04-12T11:46:39.365243+0000","last_clean":"2024-04-12T11:46:39.365243+0000","last_became_active":"2024-04-12T11:44:45.416679+0000","last_became_peered":"2024-04-12T11:44:45.416679+0000","last_unstale":"2024-04-12T11:46:39.365243+0000","last_undegraded":"2024-04-12T11:46:39.365243+0000","last_fullsized":"2024-04-12T11:46:39.365243+0000","mapping_epoch":1199,"log_start":"1186'1800","ondisk_log_start":"1186'1800","created":1199,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":5,"last_scrub":"1201'7675","last_scrub_stamp":"2024-04-12T11:44:48.777814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:44:48.777814+0000","objects_scrubbed":2673,"log_size":8287,"log_dups_size":0,"ondisk_log_size":8287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:51:22.586724+0000","scrub_duration":716,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15925248,"num_objects":243,"num_object_clones":0,"num_object_copies":729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2430,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":1789,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.323185+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:46:39.323185+0000","last_peered":"2024-04-12T11:46:39.323185+0000","last_clean":"2024-04-12T11:46:39.323185+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:46:39.323185+0000","last_undegraded":"2024-04-12T11:46:39.323185+0000","last_fullsized":"2024-04-12T11:46:39.323185+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":648,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328161+0000","last_change":"2024-04-12T11:46:10.406067+0000","last_active":"2024-04-12T11:46:39.328161+0000","last_peered":"2024-04-12T11:46:39.328161+0000","last_clean":"2024-04-12T11:46:39.328161+0000","last_became_active":"2024-04-12T11:46:10.405480+0000","last_became_peered":"2024-04-12T11:46:10.405480+0000","last_unstale":"2024-04-12T11:46:39.328161+0000","last_undegraded":"2024-04-12T11:46:39.328161+0000","last_fullsized":"2024-04-12T11:46:39.328161+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:54:55.306753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.12","version":"1217'10108","reported_seq":10216,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:40.135492+0000","last_change":"2024-04-12T11:46:11.380332+0000","last_active":"2024-04-12T11:46:40.135492+0000","last_peered":"2024-04-12T11:46:40.135492+0000","last_clean":"2024-04-12T11:46:40.135492+0000","last_became_active":"2024-04-12T11:44:45.477110+0000","last_became_peered":"2024-04-12T11:44:45.477110+0000","last_unstale":"2024-04-12T11:46:40.135492+0000","last_undegraded":"2024-04-12T11:46:40.135492+0000","last_fullsized":"2024-04-12T11:46:40.135492+0000","mapping_epoch":1199,"log_start":"1179'1300","ondisk_log_start":"1179'1300","created":1199,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":8808,"log_dups_size":0,"ondisk_log_size":8808,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-12T11:46:43.048772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20021248,"num_objects":306,"num_object_clones":0,"num_object_copies":918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6321,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1867,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.327994+0000","last_change":"2024-04-12T11:46:10.403816+0000","last_active":"2024-04-12T11:46:39.327994+0000","last_peered":"2024-04-12T11:46:39.327994+0000","last_clean":"2024-04-12T11:46:39.327994+0000","last_became_active":"2024-04-12T11:46:10.399712+0000","last_became_peered":"2024-04-12T11:46:10.399712+0000","last_unstale":"2024-04-12T11:46:39.327994+0000","last_undegraded":"2024-04-12T11:46:39.327994+0000","last_fullsized":"2024-04-12T11:46:39.327994+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:06:08.696544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1887,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328993+0000","last_change":"2024-04-12T11:46:10.395530+0000","last_active":"2024-04-12T11:46:39.328993+0000","last_peered":"2024-04-12T11:46:39.328993+0000","last_clean":"2024-04-12T11:46:39.328993+0000","last_became_active":"2024-04-12T11:46:10.395134+0000","last_became_peered":"2024-04-12T11:46:10.395134+0000","last_unstale":"2024-04-12T11:46:39.328993+0000","last_undegraded":"2024-04-12T11:46:39.328993+0000","last_fullsized":"2024-04-12T11:46:39.328993+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:16:32.508735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"1217'10310","reported_seq":10419,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328672+0000","last_change":"2024-04-12T11:44:50.089843+0000","last_active":"2024-04-12T11:46:39.328672+0000","last_peered":"2024-04-12T11:46:39.328672+0000","last_clean":"2024-04-12T11:46:39.328672+0000","last_became_active":"2024-04-12T11:44:45.422891+0000","last_became_peered":"2024-04-12T11:44:45.422891+0000","last_unstale":"2024-04-12T11:46:39.328672+0000","last_undegraded":"2024-04-12T11:46:39.328672+0000","last_fullsized":"2024-04-12T11:46:39.328672+0000","mapping_epoch":1199,"log_start":"1186'1900","ondisk_log_start":"1186'1900","created":1199,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":8410,"log_dups_size":0,"ondisk_log_size":8410,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-12T11:46:40.701611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20774912,"num_objects":317,"num_object_clones":0,"num_object_copies":951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6429,"num_write_kb":215872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":615,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328108+0000","last_change":"2024-04-12T11:46:10.391931+0000","last_active":"2024-04-12T11:46:39.328108+0000","last_peered":"2024-04-12T11:46:39.328108+0000","last_clean":"2024-04-12T11:46:39.328108+0000","last_became_active":"2024-04-12T11:46:10.391618+0000","last_became_peered":"2024-04-12T11:46:10.391618+0000","last_unstale":"2024-04-12T11:46:39.328108+0000","last_undegraded":"2024-04-12T11:46:39.328108+0000","last_fullsized":"2024-04-12T11:46:39.328108+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:05:47.259753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1165,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.045365+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:46:28.045365+0000","last_peered":"2024-04-12T11:46:28.045365+0000","last_clean":"2024-04-12T11:46:28.045365+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:46:28.045365+0000","last_undegraded":"2024-04-12T11:46:28.045365+0000","last_fullsized":"2024-04-12T11:46:28.045365+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1858,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328705+0000","last_change":"2024-04-12T11:46:10.397031+0000","last_active":"2024-04-12T11:46:39.328705+0000","last_peered":"2024-04-12T11:46:39.328705+0000","last_clean":"2024-04-12T11:46:39.328705+0000","last_became_active":"2024-04-12T11:46:10.396649+0000","last_became_peered":"2024-04-12T11:46:10.396649+0000","last_unstale":"2024-04-12T11:46:39.328705+0000","last_undegraded":"2024-04-12T11:46:39.328705+0000","last_fullsized":"2024-04-12T11:46:39.328705+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:06:45.598704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":599,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.336095+0000","last_change":"2024-04-12T11:46:10.396045+0000","last_active":"2024-04-12T11:46:39.336095+0000","last_peered":"2024-04-12T11:46:39.336095+0000","last_clean":"2024-04-12T11:46:39.336095+0000","last_became_active":"2024-04-12T11:46:10.393256+0000","last_became_peered":"2024-04-12T11:46:10.393256+0000","last_unstale":"2024-04-12T11:46:39.336095+0000","last_undegraded":"2024-04-12T11:46:39.336095+0000","last_fullsized":"2024-04-12T11:46:39.336095+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:26:31.787589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1196,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.061640+0000","last_change":"2024-04-12T11:45:52.986340+0000","last_active":"2024-04-12T11:46:28.061640+0000","last_peered":"2024-04-12T11:46:28.061640+0000","last_clean":"2024-04-12T11:46:28.061640+0000","last_became_active":"2024-04-12T11:45:52.984190+0000","last_became_peered":"2024-04-12T11:45:52.984190+0000","last_unstale":"2024-04-12T11:46:28.061640+0000","last_undegraded":"2024-04-12T11:46:28.061640+0000","last_fullsized":"2024-04-12T11:46:28.061640+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:52:19.144894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":35,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.052355+0000","last_change":"2024-04-12T11:46:10.398333+0000","last_active":"2024-04-12T11:46:28.052355+0000","last_peered":"2024-04-12T11:46:28.052355+0000","last_clean":"2024-04-12T11:46:28.052355+0000","last_became_active":"2024-04-12T11:46:10.398008+0000","last_became_peered":"2024-04-12T11:46:10.398008+0000","last_unstale":"2024-04-12T11:46:28.052355+0000","last_undegraded":"2024-04-12T11:46:28.052355+0000","last_fullsized":"2024-04-12T11:46:28.052355+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:02:35.332854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":39,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.059953+0000","last_change":"2024-04-12T11:46:10.411220+0000","last_active":"2024-04-12T11:46:28.059953+0000","last_peered":"2024-04-12T11:46:28.059953+0000","last_clean":"2024-04-12T11:46:28.059953+0000","last_became_active":"2024-04-12T11:46:10.386123+0000","last_became_peered":"2024-04-12T11:46:10.386123+0000","last_unstale":"2024-04-12T11:46:28.059953+0000","last_undegraded":"2024-04-12T11:46:28.059953+0000","last_fullsized":"2024-04-12T11:46:28.059953+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:40:12.714391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1840,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.331314+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:46:39.331314+0000","last_peered":"2024-04-12T11:46:39.331314+0000","last_clean":"2024-04-12T11:46:39.331314+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:46:39.331314+0000","last_undegraded":"2024-04-12T11:46:39.331314+0000","last_fullsized":"2024-04-12T11:46:39.331314+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1793,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.323462+0000","last_change":"2024-04-12T11:45:53.024626+0000","last_active":"2024-04-12T11:46:39.323462+0000","last_peered":"2024-04-12T11:46:39.323462+0000","last_clean":"2024-04-12T11:46:39.323462+0000","last_became_active":"2024-04-12T11:45:52.976276+0000","last_became_peered":"2024-04-12T11:45:52.976276+0000","last_unstale":"2024-04-12T11:46:39.323462+0000","last_undegraded":"2024-04-12T11:46:39.323462+0000","last_fullsized":"2024-04-12T11:46:39.323462+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:49.929210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1752,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.323702+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:46:39.323702+0000","last_peered":"2024-04-12T11:46:39.323702+0000","last_clean":"2024-04-12T11:46:39.323702+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:46:39.323702+0000","last_undegraded":"2024-04-12T11:46:39.323702+0000","last_fullsized":"2024-04-12T11:46:39.323702+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"1217'9995","reported_seq":18018,"reported_epoch":1217,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:46:42.868257+0000","last_change":"2024-04-12T11:46:11.479310+0000","last_active":"2024-04-12T11:46:42.868257+0000","last_peered":"2024-04-12T11:46:42.868257+0000","last_clean":"2024-04-12T11:43:55.127664+0000","last_became_active":"2024-04-12T11:46:11.417318+0000","last_became_peered":"2024-04-12T11:46:11.417318+0000","last_unstale":"2024-04-12T11:46:42.868257+0000","last_undegraded":"2024-04-12T11:46:11.386902+0000","last_fullsized":"2024-04-12T11:46:42.868257+0000","mapping_epoch":1214,"log_start":"1176'400","ondisk_log_start":"1176'400","created":1199,"last_epoch_clean":1185,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9595,"log_dups_size":0,"ondisk_log_size":9595,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14090240,"num_objects":215,"num_object_clones":0,"num_object_copies":645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6275,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2681,"num_bytes_recovered":175636480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":["5","0","3"],"object_location_counts":[{"shards":"0,4,5","objects":179},{"shards":"0,5,7","objects":36}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":24,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.052661+0000","last_change":"2024-04-12T11:45:53.035604+0000","last_active":"2024-04-12T11:46:28.052661+0000","last_peered":"2024-04-12T11:46:28.052661+0000","last_clean":"2024-04-12T11:46:28.052661+0000","last_became_active":"2024-04-12T11:45:52.984722+0000","last_became_peered":"2024-04-12T11:45:52.984722+0000","last_unstale":"2024-04-12T11:46:28.052661+0000","last_undegraded":"2024-04-12T11:46:28.052661+0000","last_fullsized":"2024-04-12T11:46:28.052661+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:30.674395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":605,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.327614+0000","last_change":"2024-04-12T11:46:10.398603+0000","last_active":"2024-04-12T11:46:39.327614+0000","last_peered":"2024-04-12T11:46:39.327614+0000","last_clean":"2024-04-12T11:46:39.327614+0000","last_became_active":"2024-04-12T11:46:10.390834+0000","last_became_peered":"2024-04-12T11:46:10.390834+0000","last_unstale":"2024-04-12T11:46:39.327614+0000","last_undegraded":"2024-04-12T11:46:39.327614+0000","last_fullsized":"2024-04-12T11:46:39.327614+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:10:10.149137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.19","version":"1217'10311","reported_seq":10375,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:42.595040+0000","last_change":"2024-04-12T11:44:45.463778+0000","last_active":"2024-04-12T11:46:42.595040+0000","last_peered":"2024-04-12T11:46:42.595040+0000","last_clean":"2024-04-12T11:46:42.595040+0000","last_became_active":"2024-04-12T11:44:45.453076+0000","last_became_peered":"2024-04-12T11:44:45.453076+0000","last_unstale":"2024-04-12T11:46:42.595040+0000","last_undegraded":"2024-04-12T11:46:42.595040+0000","last_fullsized":"2024-04-12T11:46:42.595040+0000","mapping_epoch":1199,"log_start":"1179'1100","ondisk_log_start":"1179'1100","created":1199,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9211,"log_dups_size":0,"ondisk_log_size":9211,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12124160,"num_objects":185,"num_object_clones":0,"num_object_copies":555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6496,"num_write_kb":213792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1871,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.329239+0000","last_change":"2024-04-12T11:46:10.396013+0000","last_active":"2024-04-12T11:46:39.329239+0000","last_peered":"2024-04-12T11:46:39.329239+0000","last_clean":"2024-04-12T11:46:39.329239+0000","last_became_active":"2024-04-12T11:46:10.395714+0000","last_became_peered":"2024-04-12T11:46:10.395714+0000","last_unstale":"2024-04-12T11:46:39.329239+0000","last_undegraded":"2024-04-12T11:46:39.329239+0000","last_fullsized":"2024-04-12T11:46:39.329239+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:27:19.237889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1213,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.065887+0000","last_change":"2024-04-12T11:46:10.397701+0000","last_active":"2024-04-12T11:46:28.065887+0000","last_peered":"2024-04-12T11:46:28.065887+0000","last_clean":"2024-04-12T11:46:28.065887+0000","last_became_active":"2024-04-12T11:46:10.397162+0000","last_became_peered":"2024-04-12T11:46:10.397162+0000","last_unstale":"2024-04-12T11:46:28.065887+0000","last_undegraded":"2024-04-12T11:46:28.065887+0000","last_fullsized":"2024-04-12T11:46:28.065887+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:16:29.772172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":18,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.052280+0000","last_change":"2024-04-12T11:45:53.006808+0000","last_active":"2024-04-12T11:46:28.052280+0000","last_peered":"2024-04-12T11:46:28.052280+0000","last_clean":"2024-04-12T11:46:28.052280+0000","last_became_active":"2024-04-12T11:45:53.006480+0000","last_became_peered":"2024-04-12T11:45:53.006480+0000","last_unstale":"2024-04-12T11:46:28.052280+0000","last_undegraded":"2024-04-12T11:46:28.052280+0000","last_fullsized":"2024-04-12T11:46:28.052280+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:47.685096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1872,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.329086+0000","last_change":"2024-04-12T11:46:10.393712+0000","last_active":"2024-04-12T11:46:39.329086+0000","last_peered":"2024-04-12T11:46:39.329086+0000","last_clean":"2024-04-12T11:46:39.329086+0000","last_became_active":"2024-04-12T11:46:10.393359+0000","last_became_peered":"2024-04-12T11:46:10.393359+0000","last_unstale":"2024-04-12T11:46:39.329086+0000","last_undegraded":"2024-04-12T11:46:39.329086+0000","last_fullsized":"2024-04-12T11:46:39.329086+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:33:56.240139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1172,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.044546+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:46:28.044546+0000","last_peered":"2024-04-12T11:46:28.044546+0000","last_clean":"2024-04-12T11:46:28.044546+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:46:28.044546+0000","last_undegraded":"2024-04-12T11:46:28.044546+0000","last_fullsized":"2024-04-12T11:46:28.044546+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":35,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.052910+0000","last_change":"2024-04-12T11:46:10.408581+0000","last_active":"2024-04-12T11:46:28.052910+0000","last_peered":"2024-04-12T11:46:28.052910+0000","last_clean":"2024-04-12T11:46:28.052910+0000","last_became_active":"2024-04-12T11:46:10.406940+0000","last_became_peered":"2024-04-12T11:46:10.406940+0000","last_unstale":"2024-04-12T11:46:28.052910+0000","last_undegraded":"2024-04-12T11:46:28.052910+0000","last_fullsized":"2024-04-12T11:46:28.052910+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:18:48.590948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"1217'10567","reported_seq":10631,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:42.884898+0000","last_change":"2024-04-12T11:44:45.426897+0000","last_active":"2024-04-12T11:46:42.884898+0000","last_peered":"2024-04-12T11:46:42.884898+0000","last_clean":"2024-04-12T11:46:42.884898+0000","last_became_active":"2024-04-12T11:44:45.420234+0000","last_became_peered":"2024-04-12T11:44:45.420234+0000","last_unstale":"2024-04-12T11:46:42.884898+0000","last_undegraded":"2024-04-12T11:46:42.884898+0000","last_fullsized":"2024-04-12T11:46:42.884898+0000","mapping_epoch":1172,"log_start":"1179'900","ondisk_log_start":"1179'900","created":1172,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9667,"log_dups_size":0,"ondisk_log_size":9667,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17989632,"num_objects":274,"num_object_clones":0,"num_object_copies":822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6617,"num_write_kb":220544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":22,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.052980+0000","last_change":"2024-04-12T11:45:53.028640+0000","last_active":"2024-04-12T11:46:28.052980+0000","last_peered":"2024-04-12T11:46:28.052980+0000","last_clean":"2024-04-12T11:46:28.052980+0000","last_became_active":"2024-04-12T11:45:52.983099+0000","last_became_peered":"2024-04-12T11:45:52.983099+0000","last_unstale":"2024-04-12T11:46:28.052980+0000","last_undegraded":"2024-04-12T11:46:28.052980+0000","last_fullsized":"2024-04-12T11:46:28.052980+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:43:01.897014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1173,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.064151+0000","last_change":"2024-04-12T11:45:52.993160+0000","last_active":"2024-04-12T11:46:28.064151+0000","last_peered":"2024-04-12T11:46:28.064151+0000","last_clean":"2024-04-12T11:46:28.064151+0000","last_became_active":"2024-04-12T11:45:52.989310+0000","last_became_peered":"2024-04-12T11:45:52.989310+0000","last_unstale":"2024-04-12T11:46:28.064151+0000","last_undegraded":"2024-04-12T11:46:28.064151+0000","last_fullsized":"2024-04-12T11:46:28.064151+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:58:47.362374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"1217'10310","reported_seq":14055,"reported_epoch":1217,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:46:42.691048+0000","last_change":"2024-04-12T11:45:54.192268+0000","last_active":"2024-04-12T11:46:42.691048+0000","last_peered":"2024-04-12T11:46:42.691048+0000","last_clean":"2024-04-12T11:43:55.132835+0000","last_became_active":"2024-04-12T11:45:54.166007+0000","last_became_peered":"2024-04-12T11:45:54.166007+0000","last_unstale":"2024-04-12T11:46:42.691048+0000","last_undegraded":"2024-04-12T11:45:53.988008+0000","last_fullsized":"2024-04-12T11:45:53.964585+0000","mapping_epoch":1210,"log_start":"1179'1100","ondisk_log_start":"1179'1100","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9210,"log_dups_size":0,"ondisk_log_size":9210,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9830400,"num_objects":150,"num_object_clones":0,"num_object_copies":450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2612,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6488,"num_write_kb":212416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1829,"num_bytes_recovered":119898112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"0,4,6","objects":217},{"shards":"4,6","objects":-496}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":22,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.054587+0000","last_change":"2024-04-12T11:45:53.031759+0000","last_active":"2024-04-12T11:46:28.054587+0000","last_peered":"2024-04-12T11:46:28.054587+0000","last_clean":"2024-04-12T11:46:28.054587+0000","last_became_active":"2024-04-12T11:45:52.984545+0000","last_became_peered":"2024-04-12T11:45:52.984545+0000","last_unstale":"2024-04-12T11:46:28.054587+0000","last_undegraded":"2024-04-12T11:46:28.054587+0000","last_fullsized":"2024-04-12T11:46:28.054587+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:16:10.421992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1793,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.322963+0000","last_change":"2024-04-12T11:46:10.400601+0000","last_active":"2024-04-12T11:46:39.322963+0000","last_peered":"2024-04-12T11:46:39.322963+0000","last_clean":"2024-04-12T11:46:39.322963+0000","last_became_active":"2024-04-12T11:46:10.400255+0000","last_became_peered":"2024-04-12T11:46:10.400255+0000","last_unstale":"2024-04-12T11:46:39.322963+0000","last_undegraded":"2024-04-12T11:46:39.322963+0000","last_fullsized":"2024-04-12T11:46:39.322963+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:03:28.857231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"1217'10158","reported_seq":15067,"reported_epoch":1217,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:46:40.919633+0000","last_change":"2024-04-12T11:45:54.168832+0000","last_active":"2024-04-12T11:46:40.919633+0000","last_peered":"2024-04-12T11:46:40.919633+0000","last_clean":"2024-04-12T11:43:55.131806+0000","last_became_active":"2024-04-12T11:45:54.140710+0000","last_became_peered":"2024-04-12T11:45:54.140710+0000","last_unstale":"2024-04-12T11:46:40.919633+0000","last_undegraded":"2024-04-12T11:45:53.983731+0000","last_fullsized":"2024-04-12T11:45:53.964552+0000","mapping_epoch":1210,"log_start":"1176'500","ondisk_log_start":"1176'500","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9658,"log_dups_size":0,"ondisk_log_size":9658,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11403264,"num_objects":174,"num_object_clones":0,"num_object_copies":522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6400,"num_write_kb":210368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1900,"num_bytes_recovered":124452864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,2,6","objects":225},{"shards":"0,6","objects":-483}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":22,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.055490+0000","last_change":"2024-04-12T11:45:53.169686+0000","last_active":"2024-04-12T11:46:28.055490+0000","last_peered":"2024-04-12T11:46:28.055490+0000","last_clean":"2024-04-12T11:46:28.055490+0000","last_became_active":"2024-04-12T11:45:53.021021+0000","last_became_peered":"2024-04-12T11:45:53.021021+0000","last_unstale":"2024-04-12T11:46:28.055490+0000","last_undegraded":"2024-04-12T11:46:28.055490+0000","last_fullsized":"2024-04-12T11:46:28.055490+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:48:23.506812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1191,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.044576+0000","last_change":"2024-04-12T11:45:52.979601+0000","last_active":"2024-04-12T11:46:28.044576+0000","last_peered":"2024-04-12T11:46:28.044576+0000","last_clean":"2024-04-12T11:46:28.044576+0000","last_became_active":"2024-04-12T11:45:52.979307+0000","last_became_peered":"2024-04-12T11:45:52.979307+0000","last_unstale":"2024-04-12T11:46:28.044576+0000","last_undegraded":"2024-04-12T11:46:28.044576+0000","last_fullsized":"2024-04-12T11:46:28.044576+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:21.367107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1217'10115","reported_seq":11649,"reported_epoch":1217,"state":"active+recovery_wait","last_fresh":"2024-04-12T11:46:39.380691+0000","last_change":"2024-04-12T11:46:10.469615+0000","last_active":"2024-04-12T11:46:39.380691+0000","last_peered":"2024-04-12T11:46:39.380691+0000","last_clean":"2024-04-12T11:45:16.333502+0000","last_became_active":"2024-04-12T11:46:10.438648+0000","last_became_peered":"2024-04-12T11:46:10.438648+0000","last_unstale":"2024-04-12T11:46:39.380691+0000","last_undegraded":"2024-04-12T11:46:39.380691+0000","last_fullsized":"2024-04-12T11:46:39.380691+0000","mapping_epoch":1213,"log_start":"1185'1700","ondisk_log_start":"1185'1700","created":1172,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":8415,"log_dups_size":0,"ondisk_log_size":8415,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12091392,"num_objects":184,"num_object_clones":0,"num_object_copies":552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6376,"num_write_kb":209952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":350,"num_bytes_recovered":22904832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":["1","2","3","6"],"object_location_counts":[{"shards":"1,2,6","objects":184}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1198,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.325929+0000","last_change":"2024-04-12T11:46:10.399997+0000","last_active":"2024-04-12T11:46:39.325929+0000","last_peered":"2024-04-12T11:46:39.325929+0000","last_clean":"2024-04-12T11:46:39.325929+0000","last_became_active":"2024-04-12T11:46:10.399692+0000","last_became_peered":"2024-04-12T11:46:10.399692+0000","last_unstale":"2024-04-12T11:46:39.325929+0000","last_undegraded":"2024-04-12T11:46:39.325929+0000","last_fullsized":"2024-04-12T11:46:39.325929+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:44.268992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1852,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328290+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:46:39.328290+0000","last_peered":"2024-04-12T11:46:39.328290+0000","last_clean":"2024-04-12T11:46:39.328290+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:46:39.328290+0000","last_undegraded":"2024-04-12T11:46:39.328290+0000","last_fullsized":"2024-04-12T11:46:39.328290+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"1217'10107","reported_seq":13838,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:42.878640+0000","last_change":"2024-04-12T11:46:11.458619+0000","last_active":"2024-04-12T11:46:42.878640+0000","last_peered":"2024-04-12T11:46:42.878640+0000","last_clean":"2024-04-12T11:46:42.878640+0000","last_became_active":"2024-04-12T11:46:11.388666+0000","last_became_peered":"2024-04-12T11:46:11.388666+0000","last_unstale":"2024-04-12T11:46:42.878640+0000","last_undegraded":"2024-04-12T11:46:42.878640+0000","last_fullsized":"2024-04-12T11:46:42.878640+0000","mapping_epoch":1214,"log_start":"1176'500","ondisk_log_start":"1176'500","created":1172,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9607,"log_dups_size":0,"ondisk_log_size":9607,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14450688,"num_objects":220,"num_object_clones":0,"num_object_copies":660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6351,"num_write_kb":210304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1440,"num_bytes_recovered":94306304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":121,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328792+0000","last_change":"2024-04-12T11:46:10.418286+0000","last_active":"2024-04-12T11:46:39.328792+0000","last_peered":"2024-04-12T11:46:39.328792+0000","last_clean":"2024-04-12T11:46:39.328792+0000","last_became_active":"2024-04-12T11:46:10.399488+0000","last_became_peered":"2024-04-12T11:46:10.399488+0000","last_unstale":"2024-04-12T11:46:39.328792+0000","last_undegraded":"2024-04-12T11:46:39.328792+0000","last_fullsized":"2024-04-12T11:46:39.328792+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:38:14.150930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1784,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.323375+0000","last_change":"2024-04-12T11:46:10.395742+0000","last_active":"2024-04-12T11:46:39.323375+0000","last_peered":"2024-04-12T11:46:39.323375+0000","last_clean":"2024-04-12T11:46:39.323375+0000","last_became_active":"2024-04-12T11:46:10.395362+0000","last_became_peered":"2024-04-12T11:46:10.395362+0000","last_unstale":"2024-04-12T11:46:39.323375+0000","last_undegraded":"2024-04-12T11:46:39.323375+0000","last_fullsized":"2024-04-12T11:46:39.323375+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:56:49.815229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"1217'10240","reported_seq":10386,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.350033+0000","last_change":"2024-04-12T11:44:46.425482+0000","last_active":"2024-04-12T11:46:39.350033+0000","last_peered":"2024-04-12T11:46:39.350033+0000","last_clean":"2024-04-12T11:46:39.350033+0000","last_became_active":"2024-04-12T11:44:45.391832+0000","last_became_peered":"2024-04-12T11:44:45.391832+0000","last_unstale":"2024-04-12T11:46:39.350033+0000","last_undegraded":"2024-04-12T11:46:39.350033+0000","last_fullsized":"2024-04-12T11:46:39.350033+0000","mapping_epoch":1172,"log_start":"1186'1800","ondisk_log_start":"1186'1800","created":1172,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":0,"last_scrub":"1200'7584","last_scrub_stamp":"2024-04-12T11:44:46.425356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:44:46.425356+0000","objects_scrubbed":2921,"log_size":8440,"log_dups_size":0,"ondisk_log_size":8440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:06:53.217565+0000","scrub_duration":329,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17367040,"num_objects":265,"num_object_clones":0,"num_object_copies":795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2656,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":792,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.324131+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:46:39.324131+0000","last_peered":"2024-04-12T11:46:39.324131+0000","last_clean":"2024-04-12T11:46:39.324131+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:46:39.324131+0000","last_undegraded":"2024-04-12T11:46:39.324131+0000","last_fullsized":"2024-04-12T11:46:39.324131+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1788,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.323509+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:46:39.323509+0000","last_peered":"2024-04-12T11:46:39.323509+0000","last_clean":"2024-04-12T11:46:39.323509+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:46:39.323509+0000","last_undegraded":"2024-04-12T11:46:39.323509+0000","last_fullsized":"2024-04-12T11:46:39.323509+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"1217'10180","reported_seq":10292,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:40.128353+0000","last_change":"2024-04-12T11:45:16.383840+0000","last_active":"2024-04-12T11:46:40.128353+0000","last_peered":"2024-04-12T11:46:40.128353+0000","last_clean":"2024-04-12T11:46:40.128353+0000","last_became_active":"2024-04-12T11:44:45.459480+0000","last_became_peered":"2024-04-12T11:44:45.459480+0000","last_unstale":"2024-04-12T11:46:40.128353+0000","last_undegraded":"2024-04-12T11:46:40.128353+0000","last_fullsized":"2024-04-12T11:46:40.128353+0000","mapping_epoch":1172,"log_start":"1179'1300","ondisk_log_start":"1179'1300","created":1172,"last_epoch_clean":1203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":8880,"log_dups_size":0,"ondisk_log_size":8880,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-12T11:46:43.051620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15302656,"num_objects":233,"num_object_clones":0,"num_object_copies":699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6392,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1786,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.323053+0000","last_change":"2024-04-12T11:45:52.976139+0000","last_active":"2024-04-12T11:46:39.323053+0000","last_peered":"2024-04-12T11:46:39.323053+0000","last_clean":"2024-04-12T11:46:39.323053+0000","last_became_active":"2024-04-12T11:45:52.975803+0000","last_became_peered":"2024-04-12T11:45:52.975803+0000","last_unstale":"2024-04-12T11:46:39.323053+0000","last_undegraded":"2024-04-12T11:46:39.323053+0000","last_fullsized":"2024-04-12T11:46:39.323053+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:46.618143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1753,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.323993+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:46:39.323993+0000","last_peered":"2024-04-12T11:46:39.323993+0000","last_clean":"2024-04-12T11:46:39.323993+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:46:39.323993+0000","last_undegraded":"2024-04-12T11:46:39.323993+0000","last_fullsized":"2024-04-12T11:46:39.323993+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"1217'10365","reported_seq":10475,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.363479+0000","last_change":"2024-04-12T11:46:10.382114+0000","last_active":"2024-04-12T11:46:39.363479+0000","last_peered":"2024-04-12T11:46:39.363479+0000","last_clean":"2024-04-12T11:46:39.363479+0000","last_became_active":"2024-04-12T11:44:45.396798+0000","last_became_peered":"2024-04-12T11:44:45.396798+0000","last_unstale":"2024-04-12T11:46:39.363479+0000","last_undegraded":"2024-04-12T11:46:39.363479+0000","last_fullsized":"2024-04-12T11:46:39.363479+0000","mapping_epoch":1172,"log_start":"1186'1900","ondisk_log_start":"1186'1900","created":1172,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":8465,"log_dups_size":0,"ondisk_log_size":8465,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-12T11:46:41.732593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17170432,"num_objects":262,"num_object_clones":0,"num_object_copies":786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6484,"num_write_kb":215872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":606,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.327715+0000","last_change":"2024-04-12T11:46:10.398575+0000","last_active":"2024-04-12T11:46:39.327715+0000","last_peered":"2024-04-12T11:46:39.327715+0000","last_clean":"2024-04-12T11:46:39.327715+0000","last_became_active":"2024-04-12T11:46:10.395695+0000","last_became_peered":"2024-04-12T11:46:10.395695+0000","last_unstale":"2024-04-12T11:46:39.327715+0000","last_undegraded":"2024-04-12T11:46:39.327715+0000","last_fullsized":"2024-04-12T11:46:39.327715+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:49:42.768675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1873,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328952+0000","last_change":"2024-04-12T11:46:10.437691+0000","last_active":"2024-04-12T11:46:39.328952+0000","last_peered":"2024-04-12T11:46:39.328952+0000","last_clean":"2024-04-12T11:46:39.328952+0000","last_became_active":"2024-04-12T11:46:10.404629+0000","last_became_peered":"2024-04-12T11:46:10.404629+0000","last_unstale":"2024-04-12T11:46:39.328952+0000","last_undegraded":"2024-04-12T11:46:39.328952+0000","last_fullsized":"2024-04-12T11:46:39.328952+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:00:29.330962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"1217'12440","reported_seq":12543,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:40.946596+0000","last_change":"2024-04-12T11:42:38.226043+0000","last_active":"2024-04-12T11:46:40.946596+0000","last_peered":"2024-04-12T11:46:40.946596+0000","last_clean":"2024-04-12T11:46:40.946596+0000","last_became_active":"2024-04-12T11:42:38.225220+0000","last_became_peered":"2024-04-12T11:42:38.225220+0000","last_unstale":"2024-04-12T11:46:40.946596+0000","last_undegraded":"2024-04-12T11:46:40.946596+0000","last_fullsized":"2024-04-12T11:46:40.946596+0000","mapping_epoch":1172,"log_start":"1196'5580","ondisk_log_start":"1196'5580","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6860,"log_dups_size":0,"ondisk_log_size":6860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:05:38.086148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31719424,"num_objects":484,"num_object_clones":0,"num_object_copies":1452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12440,"num_write_kb":413568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1863,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.327875+0000","last_change":"2024-04-12T11:45:52.980673+0000","last_active":"2024-04-12T11:46:39.327875+0000","last_peered":"2024-04-12T11:46:39.327875+0000","last_clean":"2024-04-12T11:46:39.327875+0000","last_became_active":"2024-04-12T11:45:52.978978+0000","last_became_peered":"2024-04-12T11:45:52.978978+0000","last_unstale":"2024-04-12T11:46:39.327875+0000","last_undegraded":"2024-04-12T11:46:39.327875+0000","last_fullsized":"2024-04-12T11:46:39.327875+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:32:10.303905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1214,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:32.902583+0000","last_change":"2024-04-12T11:46:10.397490+0000","last_active":"2024-04-12T11:46:32.902583+0000","last_peered":"2024-04-12T11:46:32.902583+0000","last_clean":"2024-04-12T11:46:32.902583+0000","last_became_active":"2024-04-12T11:46:10.396703+0000","last_became_peered":"2024-04-12T11:46:10.396703+0000","last_unstale":"2024-04-12T11:46:32.902583+0000","last_undegraded":"2024-04-12T11:46:32.902583+0000","last_fullsized":"2024-04-12T11:46:32.902583+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:35:28.943460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1842,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.331221+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:46:39.331221+0000","last_peered":"2024-04-12T11:46:39.331221+0000","last_clean":"2024-04-12T11:46:39.331221+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:46:39.331221+0000","last_undegraded":"2024-04-12T11:46:39.331221+0000","last_fullsized":"2024-04-12T11:46:39.331221+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"1217'10069","reported_seq":18184,"reported_epoch":1217,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:46:42.919131+0000","last_change":"2024-04-12T11:46:11.493047+0000","last_active":"2024-04-12T11:46:42.919131+0000","last_peered":"2024-04-12T11:46:42.919131+0000","last_clean":"2024-04-12T11:43:55.127664+0000","last_became_active":"2024-04-12T11:46:11.417113+0000","last_became_peered":"2024-04-12T11:46:11.417113+0000","last_unstale":"2024-04-12T11:46:42.919131+0000","last_undegraded":"2024-04-12T11:46:11.387424+0000","last_fullsized":"2024-04-12T11:46:42.919131+0000","mapping_epoch":1214,"log_start":"1176'400","ondisk_log_start":"1176'400","created":1172,"last_epoch_clean":1185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9669,"log_dups_size":0,"ondisk_log_size":9669,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9240576,"num_objects":141,"num_object_clones":0,"num_object_copies":423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2102,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6349,"num_write_kb":207680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2725,"num_bytes_recovered":178585600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":["5","0","3"],"object_location_counts":[{"shards":"0,4,5","objects":176},{"shards":"0,5,7","objects":-35}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1176,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.066080+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:46:28.066080+0000","last_peered":"2024-04-12T11:46:28.066080+0000","last_clean":"2024-04-12T11:46:28.066080+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:46:28.066080+0000","last_undegraded":"2024-04-12T11:46:28.066080+0000","last_fullsized":"2024-04-12T11:46:28.066080+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1775,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.323535+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:46:39.323535+0000","last_peered":"2024-04-12T11:46:39.323535+0000","last_clean":"2024-04-12T11:46:39.323535+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:46:39.323535+0000","last_undegraded":"2024-04-12T11:46:39.323535+0000","last_fullsized":"2024-04-12T11:46:39.323535+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.333501+0000","last_change":"2024-04-12T11:46:10.393257+0000","last_active":"2024-04-12T11:46:39.333501+0000","last_peered":"2024-04-12T11:46:39.333501+0000","last_clean":"2024-04-12T11:46:39.333501+0000","last_became_active":"2024-04-12T11:46:10.392748+0000","last_became_peered":"2024-04-12T11:46:10.392748+0000","last_unstale":"2024-04-12T11:46:39.333501+0000","last_undegraded":"2024-04-12T11:46:39.333501+0000","last_fullsized":"2024-04-12T11:46:39.333501+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:31:43.963814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.b","version":"1217'13046","reported_seq":21763,"reported_epoch":1217,"state":"active+recovery_wait+remapped","last_fresh":"2024-04-12T11:46:40.206294+0000","last_change":"2024-04-12T11:46:11.602234+0000","last_active":"2024-04-12T11:46:40.206294+0000","last_peered":"2024-04-12T11:46:40.206294+0000","last_clean":"2024-04-12T11:46:09.363231+0000","last_became_active":"2024-04-12T11:46:11.512937+0000","last_became_peered":"2024-04-12T11:46:11.512937+0000","last_unstale":"2024-04-12T11:46:40.206294+0000","last_undegraded":"2024-04-12T11:46:40.206294+0000","last_fullsized":"2024-04-12T11:46:40.206294+0000","mapping_epoch":1214,"log_start":"1196'6976","ondisk_log_start":"1196'6976","created":1172,"last_epoch_clean":1208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6070,"log_dups_size":0,"ondisk_log_size":6070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33816576,"num_objects":516,"num_object_clones":0,"num_object_copies":1548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":516,"num_objects_unfound":0,"num_objects_dirty":516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13046,"num_write_kb":433984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4294,"num_bytes_recovered":281346048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"1,3,4","objects":516}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1833,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.336752+0000","last_change":"2024-04-12T11:46:10.400420+0000","last_active":"2024-04-12T11:46:39.336752+0000","last_peered":"2024-04-12T11:46:39.336752+0000","last_clean":"2024-04-12T11:46:39.336752+0000","last_became_active":"2024-04-12T11:46:10.400096+0000","last_became_peered":"2024-04-12T11:46:10.400096+0000","last_unstale":"2024-04-12T11:46:39.336752+0000","last_undegraded":"2024-04-12T11:46:39.336752+0000","last_fullsized":"2024-04-12T11:46:39.336752+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:43:39.116532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1843,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.327623+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:46:39.327623+0000","last_peered":"2024-04-12T11:46:39.327623+0000","last_clean":"2024-04-12T11:46:39.327623+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:46:39.327623+0000","last_undegraded":"2024-04-12T11:46:39.327623+0000","last_fullsized":"2024-04-12T11:46:39.327623+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1845,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328863+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:46:39.328863+0000","last_peered":"2024-04-12T11:46:39.328863+0000","last_clean":"2024-04-12T11:46:39.328863+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:46:39.328863+0000","last_undegraded":"2024-04-12T11:46:39.328863+0000","last_fullsized":"2024-04-12T11:46:39.328863+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"1217'10298","reported_seq":10363,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:42.722974+0000","last_change":"2024-04-12T11:44:45.439592+0000","last_active":"2024-04-12T11:46:42.722974+0000","last_peered":"2024-04-12T11:46:42.722974+0000","last_clean":"2024-04-12T11:46:42.722974+0000","last_became_active":"2024-04-12T11:44:45.419469+0000","last_became_peered":"2024-04-12T11:44:45.419469+0000","last_unstale":"2024-04-12T11:46:42.722974+0000","last_undegraded":"2024-04-12T11:46:42.722974+0000","last_fullsized":"2024-04-12T11:46:42.722974+0000","mapping_epoch":1172,"log_start":"1179'1100","ondisk_log_start":"1179'1100","created":1172,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9198,"log_dups_size":0,"ondisk_log_size":9198,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12976128,"num_objects":198,"num_object_clones":0,"num_object_copies":594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6483,"num_write_kb":213792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1203,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.061953+0000","last_change":"2024-04-12T11:45:52.986504+0000","last_active":"2024-04-12T11:46:28.061953+0000","last_peered":"2024-04-12T11:46:28.061953+0000","last_clean":"2024-04-12T11:46:28.061953+0000","last_became_active":"2024-04-12T11:45:52.984346+0000","last_became_peered":"2024-04-12T11:45:52.984346+0000","last_unstale":"2024-04-12T11:46:28.061953+0000","last_undegraded":"2024-04-12T11:46:28.061953+0000","last_fullsized":"2024-04-12T11:46:28.061953+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:26.859024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1523,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.057572+0000","last_change":"2024-04-12T11:45:53.970732+0000","last_active":"2024-04-12T11:46:28.057572+0000","last_peered":"2024-04-12T11:46:28.057572+0000","last_clean":"2024-04-12T11:46:28.057572+0000","last_became_active":"2024-04-12T11:45:53.970305+0000","last_became_peered":"2024-04-12T11:45:53.970305+0000","last_unstale":"2024-04-12T11:46:28.057572+0000","last_undegraded":"2024-04-12T11:46:28.057572+0000","last_fullsized":"2024-04-12T11:46:28.057572+0000","mapping_epoch":1210,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:42:56.998830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":540,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.066202+0000","last_change":"2024-04-12T11:46:10.396169+0000","last_active":"2024-04-12T11:46:28.066202+0000","last_peered":"2024-04-12T11:46:28.066202+0000","last_clean":"2024-04-12T11:46:28.066202+0000","last_became_active":"2024-04-12T11:46:10.395392+0000","last_became_peered":"2024-04-12T11:46:10.395392+0000","last_unstale":"2024-04-12T11:46:28.066202+0000","last_undegraded":"2024-04-12T11:46:28.066202+0000","last_fullsized":"2024-04-12T11:46:28.066202+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:19:09.329529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1183,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.045513+0000","last_change":"2024-04-12T11:45:53.016616+0000","last_active":"2024-04-12T11:46:28.045513+0000","last_peered":"2024-04-12T11:46:28.045513+0000","last_clean":"2024-04-12T11:46:28.045513+0000","last_became_active":"2024-04-12T11:45:53.016292+0000","last_became_peered":"2024-04-12T11:45:53.016292+0000","last_unstale":"2024-04-12T11:46:28.045513+0000","last_undegraded":"2024-04-12T11:46:28.045513+0000","last_fullsized":"2024-04-12T11:46:28.045513+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:53:29.382293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1217'12886","reported_seq":12989,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.398340+0000","last_change":"2024-04-12T11:42:38.227777+0000","last_active":"2024-04-12T11:46:39.398340+0000","last_peered":"2024-04-12T11:46:39.398340+0000","last_clean":"2024-04-12T11:46:39.398340+0000","last_became_active":"2024-04-12T11:42:38.227462+0000","last_became_peered":"2024-04-12T11:42:38.227462+0000","last_unstale":"2024-04-12T11:46:39.398340+0000","last_undegraded":"2024-04-12T11:46:39.398340+0000","last_fullsized":"2024-04-12T11:46:39.398340+0000","mapping_epoch":1172,"log_start":"1196'6664","ondisk_log_start":"1196'6664","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6222,"log_dups_size":0,"ondisk_log_size":6222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:36:48.862582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34078720,"num_objects":520,"num_object_clones":0,"num_object_copies":1560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12886,"num_write_kb":428992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.055025+0000","last_change":"2024-04-12T11:45:52.987902+0000","last_active":"2024-04-12T11:46:28.055025+0000","last_peered":"2024-04-12T11:46:28.055025+0000","last_clean":"2024-04-12T11:46:28.055025+0000","last_became_active":"2024-04-12T11:45:52.980281+0000","last_became_peered":"2024-04-12T11:45:52.980281+0000","last_unstale":"2024-04-12T11:46:28.055025+0000","last_undegraded":"2024-04-12T11:46:28.055025+0000","last_fullsized":"2024-04-12T11:46:28.055025+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:07:12.203048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1870,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328414+0000","last_change":"2024-04-12T11:46:10.386160+0000","last_active":"2024-04-12T11:46:39.328414+0000","last_peered":"2024-04-12T11:46:39.328414+0000","last_clean":"2024-04-12T11:46:39.328414+0000","last_became_active":"2024-04-12T11:46:10.385225+0000","last_became_peered":"2024-04-12T11:46:10.385225+0000","last_unstale":"2024-04-12T11:46:39.328414+0000","last_undegraded":"2024-04-12T11:46:39.328414+0000","last_fullsized":"2024-04-12T11:46:39.328414+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:56:58.350984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1852,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.329215+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:46:39.329215+0000","last_peered":"2024-04-12T11:46:39.329215+0000","last_clean":"2024-04-12T11:46:39.329215+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:46:39.329215+0000","last_undegraded":"2024-04-12T11:46:39.329215+0000","last_fullsized":"2024-04-12T11:46:39.329215+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"1217'12901","reported_seq":15679,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:40.395108+0000","last_change":"2024-04-12T11:46:11.427748+0000","last_active":"2024-04-12T11:46:40.395108+0000","last_peered":"2024-04-12T11:46:40.395108+0000","last_clean":"2024-04-12T11:46:40.395108+0000","last_became_active":"2024-04-12T11:46:11.384339+0000","last_became_peered":"2024-04-12T11:46:11.384339+0000","last_unstale":"2024-04-12T11:46:40.395108+0000","last_undegraded":"2024-04-12T11:46:40.395108+0000","last_fullsized":"2024-04-12T11:46:40.395108+0000","mapping_epoch":1214,"log_start":"1191'4539","ondisk_log_start":"1191'4539","created":1172,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":8362,"log_dups_size":0,"ondisk_log_size":8362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:07:13.420604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29818880,"num_objects":455,"num_object_clones":0,"num_object_copies":1365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12901,"num_write_kb":427392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1328,"num_bytes_recovered":87031808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.055577+0000","last_change":"2024-04-12T11:45:53.005585+0000","last_active":"2024-04-12T11:46:28.055577+0000","last_peered":"2024-04-12T11:46:28.055577+0000","last_clean":"2024-04-12T11:46:28.055577+0000","last_became_active":"2024-04-12T11:45:52.987880+0000","last_became_peered":"2024-04-12T11:45:52.987880+0000","last_unstale":"2024-04-12T11:46:28.055577+0000","last_undegraded":"2024-04-12T11:46:28.055577+0000","last_fullsized":"2024-04-12T11:46:28.055577+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:09.819861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1842,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.327779+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:46:39.327779+0000","last_peered":"2024-04-12T11:46:39.327779+0000","last_clean":"2024-04-12T11:46:39.327779+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:46:39.327779+0000","last_undegraded":"2024-04-12T11:46:39.327779+0000","last_fullsized":"2024-04-12T11:46:39.327779+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1847,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.329010+0000","last_change":"2024-04-12T11:45:52.982299+0000","last_active":"2024-04-12T11:46:39.329010+0000","last_peered":"2024-04-12T11:46:39.329010+0000","last_clean":"2024-04-12T11:46:39.329010+0000","last_became_active":"2024-04-12T11:45:52.981942+0000","last_became_peered":"2024-04-12T11:45:52.981942+0000","last_unstale":"2024-04-12T11:46:39.329010+0000","last_undegraded":"2024-04-12T11:46:39.329010+0000","last_fullsized":"2024-04-12T11:46:39.329010+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:18:13.711200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"1217'12706","reported_seq":20071,"reported_epoch":1217,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:46:40.948083+0000","last_change":"2024-04-12T11:46:11.551868+0000","last_active":"2024-04-12T11:46:40.948083+0000","last_peered":"2024-04-12T11:46:40.948083+0000","last_clean":"2024-04-12T11:43:55.131021+0000","last_became_active":"2024-04-12T11:46:11.506810+0000","last_became_peered":"2024-04-12T11:46:11.506810+0000","last_unstale":"2024-04-12T11:46:40.948083+0000","last_undegraded":"2024-04-12T11:46:11.391157+0000","last_fullsized":"2024-04-12T11:46:40.948083+0000","mapping_epoch":1214,"log_start":"1196'5860","ondisk_log_start":"1196'5860","created":1172,"last_epoch_clean":1181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28049408,"num_objects":428,"num_object_clones":0,"num_object_copies":1284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12706,"num_write_kb":420288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3618,"num_bytes_recovered":236978176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,4,5","objects":427},{"shards":"0,5,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":43,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.052802+0000","last_change":"2024-04-12T11:46:10.391570+0000","last_active":"2024-04-12T11:46:28.052802+0000","last_peered":"2024-04-12T11:46:28.052802+0000","last_clean":"2024-04-12T11:46:28.052802+0000","last_became_active":"2024-04-12T11:46:10.386152+0000","last_became_peered":"2024-04-12T11:46:10.386152+0000","last_unstale":"2024-04-12T11:46:28.052802+0000","last_undegraded":"2024-04-12T11:46:28.052802+0000","last_fullsized":"2024-04-12T11:46:28.052802+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:19:59.541495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1842,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.327940+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:46:39.327940+0000","last_peered":"2024-04-12T11:46:39.327940+0000","last_clean":"2024-04-12T11:46:39.327940+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:46:39.327940+0000","last_undegraded":"2024-04-12T11:46:39.327940+0000","last_fullsized":"2024-04-12T11:46:39.327940+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":35,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.054825+0000","last_change":"2024-04-12T11:46:10.409235+0000","last_active":"2024-04-12T11:46:28.054825+0000","last_peered":"2024-04-12T11:46:28.054825+0000","last_clean":"2024-04-12T11:46:28.054825+0000","last_became_active":"2024-04-12T11:46:10.408701+0000","last_became_peered":"2024-04-12T11:46:10.408701+0000","last_unstale":"2024-04-12T11:46:28.054825+0000","last_undegraded":"2024-04-12T11:46:28.054825+0000","last_fullsized":"2024-04-12T11:46:28.054825+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:03:52.034900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"1217'13042","reported_seq":20482,"reported_epoch":1217,"state":"active+recovering+undersized+remapped","last_fresh":"2024-04-12T11:46:42.919015+0000","last_change":"2024-04-12T11:45:53.236178+0000","last_active":"2024-04-12T11:46:42.919015+0000","last_peered":"2024-04-12T11:46:42.919015+0000","last_clean":"2024-04-12T11:43:55.121092+0000","last_became_active":"2024-04-12T11:45:53.167972+0000","last_became_peered":"2024-04-12T11:45:53.167972+0000","last_unstale":"2024-04-12T11:46:42.919015+0000","last_undegraded":"2024-04-12T11:46:42.919015+0000","last_fullsized":"2024-04-12T11:45:52.961241+0000","mapping_epoch":1209,"log_start":"1195'5714","ondisk_log_start":"1195'5714","created":1172,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":7328,"log_dups_size":0,"ondisk_log_size":7328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:29:33.409841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24444952,"num_objects":374,"num_object_clones":0,"num_object_copies":1122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":374,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13042,"num_write_kb":429249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3683,"num_bytes_recovered":221249560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":374}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1216,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.045429+0000","last_change":"2024-04-12T11:45:52.979745+0000","last_active":"2024-04-12T11:46:28.045429+0000","last_peered":"2024-04-12T11:46:28.045429+0000","last_clean":"2024-04-12T11:46:28.045429+0000","last_became_active":"2024-04-12T11:45:52.978936+0000","last_became_peered":"2024-04-12T11:45:52.978936+0000","last_unstale":"2024-04-12T11:46:28.045429+0000","last_undegraded":"2024-04-12T11:46:28.045429+0000","last_fullsized":"2024-04-12T11:46:28.045429+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:22.710430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1855,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328882+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:46:39.328882+0000","last_peered":"2024-04-12T11:46:39.328882+0000","last_clean":"2024-04-12T11:46:39.328882+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:46:39.328882+0000","last_undegraded":"2024-04-12T11:46:39.328882+0000","last_fullsized":"2024-04-12T11:46:39.328882+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1762,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.322742+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:46:39.322742+0000","last_peered":"2024-04-12T11:46:39.322742+0000","last_clean":"2024-04-12T11:46:39.322742+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:46:39.322742+0000","last_undegraded":"2024-04-12T11:46:39.322742+0000","last_fullsized":"2024-04-12T11:46:39.322742+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"1217'10207","reported_seq":11800,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:42.904704+0000","last_change":"2024-04-12T11:46:11.458405+0000","last_active":"2024-04-12T11:46:42.904704+0000","last_peered":"2024-04-12T11:46:42.904704+0000","last_clean":"2024-04-12T11:46:42.904704+0000","last_became_active":"2024-04-12T11:46:11.388672+0000","last_became_peered":"2024-04-12T11:46:11.388672+0000","last_unstale":"2024-04-12T11:46:42.904704+0000","last_undegraded":"2024-04-12T11:46:42.904704+0000","last_fullsized":"2024-04-12T11:46:42.904704+0000","mapping_epoch":1214,"log_start":"1176'500","ondisk_log_start":"1176'500","created":1199,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9707,"log_dups_size":0,"ondisk_log_size":9707,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7897088,"num_objects":121,"num_object_clones":0,"num_object_copies":363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6452,"num_write_kb":210304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":373,"num_bytes_recovered":24379392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1856,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328903+0000","last_change":"2024-04-12T11:45:52.979689+0000","last_active":"2024-04-12T11:46:39.328903+0000","last_peered":"2024-04-12T11:46:39.328903+0000","last_clean":"2024-04-12T11:46:39.328903+0000","last_became_active":"2024-04-12T11:45:52.979286+0000","last_became_peered":"2024-04-12T11:45:52.979286+0000","last_unstale":"2024-04-12T11:46:39.328903+0000","last_undegraded":"2024-04-12T11:46:39.328903+0000","last_fullsized":"2024-04-12T11:46:39.328903+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:10.453472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":134,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.062040+0000","last_change":"2024-04-12T11:46:10.387670+0000","last_active":"2024-04-12T11:46:28.062040+0000","last_peered":"2024-04-12T11:46:28.062040+0000","last_clean":"2024-04-12T11:46:28.062040+0000","last_became_active":"2024-04-12T11:46:10.387356+0000","last_became_peered":"2024-04-12T11:46:10.387356+0000","last_unstale":"2024-04-12T11:46:28.062040+0000","last_undegraded":"2024-04-12T11:46:28.062040+0000","last_fullsized":"2024-04-12T11:46:28.062040+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:42:34.271621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.17","version":"1216'9961","reported_seq":11493,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328708+0000","last_change":"2024-04-12T11:46:10.455983+0000","last_active":"2024-04-12T11:46:39.328708+0000","last_peered":"2024-04-12T11:46:39.328708+0000","last_clean":"2024-04-12T11:46:39.328708+0000","last_became_active":"2024-04-12T11:46:10.403019+0000","last_became_peered":"2024-04-12T11:46:10.403019+0000","last_unstale":"2024-04-12T11:46:39.328708+0000","last_undegraded":"2024-04-12T11:46:39.328708+0000","last_fullsized":"2024-04-12T11:46:39.328708+0000","mapping_epoch":1213,"log_start":"1185'1700","ondisk_log_start":"1185'1700","created":1199,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":8261,"log_dups_size":0,"ondisk_log_size":8261,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-12T11:46:42.773291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22183936,"num_objects":339,"num_object_clones":0,"num_object_copies":1017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6223,"num_write_kb":209952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":350,"num_bytes_recovered":22904832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1186,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.061894+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:46:28.061894+0000","last_peered":"2024-04-12T11:46:28.061894+0000","last_clean":"2024-04-12T11:46:28.061894+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:46:28.061894+0000","last_undegraded":"2024-04-12T11:46:28.061894+0000","last_fullsized":"2024-04-12T11:46:28.061894+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1807,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.323804+0000","last_change":"2024-04-12T11:46:10.393030+0000","last_active":"2024-04-12T11:46:39.323804+0000","last_peered":"2024-04-12T11:46:39.323804+0000","last_clean":"2024-04-12T11:46:39.323804+0000","last_became_active":"2024-04-12T11:46:10.392633+0000","last_became_peered":"2024-04-12T11:46:10.392633+0000","last_unstale":"2024-04-12T11:46:39.323804+0000","last_undegraded":"2024-04-12T11:46:39.323804+0000","last_fullsized":"2024-04-12T11:46:39.323804+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:37:56.653388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"1217'10092","reported_seq":16503,"reported_epoch":1217,"state":"active+remapped+backfill_wait","last_fresh":"2024-04-12T11:46:40.889947+0000","last_change":"2024-04-12T11:45:54.168645+0000","last_active":"2024-04-12T11:46:40.889947+0000","last_peered":"2024-04-12T11:46:40.889947+0000","last_clean":"2024-04-12T11:45:52.073387+0000","last_became_active":"2024-04-12T11:45:54.139510+0000","last_became_peered":"2024-04-12T11:45:54.139510+0000","last_unstale":"2024-04-12T11:46:40.889947+0000","last_undegraded":"2024-04-12T11:46:40.889947+0000","last_fullsized":"2024-04-12T11:46:40.889947+0000","mapping_epoch":1210,"log_start":"1176'500","ondisk_log_start":"1176'500","created":1199,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9592,"log_dups_size":0,"ondisk_log_size":9592,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15728640,"num_objects":240,"num_object_clones":0,"num_object_copies":720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":240,"num_objects_unfound":0,"num_objects_dirty":240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6334,"num_write_kb":210368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2649,"num_bytes_recovered":173604864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":240}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1171,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-04-12T11:46:28.065977+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:46:28.065977+0000","last_peered":"2024-04-12T11:46:28.065977+0000","last_clean":"2024-04-12T11:46:28.065977+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:46:28.065977+0000","last_undegraded":"2024-04-12T11:46:28.065977+0000","last_fullsized":"2024-04-12T11:46:28.065977+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1798,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.324207+0000","last_change":"2024-04-12T11:46:10.390271+0000","last_active":"2024-04-12T11:46:39.324207+0000","last_peered":"2024-04-12T11:46:39.324207+0000","last_clean":"2024-04-12T11:46:39.324207+0000","last_became_active":"2024-04-12T11:46:10.389691+0000","last_became_peered":"2024-04-12T11:46:10.389691+0000","last_unstale":"2024-04-12T11:46:39.324207+0000","last_undegraded":"2024-04-12T11:46:39.324207+0000","last_fullsized":"2024-04-12T11:46:39.324207+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:11.035208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"1217'10209","reported_seq":15694,"reported_epoch":1217,"state":"active+remapped+backfill_wait","last_fresh":"2024-04-12T11:46:42.631001+0000","last_change":"2024-04-12T11:45:54.192112+0000","last_active":"2024-04-12T11:46:42.631001+0000","last_peered":"2024-04-12T11:46:42.631001+0000","last_clean":"2024-04-12T11:45:52.342744+0000","last_became_active":"2024-04-12T11:45:54.162874+0000","last_became_peered":"2024-04-12T11:45:54.162874+0000","last_unstale":"2024-04-12T11:46:42.631001+0000","last_undegraded":"2024-04-12T11:46:42.631001+0000","last_fullsized":"2024-04-12T11:46:42.631001+0000","mapping_epoch":1210,"log_start":"1179'1100","ondisk_log_start":"1179'1100","created":1199,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9109,"log_dups_size":0,"ondisk_log_size":9109,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16449536,"num_objects":251,"num_object_clones":0,"num_object_copies":753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":251,"num_objects_unfound":0,"num_objects_dirty":251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6387,"num_write_kb":212416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2698,"num_bytes_recovered":142573568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,0],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":251}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":77,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.336568+0000","last_change":"2024-04-12T11:46:10.400994+0000","last_active":"2024-04-12T11:46:39.336568+0000","last_peered":"2024-04-12T11:46:39.336568+0000","last_clean":"2024-04-12T11:46:39.336568+0000","last_became_active":"2024-04-12T11:46:10.400663+0000","last_became_peered":"2024-04-12T11:46:10.400663+0000","last_unstale":"2024-04-12T11:46:39.336568+0000","last_undegraded":"2024-04-12T11:46:39.336568+0000","last_fullsized":"2024-04-12T11:46:39.336568+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:33:02.927692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":604,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.336870+0000","last_change":"2024-04-12T11:46:10.389628+0000","last_active":"2024-04-12T11:46:39.336870+0000","last_peered":"2024-04-12T11:46:39.336870+0000","last_clean":"2024-04-12T11:46:39.336870+0000","last_became_active":"2024-04-12T11:46:10.389222+0000","last_became_peered":"2024-04-12T11:46:10.389222+0000","last_unstale":"2024-04-12T11:46:39.336870+0000","last_undegraded":"2024-04-12T11:46:39.336870+0000","last_fullsized":"2024-04-12T11:46:39.336870+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:55:17.170994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.14","version":"1217'10728","reported_seq":10791,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:42.869047+0000","last_change":"2024-04-12T11:44:45.481247+0000","last_active":"2024-04-12T11:46:42.869047+0000","last_peered":"2024-04-12T11:46:42.869047+0000","last_clean":"2024-04-12T11:46:42.869047+0000","last_became_active":"2024-04-12T11:44:45.473643+0000","last_became_peered":"2024-04-12T11:44:45.473643+0000","last_unstale":"2024-04-12T11:46:42.869047+0000","last_undegraded":"2024-04-12T11:46:42.869047+0000","last_fullsized":"2024-04-12T11:46:42.869047+0000","mapping_epoch":1199,"log_start":"1179'900","ondisk_log_start":"1179'900","created":1199,"last_epoch_clean":1200,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:42:37.207868+0000","last_clean_scrub_stamp":"2024-04-12T11:42:37.207868+0000","objects_scrubbed":0,"log_size":9828,"log_dups_size":0,"ondisk_log_size":9828,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7438336,"num_objects":114,"num_object_clones":0,"num_object_copies":342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6779,"num_write_kb":220544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1885,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.328801+0000","last_change":"2024-04-12T11:43:26.370162+0000","last_active":"2024-04-12T11:46:39.328801+0000","last_peered":"2024-04-12T11:46:39.328801+0000","last_clean":"2024-04-12T11:46:39.328801+0000","last_became_active":"2024-04-12T11:43:26.352240+0000","last_became_peered":"2024-04-12T11:43:26.352240+0000","last_unstale":"2024-04-12T11:46:39.328801+0000","last_undegraded":"2024-04-12T11:46:39.328801+0000","last_fullsized":"2024-04-12T11:46:39.328801+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:34:09.364519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1752,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-04-12T11:46:39.322978+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:46:39.322978+0000","last_peered":"2024-04-12T11:46:39.322978+0000","last_clean":"2024-04-12T11:46:39.322978+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:46:39.322978+0000","last_undegraded":"2024-04-12T11:46:39.322978+0000","last_fullsized":"2024-04-12T11:46:39.322978+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":11634,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":472383512,"num_objects":7209,"num_object_clones":0,"num_object_copies":21627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13516,"num_objects_misplaced":1007,"num_objects_unfound":0,"num_objects_dirty":7209,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":197743,"num_write_kb":6379393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29919,"num_bytes_recovered":1905852440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2703179776,"data_stored":2703163488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":223705,"ondisk_log_size":223705,"up":78,"acting":75,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234412,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1407520,"kb_used_data":1001560,"kb_used_omap":1,"kb_used_meta":405950,"kb_avail":92337632,"statfs":{"total":95995035648,"available":94553735168,"internally_reserved":0,"allocated":1025597440,"data_stored":1022167035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":415693259},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496049,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":628632,"kb_used_data":246480,"kb_used_omap":1,"kb_used_meta":382142,"kb_avail":93116520,"statfs":{"total":95995035648,"available":95351316480,"internally_reserved":0,"allocated":252395520,"data_stored":248973743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":391313864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536358,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":649388,"kb_used_data":260644,"kb_used_omap":30,"kb_used_meta":388705,"kb_avail":93095764,"statfs":{"total":95995035648,"available":95330062336,"internally_reserved":0,"allocated":266899456,"data_stored":263508123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":398034242},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190442,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":810576,"kb_used_data":313412,"kb_used_omap":43,"kb_used_meta":497108,"kb_avail":92934576,"statfs":{"total":95995035648,"available":95165005824,"internally_reserved":0,"allocated":320933888,"data_stored":317519673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":509038635},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849570,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":921928,"kb_used_data":303996,"kb_used_omap":36,"kb_used_meta":617819,"kb_avail":92823224,"statfs":{"total":95995035648,"available":95050981376,"internally_reserved":0,"allocated":311291904,"data_stored":307884701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":632646836},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":0,"seq":2534030704923,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018135,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":836760,"kb_used_data":403404,"kb_used_omap":42,"kb_used_meta":433301,"kb_avail":92908392,"statfs":{"total":95995035648,"available":95138193408,"internally_reserved":0,"allocated":413085696,"data_stored":409666953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":443701048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757685,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":721160,"kb_used_data":224960,"kb_used_omap":1,"kb_used_meta":496190,"kb_avail":93023992,"statfs":{"total":95995035648,"available":95256567808,"internally_reserved":0,"allocated":230359040,"data_stored":226948347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":508099014},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":288227328,"data_stored":288227328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":243466240,"data_stored":243466240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":279973888,"data_stored":279969816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":30605312,"data_stored":30605312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":399441920,"data_stored":399441920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":213389312,"data_stored":213385240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":235737088,"data_stored":235733016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1012338688,"data_stored":1012334616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:46:51.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool set unique_pool_3 pgp_num 26 2024-04-12T11:46:51.422 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:51.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:51.450 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:51.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:51.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:51.577 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:51.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:51.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:51.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:51.704 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:51.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:51.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:51 smithi087 ceph-mon[89014]: pgmap v1856: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 140 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 13022/6759 objects degraded (192.662%); 498/6759 objects misplaced (7.368%); 6.0 MiB/s, 130 objects/s recovering 2024-04-12T11:46:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:51.743 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:51 smithi029 ceph-mon[93650]: pgmap v1856: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 140 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 13022/6759 objects degraded (192.662%); 498/6759 objects misplaced (7.368%); 6.0 MiB/s, 130 objects/s recovering 2024-04-12T11:46:51.743 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:51.743 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:51.743 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:51 smithi029 ceph-mon[97787]: pgmap v1856: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 140 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 13022/6759 objects degraded (192.662%); 498/6759 objects misplaced (7.368%); 6.0 MiB/s, 130 objects/s recovering 2024-04-12T11:46:51.744 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:46:51.744 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:46:51.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:51.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:51.831 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:51.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:51.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:51.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:51.961 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:51.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:52.063 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:52.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:52.089 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:52.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:52.190 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:52.214 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:52.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:52.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:52.316 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:52.339 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:52.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:52.442 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:52.466 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:52.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:52.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:52.567 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:52.591 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:52.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:52.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:52.617 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:46:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:46:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:46:52.692 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:52.716 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:52.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:52.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:52.818 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:52.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:52.843 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:52.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:52.944 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:52.968 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:52.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:52.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:53.070 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:53.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:53.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:53.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:53.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:53.222 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:53.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:53.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:53.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:53.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:53.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:53.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:53.419 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:46:53.419 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:46:53.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:53 smithi029 ceph-mon[97787]: pgmap v1857: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 66 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 12846/3234 objects degraded (397.217%); 180/3234 objects misplaced (5.566%); 6.2 MiB/s, 129 objects/s recovering 2024-04-12T11:46:53.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 12846/3234 objects degraded (397.217%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:53.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:53 smithi029 ceph-mon[93650]: pgmap v1857: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 66 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 12846/3234 objects degraded (397.217%); 180/3234 objects misplaced (5.566%); 6.2 MiB/s, 129 objects/s recovering 2024-04-12T11:46:53.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 12846/3234 objects degraded (397.217%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:53.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:53.473 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:53.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:53.574 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:53.597 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:53.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:53.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:53.699 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:53 smithi087 ceph-mon[89014]: pgmap v1857: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 66 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 12846/3234 objects degraded (397.217%); 180/3234 objects misplaced (5.566%); 6.2 MiB/s, 129 objects/s recovering 2024-04-12T11:46:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 12846/3234 objects degraded (397.217%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:46:53.722 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:53.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:53.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:53.824 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:53.850 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:53.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:53.951 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:53.976 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:53.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:53.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:54.078 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:54.101 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:54.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:54.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:54.204 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:54.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:54.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:54.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:54.329 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:54.353 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:54.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:54.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:54.378 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:46:54.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:54.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:54.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:54.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:54.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:54.607 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:54.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:54.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:54.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:54.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:54 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2929180444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:54.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:54 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:54.735 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:54.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:54.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:54.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:54 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2929180444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:54.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:54 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:54.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:54 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2929180444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:54.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:54 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:46:54.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:54.864 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:54.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:54.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:54.966 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:54.993 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:54.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:54.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:55.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:55.124 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:55.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:55.225 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:55.250 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:55.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:46:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:46:54] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2024-04-12T11:46:55.351 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:55.376 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:55.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:55.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:55.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:55.505 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:55.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:55.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:55.607 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:55.632 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:55.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:55.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:55 smithi029 ceph-mon[93650]: pgmap v1858: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 37 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 12734/1803 objects degraded (706.267%); 118/1803 objects misplaced (6.545%); 5.4 MiB/s, 154 objects/s recovering 2024-04-12T11:46:55.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:55 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:46:55.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:55 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:46:55.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:55 smithi029 ceph-mon[93650]: osdmap e1218: 8 total, 8 up, 7 in 2024-04-12T11:46:55.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:55 smithi029 ceph-mon[97787]: pgmap v1858: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 37 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 12734/1803 objects degraded (706.267%); 118/1803 objects misplaced (6.545%); 5.4 MiB/s, 154 objects/s recovering 2024-04-12T11:46:55.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:55 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:46:55.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:55 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:46:55.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:55 smithi029 ceph-mon[97787]: osdmap e1218: 8 total, 8 up, 7 in 2024-04-12T11:46:55.669 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-04-12T11:46:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:55 smithi087 ceph-mon[89014]: pgmap v1858: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 37 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 12734/1803 objects degraded (706.267%); 118/1803 objects misplaced (6.545%); 5.4 MiB/s, 154 objects/s recovering 2024-04-12T11:46:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:55 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:46:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:55 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:46:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:55 smithi087 ceph-mon[89014]: osdmap e1218: 8 total, 8 up, 7 in 2024-04-12T11:46:55.735 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:55.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:55.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:55.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:55.861 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:55.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:55.887 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:55.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:55.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:56.012 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:56.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:56.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:56.049 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:46:56.069 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:46:56.089 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:46:56.109 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:46:56.114 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:56.129 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:46:56.138 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:56.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:56.148 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:46:56.167 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:46:56.186 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:46:56.188 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:46:56.188 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:46:56.188 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:46:56.188 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:46:56.188 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:46:56.188 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:46:56.188 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:46:56.188 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:46:56.240 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:56.266 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:56.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:56.368 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:56.392 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:56.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:56.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:56.494 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:56.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:56 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2937214551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:56.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:56 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2937214551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:56.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:56.519 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:56.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:56.621 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:56.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:56.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:56.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:56.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:56 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2937214551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:46:56.747 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:46:56.772 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:56.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:56.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:46:56.873 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:56.897 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:56.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:56.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:56.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:57.021 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:57.025 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:57.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:57.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:57.127 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:57.150 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:57.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:57.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:57.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:57.277 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:57.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:57.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:57.379 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:57.403 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:57.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:57.451 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:57 smithi029 ceph-mon[93650]: pgmap v1860: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 4.9 MiB/s, 144 objects/s recovering 2024-04-12T11:46:57.451 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:57 smithi029 ceph-mon[97787]: pgmap v1860: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 4.9 MiB/s, 144 objects/s recovering 2024-04-12T11:46:57.489 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:46:57.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:57.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:57.529 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:57.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:57.629 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:57.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:57.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:57.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:57 smithi087 ceph-mon[89014]: pgmap v1860: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 4.9 MiB/s, 144 objects/s recovering 2024-04-12T11:46:57.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:57.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:57.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:57.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:57.880 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:46:57.905 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:57.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:46:58.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:46:58.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:58.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:58.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:46:58.133 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:58.159 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:58.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:58.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:58.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:58.287 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:58.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:58.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:58.389 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:58.415 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:58.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:58.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:58.542 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:58.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:58.644 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:58.670 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:58.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:58.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 12734/1452 objects degraded (876.997%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:46:58.766 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:46:58.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 12734/1452 objects degraded (876.997%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:46:58.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 12734/1452 objects degraded (876.997%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:46:58.772 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:58.796 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:58.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:58.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:58.897 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:58.923 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:58.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:58.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:59.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:46:59.050 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:59.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:46:59.151 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:46:59.176 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:46:59.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:59.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:46:59.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:59.301 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:59.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:59.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:46:59.427 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:59.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:59.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:46:59.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:46:59.495 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:46:59.495 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:46:59.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:46:59.495 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:46:59.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:121579 8.3 8:c342c08f:::benchmark_data_smithi029_194418_object15269:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:44.401168+0000", 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 135.091648787, 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.007615543, 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 121579 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.496 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401168+0000", 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401168+0000", 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401170+0000", 2024-04-12T11:46:59.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.499e-06 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401175+0000", 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8189999999999999e-06 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401186+0000", 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1049999999999999e-05 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401193+0000", 2024-04-12T11:46:59.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.182e-06 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.608442+0000", 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20724941099999999 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.608454+0000", 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1792e-05 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.408699+0000", 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.800244972 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.408784+0000", 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4818e-05 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:121596 8.13 8:c8d875f1:::benchmark_data_smithi029_194418_object15286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:44.643200+0000", 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 134.84961687000001, 2024-04-12T11:46:59.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.79399299700000003, 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 121596 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643200+0000", 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643200+0000", 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643202+0000", 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.655e-06 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643210+0000", 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4759999999999998e-06 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643229+0000", 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8681e-05 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643242+0000", 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3291e-05 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643348+0000", 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105769 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643370+0000", 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2418000000000001e-05 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.429928+0000", 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78655749699999999 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.430037+0000", 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109009 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.430234+0000", 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019738399999999999 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.431275+0000", 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001040944 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.431487+0000", 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000211296 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.437078+0000", 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00559181 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.437144+0000", 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5567000000000004e-05 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.437193+0000", 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9200000000000003e-05 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.506 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160456 8.7 8:e0bb330f:::benchmark_data_smithi029_194418_object54146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.342918+0000", 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 81.149899480000002, 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37251305099999998, 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160456 2024-04-12T11:46:59.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342918+0000", 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342918+0000", 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342919+0000", 2024-04-12T11:46:59.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4580000000000001e-06 2024-04-12T11:46:59.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342924+0000", 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8860000000000003e-06 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342936+0000", 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2337000000000001e-05 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342944+0000", 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6059999999999997e-06 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343025+0000", 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1656999999999994e-05 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343181+0000", 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015508300000000001 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343604+0000", 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00042354900000000001 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346834+0000", 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032303229999999998 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347546+0000", 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000711369 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347894+0000", 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034823999999999999 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715256+0000", 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36736171200000001 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715308+0000", 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1749999999999997e-05 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715431+0000", 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012308100000000001 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160461 8.3 8:c31b08d7:::benchmark_data_smithi029_194418_object54151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.343478+0000", 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 81.149338767000003, 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.371828979, 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160461 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343478+0000", 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343478+0000", 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343479+0000", 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2399999999999997e-07 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343483+0000", 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3039999999999998e-06 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343494+0000", 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0451999999999999e-05 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343501+0000", 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6140000000000003e-06 2024-04-12T11:46:59.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.344049+0000", 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00054763100000000003 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.344163+0000", 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113725 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.344543+0000", 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00038017299999999998 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715034+0000", 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37049053999999998 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715054+0000", 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0163999999999999e-05 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715190+0000", 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000135964 2024-04-12T11:46:59.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715252+0000", 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2842000000000006e-05 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715307+0000", 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4746000000000001e-05 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.519 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160465 8.b 8:dae09cce:::benchmark_data_smithi029_194418_object54155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.346986+0000", 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 81.145830684000003, 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37148354900000002, 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160465 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346986+0000", 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346986+0000", 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346988+0000", 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3859999999999999e-06 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346993+0000", 2024-04-12T11:46:59.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.716e-06 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347005+0000", 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1119e-05 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347012+0000", 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.695e-06 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347065+0000", 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2281999999999999e-05 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347173+0000", 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108682 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,4", 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347304+0000", 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131259 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715066+0000", 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36776108400000002 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715465+0000", 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00039961 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.718246+0000", 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027803720000000001 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.718291+0000", 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5025999999999999e-05 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.718351+0000", 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0430000000000002e-05 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.718470+0000", 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118888 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.525 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPushReply(8.b 1206/1204 [PushReplyOp(8:db105a17:::benchmark_data_smithi029_194418_object67346:head)])", 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.347683+0000", 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 81.145134079000002, 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36772148199999999, 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347683+0000", 2024-04-12T11:46:59.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347683+0000", 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347683+0000", 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9999999999999999e-07 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347687+0000", 2024-04-12T11:46:59.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3500000000000001e-06 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347694+0000", 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0890000000000002e-06 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347701+0000", 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1199999999999996e-06 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715160+0000", 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36745903600000002 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715404+0000", 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00024458699999999999 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:160461 8.3 e1206/1199 ondisk, result = 0)", 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.350060+0000", 2024-04-12T11:46:59.529 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 81.142757025999998, 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36500701200000002, 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350060+0000", 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350060+0000", 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350061+0000", 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.097e-06 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350066+0000", 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8600000000000001e-06 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350077+0000", 2024-04-12T11:46:59.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0832000000000001e-05 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350084+0000", 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7190000000000001e-06 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715014+0000", 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36492922900000002 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715026+0000", 2024-04-12T11:46:59.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2103000000000001e-05 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715067+0000", 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1171999999999998e-05 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160468 8.13 8:c940b4f5:::benchmark_data_smithi029_194418_object54158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.350298+0000", 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 81.142518901000003, 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.41454260900000001, 2024-04-12T11:46:59.533 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160468 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350298+0000", 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350298+0000", 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350299+0000", 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.195e-06 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350304+0000", 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9860000000000002e-06 2024-04-12T11:46:59.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715096+0000", 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36479172900000001 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715110+0000", 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3563999999999999e-05 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715435+0000", 2024-04-12T11:46:59.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032556700000000002 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715576+0000", 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014090700000000001 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.716039+0000", 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00046330200000000002 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.759398+0000", 2024-04-12T11:46:59.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043358830000000001 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.759416+0000", 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7728999999999999e-05 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.764532+0000", 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051162480000000003 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.764800+0000", 2024-04-12T11:46:59.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000267896 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.764841+0000", 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0655999999999998e-05 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:166686 8.10 8:0f70ea2e:::benchmark_data_smithi029_194418_object60376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1207)", 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:48.383130+0000", 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 71.109686687000007, 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37085964300000002, 2024-04-12T11:46:59.539 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166686 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383130+0000", 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383130+0000", 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383132+0000", 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3570000000000001e-06 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383137+0000", 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7089999999999999e-06 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383157+0000", 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9262e-05 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383165+0000", 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9009999999999994e-06 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383213+0000", 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8167000000000001e-05 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383342+0000", 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000128885 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4,5", 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383718+0000", 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00037679800000000002 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753747+0000", 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37002887800000001 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753778+0000", 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0664000000000003e-05 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753828+0000", 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9713999999999999e-05 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753870+0000", 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2629000000000001e-05 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753912+0000", 2024-04-12T11:46:59.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1334000000000001e-05 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753990+0000", 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8344999999999996e-05 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:166687 8.3 8:c4dc7983:::benchmark_data_smithi029_194418_object60377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1207)", 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:48.383448+0000", 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 71.109368735999993, 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37355421100000002, 2024-04-12T11:46:59.545 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166687 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383448+0000", 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383448+0000", 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383449+0000", 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0400000000000005e-07 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383454+0000", 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8760000000000004e-06 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383465+0000", 2024-04-12T11:46:59.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0628000000000001e-05 2024-04-12T11:46:59.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383472+0000", 2024-04-12T11:46:59.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.413e-06 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383528+0000", 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6011999999999998e-05 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383646+0000", 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011756699999999999 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.384094+0000", 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00044867000000000003 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753559+0000", 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36946446300000002 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753688+0000", 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000129333 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.756946+0000", 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003258169 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.756976+0000", 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0133000000000001e-05 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757002+0000", 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6043000000000001e-05 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.552 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:166689 8.13 8:cf77213d:::benchmark_data_smithi029_194418_object60379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1207)", 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:48.385913+0000", 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 71.106903973000001, 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37121580300000001, 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166689 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.553 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.385913+0000", 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.385913+0000", 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.385914+0000", 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4020000000000001e-06 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.385920+0000", 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4199999999999998e-06 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753345+0000", 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36742511 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753368+0000", 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2959000000000002e-05 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753536+0000", 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000168536 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753680+0000", 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014355999999999999 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.754100+0000", 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041976999999999997 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.756970+0000", 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028701680000000002 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757009+0000", 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8596000000000001e-05 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757052+0000", 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3241999999999997e-05 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757095+0000", 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3022000000000002e-05 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757129+0000", 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4017999999999999e-05 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.558 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:166687 8.3 e1207/1199 ondisk, result = 0)", 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:48.388674+0000", 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 71.104142969999998, 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.364894616, 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.388674+0000", 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.388674+0000", 2024-04-12T11:46:59.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.388675+0000", 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9100000000000002e-07 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.388681+0000", 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5409999999999997e-06 2024-04-12T11:46:59.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753344+0000", 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36466264300000001 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753368+0000", 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3822e-05 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753543+0000", 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017491100000000001 2024-04-12T11:46:59.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753551+0000", 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5550000000000003e-06 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753569+0000", 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7252999999999999e-05 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.562 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:167527 8.b e1208/1207 8:d8b3b4b5:::benchmark_data_smithi029_194418_object61217:head v 1208'10757, mlcod=1208'10755)", 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.368934+0000", 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 69.123883340999996, 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37717215500000001, 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368934+0000", 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368934+0000", 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368935+0000", 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.048e-06 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368941+0000", 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4930000000000003e-06 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368952+0000", 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0958000000000001e-05 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368959+0000", 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6449999999999999e-06 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369095+0000", 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000136475 2024-04-12T11:46:59.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369115+0000", 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9879000000000001e-05 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.375105+0000", 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059899319999999999 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.746106+0000", 2024-04-12T11:46:59.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37100072499999998 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167531 8.f 8:f56e1a80:::benchmark_data_smithi029_194418_object61221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.369039+0000", 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 69.123777623999999, 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.45084892500000001, 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.567 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167531 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369039+0000", 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369039+0000", 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369040+0000", 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0920000000000001e-06 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369044+0000", 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4199999999999999e-06 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369051+0000", 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5800000000000003e-06 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369057+0000", 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8510000000000004e-06 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370219+0000", 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011614049999999999 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370344+0000", 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000124948 2024-04-12T11:46:59.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,4", 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370468+0000", 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012388500000000001 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752078+0000", 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38161004300000001 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752119+0000", 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1168000000000003e-05 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752172+0000", 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3448000000000001e-05 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819711+0000", 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067538835000000005 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819764+0000", 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3220000000000002e-05 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819888+0000", 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012402999999999999 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167533 8.f 8:f5e9f8d9:::benchmark_data_smithi029_194418_object61223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.369313+0000", 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 69.123504346000004, 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.45089991499999998, 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167533 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369313+0000", 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369313+0000", 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369314+0000", 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9800000000000002e-07 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369318+0000", 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.685e-06 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369329+0000", 2024-04-12T11:46:59.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0509000000000001e-05 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369336+0000", 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5850000000000002e-06 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374380+0000", 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050431699999999996 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374496+0000", 2024-04-12T11:46:59.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011610100000000001 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,4", 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374620+0000", 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012398500000000001 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752288+0000", 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37766854799999999 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752332+0000", 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3488000000000001e-05 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752374+0000", 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1845999999999999e-05 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820056+0000", 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067682592 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820100+0000", 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3464999999999998e-05 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820213+0000", 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011294300000000001 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167535 8.0 8:023edd32:::benchmark_data_smithi029_194418_object61225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:46:59.579 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 69.123469309000001, 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38064882700000002, 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167535 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9999999999999998e-07 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369350+0000", 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.351e-06 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369356+0000", 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.874e-06 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369362+0000", 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1639999999999998e-06 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369478+0000", 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115281 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369611+0000", 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000132879 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4,5", 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370066+0000", 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000455045 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.746687+0000", 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37662145200000002 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749353+0000", 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026659259999999999 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749762+0000", 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000408537 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749809+0000", 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7679000000000002e-05 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749868+0000", 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8362999999999997e-05 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749997+0000", 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012887599999999999 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167536 8.17 8:effa3b55:::benchmark_data_smithi029_194418_object61226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.372734+0000", 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 69.120083141999999, 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.447314869, 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167536 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.586 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372734+0000", 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372734+0000", 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372735+0000", 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.53e-06 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372740+0000", 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4349999999999999e-06 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372752+0000", 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2604e-05 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372761+0000", 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4349999999999997e-06 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.373764+0000", 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001003309 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.373898+0000", 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0001338 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374260+0000", 2024-04-12T11:46:59.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00036174299999999998 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752035+0000", 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37777566800000001 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752211+0000", 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000175685 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:46:59.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752248+0000", 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6981999999999999e-05 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819903+0000", 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067654598999999996 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819947+0000", 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3894999999999999e-05 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820049+0000", 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102184 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:167531 8.f e1208/1204 ondisk, result = 0)", 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.376542+0000", 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 69.116275508000001, 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37563801400000002, 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376542+0000", 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376542+0000", 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376543+0000", 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.229e-06 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376548+0000", 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2159999999999997e-06 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749229+0000", 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.372680911 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749236+0000", 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6140000000000003e-06 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752159+0000", 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029229360000000001 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752167+0000", 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1720000000000001e-06 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752180+0000", 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2935999999999999e-05 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:167536 8.17 e1208/1204 ondisk, result = 0)", 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 69.115620376999999, 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37484725600000002, 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:46:59.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7100000000000001e-07 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377202+0000", 2024-04-12T11:46:59.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8250000000000004e-06 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.745962+0000", 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36876029599999999 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.745974+0000", 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1501000000000001e-05 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752020+0000", 2024-04-12T11:46:59.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006045847 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752028+0000", 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4959999999999996e-06 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752044+0000", 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.562e-05 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:167531 8.f e1208/1204 ondisk, result = 0)", 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.379123+0000", 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 69.113694093999996, 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37296465699999998, 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379123+0000", 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379123+0000", 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379124+0000", 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6599999999999996e-07 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379128+0000", 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2030000000000002e-06 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.748158+0000", 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.369030319 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.748173+0000", 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4620999999999999e-05 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752064+0000", 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038913590000000001 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752071+0000", 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9770000000000003e-06 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752088+0000", 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6512e-05 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:46:59.603 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:46:59.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:46:59 smithi029 ceph-mon[97787]: pgmap v1861: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 1.8 MiB/s, 211 objects/s recovering 2024-04-12T11:46:59.618 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:46:59 smithi029 ceph-mon[93650]: pgmap v1861: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 1.8 MiB/s, 211 objects/s recovering 2024-04-12T11:46:59.632 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:59.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:59.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:46:59 smithi087 ceph-mon[89014]: pgmap v1861: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 1.8 MiB/s, 211 objects/s recovering 2024-04-12T11:46:59.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:46:59.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:59.757 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:59.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:46:59.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:46:59.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:46:59.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:46:59.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:46:59.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:00.006 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:00.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:00.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:00.108 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:00.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:00.133 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:00.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:00.234 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:00.258 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:00.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:00.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:00.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:00.385 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:00.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:00.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:00.511 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:00.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:00.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:00.612 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:00.636 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:00.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:00.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:00.737 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:00.761 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:00.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:00.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:00.863 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:00.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:00.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:00.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:00.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:01.011 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:01.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:01.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:01.113 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:01.138 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:01.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:01.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:01.240 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:01.265 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:01.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:01.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:01.390 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:01.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:01.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:01.458 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:01 smithi029 ceph-mon[97787]: pgmap v1862: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 212 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 1.8 MiB/s, 211 objects/s recovering 2024-04-12T11:47:01.459 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:01 smithi029 ceph-mon[93650]: pgmap v1862: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 212 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 1.8 MiB/s, 211 objects/s recovering 2024-04-12T11:47:01.485 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-04-12T11:47:01.493 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:01.516 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:01.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:01.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:01.540 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:01.541 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:01.618 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:01.642 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:01.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:01.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:01 smithi087 ceph-mon[89014]: pgmap v1862: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 212 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 1.8 MiB/s, 211 objects/s recovering 2024-04-12T11:47:01.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:01.773 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:01.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:01.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:01.898 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:01.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:01.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:02.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:02.024 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:02.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:02.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:02.126 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:02.150 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:02.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:02.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:02.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:02.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:02.279 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:02.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:02.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:02.409 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:02.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:02.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:02.511 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:02.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:02.536 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:02.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:02.637 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:02.660 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:02.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:02.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:47:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:47:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:47:02.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:02.786 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:02.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:02.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:02.890 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:02.913 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:02.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:02.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:03.015 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:03.039 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:03.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:03.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:03.165 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:03.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:03.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:03.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:03.290 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:03.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:03.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:03.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:03.401 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 8 pgp_num to 26 2024-04-12T11:47:03.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:03.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:03.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:03.523 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:03.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:03.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:03.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:03.648 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:03.672 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:03.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:03 smithi087 ceph-mon[89014]: pgmap v1863: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 717 KiB/s, 193 objects/s recovering 2024-04-12T11:47:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:03 smithi087 ceph-mon[89014]: osdmap e1219: 8 total, 8 up, 7 in 2024-04-12T11:47:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:03 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2213448852' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2024-04-12T11:47:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:03 smithi029 ceph-mon[97787]: pgmap v1863: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 717 KiB/s, 193 objects/s recovering 2024-04-12T11:47:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:03 smithi029 ceph-mon[97787]: osdmap e1219: 8 total, 8 up, 7 in 2024-04-12T11:47:03.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:03 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2213448852' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2024-04-12T11:47:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:03 smithi029 ceph-mon[93650]: pgmap v1863: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 717 KiB/s, 193 objects/s recovering 2024-04-12T11:47:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:03 smithi029 ceph-mon[93650]: osdmap e1219: 8 total, 8 up, 7 in 2024-04-12T11:47:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:03 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2213448852' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2024-04-12T11:47:03.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:03.797 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:03.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:03.899 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:03.922 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:03.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:03.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:04.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:04.045 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-04-12T11:47:04.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-04-12T11:47:04.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:04.048 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:04.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:04.149 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:04.173 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:04.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:04.275 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:04.299 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:04.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:04.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:04.428 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:04.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:04.530 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:04.555 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:04.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:04.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:04.656 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:04.680 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:04.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:04.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:04 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2213448852' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]': finished 2024-04-12T11:47:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:04 smithi087 ceph-mon[89014]: osdmap e1220: 8 total, 8 up, 7 in 2024-04-12T11:47:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:04 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2213448852' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]': finished 2024-04-12T11:47:04.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:04 smithi029 ceph-mon[93650]: osdmap e1220: 8 total, 8 up, 7 in 2024-04-12T11:47:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:04 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2213448852' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]': finished 2024-04-12T11:47:04.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:04 smithi029 ceph-mon[97787]: osdmap e1220: 8 total, 8 up, 7 in 2024-04-12T11:47:04.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:04.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:04.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:04.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:04.893 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:04.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:04.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:04.934 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:04.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:05.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:05.061 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:05.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:05.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:05.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:05.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:05.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:05.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:05.194 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:47:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:47:04] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2024-04-12T11:47:05.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:05.313 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:05.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:05.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:05.414 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:05.438 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:05.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:05.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:05.471 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:05.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:05 smithi029 ceph-mon[97787]: pgmap v1866: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 0 B/s, 369 objects/s recovering 2024-04-12T11:47:05.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:05 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 12734/1452 objects degraded (876.997%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:47:05.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:05 smithi029 ceph-mon[93650]: pgmap v1866: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 0 B/s, 369 objects/s recovering 2024-04-12T11:47:05.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:05 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 12734/1452 objects degraded (876.997%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:47:05.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:05.562 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:05.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:05.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:05.688 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:05.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:05.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:05 smithi087 ceph-mon[89014]: pgmap v1866: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 0 B/s, 369 objects/s recovering 2024-04-12T11:47:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:05 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 12734/1452 objects degraded (876.997%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:47:05.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:05.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:05.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:05.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:05.916 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:05.940 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:05.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:05.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:06.042 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:06.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:06.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:06.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:06.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:06.191 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:06.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:06.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:06.292 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:06.316 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:06.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:06.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:06.412 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:47:06.418 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:06.442 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:06.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:06.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:06.543 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:06.568 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:06.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:06.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:06.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:06.696 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:06.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4191361186' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:47:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:06 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:47:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:06.725 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4191361186' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:47:06.725 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:06 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:47:06.725 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:06.726 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4191361186' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:47:06.726 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:06 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:47:06.726 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:06.798 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:06.822 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:06.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:06.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:06.868 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:47:06.891 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:47:06.910 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:47:06.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:06.933 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:47:06.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:06.951 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:06.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:06.954 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:47:06.974 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:47:06.993 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:47:07.013 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:47:07.014 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:47:07.014 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:47:07.014 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:47:07.014 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:47:07.014 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:47:07.014 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:47:07.015 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:47:07.015 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:47:07.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:07.077 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:07.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:07.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:07.179 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:07.202 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:07.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:07.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:07.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:07.327 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:07.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:07.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:07.429 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:07.452 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:07.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:07.554 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:07.577 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:07.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:07.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:07.679 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:07.702 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:07.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:07.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:07 smithi087 ceph-mon[89014]: pgmap v1867: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 0 B/s, 200 objects/s recovering 2024-04-12T11:47:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:07 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:47:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:07 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:47:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:07 smithi087 ceph-mon[89014]: osdmap e1221: 8 total, 8 up, 7 in 2024-04-12T11:47:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:07 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3215227407' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:47:07.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:07 smithi029 ceph-mon[97787]: pgmap v1867: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 0 B/s, 200 objects/s recovering 2024-04-12T11:47:07.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:07 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:47:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:07 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:47:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:07 smithi029 ceph-mon[97787]: osdmap e1221: 8 total, 8 up, 7 in 2024-04-12T11:47:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:07 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3215227407' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:47:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:07 smithi029 ceph-mon[93650]: pgmap v1867: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 89 active+clean, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 12734/1452 objects degraded (876.997%); 118/1452 objects misplaced (8.127%); 0 B/s, 200 objects/s recovering 2024-04-12T11:47:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:07 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:47:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:07 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:47:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:07 smithi029 ceph-mon[93650]: osdmap e1221: 8 total, 8 up, 7 in 2024-04-12T11:47:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:07 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3215227407' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:47:07.805 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:07.829 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:07.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:07.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:07.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:07.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:07.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:07.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:08.056 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:08.081 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:08.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:08.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:08.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:08.207 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:08.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:08.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:08.309 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:08.333 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:08.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:08.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:08.434 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:08.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:08.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:08.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:08.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:08.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:08.586 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:08.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:08.687 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:08.711 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:08.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:08.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:08.814 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:08.840 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:08.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:08.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:08.943 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:08.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:08.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:08.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:09.045 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:47:09.045 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:47:09.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:09.096 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:09.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:09.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:09.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:09.222 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:09.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:09.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:09.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:09.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:09.349 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:09.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:09.450 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:09.476 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:09.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:09.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:09.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:09 smithi029 ceph-mon[97787]: pgmap v1869: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait; 29 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 10832/1452 objects degraded (746.006%); 118/1452 objects misplaced (8.127%); 0 B/s, 507 objects/s recovering 2024-04-12T11:47:09.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:09 smithi029 ceph-mon[97787]: osdmap e1222: 8 total, 8 up, 7 in 2024-04-12T11:47:09.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:47:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:09 smithi029 ceph-mon[93650]: pgmap v1869: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait; 29 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 10832/1452 objects degraded (746.006%); 118/1452 objects misplaced (8.127%); 0 B/s, 507 objects/s recovering 2024-04-12T11:47:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:09 smithi029 ceph-mon[93650]: osdmap e1222: 8 total, 8 up, 7 in 2024-04-12T11:47:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:47:09.578 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:09.601 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:09.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:09.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:09.703 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:09 smithi087 ceph-mon[89014]: pgmap v1869: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait; 29 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 10832/1452 objects degraded (746.006%); 118/1452 objects misplaced (8.127%); 0 B/s, 507 objects/s recovering 2024-04-12T11:47:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:09 smithi087 ceph-mon[89014]: osdmap e1222: 8 total, 8 up, 7 in 2024-04-12T11:47:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:47:09.729 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:09.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:09.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:09.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:09.856 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:09.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:09.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:09.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:09.983 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:09.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:10.053 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:47:10.085 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:10.109 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:10.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:10.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:10.210 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:10.234 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:10.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:10.336 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:10.361 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:10.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:10.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:10.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:10.487 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:10.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:10.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:10.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:10 smithi029 ceph-mon[93650]: osdmap e1223: 8 total, 8 up, 7 in 2024-04-12T11:47:10.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:10 smithi029 ceph-mon[93650]: pgmap v1872: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 91 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 10832/1452 objects degraded (746.006%); 118/1452 objects misplaced (8.127%); 0 B/s, 241 objects/s recovering 2024-04-12T11:47:10.562 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:10 smithi029 ceph-mon[97787]: osdmap e1223: 8 total, 8 up, 7 in 2024-04-12T11:47:10.562 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:10 smithi029 ceph-mon[97787]: pgmap v1872: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 91 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 10832/1452 objects degraded (746.006%); 118/1452 objects misplaced (8.127%); 0 B/s, 241 objects/s recovering 2024-04-12T11:47:10.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:10.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:10.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:10.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:10.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:47:10.714 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:10.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:10 smithi087 ceph-mon[89014]: osdmap e1223: 8 total, 8 up, 7 in 2024-04-12T11:47:10.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:10 smithi087 ceph-mon[89014]: pgmap v1872: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 91 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 29 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 10832/1452 objects degraded (746.006%); 118/1452 objects misplaced (8.127%); 0 B/s, 241 objects/s recovering 2024-04-12T11:47:10.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:10.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:10.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:10.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:10.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:10.869 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:10.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:10.971 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:10.995 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:10.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:10.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:11.097 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:11.120 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:11.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:11.222 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:11.247 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:11.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:11.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:11.307 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:47:11.307 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:47:11.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:11.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:11.373 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:11.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:11.437 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_3' removed 2024-04-12T11:47:11.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:11 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/592418969' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:47:11.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:11 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:47:11.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:11 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/592418969' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:47:11.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:11 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:47:11.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:11.533 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:11.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:11.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:11.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:11.663 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:11.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:11.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:11 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/592418969' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:47:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:11 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:47:11.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:11.788 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:11.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:11.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:11.913 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:11.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:11.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:12.015 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:12.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:12.041 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:12.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:12.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-04-12T11:47:12.142 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:12.170 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:12.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:12.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:12.272 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:12.296 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:12.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:12.395 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:12.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:12.424 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:12.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:12.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:12.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:12.549 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:12.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:12.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:12.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:12.674 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:12.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:12.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:47:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:47:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:47:12.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:12 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:47:12.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:12 smithi087 ceph-mon[89014]: osdmap e1224: 8 total, 8 up, 7 in 2024-04-12T11:47:12.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:12 smithi087 ceph-mon[89014]: pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:12.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:12 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:47:12.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:12 smithi029 ceph-mon[93650]: osdmap e1224: 8 total, 8 up, 7 in 2024-04-12T11:47:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:12 smithi029 ceph-mon[93650]: pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:12.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:12 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:47:12.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:12 smithi029 ceph-mon[97787]: osdmap e1224: 8 total, 8 up, 7 in 2024-04-12T11:47:12.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:12 smithi029 ceph-mon[97787]: pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:12.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:12.799 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:12.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:12.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:12.850 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-12T11:47:12.851 INFO:tasks.radosbench:Beginning radosbench... 2024-04-12T11:47:12.851 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-04-12T11:47:12.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-04-12T11:47:12.901 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:12.926 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:12.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:12.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:13.028 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:13.054 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:13.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:13.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:13.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:13.181 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:13.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:13.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:13.306 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:13.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:13.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:13.408 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:13.435 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:13.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:13.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:13.537 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:13.561 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:13.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:13.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:13.663 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:13.687 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:13.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:13.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:13 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10832/1452 objects degraded (746.006%), 5 pgs degraded, 2 pgs undersized) 2024-04-12T11:47:13.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:13 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10832/1452 objects degraded (746.006%), 5 pgs degraded, 2 pgs undersized) 2024-04-12T11:47:13.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:13 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10832/1452 objects degraded (746.006%), 5 pgs degraded, 2 pgs undersized) 2024-04-12T11:47:13.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:13.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:13.814 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:13.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:13.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:13.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:13.943 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:13.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:14.044 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:14.070 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:14.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:14.172 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:14.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:14.198 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:14.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:14.299 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:14.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:14.324 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:14.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:14.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:14.452 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:14.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:14.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:14.553 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:14.580 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:14.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:14.681 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:14.705 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:14.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:14.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:14.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:14 smithi087 ceph-mon[89014]: pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:14.752 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:14 smithi029 ceph-mon[93650]: pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:14.752 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:14 smithi029 ceph-mon[97787]: pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:14.807 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:14.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:14.835 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:14.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:14.936 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:14.962 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:14.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:14.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:15.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:47:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:47:14] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2024-04-12T11:47:15.064 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:15.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:15.092 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:15.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:15.193 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:15.222 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:15.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:15.323 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:15.350 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:15.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:15.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:15.452 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:15.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:15.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:15.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:15.582 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:15.610 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:15.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:15.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:15.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:15.740 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:15.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:15.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:15.842 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:15.869 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:15.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:15.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:15.971 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:15.999 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:15.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:16.100 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:16.128 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:16.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:16.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:16.229 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:16.254 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:16.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:16.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:16.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:16.382 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:16.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:16.484 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:16.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:16.512 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:16.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:16.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:16.642 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:16.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:16.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:16.772 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:16.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:16.873 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:16.901 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:16.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:16.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:17.003 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:17.033 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:17.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:17.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:17.135 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:17.163 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:17.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:17.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:17.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:16 smithi087 ceph-mon[89014]: pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:17.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:16 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:17.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:17.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:16 smithi029 ceph-mon[93650]: pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:17.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:17 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:17.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:16 smithi029 ceph-mon[97787]: pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:17.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:17 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:17.292 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:17.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:17.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:17.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:17.422 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:17.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:17.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:17.523 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:17.551 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:17.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:17.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:17.653 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:17.681 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:17.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:17.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:17.783 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:17.810 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:17.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:17.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:17.912 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:17.940 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:17.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:17.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:17.994 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:17.994 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:18.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:18.069 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:18.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:18.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:18.171 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:18.198 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:18.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:18.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:18.300 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:18.327 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:18.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:18.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:18.421 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:18.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:18.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:18.459 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:18.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:18.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:18.585 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:18.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:18.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:18.687 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:18.715 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:18.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:18.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:18.816 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:18.843 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:18.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:18.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:18.945 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:18.973 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:18.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:19.075 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:19.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:19.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:19.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:19.232 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:19.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:19.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:19.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:18 smithi029 ceph-mon[93650]: pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:19.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:18 smithi029 ceph-mon[97787]: pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:19.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:19.362 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:19.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:19.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:19.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:18 smithi087 ceph-mon[89014]: pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:19.491 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:19.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:19.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:19.593 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:19.619 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:19.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:19.721 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:19.746 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:19.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:19.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:19.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:19.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:19.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:19.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:19.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:20.005 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:20.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:20.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:20.107 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:20.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:20.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:20.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:20.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:20.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:20.266 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:20.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:20.367 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:20.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:20.397 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:20.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:20.499 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:20.530 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:20.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:20.632 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:20.660 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:20.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:20.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:20.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:20.789 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:20.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:20.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:20.891 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:20.919 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:20.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:20.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:21.021 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:21.048 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:21.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:21.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:21.149 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:21.176 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:21.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:21.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:20 smithi029 ceph-mon[93650]: pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:21.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:20 smithi029 ceph-mon[97787]: pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:21.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:21.305 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:21.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:21.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:21.335 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:20 smithi087 ceph-mon[89014]: pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:21.406 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:21.432 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:21.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:21.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:21.534 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:21.562 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:21.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:21.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:21.663 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:21.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:21.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:21.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:21.817 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:21.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:21.919 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:21.943 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:21.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:22.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:22.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:22.074 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:22.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:22.175 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:22.203 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:22.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:22.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:22.304 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:22.331 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:22.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:22.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:22.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:22.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:22.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:22.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:47:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:47:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:47:22.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:22.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:22.562 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:22.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:22.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:22.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:22.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:22.715 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:22.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:22.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:22.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:22.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:22.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:22.818 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:22.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:22.846 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:22.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:22.947 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:22.975 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:22.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:22.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:23.076 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:23.101 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:23.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:23.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:23.203 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:23.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:23.232 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:23.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:23.333 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:23.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:23.358 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:23.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:23.459 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:23.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:23 smithi087 ceph-mon[89014]: pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:23.487 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:23.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:23.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:23.584 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:23 smithi029 ceph-mon[97787]: pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:23.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:23 smithi029 ceph-mon[93650]: pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:23.589 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:23.616 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:23.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:23.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:23.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:23.746 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:23.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:23.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:23.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:23.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:23.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:23.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:23.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:24.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:24.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:24.103 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:24.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:24.127 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:24.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:24.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:24.256 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:24.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:24.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:24.357 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:24.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:24.381 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:24.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:24.482 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:24.507 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:24.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:24.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:24.609 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:24.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:24.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:24.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:24.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:24.759 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:24.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:24.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:24.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:24.885 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:24.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:24.986 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:25.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:25.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:25.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:25.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:25.135 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:25.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:25.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:25.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:25.261 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:25.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:25.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:47:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:47:24] "GET /metrics HTTP/1.1" 200 34173 "" "Prometheus/2.43.0" 2024-04-12T11:47:25.363 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:25.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:25.388 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:25.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:25.489 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:25.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:25.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:25.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:25.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:25.631 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:25 smithi029 ceph-mon[97787]: pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:25.631 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:25.631 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:25.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:25 smithi029 ceph-mon[93650]: pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:25.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:25.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:25.654 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:25.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:25.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:25 smithi087 ceph-mon[89014]: pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:25.756 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:25.784 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:25.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:25.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:25.886 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:25.913 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:25.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:26.014 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:26.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:26.039 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:26.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:26.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:26.164 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:26.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:26.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:26.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:26.290 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:26.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:26.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:26.325 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:47:26.392 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:26.417 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:26.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:26.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:26.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:26.542 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:26.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:26.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:26.644 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:26.670 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:26.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:26 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1498686328' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:47:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:26 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:47:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:26 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1498686328' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:47:26.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:26 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:47:26.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:26 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1498686328' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:47:26.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:26 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:47:26.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:26.795 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:26.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:26.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:26.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:26.923 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:26.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:26.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:27.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:27.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:27.051 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:27.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:27.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:27.165 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:47:27.165 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:47:27.182 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:27.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:27.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:27.308 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:27.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:27.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:27.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:27.434 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:27.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:27.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:27.536 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:27.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:27.565 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:27.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:27.666 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:27.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:27.690 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:27.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:27 smithi087 ceph-mon[89014]: pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:27 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:47:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:27 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:47:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:27 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:47:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:27 smithi087 ceph-mon[89014]: osdmap e1225: 8 total, 8 up, 7 in 2024-04-12T11:47:27.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:27 smithi029 ceph-mon[93650]: pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:27 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:47:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:27 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:47:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:27 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:47:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:27 smithi029 ceph-mon[93650]: osdmap e1225: 8 total, 8 up, 7 in 2024-04-12T11:47:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:27 smithi029 ceph-mon[97787]: pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:27 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:47:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:27 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:47:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:27 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:47:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:27 smithi029 ceph-mon[97787]: osdmap e1225: 8 total, 8 up, 7 in 2024-04-12T11:47:27.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:27.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:27.820 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:27.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:27.921 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:27.948 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:27.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:27.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:28.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:28.074 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:28.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:28.176 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:28.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:28.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:28.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:28.301 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:28.326 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:28.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:28.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:28.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:28.452 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:28.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:28.553 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:28.581 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:28.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:28.596 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-04-12T11:47:28.682 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:28.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:28.711 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:28.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:28.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:28.835 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:28.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:28.937 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:28.961 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:28.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:28.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:29.063 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:29.090 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:29.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:29.192 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:29.219 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:29.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:29.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:29.321 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:29.347 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:29.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:29.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:29.449 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:29.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:29.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:29.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:29.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:29 smithi029 ceph-mon[93650]: pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:29.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:29 smithi029 ceph-mon[97787]: pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:29.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:29.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:29.603 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:29.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:29.704 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:29 smithi087 ceph-mon[89014]: pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:29.732 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:29.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:29.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:29.834 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:29.862 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:29.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:29.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:29.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:29.989 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:29.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:29.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:30.091 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:30.115 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:30.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:30.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:30.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:30.241 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:30.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:30.334 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_4' created 2024-04-12T11:47:30.343 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:30.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:30.370 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:30.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:30.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:30.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:30.495 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:30.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:30.596 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:30.624 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:30.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:30.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:30 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/436830736' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2024-04-12T11:47:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:30 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2024-04-12T11:47:30.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:30.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:30.751 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:30.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:30.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:30 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/436830736' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2024-04-12T11:47:30.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:30 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2024-04-12T11:47:30.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:30 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/436830736' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2024-04-12T11:47:30.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:30 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2024-04-12T11:47:30.853 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:30.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:30.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:30.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:30.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:31.007 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:31.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:31.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:31.109 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:31.130 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-04-12T11:47:31.136 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:31.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:31.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:31.238 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:31.266 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:31.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:31.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:31.370 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:31.394 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:31.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:31.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:31.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:31.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:31 smithi029 ceph-mon[97787]: pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:31.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:31 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-04-12T11:47:31.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:31 smithi029 ceph-mon[97787]: osdmap e1226: 8 total, 8 up, 7 in 2024-04-12T11:47:31.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:31 smithi029 ceph-mon[93650]: pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:31.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:31 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-04-12T11:47:31.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:31 smithi029 ceph-mon[93650]: osdmap e1226: 8 total, 8 up, 7 in 2024-04-12T11:47:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:31.520 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:31.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:31.621 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:31.650 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:31.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:31.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:31 smithi087 ceph-mon[89014]: pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:31 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-04-12T11:47:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:31 smithi087 ceph-mon[89014]: osdmap e1226: 8 total, 8 up, 7 in 2024-04-12T11:47:31.753 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:31.781 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:31.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:31.834 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:31.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:31.911 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:31.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:31.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:32.013 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:32.040 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:32.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:32.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:32.142 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:32.168 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:32.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:32.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:32.270 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:32.296 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:32.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:32.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:32.381 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:47:32.398 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:32.413 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:47:32.422 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:32.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:32.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:32.432 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:47:32.452 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:47:32.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:47:32.472 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:47:32.493 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:47:32.511 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:32 smithi029 ceph-mon[93650]: osdmap e1227: 8 total, 8 up, 7 in 2024-04-12T11:47:32.512 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:32 smithi029 ceph-mon[97787]: osdmap e1227: 8 total, 8 up, 7 in 2024-04-12T11:47:32.512 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:47:32.524 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:32.532 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:47:32.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:47:32.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:47:32.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:47:32.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:47:32.534 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:47:32.534 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:47:32.534 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:47:32.534 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:47:32.549 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:32.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:32.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:32.650 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:32.674 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:32.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:32.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:32 smithi087 ceph-mon[89014]: osdmap e1227: 8 total, 8 up, 7 in 2024-04-12T11:47:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:47:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:47:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:47:32.777 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:32.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:32.803 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:32.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:32.904 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:32.927 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:32.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:32.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:33.029 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:33.054 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:33.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:33.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:33.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:33.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:33.181 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:33.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:33.282 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:33.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:33.310 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:33.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:33.369 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:33.411 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:33.438 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:33.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:33.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:33.540 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:33.568 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:33.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:33.670 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:33.698 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:33.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:33 smithi087 ceph-mon[89014]: pgmap v1887: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:33 smithi087 ceph-mon[89014]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:47:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:33 smithi087 ceph-mon[89014]: osdmap e1228: 8 total, 8 up, 7 in 2024-04-12T11:47:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:33 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3768354354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:47:33.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:33 smithi029 ceph-mon[93650]: pgmap v1887: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:33 smithi029 ceph-mon[93650]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:47:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:33 smithi029 ceph-mon[93650]: osdmap e1228: 8 total, 8 up, 7 in 2024-04-12T11:47:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:33 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3768354354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:47:33.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:33 smithi029 ceph-mon[97787]: pgmap v1887: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:33.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:33 smithi029 ceph-mon[97787]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:47:33.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:33 smithi029 ceph-mon[97787]: osdmap e1228: 8 total, 8 up, 7 in 2024-04-12T11:47:33.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:33 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3768354354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:47:33.801 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:33.830 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:33.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:33.931 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:33.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:33.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:34.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:34.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:34.086 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:34.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:34.187 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:34.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:34.213 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:34.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:34.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:34.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:34.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:34.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:34.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:34.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:34.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:34.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:34.573 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:34.598 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:34.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:34.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:34.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:34.700 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:34.727 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:34.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:34.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:34.828 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:34.853 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:34.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:34.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:34.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:34.960 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:47:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:47:34] "GET /metrics HTTP/1.1" 200 34173 "" "Prometheus/2.43.0" 2024-04-12T11:47:34.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:34.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:34.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:35.084 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:35.112 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:35.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:35.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:35.213 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:35.241 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:35.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:35.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:35.343 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:35.370 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:35.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:35.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:35.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:35.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:35.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:35.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:35.600 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:35.621 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:35 smithi029 ceph-mon[97787]: pgmap v1889: 89 pgs: 4 creating+peering, 4 unknown, 81 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:35.621 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:35 smithi029 ceph-mon[93650]: pgmap v1889: 89 pgs: 4 creating+peering, 4 unknown, 81 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:35.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:35.629 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:35.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:35.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:35 smithi087 ceph-mon[89014]: pgmap v1889: 89 pgs: 4 creating+peering, 4 unknown, 81 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:35.730 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:35.755 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:35.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:35.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:35.854 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:47:35.854 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:47:35.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:47:35.854 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:47:35.854 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.854 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:120703 8.f 8:fc4c3947:::benchmark_data_smithi029_194418_object14393:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1197)", 2024-04-12T11:47:35.855 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:43.064139+0000", 2024-04-12T11:47:35.855 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 172.78688949299999, 2024-04-12T11:47:35.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.18609684000000001, 2024-04-12T11:47:35.855 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.855 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.855 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.855 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:47:35.855 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:47:35.855 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:47:35.855 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 120703 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.064139+0000", 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.064139+0000", 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.064149+0000", 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0056e-05 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.064155+0000", 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8139999999999997e-06 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.064168+0000", 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3179e-05 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.064175+0000", 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5059999999999998e-06 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.064229+0000", 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3522e-05 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.064343+0000", 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011399400000000001 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.064441+0000", 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8183000000000005e-05 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.070278+0000", 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058367219999999999 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.070294+0000", 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5633000000000001e-05 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.076962+0000", 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006668495 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.250178+0000", 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.173216378 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.250236+0000", 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7358000000000002e-05 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:120708 8.f 8:fd9f18b8:::benchmark_data_smithi029_194418_object14398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1197)", 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:43.079154+0000", 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 172.77187465399999, 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.24260272199999999, 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.861 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 120708 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.079154+0000", 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.862 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.079154+0000", 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.079155+0000", 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2130000000000001e-06 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.079159+0000", 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5059999999999997e-06 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.250110+0000", 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17095099499999999 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.250130+0000", 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9953e-05 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.250271+0000", 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014032199999999999 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.250376+0000", 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105538 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.250454+0000", 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8195999999999995e-05 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.253604+0000", 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031496240000000002 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.254658+0000", 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001054052 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.321648+0000", 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.066989599999999996 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.321710+0000", 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2298000000000006e-05 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:43.321756+0000", 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6424999999999997e-05 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:130536 8.f 8:ffc3320d:::benchmark_data_smithi029_194418_object24226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:56.036392+0000", 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 159.814636495, 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19510713199999999, 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.867 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130536 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.036392+0000", 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.868 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.036392+0000", 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.036394+0000", 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6470000000000001e-06 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.036400+0000", 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9320000000000001e-06 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.036412+0000", 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2858e-05 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.036420+0000", 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7209999999999994e-06 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.036482+0000", 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1488000000000005e-05 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.036592+0000", 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110441 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.036701+0000", 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109411 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.046329+0000", 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0096276959999999998 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.046381+0000", 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2218e-05 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.231263+0000", 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.184881191 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.231452+0000", 2024-04-12T11:47:35.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018924299999999999 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.231499+0000", 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7286000000000001e-05 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:130538 8.f 8:f5061ae2:::benchmark_data_smithi029_194418_object24228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:56.040436+0000", 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 159.81059274099999, 2024-04-12T11:47:35.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.191424661, 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130538 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.040436+0000", 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.040436+0000", 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.040437+0000", 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4789999999999999e-06 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.040443+0000", 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0360000000000003e-06 2024-04-12T11:47:35.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.040456+0000", 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2689e-05 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.040464+0000", 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0709999999999995e-06 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.042345+0000", 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018807120000000001 2024-04-12T11:47:35.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.042467+0000", 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000121932 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.042591+0000", 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000124893 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.046368+0000", 2024-04-12T11:47:35.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037770059999999999 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.046389+0000", 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0154000000000001e-05 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.231740+0000", 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.18535116099999999 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.231819+0000", 2024-04-12T11:47:35.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8782000000000006e-05 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.231860+0000", 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1746000000000003e-05 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:131230 8.f 8:f3631c3a:::benchmark_data_smithi029_194418_object24920:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:56.922559+0000", 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.928469304, 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.234473344, 2024-04-12T11:47:35.879 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 131230 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.922559+0000", 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.881 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.922559+0000", 2024-04-12T11:47:35.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.881 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.881 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.881 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.922561+0000", 2024-04-12T11:47:35.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.804e-06 2024-04-12T11:47:35.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.881 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.881 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.881 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:35.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.922567+0000", 2024-04-12T11:47:35.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9830000000000001e-06 2024-04-12T11:47:35.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.922580+0000", 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3149000000000001e-05 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.922589+0000", 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6300000000000004e-06 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.924720+0000", 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021308199999999998 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.924844+0000", 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012495799999999999 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.925425+0000", 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00058051800000000003 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.928420+0000", 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029945900000000001 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.931439+0000", 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030191269999999999 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.156687+0000", 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.225248693 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.156980+0000", 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00029260499999999998 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.157032+0000", 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2466999999999998e-05 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:131236 8.f 8:f597c871:::benchmark_data_smithi029_194418_object24926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:56.945068+0000", 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.90595994099999, 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21608355100000001, 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.887 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 131236 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.945068+0000", 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.945068+0000", 2024-04-12T11:47:35.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.945069+0000", 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2099999999999995e-07 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.945073+0000", 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7620000000000001e-06 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.945083+0000", 2024-04-12T11:47:35.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0227000000000001e-05 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.945089+0000", 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9170000000000003e-06 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.945131+0000", 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1863000000000002e-05 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.945222+0000", 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0779999999999995e-05 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.945298+0000", 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6483999999999995e-05 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.143913+0000", 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19861420599999999 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:47:35.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.143983+0000", 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0526999999999997e-05 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.161037+0000", 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017054149000000001 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.161095+0000", 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8010000000000002e-05 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.161152+0000", 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6705e-05 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:131237 8.f 8:f96d7561:::benchmark_data_smithi029_194418_object24927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:56.947739+0000", 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.903289183, 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21358701299999999, 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:47:35.893 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 131237 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.947739+0000", 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.947739+0000", 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.947740+0000", 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6499999999999998e-07 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.947744+0000", 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1359999999999999e-06 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.947753+0000", 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3389999999999999e-06 2024-04-12T11:47:35.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.947760+0000", 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2890000000000003e-06 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.947879+0000", 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000119349 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.947957+0000", 2024-04-12T11:47:35.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8023999999999997e-05 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.948029+0000", 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1466999999999999e-05 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.143956+0000", 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.195927294 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.143993+0000", 2024-04-12T11:47:35.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6915000000000001e-05 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.161199+0000", 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017206009000000001 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.161284+0000", 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5147000000000003e-05 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.161326+0000", 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2279e-05 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:131236 8.f e1202/1181 ondisk, result = 0)", 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:56.952082+0000", 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.898946032, 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19183999900000001, 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.952082+0000", 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.952082+0000", 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.952090+0000", 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6229999999999997e-06 2024-04-12T11:47:35.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:56.952100+0000", 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6739999999999992e-06 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.143835+0000", 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.191735665 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.143847+0000", 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1406999999999999e-05 2024-04-12T11:47:35.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.143894+0000", 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7042000000000001e-05 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.143905+0000", 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1189000000000001e-05 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.143922+0000", 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7399000000000001e-05 2024-04-12T11:47:35.902 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:131771 8.f 8:f0bae133:::benchmark_data_smithi029_194418_object25461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:57.689719+0000", 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.161308972, 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33499846, 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:47:35.903 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 131771 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.689719+0000", 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.689719+0000", 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.689722+0000", 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0600000000000002e-06 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.689733+0000", 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1133e-05 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.689757+0000", 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4267e-05 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.689772+0000", 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4888999999999999e-05 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.689887+0000", 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115365 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.689999+0000", 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000112148 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.690076+0000", 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6847999999999993e-05 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.692941+0000", 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028650669999999998 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:57.694345+0000", 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001403301 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:58.024521+0000", 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33017678099999997 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:58.024671+0000", 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000149937 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:58.024718+0000", 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6663999999999999e-05 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.908 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:137722 8.b e1202/1185 8:d11e27d3:::benchmark_data_smithi029_194418_object31412:head v 1202'8826, mlcod=1202'8824)", 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:05.581638+0000", 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 150.269390891, 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19660349799999999, 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.581638+0000", 2024-04-12T11:47:35.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.581638+0000", 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.581638+0000", 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2399999999999998e-07 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.581642+0000", 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8820000000000003e-06 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.581650+0000", 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2360000000000004e-06 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.581655+0000", 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2279999999999998e-06 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.581698+0000", 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2942000000000003e-05 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.581711+0000", 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2996999999999999e-05 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.778189+0000", 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19647753000000001 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.778241+0000", 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2159e-05 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:137707 8.d e1202/1194 8:b8a03b9b:::benchmark_data_smithi029_194418_object31397:head v 1202'8698, mlcod=1179'980)", 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:05.584169+0000", 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 150.266859547, 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21116995199999999, 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.913 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.584169+0000", 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.584169+0000", 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.584169+0000", 2024-04-12T11:47:35.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9999999999999998e-07 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.584174+0000", 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3889999999999998e-06 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.777961+0000", 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.193787287 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.777984+0000", 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2466000000000001e-05 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.778087+0000", 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010357399999999999 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.778123+0000", 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5512999999999997e-05 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.795297+0000", 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017174188999999999 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:05.795339+0000", 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2033999999999998e-05 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:141654 8.a e1202/1172 8:5b6aa3b8:::benchmark_data_smithi029_194418_object35344:head v 1202'8878, mlcod=1202'8876)", 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:10.955939+0000", 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 144.89508963500001, 2024-04-12T11:47:35.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17924703, 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.955939+0000", 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.955939+0000", 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.955939+0000", 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7999999999999995e-07 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.955945+0000", 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2379999999999997e-06 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.955954+0000", 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2520000000000005e-06 2024-04-12T11:47:35.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.955960+0000", 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2570000000000004e-06 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956035+0000", 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5148000000000006e-05 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956049+0000", 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3939e-05 2024-04-12T11:47:35.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.135127+0000", 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17907828100000001 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.135186+0000", 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8335000000000003e-05 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(8.6 1202/1200 [PushOp(8:67f2cbb6:::benchmark_data_smithi029_194418_object35346:head, version: 1186'2199, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:67f2cbb6:::benchmark_data_smithi029_194418_object35346:head@1186'2199, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:10.956637+0000", 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 144.89439111900001, 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.18030041299999999, 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956637+0000", 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956637+0000", 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956638+0000", 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9400000000000005e-07 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956685+0000", 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6937999999999997e-05 2024-04-12T11:47:35.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956705+0000", 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9862999999999999e-05 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956717+0000", 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2585000000000001e-05 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956801+0000", 2024-04-12T11:47:35.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4153000000000006e-05 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956816+0000", 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4289e-05 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.136938+0000", 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.18012199100000001 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:141655 8.17 e1202/1199 8:ecce6f30:::benchmark_data_smithi029_194418_object35345:head v 1202'8076, mlcod=1202'8074)", 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:10.956791+0000", 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 144.89423714899999, 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.18457855400000001, 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956791+0000", 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956791+0000", 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956792+0000", 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3200000000000005e-07 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956796+0000", 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2749999999999997e-06 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956803+0000", 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.216e-06 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.956808+0000", 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5580000000000002e-06 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.133595+0000", 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17678744900000001 2024-04-12T11:47:35.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.133625+0000", 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9535000000000001e-05 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.141328+0000", 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077027229999999999 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.141370+0000", 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2265999999999998e-05 2024-04-12T11:47:35.929 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:141659 8.e e1202/1172 8:7056c44c:::benchmark_data_smithi029_194418_object35349:head v 1202'8534, mlcod=1202'8532)", 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:10.960011+0000", 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 144.891017758, 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.18045472700000001, 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.960011+0000", 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.960011+0000", 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.960011+0000", 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8100000000000002e-07 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.960017+0000", 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.575e-06 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.133450+0000", 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.173432587 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.133474+0000", 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4474999999999999e-05 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.133642+0000", 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016827199999999999 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.133666+0000", 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3266000000000001e-05 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.140421+0000", 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067555640000000004 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.140465+0000", 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4307000000000001e-05 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:141661 8.17 e1202/1199 8:eb4ee420:::benchmark_data_smithi029_194418_object35351:head v 1202'8077, mlcod=1202'8074)", 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:10.960966+0000", 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 144.89006276200001, 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.181376857, 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.934 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.960966+0000", 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.960966+0000", 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.960966+0000", 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.61e-07 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.960973+0000", 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3210000000000001e-06 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.133208+0000", 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17223571300000001 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.133228+0000", 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9729999999999999e-05 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.135216+0000", 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019882400000000001 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.135235+0000", 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8343999999999999e-05 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.142265+0000", 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0070303249999999996 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.142343+0000", 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7522999999999996e-05 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:141669 8.f 8:f80efd75:::benchmark_data_smithi029_194418_object35359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:47:35.938 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:10.973816+0000", 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 144.877212845, 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.185465396, 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 141669 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.973816+0000", 2024-04-12T11:47:35.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.973816+0000", 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.973817+0000", 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.499e-06 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:10.973824+0000", 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6630000000000004e-06 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.133442+0000", 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15961788099999999 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.133458+0000", 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6330000000000001e-05 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.137319+0000", 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003861499 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.137462+0000", 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014243599999999999 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.137606+0000", 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014415000000000001 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.142256+0000", 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046503029999999997 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.153600+0000", 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011343847000000001 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.158876+0000", 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052761880000000002 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.159231+0000", 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035495900000000001 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:11.159281+0000", 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9641000000000001e-05 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:165008 8.6 e1206/1200 8:67e67026:::benchmark_data_smithi029_194418_object58698:head v 1206'8839, mlcod=1191'3601)", 2024-04-12T11:47:35.944 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:45.961618+0000", 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 109.88941066300001, 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19456577799999999, 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.961618+0000", 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.961618+0000", 2024-04-12T11:47:35.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.961619+0000", 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2999999999999999e-07 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.961626+0000", 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0809999999999996e-06 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.961637+0000", 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0871e-05 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.961644+0000", 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1489999999999997e-06 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.962462+0000", 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00081794400000000003 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.962487+0000", 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4862e-05 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:46.156067+0000", 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.193580429 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:46.156184+0000", 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000116512 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(8.6 1206/1200 [PushOp(8:64f20bd6:::benchmark_data_smithi029_194418_object58724:head, version: 1191'3603, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:64f20bd6:::benchmark_data_smithi029_194418_object58724:head@1191'3603, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:45.964097+0000", 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 109.886930932, 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19192900900000001, 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.964097+0000", 2024-04-12T11:47:35.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.964097+0000", 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.964098+0000", 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2500000000000005e-07 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.964161+0000", 2024-04-12T11:47:35.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3305000000000005e-05 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.964179+0000", 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7983e-05 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.964195+0000", 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5197999999999999e-05 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.965421+0000", 2024-04-12T11:47:35.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012261349999999999 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.965433+0000", 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2289999999999999e-05 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:46.156026+0000", 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19059347300000001 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:165032 8.d e1206/1194 8:bcbe381f:::benchmark_data_smithi029_194418_object58722:head v 1206'10452, mlcod=1179'980)", 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:45.965894+0000", 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 109.885134497, 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.195351003, 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.965894+0000", 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.965894+0000", 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.965894+0000", 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2899999999999999e-07 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.965899+0000", 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4240000000000003e-06 2024-04-12T11:47:35.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.965905+0000", 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6939999999999998e-06 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.965910+0000", 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4220000000000002e-06 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.965948+0000", 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8504000000000003e-05 2024-04-12T11:47:35.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:45.965964+0000", 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5539000000000001e-05 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:46.161160+0000", 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.195195693 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:46.161245+0000", 2024-04-12T11:47:35.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5297999999999995e-05 2024-04-12T11:47:35.957 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.957 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.957 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.957 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:47:35.957 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:47:35.957 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:47:35.985 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:35.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:35.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:36.086 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:36.115 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:36.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:36.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:36.217 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:36.247 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:36.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:36.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:36.349 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:36.373 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:36.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:36.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:36.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:47:36.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:36.498 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:36.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:36.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:36.600 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:36.628 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:36.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:36.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:36.730 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:36.757 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:36.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:36.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:36.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:36.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:36.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:36.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:36.984 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:37.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:37.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:37.114 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:37.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:37.140 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:37.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:37.241 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:37.268 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:37.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:37.370 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:37.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:37.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:37.496 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:37.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:37 smithi029 ceph-mon[97787]: pgmap v1890: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:37.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:37.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:37 smithi029 ceph-mon[93650]: pgmap v1890: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:37.519 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:37.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:37.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:37.621 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:37.649 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:37.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:37 smithi087 ceph-mon[89014]: pgmap v1890: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:37.751 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:37.774 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:37.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:37.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:37.876 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:37.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:37.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:37.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:37.989 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:38.005 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:38.032 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:38.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:38.134 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:38.161 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:38.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:38.263 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:38.287 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:38.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:38.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:38.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:38.413 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:38.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:38.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:38.515 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:38.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:38.540 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:38.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:38.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:38.664 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:38.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:38.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:38.766 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:38.794 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:38.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:38.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:38.896 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:38.924 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:38.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:38.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:39.026 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:39.049 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:39.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:39.151 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:39.178 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:39.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:39.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:39.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:39.304 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:39.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:39.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:39.319 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-04-12T11:47:39.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:39.429 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:39.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:39.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:39.531 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:39.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-04-12T11:47:39.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:39.557 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:39.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:39.579 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:39 smithi029 ceph-mon[97787]: pgmap v1891: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:39.579 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:39 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/187689510' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:47:39.579 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:39 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:47:39.579 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:39 smithi029 ceph-mon[93650]: pgmap v1891: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:39.579 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:39 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/187689510' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:47:39.579 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:39 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:47:39.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:39.682 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:39.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:39.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:39 smithi087 ceph-mon[89014]: pgmap v1891: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:39 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/187689510' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:47:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:39 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:47:39.783 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:39.811 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:39.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:39.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:39.913 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:39.937 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:39.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:39.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:40.039 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:40.067 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:40.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:40.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:40.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:40.192 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:40.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:40.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:40.294 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:40.322 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:40.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:40.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:40.424 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:40.447 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:40.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:40.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:40.491 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:40 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:47:40.491 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:40 smithi029 ceph-mon[97787]: osdmap e1229: 8 total, 8 up, 7 in 2024-04-12T11:47:40.492 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:40 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:47:40.492 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:40 smithi029 ceph-mon[93650]: osdmap e1229: 8 total, 8 up, 7 in 2024-04-12T11:47:40.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:40.572 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:40.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:40.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:40.674 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:40.702 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:40.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:40.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:40 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:47:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:40 smithi087 ceph-mon[89014]: osdmap e1229: 8 total, 8 up, 7 in 2024-04-12T11:47:40.803 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:40.830 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:40.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:40.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:40.932 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:40.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:40.960 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:40.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:41.061 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:41.085 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:41.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:41.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:41.187 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:41.211 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:41.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:41.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:41.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:41.336 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:47:41.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:41.343 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:41.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:41.444 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:41.468 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:41.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:41.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:41.569 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:41.597 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:41.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:41.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:41.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:41 smithi087 ceph-mon[89014]: pgmap v1893: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:41 smithi087 ceph-mon[89014]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:47:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:41 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:47:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:41 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2006476630' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:47:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:41 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:47:41.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:41.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:41 smithi029 ceph-mon[97787]: pgmap v1893: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:41 smithi029 ceph-mon[97787]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:47:41.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:41 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:47:41.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:41 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2006476630' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:47:41.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:41 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:47:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:41 smithi029 ceph-mon[93650]: pgmap v1893: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2024-04-12T11:47:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:41 smithi029 ceph-mon[93650]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:47:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:41 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:47:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:41 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2006476630' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:47:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:41 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:47:41.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:41.853 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:41.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:41.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:41.954 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:41.978 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:41.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:41.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:42.080 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:42.107 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:42.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:42.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:42.209 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:42.233 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:42.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:42.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:42.359 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:42.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:42.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:42.461 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:42.477 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:42 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:47:42.477 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:42 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:47:42.477 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:42 smithi029 ceph-mon[97787]: osdmap e1230: 8 total, 8 up, 7 in 2024-04-12T11:47:42.477 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:42.477 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:42 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:47:42.477 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:42 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:47:42.477 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:42 smithi029 ceph-mon[93650]: osdmap e1230: 8 total, 8 up, 7 in 2024-04-12T11:47:42.477 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:42.489 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:42.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:42.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:42.590 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:42.617 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:42.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:47:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:47:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:47:42.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:42 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:47:42.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:42 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:47:42.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:42 smithi087 ceph-mon[89014]: osdmap e1230: 8 total, 8 up, 7 in 2024-04-12T11:47:42.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:42.719 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:42.746 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:42.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:42.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:42.848 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:42.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:42.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:42.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:42.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:43.005 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:43.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:43.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:43.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:43.134 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:43.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:43.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:43.236 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:43.263 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:43.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:43.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:43.365 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:43.390 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:43.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:43.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:43.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:43.517 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:43.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:43.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:43.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:43 smithi029 ceph-mon[93650]: pgmap v1895: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-04-12T11:47:43.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:43 smithi029 ceph-mon[97787]: pgmap v1895: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-04-12T11:47:43.619 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:43.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:43.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:43.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:43 smithi087 ceph-mon[89014]: pgmap v1895: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-04-12T11:47:43.746 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:43.775 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:43.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:43.877 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:43.905 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:43.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:43.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:44.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:44.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:44.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:44.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:44.133 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:44.160 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:44.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:44.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:44.176 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:47:44.196 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:47:44.215 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:47:44.234 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:47:44.254 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:47:44.262 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:44.273 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:47:44.289 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:44.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:44.293 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:47:44.312 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:47:44.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:47:44.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:47:44.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:47:44.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:47:44.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:47:44.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:47:44.314 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:47:44.314 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:47:44.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:44.417 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:44.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:44.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:44.453 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:44 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3260135855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:47:44.453 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:44 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3260135855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:47:44.520 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:44.548 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:44.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:44.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:44.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:44.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:44.675 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:44.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:44.688 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-04-12T11:47:44.689 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-04-12T11:47:44.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:47:44.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:44 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3260135855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:47:44.718 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-12T11:47:44.718 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-12T11:47:44.776 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:44.804 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:44.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:44.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:44.906 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:44.934 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:44.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:44.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:44.961 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2024-04-12T11:47:44.962 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-12T11:47:44.962 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_222585 2024-04-12T11:47:44.962 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:47:44.962 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2024-04-12T11:47:45.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:47:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:47:44] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2024-04-12T11:47:45.036 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:45.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:45.079 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:45.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:45.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:47:45.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:45.216 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:45.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:45.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:45.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:45.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:45.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:45.472 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:45.510 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:45.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:45.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:45 smithi029 ceph-mon[97787]: pgmap v1896: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-04-12T11:47:45.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:45 smithi029 ceph-mon[93650]: pgmap v1896: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-04-12T11:47:45.612 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:45.650 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:45.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:45.700 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:45 smithi087 ceph-mon[89014]: pgmap v1896: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-04-12T11:47:45.753 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:45.786 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:45.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:45.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:45.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:45.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:45.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:45.962 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3734 3718 232.317 232.375 0.00445412 0.00429122 2024-04-12T11:47:46.034 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:46.070 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:46.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:46.171 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:46.195 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:46.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:46.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:46.336 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:46.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:46.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:46.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:46.474 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:46.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:46.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:46.524 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:46.576 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:46.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:46.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:46.715 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:46.752 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:46.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:46.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:46.854 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:46.889 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:46.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:46.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:46.962 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 7139 7123 222.555 212.812 0.00214943 0.00448387 2024-04-12T11:47:46.992 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:46 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:46 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:47.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:46 smithi029 ceph-mon[97787]: pgmap v1897: 89 pgs: 89 active+clean; 4.8 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 544 KiB/s wr, 8 op/s 2024-04-12T11:47:47.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:46 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:47.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:46 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:47.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:46 smithi029 ceph-mon[93650]: pgmap v1897: 89 pgs: 89 active+clean; 4.8 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 544 KiB/s wr, 8 op/s 2024-04-12T11:47:47.028 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:47.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:47.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:47.131 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:47.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:47.168 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:47.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:47.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:46 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:47.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:46 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:47.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:46 smithi087 ceph-mon[89014]: pgmap v1897: 89 pgs: 89 active+clean; 4.8 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 544 KiB/s wr, 8 op/s 2024-04-12T11:47:47.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:47.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:47.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:47.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:47:47.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:47.437 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:47.469 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:47.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:47.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:47.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:47.606 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:47.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:47.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:47.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:47.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:47.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:47.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:47.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:47.881 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:47.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:47.962 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 10323 10307 214.698 199 0.00406421 0.00465075 2024-04-12T11:47:47.983 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:48.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:47:48.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:47:48.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:48.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:47:48.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:47:48.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:48.021 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:48.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:48.122 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:48.159 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:48.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:48.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:48.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:47:48.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:47:48.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:48.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:48.296 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:48.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:48.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:48.397 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:48.435 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:48.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:48.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:48.537 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:48.573 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:48.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:48.621 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:47:48.621 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:47:48.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:48.715 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:48.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:48.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:48.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:48.853 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:48.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:48.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:48 smithi029 ceph-mon[97787]: pgmap v1898: 89 pgs: 89 active+clean; 351 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s 2024-04-12T11:47:48.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:48 smithi029 ceph-mon[93650]: pgmap v1898: 89 pgs: 89 active+clean; 351 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s 2024-04-12T11:47:48.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:48.962 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 13394 13378 209.004 191.938 0.00814566 0.00477627 2024-04-12T11:47:48.994 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:48.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:49.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:49.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:49.134 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:49.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:49.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:48 smithi087 ceph-mon[89014]: pgmap v1898: 89 pgs: 89 active+clean; 351 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s 2024-04-12T11:47:49.235 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:49.272 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:49.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:49.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:49.374 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:49.411 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:49.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:49.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:49.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:49.548 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:49.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:49.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:49.650 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:49.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:49.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:49.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:49.826 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:49.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:49.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:49.928 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:49.962 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 16448 16432 205.375 190.875 0.00186072 0.00485829 2024-04-12T11:47:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:49.964 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:49.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:50.065 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:50.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:50.103 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:50.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:50.204 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:50.241 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:50.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:50.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:50.344 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:50.370 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:50.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:50.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:50.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:50.510 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:50.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:50.611 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:50.647 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:50.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:50.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:50.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:50.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:50.788 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:50.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:50.889 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:50.925 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:50.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:50.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:50.962 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 18923 18907 196.926 154.688 0.00342921 0.00507189 2024-04-12T11:47:51.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:51.069 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:51.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:51.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:51.171 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:51.195 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:47:51.208 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:51.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:51.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:51 smithi029 ceph-mon[93650]: pgmap v1899: 89 pgs: 89 active+clean; 527 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s 2024-04-12T11:47:51.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:51 smithi029 ceph-mon[97787]: pgmap v1899: 89 pgs: 89 active+clean; 527 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s 2024-04-12T11:47:51.310 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:51.360 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:51.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:51.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:51.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:51.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:51 smithi087 ceph-mon[89014]: pgmap v1899: 89 pgs: 89 active+clean; 527 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s 2024-04-12T11:47:51.503 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:51.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:51.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:51.641 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:51.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:51.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:51.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:51.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:51.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:51.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:51.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:51.917 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:51.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:51.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:51.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 21615 21599 192.827 168.25 0.00674026 0.00518033 2024-04-12T11:47:52.019 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:52.058 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:52.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:52.160 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:52.196 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:52.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:52.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:52.299 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:52.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:52.324 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:52.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:52.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:52.461 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:52.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:52.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:52.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:52.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:52.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:52.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:52.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:52.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:52.606 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:52.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:52.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:47:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:47:52.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:47:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:47:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:47:52.744 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:52.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:52.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:52.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:52.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:52.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:52.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 15 23919 23904 186.73 144.062 0.00796116 0.00535038 2024-04-12T11:47:52.984 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:53.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:53.022 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:53.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:53.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:53.146 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:53.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:53.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:53.248 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:53.288 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:53.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:53.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:53.390 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:53.427 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:53.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:53.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:53.552 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:53.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:53.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:53.654 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:53.692 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:53.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:53.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:53 smithi087 ceph-mon[89014]: pgmap v1900: 89 pgs: 89 active+clean; 823 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-12T11:47:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:53 smithi029 ceph-mon[97787]: pgmap v1900: 89 pgs: 89 active+clean; 823 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-12T11:47:53.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:53 smithi029 ceph-mon[93650]: pgmap v1900: 89 pgs: 89 active+clean; 823 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-12T11:47:53.794 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:53.833 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:53.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:53.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:53.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:53.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 25272 25256 175.37 84.5 0.00459767 0.0056972 2024-04-12T11:47:53.963 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:53.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:53.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:54.065 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:54.088 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:54.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:54.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:54.190 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:54.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:54.227 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:54.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:54.328 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:54.366 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:54.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:54.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:54.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:54.502 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:54.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:54.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:54.604 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:54.642 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:54.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:54.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:54.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:54.781 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:54.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:54.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:54.907 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:54.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:54.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 27216 27200 169.982 121.5 0.00670997 0.00587783 2024-04-12T11:47:55.008 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:55.045 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:55.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:55.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:55.180 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:55.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:55.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:47:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:47:54] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2024-04-12T11:47:55.282 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:55.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:55.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:55.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:55.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:55.456 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:55.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:55.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:55.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:55.575 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:55 smithi029 ceph-mon[97787]: pgmap v1901: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-04-12T11:47:55.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:55 smithi029 ceph-mon[93650]: pgmap v1901: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-04-12T11:47:55.594 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:55.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:55.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:55.696 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:55 smithi087 ceph-mon[89014]: pgmap v1901: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-04-12T11:47:55.733 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:55.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:55.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:55.835 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:55.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:55.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:55.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 29314 29298 166.449 131.125 0.00202293 0.00599798 2024-04-12T11:47:55.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:56.000 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:56.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:56.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:56.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:56.138 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:56.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:56.240 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:56.278 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:56.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:56.379 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:56.415 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:56.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:56.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:56.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:56.540 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:56.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:47:56.642 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:56.667 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:56.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:56.768 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:56.805 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:56.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:56.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:56.907 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:56.930 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:56.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:56.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:56.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 30786 30770 160.244 92 0.00170627 0.00618588 2024-04-12T11:47:57.032 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:47:57.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:57.056 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:57.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:47:57.157 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:57.196 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:57.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:57.298 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:57.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:57.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:57.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:57.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:57.461 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:57.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:57.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:57.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:57.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:57.600 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:57.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:57.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:57 smithi087 ceph-mon[89014]: pgmap v1902: 89 pgs: 89 active+clean; 1.4 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-04-12T11:47:57.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:57 smithi029 ceph-mon[97787]: pgmap v1902: 89 pgs: 89 active+clean; 1.4 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-04-12T11:47:57.738 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:57 smithi029 ceph-mon[93650]: pgmap v1902: 89 pgs: 89 active+clean; 1.4 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-04-12T11:47:57.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:57.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:57.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:57.841 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:57.865 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:57.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:57.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:57.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 32117 32101 154.315 83.1875 0.00202777 0.00637675 2024-04-12T11:47:57.967 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:57.991 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:57.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:57.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:58.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:58.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:58.126 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:58.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:58.227 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:58.264 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:58.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:58.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:58.366 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:58.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:58.404 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:58.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:58.505 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:47:58.528 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:58.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:58.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:47:58.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:47:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:58.679 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:58.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:47:58.780 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:58.819 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:58.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:58.921 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:58.944 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:58.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:58.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:58.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 33684 33668 150.288 97.9375 0.00245481 0.0066236 2024-04-12T11:47:59.045 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:47:59.084 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:59.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:47:59.185 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:59.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:59.226 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:59.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:59.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:59.365 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:59.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:59.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:59.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:47:59.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:59.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:59.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:47:59.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:47:59.614 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:59.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:59.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:47:59.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:47:59 smithi029 ceph-mon[93650]: pgmap v1903: 89 pgs: 89 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-04-12T11:47:59.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:47:59 smithi029 ceph-mon[97787]: pgmap v1903: 89 pgs: 89 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-04-12T11:47:59.716 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:47:59 smithi087 ceph-mon[89014]: pgmap v1903: 89 pgs: 89 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-04-12T11:47:59.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:59.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:59.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:59.841 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:47:59.865 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:47:59.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:59.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:47:59.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 34863 34847 145.181 73.6875 0.00197509 0.00676735 2024-04-12T11:47:59.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:47:59.991 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:47:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:47:59.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:00.093 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:00.120 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:00.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:00.222 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:00.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:00.262 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:00.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:00.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:00.400 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:00.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:00.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:00.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:00.537 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:00.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:00.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:00.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:00.671 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:00.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:00.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:00.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:00.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:00.802 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:00.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:00.903 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:00.937 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:00.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:00.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:00.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 36041 36025 140.708 73.625 0.010358 0.00710145 2024-04-12T11:48:01.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:01.077 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:01.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:01.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:01.179 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:01.212 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:01.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:01.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:01.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:01.338 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:01.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:01.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:01.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:01.469 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:01.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:01.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:01.570 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:01.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:01.609 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:01.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:01.710 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:01 smithi087 ceph-mon[89014]: pgmap v1904: 89 pgs: 89 active+clean; 1.9 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-04-12T11:48:01.733 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:01.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:01.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:01.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:01 smithi029 ceph-mon[97787]: pgmap v1904: 89 pgs: 89 active+clean; 1.9 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-04-12T11:48:01.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:01 smithi029 ceph-mon[93650]: pgmap v1904: 89 pgs: 89 active+clean; 1.9 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-04-12T11:48:01.836 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:01.874 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:01.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:01.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:01.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 37522 37506 137.876 92.5625 0.00566256 0.00724797 2024-04-12T11:48:01.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:02.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:02.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:02.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:02.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:02.143 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:02.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:02.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:02.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:02.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:02.278 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:02.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:02.379 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:02.415 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:02.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:02.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:02.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:02.543 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:02.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:02.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:02.644 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:02.668 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:02.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:02.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:48:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:48:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:48:02.769 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:02.804 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:02.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:02.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:02.906 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:02.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:02.930 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:02.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:02.963 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 39135 39119 135.816 100.812 0.00603459 0.00735762 2024-04-12T11:48:03.031 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:03.070 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:03.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:03.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:03.172 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:03.210 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:03.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:03.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:03.352 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:03.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:03.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:03.454 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:03.489 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:03.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:03.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:03.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:03 smithi029 ceph-mon[93650]: pgmap v1905: 89 pgs: 89 active+clean; 2.0 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-04-12T11:48:03.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:03 smithi029 ceph-mon[97787]: pgmap v1905: 89 pgs: 89 active+clean; 2.0 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-04-12T11:48:03.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:03.629 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:03.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:03.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:03 smithi087 ceph-mon[89014]: pgmap v1905: 89 pgs: 89 active+clean; 2.0 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-04-12T11:48:03.730 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:03.768 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:03.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:03.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:03.870 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:03.906 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:03.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:03.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:03.947 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-04-12T11:48:03.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 41037 41021 134.924 118.875 0.00629478 0.00740612 2024-04-12T11:48:04.008 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:04.031 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:04.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:04.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:04.132 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:04.156 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:04.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:04.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:04.257 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:04.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:04.295 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:04.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:04.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:04.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:04.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:04.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:04.523 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:04.559 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:04.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:04.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:04.661 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:04.698 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:04.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:04.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:04.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:04.833 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:04.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:04.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:04.935 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:04.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:48:04.963886+0000 min lat: 0.0012668 max lat: 0.449354 avg lat: 0.00754222 2024-04-12T11:48:04.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:48:04.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 42421 42405 132.502 86.5 0.00514427 0.00754222 2024-04-12T11:48:04.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:04.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:04.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:05.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:05.098 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:05.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:05.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:05.200 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:05.240 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:05.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:48:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:48:04] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2024-04-12T11:48:05.342 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:05.379 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:05.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:05.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:05.481 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:05.513 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:05.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:05.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:05.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:05.652 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:05.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:05.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:05 smithi087 ceph-mon[89014]: pgmap v1906: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-04-12T11:48:05.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:05 smithi029 ceph-mon[97787]: pgmap v1906: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-04-12T11:48:05.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:05 smithi029 ceph-mon[93650]: pgmap v1906: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-04-12T11:48:05.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:05.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:05.802 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:05.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:05.903 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:05.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:05.940 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:05.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:05.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 43902 43886 130.6 92.5625 0.00307799 0.00765198 2024-04-12T11:48:06.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:06.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:06.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:06.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:06.177 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:06.216 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:06.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:06.317 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:06.357 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:06.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:06.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:06.358 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:48:06.459 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:06.495 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:06.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:06.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:06.596 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:06.634 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:06.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:06.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2096516850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:48:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:06.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:06.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2096516850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:48:06.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:06.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2096516850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:48:06.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:06.774 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:06.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:06.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:06.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:06.913 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:06.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:06.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:06.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 45777 45761 129.99 117.188 0.00728204 0.00768764 2024-04-12T11:48:07.015 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:07.048 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:07.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:07.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:07.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:07.182 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:07.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:07.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:07.283 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:07.319 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:07.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:07.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:07.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:07.458 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:07.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:07.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:07.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:07.583 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:07.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:07.685 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:07.708 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:07.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:07.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:07 smithi087 ceph-mon[89014]: pgmap v1907: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-04-12T11:48:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:07 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:48:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:07 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2096516850' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:48:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:07 smithi087 ceph-mon[89014]: osdmap e1231: 8 total, 8 up, 7 in 2024-04-12T11:48:07.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:07 smithi029 ceph-mon[97787]: pgmap v1907: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-04-12T11:48:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:07 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:48:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:07 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2096516850' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:48:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:07 smithi029 ceph-mon[97787]: osdmap e1231: 8 total, 8 up, 7 in 2024-04-12T11:48:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:07 smithi029 ceph-mon[93650]: pgmap v1907: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-04-12T11:48:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:07 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:48:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:07 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2096516850' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:48:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:07 smithi029 ceph-mon[93650]: osdmap e1231: 8 total, 8 up, 7 in 2024-04-12T11:48:07.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:07.848 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:07.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:07.950 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:07.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 47247 47231 128.332 91.875 0.00822579 0.00778687 2024-04-12T11:48:07.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:07.984 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:07.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:08.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:08.120 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:08.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:08.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:08.222 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:08.247 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:08.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:08.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:08.349 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:08.372 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:08.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:08.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:08.474 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:08.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:08.511 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:08.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:08.612 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:08.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:08.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:08.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:08.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:08.781 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:08.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:08.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:08.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:08.919 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:08.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:08.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:08.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 48841 48825 127.136 99.625 0.00532169 0.00786025 2024-04-12T11:48:09.021 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:09.059 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:09.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:09.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:09.161 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:09.194 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:09.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:09.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:09.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:09.331 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:09.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:09.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:09.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:48:09.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:09.459 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:09.467 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:09.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:09.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:09.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:09 smithi029 ceph-mon[93650]: pgmap v1909: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T11:48:09.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:09 smithi029 ceph-mon[97787]: pgmap v1909: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T11:48:09.569 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:09.619 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:09.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:09 smithi087 ceph-mon[89014]: pgmap v1909: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-12T11:48:09.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:09.759 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:09.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:09.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:09.860 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:09.903 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:09.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:09.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:09.932 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:48:09.940 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:48:09.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 50455 50439 126.085 100.875 0.0129382 0.00792645 2024-04-12T11:48:10.004 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:10.044 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:10.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:10.125 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:48:10.146 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:10.162 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:48:10.185 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:10.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:10.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:10.195 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:48:10.236 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:48:10.272 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:48:10.287 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:10.312 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:48:10.324 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:10.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:10.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:10.350 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:48:10.389 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:48:10.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:48:10.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:48:10.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:48:10.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:48:10.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:48:10.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:48:10.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:48:10.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:48:10.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/833183683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:10.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/833183683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:10.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:10.466 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:10.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:10.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:10.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:10.597 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:10.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:10.698 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/833183683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:10.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:10.738 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:10.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:10.839 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:10.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:10.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:10.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:10.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 52123 52107 125.245 104.25 0.010881 0.00797924 2024-04-12T11:48:10.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:11.021 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:11.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:11.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:11.123 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:11.160 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:11.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:11.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:11.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:11.298 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:11.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:11.399 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:11.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:11.524 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:11.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:11.625 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:11.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:11.674 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:11.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:11.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:11 smithi029 ceph-mon[97787]: from='client.45338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:48:11.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:11 smithi029 ceph-mon[97787]: pgmap v1910: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-12T11:48:11.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:11 smithi029 ceph-mon[93650]: from='client.45338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:48:11.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:11 smithi029 ceph-mon[93650]: pgmap v1910: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-12T11:48:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:11 smithi087 ceph-mon[89014]: from='client.45338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:48:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:11 smithi087 ceph-mon[89014]: pgmap v1910: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-12T11:48:11.719 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1909,"stamp":"2024-04-12T11:48:07.951598+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2807699410,"num_objects":42857,"num_object_clones":0,"num_object_copies":128571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42857,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":43133,"num_write_kb":2743758,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2045426,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":43049,"ondisk_log_size":43049,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11737256,"kb_used_data":8192992,"kb_used_omap":158,"kb_used_meta":3543265,"kb_avail":644478808,"statfs":{"total":671965249536,"available":659946299392,"internally_reserved":0,"allocated":8389623808,"data_stored":8369140993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":162126,"internal_metadata":3628304050},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":850395136,"num_objects":12976,"num_object_clones":0,"num_object_copies":38928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12976,"num_write_kb":830464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002423"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1815,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333469+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:47:41.333469+0000","last_peered":"2024-04-12T11:47:41.333469+0000","last_clean":"2024-04-12T11:47:41.333469+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:47:41.333469+0000","last_undegraded":"2024-04-12T11:47:41.333469+0000","last_fullsized":"2024-04-12T11:47:41.333469+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":674,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754787+0000","last_change":"2024-04-12T11:46:10.406067+0000","last_active":"2024-04-12T11:47:41.754787+0000","last_peered":"2024-04-12T11:47:41.754787+0000","last_clean":"2024-04-12T11:47:41.754787+0000","last_became_active":"2024-04-12T11:46:10.405480+0000","last_became_peered":"2024-04-12T11:46:10.405480+0000","last_unstale":"2024-04-12T11:47:41.754787+0000","last_undegraded":"2024-04-12T11:47:41.754787+0000","last_fullsized":"2024-04-12T11:47:41.754787+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:54:55.306753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1893,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754015+0000","last_change":"2024-04-12T11:46:10.403816+0000","last_active":"2024-04-12T11:47:41.754015+0000","last_peered":"2024-04-12T11:47:41.754015+0000","last_clean":"2024-04-12T11:47:41.754015+0000","last_became_active":"2024-04-12T11:46:10.399712+0000","last_became_peered":"2024-04-12T11:46:10.399712+0000","last_unstale":"2024-04-12T11:47:41.754015+0000","last_undegraded":"2024-04-12T11:47:41.754015+0000","last_fullsized":"2024-04-12T11:47:41.754015+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:06:08.696544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1914,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754506+0000","last_change":"2024-04-12T11:46:10.395530+0000","last_active":"2024-04-12T11:47:41.754506+0000","last_peered":"2024-04-12T11:47:41.754506+0000","last_clean":"2024-04-12T11:47:41.754506+0000","last_became_active":"2024-04-12T11:46:10.395134+0000","last_became_peered":"2024-04-12T11:46:10.395134+0000","last_unstale":"2024-04-12T11:47:41.754506+0000","last_undegraded":"2024-04-12T11:47:41.754506+0000","last_fullsized":"2024-04-12T11:47:41.754506+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:16:32.508735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":641,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.753546+0000","last_change":"2024-04-12T11:46:10.391931+0000","last_active":"2024-04-12T11:47:41.753546+0000","last_peered":"2024-04-12T11:47:41.753546+0000","last_clean":"2024-04-12T11:47:41.753546+0000","last_became_active":"2024-04-12T11:46:10.391618+0000","last_became_peered":"2024-04-12T11:46:10.391618+0000","last_unstale":"2024-04-12T11:47:41.753546+0000","last_undegraded":"2024-04-12T11:47:41.753546+0000","last_fullsized":"2024-04-12T11:47:41.753546+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:05:47.259753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1179,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.541959+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:47:41.541959+0000","last_peered":"2024-04-12T11:47:41.541959+0000","last_clean":"2024-04-12T11:47:41.541959+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:47:41.541959+0000","last_undegraded":"2024-04-12T11:47:41.541959+0000","last_fullsized":"2024-04-12T11:47:41.541959+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1884,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754576+0000","last_change":"2024-04-12T11:46:10.397031+0000","last_active":"2024-04-12T11:47:41.754576+0000","last_peered":"2024-04-12T11:47:41.754576+0000","last_clean":"2024-04-12T11:47:41.754576+0000","last_became_active":"2024-04-12T11:46:10.396649+0000","last_became_peered":"2024-04-12T11:46:10.396649+0000","last_unstale":"2024-04-12T11:47:41.754576+0000","last_undegraded":"2024-04-12T11:47:41.754576+0000","last_fullsized":"2024-04-12T11:47:41.754576+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:06:45.598704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":625,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754153+0000","last_change":"2024-04-12T11:46:10.396045+0000","last_active":"2024-04-12T11:47:41.754153+0000","last_peered":"2024-04-12T11:47:41.754153+0000","last_clean":"2024-04-12T11:47:41.754153+0000","last_became_active":"2024-04-12T11:46:10.393256+0000","last_became_peered":"2024-04-12T11:46:10.393256+0000","last_unstale":"2024-04-12T11:47:41.754153+0000","last_undegraded":"2024-04-12T11:47:41.754153+0000","last_fullsized":"2024-04-12T11:47:41.754153+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:26:31.787589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1210,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-04-12T11:47:39.546817+0000","last_change":"2024-04-12T11:45:52.986340+0000","last_active":"2024-04-12T11:47:39.546817+0000","last_peered":"2024-04-12T11:47:39.546817+0000","last_clean":"2024-04-12T11:47:39.546817+0000","last_became_active":"2024-04-12T11:45:52.984190+0000","last_became_peered":"2024-04-12T11:45:52.984190+0000","last_unstale":"2024-04-12T11:47:39.546817+0000","last_undegraded":"2024-04-12T11:47:39.546817+0000","last_fullsized":"2024-04-12T11:47:39.546817+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:52:19.144894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":49,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754455+0000","last_change":"2024-04-12T11:46:10.398333+0000","last_active":"2024-04-12T11:47:41.754455+0000","last_peered":"2024-04-12T11:47:41.754455+0000","last_clean":"2024-04-12T11:47:41.754455+0000","last_became_active":"2024-04-12T11:46:10.398008+0000","last_became_peered":"2024-04-12T11:46:10.398008+0000","last_unstale":"2024-04-12T11:47:41.754455+0000","last_undegraded":"2024-04-12T11:47:41.754455+0000","last_fullsized":"2024-04-12T11:47:41.754455+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:02:35.332854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":53,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754741+0000","last_change":"2024-04-12T11:46:10.411220+0000","last_active":"2024-04-12T11:47:41.754741+0000","last_peered":"2024-04-12T11:47:41.754741+0000","last_clean":"2024-04-12T11:47:41.754741+0000","last_became_active":"2024-04-12T11:46:10.386123+0000","last_became_peered":"2024-04-12T11:46:10.386123+0000","last_unstale":"2024-04-12T11:47:41.754741+0000","last_undegraded":"2024-04-12T11:47:41.754741+0000","last_fullsized":"2024-04-12T11:47:41.754741+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:40:12.714391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1867,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754341+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:47:41.754341+0000","last_peered":"2024-04-12T11:47:41.754341+0000","last_clean":"2024-04-12T11:47:41.754341+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:47:41.754341+0000","last_undegraded":"2024-04-12T11:47:41.754341+0000","last_fullsized":"2024-04-12T11:47:41.754341+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1820,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.334327+0000","last_change":"2024-04-12T11:45:53.024626+0000","last_active":"2024-04-12T11:47:41.334327+0000","last_peered":"2024-04-12T11:47:41.334327+0000","last_clean":"2024-04-12T11:47:41.334327+0000","last_became_active":"2024-04-12T11:45:52.976276+0000","last_became_peered":"2024-04-12T11:45:52.976276+0000","last_unstale":"2024-04-12T11:47:41.334327+0000","last_undegraded":"2024-04-12T11:47:41.334327+0000","last_fullsized":"2024-04-12T11:47:41.334327+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:49.929210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1779,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333912+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:47:41.333912+0000","last_peered":"2024-04-12T11:47:41.333912+0000","last_clean":"2024-04-12T11:47:41.333912+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:47:41.333912+0000","last_undegraded":"2024-04-12T11:47:41.333912+0000","last_fullsized":"2024-04-12T11:47:41.333912+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":38,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754408+0000","last_change":"2024-04-12T11:45:53.035604+0000","last_active":"2024-04-12T11:47:41.754408+0000","last_peered":"2024-04-12T11:47:41.754408+0000","last_clean":"2024-04-12T11:47:41.754408+0000","last_became_active":"2024-04-12T11:45:52.984722+0000","last_became_peered":"2024-04-12T11:45:52.984722+0000","last_unstale":"2024-04-12T11:47:41.754408+0000","last_undegraded":"2024-04-12T11:47:41.754408+0000","last_fullsized":"2024-04-12T11:47:41.754408+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:30.674395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":631,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754352+0000","last_change":"2024-04-12T11:46:10.398603+0000","last_active":"2024-04-12T11:47:41.754352+0000","last_peered":"2024-04-12T11:47:41.754352+0000","last_clean":"2024-04-12T11:47:41.754352+0000","last_became_active":"2024-04-12T11:46:10.390834+0000","last_became_peered":"2024-04-12T11:46:10.390834+0000","last_unstale":"2024-04-12T11:47:41.754352+0000","last_undegraded":"2024-04-12T11:47:41.754352+0000","last_fullsized":"2024-04-12T11:47:41.754352+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:10:10.149137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1898,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.753752+0000","last_change":"2024-04-12T11:46:10.396013+0000","last_active":"2024-04-12T11:47:41.753752+0000","last_peered":"2024-04-12T11:47:41.753752+0000","last_clean":"2024-04-12T11:47:41.753752+0000","last_became_active":"2024-04-12T11:46:10.395714+0000","last_became_peered":"2024-04-12T11:46:10.395714+0000","last_unstale":"2024-04-12T11:47:41.753752+0000","last_undegraded":"2024-04-12T11:47:41.753752+0000","last_fullsized":"2024-04-12T11:47:41.753752+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:27:19.237889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1227,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-04-12T11:47:39.546902+0000","last_change":"2024-04-12T11:46:10.397701+0000","last_active":"2024-04-12T11:47:39.546902+0000","last_peered":"2024-04-12T11:47:39.546902+0000","last_clean":"2024-04-12T11:47:39.546902+0000","last_became_active":"2024-04-12T11:46:10.397162+0000","last_became_peered":"2024-04-12T11:46:10.397162+0000","last_unstale":"2024-04-12T11:47:39.546902+0000","last_undegraded":"2024-04-12T11:47:39.546902+0000","last_fullsized":"2024-04-12T11:47:39.546902+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:16:29.772172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":32,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754301+0000","last_change":"2024-04-12T11:45:53.006808+0000","last_active":"2024-04-12T11:47:41.754301+0000","last_peered":"2024-04-12T11:47:41.754301+0000","last_clean":"2024-04-12T11:47:41.754301+0000","last_became_active":"2024-04-12T11:45:53.006480+0000","last_became_peered":"2024-04-12T11:45:53.006480+0000","last_unstale":"2024-04-12T11:47:41.754301+0000","last_undegraded":"2024-04-12T11:47:41.754301+0000","last_fullsized":"2024-04-12T11:47:41.754301+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:47.685096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1898,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754559+0000","last_change":"2024-04-12T11:46:10.393712+0000","last_active":"2024-04-12T11:47:41.754559+0000","last_peered":"2024-04-12T11:47:41.754559+0000","last_clean":"2024-04-12T11:47:41.754559+0000","last_became_active":"2024-04-12T11:46:10.393359+0000","last_became_peered":"2024-04-12T11:46:10.393359+0000","last_unstale":"2024-04-12T11:47:41.754559+0000","last_undegraded":"2024-04-12T11:47:41.754559+0000","last_fullsized":"2024-04-12T11:47:41.754559+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:33:56.240139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1187,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:57.185107+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:47:57.185107+0000","last_peered":"2024-04-12T11:47:57.185107+0000","last_clean":"2024-04-12T11:47:57.185107+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:47:57.185107+0000","last_undegraded":"2024-04-12T11:47:57.185107+0000","last_fullsized":"2024-04-12T11:47:57.185107+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":49,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754753+0000","last_change":"2024-04-12T11:46:10.408581+0000","last_active":"2024-04-12T11:47:41.754753+0000","last_peered":"2024-04-12T11:47:41.754753+0000","last_clean":"2024-04-12T11:47:41.754753+0000","last_became_active":"2024-04-12T11:46:10.406940+0000","last_became_peered":"2024-04-12T11:46:10.406940+0000","last_unstale":"2024-04-12T11:47:41.754753+0000","last_undegraded":"2024-04-12T11:47:41.754753+0000","last_fullsized":"2024-04-12T11:47:41.754753+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:18:48.590948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"1230'2806","reported_seq":2824,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:06.033168+0000","last_change":"2024-04-12T11:47:31.337856+0000","last_active":"2024-04-12T11:48:06.033168+0000","last_peered":"2024-04-12T11:48:06.033168+0000","last_clean":"2024-04-12T11:48:06.033168+0000","last_became_active":"2024-04-12T11:47:31.337539+0000","last_became_peered":"2024-04-12T11:47:31.337539+0000","last_unstale":"2024-04-12T11:48:06.033168+0000","last_undegraded":"2024-04-12T11:48:06.033168+0000","last_fullsized":"2024-04-12T11:48:06.033168+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2806,"log_dups_size":0,"ondisk_log_size":2806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:55:13.678253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183894016,"num_objects":2806,"num_object_clones":0,"num_object_copies":8418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2806,"num_write_kb":179584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":36,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754801+0000","last_change":"2024-04-12T11:45:53.028640+0000","last_active":"2024-04-12T11:47:41.754801+0000","last_peered":"2024-04-12T11:47:41.754801+0000","last_clean":"2024-04-12T11:47:41.754801+0000","last_became_active":"2024-04-12T11:45:52.983099+0000","last_became_peered":"2024-04-12T11:45:52.983099+0000","last_unstale":"2024-04-12T11:47:41.754801+0000","last_undegraded":"2024-04-12T11:47:41.754801+0000","last_fullsized":"2024-04-12T11:47:41.754801+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:43:01.897014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1187,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.332693+0000","last_change":"2024-04-12T11:45:52.993160+0000","last_active":"2024-04-12T11:47:41.332693+0000","last_peered":"2024-04-12T11:47:41.332693+0000","last_clean":"2024-04-12T11:47:41.332693+0000","last_became_active":"2024-04-12T11:45:52.989310+0000","last_became_peered":"2024-04-12T11:45:52.989310+0000","last_unstale":"2024-04-12T11:47:41.332693+0000","last_undegraded":"2024-04-12T11:47:41.332693+0000","last_fullsized":"2024-04-12T11:47:41.332693+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:58:47.362374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1230'2658","reported_seq":2676,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:04.622671+0000","last_change":"2024-04-12T11:47:31.334339+0000","last_active":"2024-04-12T11:48:04.622671+0000","last_peered":"2024-04-12T11:48:04.622671+0000","last_clean":"2024-04-12T11:48:04.622671+0000","last_became_active":"2024-04-12T11:47:31.333922+0000","last_became_peered":"2024-04-12T11:47:31.333922+0000","last_unstale":"2024-04-12T11:48:04.622671+0000","last_undegraded":"2024-04-12T11:48:04.622671+0000","last_fullsized":"2024-04-12T11:48:04.622671+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2658,"log_dups_size":0,"ondisk_log_size":2658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:33:04.664441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174194688,"num_objects":2658,"num_object_clones":0,"num_object_copies":7974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2658,"num_write_kb":170112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":37,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-04-12T11:48:06.364123+0000","last_change":"2024-04-12T11:45:53.031759+0000","last_active":"2024-04-12T11:48:06.364123+0000","last_peered":"2024-04-12T11:48:06.364123+0000","last_clean":"2024-04-12T11:48:06.364123+0000","last_became_active":"2024-04-12T11:45:52.984545+0000","last_became_peered":"2024-04-12T11:45:52.984545+0000","last_unstale":"2024-04-12T11:48:06.364123+0000","last_undegraded":"2024-04-12T11:48:06.364123+0000","last_fullsized":"2024-04-12T11:48:06.364123+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:16:10.421992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1820,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333522+0000","last_change":"2024-04-12T11:46:10.400601+0000","last_active":"2024-04-12T11:47:41.333522+0000","last_peered":"2024-04-12T11:47:41.333522+0000","last_clean":"2024-04-12T11:47:41.333522+0000","last_became_active":"2024-04-12T11:46:10.400255+0000","last_became_peered":"2024-04-12T11:46:10.400255+0000","last_unstale":"2024-04-12T11:47:41.333522+0000","last_undegraded":"2024-04-12T11:47:41.333522+0000","last_fullsized":"2024-04-12T11:47:41.333522+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:03:28.857231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1230'2446","reported_seq":2458,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:02.943446+0000","last_change":"2024-04-12T11:47:31.345676+0000","last_active":"2024-04-12T11:48:02.943446+0000","last_peered":"2024-04-12T11:48:02.943446+0000","last_clean":"2024-04-12T11:48:02.943446+0000","last_became_active":"2024-04-12T11:47:31.345389+0000","last_became_peered":"2024-04-12T11:47:31.345389+0000","last_unstale":"2024-04-12T11:48:02.943446+0000","last_undegraded":"2024-04-12T11:48:02.943446+0000","last_fullsized":"2024-04-12T11:48:02.943446+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2446,"log_dups_size":0,"ondisk_log_size":2446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:47:18.346324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160301056,"num_objects":2446,"num_object_clones":0,"num_object_copies":7338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2446,"num_write_kb":156544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":37,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-04-12T11:48:06.363982+0000","last_change":"2024-04-12T11:45:53.169686+0000","last_active":"2024-04-12T11:48:06.363982+0000","last_peered":"2024-04-12T11:48:06.363982+0000","last_clean":"2024-04-12T11:48:06.363982+0000","last_became_active":"2024-04-12T11:45:53.021021+0000","last_became_peered":"2024-04-12T11:45:53.021021+0000","last_unstale":"2024-04-12T11:48:06.363982+0000","last_undegraded":"2024-04-12T11:48:06.363982+0000","last_fullsized":"2024-04-12T11:48:06.363982+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:48:23.506812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1206,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:47.179655+0000","last_change":"2024-04-12T11:45:52.979601+0000","last_active":"2024-04-12T11:47:47.179655+0000","last_peered":"2024-04-12T11:47:47.179655+0000","last_clean":"2024-04-12T11:47:47.179655+0000","last_became_active":"2024-04-12T11:45:52.979307+0000","last_became_peered":"2024-04-12T11:45:52.979307+0000","last_unstale":"2024-04-12T11:47:47.179655+0000","last_undegraded":"2024-04-12T11:47:47.179655+0000","last_fullsized":"2024-04-12T11:47:47.179655+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:21.367107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1230'2573","reported_seq":2586,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:02.944430+0000","last_change":"2024-04-12T11:47:31.345659+0000","last_active":"2024-04-12T11:48:02.944430+0000","last_peered":"2024-04-12T11:48:02.944430+0000","last_clean":"2024-04-12T11:48:02.944430+0000","last_became_active":"2024-04-12T11:47:31.345313+0000","last_became_peered":"2024-04-12T11:47:31.345313+0000","last_unstale":"2024-04-12T11:48:02.944430+0000","last_undegraded":"2024-04-12T11:48:02.944430+0000","last_fullsized":"2024-04-12T11:48:02.944430+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2573,"log_dups_size":0,"ondisk_log_size":2573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:03:10.526704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168624128,"num_objects":2573,"num_object_clones":0,"num_object_copies":7719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2573,"num_write_kb":164672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1211,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-04-12T11:47:39.546850+0000","last_change":"2024-04-12T11:46:10.399997+0000","last_active":"2024-04-12T11:47:39.546850+0000","last_peered":"2024-04-12T11:47:39.546850+0000","last_clean":"2024-04-12T11:47:39.546850+0000","last_became_active":"2024-04-12T11:46:10.399692+0000","last_became_peered":"2024-04-12T11:46:10.399692+0000","last_unstale":"2024-04-12T11:47:39.546850+0000","last_undegraded":"2024-04-12T11:47:39.546850+0000","last_fullsized":"2024-04-12T11:47:39.546850+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:44.268992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1878,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754068+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:47:41.754068+0000","last_peered":"2024-04-12T11:47:41.754068+0000","last_clean":"2024-04-12T11:47:41.754068+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:47:41.754068+0000","last_undegraded":"2024-04-12T11:47:41.754068+0000","last_fullsized":"2024-04-12T11:47:41.754068+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"1231'2886","reported_seq":2900,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-04-12T11:48:07.173259+0000","last_change":"2024-04-12T11:47:31.340063+0000","last_active":"2024-04-12T11:48:07.173259+0000","last_peered":"2024-04-12T11:48:07.173259+0000","last_clean":"2024-04-12T11:48:07.173259+0000","last_became_active":"2024-04-12T11:47:31.339760+0000","last_became_peered":"2024-04-12T11:47:31.339760+0000","last_unstale":"2024-04-12T11:48:07.173259+0000","last_undegraded":"2024-04-12T11:48:07.173259+0000","last_fullsized":"2024-04-12T11:48:07.173259+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2886,"log_dups_size":0,"ondisk_log_size":2886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:26:14.945517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189136896,"num_objects":2886,"num_object_clones":0,"num_object_copies":8658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2886,"num_write_kb":184704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":148,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754076+0000","last_change":"2024-04-12T11:46:10.418286+0000","last_active":"2024-04-12T11:47:41.754076+0000","last_peered":"2024-04-12T11:47:41.754076+0000","last_clean":"2024-04-12T11:47:41.754076+0000","last_became_active":"2024-04-12T11:46:10.399488+0000","last_became_peered":"2024-04-12T11:46:10.399488+0000","last_unstale":"2024-04-12T11:47:41.754076+0000","last_undegraded":"2024-04-12T11:47:41.754076+0000","last_fullsized":"2024-04-12T11:47:41.754076+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:38:14.150930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1810,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.334235+0000","last_change":"2024-04-12T11:46:10.395742+0000","last_active":"2024-04-12T11:47:41.334235+0000","last_peered":"2024-04-12T11:47:41.334235+0000","last_clean":"2024-04-12T11:47:41.334235+0000","last_became_active":"2024-04-12T11:46:10.395362+0000","last_became_peered":"2024-04-12T11:46:10.395362+0000","last_unstale":"2024-04-12T11:47:41.334235+0000","last_undegraded":"2024-04-12T11:47:41.334235+0000","last_fullsized":"2024-04-12T11:47:41.334235+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:56:49.815229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1230'2570","reported_seq":2588,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:04.621507+0000","last_change":"2024-04-12T11:47:31.345847+0000","last_active":"2024-04-12T11:48:04.621507+0000","last_peered":"2024-04-12T11:48:04.621507+0000","last_clean":"2024-04-12T11:48:04.621507+0000","last_became_active":"2024-04-12T11:47:31.345534+0000","last_became_peered":"2024-04-12T11:47:31.345534+0000","last_unstale":"2024-04-12T11:48:04.621507+0000","last_undegraded":"2024-04-12T11:48:04.621507+0000","last_fullsized":"2024-04-12T11:48:04.621507+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2570,"log_dups_size":0,"ondisk_log_size":2570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:30:47.116045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168427520,"num_objects":2570,"num_object_clones":0,"num_object_copies":7710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2570,"num_write_kb":164480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":819,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333817+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:47:41.333817+0000","last_peered":"2024-04-12T11:47:41.333817+0000","last_clean":"2024-04-12T11:47:41.333817+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:47:41.333817+0000","last_undegraded":"2024-04-12T11:47:41.333817+0000","last_fullsized":"2024-04-12T11:47:41.333817+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1814,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333776+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:47:41.333776+0000","last_peered":"2024-04-12T11:47:41.333776+0000","last_clean":"2024-04-12T11:47:41.333776+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:47:41.333776+0000","last_undegraded":"2024-04-12T11:47:41.333776+0000","last_fullsized":"2024-04-12T11:47:41.333776+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1231'2892","reported_seq":2904,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-04-12T11:48:07.192158+0000","last_change":"2024-04-12T11:47:31.340770+0000","last_active":"2024-04-12T11:48:07.192158+0000","last_peered":"2024-04-12T11:48:07.192158+0000","last_clean":"2024-04-12T11:48:07.192158+0000","last_became_active":"2024-04-12T11:47:31.340454+0000","last_became_peered":"2024-04-12T11:47:31.340454+0000","last_unstale":"2024-04-12T11:48:07.192158+0000","last_undegraded":"2024-04-12T11:48:07.192158+0000","last_fullsized":"2024-04-12T11:48:07.192158+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2892,"log_dups_size":0,"ondisk_log_size":2892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:38:55.000492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189530112,"num_objects":2892,"num_object_clones":0,"num_object_copies":8676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2892,"num_write_kb":185088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1813,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333631+0000","last_change":"2024-04-12T11:45:52.976139+0000","last_active":"2024-04-12T11:47:41.333631+0000","last_peered":"2024-04-12T11:47:41.333631+0000","last_clean":"2024-04-12T11:47:41.333631+0000","last_became_active":"2024-04-12T11:45:52.975803+0000","last_became_peered":"2024-04-12T11:45:52.975803+0000","last_unstale":"2024-04-12T11:47:41.333631+0000","last_undegraded":"2024-04-12T11:47:41.333631+0000","last_fullsized":"2024-04-12T11:47:41.333631+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:46.618143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1779,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.334692+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:47:41.334692+0000","last_peered":"2024-04-12T11:47:41.334692+0000","last_clean":"2024-04-12T11:47:41.334692+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:47:41.334692+0000","last_undegraded":"2024-04-12T11:47:41.334692+0000","last_fullsized":"2024-04-12T11:47:41.334692+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1230'2553","reported_seq":2571,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:04.609873+0000","last_change":"2024-04-12T11:47:31.345891+0000","last_active":"2024-04-12T11:48:04.609873+0000","last_peered":"2024-04-12T11:48:04.609873+0000","last_clean":"2024-04-12T11:48:04.609873+0000","last_became_active":"2024-04-12T11:47:31.345543+0000","last_became_peered":"2024-04-12T11:47:31.345543+0000","last_unstale":"2024-04-12T11:48:04.609873+0000","last_undegraded":"2024-04-12T11:48:04.609873+0000","last_fullsized":"2024-04-12T11:48:04.609873+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2553,"log_dups_size":0,"ondisk_log_size":2553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:50:02.041176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167313408,"num_objects":2553,"num_object_clones":0,"num_object_copies":7659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2553,"num_write_kb":163392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":633,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.753867+0000","last_change":"2024-04-12T11:46:10.398575+0000","last_active":"2024-04-12T11:47:41.753867+0000","last_peered":"2024-04-12T11:47:41.753867+0000","last_clean":"2024-04-12T11:47:41.753867+0000","last_became_active":"2024-04-12T11:46:10.395695+0000","last_became_peered":"2024-04-12T11:46:10.395695+0000","last_unstale":"2024-04-12T11:47:41.753867+0000","last_undegraded":"2024-04-12T11:47:41.753867+0000","last_fullsized":"2024-04-12T11:47:41.753867+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:49:42.768675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1900,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.755000+0000","last_change":"2024-04-12T11:46:10.437691+0000","last_active":"2024-04-12T11:47:41.755000+0000","last_peered":"2024-04-12T11:47:41.755000+0000","last_clean":"2024-04-12T11:47:41.755000+0000","last_became_active":"2024-04-12T11:46:10.404629+0000","last_became_peered":"2024-04-12T11:46:10.404629+0000","last_unstale":"2024-04-12T11:47:41.755000+0000","last_undegraded":"2024-04-12T11:47:41.755000+0000","last_fullsized":"2024-04-12T11:47:41.755000+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:00:29.330962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"1230'2415","reported_seq":2428,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:02.854950+0000","last_change":"2024-04-12T11:47:31.345727+0000","last_active":"2024-04-12T11:48:02.854950+0000","last_peered":"2024-04-12T11:48:02.854950+0000","last_clean":"2024-04-12T11:48:02.854950+0000","last_became_active":"2024-04-12T11:47:31.345433+0000","last_became_peered":"2024-04-12T11:47:31.345433+0000","last_unstale":"2024-04-12T11:48:02.854950+0000","last_undegraded":"2024-04-12T11:48:02.854950+0000","last_fullsized":"2024-04-12T11:48:02.854950+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2415,"log_dups_size":0,"ondisk_log_size":2415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:51:31.021628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158269440,"num_objects":2415,"num_object_clones":0,"num_object_copies":7245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2415,"num_write_kb":154560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1890,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754594+0000","last_change":"2024-04-12T11:45:52.980673+0000","last_active":"2024-04-12T11:47:41.754594+0000","last_peered":"2024-04-12T11:47:41.754594+0000","last_clean":"2024-04-12T11:47:41.754594+0000","last_became_active":"2024-04-12T11:45:52.978978+0000","last_became_peered":"2024-04-12T11:45:52.978978+0000","last_unstale":"2024-04-12T11:47:41.754594+0000","last_undegraded":"2024-04-12T11:47:41.754594+0000","last_fullsized":"2024-04-12T11:47:41.754594+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:32:10.303905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1227,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-04-12T11:47:39.546908+0000","last_change":"2024-04-12T11:46:10.397490+0000","last_active":"2024-04-12T11:47:39.546908+0000","last_peered":"2024-04-12T11:47:39.546908+0000","last_clean":"2024-04-12T11:47:39.546908+0000","last_became_active":"2024-04-12T11:46:10.396703+0000","last_became_peered":"2024-04-12T11:46:10.396703+0000","last_unstale":"2024-04-12T11:47:39.546908+0000","last_undegraded":"2024-04-12T11:47:39.546908+0000","last_fullsized":"2024-04-12T11:47:39.546908+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:35:28.943460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1869,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754209+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:47:41.754209+0000","last_peered":"2024-04-12T11:47:41.754209+0000","last_clean":"2024-04-12T11:47:41.754209+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:47:41.754209+0000","last_undegraded":"2024-04-12T11:47:41.754209+0000","last_fullsized":"2024-04-12T11:47:41.754209+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"1230'2695","reported_seq":2713,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:06.005642+0000","last_change":"2024-04-12T11:47:31.337684+0000","last_active":"2024-04-12T11:48:06.005642+0000","last_peered":"2024-04-12T11:48:06.005642+0000","last_clean":"2024-04-12T11:48:06.005642+0000","last_became_active":"2024-04-12T11:47:31.337391+0000","last_became_peered":"2024-04-12T11:47:31.337391+0000","last_unstale":"2024-04-12T11:48:06.005642+0000","last_undegraded":"2024-04-12T11:48:06.005642+0000","last_fullsized":"2024-04-12T11:48:06.005642+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2695,"log_dups_size":0,"ondisk_log_size":2695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:11:18.213572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176619520,"num_objects":2695,"num_object_clones":0,"num_object_copies":8085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2695,"num_write_kb":172480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1190,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-04-12T11:47:39.546518+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:47:39.546518+0000","last_peered":"2024-04-12T11:47:39.546518+0000","last_clean":"2024-04-12T11:47:39.546518+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:47:39.546518+0000","last_undegraded":"2024-04-12T11:47:39.546518+0000","last_fullsized":"2024-04-12T11:47:39.546518+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1801,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333778+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:47:41.333778+0000","last_peered":"2024-04-12T11:47:41.333778+0000","last_clean":"2024-04-12T11:47:41.333778+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:47:41.333778+0000","last_undegraded":"2024-04-12T11:47:41.333778+0000","last_fullsized":"2024-04-12T11:47:41.333778+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":107,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.753772+0000","last_change":"2024-04-12T11:46:10.393257+0000","last_active":"2024-04-12T11:47:41.753772+0000","last_peered":"2024-04-12T11:47:41.753772+0000","last_clean":"2024-04-12T11:47:41.753772+0000","last_became_active":"2024-04-12T11:46:10.392748+0000","last_became_peered":"2024-04-12T11:46:10.392748+0000","last_unstale":"2024-04-12T11:47:41.753772+0000","last_undegraded":"2024-04-12T11:47:41.753772+0000","last_fullsized":"2024-04-12T11:47:41.753772+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:31:43.963814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"1230'2413","reported_seq":2426,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:02.869679+0000","last_change":"2024-04-12T11:47:31.341656+0000","last_active":"2024-04-12T11:48:02.869679+0000","last_peered":"2024-04-12T11:48:02.869679+0000","last_clean":"2024-04-12T11:48:02.869679+0000","last_became_active":"2024-04-12T11:47:31.341335+0000","last_became_peered":"2024-04-12T11:47:31.341335+0000","last_unstale":"2024-04-12T11:48:02.869679+0000","last_undegraded":"2024-04-12T11:48:02.869679+0000","last_fullsized":"2024-04-12T11:48:02.869679+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2413,"log_dups_size":0,"ondisk_log_size":2413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:16:26.010127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158138368,"num_objects":2413,"num_object_clones":0,"num_object_copies":7239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2413,"num_write_kb":154432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1860,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754646+0000","last_change":"2024-04-12T11:46:10.400420+0000","last_active":"2024-04-12T11:47:41.754646+0000","last_peered":"2024-04-12T11:47:41.754646+0000","last_clean":"2024-04-12T11:47:41.754646+0000","last_became_active":"2024-04-12T11:46:10.400096+0000","last_became_peered":"2024-04-12T11:46:10.400096+0000","last_unstale":"2024-04-12T11:47:41.754646+0000","last_undegraded":"2024-04-12T11:47:41.754646+0000","last_fullsized":"2024-04-12T11:47:41.754646+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:43:39.116532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1869,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.753762+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:47:41.753762+0000","last_peered":"2024-04-12T11:47:41.753762+0000","last_clean":"2024-04-12T11:47:41.753762+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:47:41.753762+0000","last_undegraded":"2024-04-12T11:47:41.753762+0000","last_fullsized":"2024-04-12T11:47:41.753762+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1871,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754911+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:47:41.754911+0000","last_peered":"2024-04-12T11:47:41.754911+0000","last_clean":"2024-04-12T11:47:41.754911+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:47:41.754911+0000","last_undegraded":"2024-04-12T11:47:41.754911+0000","last_fullsized":"2024-04-12T11:47:41.754911+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1230'2797","reported_seq":2815,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:06.034968+0000","last_change":"2024-04-12T11:47:31.337893+0000","last_active":"2024-04-12T11:48:06.034968+0000","last_peered":"2024-04-12T11:48:06.034968+0000","last_clean":"2024-04-12T11:48:06.034968+0000","last_became_active":"2024-04-12T11:47:31.337115+0000","last_became_peered":"2024-04-12T11:47:31.337115+0000","last_unstale":"2024-04-12T11:48:06.034968+0000","last_undegraded":"2024-04-12T11:48:06.034968+0000","last_fullsized":"2024-04-12T11:48:06.034968+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2797,"log_dups_size":0,"ondisk_log_size":2797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:48:50.148294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183304192,"num_objects":2797,"num_object_clones":0,"num_object_copies":8391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2797,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1217,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-04-12T11:47:39.546707+0000","last_change":"2024-04-12T11:45:52.986504+0000","last_active":"2024-04-12T11:47:39.546707+0000","last_peered":"2024-04-12T11:47:39.546707+0000","last_clean":"2024-04-12T11:47:39.546707+0000","last_became_active":"2024-04-12T11:45:52.984346+0000","last_became_peered":"2024-04-12T11:45:52.984346+0000","last_unstale":"2024-04-12T11:47:39.546707+0000","last_undegraded":"2024-04-12T11:47:39.546707+0000","last_fullsized":"2024-04-12T11:47:39.546707+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:26.859024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1538,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:02.196811+0000","last_change":"2024-04-12T11:45:53.970732+0000","last_active":"2024-04-12T11:48:02.196811+0000","last_peered":"2024-04-12T11:48:02.196811+0000","last_clean":"2024-04-12T11:48:02.196811+0000","last_became_active":"2024-04-12T11:45:53.970305+0000","last_became_peered":"2024-04-12T11:45:53.970305+0000","last_unstale":"2024-04-12T11:48:02.196811+0000","last_undegraded":"2024-04-12T11:48:02.196811+0000","last_fullsized":"2024-04-12T11:48:02.196811+0000","mapping_epoch":1210,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:42:56.998830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":554,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-04-12T11:47:39.546618+0000","last_change":"2024-04-12T11:46:10.396169+0000","last_active":"2024-04-12T11:47:39.546618+0000","last_peered":"2024-04-12T11:47:39.546618+0000","last_clean":"2024-04-12T11:47:39.546618+0000","last_became_active":"2024-04-12T11:46:10.395392+0000","last_became_peered":"2024-04-12T11:46:10.395392+0000","last_unstale":"2024-04-12T11:47:39.546618+0000","last_undegraded":"2024-04-12T11:47:39.546618+0000","last_fullsized":"2024-04-12T11:47:39.546618+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:19:09.329529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1197,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.541865+0000","last_change":"2024-04-12T11:45:53.016616+0000","last_active":"2024-04-12T11:47:41.541865+0000","last_peered":"2024-04-12T11:47:41.541865+0000","last_clean":"2024-04-12T11:47:41.541865+0000","last_became_active":"2024-04-12T11:45:53.016292+0000","last_became_peered":"2024-04-12T11:45:53.016292+0000","last_unstale":"2024-04-12T11:47:41.541865+0000","last_undegraded":"2024-04-12T11:47:41.541865+0000","last_fullsized":"2024-04-12T11:47:41.541865+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:53:29.382293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1230'2675","reported_seq":2693,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:06.023894+0000","last_change":"2024-04-12T11:47:31.337491+0000","last_active":"2024-04-12T11:48:06.023894+0000","last_peered":"2024-04-12T11:48:06.023894+0000","last_clean":"2024-04-12T11:48:06.023894+0000","last_became_active":"2024-04-12T11:47:31.337137+0000","last_became_peered":"2024-04-12T11:47:31.337137+0000","last_unstale":"2024-04-12T11:48:06.023894+0000","last_undegraded":"2024-04-12T11:48:06.023894+0000","last_fullsized":"2024-04-12T11:48:06.023894+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2675,"log_dups_size":0,"ondisk_log_size":2675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:21:25.556372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175308800,"num_objects":2675,"num_object_clones":0,"num_object_copies":8025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2675,"num_write_kb":171200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":33,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:57.185366+0000","last_change":"2024-04-12T11:45:52.987902+0000","last_active":"2024-04-12T11:47:57.185366+0000","last_peered":"2024-04-12T11:47:57.185366+0000","last_clean":"2024-04-12T11:47:57.185366+0000","last_became_active":"2024-04-12T11:45:52.980281+0000","last_became_peered":"2024-04-12T11:45:52.980281+0000","last_unstale":"2024-04-12T11:47:57.185366+0000","last_undegraded":"2024-04-12T11:47:57.185366+0000","last_fullsized":"2024-04-12T11:47:57.185366+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:07:12.203048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1897,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.753850+0000","last_change":"2024-04-12T11:46:10.386160+0000","last_active":"2024-04-12T11:47:41.753850+0000","last_peered":"2024-04-12T11:47:41.753850+0000","last_clean":"2024-04-12T11:47:41.753850+0000","last_became_active":"2024-04-12T11:46:10.385225+0000","last_became_peered":"2024-04-12T11:46:10.385225+0000","last_unstale":"2024-04-12T11:47:41.753850+0000","last_undegraded":"2024-04-12T11:47:41.753850+0000","last_fullsized":"2024-04-12T11:47:41.753850+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:56:58.350984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1879,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754634+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:47:41.754634+0000","last_peered":"2024-04-12T11:47:41.754634+0000","last_clean":"2024-04-12T11:47:41.754634+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:47:41.754634+0000","last_undegraded":"2024-04-12T11:47:41.754634+0000","last_fullsized":"2024-04-12T11:47:41.754634+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"1231'2906","reported_seq":2920,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-04-12T11:48:07.182461+0000","last_change":"2024-04-12T11:47:31.338497+0000","last_active":"2024-04-12T11:48:07.182461+0000","last_peered":"2024-04-12T11:48:07.182461+0000","last_clean":"2024-04-12T11:48:07.182461+0000","last_became_active":"2024-04-12T11:47:31.338160+0000","last_became_peered":"2024-04-12T11:47:31.338160+0000","last_unstale":"2024-04-12T11:48:07.182461+0000","last_undegraded":"2024-04-12T11:48:07.182461+0000","last_fullsized":"2024-04-12T11:48:07.182461+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2906,"log_dups_size":0,"ondisk_log_size":2906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:22:37.585891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190447616,"num_objects":2906,"num_object_clones":0,"num_object_copies":8718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2906,"num_write_kb":185984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":33,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:02.197165+0000","last_change":"2024-04-12T11:45:53.005585+0000","last_active":"2024-04-12T11:48:02.197165+0000","last_peered":"2024-04-12T11:48:02.197165+0000","last_clean":"2024-04-12T11:48:02.197165+0000","last_became_active":"2024-04-12T11:45:52.987880+0000","last_became_peered":"2024-04-12T11:45:52.987880+0000","last_unstale":"2024-04-12T11:48:02.197165+0000","last_undegraded":"2024-04-12T11:48:02.197165+0000","last_fullsized":"2024-04-12T11:48:02.197165+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:09.819861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1869,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754507+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:47:41.754507+0000","last_peered":"2024-04-12T11:47:41.754507+0000","last_clean":"2024-04-12T11:47:41.754507+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:47:41.754507+0000","last_undegraded":"2024-04-12T11:47:41.754507+0000","last_fullsized":"2024-04-12T11:47:41.754507+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1874,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754187+0000","last_change":"2024-04-12T11:45:52.982299+0000","last_active":"2024-04-12T11:47:41.754187+0000","last_peered":"2024-04-12T11:47:41.754187+0000","last_clean":"2024-04-12T11:47:41.754187+0000","last_became_active":"2024-04-12T11:45:52.981942+0000","last_became_peered":"2024-04-12T11:45:52.981942+0000","last_unstale":"2024-04-12T11:47:41.754187+0000","last_undegraded":"2024-04-12T11:47:41.754187+0000","last_fullsized":"2024-04-12T11:47:41.754187+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:18:13.711200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"1231'2908","reported_seq":2922,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-04-12T11:48:07.176692+0000","last_change":"2024-04-12T11:47:31.339988+0000","last_active":"2024-04-12T11:48:07.176692+0000","last_peered":"2024-04-12T11:48:07.176692+0000","last_clean":"2024-04-12T11:48:07.176692+0000","last_became_active":"2024-04-12T11:47:31.339650+0000","last_became_peered":"2024-04-12T11:47:31.339650+0000","last_unstale":"2024-04-12T11:48:07.176692+0000","last_undegraded":"2024-04-12T11:48:07.176692+0000","last_fullsized":"2024-04-12T11:48:07.176692+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2908,"log_dups_size":0,"ondisk_log_size":2908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:02:57.059787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190578688,"num_objects":2908,"num_object_clones":0,"num_object_copies":8724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2908,"num_write_kb":186112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":57,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754673+0000","last_change":"2024-04-12T11:46:10.391570+0000","last_active":"2024-04-12T11:47:41.754673+0000","last_peered":"2024-04-12T11:47:41.754673+0000","last_clean":"2024-04-12T11:47:41.754673+0000","last_became_active":"2024-04-12T11:46:10.386152+0000","last_became_peered":"2024-04-12T11:46:10.386152+0000","last_unstale":"2024-04-12T11:47:41.754673+0000","last_undegraded":"2024-04-12T11:47:41.754673+0000","last_fullsized":"2024-04-12T11:47:41.754673+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:19:59.541495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1868,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.755059+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:47:41.755059+0000","last_peered":"2024-04-12T11:47:41.755059+0000","last_clean":"2024-04-12T11:47:41.755059+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:47:41.755059+0000","last_undegraded":"2024-04-12T11:47:41.755059+0000","last_fullsized":"2024-04-12T11:47:41.755059+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":49,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.753976+0000","last_change":"2024-04-12T11:46:10.409235+0000","last_active":"2024-04-12T11:47:41.753976+0000","last_peered":"2024-04-12T11:47:41.753976+0000","last_clean":"2024-04-12T11:47:41.753976+0000","last_became_active":"2024-04-12T11:46:10.408701+0000","last_became_peered":"2024-04-12T11:46:10.408701+0000","last_unstale":"2024-04-12T11:47:41.753976+0000","last_undegraded":"2024-04-12T11:47:41.753976+0000","last_fullsized":"2024-04-12T11:47:41.753976+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:03:52.034900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"1230'2640","reported_seq":2656,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:48:05.236786+0000","last_change":"2024-04-12T11:47:31.336555+0000","last_active":"2024-04-12T11:48:05.236786+0000","last_peered":"2024-04-12T11:48:05.236786+0000","last_clean":"2024-04-12T11:48:05.236786+0000","last_became_active":"2024-04-12T11:47:31.336202+0000","last_became_peered":"2024-04-12T11:47:31.336202+0000","last_unstale":"2024-04-12T11:48:05.236786+0000","last_undegraded":"2024-04-12T11:48:05.236786+0000","last_fullsized":"2024-04-12T11:48:05.236786+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":2640,"log_dups_size":0,"ondisk_log_size":2640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:17:03.188494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173015040,"num_objects":2640,"num_object_clones":0,"num_object_copies":7920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2640,"num_write_kb":168960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1230,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.542000+0000","last_change":"2024-04-12T11:45:52.979745+0000","last_active":"2024-04-12T11:47:41.542000+0000","last_peered":"2024-04-12T11:47:41.542000+0000","last_clean":"2024-04-12T11:47:41.542000+0000","last_became_active":"2024-04-12T11:45:52.978936+0000","last_became_peered":"2024-04-12T11:45:52.978936+0000","last_unstale":"2024-04-12T11:47:41.542000+0000","last_undegraded":"2024-04-12T11:47:41.542000+0000","last_fullsized":"2024-04-12T11:47:41.542000+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:22.710430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1882,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754414+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:47:41.754414+0000","last_peered":"2024-04-12T11:47:41.754414+0000","last_clean":"2024-04-12T11:47:41.754414+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:47:41.754414+0000","last_undegraded":"2024-04-12T11:47:41.754414+0000","last_fullsized":"2024-04-12T11:47:41.754414+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1788,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333102+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:47:41.333102+0000","last_peered":"2024-04-12T11:47:41.333102+0000","last_clean":"2024-04-12T11:47:41.333102+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:47:41.333102+0000","last_undegraded":"2024-04-12T11:47:41.333102+0000","last_fullsized":"2024-04-12T11:47:41.333102+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1883,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754171+0000","last_change":"2024-04-12T11:45:52.979689+0000","last_active":"2024-04-12T11:47:41.754171+0000","last_peered":"2024-04-12T11:47:41.754171+0000","last_clean":"2024-04-12T11:47:41.754171+0000","last_became_active":"2024-04-12T11:45:52.979286+0000","last_became_peered":"2024-04-12T11:45:52.979286+0000","last_unstale":"2024-04-12T11:47:41.754171+0000","last_undegraded":"2024-04-12T11:47:41.754171+0000","last_fullsized":"2024-04-12T11:47:41.754171+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:10.453472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":147,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-04-12T11:47:39.546890+0000","last_change":"2024-04-12T11:46:10.387670+0000","last_active":"2024-04-12T11:47:39.546890+0000","last_peered":"2024-04-12T11:47:39.546890+0000","last_clean":"2024-04-12T11:47:39.546890+0000","last_became_active":"2024-04-12T11:46:10.387356+0000","last_became_peered":"2024-04-12T11:46:10.387356+0000","last_unstale":"2024-04-12T11:47:39.546890+0000","last_undegraded":"2024-04-12T11:47:39.546890+0000","last_fullsized":"2024-04-12T11:47:39.546890+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:42:34.271621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1200,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333637+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:47:41.333637+0000","last_peered":"2024-04-12T11:47:41.333637+0000","last_clean":"2024-04-12T11:47:41.333637+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:47:41.333637+0000","last_undegraded":"2024-04-12T11:47:41.333637+0000","last_fullsized":"2024-04-12T11:47:41.333637+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1833,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.334166+0000","last_change":"2024-04-12T11:46:10.393030+0000","last_active":"2024-04-12T11:47:41.334166+0000","last_peered":"2024-04-12T11:47:41.334166+0000","last_clean":"2024-04-12T11:47:41.334166+0000","last_became_active":"2024-04-12T11:46:10.392633+0000","last_became_peered":"2024-04-12T11:46:10.392633+0000","last_unstale":"2024-04-12T11:47:41.334166+0000","last_undegraded":"2024-04-12T11:47:41.334166+0000","last_fullsized":"2024-04-12T11:47:41.334166+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:37:56.653388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1185,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333789+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:47:41.333789+0000","last_peered":"2024-04-12T11:47:41.333789+0000","last_clean":"2024-04-12T11:47:41.333789+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:47:41.333789+0000","last_undegraded":"2024-04-12T11:47:41.333789+0000","last_fullsized":"2024-04-12T11:47:41.333789+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1824,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333900+0000","last_change":"2024-04-12T11:46:10.390271+0000","last_active":"2024-04-12T11:47:41.333900+0000","last_peered":"2024-04-12T11:47:41.333900+0000","last_clean":"2024-04-12T11:47:41.333900+0000","last_became_active":"2024-04-12T11:46:10.389691+0000","last_became_peered":"2024-04-12T11:46:10.389691+0000","last_unstale":"2024-04-12T11:47:41.333900+0000","last_undegraded":"2024-04-12T11:47:41.333900+0000","last_fullsized":"2024-04-12T11:47:41.333900+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:11.035208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":104,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.755409+0000","last_change":"2024-04-12T11:46:10.400994+0000","last_active":"2024-04-12T11:47:41.755409+0000","last_peered":"2024-04-12T11:47:41.755409+0000","last_clean":"2024-04-12T11:47:41.755409+0000","last_became_active":"2024-04-12T11:46:10.400663+0000","last_became_peered":"2024-04-12T11:46:10.400663+0000","last_unstale":"2024-04-12T11:47:41.755409+0000","last_undegraded":"2024-04-12T11:47:41.755409+0000","last_fullsized":"2024-04-12T11:47:41.755409+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:33:02.927692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":630,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754847+0000","last_change":"2024-04-12T11:46:10.389628+0000","last_active":"2024-04-12T11:47:41.754847+0000","last_peered":"2024-04-12T11:47:41.754847+0000","last_clean":"2024-04-12T11:47:41.754847+0000","last_became_active":"2024-04-12T11:46:10.389222+0000","last_became_peered":"2024-04-12T11:46:10.389222+0000","last_unstale":"2024-04-12T11:47:41.754847+0000","last_undegraded":"2024-04-12T11:47:41.754847+0000","last_fullsized":"2024-04-12T11:47:41.754847+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:55:17.170994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1912,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.754628+0000","last_change":"2024-04-12T11:43:26.370162+0000","last_active":"2024-04-12T11:47:41.754628+0000","last_peered":"2024-04-12T11:47:41.754628+0000","last_clean":"2024-04-12T11:47:41.754628+0000","last_became_active":"2024-04-12T11:43:26.352240+0000","last_became_peered":"2024-04-12T11:43:26.352240+0000","last_unstale":"2024-04-12T11:47:41.754628+0000","last_undegraded":"2024-04-12T11:47:41.754628+0000","last_fullsized":"2024-04-12T11:47:41.754628+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:34:09.364519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1779,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-04-12T11:47:41.333183+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:47:41.333183+0000","last_peered":"2024-04-12T11:47:41.333183+0000","last_clean":"2024-04-12T11:47:41.333183+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:47:41.333183+0000","last_undegraded":"2024-04-12T11:47:41.333183+0000","last_fullsized":"2024-04-12T11:47:41.333183+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":11634,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2807103488,"num_objects":42833,"num_object_clones":0,"num_object_copies":128499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":42833,"num_write_kb":2741312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8460566528,"data_stored":8460566528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42833,"ondisk_log_size":42833,"up":48,"acting":48,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234429,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2490736,"kb_used_data":1985952,"kb_used_omap":1,"kb_used_meta":504382,"kb_avail":91254416,"statfs":{"total":95995035648,"available":93444521984,"internally_reserved":0,"allocated":2033614848,"data_stored":2030997415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":516487627},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496066,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":919720,"kb_used_data":549464,"kb_used_omap":1,"kb_used_meta":369854,"kb_avail":92825432,"statfs":{"total":95995035648,"available":95053242368,"internally_reserved":0,"allocated":562651136,"data_stored":560042331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":378730952},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536375,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1935704,"kb_used_data":1498384,"kb_used_omap":30,"kb_used_meta":437281,"kb_avail":91809448,"statfs":{"total":95995035648,"available":94012874752,"internally_reserved":0,"allocated":1534345216,"data_stored":1531221775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":447776066},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190459,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1624104,"kb_used_data":1042272,"kb_used_omap":43,"kb_used_meta":581780,"kb_avail":92121048,"statfs":{"total":95995035648,"available":94331953152,"internally_reserved":0,"allocated":1067286528,"data_stored":1064147941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":595742763},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849587,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1504668,"kb_used_data":864596,"kb_used_omap":36,"kb_used_meta":640027,"kb_avail":92240484,"statfs":{"total":95995035648,"available":94454255616,"internally_reserved":0,"allocated":885346304,"data_stored":882218801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":655387828},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":0,"seq":2534030704940,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018152,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1148080,"kb_used_data":692520,"kb_used_omap":42,"kb_used_meta":455509,"kb_avail":92597072,"statfs":{"total":95995035648,"available":94819401728,"internally_reserved":0,"allocated":709140480,"data_stored":705997365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":466442040},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757701,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2114244,"kb_used_data":1559804,"kb_used_omap":1,"kb_used_meta":554430,"kb_avail":91630908,"statfs":{"total":95995035648,"available":93830049792,"internally_reserved":0,"allocated":1597239296,"data_stored":1594515365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":567736774},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1082458112,"data_stored":1082458112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1544486912,"data_stored":1544486912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":881721344,"data_stored":881721344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":705167360,"data_stored":705167360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1608056832,"data_stored":1608056832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":559415296,"data_stored":559415296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2079260672,"data_stored":2079260672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:48:11.724 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-04-12T11:48:11.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:11.886 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:11.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:11.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 53716 53700 124.293 99.5625 0.0125841 0.00804052 2024-04-12T11:48:11.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:12.032 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:12.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:12.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:12.134 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:12.180 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:12.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:12.281 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:12.323 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:12.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:12.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:12.425 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:12.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:12.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:12.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:12.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:12.602 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:12.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:12.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:48:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:48:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:48:12.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:12.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:12.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:12.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:12.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:12.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:12.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:12.964 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 55345 55329 123.49 101.812 0.0161441 0.00809279 2024-04-12T11:48:12.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:13.021 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:13.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:13.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:13.123 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:13.163 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:13.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:13.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:13.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:13.300 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:13.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:13.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:13.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:13.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:13.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:13.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:13.501 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:13 smithi029 ceph-mon[93650]: pgmap v1911: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-04-12T11:48:13.502 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:13 smithi029 ceph-mon[97787]: pgmap v1911: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-04-12T11:48:13.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:13.574 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:13.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:13.676 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:13.713 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:13.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:13.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:13 smithi087 ceph-mon[89014]: pgmap v1911: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-04-12T11:48:13.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:13.850 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:13.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:13.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:13.952 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:13.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 56914 56898 122.612 98.0625 0.00531068 0.00815047 2024-04-12T11:48:13.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:13.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:13.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:14.029 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:14.084 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:14.091 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:14.128 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:48:14.128 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:48:14.128 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:48:14.128 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:48:14.128 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:48:14.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:14.146 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:14.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:14.247 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:14.280 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:14.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:14.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:14.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:14.416 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:14.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:14.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:14.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:14.553 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:14.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:14.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:14.687 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:14.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:14.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:14.789 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:14.827 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:14.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:14.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:14.928 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:14.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 58467 58451 121.76 97.0625 0.00819786 0.00820781 2024-04-12T11:48:14.965 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:14.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:14.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:15.067 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:15.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:15.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:15.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:15.204 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:15.207 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:48:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:48:14] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2024-04-12T11:48:15.239 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:15.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:15.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:15.341 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:15.378 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:15.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:15.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:15.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:15.515 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:15.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:15.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:15.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:15 smithi029 ceph-mon[97787]: pgmap v1912: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-04-12T11:48:15.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:15 smithi029 ceph-mon[93650]: pgmap v1912: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-04-12T11:48:15.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:48:15.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:15.656 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:15.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:15.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:15 smithi087 ceph-mon[89014]: pgmap v1912: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-04-12T11:48:15.757 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:15.792 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:15.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:15.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:15.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:15.927 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:15.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:15.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 60019 60003 120.961 97 0.00826707 0.00826206 2024-04-12T11:48:16.029 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:16.074 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:16.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:16.174 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:16.211 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:16.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:16.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:16.313 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:16.350 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:16.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:16.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:16.452 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:16.491 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:16.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:16.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:16.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:16.625 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:16.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:16.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:16.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:16.763 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:16.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:16.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:16.865 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:16.902 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:16.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:16.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:16.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 61494 61478 120.062 92.1875 0.00842501 0.00832439 2024-04-12T11:48:17.003 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:17.034 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:17.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:17.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:17.136 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:17.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-04-12T11:48:17.179 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:17.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:17.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:17.281 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:17.326 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:17.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:17.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:17.428 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:17.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:17.463 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:17.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:17.564 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:17.598 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:17.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:17.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:17 smithi029 ceph-mon[97787]: pgmap v1913: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-04-12T11:48:17.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:17 smithi029 ceph-mon[93650]: pgmap v1913: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-04-12T11:48:17.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:17 smithi087 ceph-mon[89014]: pgmap v1913: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-04-12T11:48:17.733 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:17.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:17.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:17.835 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:17.871 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:17.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:17.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:17.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 62957 62941 119.194 91.4375 0.0047188 0.0083841 2024-04-12T11:48:17.973 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:18.007 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:18.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:18.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:18.108 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:18.146 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:18.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:18.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:18.233 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:18.249 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:18.300 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:18.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:18.402 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:18.441 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:18.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:18.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:18.543 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:18.583 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:18.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:18.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:18.685 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:18.720 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:18.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:18.760 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:18.823 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:18.870 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:18.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:18.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:18.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 64359 64343 118.266 87.625 0.00927579 0.00845055 2024-04-12T11:48:18.973 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:19.009 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:19.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:19.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:19.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:19.146 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:19.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:19.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:19.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:19.283 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:19.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:19.384 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:19.397 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 1 pgp_num to 1 2024-04-12T11:48:19.399 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:48:19.422 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:19.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:19.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:19.524 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:19.554 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:19.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:19.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:19.656 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:19.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:19.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:19.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:19 smithi029 ceph-mon[97787]: pgmap v1914: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-12T11:48:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:19 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1206368486' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2024-04-12T11:48:19.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:19 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3159944256' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:48:19.714 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:19 smithi029 ceph-mon[93650]: pgmap v1914: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-12T11:48:19.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:19 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1206368486' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2024-04-12T11:48:19.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:19 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3159944256' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:48:19.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:19 smithi087 ceph-mon[89014]: pgmap v1914: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-12T11:48:19.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:19 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1206368486' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2024-04-12T11:48:19.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:19 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3159944256' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:48:19.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:19.824 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:19.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:19.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:19.925 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:19.963 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:19.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:19.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:19.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 65794 65778 117.449 89.6875 0.0197996 0.00850895 2024-04-12T11:48:20.065 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:20.099 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:20.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:20.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:20.201 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:20.243 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:20.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:20.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:20.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:20.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:20.388 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:20.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:20.489 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:20.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:20 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:48:20.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:20 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1206368486' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-04-12T11:48:20.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:20 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3159944256' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:48:20.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:20 smithi029 ceph-mon[97787]: osdmap e1232: 8 total, 8 up, 7 in 2024-04-12T11:48:20.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:20 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:48:20.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:20 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1206368486' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-04-12T11:48:20.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:20 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3159944256' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:48:20.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:20 smithi029 ceph-mon[93650]: osdmap e1232: 8 total, 8 up, 7 in 2024-04-12T11:48:20.524 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:20.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:20.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:20.626 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:20.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:20.664 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:20.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:20.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:20 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:48:20.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:20 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1206368486' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-04-12T11:48:20.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:20 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3159944256' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:48:20.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:20 smithi087 ceph-mon[89014]: osdmap e1232: 8 total, 8 up, 7 in 2024-04-12T11:48:20.765 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:20.801 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:20.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:20.903 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:20.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:20.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:20.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:20.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 67217 67201 116.657 88.9375 0.0220024 0.00856704 2024-04-12T11:48:21.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:21.081 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:21.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:21.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:21.183 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:21.221 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:21.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:21.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:21.323 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:21.360 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:21.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:21.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:21.462 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:21.499 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:21.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:21.601 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:21.638 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:21.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:21.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:21 smithi029 ceph-mon[97787]: pgmap v1916: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-12T11:48:21.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:21.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:21 smithi029 ceph-mon[93650]: pgmap v1916: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-12T11:48:21.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:21 smithi087 ceph-mon[89014]: pgmap v1916: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-12T11:48:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:21.739 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:21.778 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:21.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:21.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:21.802 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:48:21.842 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:48:21.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:21.886 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:48:21.919 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:21.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:21.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:21.923 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:48:21.964 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:48:21.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 68614 68598 115.863 87.3125 0.00717137 0.00862597 2024-04-12T11:48:22.002 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:48:22.021 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:22.038 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:48:22.056 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:22.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:22.077 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:48:22.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:48:22.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:48:22.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:48:22.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:48:22.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:48:22.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:48:22.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:48:22.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:48:22.158 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:22.194 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:22.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:22.296 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:22.320 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:22.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:22.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:22.422 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:22.461 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:22.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:22.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:22.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:22.598 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:22.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:22.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:22.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:48:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:48:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:48:22.618 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:22 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2977333514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:22.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:22.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:22.738 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:22.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:22.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:22 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2977333514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:22 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2977333514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:22.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:22.876 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:22.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:22.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:22.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 69921 69905 114.964 81.6875 0.0142475 0.00869351 2024-04-12T11:48:22.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:23.013 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:23.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:23.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:23.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:23.153 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:23.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:23.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:23.255 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:23.291 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:23.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:23.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:23.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:23.431 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:23.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:23.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:23.532 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:23.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:23.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:23.668 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:23.698 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:23.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:23.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:23 smithi087 ceph-mon[89014]: pgmap v1917: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-04-12T11:48:23.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:23 smithi029 ceph-mon[97787]: pgmap v1917: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-04-12T11:48:23.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:23 smithi029 ceph-mon[93650]: pgmap v1917: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-04-12T11:48:23.800 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:23.840 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:23.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:23.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:23.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 71285 71269 114.202 85.25 0.0123978 0.00875156 2024-04-12T11:48:23.973 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:23.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:23.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:24.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:24.109 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:24.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:24.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:24.211 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:24.234 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:24.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:24.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:24.336 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:24.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:24.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:24.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:24.503 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:24.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:24.604 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:24.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:24.644 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:24.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:24.745 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:24.781 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:24.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:24.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:24.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:24.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:24.923 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:24.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:24.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:48:24.965900+0000 min lat: 0.0012668 max lat: 0.449354 avg lat: 0.00883727 2024-04-12T11:48:24.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:48:24.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 72404 72388 113.095 69.9375 0.0171884 0.00883727 2024-04-12T11:48:25.014 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:48:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:48:24] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2024-04-12T11:48:25.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:25.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:25.054 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:25.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:25.155 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:25.193 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:25.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:25.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:25.295 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:25.331 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:25.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:25.408 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:48:25.408 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:48:25.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:25.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:25.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:25.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:25.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:25.602 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:48:25.613 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:25.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:25.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:25.648 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:25 smithi029 ceph-mon[97787]: pgmap v1918: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-04-12T11:48:25.648 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:25 smithi029 ceph-mon[93650]: pgmap v1918: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-04-12T11:48:25.715 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:25 smithi087 ceph-mon[89014]: pgmap v1918: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-04-12T11:48:25.749 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:25.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:25.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:25.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:25.888 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:25.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:25.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:25.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 73641 73625 112.222 77.3125 0.00925551 0.00890596 2024-04-12T11:48:25.990 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:26.028 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:26.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:26.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:26.161 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:26.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:26.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:26.263 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:26.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:26.301 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:26.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:26.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:26.443 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:26.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:26.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:26.520 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:48:26.521 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:48:26.521 INFO:tasks.thrashosds.thrasher:Adding osd 3 2024-04-12T11:48:26.521 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd in 3 2024-04-12T11:48:26.544 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:26.607 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:26.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:26.709 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:26.716 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-04-12T11:48:26.744 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:26.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:26.766 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:48:26.818 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:26.846 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:26.883 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:26.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:26.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:26.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 74812 74796 111.292 73.1875 0.0166451 0.00898042 2024-04-12T11:48:26.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:27.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:27.043 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:27.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:27.102 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:48:27.144 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:27.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:27.190 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:27.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:27.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:27.324 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:27.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:27.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:27.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:27.459 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:27.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:27.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:27.593 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:27.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:27.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:27.695 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:27 smithi087 ceph-mon[89014]: pgmap v1919: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-12T11:48:27.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:27.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:27.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:27 smithi029 ceph-mon[97787]: pgmap v1919: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-12T11:48:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:27 smithi029 ceph-mon[93650]: pgmap v1919: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-12T11:48:27.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:27.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:27.879 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:27.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:27.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 76043 76027 110.493 76.9375 0.00589464 0.009045 2024-04-12T11:48:27.980 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:28.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:28.012 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:28.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:28.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:28.153 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:28.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:28.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:28.255 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:28.266 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:28.294 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:28.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:28.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:28.331 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:28.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:28.427 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:28.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:28.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:28.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:28.567 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:28.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:28.669 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:28.700 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:28.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:28.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:28.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:28.835 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:28.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:28.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:28.937 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:28.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 77195 77179 109.618 72 0.0112761 0.0091177 2024-04-12T11:48:28.973 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:28.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:28.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:29.075 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:29.112 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:29.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:29.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:29.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:29.278 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:29.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:29.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:29.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:29.419 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:29.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:29.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:29.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:29 smithi029 ceph-mon[97787]: pgmap v1920: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T11:48:29.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:29 smithi029 ceph-mon[93650]: pgmap v1920: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T11:48:29.521 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:29.555 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:29.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:29.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:29.657 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:29.695 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:29.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:29.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:29 smithi087 ceph-mon[89014]: pgmap v1920: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T11:48:29.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:29.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:29.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:29.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:29.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:29.961 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:29.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:29.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 78399 78383 108.854 75.25 0.00428893 0.00918023 2024-04-12T11:48:30.064 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:30.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:30.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:30.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:30.204 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:30.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:30.244 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:30.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:30.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:30.379 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:30.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:30.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:30.437 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:48:30.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:30 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3665377525' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:48:30.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:30 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:48:30.481 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:30.497 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:48:30.497 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:48:30.510 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:30.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:30.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:30.612 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:30.645 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:30.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:30.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:30.747 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:30 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3665377525' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:48:30.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:30 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:48:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:30 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3665377525' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:48:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:30 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:48:30.798 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:30.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:30.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:30.900 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:30.937 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:30.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:30.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:30.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 79394 79378 107.84 62.1875 0.0126038 0.00926762 2024-04-12T11:48:31.039 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:31.075 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:31.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:31.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:31.177 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:31.222 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:31.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:31.323 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:31.369 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:31.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:31.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:31.470 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-04-12T11:48:31.473 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:31.519 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:31.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:31.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:31.668 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:31.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:31.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:31.694 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:31 smithi087 ceph-mon[89014]: pgmap v1921: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T11:48:31.694 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:31 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:48:31.694 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:31 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:48:31.694 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:31 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:48:31.694 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:31 smithi087 ceph-mon[89014]: osdmap e1233: 8 total, 8 up, 7 in 2024-04-12T11:48:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:31 smithi029 ceph-mon[93650]: pgmap v1921: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T11:48:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:31 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:48:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:31 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:48:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:31 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:48:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:31 smithi029 ceph-mon[93650]: osdmap e1233: 8 total, 8 up, 7 in 2024-04-12T11:48:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:31 smithi029 ceph-mon[97787]: pgmap v1921: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-12T11:48:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:31 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:48:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:31 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:48:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:31 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:48:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:31 smithi029 ceph-mon[97787]: osdmap e1233: 8 total, 8 up, 7 in 2024-04-12T11:48:31.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:31.802 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:31.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:31.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:31.824 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:48:31.904 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:31.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:31.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:31.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:31.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 80446 80430 106.944 65.75 0.00439858 0.00934548 2024-04-12T11:48:32.060 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:32.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:32.103 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:32.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:32.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:32.233 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:32.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:32.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:32.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:32.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:32.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:32.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:32.465 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:32.502 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:32.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:32.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:32.605 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:32.627 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:32.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:32.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:48:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:48:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:48:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:32 smithi087 ceph-mon[89014]: pgmap v1923: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-04-12T11:48:32.730 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:32.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:32.765 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:32.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:32.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:32 smithi029 ceph-mon[93650]: pgmap v1923: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-04-12T11:48:32.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:32 smithi029 ceph-mon[97787]: pgmap v1923: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-04-12T11:48:32.866 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:32.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:32.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:32.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:32.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 81213 81197 105.715 47.9375 0.0044332 0.00944507 2024-04-12T11:48:33.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:33.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:33.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:33.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:33.133 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:33.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:33.168 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:33.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:33.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:33.301 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:33.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:33.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:33.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:33.428 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:33.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:33.530 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:33.555 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:33.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:33.657 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:33.698 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:33.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:33.796 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:48:33.796 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:48:33.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:33.824 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:33.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:33.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:33.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:33.958 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:33.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:33.966 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 81882 81866 104.41 41.8125 0.00812121 0.00957328 2024-04-12T11:48:34.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:34.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:34.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:34.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:34.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:34.241 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:34.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:34.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:34.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:34.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:34.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:34.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:34.473 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-04-12T11:48:34.501 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:34.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:34.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:34.603 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:34.658 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:34.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:34.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:34.760 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:34.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:34.808 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:34.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:34.852 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:48:34.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:34.948 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:34.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:34.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:34.967 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 82801 82785 103.471 57.4375 0.0291658 0.00965771 2024-04-12T11:48:35.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:35.082 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:35.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:35.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:35.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:35.219 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:35.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:35.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:35.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:35 smithi029 ceph-mon[93650]: pgmap v1924: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-04-12T11:48:35.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:35 smithi029 ceph-mon[97787]: pgmap v1924: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-04-12T11:48:35.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:48:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:48:34] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2024-04-12T11:48:35.279 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:35 smithi087 ceph-mon[89014]: pgmap v1924: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-04-12T11:48:35.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:35.345 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:35.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:35.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:35.446 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:35.470 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:35.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:35.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:35.572 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:35.606 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:35.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:35.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:35.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:35.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:35.746 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:35.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:35.847 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:35.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:35.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:35.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:35.967 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 83746 83730 102.6 59.0625 0.00617778 0.00974165 2024-04-12T11:48:35.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:36.020 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:36.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:36.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:36.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:36.146 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:36.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:36.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:36.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:36.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:36.289 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:36.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:36.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:36.431 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:36.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:36.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:36.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:36.556 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:36.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:36.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:36.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:36.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:36.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:36.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:36.705 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:48:36.705 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:48:36.804 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:36.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:36.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:36.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:36.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:36.967 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 84552 84536 101.595 50.375 0.0109101 0.009838 2024-04-12T11:48:36.985 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:36.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:36.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:37.087 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:37.125 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:37.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:37.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:37.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:37.267 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:37.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:37.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:37.369 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:37.392 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:37.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:37.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:37.428 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:37 smithi087 ceph-mon[89014]: pgmap v1925: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T11:48:37.428 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:48:37.428 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:37.457 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-04-12T11:48:37.494 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:37.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:37.524 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:37.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:37.572 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:37 smithi029 ceph-mon[97787]: pgmap v1925: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T11:48:37.572 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:48:37.572 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:37.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:37 smithi029 ceph-mon[93650]: pgmap v1925: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-12T11:48:37.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:48:37.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:37.625 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:37.660 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:37.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:37.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:37.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:37.793 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:37.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:37.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:37.853 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:48:37.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:37.925 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:37.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:37.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:37.967 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 85424 85408 100.707 54.5 0.0155562 0.00992494 2024-04-12T11:48:38.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:38.050 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:38.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:38.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:38.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:38.175 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:38.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:38.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:38.276 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:38.320 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:38.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:38.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:38.454 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:38.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:38.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:38.555 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:38.579 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:38.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:38.681 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:38.704 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:38.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:38.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:38.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:38.839 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:38.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:38.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:38.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:38.967 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:38.967 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 86059 86043 99.5766 39.6875 0.0328213 0.0100347 2024-04-12T11:48:38.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:38.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:39.068 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:39.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:39.104 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:39.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:39.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:39.242 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:39.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:39.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:39.344 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:39.380 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:39.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:39.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:39.442 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:39 smithi087 ceph-mon[89014]: pgmap v1926: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:48:39.482 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:39.516 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:39.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:39.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:39.618 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:39.648 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:39.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:39.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:39.750 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:39.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:39 smithi029 ceph-mon[97787]: pgmap v1926: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:48:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:39 smithi029 ceph-mon[93650]: pgmap v1926: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:48:39.778 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:48:39.778 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:48:39.780 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:39.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:39.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:39.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:39.911 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:39.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:39.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:39.967 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 87087 87071 98.9342 64.25 0.0110637 0.0101027 2024-04-12T11:48:40.013 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:40.048 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:40.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:40.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:40.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:40.183 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:40.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:40.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:40.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:40.319 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:40.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:40.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:40.420 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:40.435 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-04-12T11:48:40.444 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:40.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:40.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:40.546 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:40.578 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:40.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:40.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:40.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:40.717 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:40.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:40.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:40.818 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:40.822 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:48:40.848 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:40.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:40.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:40.950 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:40.967 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 88151 88135 98.3547 66.5 0.00863419 0.0101622 2024-04-12T11:48:40.984 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:40.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:40.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:41.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:41.111 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:41.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:41.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:41.213 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:41.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:41.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:41.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:41.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:41.375 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:41.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:41.477 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:41.513 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:41.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:41.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:41.615 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:41.657 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:41.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:41.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:41 smithi087 ceph-mon[89014]: pgmap v1927: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 958 op/s 2024-04-12T11:48:41.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:41 smithi029 ceph-mon[97787]: pgmap v1927: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 958 op/s 2024-04-12T11:48:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:41 smithi029 ceph-mon[93650]: pgmap v1927: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 958 op/s 2024-04-12T11:48:41.792 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:41.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:41.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:41.894 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:41.933 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:41.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:41.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 89163 89147 97.7387 63.25 0.00777671 0.0102264 2024-04-12T11:48:42.034 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:42.069 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:42.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:42.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:42.172 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:42.215 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:42.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:42.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:42.316 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:42.365 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:42.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:42.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:42.441 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:48:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:48:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:48:42.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:42.500 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:42.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:42.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:42.601 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:42.648 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:42.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:42.750 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:42.789 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:42.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:42.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:42.798 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:48:42.798 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:48:42.891 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:42.924 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:42.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:42.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:42.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 90210 90194 97.1817 65.4375 0.025469 0.010285 2024-04-12T11:48:43.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:43.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:43.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:43.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:43.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:43.202 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:43.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:43.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:43.303 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:43.341 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:43.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:43.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:43.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:43.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:43 smithi087 ceph-mon[89014]: pgmap v1928: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s 2024-04-12T11:48:43.485 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:43.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:43.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:43.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:43 smithi029 ceph-mon[97787]: pgmap v1928: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s 2024-04-12T11:48:43.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:43 smithi029 ceph-mon[93650]: pgmap v1928: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 57 MiB/s wr, 918 op/s 2024-04-12T11:48:43.568 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-04-12T11:48:43.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:43.621 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:43.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:43.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:43.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:43.752 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:43.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:43.854 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:43.900 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:43.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:43.939 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:48:43.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 91222 91206 96.6065 63.25 0.036004 0.0103453 2024-04-12T11:48:44.002 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:44.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:43 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T11:48:43.681+0000 7fae24212700 -1 mon.a@0(leader).osd e1233 definitely_dead 0 2024-04-12T11:48:44.046 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:44.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:44.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:44.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:48:44.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:44.186 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:44.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:44.287 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:44.327 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:44.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:44.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:44.397 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.3. 2024-04-12T11:48:44.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:44.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:44.453 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:44.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:44.554 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:44.577 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:44.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:44.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:44.679 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:44.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:44.703 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:44.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:44.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:44 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/520435042' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:48:44.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:44 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:48:44.742 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:44 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/520435042' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:48:44.742 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:44 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:48:44.742 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:44 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/520435042' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:48:44.742 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:44 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2024-04-12T11:48:44.805 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:44.829 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:44.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:44.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:44.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:44.956 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:44.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:44.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:44.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:48:44.968089+0000 min lat: 0.0012668 max lat: 0.449354 avg lat: 0.0103772 2024-04-12T11:48:44.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:48:44.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 91632 91616 95.4235 25.625 0.00456978 0.0103772 2024-04-12T11:48:45.015 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:48:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:48:44] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.43.0" 2024-04-12T11:48:45.058 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:45.083 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:45.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:45.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:45.185 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:45.208 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:45.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:45.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:45.310 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:45.345 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:45.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:45.446 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:45.469 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:45.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:45.570 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:45.594 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:45.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:45.696 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:45 smithi087 ceph-mon[89014]: pgmap v1929: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2024-04-12T11:48:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:45 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-04-12T11:48:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:45 smithi087 ceph-mon[89014]: osdmap e1234: 8 total, 8 up, 8 in 2024-04-12T11:48:45.720 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:45.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:45.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:45 smithi029 ceph-mon[97787]: pgmap v1929: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2024-04-12T11:48:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:45 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-04-12T11:48:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:45 smithi029 ceph-mon[97787]: osdmap e1234: 8 total, 8 up, 8 in 2024-04-12T11:48:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:45 smithi029 ceph-mon[93650]: pgmap v1929: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 60 MiB/s wr, 957 op/s 2024-04-12T11:48:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:45 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-04-12T11:48:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:45 smithi029 ceph-mon[93650]: osdmap e1234: 8 total, 8 up, 8 in 2024-04-12T11:48:45.822 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:45.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:45.846 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:45.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:45.947 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:45.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 91632 91616 93.8591 0 - 0.0103772 2024-04-12T11:48:45.973 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:45.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:45.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:46.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:46.101 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:46.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:46.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:46.203 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:46.230 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:46.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:46.331 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:46.370 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:46.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:46.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:46.472 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:46.497 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:46.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:46.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:46.637 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:46.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:46.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:46.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:46 smithi029 ceph-mon[97787]: osdmap e1235: 8 total, 8 up, 8 in 2024-04-12T11:48:46.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:46 smithi029 ceph-mon[97787]: osdmap e1236: 8 total, 8 up, 8 in 2024-04-12T11:48:46.707 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:46 smithi029 ceph-mon[93650]: osdmap e1235: 8 total, 8 up, 8 in 2024-04-12T11:48:46.708 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:46 smithi029 ceph-mon[93650]: osdmap e1236: 8 total, 8 up, 8 in 2024-04-12T11:48:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:46 smithi087 ceph-mon[89014]: osdmap e1235: 8 total, 8 up, 8 in 2024-04-12T11:48:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:46 smithi087 ceph-mon[89014]: osdmap e1236: 8 total, 8 up, 8 in 2024-04-12T11:48:46.739 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:46.741 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:46.747 INFO:tasks.thrashosds.thrasher:Added osd 3 2024-04-12T11:48:46.764 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:46.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:46.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:46.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:46.909 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:46.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:46.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:46.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 91730 91714 92.444 3.0625 0.0046645 0.0107227 2024-04-12T11:48:47.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:47.049 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:47.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:47.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:47.151 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:47.186 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:47.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:47.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:47.287 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:47.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:47.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:47.419 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:47.460 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:47.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:47.562 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:47.600 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:47 smithi087 ceph-mon[89014]: pgmap v1932: 89 pgs: 7 peering, 82 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2024-04-12T11:48:47.604 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:47.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:47.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:47.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:47.759 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:47 smithi029 ceph-mon[93650]: pgmap v1932: 89 pgs: 7 peering, 82 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2024-04-12T11:48:47.759 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:47 smithi029 ceph-mon[97787]: pgmap v1932: 89 pgs: 7 peering, 82 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2024-04-12T11:48:47.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:47.764 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:47.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:47.785 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:48:47.785 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:48:47.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:47.932 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:47.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:47.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:47.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 92355 92339 91.5967 39.0625 0.0140505 0.010912 2024-04-12T11:48:48.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:48.048 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:48:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:48.100 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:48.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:48.106 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:48:48.153 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:48:48.201 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:48.201 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:48:48.249 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:48:48.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:48.254 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:48.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:48.291 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:48:48.335 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:48:48.354 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:48.380 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:48:48.385 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:48.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:48.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:48.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:48:48.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:48:48.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:48:48.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:48:48.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:48:48.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:48:48.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:48:48.395 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:48:48.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:48.512 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-04-12T11:48:48.534 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:48.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:48.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:48.636 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:48.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:48.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:48.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:48.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:48 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:48:48.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:48 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3315617643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:48 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:48:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:48 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3315617643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:48.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:48 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:48:48.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:48 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3315617643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:48.784 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:48.824 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:48.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:48.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:48.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:48.963 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:48.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:48.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:48.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 92986 92970 90.7817 39.4375 0.0148263 0.0110104 2024-04-12T11:48:49.065 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:49.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:49.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:49.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:49.203 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:49.237 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:49.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:49.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:49.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:49.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:49.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:49.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:49.506 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:49.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:49.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:49.541 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:48:49.607 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:49.648 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:49.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:49.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:49.688 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:49 smithi087 ceph-mon[89014]: pgmap v1934: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 7 peering, 79 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 5634/272586 objects degraded (2.067%); 14 B/s, 0 objects/s recovering 2024-04-12T11:48:49.688 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:49 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 5634/272586 objects degraded (2.067%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:48:49.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:49 smithi029 ceph-mon[97787]: pgmap v1934: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 7 peering, 79 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 5634/272586 objects degraded (2.067%); 14 B/s, 0 objects/s recovering 2024-04-12T11:48:49.725 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:49 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 5634/272586 objects degraded (2.067%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:48:49.725 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:49 smithi029 ceph-mon[93650]: pgmap v1934: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 7 peering, 79 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s; 5634/272586 objects degraded (2.067%); 14 B/s, 0 objects/s recovering 2024-04-12T11:48:49.725 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:49 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 5634/272586 objects degraded (2.067%), 1 pg degraded (PG_DEGRADED) 2024-04-12T11:48:49.749 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:49.789 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:49.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:49.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:49.891 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:49.934 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:49.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:49.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:49.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 93665 93649 90.0378 42.4375 0.0349995 0.0110996 2024-04-12T11:48:50.036 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:50.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:50.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:50.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:50.167 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:50.197 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:50.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:50.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:50.299 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:50.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:50.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:50.463 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:50.506 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:50.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:50.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:50.608 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:50.654 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:50.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:50.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:50.756 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:50.800 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:50.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:50.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:50.901 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:50.928 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:50.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:50.968 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 94316 94300 89.2901 40.6875 0.0208562 0.0111933 2024-04-12T11:48:51.029 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:51.067 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:51.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:51.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:51.168 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:51.195 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:51.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:51.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:51.297 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:51.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:51.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:51.397 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:48:51.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:51.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:51.478 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:51.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:51.579 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:51.621 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:51.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:51.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:51.640 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:51 smithi029 ceph-mon[93650]: pgmap v1935: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 4 peering, 78 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35 MiB/s wr, 553 op/s; 28577/276246 objects degraded (10.345%); 11 KiB/s, 0 objects/s recovering 2024-04-12T11:48:51.640 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:51 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2268638793' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:48:51.640 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:48:51.640 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:51.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:51 smithi029 ceph-mon[97787]: pgmap v1935: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 4 peering, 78 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35 MiB/s wr, 553 op/s; 28577/276246 objects degraded (10.345%); 11 KiB/s, 0 objects/s recovering 2024-04-12T11:48:51.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:51 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2268638793' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:48:51.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:48:51.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:51 smithi087 ceph-mon[89014]: pgmap v1935: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 4 peering, 78 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35 MiB/s wr, 553 op/s; 28577/276246 objects degraded (10.345%); 11 KiB/s, 0 objects/s recovering 2024-04-12T11:48:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:51 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2268638793' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:48:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:48:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:48:51.724 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:51.747 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:48:51.747 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:48:51.748 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-04-12T11:48:51.748 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:48:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:51.773 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:51.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:51.874 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:51.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:51.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:51.971 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 94917 94901 88.518 37.5625 0.0149454 0.0112907 2024-04-12T11:48:52.004 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:52.042 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:52.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:52.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:52.144 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:52.198 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:52.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:52.297 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:48:52.300 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:52.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:52.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:52.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:52.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:52.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:52.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:52.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:52 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:48:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:52 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2268638793' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:48:52.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:52 smithi029 ceph-mon[97787]: osdmap e1237: 8 total, 8 up, 8 in 2024-04-12T11:48:52.523 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:52 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:48:52.523 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:52 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2268638793' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:48:52.523 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:52 smithi029 ceph-mon[93650]: osdmap e1237: 8 total, 8 up, 8 in 2024-04-12T11:48:52.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:52.609 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:48:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:48:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:48:52.609 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:52 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:48:52.609 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:52 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2268638793' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:48:52.609 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:52 smithi087 ceph-mon[89014]: osdmap e1237: 8 total, 8 up, 8 in 2024-04-12T11:48:52.613 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:52.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:52.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:52.715 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:52.756 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:52.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:52.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:52.858 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:52.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:52.906 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:52.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:52.969 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 95568 95552 87.8145 40.6875 0.044324 0.0113784 2024-04-12T11:48:53.007 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:53.037 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:53.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:53.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:53.126 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:53.135 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:53.139 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:53.167 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:53.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:53.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:53.324 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:53.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:53.426 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:53.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:53.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:53.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:53.496 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:53 smithi029 ceph-mon[97787]: pgmap v1937: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 36 MiB/s wr, 582 op/s; 34303/279738 objects degraded (12.263%); 11280/279738 objects misplaced (4.032%); 926 KiB/s, 14 objects/s recovering 2024-04-12T11:48:53.496 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:53 smithi029 ceph-mon[93650]: pgmap v1937: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 36 MiB/s wr, 582 op/s; 34303/279738 objects degraded (12.263%); 11280/279738 objects misplaced (4.032%); 926 KiB/s, 14 objects/s recovering 2024-04-12T11:48:53.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:53.603 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:53.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:53.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:53 smithi087 ceph-mon[89014]: pgmap v1937: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 36 MiB/s wr, 582 op/s; 34303/279738 objects degraded (12.263%); 11280/279738 objects misplaced (4.032%); 926 KiB/s, 14 objects/s recovering 2024-04-12T11:48:53.750 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:53.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:53.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:53.852 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:53.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:53.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:53.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:53.969 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 96202 96186 87.1161 39.625 0.0129173 0.0114736 2024-04-12T11:48:53.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:54.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:54.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:54.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:54.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:54.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:54.167 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:54.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:54.268 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:54.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:54.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:54.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:54.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:54.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:54.459 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:54.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:54.560 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:54.589 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:54.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:54.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:54.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:48:54.691 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:54.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:54 smithi029 ceph-mon[93650]: pgmap v1938: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 34303/284004 objects degraded (12.078%); 11280/284004 objects misplaced (3.972%); 768 KiB/s, 12 objects/s recovering 2024-04-12T11:48:54.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:54 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/284004 objects degraded (12.078%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:48:54.716 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:54 smithi029 ceph-mon[97787]: pgmap v1938: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 34303/284004 objects degraded (12.078%); 11280/284004 objects misplaced (3.972%); 768 KiB/s, 12 objects/s recovering 2024-04-12T11:48:54.716 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:54 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/284004 objects degraded (12.078%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:48:54.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:54 smithi087 ceph-mon[89014]: pgmap v1938: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 34303/284004 objects degraded (12.078%); 11280/284004 objects misplaced (3.972%); 768 KiB/s, 12 objects/s recovering 2024-04-12T11:48:54.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:54 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/284004 objects degraded (12.078%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:48:54.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:54.765 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:54.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:54.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:54.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:54.909 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:54.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:54.969 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 96800 96784 86.4054 37.375 0.021859 0.0115665 2024-04-12T11:48:54.983 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:48:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:48:54] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.43.0" 2024-04-12T11:48:55.010 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:55.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:55.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:55.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:55.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:55.193 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:55.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:55.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:55.295 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:55.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:55.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:55.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:55.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:55.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:55.482 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:55.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35445.0:97150 9.0 e1237/1226 9:01bc85fd:::benchmark_data_smithi029_222585_object97149:head v 1237'6089, mlcod=1237'6086)", 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:48:55.523266+0000", 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.001161182, 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001191835, 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:48:55.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:48:55.523266+0000", 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:48:55.523266+0000", 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:48:55.523265+0000", 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:48:55.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:48:55.523310+0000", 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5738999999999997e-05 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:48:55.523331+0000", 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0265e-05 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:48:55.523341+0000", 2024-04-12T11:48:55.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0003000000000001e-05 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:48:55.523514+0000", 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017322400000000001 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:48:55.523567+0000", 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2651000000000001e-05 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:48:55.529 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:48:55.530 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2024-04-12T11:48:55.530 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 1 2024-04-12T11:48:55.530 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:48:55.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:55.620 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:55.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:55.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:55.722 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:55.757 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:55.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:55.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:55.890 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:55.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:55.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:55.969 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 97455 97439 85.765 40.9375 0.0169851 0.011652 2024-04-12T11:48:55.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:56.037 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:56.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:56.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:56.139 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:56.177 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:56.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:56.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:56.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-04-12T11:48:56.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:56.323 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:56.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:56.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:56.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:48:56.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:56.490 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:56.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:48:56.591 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:56.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:56.625 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:56.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:56.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:56.769 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:56.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:56.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:56.871 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:56.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:56.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:56.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:56.972 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 15 98046 98031 85.0842 37 0.042779 0.0117468 2024-04-12T11:48:56.999 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:57.027 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:57.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:57.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:57.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:57.177 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:57.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:57.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:57.207 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:48:57.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:57 smithi087 ceph-mon[89014]: pgmap v1939: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 34303/286305 objects degraded (11.981%); 11131/286305 objects misplaced (3.888%); 1.6 MiB/s, 25 objects/s recovering 2024-04-12T11:48:57.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:57 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:48:57.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:57 smithi029 ceph-mon[97787]: pgmap v1939: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 34303/286305 objects degraded (11.981%); 11131/286305 objects misplaced (3.888%); 1.6 MiB/s, 25 objects/s recovering 2024-04-12T11:48:57.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:57 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:48:57.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:57 smithi029 ceph-mon[93650]: pgmap v1939: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 521 op/s; 34303/286305 objects degraded (11.981%); 11131/286305 objects misplaced (3.888%); 1.6 MiB/s, 25 objects/s recovering 2024-04-12T11:48:57.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:57 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:48:57.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:57.318 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:57.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:57.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:57.420 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:57.442 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:48:57.479 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:57.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:57.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:57.581 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:57.601 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:48:57.608 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:57.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:57.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:57.623 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:48:57.711 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:57.740 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:57.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:57.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:57.811 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:48:57.841 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:57.856 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:48:57.884 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:57.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:57.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:57.900 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:48:57.945 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:48:57.972 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 98536 98520 84.3373 30.5625 0.0764277 0.0118507 2024-04-12T11:48:57.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:57.991 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:48:58.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:58.028 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:58.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:58.040 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:48:58.081 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:48:58.121 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:48:58.121 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:58 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/718993455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:58.121 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:58 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/718993455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:58.128 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:48:58.128 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:48:58.128 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:48:58.128 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:48:58.129 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:48:58.129 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:48:58.129 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:48:58.129 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:48:58.129 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:58.160 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:58.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:58.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:58.262 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:58.292 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:58.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:58.308 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:58 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/718993455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:48:58.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:58.434 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:58.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:58.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:58.535 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:48:58.555 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1939,"stamp":"2024-04-12T11:48:55.967071+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6253451218,"num_objects":95435,"num_object_clones":0,"num_object_copies":286305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34303,"num_objects_misplaced":11131,"num_objects_unfound":0,"num_objects_dirty":95435,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":95711,"num_write_kb":6108750,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":283,"num_bytes_recovered":18101836,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95627,"ondisk_log_size":95627,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":275,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22832292,"kb_used_data":18493464,"kb_used_omap":175,"kb_used_meta":4338576,"kb_avail":727128924,"statfs":{"total":767960285184,"available":744580018176,"internally_reserved":0,"allocated":18937307136,"data_stored":18916740672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":4442701993},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,3,6,6,5,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":329515008,"num_objects":5028,"num_object_clones":0,"num_object_copies":15084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34303,"num_objects_misplaced":11131,"num_objects_unfound":0,"num_objects_dirty":5028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5028,"num_write_kb":321792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":247,"num_bytes_recovered":16056410,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.634147"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1830,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.384780+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:48:51.384780+0000","last_peered":"2024-04-12T11:48:51.384780+0000","last_clean":"2024-04-12T11:48:51.384780+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:48:51.384780+0000","last_undegraded":"2024-04-12T11:48:51.384780+0000","last_fullsized":"2024-04-12T11:48:51.384780+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.409082+0000","last_change":"2024-04-12T11:48:45.352231+0000","last_active":"2024-04-12T11:48:51.409082+0000","last_peered":"2024-04-12T11:48:51.409082+0000","last_clean":"2024-04-12T11:48:51.409082+0000","last_became_active":"2024-04-12T11:48:45.351324+0000","last_became_peered":"2024-04-12T11:48:45.351324+0000","last_unstale":"2024-04-12T11:48:51.409082+0000","last_undegraded":"2024-04-12T11:48:51.409082+0000","last_fullsized":"2024-04-12T11:48:51.409082+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:01.615486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1914,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.391588+0000","last_change":"2024-04-12T11:48:45.353042+0000","last_active":"2024-04-12T11:48:51.391588+0000","last_peered":"2024-04-12T11:48:51.391588+0000","last_clean":"2024-04-12T11:48:51.391588+0000","last_became_active":"2024-04-12T11:48:45.352721+0000","last_became_peered":"2024-04-12T11:48:45.352721+0000","last_unstale":"2024-04-12T11:48:51.391588+0000","last_undegraded":"2024-04-12T11:48:51.391588+0000","last_fullsized":"2024-04-12T11:48:51.391588+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:00:32.520262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1932,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.336965+0000","last_change":"2024-04-12T11:48:45.353306+0000","last_active":"2024-04-12T11:48:46.336965+0000","last_peered":"2024-04-12T11:48:46.336965+0000","last_clean":"2024-04-12T11:48:46.336965+0000","last_became_active":"2024-04-12T11:48:45.352065+0000","last_became_peered":"2024-04-12T11:48:45.352065+0000","last_unstale":"2024-04-12T11:48:46.336965+0000","last_undegraded":"2024-04-12T11:48:46.336965+0000","last_fullsized":"2024-04-12T11:48:46.336965+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:26:36.201291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.406653+0000","last_change":"2024-04-12T11:48:45.351604+0000","last_active":"2024-04-12T11:48:51.406653+0000","last_peered":"2024-04-12T11:48:51.406653+0000","last_clean":"2024-04-12T11:48:51.406653+0000","last_became_active":"2024-04-12T11:48:45.349968+0000","last_became_peered":"2024-04-12T11:48:45.349968+0000","last_unstale":"2024-04-12T11:48:51.406653+0000","last_undegraded":"2024-04-12T11:48:51.406653+0000","last_fullsized":"2024-04-12T11:48:51.406653+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:56:42.419931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1186,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.375725+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:48:46.375725+0000","last_peered":"2024-04-12T11:48:46.375725+0000","last_clean":"2024-04-12T11:48:46.375725+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:48:46.375725+0000","last_undegraded":"2024-04-12T11:48:46.375725+0000","last_fullsized":"2024-04-12T11:48:46.375725+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1903,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.337199+0000","last_change":"2024-04-12T11:48:45.351162+0000","last_active":"2024-04-12T11:48:46.337199+0000","last_peered":"2024-04-12T11:48:46.337199+0000","last_clean":"2024-04-12T11:48:46.337199+0000","last_became_active":"2024-04-12T11:48:45.350817+0000","last_became_peered":"2024-04-12T11:48:45.350817+0000","last_unstale":"2024-04-12T11:48:46.337199+0000","last_undegraded":"2024-04-12T11:48:46.337199+0000","last_fullsized":"2024-04-12T11:48:46.337199+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:59.627216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.406807+0000","last_change":"2024-04-12T11:48:45.420195+0000","last_active":"2024-04-12T11:48:51.406807+0000","last_peered":"2024-04-12T11:48:51.406807+0000","last_clean":"2024-04-12T11:48:51.406807+0000","last_became_active":"2024-04-12T11:48:45.419810+0000","last_became_peered":"2024-04-12T11:48:45.419810+0000","last_unstale":"2024-04-12T11:48:51.406807+0000","last_undegraded":"2024-04-12T11:48:51.406807+0000","last_fullsized":"2024-04-12T11:48:51.406807+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:33:11.234641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1217,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.334913+0000","last_change":"2024-04-12T11:45:52.986340+0000","last_active":"2024-04-12T11:48:46.334913+0000","last_peered":"2024-04-12T11:48:46.334913+0000","last_clean":"2024-04-12T11:48:46.334913+0000","last_became_active":"2024-04-12T11:45:52.984190+0000","last_became_peered":"2024-04-12T11:45:52.984190+0000","last_unstale":"2024-04-12T11:48:46.334913+0000","last_undegraded":"2024-04-12T11:48:46.334913+0000","last_fullsized":"2024-04-12T11:48:46.334913+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:52:19.144894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.399516+0000","last_change":"2024-04-12T11:48:45.349940+0000","last_active":"2024-04-12T11:48:51.399516+0000","last_peered":"2024-04-12T11:48:51.399516+0000","last_clean":"2024-04-12T11:48:51.399516+0000","last_became_active":"2024-04-12T11:48:45.349603+0000","last_became_peered":"2024-04-12T11:48:45.349603+0000","last_unstale":"2024-04-12T11:48:51.399516+0000","last_undegraded":"2024-04-12T11:48:51.399516+0000","last_fullsized":"2024-04-12T11:48:51.399516+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:04:27.503011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":75,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.338467+0000","last_change":"2024-04-12T11:48:45.357689+0000","last_active":"2024-04-12T11:48:46.338467+0000","last_peered":"2024-04-12T11:48:46.338467+0000","last_clean":"2024-04-12T11:48:46.338467+0000","last_became_active":"2024-04-12T11:48:45.349500+0000","last_became_peered":"2024-04-12T11:48:45.349500+0000","last_unstale":"2024-04-12T11:48:46.338467+0000","last_undegraded":"2024-04-12T11:48:46.338467+0000","last_fullsized":"2024-04-12T11:48:46.338467+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:54:58.532082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1881,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.396405+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:48:51.396405+0000","last_peered":"2024-04-12T11:48:51.396405+0000","last_clean":"2024-04-12T11:48:51.396405+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:48:51.396405+0000","last_undegraded":"2024-04-12T11:48:51.396405+0000","last_fullsized":"2024-04-12T11:48:51.396405+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1834,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.387005+0000","last_change":"2024-04-12T11:45:53.024626+0000","last_active":"2024-04-12T11:48:51.387005+0000","last_peered":"2024-04-12T11:48:51.387005+0000","last_clean":"2024-04-12T11:48:51.387005+0000","last_became_active":"2024-04-12T11:45:52.976276+0000","last_became_peered":"2024-04-12T11:45:52.976276+0000","last_unstale":"2024-04-12T11:48:51.387005+0000","last_undegraded":"2024-04-12T11:48:51.387005+0000","last_fullsized":"2024-04-12T11:48:51.387005+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:49.929210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1793,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.385687+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:48:51.385687+0000","last_peered":"2024-04-12T11:48:51.385687+0000","last_clean":"2024-04-12T11:48:51.385687+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:48:51.385687+0000","last_undegraded":"2024-04-12T11:48:51.385687+0000","last_fullsized":"2024-04-12T11:48:51.385687+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":45,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.377842+0000","last_change":"2024-04-12T11:45:53.035604+0000","last_active":"2024-04-12T11:48:46.377842+0000","last_peered":"2024-04-12T11:48:46.377842+0000","last_clean":"2024-04-12T11:48:46.377842+0000","last_became_active":"2024-04-12T11:45:52.984722+0000","last_became_peered":"2024-04-12T11:45:52.984722+0000","last_unstale":"2024-04-12T11:48:46.377842+0000","last_undegraded":"2024-04-12T11:48:46.377842+0000","last_fullsized":"2024-04-12T11:48:46.377842+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:30.674395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.409294+0000","last_change":"2024-04-12T11:48:45.353499+0000","last_active":"2024-04-12T11:48:51.409294+0000","last_peered":"2024-04-12T11:48:51.409294+0000","last_clean":"2024-04-12T11:48:51.409294+0000","last_became_active":"2024-04-12T11:48:45.350015+0000","last_became_peered":"2024-04-12T11:48:45.350015+0000","last_unstale":"2024-04-12T11:48:51.409294+0000","last_undegraded":"2024-04-12T11:48:51.409294+0000","last_fullsized":"2024-04-12T11:48:51.409294+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:24:17.358128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1916,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.336602+0000","last_change":"2024-04-12T11:48:45.351398+0000","last_active":"2024-04-12T11:48:46.336602+0000","last_peered":"2024-04-12T11:48:46.336602+0000","last_clean":"2024-04-12T11:48:46.336602+0000","last_became_active":"2024-04-12T11:48:45.350886+0000","last_became_peered":"2024-04-12T11:48:45.350886+0000","last_unstale":"2024-04-12T11:48:46.336602+0000","last_undegraded":"2024-04-12T11:48:46.336602+0000","last_fullsized":"2024-04-12T11:48:46.336602+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:02.835400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1243,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.337417+0000","last_change":"2024-04-12T11:48:45.350844+0000","last_active":"2024-04-12T11:48:46.337417+0000","last_peered":"2024-04-12T11:48:46.337417+0000","last_clean":"2024-04-12T11:48:46.337417+0000","last_became_active":"2024-04-12T11:48:45.350024+0000","last_became_peered":"2024-04-12T11:48:45.350024+0000","last_unstale":"2024-04-12T11:48:46.337417+0000","last_undegraded":"2024-04-12T11:48:46.337417+0000","last_fullsized":"2024-04-12T11:48:46.337417+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:44:47.030599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":39,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.338149+0000","last_change":"2024-04-12T11:45:53.006808+0000","last_active":"2024-04-12T11:48:46.338149+0000","last_peered":"2024-04-12T11:48:46.338149+0000","last_clean":"2024-04-12T11:48:46.338149+0000","last_became_active":"2024-04-12T11:45:53.006480+0000","last_became_peered":"2024-04-12T11:45:53.006480+0000","last_unstale":"2024-04-12T11:48:46.338149+0000","last_undegraded":"2024-04-12T11:48:46.338149+0000","last_fullsized":"2024-04-12T11:48:46.338149+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:47.685096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1917,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.337066+0000","last_change":"2024-04-12T11:48:45.352862+0000","last_active":"2024-04-12T11:48:46.337066+0000","last_peered":"2024-04-12T11:48:46.337066+0000","last_clean":"2024-04-12T11:48:46.337066+0000","last_became_active":"2024-04-12T11:48:45.352485+0000","last_became_peered":"2024-04-12T11:48:45.352485+0000","last_unstale":"2024-04-12T11:48:46.337066+0000","last_undegraded":"2024-04-12T11:48:46.337066+0000","last_fullsized":"2024-04-12T11:48:46.337066+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:11:20.916845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1193,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.375490+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:48:46.375490+0000","last_peered":"2024-04-12T11:48:46.375490+0000","last_clean":"2024-04-12T11:48:46.375490+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:48:46.375490+0000","last_undegraded":"2024-04-12T11:48:46.375490+0000","last_fullsized":"2024-04-12T11:48:46.375490+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.400619+0000","last_change":"2024-04-12T11:48:45.361653+0000","last_active":"2024-04-12T11:48:51.400619+0000","last_peered":"2024-04-12T11:48:51.400619+0000","last_clean":"2024-04-12T11:48:51.400619+0000","last_became_active":"2024-04-12T11:48:45.361334+0000","last_became_peered":"2024-04-12T11:48:45.361334+0000","last_unstale":"2024-04-12T11:48:51.400619+0000","last_undegraded":"2024-04-12T11:48:51.400619+0000","last_fullsized":"2024-04-12T11:48:51.400619+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:58:25.548034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"1236'5984","reported_seq":6012,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.206606+0000","last_change":"2024-04-12T11:47:31.337856+0000","last_active":"2024-04-12T11:48:51.206606+0000","last_peered":"2024-04-12T11:48:51.206606+0000","last_clean":"2024-04-12T11:48:51.206606+0000","last_became_active":"2024-04-12T11:47:31.337539+0000","last_became_peered":"2024-04-12T11:47:31.337539+0000","last_unstale":"2024-04-12T11:48:51.206606+0000","last_undegraded":"2024-04-12T11:48:51.206606+0000","last_fullsized":"2024-04-12T11:48:51.206606+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5984,"log_dups_size":0,"ondisk_log_size":5984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:55:13.678253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392167424,"num_objects":5984,"num_object_clones":0,"num_object_copies":17952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5984,"num_write_kb":382976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":43,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.340014+0000","last_change":"2024-04-12T11:45:53.028640+0000","last_active":"2024-04-12T11:48:46.340014+0000","last_peered":"2024-04-12T11:48:46.340014+0000","last_clean":"2024-04-12T11:48:46.340014+0000","last_became_active":"2024-04-12T11:45:52.983099+0000","last_became_peered":"2024-04-12T11:45:52.983099+0000","last_unstale":"2024-04-12T11:48:46.340014+0000","last_undegraded":"2024-04-12T11:48:46.340014+0000","last_fullsized":"2024-04-12T11:48:46.340014+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:43:01.897014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1194,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.334235+0000","last_change":"2024-04-12T11:45:52.993160+0000","last_active":"2024-04-12T11:48:46.334235+0000","last_peered":"2024-04-12T11:48:46.334235+0000","last_clean":"2024-04-12T11:48:46.334235+0000","last_became_active":"2024-04-12T11:45:52.989310+0000","last_became_peered":"2024-04-12T11:45:52.989310+0000","last_unstale":"2024-04-12T11:48:46.334235+0000","last_undegraded":"2024-04-12T11:48:46.334235+0000","last_fullsized":"2024-04-12T11:48:46.334235+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:58:47.362374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1237'6154","reported_seq":6189,"reported_epoch":1237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:48:54.528375+0000","last_change":"2024-04-12T11:48:46.804539+0000","last_active":"2024-04-12T11:48:54.528375+0000","last_peered":"2024-04-12T11:48:54.528375+0000","last_clean":"2024-04-12T11:48:44.288786+0000","last_became_active":"2024-04-12T11:48:46.787738+0000","last_became_peered":"2024-04-12T11:48:46.787738+0000","last_unstale":"2024-04-12T11:48:54.528375+0000","last_undegraded":"2024-04-12T11:48:46.374623+0000","last_fullsized":"2024-04-12T11:48:46.337433+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6154,"log_dups_size":0,"ondisk_log_size":6154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403308544,"num_objects":6154,"num_object_clones":0,"num_object_copies":18462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5831,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6154,"num_write_kb":393856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":5831},{"shards":"1,4","objects":323}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":43,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.338980+0000","last_change":"2024-04-12T11:45:53.031759+0000","last_active":"2024-04-12T11:48:46.338980+0000","last_peered":"2024-04-12T11:48:46.338980+0000","last_clean":"2024-04-12T11:48:46.338980+0000","last_became_active":"2024-04-12T11:45:52.984545+0000","last_became_peered":"2024-04-12T11:45:52.984545+0000","last_unstale":"2024-04-12T11:48:46.338980+0000","last_undegraded":"2024-04-12T11:48:46.338980+0000","last_fullsized":"2024-04-12T11:48:46.338980+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:16:10.421992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1840,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.386724+0000","last_change":"2024-04-12T11:48:45.351213+0000","last_active":"2024-04-12T11:48:51.386724+0000","last_peered":"2024-04-12T11:48:51.386724+0000","last_clean":"2024-04-12T11:48:51.386724+0000","last_became_active":"2024-04-12T11:48:45.350772+0000","last_became_peered":"2024-04-12T11:48:45.350772+0000","last_unstale":"2024-04-12T11:48:51.386724+0000","last_undegraded":"2024-04-12T11:48:51.386724+0000","last_fullsized":"2024-04-12T11:48:51.386724+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:17:48.831705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1237'5980","reported_seq":6008,"reported_epoch":1237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:48:52.906020+0000","last_change":"2024-04-12T11:48:46.932869+0000","last_active":"2024-04-12T11:48:52.906020+0000","last_peered":"2024-04-12T11:48:52.906020+0000","last_clean":"2024-04-12T11:48:44.337474+0000","last_became_active":"2024-04-12T11:48:46.916781+0000","last_became_peered":"2024-04-12T11:48:46.916781+0000","last_unstale":"2024-04-12T11:48:52.906020+0000","last_undegraded":"2024-04-12T11:48:46.385475+0000","last_fullsized":"2024-04-12T11:48:46.335429+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5980,"log_dups_size":0,"ondisk_log_size":5980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:02:58.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391905280,"num_objects":5980,"num_object_clones":0,"num_object_copies":17940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5745,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5980,"num_write_kb":382720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":5745},{"shards":"5,7","objects":235}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":43,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.338296+0000","last_change":"2024-04-12T11:45:53.169686+0000","last_active":"2024-04-12T11:48:46.338296+0000","last_peered":"2024-04-12T11:48:46.338296+0000","last_clean":"2024-04-12T11:48:46.338296+0000","last_became_active":"2024-04-12T11:45:53.021021+0000","last_became_peered":"2024-04-12T11:45:53.021021+0000","last_unstale":"2024-04-12T11:48:46.338296+0000","last_undegraded":"2024-04-12T11:48:46.338296+0000","last_fullsized":"2024-04-12T11:48:46.338296+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:48:23.506812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1212,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.375656+0000","last_change":"2024-04-12T11:45:52.979601+0000","last_active":"2024-04-12T11:48:46.375656+0000","last_peered":"2024-04-12T11:48:46.375656+0000","last_clean":"2024-04-12T11:48:46.375656+0000","last_became_active":"2024-04-12T11:45:52.979307+0000","last_became_peered":"2024-04-12T11:45:52.979307+0000","last_unstale":"2024-04-12T11:48:46.375656+0000","last_undegraded":"2024-04-12T11:48:46.375656+0000","last_fullsized":"2024-04-12T11:48:46.375656+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:21.367107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1237'6122","reported_seq":6142,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:52.932142+0000","last_change":"2024-04-12T11:47:31.345659+0000","last_active":"2024-04-12T11:48:52.932142+0000","last_peered":"2024-04-12T11:48:52.932142+0000","last_clean":"2024-04-12T11:48:52.932142+0000","last_became_active":"2024-04-12T11:47:31.345313+0000","last_became_peered":"2024-04-12T11:47:31.345313+0000","last_unstale":"2024-04-12T11:48:52.932142+0000","last_undegraded":"2024-04-12T11:48:52.932142+0000","last_fullsized":"2024-04-12T11:48:52.932142+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6122,"log_dups_size":0,"ondisk_log_size":6122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:03:10.526704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401211392,"num_objects":6122,"num_object_clones":0,"num_object_copies":18366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6122,"num_write_kb":391808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1227,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.334673+0000","last_change":"2024-04-12T11:48:45.344961+0000","last_active":"2024-04-12T11:48:46.334673+0000","last_peered":"2024-04-12T11:48:46.334673+0000","last_clean":"2024-04-12T11:48:46.334673+0000","last_became_active":"2024-04-12T11:48:45.344344+0000","last_became_peered":"2024-04-12T11:48:45.344344+0000","last_unstale":"2024-04-12T11:48:46.334673+0000","last_undegraded":"2024-04-12T11:48:46.334673+0000","last_fullsized":"2024-04-12T11:48:46.334673+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:47:55.659541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1890,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.337361+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:48:46.337361+0000","last_peered":"2024-04-12T11:48:46.337361+0000","last_clean":"2024-04-12T11:48:46.337361+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:48:46.337361+0000","last_undegraded":"2024-04-12T11:48:46.337361+0000","last_fullsized":"2024-04-12T11:48:46.337361+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"1237'6069","reported_seq":6585,"reported_epoch":1237,"state":"active+recovering+remapped","last_fresh":"2024-04-12T11:48:55.238922+0000","last_change":"2024-04-12T11:48:46.724770+0000","last_active":"2024-04-12T11:48:55.238922+0000","last_peered":"2024-04-12T11:48:55.238922+0000","last_clean":"2024-04-12T11:48:44.315992+0000","last_became_active":"2024-04-12T11:48:46.712972+0000","last_became_peered":"2024-04-12T11:48:46.712972+0000","last_unstale":"2024-04-12T11:48:55.238922+0000","last_undegraded":"2024-04-12T11:48:55.238922+0000","last_fullsized":"2024-04-12T11:48:55.238922+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6069,"log_dups_size":0,"ondisk_log_size":6069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397737984,"num_objects":6069,"num_object_clones":0,"num_object_copies":18207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5520,"num_objects_unfound":0,"num_objects_dirty":6069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6069,"num_write_kb":388416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":244,"num_bytes_recovered":15990784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[4,6,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":549}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":19,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.392682+0000","last_change":"2024-04-12T11:48:45.359254+0000","last_active":"2024-04-12T11:48:51.392682+0000","last_peered":"2024-04-12T11:48:51.392682+0000","last_clean":"2024-04-12T11:48:51.392682+0000","last_became_active":"2024-04-12T11:48:45.348788+0000","last_became_peered":"2024-04-12T11:48:45.348788+0000","last_unstale":"2024-04-12T11:48:51.392682+0000","last_undegraded":"2024-04-12T11:48:51.392682+0000","last_fullsized":"2024-04-12T11:48:51.392682+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:15:04.501653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1831,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.387086+0000","last_change":"2024-04-12T11:48:45.351839+0000","last_active":"2024-04-12T11:48:51.387086+0000","last_peered":"2024-04-12T11:48:51.387086+0000","last_clean":"2024-04-12T11:48:51.387086+0000","last_became_active":"2024-04-12T11:48:45.351276+0000","last_became_peered":"2024-04-12T11:48:45.351276+0000","last_unstale":"2024-04-12T11:48:51.387086+0000","last_undegraded":"2024-04-12T11:48:51.387086+0000","last_fullsized":"2024-04-12T11:48:51.387086+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:23:03.106998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1237'6055","reported_seq":6087,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:54.553097+0000","last_change":"2024-04-12T11:47:31.345847+0000","last_active":"2024-04-12T11:48:54.553097+0000","last_peered":"2024-04-12T11:48:54.553097+0000","last_clean":"2024-04-12T11:48:54.553097+0000","last_became_active":"2024-04-12T11:47:31.345534+0000","last_became_peered":"2024-04-12T11:47:31.345534+0000","last_unstale":"2024-04-12T11:48:54.553097+0000","last_undegraded":"2024-04-12T11:48:54.553097+0000","last_fullsized":"2024-04-12T11:48:54.553097+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6055,"log_dups_size":0,"ondisk_log_size":6055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:30:47.116045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396820480,"num_objects":6055,"num_object_clones":0,"num_object_copies":18165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6055,"num_write_kb":387520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":833,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.384958+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:48:51.384958+0000","last_peered":"2024-04-12T11:48:51.384958+0000","last_clean":"2024-04-12T11:48:51.384958+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:48:51.384958+0000","last_undegraded":"2024-04-12T11:48:51.384958+0000","last_fullsized":"2024-04-12T11:48:51.384958+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1829,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.385595+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:48:51.385595+0000","last_peered":"2024-04-12T11:48:51.385595+0000","last_clean":"2024-04-12T11:48:51.385595+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:48:51.385595+0000","last_undegraded":"2024-04-12T11:48:51.385595+0000","last_fullsized":"2024-04-12T11:48:51.385595+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1237'5866","reported_seq":5893,"reported_epoch":1237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:48:52.264735+0000","last_change":"2024-04-12T11:48:46.758392+0000","last_active":"2024-04-12T11:48:52.264735+0000","last_peered":"2024-04-12T11:48:52.264735+0000","last_clean":"2024-04-12T11:48:44.327141+0000","last_became_active":"2024-04-12T11:48:46.750040+0000","last_became_peered":"2024-04-12T11:48:46.750040+0000","last_unstale":"2024-04-12T11:48:52.264735+0000","last_undegraded":"2024-04-12T11:48:46.373778+0000","last_fullsized":"2024-04-12T11:48:46.339427+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5866,"log_dups_size":0,"ondisk_log_size":5866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:54:20.338800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384434176,"num_objects":5866,"num_object_clones":0,"num_object_copies":17598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5634,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5866,"num_write_kb":375424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,1],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":5634},{"shards":"1,7","objects":232}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1827,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.386821+0000","last_change":"2024-04-12T11:45:52.976139+0000","last_active":"2024-04-12T11:48:51.386821+0000","last_peered":"2024-04-12T11:48:51.386821+0000","last_clean":"2024-04-12T11:48:51.386821+0000","last_became_active":"2024-04-12T11:45:52.975803+0000","last_became_peered":"2024-04-12T11:45:52.975803+0000","last_unstale":"2024-04-12T11:48:51.386821+0000","last_undegraded":"2024-04-12T11:48:51.386821+0000","last_fullsized":"2024-04-12T11:48:51.386821+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:46.618143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1794,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.386354+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:48:51.386354+0000","last_peered":"2024-04-12T11:48:51.386354+0000","last_clean":"2024-04-12T11:48:51.386354+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:48:51.386354+0000","last_undegraded":"2024-04-12T11:48:51.386354+0000","last_fullsized":"2024-04-12T11:48:51.386354+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1237'5958","reported_seq":5992,"reported_epoch":1237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:48:54.577358+0000","last_change":"2024-04-12T11:48:46.765801+0000","last_active":"2024-04-12T11:48:54.577358+0000","last_peered":"2024-04-12T11:48:54.577358+0000","last_clean":"2024-04-12T11:48:44.392445+0000","last_became_active":"2024-04-12T11:48:46.751638+0000","last_became_peered":"2024-04-12T11:48:46.751638+0000","last_unstale":"2024-04-12T11:48:54.577358+0000","last_undegraded":"2024-04-12T11:48:46.375433+0000","last_fullsized":"2024-04-12T11:48:46.336636+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5958,"log_dups_size":0,"ondisk_log_size":5958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390463488,"num_objects":5958,"num_object_clones":0,"num_object_copies":17874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5646,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5958,"num_write_kb":381312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2","objects":312},{"shards":"1,2,7","objects":5646}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.406885+0000","last_change":"2024-04-12T11:48:45.351852+0000","last_active":"2024-04-12T11:48:51.406885+0000","last_peered":"2024-04-12T11:48:51.406885+0000","last_clean":"2024-04-12T11:48:51.406885+0000","last_became_active":"2024-04-12T11:48:45.350286+0000","last_became_peered":"2024-04-12T11:48:45.350286+0000","last_unstale":"2024-04-12T11:48:51.406885+0000","last_undegraded":"2024-04-12T11:48:51.406885+0000","last_fullsized":"2024-04-12T11:48:51.406885+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:51:26.610188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1920,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.392185+0000","last_change":"2024-04-12T11:48:45.361378+0000","last_active":"2024-04-12T11:48:51.392185+0000","last_peered":"2024-04-12T11:48:51.392185+0000","last_clean":"2024-04-12T11:48:51.392185+0000","last_became_active":"2024-04-12T11:48:45.360917+0000","last_became_peered":"2024-04-12T11:48:45.360917+0000","last_unstale":"2024-04-12T11:48:51.392185+0000","last_undegraded":"2024-04-12T11:48:51.392185+0000","last_fullsized":"2024-04-12T11:48:51.392185+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:46:07.334663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"1237'5969","reported_seq":5998,"reported_epoch":1237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:48:52.979972+0000","last_change":"2024-04-12T11:48:46.932836+0000","last_active":"2024-04-12T11:48:52.979972+0000","last_peered":"2024-04-12T11:48:52.979972+0000","last_clean":"2024-04-12T11:48:44.299270+0000","last_became_active":"2024-04-12T11:48:46.917768+0000","last_became_peered":"2024-04-12T11:48:46.917768+0000","last_unstale":"2024-04-12T11:48:52.979972+0000","last_undegraded":"2024-04-12T11:48:46.478160+0000","last_fullsized":"2024-04-12T11:48:46.335134+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5969,"log_dups_size":0,"ondisk_log_size":5969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:00:52.272820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391184384,"num_objects":5969,"num_object_clones":0,"num_object_copies":17907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5721,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5969,"num_write_kb":382016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":5721},{"shards":"1,5","objects":248}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1904,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.399838+0000","last_change":"2024-04-12T11:45:52.980673+0000","last_active":"2024-04-12T11:48:51.399838+0000","last_peered":"2024-04-12T11:48:51.399838+0000","last_clean":"2024-04-12T11:48:51.399838+0000","last_became_active":"2024-04-12T11:45:52.978978+0000","last_became_peered":"2024-04-12T11:45:52.978978+0000","last_unstale":"2024-04-12T11:48:51.399838+0000","last_undegraded":"2024-04-12T11:48:51.399838+0000","last_fullsized":"2024-04-12T11:48:51.399838+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:32:10.303905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1244,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.336645+0000","last_change":"2024-04-12T11:48:45.419824+0000","last_active":"2024-04-12T11:48:46.336645+0000","last_peered":"2024-04-12T11:48:46.336645+0000","last_clean":"2024-04-12T11:48:46.336645+0000","last_became_active":"2024-04-12T11:48:45.419375+0000","last_became_peered":"2024-04-12T11:48:45.419375+0000","last_unstale":"2024-04-12T11:48:46.336645+0000","last_undegraded":"2024-04-12T11:48:46.336645+0000","last_fullsized":"2024-04-12T11:48:46.336645+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:19:30.300350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1883,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.396321+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:48:51.396321+0000","last_peered":"2024-04-12T11:48:51.396321+0000","last_clean":"2024-04-12T11:48:51.396321+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:48:51.396321+0000","last_undegraded":"2024-04-12T11:48:51.396321+0000","last_fullsized":"2024-04-12T11:48:51.396321+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"1236'5858","reported_seq":5888,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.155157+0000","last_change":"2024-04-12T11:47:31.337684+0000","last_active":"2024-04-12T11:48:51.155157+0000","last_peered":"2024-04-12T11:48:51.155157+0000","last_clean":"2024-04-12T11:48:51.155157+0000","last_became_active":"2024-04-12T11:47:31.337391+0000","last_became_peered":"2024-04-12T11:47:31.337391+0000","last_unstale":"2024-04-12T11:48:51.155157+0000","last_undegraded":"2024-04-12T11:48:51.155157+0000","last_fullsized":"2024-04-12T11:48:51.155157+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5858,"log_dups_size":0,"ondisk_log_size":5858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:11:18.213572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383909888,"num_objects":5858,"num_object_clones":0,"num_object_copies":17574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5858,"num_write_kb":374912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1197,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.334369+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:48:46.334369+0000","last_peered":"2024-04-12T11:48:46.334369+0000","last_clean":"2024-04-12T11:48:46.334369+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:48:46.334369+0000","last_undegraded":"2024-04-12T11:48:46.334369+0000","last_fullsized":"2024-04-12T11:48:46.334369+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1816,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.384780+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:48:51.384780+0000","last_peered":"2024-04-12T11:48:51.384780+0000","last_clean":"2024-04-12T11:48:51.384780+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:48:51.384780+0000","last_undegraded":"2024-04-12T11:48:51.384780+0000","last_fullsized":"2024-04-12T11:48:51.384780+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":125,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.388478+0000","last_change":"2024-04-12T11:48:45.418653+0000","last_active":"2024-04-12T11:48:46.388478+0000","last_peered":"2024-04-12T11:48:46.388478+0000","last_clean":"2024-04-12T11:48:46.388478+0000","last_became_active":"2024-04-12T11:48:45.418274+0000","last_became_peered":"2024-04-12T11:48:45.418274+0000","last_unstale":"2024-04-12T11:48:46.388478+0000","last_undegraded":"2024-04-12T11:48:46.388478+0000","last_fullsized":"2024-04-12T11:48:46.388478+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:39:24.012667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"1237'6020","reported_seq":6040,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:52.991617+0000","last_change":"2024-04-12T11:47:31.341656+0000","last_active":"2024-04-12T11:48:52.991617+0000","last_peered":"2024-04-12T11:48:52.991617+0000","last_clean":"2024-04-12T11:48:52.991617+0000","last_became_active":"2024-04-12T11:47:31.341335+0000","last_became_peered":"2024-04-12T11:47:31.341335+0000","last_unstale":"2024-04-12T11:48:52.991617+0000","last_undegraded":"2024-04-12T11:48:52.991617+0000","last_fullsized":"2024-04-12T11:48:52.991617+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6020,"log_dups_size":0,"ondisk_log_size":6020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:16:26.010127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394526720,"num_objects":6020,"num_object_clones":0,"num_object_copies":18060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6020,"num_write_kb":385280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1880,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.416638+0000","last_change":"2024-04-12T11:48:45.417421+0000","last_active":"2024-04-12T11:48:51.416638+0000","last_peered":"2024-04-12T11:48:51.416638+0000","last_clean":"2024-04-12T11:48:51.416638+0000","last_became_active":"2024-04-12T11:48:45.417046+0000","last_became_peered":"2024-04-12T11:48:45.417046+0000","last_unstale":"2024-04-12T11:48:51.416638+0000","last_undegraded":"2024-04-12T11:48:51.416638+0000","last_fullsized":"2024-04-12T11:48:51.416638+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:17:29.993647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1884,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.394680+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:48:51.394680+0000","last_peered":"2024-04-12T11:48:51.394680+0000","last_clean":"2024-04-12T11:48:51.394680+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:48:51.394680+0000","last_undegraded":"2024-04-12T11:48:51.394680+0000","last_fullsized":"2024-04-12T11:48:51.394680+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1886,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.392083+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:48:51.392083+0000","last_peered":"2024-04-12T11:48:51.392083+0000","last_clean":"2024-04-12T11:48:51.392083+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:48:51.392083+0000","last_undegraded":"2024-04-12T11:48:51.392083+0000","last_fullsized":"2024-04-12T11:48:51.392083+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1236'5899","reported_seq":5931,"reported_epoch":1236,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:48:51.098910+0000","last_change":"2024-04-12T11:48:46.811757+0000","last_active":"2024-04-12T11:48:51.098910+0000","last_peered":"2024-04-12T11:48:51.098910+0000","last_clean":"2024-04-12T11:48:44.316482+0000","last_became_active":"2024-04-12T11:48:46.789368+0000","last_became_peered":"2024-04-12T11:48:46.789368+0000","last_unstale":"2024-04-12T11:48:51.098910+0000","last_undegraded":"2024-04-12T11:48:46.382926+0000","last_fullsized":"2024-04-12T11:48:46.336397+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5899,"log_dups_size":0,"ondisk_log_size":5899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386596864,"num_objects":5899,"num_object_clones":0,"num_object_copies":17697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5726,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5899,"num_write_kb":377536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":173},{"shards":"0,5,7","objects":5726}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1224,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.334619+0000","last_change":"2024-04-12T11:45:52.986504+0000","last_active":"2024-04-12T11:48:46.334619+0000","last_peered":"2024-04-12T11:48:46.334619+0000","last_clean":"2024-04-12T11:48:46.334619+0000","last_became_active":"2024-04-12T11:45:52.984346+0000","last_became_peered":"2024-04-12T11:45:52.984346+0000","last_unstale":"2024-04-12T11:48:46.334619+0000","last_undegraded":"2024-04-12T11:48:46.334619+0000","last_fullsized":"2024-04-12T11:48:46.334619+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:26.859024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1544,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.338981+0000","last_change":"2024-04-12T11:45:53.970732+0000","last_active":"2024-04-12T11:48:46.338981+0000","last_peered":"2024-04-12T11:48:46.338981+0000","last_clean":"2024-04-12T11:48:46.338981+0000","last_became_active":"2024-04-12T11:45:53.970305+0000","last_became_peered":"2024-04-12T11:45:53.970305+0000","last_unstale":"2024-04-12T11:48:46.338981+0000","last_undegraded":"2024-04-12T11:48:46.338981+0000","last_fullsized":"2024-04-12T11:48:46.338981+0000","mapping_epoch":1210,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:42:56.998830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.405505+0000","last_change":"2024-04-12T11:48:45.352874+0000","last_active":"2024-04-12T11:48:51.405505+0000","last_peered":"2024-04-12T11:48:51.405505+0000","last_clean":"2024-04-12T11:48:51.405505+0000","last_became_active":"2024-04-12T11:48:45.352350+0000","last_became_peered":"2024-04-12T11:48:45.352350+0000","last_unstale":"2024-04-12T11:48:51.405505+0000","last_undegraded":"2024-04-12T11:48:51.405505+0000","last_fullsized":"2024-04-12T11:48:51.405505+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:44:46.925487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1204,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.375821+0000","last_change":"2024-04-12T11:45:53.016616+0000","last_active":"2024-04-12T11:48:46.375821+0000","last_peered":"2024-04-12T11:48:46.375821+0000","last_clean":"2024-04-12T11:48:46.375821+0000","last_became_active":"2024-04-12T11:45:53.016292+0000","last_became_peered":"2024-04-12T11:45:53.016292+0000","last_unstale":"2024-04-12T11:48:46.375821+0000","last_undegraded":"2024-04-12T11:48:46.375821+0000","last_fullsized":"2024-04-12T11:48:46.375821+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:53:29.382293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1236'5766","reported_seq":5796,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.204498+0000","last_change":"2024-04-12T11:47:31.337491+0000","last_active":"2024-04-12T11:48:51.204498+0000","last_peered":"2024-04-12T11:48:51.204498+0000","last_clean":"2024-04-12T11:48:51.204498+0000","last_became_active":"2024-04-12T11:47:31.337137+0000","last_became_peered":"2024-04-12T11:47:31.337137+0000","last_unstale":"2024-04-12T11:48:51.204498+0000","last_undegraded":"2024-04-12T11:48:51.204498+0000","last_fullsized":"2024-04-12T11:48:51.204498+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5766,"log_dups_size":0,"ondisk_log_size":5766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:21:25.556372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377880576,"num_objects":5766,"num_object_clones":0,"num_object_copies":17298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5766,"num_write_kb":369024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":39,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.338874+0000","last_change":"2024-04-12T11:45:52.987902+0000","last_active":"2024-04-12T11:48:46.338874+0000","last_peered":"2024-04-12T11:48:46.338874+0000","last_clean":"2024-04-12T11:48:46.338874+0000","last_became_active":"2024-04-12T11:45:52.980281+0000","last_became_peered":"2024-04-12T11:45:52.980281+0000","last_unstale":"2024-04-12T11:48:46.338874+0000","last_undegraded":"2024-04-12T11:48:46.338874+0000","last_fullsized":"2024-04-12T11:48:46.338874+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:07:12.203048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1915,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.336622+0000","last_change":"2024-04-12T11:48:45.351359+0000","last_active":"2024-04-12T11:48:46.336622+0000","last_peered":"2024-04-12T11:48:46.336622+0000","last_clean":"2024-04-12T11:48:46.336622+0000","last_became_active":"2024-04-12T11:48:45.350809+0000","last_became_peered":"2024-04-12T11:48:45.350809+0000","last_unstale":"2024-04-12T11:48:46.336622+0000","last_undegraded":"2024-04-12T11:48:46.336622+0000","last_fullsized":"2024-04-12T11:48:46.336622+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:58:36.906714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1891,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.337193+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:48:46.337193+0000","last_peered":"2024-04-12T11:48:46.337193+0000","last_clean":"2024-04-12T11:48:46.337193+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:48:46.337193+0000","last_undegraded":"2024-04-12T11:48:46.337193+0000","last_fullsized":"2024-04-12T11:48:46.337193+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"1237'5922","reported_seq":5942,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:52.246313+0000","last_change":"2024-04-12T11:47:31.338497+0000","last_active":"2024-04-12T11:48:52.246313+0000","last_peered":"2024-04-12T11:48:52.246313+0000","last_clean":"2024-04-12T11:48:52.246313+0000","last_became_active":"2024-04-12T11:47:31.338160+0000","last_became_peered":"2024-04-12T11:47:31.338160+0000","last_unstale":"2024-04-12T11:48:52.246313+0000","last_undegraded":"2024-04-12T11:48:52.246313+0000","last_fullsized":"2024-04-12T11:48:52.246313+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5922,"log_dups_size":0,"ondisk_log_size":5922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:22:37.585891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388104192,"num_objects":5922,"num_object_clones":0,"num_object_copies":17766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5922,"num_write_kb":379008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":39,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.338381+0000","last_change":"2024-04-12T11:45:53.005585+0000","last_active":"2024-04-12T11:48:46.338381+0000","last_peered":"2024-04-12T11:48:46.338381+0000","last_clean":"2024-04-12T11:48:46.338381+0000","last_became_active":"2024-04-12T11:45:52.987880+0000","last_became_peered":"2024-04-12T11:45:52.987880+0000","last_unstale":"2024-04-12T11:48:46.338381+0000","last_undegraded":"2024-04-12T11:48:46.338381+0000","last_fullsized":"2024-04-12T11:48:46.338381+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:09.819861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1883,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.399753+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:48:51.399753+0000","last_peered":"2024-04-12T11:48:51.399753+0000","last_clean":"2024-04-12T11:48:51.399753+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:48:51.399753+0000","last_undegraded":"2024-04-12T11:48:51.399753+0000","last_fullsized":"2024-04-12T11:48:51.399753+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1886,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.337235+0000","last_change":"2024-04-12T11:45:52.982299+0000","last_active":"2024-04-12T11:48:46.337235+0000","last_peered":"2024-04-12T11:48:46.337235+0000","last_clean":"2024-04-12T11:48:46.337235+0000","last_became_active":"2024-04-12T11:45:52.981942+0000","last_became_peered":"2024-04-12T11:45:52.981942+0000","last_unstale":"2024-04-12T11:48:46.337235+0000","last_undegraded":"2024-04-12T11:48:46.337235+0000","last_fullsized":"2024-04-12T11:48:46.337235+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:18:13.711200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"1237'5998","reported_seq":6018,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:52.233385+0000","last_change":"2024-04-12T11:47:31.339988+0000","last_active":"2024-04-12T11:48:52.233385+0000","last_peered":"2024-04-12T11:48:52.233385+0000","last_clean":"2024-04-12T11:48:52.233385+0000","last_became_active":"2024-04-12T11:47:31.339650+0000","last_became_peered":"2024-04-12T11:47:31.339650+0000","last_unstale":"2024-04-12T11:48:52.233385+0000","last_undegraded":"2024-04-12T11:48:52.233385+0000","last_fullsized":"2024-04-12T11:48:52.233385+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5998,"log_dups_size":0,"ondisk_log_size":5998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:02:57.059787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393084928,"num_objects":5998,"num_object_clones":0,"num_object_copies":17994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5998,"num_write_kb":383872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":19,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.400923+0000","last_change":"2024-04-12T11:48:45.361376+0000","last_active":"2024-04-12T11:48:51.400923+0000","last_peered":"2024-04-12T11:48:51.400923+0000","last_clean":"2024-04-12T11:48:51.400923+0000","last_became_active":"2024-04-12T11:48:45.354494+0000","last_became_peered":"2024-04-12T11:48:45.354494+0000","last_unstale":"2024-04-12T11:48:51.400923+0000","last_undegraded":"2024-04-12T11:48:51.400923+0000","last_fullsized":"2024-04-12T11:48:51.400923+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:42:08.500124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1883,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.392115+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:48:51.392115+0000","last_peered":"2024-04-12T11:48:51.392115+0000","last_clean":"2024-04-12T11:48:51.392115+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:48:51.392115+0000","last_undegraded":"2024-04-12T11:48:51.392115+0000","last_fullsized":"2024-04-12T11:48:51.392115+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.406559+0000","last_change":"2024-04-12T11:48:45.349034+0000","last_active":"2024-04-12T11:48:51.406559+0000","last_peered":"2024-04-12T11:48:51.406559+0000","last_clean":"2024-04-12T11:48:51.406559+0000","last_became_active":"2024-04-12T11:48:45.348579+0000","last_became_peered":"2024-04-12T11:48:45.348579+0000","last_unstale":"2024-04-12T11:48:51.406559+0000","last_undegraded":"2024-04-12T11:48:51.406559+0000","last_fullsized":"2024-04-12T11:48:51.406559+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:57.578035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"1236'5791","reported_seq":5824,"reported_epoch":1236,"state":"active+recovery_wait+remapped","last_fresh":"2024-04-12T11:48:51.158488+0000","last_change":"2024-04-12T11:48:46.816570+0000","last_active":"2024-04-12T11:48:51.158488+0000","last_peered":"2024-04-12T11:48:51.158488+0000","last_clean":"2024-04-12T11:48:44.312876+0000","last_became_active":"2024-04-12T11:48:46.789195+0000","last_became_peered":"2024-04-12T11:48:46.789195+0000","last_unstale":"2024-04-12T11:48:51.158488+0000","last_undegraded":"2024-04-12T11:48:51.158488+0000","last_fullsized":"2024-04-12T11:48:51.158488+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5791,"log_dups_size":0,"ondisk_log_size":5791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379518976,"num_objects":5791,"num_object_clones":0,"num_object_copies":17373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5611,"num_objects_unfound":0,"num_objects_dirty":5791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5791,"num_write_kb":370624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":180}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1237,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.375882+0000","last_change":"2024-04-12T11:45:52.979745+0000","last_active":"2024-04-12T11:48:46.375882+0000","last_peered":"2024-04-12T11:48:46.375882+0000","last_clean":"2024-04-12T11:48:46.375882+0000","last_became_active":"2024-04-12T11:45:52.978936+0000","last_became_peered":"2024-04-12T11:45:52.978936+0000","last_unstale":"2024-04-12T11:48:46.375882+0000","last_undegraded":"2024-04-12T11:48:46.375882+0000","last_fullsized":"2024-04-12T11:48:46.375882+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:22.710430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1894,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.336793+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:48:46.336793+0000","last_peered":"2024-04-12T11:48:46.336793+0000","last_clean":"2024-04-12T11:48:46.336793+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:48:46.336793+0000","last_undegraded":"2024-04-12T11:48:46.336793+0000","last_fullsized":"2024-04-12T11:48:46.336793+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1803,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.386464+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:48:51.386464+0000","last_peered":"2024-04-12T11:48:51.386464+0000","last_clean":"2024-04-12T11:48:51.386464+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:48:51.386464+0000","last_undegraded":"2024-04-12T11:48:51.386464+0000","last_fullsized":"2024-04-12T11:48:51.386464+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1895,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.337078+0000","last_change":"2024-04-12T11:45:52.979689+0000","last_active":"2024-04-12T11:48:46.337078+0000","last_peered":"2024-04-12T11:48:46.337078+0000","last_clean":"2024-04-12T11:48:46.337078+0000","last_became_active":"2024-04-12T11:45:52.979286+0000","last_became_peered":"2024-04-12T11:45:52.979286+0000","last_unstale":"2024-04-12T11:48:46.337078+0000","last_undegraded":"2024-04-12T11:48:46.337078+0000","last_fullsized":"2024-04-12T11:48:46.337078+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:10.453472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":164,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.334770+0000","last_change":"2024-04-12T11:48:45.344748+0000","last_active":"2024-04-12T11:48:46.334770+0000","last_peered":"2024-04-12T11:48:46.334770+0000","last_clean":"2024-04-12T11:48:46.334770+0000","last_became_active":"2024-04-12T11:48:45.344290+0000","last_became_peered":"2024-04-12T11:48:45.344290+0000","last_unstale":"2024-04-12T11:48:46.334770+0000","last_undegraded":"2024-04-12T11:48:46.334770+0000","last_fullsized":"2024-04-12T11:48:46.334770+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:13.440125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1207,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.334407+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:48:46.334407+0000","last_peered":"2024-04-12T11:48:46.334407+0000","last_clean":"2024-04-12T11:48:46.334407+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:48:46.334407+0000","last_undegraded":"2024-04-12T11:48:46.334407+0000","last_fullsized":"2024-04-12T11:48:46.334407+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1854,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.385827+0000","last_change":"2024-04-12T11:48:45.419685+0000","last_active":"2024-04-12T11:48:51.385827+0000","last_peered":"2024-04-12T11:48:51.385827+0000","last_clean":"2024-04-12T11:48:51.385827+0000","last_became_active":"2024-04-12T11:48:45.419344+0000","last_became_peered":"2024-04-12T11:48:45.419344+0000","last_unstale":"2024-04-12T11:48:51.385827+0000","last_undegraded":"2024-04-12T11:48:51.385827+0000","last_fullsized":"2024-04-12T11:48:51.385827+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:24:01.383355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1192,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.336760+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:48:46.336760+0000","last_peered":"2024-04-12T11:48:46.336760+0000","last_clean":"2024-04-12T11:48:46.336760+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:48:46.336760+0000","last_undegraded":"2024-04-12T11:48:46.336760+0000","last_fullsized":"2024-04-12T11:48:46.336760+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1845,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.385030+0000","last_change":"2024-04-12T11:48:45.351004+0000","last_active":"2024-04-12T11:48:51.385030+0000","last_peered":"2024-04-12T11:48:51.385030+0000","last_clean":"2024-04-12T11:48:51.385030+0000","last_became_active":"2024-04-12T11:48:45.350567+0000","last_became_peered":"2024-04-12T11:48:45.350567+0000","last_unstale":"2024-04-12T11:48:51.385030+0000","last_undegraded":"2024-04-12T11:48:51.385030+0000","last_fullsized":"2024-04-12T11:48:51.385030+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:48:43.488302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.408162+0000","last_change":"2024-04-12T11:48:45.418133+0000","last_active":"2024-04-12T11:48:51.408162+0000","last_peered":"2024-04-12T11:48:51.408162+0000","last_clean":"2024-04-12T11:48:51.408162+0000","last_became_active":"2024-04-12T11:48:45.417690+0000","last_became_peered":"2024-04-12T11:48:45.417690+0000","last_unstale":"2024-04-12T11:48:51.408162+0000","last_undegraded":"2024-04-12T11:48:51.408162+0000","last_fullsized":"2024-04-12T11:48:51.408162+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:09:21.940487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.399417+0000","last_change":"2024-04-12T11:48:45.418845+0000","last_active":"2024-04-12T11:48:51.399417+0000","last_peered":"2024-04-12T11:48:51.399417+0000","last_clean":"2024-04-12T11:48:51.399417+0000","last_became_active":"2024-04-12T11:48:45.417891+0000","last_became_peered":"2024-04-12T11:48:45.417891+0000","last_unstale":"2024-04-12T11:48:51.399417+0000","last_undegraded":"2024-04-12T11:48:51.399417+0000","last_fullsized":"2024-04-12T11:48:51.399417+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:12:05.184852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1924,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-04-12T11:48:46.337318+0000","last_change":"2024-04-12T11:43:26.370162+0000","last_active":"2024-04-12T11:48:46.337318+0000","last_peered":"2024-04-12T11:48:46.337318+0000","last_clean":"2024-04-12T11:48:46.337318+0000","last_became_active":"2024-04-12T11:43:26.352240+0000","last_became_peered":"2024-04-12T11:43:26.352240+0000","last_unstale":"2024-04-12T11:48:46.337318+0000","last_undegraded":"2024-04-12T11:48:46.337318+0000","last_fullsized":"2024-04-12T11:48:46.337318+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:34:09.364519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1793,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-04-12T11:48:51.385209+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:48:51.385209+0000","last_peered":"2024-04-12T11:48:51.385209+0000","last_clean":"2024-04-12T11:48:51.385209+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:48:51.385209+0000","last_undegraded":"2024-04-12T11:48:51.385209+0000","last_fullsized":"2024-04-12T11:48:51.385209+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":11724,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6252855296,"num_objects":95411,"num_object_clones":0,"num_object_copies":286233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34303,"num_objects_misplaced":11131,"num_objects_unfound":0,"num_objects_dirty":95411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":95411,"num_write_kb":6106304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":245,"num_bytes_recovered":16056320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18932629504,"data_stored":18932629504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95411,"ondisk_log_size":95411,"up":48,"acting":42,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234438,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4697876,"kb_used_data":4143432,"kb_used_omap":1,"kb_used_meta":554430,"kb_avail":89047276,"statfs":{"total":95995035648,"available":91184410624,"internally_reserved":0,"allocated":4242874368,"data_stored":4240532962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":567736779},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496075,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1547980,"kb_used_data":1137920,"kb_used_omap":1,"kb_used_meta":410046,"kb_avail":92197172,"statfs":{"total":95995035648,"available":94409904128,"internally_reserved":0,"allocated":1165230080,"data_stored":1162897302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":419887560},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536385,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3999264,"kb_used_data":3493912,"kb_used_omap":30,"kb_used_meta":505313,"kb_avail":89745888,"statfs":{"total":95995035648,"available":91899789312,"internally_reserved":0,"allocated":3577765888,"data_stored":3575073424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":517440834},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190468,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2838632,"kb_used_data":2251232,"kb_used_omap":43,"kb_used_meta":587348,"kb_avail":90906520,"statfs":{"total":95995035648,"available":93088276480,"internally_reserved":0,"allocated":2305261568,"data_stored":2302488580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":601444395},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849597,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2583524,"kb_used_data":1911516,"kb_used_omap":36,"kb_used_meta":671963,"kb_avail":91161628,"statfs":{"total":95995035648,"available":93349507072,"internally_reserved":0,"allocated":1957392384,"data_stored":1954695858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":688090292},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704950,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":730084,"kb_used_data":202716,"kb_used_omap":17,"kb_used_meta":527342,"kb_avail":93015068,"statfs":{"total":95995035648,"available":95247429632,"internally_reserved":0,"allocated":207581184,"data_stored":204884714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":539998711},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,3,5,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018162,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2050424,"kb_used_data":1558768,"kb_used_omap":42,"kb_used_meta":491605,"kb_avail":91694728,"statfs":{"total":95995035648,"available":93895401472,"internally_reserved":0,"allocated":1596178432,"data_stored":1593466294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":503404344},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757711,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4384508,"kb_used_data":3793968,"kb_used_omap":1,"kb_used_meta":590526,"kb_avail":89360644,"statfs":{"total":95995035648,"available":91505299456,"internally_reserved":0,"allocated":3885023232,"data_stored":3882701538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":604699078},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2302410752,"data_stored":2302410752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3572629504,"data_stored":3572629504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1950089216,"data_stored":1950089216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":204144640,"data_stored":204144640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1588854784,"data_stored":1588854784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3903651840,"data_stored":3903651840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1162674176,"data_stored":1162674176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4248174592,"data_stored":4248174592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:48:58.588 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-04-12T11:48:58.589 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool set .mgr pg_num 11 2024-04-12T11:48:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:58.590 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:58.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:48:58.691 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:58.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:58.749 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:58.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:48:58.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:48:58.897 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:58.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:58.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:48:58.972 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 99133 99117 83.7018 37.3125 0.0270541 0.0119407 2024-04-12T11:48:58.999 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:59.028 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:59.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:59.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:59.130 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:59.170 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:59.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:59.272 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:48:59.303 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:59.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:48:59.323 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:59 smithi087 ceph-mon[89014]: from='client.35493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:48:59.324 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:48:59 smithi087 ceph-mon[89014]: pgmap v1940: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 34303/291048 objects degraded (11.786%); 11131/291048 objects misplaced (3.824%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T11:48:59.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:48:59.448 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:48:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:59.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:48:59.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:59 smithi029 ceph-mon[97787]: from='client.35493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:48:59.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:48:59 smithi029 ceph-mon[97787]: pgmap v1940: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 34303/291048 objects degraded (11.786%); 11131/291048 objects misplaced (3.824%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T11:48:59.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:59 smithi029 ceph-mon[93650]: from='client.35493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:48:59.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:48:59 smithi029 ceph-mon[93650]: pgmap v1940: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 34303/291048 objects degraded (11.786%); 11131/291048 objects misplaced (3.824%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T11:48:59.550 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:59.585 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:59.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:59.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:59.687 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:48:59.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:59.727 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:59.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:48:59.804 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:48:59.805 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:48:59.827 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:48:59.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:48:59.866 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:48:59.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:48:59.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:48:59.972 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 99827 99811 83.1641 43.375 0.00658623 0.0120178 2024-04-12T11:49:00.011 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:00.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:00.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:00.113 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:00.142 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:00.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:00.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:00.243 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:00.273 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:00.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:00.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:00.375 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:00.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:00 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/295044 objects degraded (11.626%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:00.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:00 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/295044 objects degraded (11.626%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:00.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:00.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:00.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:00.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:00 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/295044 objects degraded (11.626%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:00.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:00.579 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:00.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:00.680 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:00.713 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:00.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:00.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:00.814 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:00.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:00.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:00.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:00.890 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:00.945 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:00.977 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 100474 100458 82.5984 40.4375 0.0262431 0.0121 2024-04-12T11:49:01.017 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:01.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:01.119 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:01.151 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:01.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:01.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:01 smithi087 ceph-mon[89014]: pgmap v1941: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s; 34303/295044 objects degraded (11.626%); 11131/295044 objects misplaced (3.773%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T11:49:01.252 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:01.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:01.318 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:01.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:01.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:01 smithi029 ceph-mon[93650]: pgmap v1941: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s; 34303/295044 objects degraded (11.626%); 11131/295044 objects misplaced (3.773%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T11:49:01.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:01 smithi029 ceph-mon[97787]: pgmap v1941: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s; 34303/295044 objects degraded (11.626%); 11131/295044 objects misplaced (3.773%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T11:49:01.418 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:01.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:01.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:01.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:01.563 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:01.592 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:01.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:01.694 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:01.727 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:01.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:01.828 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:01.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:01.869 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:01.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:01.970 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:01.976 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 101095 101079 82.0293 38.8125 0.0174552 0.0121838 2024-04-12T11:49:02.012 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:02.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:02.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:02.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:02.133 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:49:02.170 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:02.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:02.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:02.272 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:02.301 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:02.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:02.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:02.319 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:02 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3409805352' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:49:02.384 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:02 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3409805352' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:49:02.384 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:02 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3409805352' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:49:02.400 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-04-12T11:49:02.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:02.405 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:49:02.447 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:02.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:02.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:02.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:02.592 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:02.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:02.694 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:49:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:49:02.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:49:02.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:02.740 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:02.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:02.775 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:49:02.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:02.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:02.885 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:02.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:02.976 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 101726 101710 81.4832 39.4375 0.0422214 0.012267 2024-04-12T11:49:02.986 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:03.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:03.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:03.129 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:03.172 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:03.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:03.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:03.212 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:03 smithi087 ceph-mon[89014]: pgmap v1942: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 37 MiB/s wr, 584 op/s; 34303/298296 objects degraded (11.500%); 10994/298296 objects misplaced (3.686%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:49:03.212 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:03 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:49:03.212 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:03 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3409805352' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:49:03.212 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:03 smithi087 ceph-mon[89014]: osdmap e1238: 8 total, 8 up, 8 in 2024-04-12T11:49:03.212 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:03.212 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:03.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:03.315 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:03.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:03.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:03.417 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:03.452 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:03.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:03.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:03.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[97787]: pgmap v1942: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 37 MiB/s wr, 584 op/s; 34303/298296 objects degraded (11.500%); 10994/298296 objects misplaced (3.686%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3409805352' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[97787]: osdmap e1238: 8 total, 8 up, 8 in 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[93650]: pgmap v1942: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 37 MiB/s wr, 584 op/s; 34303/298296 objects degraded (11.500%); 10994/298296 objects misplaced (3.686%); 1.7 MiB/s, 27 objects/s recovering 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3409805352' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[93650]: osdmap e1238: 8 total, 8 up, 8 in 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:03.554 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:03.591 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:03.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:03.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:03.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:03.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:03.744 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:03.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:03.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:03.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:03.877 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:03.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:03.977 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 102394 102378 80.9799 41.75 0.0126885 0.012343 2024-04-12T11:49:03.978 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:04.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:04.028 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:04.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:04.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:04.165 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:04.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:04.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:04.306 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:04.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:04.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:04.408 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:04.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:04.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:04.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:04.558 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:04.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:04.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:04.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:04.694 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:49:04.694 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:49:04.702 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:04.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:04.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:04.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:04.841 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:04.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:04.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:04.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:04.972 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:04.981 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:49:04.981158+0000 min lat: 0.0012668 max lat: 2.66246 avg lat: 0.0124227 2024-04-12T11:49:04.981 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:49:04.981 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 102992 102976 80.4306 37.375 0.0201505 0.0124227 2024-04-12T11:49:04.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:05.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:05.056 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:05.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:05.145 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 1 pg_num to 11 2024-04-12T11:49:05.157 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:05.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:05 smithi087 ceph-mon[89014]: pgmap v1944: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 34303/302481 objects degraded (11.341%); 10994/302481 objects misplaced (3.635%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T11:49:05.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:05 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3955691595' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2024-04-12T11:49:05.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:05 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2024-04-12T11:49:05.222 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:05.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:05.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:05.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:49:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:49:04] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.43.0" 2024-04-12T11:49:05.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:05 smithi029 ceph-mon[97787]: pgmap v1944: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 34303/302481 objects degraded (11.341%); 10994/302481 objects misplaced (3.635%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T11:49:05.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:05 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3955691595' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2024-04-12T11:49:05.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:05 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2024-04-12T11:49:05.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:05 smithi029 ceph-mon[93650]: pgmap v1944: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 34303/302481 objects degraded (11.341%); 10994/302481 objects misplaced (3.635%); 1.8 MiB/s, 28 objects/s recovering 2024-04-12T11:49:05.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:05 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3955691595' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2024-04-12T11:49:05.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:05 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2024-04-12T11:49:05.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:05.367 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:05.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:05.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:05.422 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-04-12T11:49:05.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:05.507 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:05.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:05.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:05.609 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:05.661 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:05.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:05.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:05.763 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:05.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:49:05.769 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:49:05.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:05.802 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:05.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:05.903 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:05.937 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:05.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:05.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:05.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 103551 103535 79.869 34.9375 0.0502982 0.0125131 2024-04-12T11:49:06.039 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:06.077 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:06.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:06.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:06.178 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:06.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:06 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-04-12T11:49:06.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:06 smithi087 ceph-mon[89014]: osdmap e1239: 8 total, 8 up, 8 in 2024-04-12T11:49:06.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2024-04-12T11:49:06.223 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:06.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:06.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:06.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:06.368 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:06.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:06.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:06.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:06 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-04-12T11:49:06.407 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:06 smithi029 ceph-mon[97787]: osdmap e1239: 8 total, 8 up, 8 in 2024-04-12T11:49:06.407 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2024-04-12T11:49:06.407 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:06 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-04-12T11:49:06.407 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:06 smithi029 ceph-mon[93650]: osdmap e1239: 8 total, 8 up, 8 in 2024-04-12T11:49:06.407 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2024-04-12T11:49:06.470 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:06.513 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:06.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:06.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:06.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:06.656 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:06.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:06.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:06.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:06.801 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:06.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:06.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:06.903 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:06.936 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:06.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:06.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:06.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 104184 104168 79.3775 39.5625 0.0324807 0.0125924 2024-04-12T11:49:07.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:07.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:07.073 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:07.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:07.174 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:07.215 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:07.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:07.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:07.317 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:07.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:07.371 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:07.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:07 smithi087 ceph-mon[89014]: pgmap v1946: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 34303/304935 objects degraded (11.249%); 10853/304935 objects misplaced (3.559%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:49:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:07 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/304935 objects degraded (11.249%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:07.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-04-12T11:49:07.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:07 smithi087 ceph-mon[89014]: osdmap e1240: 8 total, 8 up, 8 in 2024-04-12T11:49:07.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:07.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:07.471 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:07.508 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:07.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:07.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:07.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[97787]: pgmap v1946: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 34303/304935 objects degraded (11.249%); 10853/304935 objects misplaced (3.559%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:49:07.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/304935 objects degraded (11.249%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:07.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-04-12T11:49:07.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[97787]: osdmap e1240: 8 total, 8 up, 8 in 2024-04-12T11:49:07.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:07.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:07.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[93650]: pgmap v1946: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 34303/304935 objects degraded (11.249%); 10853/304935 objects misplaced (3.559%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:49:07.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/304935 objects degraded (11.249%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:07.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-04-12T11:49:07.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[93650]: osdmap e1240: 8 total, 8 up, 8 in 2024-04-12T11:49:07.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:07.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:07.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:07.650 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:07.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:07.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:07.752 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:07.784 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:07.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:07.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:07.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:07.922 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:07.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:07.981 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 104828 104812 78.9061 40.25 0.0676544 0.0126671 2024-04-12T11:49:08.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:08.058 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:08.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:08.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:08.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:08.210 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:08.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:08.228 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:08.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:08.329 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:08.351 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:08 smithi029 ceph-mon[93650]: osdmap e1241: 8 total, 8 up, 8 in 2024-04-12T11:49:08.351 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:08 smithi029 ceph-mon[97787]: osdmap e1241: 8 total, 8 up, 8 in 2024-04-12T11:49:08.367 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:08.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:08.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:08.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:08 smithi087 ceph-mon[89014]: osdmap e1241: 8 total, 8 up, 8 in 2024-04-12T11:49:08.468 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:08.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:08.498 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:08.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:08.599 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:08.635 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:08.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:08.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:08.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:08.773 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:08.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:08.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:08.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:08.907 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:08.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:08.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:08.981 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 105458 105442 78.4355 39.375 0.0178557 0.012744 2024-04-12T11:49:09.010 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:09.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:09.054 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:09.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:09.155 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:09.200 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:09.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:09.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:09.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:09.341 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:09.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:09.402 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:09 smithi029 ceph-mon[97787]: pgmap v1949: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 34303/309510 objects degraded (11.083%); 10853/309510 objects misplaced (3.507%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T11:49:09.403 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:09 smithi029 ceph-mon[93650]: pgmap v1949: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 34303/309510 objects degraded (11.083%); 10853/309510 objects misplaced (3.507%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T11:49:09.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:09 smithi087 ceph-mon[89014]: pgmap v1949: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 34303/309510 objects degraded (11.083%); 10853/309510 objects misplaced (3.507%); 1.5 MiB/s, 24 objects/s recovering 2024-04-12T11:49:09.472 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:09.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:09.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:09.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:09.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:09.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:09.649 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:49:09.686 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:49:09.717 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:09.727 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:49:09.752 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:09.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:09.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:09.773 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:49:09.818 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:49:09.818 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:49:09.821 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:49:09.854 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:09.863 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:49:09.892 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:09.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:09.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:09.899 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:49:09.936 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:49:09.953 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:49:09.954 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:49:09.954 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:49:09.954 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:49:09.954 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:49:09.954 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:49:09.954 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:49:09.954 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:49:09.981 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 106062 106046 77.9569 37.75 0.0134509 0.0128193 2024-04-12T11:49:09.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:10.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:10.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:10.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:10.133 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:10.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:10.168 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:10.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:10.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1405132214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:49:10.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1405132214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:49:10.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:10.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:10.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:10.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:10.412 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:10.445 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:10.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:10.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:10.462 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1405132214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:49:10.489 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-04-12T11:49:10.547 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:10.607 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:10.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:10.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:10.761 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:10.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:10.830 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:49:10.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:10.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:10.905 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:10.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:10.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 106710 106694 77.5214 40.5 0.0182339 0.0128932 2024-04-12T11:49:11.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:11.063 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:11.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:11.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:11.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:11.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:11 smithi087 ceph-mon[89014]: pgmap v1950: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 34303/313644 objects degraded (10.937%); 10853/313644 objects misplaced (3.460%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T11:49:11.218 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:11.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:11.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:11.233 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:49:11.234 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:49:11.234 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2024-04-12T11:49:11.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:49:11.320 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:11.356 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:11.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:11.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:11.458 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:11.487 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:11.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:11.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:11 smithi029 ceph-mon[97787]: pgmap v1950: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 34303/313644 objects degraded (10.937%); 10853/313644 objects misplaced (3.460%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T11:49:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:11 smithi029 ceph-mon[93650]: pgmap v1950: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 34303/313644 objects degraded (10.937%); 10853/313644 objects misplaced (3.460%); 1.5 MiB/s, 23 objects/s recovering 2024-04-12T11:49:11.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:11.617 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:11.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:11.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:11.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:11.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:11.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:11.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:11.931 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:11.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:11.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:11.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 107332 107316 77.0772 38.875 0.0136994 0.0129677 2024-04-12T11:49:12.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:12.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:12.066 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:12.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:12.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:12.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:12.212 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:12.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:12.272 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:12 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/317238 objects degraded (10.813%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:12.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:12.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:12.343 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:12.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:12.444 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:12.472 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:12.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:12.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:12 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/317238 objects degraded (10.813%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:12.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:12 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/317238 objects degraded (10.813%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:12.544 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:49:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:49:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:49:12.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:12.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:12.618 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:12.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:12.694 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:49:12.695 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:49:12.695 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:49:12.695 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:49:12.695 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:49:12.719 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:12.752 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:12.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:12.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:12.854 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:12.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:12.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:12.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 107860 107844 76.5763 33 0.0155589 0.0130524 2024-04-12T11:49:12.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:13.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:13.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:13.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:13.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:13.161 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:13.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:13.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:13.262 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:13.295 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:13.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:13.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:13.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:13.448 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-04-12T11:49:13.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:13.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:13.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:13.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:13 smithi029 ceph-mon[97787]: pgmap v1951: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s; 34303/317238 objects degraded (10.813%); 10714/317238 objects misplaced (3.377%); 2.6 MiB/s, 40 objects/s recovering 2024-04-12T11:49:13.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:13 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2123311022' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:49:13.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:13 smithi029 ceph-mon[93650]: pgmap v1951: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s; 34303/317238 objects degraded (10.813%); 10714/317238 objects misplaced (3.377%); 2.6 MiB/s, 40 objects/s recovering 2024-04-12T11:49:13.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:13 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2123311022' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:49:13.566 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:13.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:49:13.591 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:13.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:13.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:13.693 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:13.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:13 smithi087 ceph-mon[89014]: pgmap v1951: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 627 op/s; 34303/317238 objects degraded (10.813%); 10714/317238 objects misplaced (3.377%); 2.6 MiB/s, 40 objects/s recovering 2024-04-12T11:49:13.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:13 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2123311022' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:49:13.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:13.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:13.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:13.746 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:49:13.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:13.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:13.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:13.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:13.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:13.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 108299 108283 76.0243 27.4375 0.20235 0.0131289 2024-04-12T11:49:13.993 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:13.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:13.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:14.096 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:14.127 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:14.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:14.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:14.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:14.256 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:14.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:14.358 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:14.383 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:14.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:14.485 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:14.516 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:14.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:14.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:14.588 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:14 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:49:14.588 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:14 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2123311022' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:49:14.588 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:14 smithi087 ceph-mon[89014]: osdmap e1242: 8 total, 8 up, 8 in 2024-04-12T11:49:14.618 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:14.658 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:14.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:14.759 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:14.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:14 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:49:14.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:14 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2123311022' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:49:14.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:14 smithi029 ceph-mon[97787]: osdmap e1242: 8 total, 8 up, 8 in 2024-04-12T11:49:14.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:14 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:49:14.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:14 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2123311022' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:49:14.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:14 smithi029 ceph-mon[93650]: osdmap e1242: 8 total, 8 up, 8 in 2024-04-12T11:49:14.786 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:14.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:14.888 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:14.929 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:14.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:14.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:14.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.012 2024-04-12T11:49:14.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 108774 2024-04-12T11:49:14.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2024-04-12T11:49:14.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2024-04-12T11:49:14.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 75.5275 2024-04-12T11:49:14.982 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 45.1492 2024-04-12T11:49:14.983 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 232.375 2024-04-12T11:49:14.983 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2024-04-12T11:49:14.983 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1208 2024-04-12T11:49:14.983 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 722.386 2024-04-12T11:49:14.983 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3718 2024-04-12T11:49:14.983 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2024-04-12T11:49:14.983 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0132352 2024-04-12T11:49:14.983 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0340777 2024-04-12T11:49:14.983 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 2.66246 2024-04-12T11:49:14.983 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.0012668 2024-04-12T11:49:14.989 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2024-04-12T11:49:15.031 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:15.063 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:15.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:15.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:15.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:15.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:15.190 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:15.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:49:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:49:14] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.43.0" 2024-04-12T11:49:15.291 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:15.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:15.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:15.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:15.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:15.457 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:15.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:15.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:15.559 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:15.562 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:15 smithi087 ceph-mon[89014]: pgmap v1953: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s; 34303/321048 objects degraded (10.685%); 10714/321048 objects misplaced (3.337%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T11:49:15.588 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:15.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:15.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:15.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:15.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:15.719 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:15.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:15.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:15 smithi029 ceph-mon[93650]: pgmap v1953: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s; 34303/321048 objects degraded (10.685%); 10714/321048 objects misplaced (3.337%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T11:49:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:15 smithi029 ceph-mon[97787]: pgmap v1953: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s; 34303/321048 objects degraded (10.685%); 10714/321048 objects misplaced (3.337%); 1.1 MiB/s, 17 objects/s recovering 2024-04-12T11:49:15.821 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:15.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:15.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:15.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:15.960 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:16.000 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:16.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:16.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:16.088 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:49:16.088 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:49:16.088 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:168464 8.c e1211/1210 8:30f6b6f9:::benchmark_data_smithi029_194418_object62154:head v 1211'10350, mlcod=1192'5288)", 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:54.361322+0000", 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 201.722497051, 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.11230293399999999, 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.361322+0000", 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.089 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.361322+0000", 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.361322+0000", 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.090 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.361332+0000", 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0964999999999999e-05 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.361354+0000", 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1257e-05 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.361363+0000", 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6430000000000007e-06 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.406883+0000", 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.045520089999999999 2024-04-12T11:49:16.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.406908+0000", 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4176e-05 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.473585+0000", 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.066677782000000005 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.473625+0000", 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9956999999999997e-05 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(8.d 1211/1209 [PushOp(8:b5dbce15:::benchmark_data_smithi029_194418_object87342:head, version: 1195'5490, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b5dbce15:::benchmark_data_smithi029_194418_object87342:head@1195'5490, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:54.545856+0000", 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 201.53796364300001, 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.124948083, 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.545856+0000", 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.545856+0000", 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.545852+0000", 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999962 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.545898+0000", 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6459999999999999e-05 2024-04-12T11:49:16.094 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.545932+0000", 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4118000000000001e-05 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.545963+0000", 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.0026999999999999e-05 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.665068+0000", 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.11910523100000001 2024-04-12T11:49:16.095 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.665083+0000", 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4895000000000001e-05 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.670804+0000", 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0057212809999999999 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.096 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:170616 8.c e1211/1210 8:35fcf1c1:::benchmark_data_smithi029_194418_object64306:head v 1211'10496, mlcod=1192'5288)", 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:56.510413+0000", 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 199.57340684600001, 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.106395007, 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:56.510413+0000", 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.097 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:56.510413+0000", 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:56.510411+0000", 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:56.510424+0000", 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3056e-05 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.098 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:56.510442+0000", 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8312999999999999e-05 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:56.510453+0000", 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0176999999999999e-05 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:56.526367+0000", 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.015914055999999999 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.099 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:56.526399+0000", 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.2620999999999998e-05 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:56.616771+0000", 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.090371429000000003 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:56.616808+0000", 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6996000000000003e-05 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.100 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(8.d 1211/1209 [PushOp(8:b0760182:::benchmark_data_smithi029_194418_object88037:head, version: 1195'5537, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b0760182:::benchmark_data_smithi029_194418_object88037:head@1195'5537, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:57.138220+0000", 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 198.94559948599999, 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.22547447800000001, 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138220+0000", 2024-04-12T11:49:16.101 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138220+0000", 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138213+0000", 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999933 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138258+0000", 2024-04-12T11:49:16.102 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4864999999999998e-05 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138278+0000", 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9587000000000001e-05 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138294+0000", 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6532999999999999e-05 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.360093+0000", 2024-04-12T11:49:16.103 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.221798943 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.360105+0000", 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1331e-05 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.363694+0000", 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0035898330000000002 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(8.d 1211/1209 [PushOp(8:b9eded05:::benchmark_data_smithi029_194418_object88441:head, version: 1195'5559, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b9eded05:::benchmark_data_smithi029_194418_object88441:head@1195'5559, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:58.071527+0000", 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 198.012292411, 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27444023299999998, 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071527+0000", 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.105 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071527+0000", 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071526+0000", 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071565+0000", 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9232e-05 2024-04-12T11:49:16.106 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071591+0000", 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5477e-05 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071609+0000", 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7731999999999999e-05 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.341174+0000", 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.26956538899999999 2024-04-12T11:49:16.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.341186+0000", 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2300000000000001e-05 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.345967+0000", 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0047811040000000004 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(8.d 1211/1209 [PushOp(8:b10db278:::benchmark_data_smithi029_194418_object88780:head, version: 1195'5581, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b10db278:::benchmark_data_smithi029_194418_object88780:head@1195'5581, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:59.765684+0000", 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 196.318135799, 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.316417805, 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765684+0000", 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.109 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765684+0000", 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765682+0000", 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765726+0000", 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3307999999999999e-05 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.110 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765751+0000", 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5627e-05 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765769+0000", 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7890999999999999e-05 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.076685+0000", 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.31091553999999999 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.111 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.076700+0000", 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5271e-05 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.082101+0000", 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.005401622 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:173690 8.d e1211/1209 8:b60a8545:::benchmark_data_smithi029_194418_object67380:head v 1211'10983, mlcod=1195'5580)", 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:59.874616+0000", 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 196.209203379, 2024-04-12T11:49:16.112 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.10220572, 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.874616+0000", 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.874616+0000", 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.113 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.874615+0000", 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.874624+0000", 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1209999999999996e-06 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.874641+0000", 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7527e-05 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.114 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.874650+0000", 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3489999999999992e-06 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.929080+0000", 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.054429836000000002 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.929103+0000", 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.2886000000000001e-05 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.976740+0000", 2024-04-12T11:49:16.115 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.047637946 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.976822+0000", 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1363e-05 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(8.d 1211/1209 [PushOp(8:b4918f04:::benchmark_data_smithi029_194418_object88967:head, version: 1195'5591, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b4918f04:::benchmark_data_smithi029_194418_object88967:head@1195'5591, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:46:00.614183+0000", 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 195.469636576, 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.14978947500000001, 2024-04-12T11:49:16.116 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.614183+0000", 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.614183+0000", 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.117 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.614182+0000", 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.614226+0000", 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4468999999999997e-05 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.614244+0000", 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7374999999999999e-05 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.118 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.614259+0000", 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.504e-05 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.758604+0000", 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.14434498400000001 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.758612+0000", 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6119999999999999e-06 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.119 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.763972+0000", 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0053599659999999999 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:174462 8.d e1211/1209 8:bcae3fa4:::benchmark_data_smithi029_194418_object68152:head v 1211'11029, mlcod=1195'5590)", 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:46:00.652589+0000", 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 195.43123029399999, 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.111293796, 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.120 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.652589+0000", 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.652589+0000", 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.652588+0000", 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.652597+0000", 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3309999999999993e-06 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.652614+0000", 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.66e-05 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.652622+0000", 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3329999999999997e-06 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.682536+0000", 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.029914017000000001 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.682591+0000", 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5096999999999998e-05 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.123 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.123 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:16.124 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.763840+0000", 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.081248227000000006 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.763883+0000", 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3336e-05 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:174467 8.d e1211/1209 8:bbb47418:::benchmark_data_smithi029_194418_object68157:head v 1211'11030, mlcod=1195'5590)", 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:46:00.654210+0000", 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 195.4296095, 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.109695283, 2024-04-12T11:49:16.125 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.654210+0000", 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.654210+0000", 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.126 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.654209+0000", 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.654224+0000", 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5335000000000001e-05 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.654240+0000", 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6082e-05 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.654249+0000", 2024-04-12T11:49:16.127 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7390000000000005e-06 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.693133+0000", 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.038884662 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.693168+0000", 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4418999999999997e-05 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.763886+0000", 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.070718106000000003 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.128 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.763905+0000", 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9258000000000001e-05 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:174472 8.d e1211/1209 8:bbf27316:::benchmark_data_smithi029_194418_object68162:head v 1211'11031, mlcod=1195'5590)", 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:46:00.658635+0000", 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 195.425184401, 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.105334961, 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.129 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.658635+0000", 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.658635+0000", 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.658634+0000", 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:49:16.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.658644+0000", 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3070000000000008e-06 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.658660+0000", 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6860000000000001e-05 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.658669+0000", 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7409999999999998e-06 2024-04-12T11:49:16.131 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.703549+0000", 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.044879558999999999 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.703576+0000", 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6928999999999999e-05 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.763907+0000", 2024-04-12T11:49:16.132 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.060331665 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.763970+0000", 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2726999999999998e-05 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.35217.0:174475 8.d e1211/1209 8:b6a532a4:::benchmark_data_smithi029_194418_object68165:head v 1211'11032, mlcod=1195'5590)", 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:46:00.663878+0000", 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 195.41994159800001, 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.11207592299999999, 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.133 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.663878+0000", 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.663878+0000", 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.663877+0000", 2024-04-12T11:49:16.134 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.663886+0000", 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8020000000000003e-06 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.663912+0000", 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5069e-05 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.663921+0000", 2024-04-12T11:49:16.135 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3379999999999994e-06 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.716474+0000", 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.052552725000000002 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.716531+0000", 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7759e-05 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.774970+0000", 2024-04-12T11:49:16.136 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.058438504000000002 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.775954+0000", 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00098393999999999999 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(8.d 1215/1209 [PushOp(8:b1dcb5bc:::benchmark_data_smithi029_194418_object102114:head, version: 1196'6459, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b1dcb5bc:::benchmark_data_smithi029_194418_object102114:head@1196'6459, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:46:25.657846+0000", 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 170.42597394699999, 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.114409003, 2024-04-12T11:49:16.137 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:25.657846+0000", 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:25.657846+0000", 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.138 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:25.657845+0000", 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:25.657879+0000", 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4589999999999999e-05 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:25.657897+0000", 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7402e-05 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:25.657912+0000", 2024-04-12T11:49:16.139 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5174000000000001e-05 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:25.768748+0000", 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.110836238 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:25.768760+0000", 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1473e-05 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:25.772255+0000", 2024-04-12T11:49:16.140 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003494998 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(8.d 1216/1209 [PushOp(8:b092e064:::benchmark_data_smithi029_194418_object104464:head, version: 1196'6582, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b092e064:::benchmark_data_smithi029_194418_object104464:head@1196'6582, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:46:29.595359+0000", 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 166.48846005799999, 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.10340793299999999, 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.141 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:29.595359+0000", 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:29.595359+0000", 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:29.595359+0000", 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.142 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:29.597435+0000", 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0020766140000000001 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:29.598720+0000", 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0012847850000000001 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:29.598741+0000", 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0602999999999999e-05 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.143 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:29.695223+0000", 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.096481856000000005 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:29.695236+0000", 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3315999999999999e-05 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:29.698767+0000", 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0035313940000000002 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.144 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:107693 9.3 9:cb8ca4e0:::benchmark_data_smithi029_222585_object107692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:49:12.636981+0000", 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 3.4468389610000001, 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.11220524699999999, 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 107693 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.636981+0000", 2024-04-12T11:49:16.145 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.636981+0000", 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.636979+0000", 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.146 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.637028+0000", 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8798000000000003e-05 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.637037+0000", 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3810000000000005e-06 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.637045+0000", 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8909999999999995e-06 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.147 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.645287+0000", 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0082415200000000004 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.645412+0000", 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012534800000000001 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.645539+0000", 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012650500000000001 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.648045+0000", 2024-04-12T11:49:16.148 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0025065719999999999 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.739081+0000", 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.091035252999999997 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.744769+0000", 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0056880209999999997 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.744847+0000", 2024-04-12T11:49:16.149 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8879000000000002e-05 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:12.749186+0000", 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0043383420000000002 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:108150 9.3 9:cb8e0688:::benchmark_data_smithi029_222585_object108149:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:49:13.490270+0000", 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 2.5935497999999999, 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.1121404, 2024-04-12T11:49:16.150 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 108150 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.490270+0000", 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.151 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.490270+0000", 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.490268+0000", 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.490346+0000", 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8040999999999994e-05 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.152 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.490369+0000", 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.2928000000000001e-05 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.490379+0000", 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0394000000000001e-05 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.490440+0000", 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0744e-05 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.153 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.492546+0000", 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0021062609999999999 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.492739+0000", 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000193086 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.500811+0000", 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0080714910000000001 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.154 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.509034+0000", 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0082234319999999993 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.597682+0000", 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.088647570999999994 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.597752+0000", 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0621e-05 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.155 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.602410+0000", 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0046578110000000004 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:108156 9.3 9:ca05c9c4:::benchmark_data_smithi029_222585_object108155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:49:13.496918+0000", 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 2.5869012119999999, 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.109627448, 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:49:16.156 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 108156 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.496918+0000", 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.496918+0000", 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.157 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.496916+0000", 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.496950+0000", 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3964000000000002e-05 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.496978+0000", 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.7577999999999999e-05 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.496988+0000", 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0276e-05 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.497056+0000", 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7989000000000002e-05 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.497241+0000", 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018504099999999999 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.159 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.497449+0000", 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00020832399999999999 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.500872+0000", 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034227789999999999 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.509055+0000", 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0081826239999999995 2024-04-12T11:49:16.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.161 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.161 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.161 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.602476+0000", 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.093421758999999993 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.602537+0000", 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0925999999999999e-05 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:13.606546+0000", 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0040083039999999999 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.162 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:108890 9.3 9:c96a146a:::benchmark_data_smithi029_222585_object113:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:49:15.142504+0000", 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.94131509400000002, 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.17018455900000001, 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:49:16.163 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 108890 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.142504+0000", 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.142504+0000", 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.164 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.142503+0000", 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.142510+0000", 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6420000000000001e-06 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.165 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.142525+0000", 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5820000000000001e-05 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.142534+0000", 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9979999999999992e-06 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.152683+0000", 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.010148300000000001 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.166 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.152848+0000", 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016553499999999999 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.153009+0000", 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016098999999999999 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.160632+0000", 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.007622551 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.167 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.164024+0000", 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00339237 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.308385+0000", 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.144360456 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.308512+0000", 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000127551 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.168 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.312689+0000", 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0041767439999999996 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:108891 9.3 9:c60e3d25:::benchmark_data_smithi029_222585_object114:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:49:15.146417+0000", 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.93740221999999995, 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.17430869600000001, 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 108891 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146417+0000", 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146417+0000", 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146415+0000", 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146424+0000", 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1929999999999999e-06 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146440+0000", 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5299999999999999e-05 2024-04-12T11:49:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146451+0000", 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1250000000000001e-05 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.160678+0000", 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014226518 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.160865+0000", 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018702000000000001 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.161061+0000", 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000196367 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.164151+0000", 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030902199999999999 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.304438+0000", 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.14028655100000001 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.316514+0000", 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012076304 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.316573+0000", 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9247999999999999e-05 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.320726+0000", 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0041527200000000004 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:108904 9.e 9:754bbf7e:::benchmark_data_smithi029_222585_object127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:49:15.173842+0000", 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.90997711999999997, 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.14167644700000001, 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 108904 2024-04-12T11:49:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.173842+0000", 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.173842+0000", 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:49:16.176 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.173840+0000", 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.173851+0000", 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0542e-05 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.173874+0000", 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.2977e-05 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.177 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.173884+0000", 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0224000000000001e-05 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.173932+0000", 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7760999999999999e-05 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.174044+0000", 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011223800000000001 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.178 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.174142+0000", 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7922000000000002e-05 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.177324+0000", 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0031822370000000001 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.177592+0000", 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00026813199999999997 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.179 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.315431+0000", 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.13783894099999999 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.315480+0000", 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.871e-05 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.315519+0000", 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8832999999999999e-05 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.180 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.181 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:49:16.181 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:49:16.181 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:49:16.190 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:16.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:16.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:16.292 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:16.325 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:16.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:16.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:16.427 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:16.468 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:16.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:16.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:16.570 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:16.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:16.606 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:16.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:16.707 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:16.737 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:16.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:16.838 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:16.862 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:16.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:16.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:16.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:17.002 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:17.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:17.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:17.110 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-04-12T11:49:17.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:17.155 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:17.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:17.256 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:17.288 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:17.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:17.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:17.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:17.446 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:17.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:17.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:17 smithi087 ceph-mon[89014]: pgmap v1954: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 36 MiB/s wr, 582 op/s; 34303/323157 objects degraded (10.615%); 10553/323157 objects misplaced (3.266%); 2.1 MiB/s, 34 objects/s recovering 2024-04-12T11:49:17.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:17 smithi029 ceph-mon[97787]: pgmap v1954: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 36 MiB/s wr, 582 op/s; 34303/323157 objects degraded (10.615%); 10553/323157 objects misplaced (3.266%); 2.1 MiB/s, 34 objects/s recovering 2024-04-12T11:49:17.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:17 smithi029 ceph-mon[93650]: pgmap v1954: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 36 MiB/s wr, 582 op/s; 34303/323157 objects degraded (10.615%); 10553/323157 objects misplaced (3.266%); 2.1 MiB/s, 34 objects/s recovering 2024-04-12T11:49:17.548 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:17.583 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:17.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:17.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:17.684 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:17.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:17.734 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:17.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:17.759 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:49:17.835 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:17.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:17.868 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:17.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:17.969 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:17.998 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:17.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:18.100 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:18.150 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:18.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:18.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:18.281 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:18.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:18.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:18.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:18.427 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:18.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:18.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:18.529 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:18.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:18.561 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:18.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:18.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:18.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:18.708 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:18.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:18.809 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:18.837 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:18.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:18.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:18.938 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:18.943 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:18.951 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:18.954 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:18.967 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:18 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:18 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:18 smithi087 ceph-mon[89014]: pgmap v1955: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 581 op/s; 34303/324225 objects degraded (10.580%); 10553/324225 objects misplaced (3.255%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T11:49:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/323157 objects degraded (10.615%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:18.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:18.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:19.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:18 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:19.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:18 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:19.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:18 smithi029 ceph-mon[93650]: pgmap v1955: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 581 op/s; 34303/324225 objects degraded (10.580%); 10553/324225 objects misplaced (3.255%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T11:49:19.021 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/323157 objects degraded (10.615%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:19.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:18 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:19.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:18 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:19.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:18 smithi029 ceph-mon[97787]: pgmap v1955: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 581 op/s; 34303/324225 objects degraded (10.580%); 10553/324225 objects misplaced (3.255%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T11:49:19.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/323157 objects degraded (10.615%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:19.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:19.098 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:19.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:19.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:19.199 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:19.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:19.236 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:19.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:19.337 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:19.382 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:19.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:19.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:19.484 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:19.514 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:19.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:19.616 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:19.660 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:19.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:19.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:19.763 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:19.806 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:19.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:19.907 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:19.934 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:19.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:19.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:49:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:49:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:19.967 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:49:19.968 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:49:19.968 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:19.968 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:49:19.968 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:49:19.968 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:20.036 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:20.063 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:20.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:20.165 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:20.192 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:20.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:20.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:20.294 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:20.338 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:20.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:20.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:20.440 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:20.465 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:20.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:20.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:20.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:20.606 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:20.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:20.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:20.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:20.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:20.743 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:20.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:20.844 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:20.852 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:49:20.874 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:20.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:20.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:20 smithi087 ceph-mon[89014]: pgmap v1956: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 563 op/s; 34303/322233 objects degraded (10.645%); 10553/322233 objects misplaced (3.275%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T11:49:20.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:21.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:20 smithi029 ceph-mon[93650]: pgmap v1956: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 563 op/s; 34303/322233 objects degraded (10.645%); 10553/322233 objects misplaced (3.275%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T11:49:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:20 smithi029 ceph-mon[97787]: pgmap v1956: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 563 op/s; 34303/322233 objects degraded (10.645%); 10553/322233 objects misplaced (3.275%); 1.9 MiB/s, 29 objects/s recovering 2024-04-12T11:49:21.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:21.025 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:21.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:21.126 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:21.189 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:21.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:21.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:21.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:21.333 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:21.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:21.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:21.435 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:21.465 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:21.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:21.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:21.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:21.613 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:21.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:21.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:21.715 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:21.744 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:21.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:21.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:21.846 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:21.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:21.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:21.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:21.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:22.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:22.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:22.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:22.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:22.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:22.175 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:22.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:22.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:22.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:22.336 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:22.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:22.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:22.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:22.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:22.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:22.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:22.572 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:22.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:22.605 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:22.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:22.706 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:49:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:49:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:49:22.735 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:22.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:22.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:22.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:22.881 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:22.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:22.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:23.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:22 smithi029 ceph-mon[97787]: pgmap v1957: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 557 op/s; 34303/318813 objects degraded (10.760%); 10390/318813 objects misplaced (3.259%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:23.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:22 smithi029 ceph-mon[93650]: pgmap v1957: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 557 op/s; 34303/318813 objects degraded (10.760%); 10390/318813 objects misplaced (3.259%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:23.059 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:23.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:23.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:23.161 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:23.181 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:22 smithi087 ceph-mon[89014]: pgmap v1957: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 557 op/s; 34303/318813 objects degraded (10.760%); 10390/318813 objects misplaced (3.259%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:23.228 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:23.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:23.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:23.330 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:23.368 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:23.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:23.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:23.470 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:23.524 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:23.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:23.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:23.633 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:23.671 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:23.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:23.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:23.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:23.789 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:49:23.789 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:49:23.789 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:114156 9.8 9:16aa5832:::benchmark_data_smithi029_222585_object5379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:23.731833+0000", 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.055364084000000001, 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.055399265000000003, 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 114156 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.790 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.731833+0000", 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.731833+0000", 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:23.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.731835+0000", 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.734e-06 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.731840+0000", 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9790000000000001e-06 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.731851+0000", 2024-04-12T11:49:23.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1382999999999999e-05 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.731859+0000", 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9729999999999997e-06 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.757275+0000", 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025415571000000001 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:23.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.757477+0000", 2024-04-12T11:49:23.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020191600000000001 2024-04-12T11:49:23.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5", 2024-04-12T11:49:23.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.757710+0000", 2024-04-12T11:49:23.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023361000000000001 2024-04-12T11:49:23.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.767698+0000", 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099880839999999995 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.767750+0000", 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1408e-05 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.795 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:114163 9.c 9:36592293:::benchmark_data_smithi029_222585_object5386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:23.749656+0000", 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.037541039999999998, 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.037683110999999998, 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 114163 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:23.796 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.749656+0000", 2024-04-12T11:49:23.797 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:23.797 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.797 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.797 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:23.797 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.749656+0000", 2024-04-12T11:49:23.797 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:23.797 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.797 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.798 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:23.798 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.749658+0000", 2024-04-12T11:49:23.798 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.685e-06 2024-04-12T11:49:23.798 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.798 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.798 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:23.798 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.749664+0000", 2024-04-12T11:49:23.798 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3520000000000003e-06 2024-04-12T11:49:23.798 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.798 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:23.799 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.749677+0000", 2024-04-12T11:49:23.799 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2884000000000001e-05 2024-04-12T11:49:23.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.749687+0000", 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.007e-05 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.774217+0000", 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024529809999999999 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:23.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.774721+0000", 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00050355099999999998 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4,7", 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.775282+0000", 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00056184500000000005 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.780755+0000", 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054726499999999999 2024-04-12T11:49:23.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.782720+0000", 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019652519999999998 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.783723+0000", 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001002609 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.802 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:114172 9.8 9:18944211:::benchmark_data_smithi029_222585_object5395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:23.762453+0000", 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.024744489000000001, 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025006646, 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:23.803 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 114172 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.762453+0000", 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.762453+0000", 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:23.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.762454+0000", 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2589999999999999e-06 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.762459+0000", 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3789999999999997e-06 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.762472+0000", 2024-04-12T11:49:23.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2485e-05 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.762482+0000", 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6809999999999993e-06 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.763466+0000", 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00098464499999999992 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.806 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.763641+0000", 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017471500000000001 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5", 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.763811+0000", 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016970399999999999 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.767710+0000", 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038994300000000002 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.772758+0000", 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050478449999999996 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:114182 9.c 9:37de5074:::benchmark_data_smithi029_222585_object5405:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:23.786605+0000", 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.00059219999999999997, 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00096034199999999999, 2024-04-12T11:49:23.808 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 114182 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786605+0000", 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:23.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786605+0000", 2024-04-12T11:49:23.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:23.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:23.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786607+0000", 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2400000000000002e-06 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786618+0000", 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0285999999999999e-05 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786642+0000", 2024-04-12T11:49:23.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4510000000000001e-05 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786654+0000", 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1705000000000001e-05 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786883+0000", 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000229027 2024-04-12T11:49:23.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:23.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.787312+0000", 2024-04-12T11:49:23.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00042894 2024-04-12T11:49:23.813 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.814 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:23.814 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.814 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:114183 9.c 9.bdaa4a9c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:23.814 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:23.786686+0000", 2024-04-12T11:49:23.814 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.00051158400000000004, 2024-04-12T11:49:23.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00094916, 2024-04-12T11:49:23.814 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 114183 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786686+0000", 2024-04-12T11:49:23.815 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786686+0000", 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786686+0000", 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6000000000000004e-07 2024-04-12T11:49:23.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786690+0000", 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8099999999999999e-06 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786697+0000", 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4100000000000002e-06 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:23.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:23.818 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:23.818 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:23.786704+0000", 2024-04-12T11:49:23.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2099999999999998e-06 2024-04-12T11:49:23.818 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.818 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:23.818 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.818 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:23.818 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2024-04-12T11:49:23.818 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 5 2024-04-12T11:49:23.818 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:49:23.824 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:23.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:23.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:23.927 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:23.945 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/318813 objects degraded (10.760%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:23.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/318813 objects degraded (10.760%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:23.964 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:23.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:23.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/318813 objects degraded (10.760%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:24.067 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:24.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:24.106 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:24.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:24.207 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:24.252 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:24.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:24.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:24.354 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:24.397 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:24.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:24.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:24.500 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:24.530 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:24.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:24.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:24.532 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:49:24.546 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:49:24.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:24.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:24.677 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:24.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:24.778 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:24.813 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:24.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:24.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:24.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:24 smithi029 ceph-mon[97787]: pgmap v1958: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 95 B/s rd, 10 MiB/s wr, 541 op/s; 34303/314319 objects degraded (10.913%); 10390/314319 objects misplaced (3.306%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:49:24.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:24 smithi029 ceph-mon[93650]: pgmap v1958: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 95 B/s rd, 10 MiB/s wr, 541 op/s; 34303/314319 objects degraded (10.913%); 10390/314319 objects misplaced (3.306%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:49:24.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-04-12T11:49:24.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:24 smithi087 ceph-mon[89014]: pgmap v1958: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 95 B/s rd, 10 MiB/s wr, 541 op/s; 34303/314319 objects degraded (10.913%); 10390/314319 objects misplaced (3.306%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:49:24.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:24.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:24.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:25.070 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:25.118 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:25.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:25.127 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:25.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:25.222 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:49:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:49:24] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2024-04-12T11:49:25.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:25.287 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:25.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:25.388 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:25.425 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:25.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:25.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:25.527 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:25.557 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:25.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:25.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:25.650 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1958,"stamp":"2024-04-12T11:49:23.979709+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6865360874,"num_objects":104773,"num_object_clones":0,"num_object_copies":314319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34303,"num_objects_misplaced":10390,"num_objects_unfound":0,"num_objects_dirty":104773,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":113101,"num_write_kb":6963983,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1024,"num_bytes_recovered":66664012,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109767,"ondisk_log_size":109767,"up":297,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":305,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24952580,"kb_used_data":20523332,"kb_used_omap":175,"kb_used_meta":4428688,"kb_avail":725008636,"statfs":{"total":767960285184,"available":742408843264,"internally_reserved":0,"allocated":21015891968,"data_stored":20996485310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":4534976681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,8,5,8,6],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-147062760,"num_objects":-2243,"num_object_clones":0,"num_object_copies":-6729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-324,"num_objects_unfound":0,"num_objects_dirty":-2243,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5809,"num_write_kb":114049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":324,"num_bytes_recovered":21233664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.724519"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1840,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257022+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:49:13.257022+0000","last_peered":"2024-04-12T11:49:13.257022+0000","last_clean":"2024-04-12T11:49:13.257022+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:49:13.257022+0000","last_undegraded":"2024-04-12T11:49:13.257022+0000","last_fullsized":"2024-04-12T11:49:13.257022+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.258680+0000","last_change":"2024-04-12T11:48:45.352231+0000","last_active":"2024-04-12T11:49:13.258680+0000","last_peered":"2024-04-12T11:49:13.258680+0000","last_clean":"2024-04-12T11:49:13.258680+0000","last_became_active":"2024-04-12T11:48:45.351324+0000","last_became_peered":"2024-04-12T11:48:45.351324+0000","last_unstale":"2024-04-12T11:49:13.258680+0000","last_undegraded":"2024-04-12T11:49:13.258680+0000","last_fullsized":"2024-04-12T11:49:13.258680+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:01.615486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1924,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.273659+0000","last_change":"2024-04-12T11:48:45.353042+0000","last_active":"2024-04-12T11:49:13.273659+0000","last_peered":"2024-04-12T11:49:13.273659+0000","last_clean":"2024-04-12T11:49:13.273659+0000","last_became_active":"2024-04-12T11:48:45.352721+0000","last_became_peered":"2024-04-12T11:48:45.352721+0000","last_unstale":"2024-04-12T11:49:13.273659+0000","last_undegraded":"2024-04-12T11:49:13.273659+0000","last_fullsized":"2024-04-12T11:49:13.273659+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:00:32.520262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1944,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.265652+0000","last_change":"2024-04-12T11:48:45.353306+0000","last_active":"2024-04-12T11:49:13.265652+0000","last_peered":"2024-04-12T11:49:13.265652+0000","last_clean":"2024-04-12T11:49:13.265652+0000","last_became_active":"2024-04-12T11:48:45.352065+0000","last_became_peered":"2024-04-12T11:48:45.352065+0000","last_unstale":"2024-04-12T11:49:13.265652+0000","last_undegraded":"2024-04-12T11:49:13.265652+0000","last_fullsized":"2024-04-12T11:49:13.265652+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:26:36.201291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.258953+0000","last_change":"2024-04-12T11:48:45.351604+0000","last_active":"2024-04-12T11:49:13.258953+0000","last_peered":"2024-04-12T11:49:13.258953+0000","last_clean":"2024-04-12T11:49:13.258953+0000","last_became_active":"2024-04-12T11:48:45.349968+0000","last_became_peered":"2024-04-12T11:48:45.349968+0000","last_unstale":"2024-04-12T11:49:13.258953+0000","last_undegraded":"2024-04-12T11:49:13.258953+0000","last_fullsized":"2024-04-12T11:49:13.258953+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:56:42.419931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1191,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.164053+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:49:07.164053+0000","last_peered":"2024-04-12T11:49:07.164053+0000","last_clean":"2024-04-12T11:49:07.164053+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:49:07.164053+0000","last_undegraded":"2024-04-12T11:49:07.164053+0000","last_fullsized":"2024-04-12T11:49:07.164053+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1915,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.266903+0000","last_change":"2024-04-12T11:48:45.351162+0000","last_active":"2024-04-12T11:49:13.266903+0000","last_peered":"2024-04-12T11:49:13.266903+0000","last_clean":"2024-04-12T11:49:13.266903+0000","last_became_active":"2024-04-12T11:48:45.350817+0000","last_became_peered":"2024-04-12T11:48:45.350817+0000","last_unstale":"2024-04-12T11:49:13.266903+0000","last_undegraded":"2024-04-12T11:49:13.266903+0000","last_fullsized":"2024-04-12T11:49:13.266903+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:59.627216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.264114+0000","last_change":"2024-04-12T11:48:45.420195+0000","last_active":"2024-04-12T11:49:13.264114+0000","last_peered":"2024-04-12T11:49:13.264114+0000","last_clean":"2024-04-12T11:49:13.264114+0000","last_became_active":"2024-04-12T11:48:45.419810+0000","last_became_peered":"2024-04-12T11:48:45.419810+0000","last_unstale":"2024-04-12T11:49:13.264114+0000","last_undegraded":"2024-04-12T11:49:13.264114+0000","last_fullsized":"2024-04-12T11:49:13.264114+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:33:11.234641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1223,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.024017+0000","last_change":"2024-04-12T11:45:52.986340+0000","last_active":"2024-04-12T11:49:13.024017+0000","last_peered":"2024-04-12T11:49:13.024017+0000","last_clean":"2024-04-12T11:49:13.024017+0000","last_became_active":"2024-04-12T11:45:52.984190+0000","last_became_peered":"2024-04-12T11:45:52.984190+0000","last_unstale":"2024-04-12T11:49:13.024017+0000","last_undegraded":"2024-04-12T11:49:13.024017+0000","last_fullsized":"2024-04-12T11:49:13.024017+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:52:19.144894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.283093+0000","last_change":"2024-04-12T11:48:45.349940+0000","last_active":"2024-04-12T11:49:13.283093+0000","last_peered":"2024-04-12T11:49:13.283093+0000","last_clean":"2024-04-12T11:49:13.283093+0000","last_became_active":"2024-04-12T11:48:45.349603+0000","last_became_peered":"2024-04-12T11:48:45.349603+0000","last_unstale":"2024-04-12T11:49:13.283093+0000","last_undegraded":"2024-04-12T11:49:13.283093+0000","last_fullsized":"2024-04-12T11:49:13.283093+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:04:27.503011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":80,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.168627+0000","last_change":"2024-04-12T11:48:45.357689+0000","last_active":"2024-04-12T11:49:07.168627+0000","last_peered":"2024-04-12T11:49:07.168627+0000","last_clean":"2024-04-12T11:49:07.168627+0000","last_became_active":"2024-04-12T11:48:45.349500+0000","last_became_peered":"2024-04-12T11:48:45.349500+0000","last_unstale":"2024-04-12T11:49:07.168627+0000","last_undegraded":"2024-04-12T11:49:07.168627+0000","last_fullsized":"2024-04-12T11:49:07.168627+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:54:58.532082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1891,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.277790+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:49:13.277790+0000","last_peered":"2024-04-12T11:49:13.277790+0000","last_clean":"2024-04-12T11:49:13.277790+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:49:13.277790+0000","last_undegraded":"2024-04-12T11:49:13.277790+0000","last_fullsized":"2024-04-12T11:49:13.277790+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1845,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257560+0000","last_change":"2024-04-12T11:45:53.024626+0000","last_active":"2024-04-12T11:49:13.257560+0000","last_peered":"2024-04-12T11:49:13.257560+0000","last_clean":"2024-04-12T11:49:13.257560+0000","last_became_active":"2024-04-12T11:45:52.976276+0000","last_became_peered":"2024-04-12T11:45:52.976276+0000","last_unstale":"2024-04-12T11:49:13.257560+0000","last_undegraded":"2024-04-12T11:49:13.257560+0000","last_fullsized":"2024-04-12T11:49:13.257560+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:49.929210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1804,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257399+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:49:13.257399+0000","last_peered":"2024-04-12T11:49:13.257399+0000","last_clean":"2024-04-12T11:49:13.257399+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:49:13.257399+0000","last_undegraded":"2024-04-12T11:49:13.257399+0000","last_fullsized":"2024-04-12T11:49:13.257399+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":50,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.174292+0000","last_change":"2024-04-12T11:45:53.035604+0000","last_active":"2024-04-12T11:49:07.174292+0000","last_peered":"2024-04-12T11:49:07.174292+0000","last_clean":"2024-04-12T11:49:07.174292+0000","last_became_active":"2024-04-12T11:45:52.984722+0000","last_became_peered":"2024-04-12T11:45:52.984722+0000","last_unstale":"2024-04-12T11:49:07.174292+0000","last_undegraded":"2024-04-12T11:49:07.174292+0000","last_fullsized":"2024-04-12T11:49:07.174292+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:30.674395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.276728+0000","last_change":"2024-04-12T11:48:45.353499+0000","last_active":"2024-04-12T11:49:13.276728+0000","last_peered":"2024-04-12T11:49:13.276728+0000","last_clean":"2024-04-12T11:49:13.276728+0000","last_became_active":"2024-04-12T11:48:45.350015+0000","last_became_peered":"2024-04-12T11:48:45.350015+0000","last_unstale":"2024-04-12T11:49:13.276728+0000","last_undegraded":"2024-04-12T11:49:13.276728+0000","last_fullsized":"2024-04-12T11:49:13.276728+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:24:17.358128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1928,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.265318+0000","last_change":"2024-04-12T11:48:45.351398+0000","last_active":"2024-04-12T11:49:13.265318+0000","last_peered":"2024-04-12T11:49:13.265318+0000","last_clean":"2024-04-12T11:49:13.265318+0000","last_became_active":"2024-04-12T11:48:45.350886+0000","last_became_peered":"2024-04-12T11:48:45.350886+0000","last_unstale":"2024-04-12T11:49:13.265318+0000","last_undegraded":"2024-04-12T11:49:13.265318+0000","last_fullsized":"2024-04-12T11:49:13.265318+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:02.835400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1249,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.023961+0000","last_change":"2024-04-12T11:48:45.350844+0000","last_active":"2024-04-12T11:49:13.023961+0000","last_peered":"2024-04-12T11:49:13.023961+0000","last_clean":"2024-04-12T11:49:13.023961+0000","last_became_active":"2024-04-12T11:48:45.350024+0000","last_became_peered":"2024-04-12T11:48:45.350024+0000","last_unstale":"2024-04-12T11:49:13.023961+0000","last_undegraded":"2024-04-12T11:49:13.023961+0000","last_fullsized":"2024-04-12T11:49:13.023961+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:44:47.030599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":44,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.166374+0000","last_change":"2024-04-12T11:45:53.006808+0000","last_active":"2024-04-12T11:49:07.166374+0000","last_peered":"2024-04-12T11:49:07.166374+0000","last_clean":"2024-04-12T11:49:07.166374+0000","last_became_active":"2024-04-12T11:45:53.006480+0000","last_became_peered":"2024-04-12T11:45:53.006480+0000","last_unstale":"2024-04-12T11:49:07.166374+0000","last_undegraded":"2024-04-12T11:49:07.166374+0000","last_fullsized":"2024-04-12T11:49:07.166374+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:47.685096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1929,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.265741+0000","last_change":"2024-04-12T11:48:45.352862+0000","last_active":"2024-04-12T11:49:13.265741+0000","last_peered":"2024-04-12T11:49:13.265741+0000","last_clean":"2024-04-12T11:49:13.265741+0000","last_became_active":"2024-04-12T11:48:45.352485+0000","last_became_peered":"2024-04-12T11:48:45.352485+0000","last_unstale":"2024-04-12T11:49:13.265741+0000","last_undegraded":"2024-04-12T11:49:13.265741+0000","last_fullsized":"2024-04-12T11:49:13.265741+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:11:20.916845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1198,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.163752+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:49:07.163752+0000","last_peered":"2024-04-12T11:49:07.163752+0000","last_clean":"2024-04-12T11:49:07.163752+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:49:07.163752+0000","last_undegraded":"2024-04-12T11:49:07.163752+0000","last_fullsized":"2024-04-12T11:49:07.163752+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.259470+0000","last_change":"2024-04-12T11:48:45.361653+0000","last_active":"2024-04-12T11:49:13.259470+0000","last_peered":"2024-04-12T11:49:13.259470+0000","last_clean":"2024-04-12T11:49:13.259470+0000","last_became_active":"2024-04-12T11:48:45.361334+0000","last_became_peered":"2024-04-12T11:48:45.361334+0000","last_unstale":"2024-04-12T11:49:13.259470+0000","last_undegraded":"2024-04-12T11:49:13.259470+0000","last_fullsized":"2024-04-12T11:49:13.259470+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:58:25.548034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"1242'7103","reported_seq":7145,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:21.188316+0000","last_change":"2024-04-12T11:47:31.337856+0000","last_active":"2024-04-12T11:49:21.188316+0000","last_peered":"2024-04-12T11:49:21.188316+0000","last_clean":"2024-04-12T11:49:21.188316+0000","last_became_active":"2024-04-12T11:47:31.337539+0000","last_became_peered":"2024-04-12T11:47:31.337539+0000","last_unstale":"2024-04-12T11:49:21.188316+0000","last_undegraded":"2024-04-12T11:49:21.188316+0000","last_fullsized":"2024-04-12T11:49:21.188316+0000","mapping_epoch":1226,"log_start":"1230'400","ondisk_log_start":"1230'400","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6703,"log_dups_size":0,"ondisk_log_size":6703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:55:13.678253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436404224,"num_objects":6659,"num_object_clones":0,"num_object_copies":19977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7103,"num_write_kb":440384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":48,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.166526+0000","last_change":"2024-04-12T11:45:53.028640+0000","last_active":"2024-04-12T11:49:07.166526+0000","last_peered":"2024-04-12T11:49:07.166526+0000","last_clean":"2024-04-12T11:49:07.166526+0000","last_became_active":"2024-04-12T11:45:52.983099+0000","last_became_peered":"2024-04-12T11:45:52.983099+0000","last_unstale":"2024-04-12T11:49:07.166526+0000","last_undegraded":"2024-04-12T11:49:07.166526+0000","last_fullsized":"2024-04-12T11:49:07.166526+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:43:01.897014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1199,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.181077+0000","last_change":"2024-04-12T11:45:52.993160+0000","last_active":"2024-04-12T11:49:07.181077+0000","last_peered":"2024-04-12T11:49:07.181077+0000","last_clean":"2024-04-12T11:49:07.181077+0000","last_became_active":"2024-04-12T11:45:52.989310+0000","last_became_peered":"2024-04-12T11:45:52.989310+0000","last_unstale":"2024-04-12T11:49:07.181077+0000","last_undegraded":"2024-04-12T11:49:07.181077+0000","last_fullsized":"2024-04-12T11:49:07.181077+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:58:47.362374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1242'7105","reported_seq":7150,"reported_epoch":1242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:19.759441+0000","last_change":"2024-04-12T11:48:46.804539+0000","last_active":"2024-04-12T11:49:19.759441+0000","last_peered":"2024-04-12T11:49:19.759441+0000","last_clean":"2024-04-12T11:48:44.288786+0000","last_became_active":"2024-04-12T11:48:46.787738+0000","last_became_peered":"2024-04-12T11:48:46.787738+0000","last_unstale":"2024-04-12T11:49:19.759441+0000","last_undegraded":"2024-04-12T11:48:46.374623+0000","last_fullsized":"2024-04-12T11:48:46.337433+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7105,"log_dups_size":0,"ondisk_log_size":7105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443744256,"num_objects":6771,"num_object_clones":0,"num_object_copies":20313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5831,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7105,"num_write_kb":444032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":5664},{"shards":"1,4","objects":940}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":48,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.166250+0000","last_change":"2024-04-12T11:45:53.031759+0000","last_active":"2024-04-12T11:49:07.166250+0000","last_peered":"2024-04-12T11:49:07.166250+0000","last_clean":"2024-04-12T11:49:07.166250+0000","last_became_active":"2024-04-12T11:45:52.984545+0000","last_became_peered":"2024-04-12T11:45:52.984545+0000","last_unstale":"2024-04-12T11:49:07.166250+0000","last_undegraded":"2024-04-12T11:49:07.166250+0000","last_fullsized":"2024-04-12T11:49:07.166250+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:16:10.421992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1850,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257239+0000","last_change":"2024-04-12T11:48:45.351213+0000","last_active":"2024-04-12T11:49:13.257239+0000","last_peered":"2024-04-12T11:49:13.257239+0000","last_clean":"2024-04-12T11:49:13.257239+0000","last_became_active":"2024-04-12T11:48:45.350772+0000","last_became_peered":"2024-04-12T11:48:45.350772+0000","last_unstale":"2024-04-12T11:49:13.257239+0000","last_undegraded":"2024-04-12T11:49:13.257239+0000","last_fullsized":"2024-04-12T11:49:13.257239+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:17:48.831705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1242'7108","reported_seq":7141,"reported_epoch":1242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:22.948508+0000","last_change":"2024-04-12T11:48:46.932869+0000","last_active":"2024-04-12T11:49:22.948508+0000","last_peered":"2024-04-12T11:49:22.948508+0000","last_clean":"2024-04-12T11:48:44.337474+0000","last_became_active":"2024-04-12T11:48:46.916781+0000","last_became_peered":"2024-04-12T11:48:46.916781+0000","last_unstale":"2024-04-12T11:49:22.948508+0000","last_undegraded":"2024-04-12T11:48:46.385475+0000","last_fullsized":"2024-04-12T11:48:46.335429+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7108,"log_dups_size":0,"ondisk_log_size":7108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:02:58.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422445056,"num_objects":6446,"num_object_clones":0,"num_object_copies":19338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5745,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7108,"num_write_kb":433728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":5414},{"shards":"5,7","objects":701}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":48,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.168564+0000","last_change":"2024-04-12T11:45:53.169686+0000","last_active":"2024-04-12T11:49:07.168564+0000","last_peered":"2024-04-12T11:49:07.168564+0000","last_clean":"2024-04-12T11:49:07.168564+0000","last_became_active":"2024-04-12T11:45:53.021021+0000","last_became_peered":"2024-04-12T11:45:53.021021+0000","last_unstale":"2024-04-12T11:49:07.168564+0000","last_undegraded":"2024-04-12T11:49:07.168564+0000","last_fullsized":"2024-04-12T11:49:07.168564+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:48:23.506812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1217,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.162352+0000","last_change":"2024-04-12T11:45:52.979601+0000","last_active":"2024-04-12T11:49:07.162352+0000","last_peered":"2024-04-12T11:49:07.162352+0000","last_clean":"2024-04-12T11:49:07.162352+0000","last_became_active":"2024-04-12T11:45:52.979307+0000","last_became_peered":"2024-04-12T11:45:52.979307+0000","last_unstale":"2024-04-12T11:49:07.162352+0000","last_undegraded":"2024-04-12T11:49:07.162352+0000","last_fullsized":"2024-04-12T11:49:07.162352+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:21.367107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1242'7251","reported_seq":7276,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:23.005797+0000","last_change":"2024-04-12T11:47:31.345659+0000","last_active":"2024-04-12T11:49:23.005797+0000","last_peered":"2024-04-12T11:49:23.005797+0000","last_clean":"2024-04-12T11:49:23.005797+0000","last_became_active":"2024-04-12T11:47:31.345313+0000","last_became_peered":"2024-04-12T11:47:31.345313+0000","last_unstale":"2024-04-12T11:49:23.005797+0000","last_undegraded":"2024-04-12T11:49:23.005797+0000","last_fullsized":"2024-04-12T11:49:23.005797+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7251,"log_dups_size":0,"ondisk_log_size":7251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:03:10.526704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431292416,"num_objects":6581,"num_object_clones":0,"num_object_copies":19743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7251,"num_write_kb":442624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1233,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.181349+0000","last_change":"2024-04-12T11:48:45.344961+0000","last_active":"2024-04-12T11:49:07.181349+0000","last_peered":"2024-04-12T11:49:07.181349+0000","last_clean":"2024-04-12T11:49:07.181349+0000","last_became_active":"2024-04-12T11:48:45.344344+0000","last_became_peered":"2024-04-12T11:48:45.344344+0000","last_unstale":"2024-04-12T11:49:07.181349+0000","last_undegraded":"2024-04-12T11:49:07.181349+0000","last_fullsized":"2024-04-12T11:49:07.181349+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:47:55.659541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1903,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.267860+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:49:13.267860+0000","last_peered":"2024-04-12T11:49:13.267860+0000","last_clean":"2024-04-12T11:49:13.267860+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:49:13.267860+0000","last_undegraded":"2024-04-12T11:49:13.267860+0000","last_fullsized":"2024-04-12T11:49:13.267860+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"249'161","reported_seq":1559,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.292990+0000","last_change":"2024-04-12T11:49:07.188158+0000","last_active":"2024-04-12T11:49:07.292990+0000","last_peered":"2024-04-12T11:49:07.292990+0000","last_clean":"2024-04-12T11:49:07.292990+0000","last_became_active":"2024-04-12T11:49:07.187847+0000","last_became_peered":"2024-04-12T11:49:07.187847+0000","last_unstale":"2024-04-12T11:49:07.292990+0000","last_undegraded":"2024-04-12T11:49:07.292990+0000","last_fullsized":"2024-04-12T11:49:07.292990+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.1","version":"1242'6996","reported_seq":9004,"reported_epoch":1242,"state":"active+recovering+remapped","last_fresh":"2024-04-12T11:49:20.277472+0000","last_change":"2024-04-12T11:48:46.724770+0000","last_active":"2024-04-12T11:49:20.277472+0000","last_peered":"2024-04-12T11:49:20.277472+0000","last_clean":"2024-04-12T11:48:44.315992+0000","last_became_active":"2024-04-12T11:48:46.712972+0000","last_became_peered":"2024-04-12T11:48:46.712972+0000","last_unstale":"2024-04-12T11:49:20.277472+0000","last_undegraded":"2024-04-12T11:49:20.277472+0000","last_fullsized":"2024-04-12T11:49:20.277472+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6996,"log_dups_size":0,"ondisk_log_size":6996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432144384,"num_objects":6594,"num_object_clones":0,"num_object_copies":19782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4779,"num_objects_unfound":0,"num_objects_dirty":6594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6996,"num_write_kb":434880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":985,"num_bytes_recovered":64552960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[4,6,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":1815}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":29,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.258014+0000","last_change":"2024-04-12T11:48:45.359254+0000","last_active":"2024-04-12T11:49:13.258014+0000","last_peered":"2024-04-12T11:49:13.258014+0000","last_clean":"2024-04-12T11:49:13.258014+0000","last_became_active":"2024-04-12T11:48:45.348788+0000","last_became_peered":"2024-04-12T11:48:45.348788+0000","last_unstale":"2024-04-12T11:49:13.258014+0000","last_undegraded":"2024-04-12T11:49:13.258014+0000","last_fullsized":"2024-04-12T11:49:13.258014+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:15:04.501653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1841,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257424+0000","last_change":"2024-04-12T11:48:45.351839+0000","last_active":"2024-04-12T11:49:13.257424+0000","last_peered":"2024-04-12T11:49:13.257424+0000","last_clean":"2024-04-12T11:49:13.257424+0000","last_became_active":"2024-04-12T11:48:45.351276+0000","last_became_peered":"2024-04-12T11:48:45.351276+0000","last_unstale":"2024-04-12T11:49:13.257424+0000","last_undegraded":"2024-04-12T11:49:13.257424+0000","last_fullsized":"2024-04-12T11:49:13.257424+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:23:03.106998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"249'161","reported_seq":1559,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.292914+0000","last_change":"2024-04-12T11:49:07.233588+0000","last_active":"2024-04-12T11:49:07.292914+0000","last_peered":"2024-04-12T11:49:07.292914+0000","last_clean":"2024-04-12T11:49:07.292914+0000","last_became_active":"2024-04-12T11:49:07.233149+0000","last_became_peered":"2024-04-12T11:49:07.233149+0000","last_unstale":"2024-04-12T11:49:07.292914+0000","last_undegraded":"2024-04-12T11:49:07.292914+0000","last_fullsized":"2024-04-12T11:49:07.292914+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.0","version":"1242'7010","reported_seq":7052,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:19.736638+0000","last_change":"2024-04-12T11:47:31.345847+0000","last_active":"2024-04-12T11:49:19.736638+0000","last_peered":"2024-04-12T11:49:19.736638+0000","last_clean":"2024-04-12T11:49:19.736638+0000","last_became_active":"2024-04-12T11:47:31.345534+0000","last_became_peered":"2024-04-12T11:47:31.345534+0000","last_unstale":"2024-04-12T11:49:19.736638+0000","last_undegraded":"2024-04-12T11:49:19.736638+0000","last_fullsized":"2024-04-12T11:49:19.736638+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:30:47.116045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438042624,"num_objects":6684,"num_object_clones":0,"num_object_copies":20052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7010,"num_write_kb":438208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":843,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257151+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:49:13.257151+0000","last_peered":"2024-04-12T11:49:13.257151+0000","last_clean":"2024-04-12T11:49:13.257151+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:49:13.257151+0000","last_undegraded":"2024-04-12T11:49:13.257151+0000","last_fullsized":"2024-04-12T11:49:13.257151+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1839,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257284+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:49:13.257284+0000","last_peered":"2024-04-12T11:49:13.257284+0000","last_clean":"2024-04-12T11:49:13.257284+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:49:13.257284+0000","last_undegraded":"2024-04-12T11:49:13.257284+0000","last_fullsized":"2024-04-12T11:49:13.257284+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1242'6980","reported_seq":7012,"reported_epoch":1242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:22.299878+0000","last_change":"2024-04-12T11:48:46.758392+0000","last_active":"2024-04-12T11:49:22.299878+0000","last_peered":"2024-04-12T11:49:22.299878+0000","last_clean":"2024-04-12T11:48:44.327141+0000","last_became_active":"2024-04-12T11:48:46.750040+0000","last_became_peered":"2024-04-12T11:48:46.750040+0000","last_unstale":"2024-04-12T11:49:22.299878+0000","last_undegraded":"2024-04-12T11:48:46.373778+0000","last_fullsized":"2024-04-12T11:48:46.339427+0000","mapping_epoch":1235,"log_start":"1230'1012","ondisk_log_start":"1230'1012","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5968,"log_dups_size":0,"ondisk_log_size":5968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:54:20.338800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419299328,"num_objects":6398,"num_object_clones":0,"num_object_copies":19194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5634,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6980,"num_write_kb":428096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,1],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":5343},{"shards":"1,7","objects":764}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1837,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257441+0000","last_change":"2024-04-12T11:45:52.976139+0000","last_active":"2024-04-12T11:49:13.257441+0000","last_peered":"2024-04-12T11:49:13.257441+0000","last_clean":"2024-04-12T11:49:13.257441+0000","last_became_active":"2024-04-12T11:45:52.975803+0000","last_became_peered":"2024-04-12T11:45:52.975803+0000","last_unstale":"2024-04-12T11:49:13.257441+0000","last_undegraded":"2024-04-12T11:49:13.257441+0000","last_fullsized":"2024-04-12T11:49:13.257441+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:46.618143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1804,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257673+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:49:13.257673+0000","last_peered":"2024-04-12T11:49:13.257673+0000","last_clean":"2024-04-12T11:49:13.257673+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:49:13.257673+0000","last_undegraded":"2024-04-12T11:49:13.257673+0000","last_fullsized":"2024-04-12T11:49:13.257673+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"249'161","reported_seq":1559,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.293009+0000","last_change":"2024-04-12T11:49:07.186288+0000","last_active":"2024-04-12T11:49:07.293009+0000","last_peered":"2024-04-12T11:49:07.293009+0000","last_clean":"2024-04-12T11:49:07.293009+0000","last_became_active":"2024-04-12T11:49:07.185683+0000","last_became_peered":"2024-04-12T11:49:07.185683+0000","last_unstale":"2024-04-12T11:49:07.293009+0000","last_undegraded":"2024-04-12T11:49:07.293009+0000","last_fullsized":"2024-04-12T11:49:07.293009+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2","version":"1242'6954","reported_seq":6998,"reported_epoch":1242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:19.769714+0000","last_change":"2024-04-12T11:48:46.765801+0000","last_active":"2024-04-12T11:49:19.769714+0000","last_peered":"2024-04-12T11:49:19.769714+0000","last_clean":"2024-04-12T11:48:44.392445+0000","last_became_active":"2024-04-12T11:48:46.751638+0000","last_became_peered":"2024-04-12T11:48:46.751638+0000","last_unstale":"2024-04-12T11:49:19.769714+0000","last_undegraded":"2024-04-12T11:48:46.375433+0000","last_fullsized":"2024-04-12T11:48:46.336636+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6954,"log_dups_size":0,"ondisk_log_size":6954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433848320,"num_objects":6620,"num_object_clones":0,"num_object_copies":19860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5646,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6954,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2","objects":974},{"shards":"1,2,7","objects":5479}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.264386+0000","last_change":"2024-04-12T11:48:45.351852+0000","last_active":"2024-04-12T11:49:13.264386+0000","last_peered":"2024-04-12T11:49:13.264386+0000","last_clean":"2024-04-12T11:49:13.264386+0000","last_became_active":"2024-04-12T11:48:45.350286+0000","last_became_peered":"2024-04-12T11:48:45.350286+0000","last_unstale":"2024-04-12T11:49:13.264386+0000","last_undegraded":"2024-04-12T11:49:13.264386+0000","last_fullsized":"2024-04-12T11:49:13.264386+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:51:26.610188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1931,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.274063+0000","last_change":"2024-04-12T11:48:45.361378+0000","last_active":"2024-04-12T11:49:13.274063+0000","last_peered":"2024-04-12T11:49:13.274063+0000","last_clean":"2024-04-12T11:49:13.274063+0000","last_became_active":"2024-04-12T11:48:45.360917+0000","last_became_peered":"2024-04-12T11:48:45.360917+0000","last_unstale":"2024-04-12T11:49:13.274063+0000","last_undegraded":"2024-04-12T11:49:13.274063+0000","last_fullsized":"2024-04-12T11:49:13.274063+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:46:07.334663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"249'161","reported_seq":1559,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.293210+0000","last_change":"2024-04-12T11:49:07.185970+0000","last_active":"2024-04-12T11:49:07.293210+0000","last_peered":"2024-04-12T11:49:07.293210+0000","last_clean":"2024-04-12T11:49:07.293210+0000","last_became_active":"2024-04-12T11:49:07.185643+0000","last_became_peered":"2024-04-12T11:49:07.185643+0000","last_unstale":"2024-04-12T11:49:07.293210+0000","last_undegraded":"2024-04-12T11:49:07.293210+0000","last_fullsized":"2024-04-12T11:49:07.293210+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.f","version":"1242'7123","reported_seq":7155,"reported_epoch":1242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:22.976385+0000","last_change":"2024-04-12T11:48:46.932836+0000","last_active":"2024-04-12T11:49:22.976385+0000","last_peered":"2024-04-12T11:49:22.976385+0000","last_clean":"2024-04-12T11:48:44.299270+0000","last_became_active":"2024-04-12T11:48:46.917768+0000","last_became_peered":"2024-04-12T11:48:46.917768+0000","last_unstale":"2024-04-12T11:49:22.976385+0000","last_undegraded":"2024-04-12T11:48:46.478160+0000","last_fullsized":"2024-04-12T11:48:46.335134+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7123,"log_dups_size":0,"ondisk_log_size":7123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:00:52.272820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424083456,"num_objects":6471,"num_object_clones":0,"num_object_copies":19413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5721,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7123,"num_write_kb":435008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":5395},{"shards":"1,5","objects":750}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1914,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.277912+0000","last_change":"2024-04-12T11:45:52.980673+0000","last_active":"2024-04-12T11:49:13.277912+0000","last_peered":"2024-04-12T11:49:13.277912+0000","last_clean":"2024-04-12T11:49:13.277912+0000","last_became_active":"2024-04-12T11:45:52.978978+0000","last_became_peered":"2024-04-12T11:45:52.978978+0000","last_unstale":"2024-04-12T11:49:13.277912+0000","last_undegraded":"2024-04-12T11:49:13.277912+0000","last_fullsized":"2024-04-12T11:49:13.277912+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:32:10.303905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1249,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.181602+0000","last_change":"2024-04-12T11:48:45.419824+0000","last_active":"2024-04-12T11:49:07.181602+0000","last_peered":"2024-04-12T11:49:07.181602+0000","last_clean":"2024-04-12T11:49:07.181602+0000","last_became_active":"2024-04-12T11:48:45.419375+0000","last_became_peered":"2024-04-12T11:48:45.419375+0000","last_unstale":"2024-04-12T11:49:07.181602+0000","last_undegraded":"2024-04-12T11:49:07.181602+0000","last_fullsized":"2024-04-12T11:49:07.181602+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:19:30.300350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1893,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.277696+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:49:13.277696+0000","last_peered":"2024-04-12T11:49:13.277696+0000","last_clean":"2024-04-12T11:49:13.277696+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:49:13.277696+0000","last_undegraded":"2024-04-12T11:49:13.277696+0000","last_fullsized":"2024-04-12T11:49:13.277696+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"249'161","reported_seq":1558,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.292963+0000","last_change":"2024-04-12T11:49:07.180601+0000","last_active":"2024-04-12T11:49:07.292963+0000","last_peered":"2024-04-12T11:49:07.292963+0000","last_clean":"2024-04-12T11:49:07.292963+0000","last_became_active":"2024-04-12T11:49:07.180271+0000","last_became_peered":"2024-04-12T11:49:07.180271+0000","last_unstale":"2024-04-12T11:49:07.292963+0000","last_undegraded":"2024-04-12T11:49:07.292963+0000","last_fullsized":"2024-04-12T11:49:07.292963+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184891,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.9","version":"1242'6975","reported_seq":7017,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:21.221320+0000","last_change":"2024-04-12T11:47:31.337684+0000","last_active":"2024-04-12T11:49:21.221320+0000","last_peered":"2024-04-12T11:49:21.221320+0000","last_clean":"2024-04-12T11:49:21.221320+0000","last_became_active":"2024-04-12T11:47:31.337391+0000","last_became_peered":"2024-04-12T11:47:31.337391+0000","last_unstale":"2024-04-12T11:49:21.221320+0000","last_undegraded":"2024-04-12T11:49:21.221320+0000","last_fullsized":"2024-04-12T11:49:21.221320+0000","mapping_epoch":1226,"log_start":"1230'300","ondisk_log_start":"1230'300","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6675,"log_dups_size":0,"ondisk_log_size":6675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:11:18.213572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426704896,"num_objects":6511,"num_object_clones":0,"num_object_copies":19533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6975,"num_write_kb":431552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1202,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.181196+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:49:07.181196+0000","last_peered":"2024-04-12T11:49:07.181196+0000","last_clean":"2024-04-12T11:49:07.181196+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:49:07.181196+0000","last_undegraded":"2024-04-12T11:49:07.181196+0000","last_fullsized":"2024-04-12T11:49:07.181196+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1826,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257183+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:49:13.257183+0000","last_peered":"2024-04-12T11:49:13.257183+0000","last_clean":"2024-04-12T11:49:13.257183+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:49:13.257183+0000","last_undegraded":"2024-04-12T11:49:13.257183+0000","last_fullsized":"2024-04-12T11:49:13.257183+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":137,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.267146+0000","last_change":"2024-04-12T11:48:45.418653+0000","last_active":"2024-04-12T11:49:13.267146+0000","last_peered":"2024-04-12T11:49:13.267146+0000","last_clean":"2024-04-12T11:49:13.267146+0000","last_became_active":"2024-04-12T11:48:45.418274+0000","last_became_peered":"2024-04-12T11:48:45.418274+0000","last_unstale":"2024-04-12T11:49:13.267146+0000","last_undegraded":"2024-04-12T11:49:13.267146+0000","last_fullsized":"2024-04-12T11:49:13.267146+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:39:24.012667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"249'161","reported_seq":1559,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.293028+0000","last_change":"2024-04-12T11:49:07.185964+0000","last_active":"2024-04-12T11:49:07.293028+0000","last_peered":"2024-04-12T11:49:07.293028+0000","last_clean":"2024-04-12T11:49:07.293028+0000","last_became_active":"2024-04-12T11:49:07.185523+0000","last_became_peered":"2024-04-12T11:49:07.185523+0000","last_unstale":"2024-04-12T11:49:07.293028+0000","last_undegraded":"2024-04-12T11:49:07.293028+0000","last_fullsized":"2024-04-12T11:49:07.293028+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184891,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.a","version":"1242'7128","reported_seq":7152,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:23.002484+0000","last_change":"2024-04-12T11:47:31.341656+0000","last_active":"2024-04-12T11:49:23.002484+0000","last_peered":"2024-04-12T11:49:23.002484+0000","last_clean":"2024-04-12T11:49:23.002484+0000","last_became_active":"2024-04-12T11:47:31.341335+0000","last_became_peered":"2024-04-12T11:47:31.341335+0000","last_unstale":"2024-04-12T11:49:23.002484+0000","last_undegraded":"2024-04-12T11:49:23.002484+0000","last_fullsized":"2024-04-12T11:49:23.002484+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7128,"log_dups_size":0,"ondisk_log_size":7128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:16:26.010127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426901504,"num_objects":6514,"num_object_clones":0,"num_object_copies":19542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7128,"num_write_kb":436544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1890,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.265961+0000","last_change":"2024-04-12T11:48:45.417421+0000","last_active":"2024-04-12T11:49:13.265961+0000","last_peered":"2024-04-12T11:49:13.265961+0000","last_clean":"2024-04-12T11:49:13.265961+0000","last_became_active":"2024-04-12T11:48:45.417046+0000","last_became_peered":"2024-04-12T11:48:45.417046+0000","last_unstale":"2024-04-12T11:49:13.265961+0000","last_undegraded":"2024-04-12T11:49:13.265961+0000","last_fullsized":"2024-04-12T11:49:13.265961+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:17:29.993647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1894,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.275240+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:49:13.275240+0000","last_peered":"2024-04-12T11:49:13.275240+0000","last_clean":"2024-04-12T11:49:13.275240+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:49:13.275240+0000","last_undegraded":"2024-04-12T11:49:13.275240+0000","last_fullsized":"2024-04-12T11:49:13.275240+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1896,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.273924+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:49:13.273924+0000","last_peered":"2024-04-12T11:49:13.273924+0000","last_clean":"2024-04-12T11:49:13.273924+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:49:13.273924+0000","last_undegraded":"2024-04-12T11:49:13.273924+0000","last_fullsized":"2024-04-12T11:49:13.273924+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1242'7102","reported_seq":7146,"reported_epoch":1242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:21.218508+0000","last_change":"2024-04-12T11:48:46.811757+0000","last_active":"2024-04-12T11:49:21.218508+0000","last_peered":"2024-04-12T11:49:21.218508+0000","last_clean":"2024-04-12T11:48:44.316482+0000","last_became_active":"2024-04-12T11:48:46.789368+0000","last_became_peered":"2024-04-12T11:48:46.789368+0000","last_unstale":"2024-04-12T11:49:21.218508+0000","last_undegraded":"2024-04-12T11:48:46.382926+0000","last_fullsized":"2024-04-12T11:48:46.336397+0000","mapping_epoch":1235,"log_start":"1230'400","ondisk_log_start":"1230'400","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6702,"log_dups_size":0,"ondisk_log_size":6702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431751168,"num_objects":6588,"num_object_clones":0,"num_object_copies":19764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5726,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7102,"num_write_kb":438080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":862},{"shards":"0,5,7","objects":5469}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1229,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.181245+0000","last_change":"2024-04-12T11:45:52.986504+0000","last_active":"2024-04-12T11:49:07.181245+0000","last_peered":"2024-04-12T11:49:07.181245+0000","last_clean":"2024-04-12T11:49:07.181245+0000","last_became_active":"2024-04-12T11:45:52.984346+0000","last_became_peered":"2024-04-12T11:45:52.984346+0000","last_unstale":"2024-04-12T11:49:07.181245+0000","last_undegraded":"2024-04-12T11:49:07.181245+0000","last_fullsized":"2024-04-12T11:49:07.181245+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:26.859024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1559,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.218961+0000","last_change":"2024-04-12T11:49:07.218311+0000","last_active":"2024-04-12T11:49:07.218961+0000","last_peered":"2024-04-12T11:49:07.218961+0000","last_clean":"2024-04-12T11:49:07.218961+0000","last_became_active":"2024-04-12T11:49:07.217921+0000","last_became_peered":"2024-04-12T11:49:07.217921+0000","last_unstale":"2024-04-12T11:49:07.218961+0000","last_undegraded":"2024-04-12T11:49:07.218961+0000","last_fullsized":"2024-04-12T11:49:07.218961+0000","mapping_epoch":1210,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257785+0000","last_change":"2024-04-12T11:48:45.352874+0000","last_active":"2024-04-12T11:49:13.257785+0000","last_peered":"2024-04-12T11:49:13.257785+0000","last_clean":"2024-04-12T11:49:13.257785+0000","last_became_active":"2024-04-12T11:48:45.352350+0000","last_became_peered":"2024-04-12T11:48:45.352350+0000","last_unstale":"2024-04-12T11:49:13.257785+0000","last_undegraded":"2024-04-12T11:49:13.257785+0000","last_fullsized":"2024-04-12T11:49:13.257785+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:44:46.925487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1209,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.163584+0000","last_change":"2024-04-12T11:45:53.016616+0000","last_active":"2024-04-12T11:49:07.163584+0000","last_peered":"2024-04-12T11:49:07.163584+0000","last_clean":"2024-04-12T11:49:07.163584+0000","last_became_active":"2024-04-12T11:45:53.016292+0000","last_became_peered":"2024-04-12T11:45:53.016292+0000","last_unstale":"2024-04-12T11:49:07.163584+0000","last_undegraded":"2024-04-12T11:49:07.163584+0000","last_fullsized":"2024-04-12T11:49:07.163584+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:53:29.382293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"249'161","reported_seq":1559,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.293046+0000","last_change":"2024-04-12T11:49:07.187065+0000","last_active":"2024-04-12T11:49:07.293046+0000","last_peered":"2024-04-12T11:49:07.293046+0000","last_clean":"2024-04-12T11:49:07.293046+0000","last_became_active":"2024-04-12T11:49:07.186776+0000","last_became_peered":"2024-04-12T11:49:07.186776+0000","last_unstale":"2024-04-12T11:49:07.293046+0000","last_undegraded":"2024-04-12T11:49:07.293046+0000","last_fullsized":"2024-04-12T11:49:07.293046+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"1242'6893","reported_seq":6935,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:21.225173+0000","last_change":"2024-04-12T11:47:31.337491+0000","last_active":"2024-04-12T11:49:21.225173+0000","last_peered":"2024-04-12T11:49:21.225173+0000","last_clean":"2024-04-12T11:49:21.225173+0000","last_became_active":"2024-04-12T11:47:31.337137+0000","last_became_peered":"2024-04-12T11:47:31.337137+0000","last_unstale":"2024-04-12T11:49:21.225173+0000","last_undegraded":"2024-04-12T11:49:21.225173+0000","last_fullsized":"2024-04-12T11:49:21.225173+0000","mapping_epoch":1226,"log_start":"1230'200","ondisk_log_start":"1230'200","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6693,"log_dups_size":0,"ondisk_log_size":6693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:21:25.556372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420806656,"num_objects":6421,"num_object_clones":0,"num_object_copies":19263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6893,"num_write_kb":426048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.166005+0000","last_change":"2024-04-12T11:45:52.987902+0000","last_active":"2024-04-12T11:49:07.166005+0000","last_peered":"2024-04-12T11:49:07.166005+0000","last_clean":"2024-04-12T11:49:07.166005+0000","last_became_active":"2024-04-12T11:45:52.980281+0000","last_became_peered":"2024-04-12T11:45:52.980281+0000","last_unstale":"2024-04-12T11:49:07.166005+0000","last_undegraded":"2024-04-12T11:49:07.166005+0000","last_fullsized":"2024-04-12T11:49:07.166005+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:07:12.203048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1928,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.265416+0000","last_change":"2024-04-12T11:48:45.351359+0000","last_active":"2024-04-12T11:49:13.265416+0000","last_peered":"2024-04-12T11:49:13.265416+0000","last_clean":"2024-04-12T11:49:13.265416+0000","last_became_active":"2024-04-12T11:48:45.350809+0000","last_became_peered":"2024-04-12T11:48:45.350809+0000","last_unstale":"2024-04-12T11:49:13.265416+0000","last_undegraded":"2024-04-12T11:49:13.265416+0000","last_fullsized":"2024-04-12T11:49:13.265416+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:58:36.906714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1903,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.265824+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:49:13.265824+0000","last_peered":"2024-04-12T11:49:13.265824+0000","last_clean":"2024-04-12T11:49:13.265824+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:49:13.265824+0000","last_undegraded":"2024-04-12T11:49:13.265824+0000","last_fullsized":"2024-04-12T11:49:13.265824+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"249'161","reported_seq":1559,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.293181+0000","last_change":"2024-04-12T11:49:07.186438+0000","last_active":"2024-04-12T11:49:07.293181+0000","last_peered":"2024-04-12T11:49:07.293181+0000","last_clean":"2024-04-12T11:49:07.293181+0000","last_became_active":"2024-04-12T11:49:07.185633+0000","last_became_peered":"2024-04-12T11:49:07.185633+0000","last_unstale":"2024-04-12T11:49:07.293181+0000","last_undegraded":"2024-04-12T11:49:07.293181+0000","last_fullsized":"2024-04-12T11:49:07.293181+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.e","version":"1242'7066","reported_seq":7091,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:22.244367+0000","last_change":"2024-04-12T11:47:31.338497+0000","last_active":"2024-04-12T11:49:22.244367+0000","last_peered":"2024-04-12T11:49:22.244367+0000","last_clean":"2024-04-12T11:49:22.244367+0000","last_became_active":"2024-04-12T11:47:31.338160+0000","last_became_peered":"2024-04-12T11:47:31.338160+0000","last_unstale":"2024-04-12T11:49:22.244367+0000","last_undegraded":"2024-04-12T11:49:22.244367+0000","last_fullsized":"2024-04-12T11:49:22.244367+0000","mapping_epoch":1226,"log_start":"1230'1180","ondisk_log_start":"1230'1180","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5886,"log_dups_size":0,"ondisk_log_size":5886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:22:37.585891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426901504,"num_objects":6514,"num_object_clones":0,"num_object_copies":19542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7066,"num_write_kb":434560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":44,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.168547+0000","last_change":"2024-04-12T11:45:53.005585+0000","last_active":"2024-04-12T11:49:07.168547+0000","last_peered":"2024-04-12T11:49:07.168547+0000","last_clean":"2024-04-12T11:49:07.168547+0000","last_became_active":"2024-04-12T11:45:52.987880+0000","last_became_peered":"2024-04-12T11:45:52.987880+0000","last_unstale":"2024-04-12T11:49:07.168547+0000","last_undegraded":"2024-04-12T11:49:07.168547+0000","last_fullsized":"2024-04-12T11:49:07.168547+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:09.819861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1893,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.277810+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:49:13.277810+0000","last_peered":"2024-04-12T11:49:13.277810+0000","last_clean":"2024-04-12T11:49:13.277810+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:49:13.277810+0000","last_undegraded":"2024-04-12T11:49:13.277810+0000","last_fullsized":"2024-04-12T11:49:13.277810+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1899,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.268076+0000","last_change":"2024-04-12T11:45:52.982299+0000","last_active":"2024-04-12T11:49:13.268076+0000","last_peered":"2024-04-12T11:49:13.268076+0000","last_clean":"2024-04-12T11:49:13.268076+0000","last_became_active":"2024-04-12T11:45:52.981942+0000","last_became_peered":"2024-04-12T11:45:52.981942+0000","last_unstale":"2024-04-12T11:49:13.268076+0000","last_undegraded":"2024-04-12T11:49:13.268076+0000","last_fullsized":"2024-04-12T11:49:13.268076+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:18:13.711200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"249'161","reported_seq":1559,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.293119+0000","last_change":"2024-04-12T11:49:07.187534+0000","last_active":"2024-04-12T11:49:07.293119+0000","last_peered":"2024-04-12T11:49:07.293119+0000","last_clean":"2024-04-12T11:49:07.293119+0000","last_became_active":"2024-04-12T11:49:07.187286+0000","last_became_peered":"2024-04-12T11:49:07.187286+0000","last_unstale":"2024-04-12T11:49:07.293119+0000","last_undegraded":"2024-04-12T11:49:07.293119+0000","last_fullsized":"2024-04-12T11:49:07.293119+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"1242'7126","reported_seq":7152,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:22.262220+0000","last_change":"2024-04-12T11:47:31.339988+0000","last_active":"2024-04-12T11:49:22.262220+0000","last_peered":"2024-04-12T11:49:22.262220+0000","last_clean":"2024-04-12T11:49:22.262220+0000","last_became_active":"2024-04-12T11:47:31.339650+0000","last_became_peered":"2024-04-12T11:47:31.339650+0000","last_unstale":"2024-04-12T11:49:22.262220+0000","last_undegraded":"2024-04-12T11:49:22.262220+0000","last_fullsized":"2024-04-12T11:49:22.262220+0000","mapping_epoch":1226,"log_start":"1230'1168","ondisk_log_start":"1230'1168","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5958,"log_dups_size":0,"ondisk_log_size":5958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:02:57.059787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429457432,"num_objects":6554,"num_object_clones":0,"num_object_copies":19662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6554,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7126,"num_write_kb":437697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":29,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.259557+0000","last_change":"2024-04-12T11:48:45.361376+0000","last_active":"2024-04-12T11:49:13.259557+0000","last_peered":"2024-04-12T11:49:13.259557+0000","last_clean":"2024-04-12T11:49:13.259557+0000","last_became_active":"2024-04-12T11:48:45.354494+0000","last_became_peered":"2024-04-12T11:48:45.354494+0000","last_unstale":"2024-04-12T11:49:13.259557+0000","last_undegraded":"2024-04-12T11:49:13.259557+0000","last_fullsized":"2024-04-12T11:49:13.259557+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:42:08.500124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1893,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.277488+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:49:13.277488+0000","last_peered":"2024-04-12T11:49:13.277488+0000","last_clean":"2024-04-12T11:49:13.277488+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:49:13.277488+0000","last_undegraded":"2024-04-12T11:49:13.277488+0000","last_fullsized":"2024-04-12T11:49:13.277488+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.258850+0000","last_change":"2024-04-12T11:48:45.349034+0000","last_active":"2024-04-12T11:49:13.258850+0000","last_peered":"2024-04-12T11:49:13.258850+0000","last_clean":"2024-04-12T11:49:13.258850+0000","last_became_active":"2024-04-12T11:48:45.348579+0000","last_became_peered":"2024-04-12T11:48:45.348579+0000","last_unstale":"2024-04-12T11:49:13.258850+0000","last_undegraded":"2024-04-12T11:49:13.258850+0000","last_fullsized":"2024-04-12T11:49:13.258850+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:57.578035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"249'161","reported_seq":1559,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.293099+0000","last_change":"2024-04-12T11:49:07.199005+0000","last_active":"2024-04-12T11:49:07.293099+0000","last_peered":"2024-04-12T11:49:07.293099+0000","last_clean":"2024-04-12T11:49:07.293099+0000","last_became_active":"2024-04-12T11:49:07.198633+0000","last_became_peered":"2024-04-12T11:49:07.198633+0000","last_unstale":"2024-04-12T11:49:07.293099+0000","last_undegraded":"2024-04-12T11:49:07.293099+0000","last_fullsized":"2024-04-12T11:49:07.293099+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"1242'6881","reported_seq":6924,"reported_epoch":1242,"state":"active+recovery_wait+remapped","last_fresh":"2024-04-12T11:49:21.206311+0000","last_change":"2024-04-12T11:48:46.816570+0000","last_active":"2024-04-12T11:49:21.206311+0000","last_peered":"2024-04-12T11:49:21.206311+0000","last_clean":"2024-04-12T11:48:44.312876+0000","last_became_active":"2024-04-12T11:48:46.789195+0000","last_became_peered":"2024-04-12T11:48:46.789195+0000","last_unstale":"2024-04-12T11:49:21.206311+0000","last_undegraded":"2024-04-12T11:49:21.206311+0000","last_fullsized":"2024-04-12T11:49:21.206311+0000","mapping_epoch":1235,"log_start":"1230'200","ondisk_log_start":"1230'200","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420937728,"num_objects":6423,"num_object_clones":0,"num_object_copies":19269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5611,"num_objects_unfound":0,"num_objects_dirty":6423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6881,"num_write_kb":425728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":812}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1242,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.164127+0000","last_change":"2024-04-12T11:45:52.979745+0000","last_active":"2024-04-12T11:49:07.164127+0000","last_peered":"2024-04-12T11:49:07.164127+0000","last_clean":"2024-04-12T11:49:07.164127+0000","last_became_active":"2024-04-12T11:45:52.978936+0000","last_became_peered":"2024-04-12T11:45:52.978936+0000","last_unstale":"2024-04-12T11:49:07.164127+0000","last_undegraded":"2024-04-12T11:49:07.164127+0000","last_fullsized":"2024-04-12T11:49:07.164127+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:22.710430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1906,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.265551+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:49:13.265551+0000","last_peered":"2024-04-12T11:49:13.265551+0000","last_clean":"2024-04-12T11:49:13.265551+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:49:13.265551+0000","last_undegraded":"2024-04-12T11:49:13.265551+0000","last_fullsized":"2024-04-12T11:49:13.265551+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1813,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257014+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:49:13.257014+0000","last_peered":"2024-04-12T11:49:13.257014+0000","last_clean":"2024-04-12T11:49:13.257014+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:49:13.257014+0000","last_undegraded":"2024-04-12T11:49:13.257014+0000","last_fullsized":"2024-04-12T11:49:13.257014+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1907,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.267992+0000","last_change":"2024-04-12T11:45:52.979689+0000","last_active":"2024-04-12T11:49:13.267992+0000","last_peered":"2024-04-12T11:49:13.267992+0000","last_clean":"2024-04-12T11:49:13.267992+0000","last_became_active":"2024-04-12T11:45:52.979286+0000","last_became_peered":"2024-04-12T11:45:52.979286+0000","last_unstale":"2024-04-12T11:49:13.267992+0000","last_undegraded":"2024-04-12T11:49:13.267992+0000","last_fullsized":"2024-04-12T11:49:13.267992+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:10.453472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":169,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.181433+0000","last_change":"2024-04-12T11:48:45.344748+0000","last_active":"2024-04-12T11:49:07.181433+0000","last_peered":"2024-04-12T11:49:07.181433+0000","last_clean":"2024-04-12T11:49:07.181433+0000","last_became_active":"2024-04-12T11:48:45.344290+0000","last_became_peered":"2024-04-12T11:48:45.344290+0000","last_unstale":"2024-04-12T11:49:07.181433+0000","last_undegraded":"2024-04-12T11:49:07.181433+0000","last_fullsized":"2024-04-12T11:49:07.181433+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:13.440125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1212,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.181183+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:49:07.181183+0000","last_peered":"2024-04-12T11:49:07.181183+0000","last_clean":"2024-04-12T11:49:07.181183+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:49:07.181183+0000","last_undegraded":"2024-04-12T11:49:07.181183+0000","last_fullsized":"2024-04-12T11:49:07.181183+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1864,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257510+0000","last_change":"2024-04-12T11:48:45.419685+0000","last_active":"2024-04-12T11:49:13.257510+0000","last_peered":"2024-04-12T11:49:13.257510+0000","last_clean":"2024-04-12T11:49:13.257510+0000","last_became_active":"2024-04-12T11:48:45.419344+0000","last_became_peered":"2024-04-12T11:48:45.419344+0000","last_unstale":"2024-04-12T11:49:13.257510+0000","last_undegraded":"2024-04-12T11:49:13.257510+0000","last_fullsized":"2024-04-12T11:49:13.257510+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:24:01.383355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1197,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-04-12T11:49:07.181680+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:49:07.181680+0000","last_peered":"2024-04-12T11:49:07.181680+0000","last_clean":"2024-04-12T11:49:07.181680+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:49:07.181680+0000","last_undegraded":"2024-04-12T11:49:07.181680+0000","last_fullsized":"2024-04-12T11:49:07.181680+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1855,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.257263+0000","last_change":"2024-04-12T11:48:45.351004+0000","last_active":"2024-04-12T11:49:13.257263+0000","last_peered":"2024-04-12T11:49:13.257263+0000","last_clean":"2024-04-12T11:49:13.257263+0000","last_became_active":"2024-04-12T11:48:45.350567+0000","last_became_peered":"2024-04-12T11:48:45.350567+0000","last_unstale":"2024-04-12T11:49:13.257263+0000","last_undegraded":"2024-04-12T11:49:13.257263+0000","last_fullsized":"2024-04-12T11:49:13.257263+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:48:43.488302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.259043+0000","last_change":"2024-04-12T11:48:45.418133+0000","last_active":"2024-04-12T11:49:13.259043+0000","last_peered":"2024-04-12T11:49:13.259043+0000","last_clean":"2024-04-12T11:49:13.259043+0000","last_became_active":"2024-04-12T11:48:45.417690+0000","last_became_peered":"2024-04-12T11:48:45.417690+0000","last_unstale":"2024-04-12T11:49:13.259043+0000","last_undegraded":"2024-04-12T11:49:13.259043+0000","last_fullsized":"2024-04-12T11:49:13.259043+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:09:21.940487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":25,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.282991+0000","last_change":"2024-04-12T11:48:45.418845+0000","last_active":"2024-04-12T11:49:13.282991+0000","last_peered":"2024-04-12T11:49:13.282991+0000","last_clean":"2024-04-12T11:49:13.282991+0000","last_became_active":"2024-04-12T11:48:45.417891+0000","last_became_peered":"2024-04-12T11:48:45.417891+0000","last_unstale":"2024-04-12T11:49:13.282991+0000","last_undegraded":"2024-04-12T11:49:13.282991+0000","last_fullsized":"2024-04-12T11:49:13.282991+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:12:05.184852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1937,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.266946+0000","last_change":"2024-04-12T11:43:26.370162+0000","last_active":"2024-04-12T11:49:13.266946+0000","last_peered":"2024-04-12T11:49:13.266946+0000","last_clean":"2024-04-12T11:49:13.266946+0000","last_became_active":"2024-04-12T11:43:26.352240+0000","last_became_peered":"2024-04-12T11:43:26.352240+0000","last_unstale":"2024-04-12T11:49:13.266946+0000","last_undegraded":"2024-04-12T11:49:13.266946+0000","last_fullsized":"2024-04-12T11:49:13.266946+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:34:09.364519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1803,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-04-12T11:49:13.256967+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:49:13.256967+0000","last_peered":"2024-04-12T11:49:13.256967+0000","last_clean":"2024-04-12T11:49:13.256967+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:49:13.256967+0000","last_undegraded":"2024-04-12T11:49:13.256967+0000","last_fullsized":"2024-04-12T11:49:13.256967+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":11724,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6864764952,"num_objects":104749,"num_object_clones":0,"num_object_copies":314247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34303,"num_objects_misplaced":10390,"num_objects_unfound":0,"num_objects_dirty":104749,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112801,"num_write_kb":6961537,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":986,"num_bytes_recovered":64618496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20902457344,"data_stored":20902445128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107941,"ondisk_log_size":107941,"up":48,"acting":42,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234444,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5057828,"kb_used_data":4481180,"kb_used_omap":1,"kb_used_meta":576574,"kb_avail":88687324,"statfs":{"total":95995035648,"available":90815819776,"internally_reserved":0,"allocated":4588728320,"data_stored":4586345188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":590412235},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496081,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1671576,"kb_used_data":1261456,"kb_used_omap":1,"kb_used_meta":410046,"kb_avail":92073576,"statfs":{"total":95995035648,"available":94283341824,"internally_reserved":0,"allocated":1291730944,"data_stored":1289360512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":419887560},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536390,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4365856,"kb_used_data":3846552,"kb_used_omap":30,"kb_used_meta":519201,"kb_avail":89379296,"statfs":{"total":95995035648,"available":91524399104,"internally_reserved":0,"allocated":3938869248,"data_stored":3936415832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":531662146},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190474,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3086632,"kb_used_data":2476960,"kb_used_omap":43,"kb_used_meta":609556,"kb_avail":90658520,"statfs":{"total":95995035648,"available":92834324480,"internally_reserved":0,"allocated":2536407040,"data_stored":2533946556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":624185387},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,4,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849602,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2687712,"kb_used_data":2019864,"kb_used_omap":36,"kb_used_meta":667803,"kb_avail":91057440,"statfs":{"total":95995035648,"available":93242818560,"internally_reserved":0,"allocated":2068340736,"data_stored":2065887330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":683830452},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704955,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1149280,"kb_used_data":607960,"kb_used_omap":17,"kb_used_meta":541230,"kb_avail":92595872,"statfs":{"total":95995035648,"available":94818172928,"internally_reserved":0,"allocated":622551040,"data_stored":620097690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":554220023},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018167,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2198068,"kb_used_data":1706412,"kb_used_omap":42,"kb_used_meta":491605,"kb_avail":91547084,"statfs":{"total":95995035648,"available":93744214016,"internally_reserved":0,"allocated":1747365888,"data_stored":1744896870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":503404344},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757717,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4735628,"kb_used_data":4122948,"kb_used_omap":1,"kb_used_meta":612670,"kb_avail":89009524,"statfs":{"total":95995035648,"available":91145752576,"internally_reserved":0,"allocated":4221898752,"data_stored":4219535332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":627374534},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2518482944,"data_stored":2518482944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3922923520,"data_stored":3922919448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2059337728,"data_stored":2059337728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":615514112,"data_stored":615514112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1730347008,"data_stored":1730347008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4199747584,"data_stored":4199743512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1277952000,"data_stored":1277952000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4578152448,"data_stored":4578148376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:49:25.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool set unique_pool_4 pgp_num 16 2024-04-12T11:49:25.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:25.747 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:25.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:25.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:25.904 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:25 smithi029 ceph-mon[97787]: from='client.45404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:49:25.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:25 smithi029 ceph-mon[93650]: from='client.45404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:49:25.918 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:25.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:25.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:26.020 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:26.091 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:26.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:26.192 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:26.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:25 smithi087 ceph-mon[89014]: from='client.45404 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:49:26.221 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:26.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:26.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:26.322 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:26.352 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:26.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:26.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:26.453 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:26.485 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:26.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:26.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:26.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:26.596 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:49:26.626 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:26.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:26.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:26.640 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:49:26.679 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:49:26.724 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:49:26.729 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:26.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:26.761 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:26.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:26.762 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:49:26.799 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:49:26.836 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:49:26.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:26.888 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:49:26.889 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:49:26.889 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:49:26.889 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:49:26.889 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:49:26.889 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:49:26.889 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:49:26.889 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:49:26.889 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:49:26.909 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:26.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:26.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:26.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:26 smithi029 ceph-mon[93650]: pgmap v1959: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 547 op/s; 34303/312009 objects degraded (10.994%); 10238/312009 objects misplaced (3.281%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:49:26.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:26 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2843715335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:49:26.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:26 smithi029 ceph-mon[97787]: pgmap v1959: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 547 op/s; 34303/312009 objects degraded (10.994%); 10238/312009 objects misplaced (3.281%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:49:26.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:26 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2843715335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:49:27.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:27.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:27.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:27.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:27.154 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:27.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:27.191 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:27.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:27.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:26 smithi087 ceph-mon[89014]: pgmap v1959: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 9.3 MiB/s wr, 547 op/s; 34303/312009 objects degraded (10.994%); 10238/312009 objects misplaced (3.281%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:49:27.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:26 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2843715335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:49:27.293 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:27.324 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:27.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:27.426 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:27.453 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:27.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:27.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:27.555 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:27.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:27.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:27.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:27.687 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:27.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:27.721 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:27.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:27.822 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:27.855 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:27.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:27.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:27.956 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:27.990 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:27.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:27.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:28.092 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:28.119 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:28.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:28.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:28.122 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:28.141 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:28.222 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:28.265 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:28.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:28.351 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/312009 objects degraded (10.994%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:28.352 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/312009 objects degraded (10.994%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:28.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:28.410 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:28.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:28.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/312009 objects degraded (10.994%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:28.511 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:28.539 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:28.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:28.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:28.703 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:28.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:28.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:28.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:28.869 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:28.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:28.970 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:29.009 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:29.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:29.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:29.138 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:29.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:29.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:29.240 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:29.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:29 smithi029 ceph-mon[93650]: pgmap v1960: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.6 MiB/s wr, 611 op/s; 34303/307572 objects degraded (11.153%); 10238/307572 objects misplaced (3.329%); 1.6 MiB/s, 26 objects/s recovering 2024-04-12T11:49:29.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:29 smithi029 ceph-mon[97787]: pgmap v1960: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.6 MiB/s wr, 611 op/s; 34303/307572 objects degraded (11.153%); 10238/307572 objects misplaced (3.329%); 1.6 MiB/s, 26 objects/s recovering 2024-04-12T11:49:29.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:29.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:29.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:29.371 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:29.406 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:29.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:29.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:29 smithi087 ceph-mon[89014]: pgmap v1960: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.6 MiB/s wr, 611 op/s; 34303/307572 objects degraded (11.153%); 10238/307572 objects misplaced (3.329%); 1.6 MiB/s, 26 objects/s recovering 2024-04-12T11:49:29.508 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:29.538 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:29.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:29.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:29.640 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:29.694 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:29.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:29.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:29.837 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:29.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:29.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:29.939 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:29.984 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:29.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:29.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:30.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:49:30.086 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:30.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:30.118 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:30.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:30.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:30.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:30.266 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:30.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:30.367 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:30.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:30.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:30.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:30 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1618279268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:49:30.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:30.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:30 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1618279268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:49:30.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:30 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1618279268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:49:30.558 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:30.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:30.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:30.660 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:30.704 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:30.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:30.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:30.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:30.853 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:30.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:30.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:30.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:30.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:30.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:30.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:31.045 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:49:31.084 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:31.132 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:31.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:31.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:31.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:31.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:31 smithi029 ceph-mon[97787]: pgmap v1961: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.4 MiB/s wr, 618 op/s; 34303/303546 objects degraded (11.301%); 10238/303546 objects misplaced (3.373%); 1.6 MiB/s, 26 objects/s recovering 2024-04-12T11:49:31.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:31 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:49:31.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:31 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1618279268' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:49:31.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:31 smithi029 ceph-mon[97787]: osdmap e1243: 8 total, 8 up, 8 in 2024-04-12T11:49:31.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:31 smithi029 ceph-mon[93650]: pgmap v1961: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.4 MiB/s wr, 618 op/s; 34303/303546 objects degraded (11.301%); 10238/303546 objects misplaced (3.373%); 1.6 MiB/s, 26 objects/s recovering 2024-04-12T11:49:31.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:31 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:49:31.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:31 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1618279268' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:49:31.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:31 smithi029 ceph-mon[93650]: osdmap e1243: 8 total, 8 up, 8 in 2024-04-12T11:49:31.280 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:31.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:31.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:31.382 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:31.415 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:31.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:31.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:31 smithi087 ceph-mon[89014]: pgmap v1961: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.4 MiB/s wr, 618 op/s; 34303/303546 objects degraded (11.301%); 10238/303546 objects misplaced (3.373%); 1.6 MiB/s, 26 objects/s recovering 2024-04-12T11:49:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:31 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:49:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:31 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1618279268' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:49:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:31 smithi087 ceph-mon[89014]: osdmap e1243: 8 total, 8 up, 8 in 2024-04-12T11:49:31.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:31.562 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:31.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:31.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:31.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:31.699 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:31.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:31.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:31.801 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:31.831 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:31.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:31.933 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:31.967 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:31.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:31.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:32.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:32.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:32.106 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:32.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:32.207 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:32.246 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:32.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:32.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:32.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:32.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:32.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:32.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:32.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:32.530 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:32.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:32.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:32.632 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:32.666 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:32.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:32.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:49:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:49:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:49:32.768 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:32.800 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:32.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:32.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:32.902 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:32.966 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:32.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:32.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:33.068 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:33.114 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:33.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:33.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:33.216 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:33.254 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:33.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:33.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:33 smithi029 ceph-mon[97787]: pgmap v1963: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 34303/299973 objects degraded (11.435%); 10081/299973 objects misplaced (3.361%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:49:33.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/299973 objects degraded (11.435%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:33.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:33 smithi029 ceph-mon[93650]: pgmap v1963: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 34303/299973 objects degraded (11.435%); 10081/299973 objects misplaced (3.361%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:49:33.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/299973 objects degraded (11.435%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:33.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:33.387 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:33.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:33.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:33 smithi087 ceph-mon[89014]: pgmap v1963: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 34303/299973 objects degraded (11.435%); 10081/299973 objects misplaced (3.361%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:49:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/299973 objects degraded (11.435%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:33.490 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:33.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:33.519 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:33.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:33.620 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:33.660 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:33.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:33.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:33.762 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:33.796 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:33.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:33.897 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:33.944 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:33.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:33.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:34.045 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:34.071 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:34.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:34.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:34.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:34.210 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:34.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:34.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:34.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:34.337 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:34.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:34.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:34.339 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:49:34.339 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:49:34.439 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:34.470 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:34.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:34.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:34.572 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:34.601 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:34.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:34.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:34.616 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:34.703 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:34.757 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:34.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:34.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:34.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:34.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:34.899 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:34.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:35.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:35.037 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:35.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:35.095 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 9 pgp_num to 16 2024-04-12T11:49:35.139 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:35.181 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:35.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:35.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:35.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:35 smithi029 ceph-mon[97787]: pgmap v1964: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 34303/295779 objects degraded (11.598%); 10081/295779 objects misplaced (3.408%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:49:35.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:35 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2323361592' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2024-04-12T11:49:35.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:35 smithi029 ceph-mon[93650]: pgmap v1964: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 34303/295779 objects degraded (11.598%); 10081/295779 objects misplaced (3.408%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:49:35.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:35 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2323361592' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2024-04-12T11:49:35.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:49:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:49:34] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2024-04-12T11:49:35.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-04-12T11:49:35.284 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:35.313 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:35.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:35.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:35.414 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:35.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:35.467 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:35.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:35.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:35 smithi087 ceph-mon[89014]: pgmap v1964: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 34303/295779 objects degraded (11.598%); 10081/295779 objects misplaced (3.408%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:49:35.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:35 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2323361592' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2024-04-12T11:49:35.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:49:35.568 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:35.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:35.596 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:35.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:35.697 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:35.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:35.755 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:35.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:35.856 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:35.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:35.904 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:35.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:36.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:36.055 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:36.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:36.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:36.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:36 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2323361592' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-04-12T11:49:36.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:36 smithi029 ceph-mon[97787]: osdmap e1244: 8 total, 8 up, 8 in 2024-04-12T11:49:36.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:36 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2323361592' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-04-12T11:49:36.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:36 smithi029 ceph-mon[93650]: osdmap e1244: 8 total, 8 up, 8 in 2024-04-12T11:49:36.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:36.196 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:36.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:36.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:36.334 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:36.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:36.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:36.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:36.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:36 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2323361592' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-04-12T11:49:36.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:36 smithi087 ceph-mon[89014]: osdmap e1244: 8 total, 8 up, 8 in 2024-04-12T11:49:36.468 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:36.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:36.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:36.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:36.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:36.604 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:36.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:36.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:36.753 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:36.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:36.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:36.855 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:36.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:36.883 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:36.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:36.984 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:37.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:37.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:37.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:37.115 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:37.149 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:37.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:37.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:37.286 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:37.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:37.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:37.388 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:37.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:37.452 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:37.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:37.554 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:37.592 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:37.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:37.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:37.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:37 smithi087 ceph-mon[89014]: pgmap v1966: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 34303/293286 objects degraded (11.696%); 9919/293286 objects misplaced (3.382%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:49:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:37.725 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:37 smithi029 ceph-mon[97787]: pgmap v1966: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 34303/293286 objects degraded (11.696%); 9919/293286 objects misplaced (3.382%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:49:37.726 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:37.726 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:37.726 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:37 smithi029 ceph-mon[93650]: pgmap v1966: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 34303/293286 objects degraded (11.696%); 9919/293286 objects misplaced (3.382%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:49:37.726 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:37.726 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:37.735 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:37.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:37.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:37.740 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:37.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:37.878 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:37.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:37.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:37.899 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:37.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:38.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:38.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:38.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:38.130 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:38.159 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:38.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:38.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:38.293 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:38.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:38.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:38.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:38.440 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:38.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:38.540 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:38.579 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:38.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:38.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/293286 objects degraded (11.696%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:38.593 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/293286 objects degraded (11.696%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:38.682 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:38.713 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:38.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:38.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/293286 objects degraded (11.696%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:38.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:38.857 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:38.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:38.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:38.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:38.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:38.999 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:39.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:39.101 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:39.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:39.134 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:39.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:39.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:39.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:39.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:39.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:39.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:39.411 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:39.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:39.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:39.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:39.547 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:39.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:39.648 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:39.649 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:39 smithi029 ceph-mon[97787]: pgmap v1967: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 34303/288843 objects degraded (11.876%); 9919/288843 objects misplaced (3.434%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:49:39.649 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:39 smithi029 ceph-mon[93650]: pgmap v1967: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 34303/288843 objects degraded (11.876%); 9919/288843 objects misplaced (3.434%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:49:39.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:39.680 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:39.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:39.680 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:39 smithi087 ceph-mon[89014]: pgmap v1967: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 34303/288843 objects degraded (11.876%); 9919/288843 objects misplaced (3.434%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:49:39.781 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:39.821 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:39.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:39.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:39.923 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:39.966 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:39.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:39.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:40.068 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:40.111 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:40.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:40.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:40.212 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:40.237 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:49:40.253 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:40.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:40.288 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:49:40.328 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:49:40.354 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:40.365 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:49:40.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:40.383 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:40.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:40.403 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:49:40.438 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:49:40.474 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:49:40.484 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:40.512 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:49:40.513 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:40.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:40.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:40.517 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:49:40.517 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:49:40.517 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:49:40.517 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:49:40.517 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:49:40.517 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:49:40.518 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:49:40.518 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:49:40.537 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:40 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1926437839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:49:40.538 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:40 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1926437839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:49:40.614 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:40.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:40.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:40.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:40 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1926437839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:49:40.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:40.788 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:40.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:40.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:40.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:40.929 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:40.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:40.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:41.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:41.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:41.073 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:41.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:41.174 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:41.197 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:41.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:41.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:41.299 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:41.343 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:41.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:41.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:41.444 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:41.487 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:41.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:41.588 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:41.617 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:41.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:41.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:41 smithi087 ceph-mon[89014]: pgmap v1968: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 34303/284679 objects degraded (12.050%); 9919/284679 objects misplaced (3.484%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:41.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:41.754 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:41.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:41.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:41 smithi029 ceph-mon[97787]: pgmap v1968: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 34303/284679 objects degraded (12.050%); 9919/284679 objects misplaced (3.484%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:41.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:41 smithi029 ceph-mon[93650]: pgmap v1968: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 34303/284679 objects degraded (12.050%); 9919/284679 objects misplaced (3.484%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:41.855 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:41.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:41.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:41.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:41.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:42.012 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:42.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:42.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:42.114 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:42.144 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:42.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:42.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:42.246 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:42.287 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:42.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:42.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:42.389 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:42.420 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:42.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:42.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:42.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:42.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:42.552 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:42.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:42.653 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:42.677 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:42.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:42.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:49:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:49:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:49:42.780 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:42.788 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:49:42.788 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:49:42.789 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2024-04-12T11:49:42.789 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:49:42.808 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:42.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:42.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:42.910 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:42.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:42.960 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:42.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:43.062 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:43.089 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:43.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:43.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:43.190 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:43.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:43.231 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:43.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:43.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:43.376 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:43.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:43.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:43.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:43.507 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:43.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:43.609 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:43.635 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:43.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:43.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:43 smithi087 ceph-mon[89014]: pgmap v1969: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 34303/281109 objects degraded (12.203%); 9756/281109 objects misplaced (3.471%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/281109 objects degraded (12.203%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:43.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:43 smithi029 ceph-mon[97787]: pgmap v1969: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 34303/281109 objects degraded (12.203%); 9756/281109 objects misplaced (3.471%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/281109 objects degraded (12.203%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:43 smithi029 ceph-mon[93650]: pgmap v1969: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 34303/281109 objects degraded (12.203%); 9756/281109 objects misplaced (3.471%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/281109 objects degraded (12.203%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:43.777 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:43.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:43.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:43.879 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:43.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:43.917 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:43.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:44.018 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:44.059 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:44.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:44.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:44.161 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:44.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:44.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:44.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:44.286 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:44.319 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:44.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:44.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:44.421 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:44.454 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:44.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:44.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:44.556 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:44.584 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:44.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:44.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:44.686 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:44.726 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:44.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:44.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:44.828 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:44.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:44.851 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:44.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:44.953 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:44.991 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:44.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:44.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:45.093 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:45.130 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:45.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:45.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:45.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:49:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:49:44] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.43.0" 2024-04-12T11:49:45.271 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:45.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:45.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:45.373 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:45.407 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:45.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:45.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:45.510 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:45.539 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:45.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:45.640 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:45.676 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:45.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:45.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:45 smithi087 ceph-mon[89014]: pgmap v1970: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 34303/277377 objects degraded (12.367%); 9756/277377 objects misplaced (3.517%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:45 smithi029 ceph-mon[97787]: pgmap v1970: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 34303/277377 objects degraded (12.367%); 9756/277377 objects misplaced (3.517%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:45 smithi029 ceph-mon[93650]: pgmap v1970: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 34303/277377 objects degraded (12.367%); 9756/277377 objects misplaced (3.517%); 2.0 MiB/s, 32 objects/s recovering 2024-04-12T11:49:45.777 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:45.802 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:45.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:45.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:45.903 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:45.938 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:45.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:45.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:46.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:46.075 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:46.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:46.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:46.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:46.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:46.214 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:46.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:46.315 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:46.347 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:46.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:46.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:46.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:46.492 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:46.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:46.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:46.594 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:46.640 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:46.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:46.741 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:46.767 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:46.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:46.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:46.870 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:46.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:46.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:46.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:46.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:47.057 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:47.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:47.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:47.201 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:47.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:47.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:47.352 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:47.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:47.411 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:47 smithi029 ceph-mon[97787]: pgmap v1971: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 34303/275301 objects degraded (12.460%); 9544/275301 objects misplaced (3.467%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:49:47.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:47 smithi029 ceph-mon[93650]: pgmap v1971: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 34303/275301 objects degraded (12.460%); 9544/275301 objects misplaced (3.467%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:49:47.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:47.491 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:47.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:47.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:47.594 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:47.623 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:47.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:47 smithi087 ceph-mon[89014]: pgmap v1971: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 34303/275301 objects degraded (12.460%); 9544/275301 objects misplaced (3.467%); 3.1 MiB/s, 49 objects/s recovering 2024-04-12T11:49:47.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:47.772 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:47.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:47.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:47.874 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:47.915 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:47.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:47.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:48.017 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:48.047 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:48.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:48.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:48.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:48.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:48.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:48.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:48.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:48.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:48.335 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:48.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:48.436 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:48.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:48.473 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:48.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:48.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/275301 objects degraded (12.460%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:48.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/275301 objects degraded (12.460%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:48.574 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:48.612 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:48.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:48.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:48.714 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/275301 objects degraded (12.460%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T11:49:48.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:48.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:48.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:48.856 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:48.907 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:48.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:48.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:49.009 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:49.036 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:49.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:49.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:49.138 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:49.182 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:49.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:49.283 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:49.312 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:49.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:49.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:49.373 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:49:49.413 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:49.449 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:49.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:49.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:49.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:49 smithi029 ceph-mon[97787]: pgmap v1972: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 34303/271107 objects degraded (12.653%); 9544/271107 objects misplaced (3.520%); 2.0 MiB/s, 31 objects/s recovering 2024-04-12T11:49:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:49 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1328607675' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:49:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:49 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:49:49.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:49 smithi029 ceph-mon[93650]: pgmap v1972: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 34303/271107 objects degraded (12.653%); 9544/271107 objects misplaced (3.520%); 2.0 MiB/s, 31 objects/s recovering 2024-04-12T11:49:49.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:49 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1328607675' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:49:49.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:49 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86173 8.c 8:3a57f77c:::benchmark_data_smithi029_194418_object86172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:55.135336+0000", 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 354.08138112900002, 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1975276350000001, 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.521 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86173 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135336+0000", 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135336+0000", 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135338+0000", 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5659999999999999e-06 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135360+0000", 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2385999999999999e-05 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135375+0000", 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5014000000000001e-05 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135382+0000", 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4290000000000004e-06 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135436+0000", 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3847999999999997e-05 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135623+0000", 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018665899999999999 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.135802+0000", 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017909800000000001 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.144228+0000", 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084262569999999995 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.148863+0000", 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046350970000000003 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.161556+0000", 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012692075000000001 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.161570+0000", 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4482e-05 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.138838+0000", 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97726798400000003 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.138854+0000", 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.611e-05 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.332789+0000", 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.19393528 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.332864+0000", 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4350000000000005e-05 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86183 8.c 8:359770e0:::benchmark_data_smithi029_194418_object86182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:55.164650+0000", 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 354.05206770299998, 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1898649269999999, 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.528 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86183 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164650+0000", 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164650+0000", 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164650+0000", 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8899999999999999e-07 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164675+0000", 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5018e-05 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164683+0000", 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0479999999999999e-06 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164690+0000", 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.494e-06 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164770+0000", 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0673999999999994e-05 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.164783+0000", 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2289e-05 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.139700+0000", 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.974917755 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.139712+0000", 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1844e-05 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.336155+0000", 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1964429970000001 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.336322+0000", 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000167233 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5", 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.336494+0000", 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000171206 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.342386+0000", 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058923559999999996 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.342480+0000", 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3707000000000003e-05 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.351322+0000", 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0088422550000000003 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.351382+0000", 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0513999999999997e-05 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.354514+0000", 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031319479999999999 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86185 8.6 8:67dce7ce:::benchmark_data_smithi029_194418_object86184:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:55.167892+0000", 2024-04-12T11:49:49.535 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 354.04882530600003, 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1736517800000001, 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86185 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167892+0000", 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167892+0000", 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167894+0000", 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5850000000000001e-06 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167914+0000", 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0137999999999999e-05 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167924+0000", 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0271e-05 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167930+0000", 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5180000000000001e-06 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.173181+0000", 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005250083 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.173191+0000", 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0868e-05 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141604+0000", 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96841302900000004 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141617+0000", 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2167000000000001e-05 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.327809+0000", 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1861921360000001 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.327970+0000", 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000161464 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.328086+0000", 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115804 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.337011+0000", 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089251799999999996 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.337034+0000", 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3028e-05 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.337074+0000", 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9662000000000001e-05 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.337116+0000", 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1924e-05 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.341544+0000", 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044279230000000003 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86186 8.6 8:62e71c40:::benchmark_data_smithi029_194418_object86185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:55.167943+0000", 2024-04-12T11:49:49.542 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 354.04877455100001, 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.180328485, 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86186 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167943+0000", 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167943+0000", 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167943+0000", 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9999999999999998e-07 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167960+0000", 2024-04-12T11:49:49.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7036000000000001e-05 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167967+0000", 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6080000000000001e-06 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.167972+0000", 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5910000000000003e-06 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.173221+0000", 2024-04-12T11:49:49.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005248356 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.173229+0000", 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0139999999999998e-06 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141646+0000", 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96841718899999996 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141655+0000", 2024-04-12T11:49:49.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4879999999999996e-06 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.334011+0000", 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.192355271 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.334193+0000", 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018225399999999999 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:49:49.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.334396+0000", 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020332999999999999 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.341686+0000", 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072902039999999998 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.341752+0000", 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5209999999999994e-05 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.344686+0000", 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029342679999999999 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.344726+0000", 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0436999999999999e-05 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.348271+0000", 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035448290000000002 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:86189 8.6 8:69abc3bd:::benchmark_data_smithi029_194418_object86188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:43:55.168015+0000", 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 354.04870204700001, 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.183327738, 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86189 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168015+0000", 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168015+0000", 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168016+0000", 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5999999999999999e-07 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168035+0000", 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9789999999999999e-05 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168042+0000", 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1809999999999998e-06 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.168046+0000", 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9799999999999998e-06 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.181852+0000", 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013805784999999999 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:55.181863+0000", 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0898e-05 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141682+0000", 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95981845899999996 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:56.141691+0000", 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1929999999999999e-06 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.341802+0000", 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.200110867 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.341985+0000", 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018331000000000001 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.342199+0000", 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021373900000000001 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.348371+0000", 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061724989999999997 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.348382+0000", 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1092e-05 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.348412+0000", 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9864000000000001e-05 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.348452+0000", 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0018999999999997e-05 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:43:57.351343+0000", 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028907020000000002 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.556 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:121589 8.6 8:671b29b8:::benchmark_data_smithi029_194418_object15279:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:44.398957+0000", 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 304.81775995300001, 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0599646630000001, 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 121589 2024-04-12T11:49:49.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398957+0000", 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398957+0000", 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398959+0000", 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9599999999999999e-06 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398969+0000", 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6709999999999994e-06 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398985+0000", 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5784999999999999e-05 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398997+0000", 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2522e-05 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.657919+0000", 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25892183699999999 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.657929+0000", 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5419999999999994e-06 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.391598+0000", 2024-04-12T11:49:49.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73366960800000003 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.391612+0000", 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4151000000000001e-05 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.427659+0000", 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.036046614 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.427796+0000", 2024-04-12T11:49:49.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013716499999999999 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.458097+0000", 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030300494000000001 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.458922+0000", 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00082531399999999995 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168251 8.c 8:3d814e3d:::benchmark_data_smithi029_194418_object61941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.323733+0000", 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 236.892983929, 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.010112119, 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168251 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323733+0000", 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323733+0000", 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323735+0000", 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4589999999999999e-06 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323741+0000", 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0070000000000003e-06 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323757+0000", 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5869999999999999e-05 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323765+0000", 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0649999999999999e-06 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.360710+0000", 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036944880999999999 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.363394+0000", 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002683983 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.378917+0000", 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015522948 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.379027+0000", 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011003899999999999 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4,5", 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.379202+0000", 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000175716 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394056+0000", 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014854192 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394068+0000", 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1245e-05 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.967696+0000", 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.57362859499999996 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.967709+0000", 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2986000000000001e-05 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.314657+0000", 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.346947624 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.314840+0000", 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018301 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.333767+0000", 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018927557000000001 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.333845+0000", 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7941999999999993e-05 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168281 8.16 8:69d01196:::benchmark_data_smithi029_194418_object61971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.354640+0000", 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 236.86207758699999, 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8219508370000002, 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:49:49.570 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168281 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354640+0000", 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354640+0000", 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354641+0000", 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3090000000000001e-06 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354645+0000", 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4440000000000001e-06 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354656+0000", 2024-04-12T11:49:49.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0263e-05 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354664+0000", 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9300000000000003e-06 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354959+0000", 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000295481 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.355057+0000", 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7807999999999996e-05 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.357030+0000", 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019734489999999999 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.364390+0000", 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073593649999999997 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388599+0000", 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024208994000000001 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388612+0000", 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3397000000000001e-05 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973262+0000", 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.58464973899999995 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973272+0000", 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0592e-05 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176509+0000", 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2032365409999999 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176590+0000", 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1525000000000003e-05 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.576 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168294 8.c 8:33ff478c:::benchmark_data_smithi029_194418_object61984:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.387000+0000", 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 236.82971682799999, 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9560116860000001, 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168294 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.577 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387000+0000", 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387000+0000", 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387002+0000", 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3769999999999999e-06 2024-04-12T11:49:49.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387006+0000", 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.206e-06 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387016+0000", 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6930000000000002e-06 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387023+0000", 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8180000000000004e-06 2024-04-12T11:49:49.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394098+0000", 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0070740580000000003 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394105+0000", 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2269999999999996e-06 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.968399+0000", 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.57429384699999997 2024-04-12T11:49:49.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.968408+0000", 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2350000000000005e-06 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.318138+0000", 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3497297449999999 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.318280+0000", 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000142356 2024-04-12T11:49:49.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5,7", 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.318409+0000", 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000128713 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.330729+0000", 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012320516 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.330916+0000", 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018688 2024-04-12T11:49:49.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.331041+0000", 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012471800000000001 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.341131+0000", 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010089964 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.341357+0000", 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000226734 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.343012+0000", 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016545990000000001 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168295 8.6 8:63357634:::benchmark_data_smithi029_194418_object61985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.387036+0000", 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 236.82968115599999, 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.799481667, 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.584 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168295 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387036+0000", 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387036+0000", 2024-04-12T11:49:49.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387037+0000", 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1799999999999995e-07 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387039+0000", 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6240000000000002e-06 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387046+0000", 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.888e-06 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387053+0000", 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5780000000000004e-06 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388702+0000", 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001649347 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388711+0000", 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8720000000000008e-06 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973114+0000", 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.58440336500000001 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973125+0000", 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0274e-05 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.165152+0000", 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.192027036 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.165316+0000", 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000164561 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.165826+0000", 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000509495 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170021+0000", 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041956379999999998 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.173298+0000", 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003276251 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.185462+0000", 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012164071 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.185532+0000", 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0222999999999995e-05 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.186518+0000", 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00098592600000000003 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.591 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168309 8.16 8:68816e76:::benchmark_data_smithi029_194418_object61999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.404119+0000", 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 236.81259848400001, 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.786614266, 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168309 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.592 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404119+0000", 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404119+0000", 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404120+0000", 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.393e-06 2024-04-12T11:49:49.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404126+0000", 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9630000000000003e-06 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404169+0000", 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2865999999999999e-05 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404179+0000", 2024-04-12T11:49:49.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0002e-05 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404244+0000", 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4998000000000003e-05 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404255+0000", 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0845e-05 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973300+0000", 2024-04-12T11:49:49.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56904551800000003 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973305+0000", 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8539999999999996e-06 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176619+0000", 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2033139099999999 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176694+0000", 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4870999999999995e-05 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6,7", 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.177067+0000", 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000373023 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.181264+0000", 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041970030000000004 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.183957+0000", 2024-04-12T11:49:49.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026927610000000001 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.184058+0000", 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100765 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190488+0000", 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0064309559999999998 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190552+0000", 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3929000000000004e-05 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190733+0000", 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000180609 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168311 8.6 8:65642b03:::benchmark_data_smithi029_194418_object62001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.408072+0000", 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 236.80864549699999, 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.782363267, 2024-04-12T11:49:49.599 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168311 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408072+0000", 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408072+0000", 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408073+0000", 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4950000000000001e-06 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408080+0000", 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6250000000000001e-06 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408097+0000", 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7031999999999999e-05 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408107+0000", 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8470000000000008e-06 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408190+0000", 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3578000000000005e-05 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408206+0000", 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5577e-05 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973171+0000", 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56496518699999998 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973182+0000", 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0550000000000001e-05 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170067+0000", 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1968853310000001 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170241+0000", 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017409500000000001 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170963+0000", 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00072238800000000002 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176537+0000", 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055731390000000004 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176553+0000", 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6648999999999999e-05 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.188471+0000", 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011917366 2024-04-12T11:49:49.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.188530+0000", 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9120000000000003e-05 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190435+0000", 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001905288 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.606 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168312 8.6 8:679ba278:::benchmark_data_smithi029_194418_object62002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.409075+0000", 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 236.807642375, 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.782290435, 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168312 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409075+0000", 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409075+0000", 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409076+0000", 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2200000000000002e-07 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409080+0000", 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4240000000000003e-06 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409092+0000", 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1416e-05 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409100+0000", 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7260000000000008e-06 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409233+0000", 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013282999999999999 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409245+0000", 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2078e-05 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973213+0000", 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56396750699999998 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973222+0000", 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1900000000000001e-06 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.173473+0000", 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.200250587 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.173613+0000", 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000140218 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2024-04-12T11:49:49.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.174102+0000", 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00048906700000000004 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.178907+0000", 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048052770000000002 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.181255+0000", 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023480789999999999 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190540+0000", 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092848930000000007 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190596+0000", 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6042999999999999e-05 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.191365+0000", 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00076917800000000003 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.613 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168313 8.16 8:6d52a699:::benchmark_data_smithi029_194418_object62003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.410304+0000", 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 236.80641347299999, 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.780861611, 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168313 2024-04-12T11:49:49.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410304+0000", 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410304+0000", 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410305+0000", 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7499999999999998e-07 2024-04-12T11:49:49.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410310+0000", 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7209999999999999e-06 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410327+0000", 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6297999999999999e-05 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410336+0000", 2024-04-12T11:49:49.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0359999999999995e-06 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410401+0000", 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5167999999999997e-05 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410413+0000", 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1912999999999999e-05 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973335+0000", 2024-04-12T11:49:49.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56292207400000005 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973345+0000", 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.91e-06 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.178899+0000", 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2055542189999999 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.178983+0000", 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4301999999999993e-05 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6,7", 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.179305+0000", 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032191099999999997 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.181339+0000", 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020332520000000001 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.184110+0000", 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027713640000000001 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.184229+0000", 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011907499999999999 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190999+0000", 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067697510000000001 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.191053+0000", 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3785000000000001e-05 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.191165+0000", 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011285699999999999 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:108277 9.b 9:dfed8a48:::benchmark_data_smithi029_222585_object108276:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:13.782256+0000", 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 35.434460760999997, 2024-04-12T11:49:49.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.218206232, 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108277 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.782256+0000", 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.782256+0000", 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.782259+0000", 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0530000000000001e-06 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.782281+0000", 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2518e-05 2024-04-12T11:49:49.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.782293+0000", 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2140000000000001e-05 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.782301+0000", 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1130000000000001e-06 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.782359+0000", 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7559999999999998e-05 2024-04-12T11:49:49.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.782590+0000", 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000230693 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.782735+0000", 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014499099999999999 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.786862+0000", 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041272349999999999 2024-04-12T11:49:49.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.789188+0000", 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023257989999999999 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.000337+0000", 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.211149844 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.000410+0000", 2024-04-12T11:49:49.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2099000000000002e-05 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.000463+0000", 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3186999999999997e-05 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:108280 9.5 9:ad984c7a:::benchmark_data_smithi029_222585_object108279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:13.783300+0000", 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 35.433417710999997, 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21849507300000001, 2024-04-12T11:49:49.627 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108280 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783300+0000", 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783300+0000", 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783301+0000", 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3799999999999999e-06 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783340+0000", 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8865999999999997e-05 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783351+0000", 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1418e-05 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783359+0000", 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5499999999999997e-06 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783422+0000", 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2908000000000001e-05 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783553+0000", 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131738 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783694+0000", 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014067800000000001 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.787401+0000", 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00370653 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793943+0000", 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065420690000000002 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.001698+0000", 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20775549400000001 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.001748+0000", 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9866e-05 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.001795+0000", 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6576000000000003e-05 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:108281 9.9 9:92b4d65b:::benchmark_data_smithi029_222585_object108280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:13.783381+0000", 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 35.433336433999997, 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.22128258100000001, 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.633 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108281 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783381+0000", 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783381+0000", 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783382+0000", 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1699999999999997e-07 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783402+0000", 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0242e-05 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783410+0000", 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5520000000000005e-06 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783417+0000", 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7209999999999997e-06 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783468+0000", 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0695999999999997e-05 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783598+0000", 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013075599999999999 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.783734+0000", 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013507699999999999 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.787546+0000", 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003812115 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.003008+0000", 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21546225999999999 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.003484+0000", 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000476343 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.003713+0000", 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000228266 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.004663+0000", 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00095083600000000004 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.638 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:108284 9.9 9:9dcc94d2:::benchmark_data_smithi029_222585_object108283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:13.793343+0000", 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 35.423373789999999, 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21758031799999999, 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108284 2024-04-12T11:49:49.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793343+0000", 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793343+0000", 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793345+0000", 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.362e-06 2024-04-12T11:49:49.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793367+0000", 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2444999999999999e-05 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793376+0000", 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.183e-06 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793383+0000", 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6719999999999998e-06 2024-04-12T11:49:49.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793445+0000", 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1914000000000004e-05 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793602+0000", 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015722199999999999 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793748+0000", 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014540700000000001 2024-04-12T11:49:49.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.003197+0000", 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20944932599999999 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.004706+0000", 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015092549999999999 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.010622+0000", 2024-04-12T11:49:49.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059161650000000001 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.010789+0000", 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016693200000000001 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.010924+0000", 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000134435 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.644 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:108285 9.c 9:336bf6ef:::benchmark_data_smithi029_222585_object108284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:13.793674+0000", 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 35.423043503000002, 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.24201515700000001, 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108285 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793674+0000", 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793674+0000", 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793675+0000", 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6700000000000002e-07 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793697+0000", 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2039999999999999e-05 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793708+0000", 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1691e-05 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793716+0000", 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6079999999999998e-06 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.793898+0000", 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018201099999999999 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.794055+0000", 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015663 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4,7", 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:13.794220+0000", 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016521799999999999 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.007827+0000", 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21360694499999999 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.007903+0000", 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5743999999999999e-05 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.008025+0000", 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012255799999999999 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.031387+0000", 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023361933000000001 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.031450+0000", 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2827e-05 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:14.035689+0000", 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042389849999999998 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.650 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:109774 9.8 9:1bbfee3f:::benchmark_data_smithi029_222585_object997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:16.703829+0000", 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 32.512888625999999, 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.240810052, 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 109774 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.703829+0000", 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.703829+0000", 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.703830+0000", 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4109999999999999e-06 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.703835+0000", 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0610000000000003e-06 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.703853+0000", 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7527999999999999e-05 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.703861+0000", 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.14e-06 2024-04-12T11:49:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.880134+0000", 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.176272765 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.880315+0000", 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018196799999999999 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5", 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.880512+0000", 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019611300000000001 2024-04-12T11:49:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.893013+0000", 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012501119999999999 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.897250+0000", 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042376879999999999 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.939488+0000", 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.042237929 2024-04-12T11:49:49.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.939550+0000", 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1457999999999993e-05 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:16.944639+0000", 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005088871 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:49.656 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:49:49.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:49.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:49.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:49.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:49 smithi087 ceph-mon[89014]: pgmap v1972: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 34303/271107 objects degraded (12.653%); 9544/271107 objects misplaced (3.520%); 2.0 MiB/s, 31 objects/s recovering 2024-04-12T11:49:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:49 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1328607675' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:49:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:49 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:49:49.794 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:49.832 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:49.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:49.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:49.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:49.975 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:49.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:49.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:50.077 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:50.106 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:50.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:50.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:50.208 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:50.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:50.237 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:50.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:50.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:50.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:50.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:50.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:50.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:50.496 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:50.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:50.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:50.598 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:50.630 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:50.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:50.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:50 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:49:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:50 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:49:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:50 smithi087 ceph-mon[89014]: osdmap e1245: 8 total, 8 up, 8 in 2024-04-12T11:49:50.732 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:50.761 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:50.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:50.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:50.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:50 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:49:50.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:50 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:49:50.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:50 smithi029 ceph-mon[93650]: osdmap e1245: 8 total, 8 up, 8 in 2024-04-12T11:49:50.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:50 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:49:50.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:50 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:49:50.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:50 smithi029 ceph-mon[97787]: osdmap e1245: 8 total, 8 up, 8 in 2024-04-12T11:49:50.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:49:50.862 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:50.891 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:50.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:50.993 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:51.055 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:51.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:51.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:51.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:51.207 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:51.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:51.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:51.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:51.320 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-04-12T11:49:51.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:51.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:51.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:51.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:51.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:51.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:51.576 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:51.608 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:51.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:51.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:51.710 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:51 smithi087 ceph-mon[89014]: pgmap v1974: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 34303/267186 objects degraded (12.839%); 9544/267186 objects misplaced (3.572%); 2.3 MiB/s, 37 objects/s recovering 2024-04-12T11:49:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:51.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:51.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:51.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:51 smithi029 ceph-mon[93650]: pgmap v1974: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 34303/267186 objects degraded (12.839%); 9544/267186 objects misplaced (3.572%); 2.3 MiB/s, 37 objects/s recovering 2024-04-12T11:49:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:51 smithi029 ceph-mon[97787]: pgmap v1974: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 34303/267186 objects degraded (12.839%); 9544/267186 objects misplaced (3.572%); 2.3 MiB/s, 37 objects/s recovering 2024-04-12T11:49:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:49:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:49:51.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:51.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:51.892 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:51.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:51.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:52.027 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:52.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:52.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:52.147 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:52.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:52.185 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:52.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:52.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:52.336 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:52.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:52.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:52.438 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:52.485 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:52.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:52.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:52.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:49:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:49:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:49:52.625 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:52.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:52.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:52.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:52.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:52.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:52.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:52.873 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:52.899 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:52.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:52.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:52.934 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:52.982 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:53.001 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:53.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:53.065 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:53.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:53.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:53.213 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:53.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:53.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:53.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:53.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:53.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:53.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:53.445 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:53.476 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:53.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:53.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:53.578 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:53.611 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:53.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:53.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:53.712 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:53 smithi087 ceph-mon[89014]: pgmap v1975: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 34303/263694 objects degraded (13.009%); 9369/263694 objects misplaced (3.553%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:49:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/263694 objects degraded (13.009%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:49:53.743 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:53.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:53 smithi029 ceph-mon[97787]: pgmap v1975: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 34303/263694 objects degraded (13.009%); 9369/263694 objects misplaced (3.553%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:49:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/263694 objects degraded (13.009%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:49:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:53 smithi029 ceph-mon[93650]: pgmap v1975: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 34303/263694 objects degraded (13.009%); 9369/263694 objects misplaced (3.553%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:49:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/263694 objects degraded (13.009%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:49:53.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:49:53.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:53.886 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:53.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:49:53.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:54.021 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:54.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:54.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:54.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:54.158 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:54.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:54.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:54.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:49:54.259 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:54.289 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:54.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:54.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:54.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:54.444 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:54.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:54.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:54.584 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:54.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:54.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:54.686 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:54.722 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:56.017 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:49:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:49:54] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2024-04-12T11:49:56.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:55 smithi029 ceph-mon[93650]: pgmap v1976: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 34303/259683 objects degraded (13.210%); 9369/259683 objects misplaced (3.608%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:49:56.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:56.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:55 smithi029 ceph-mon[97787]: pgmap v1976: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 34303/259683 objects degraded (13.210%); 9369/259683 objects misplaced (3.608%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:49:56.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:56.021 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:55 smithi087 ceph-mon[89014]: pgmap v1976: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 34303/259683 objects degraded (13.210%); 9369/259683 objects misplaced (3.608%); 2.4 MiB/s, 38 objects/s recovering 2024-04-12T11:49:56.122 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:56.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:56.153 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:56.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:56.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:56.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:56.306 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:56.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:56.377 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:49:56.387 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:49:56.407 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:56.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:56.467 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:56.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:56.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:56.630 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:56.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:56.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:56.731 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:56.776 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:56.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:56.878 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:49:56.911 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:56.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:56.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:49:57.013 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:57.041 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:57.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:57.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:57.143 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:57.177 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:57.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:57.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:57.279 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:57.325 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:57.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:57.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:57.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:57.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:57.475 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:57.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:57.505 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:57 smithi029 ceph-mon[97787]: pgmap v1977: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 34303/257346 objects degraded (13.330%); 9177/257346 objects misplaced (3.566%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T11:49:57.505 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:57 smithi029 ceph-mon[97787]: from='client.45434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:49:57.505 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:57 smithi029 ceph-mon[93650]: pgmap v1977: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 34303/257346 objects degraded (13.330%); 9177/257346 objects misplaced (3.566%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T11:49:57.505 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:57 smithi029 ceph-mon[93650]: from='client.45434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:49:57.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:57.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:57.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:57.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:57 smithi087 ceph-mon[89014]: pgmap v1977: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 34303/257346 objects degraded (13.330%); 9177/257346 objects misplaced (3.566%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T11:49:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:57 smithi087 ceph-mon[89014]: from='client.45434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:49:57.724 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:57.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:57.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:57.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:57.786 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1977,"stamp":"2024-04-12T11:49:55.994169+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5620766698,"num_objects":85782,"num_object_clones":0,"num_object_copies":257346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34303,"num_objects_misplaced":9177,"num_objects_unfound":0,"num_objects_dirty":85782,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":132092,"num_write_kb":6963983,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2237,"num_bytes_recovered":146159180,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118458,"ondisk_log_size":118458,"up":297,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":305,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21751744,"kb_used_data":17242104,"kb_used_omap":175,"kb_used_meta":4509264,"kb_avail":728209472,"statfs":{"total":767960285184,"available":745686499328,"internally_reserved":0,"allocated":17655914496,"data_stored":17636813930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":4617486505},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,8,13,6,2,10],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-392232960,"num_objects":-5985,"num_object_clones":0,"num_object_copies":-17955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-367,"num_objects_unfound":0,"num_objects_dirty":-5985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5985,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":367,"num_bytes_recovered":24051712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003188"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":1846,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.367959+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:49:49.367959+0000","last_peered":"2024-04-12T11:49:49.367959+0000","last_clean":"2024-04-12T11:49:49.367959+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:49:49.367959+0000","last_undegraded":"2024-04-12T11:49:49.367959+0000","last_fullsized":"2024-04-12T11:49:49.367959+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.429761+0000","last_change":"2024-04-12T11:48:45.352231+0000","last_active":"2024-04-12T11:49:49.429761+0000","last_peered":"2024-04-12T11:49:49.429761+0000","last_clean":"2024-04-12T11:49:49.429761+0000","last_became_active":"2024-04-12T11:48:45.351324+0000","last_became_peered":"2024-04-12T11:48:45.351324+0000","last_unstale":"2024-04-12T11:49:49.429761+0000","last_undegraded":"2024-04-12T11:49:49.429761+0000","last_fullsized":"2024-04-12T11:49:49.429761+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:01.615486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1930,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.438214+0000","last_change":"2024-04-12T11:48:45.353042+0000","last_active":"2024-04-12T11:49:49.438214+0000","last_peered":"2024-04-12T11:49:49.438214+0000","last_clean":"2024-04-12T11:49:49.438214+0000","last_became_active":"2024-04-12T11:48:45.352721+0000","last_became_peered":"2024-04-12T11:48:45.352721+0000","last_unstale":"2024-04-12T11:49:49.438214+0000","last_undegraded":"2024-04-12T11:49:49.438214+0000","last_fullsized":"2024-04-12T11:49:49.438214+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:00:32.520262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1950,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368265+0000","last_change":"2024-04-12T11:48:45.353306+0000","last_active":"2024-04-12T11:49:49.368265+0000","last_peered":"2024-04-12T11:49:49.368265+0000","last_clean":"2024-04-12T11:49:49.368265+0000","last_became_active":"2024-04-12T11:48:45.352065+0000","last_became_peered":"2024-04-12T11:48:45.352065+0000","last_unstale":"2024-04-12T11:49:49.368265+0000","last_undegraded":"2024-04-12T11:49:49.368265+0000","last_fullsized":"2024-04-12T11:49:49.368265+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:26:36.201291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.452743+0000","last_change":"2024-04-12T11:48:45.351604+0000","last_active":"2024-04-12T11:49:49.452743+0000","last_peered":"2024-04-12T11:49:49.452743+0000","last_clean":"2024-04-12T11:49:49.452743+0000","last_became_active":"2024-04-12T11:48:45.349968+0000","last_became_peered":"2024-04-12T11:48:45.349968+0000","last_unstale":"2024-04-12T11:49:49.452743+0000","last_undegraded":"2024-04-12T11:49:49.452743+0000","last_fullsized":"2024-04-12T11:49:49.452743+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:56:42.419931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1194,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.088264+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:49:35.088264+0000","last_peered":"2024-04-12T11:49:35.088264+0000","last_clean":"2024-04-12T11:49:35.088264+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:49:35.088264+0000","last_undegraded":"2024-04-12T11:49:35.088264+0000","last_fullsized":"2024-04-12T11:49:35.088264+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1921,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368072+0000","last_change":"2024-04-12T11:48:45.351162+0000","last_active":"2024-04-12T11:49:49.368072+0000","last_peered":"2024-04-12T11:49:49.368072+0000","last_clean":"2024-04-12T11:49:49.368072+0000","last_became_active":"2024-04-12T11:48:45.350817+0000","last_became_peered":"2024-04-12T11:48:45.350817+0000","last_unstale":"2024-04-12T11:49:49.368072+0000","last_undegraded":"2024-04-12T11:49:49.368072+0000","last_fullsized":"2024-04-12T11:49:49.368072+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:59.627216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.461855+0000","last_change":"2024-04-12T11:48:45.420195+0000","last_active":"2024-04-12T11:49:49.461855+0000","last_peered":"2024-04-12T11:49:49.461855+0000","last_clean":"2024-04-12T11:49:49.461855+0000","last_became_active":"2024-04-12T11:48:45.419810+0000","last_became_peered":"2024-04-12T11:48:45.419810+0000","last_unstale":"2024-04-12T11:49:49.461855+0000","last_undegraded":"2024-04-12T11:49:49.461855+0000","last_fullsized":"2024-04-12T11:49:49.461855+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:33:11.234641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1226,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.096725+0000","last_change":"2024-04-12T11:45:52.986340+0000","last_active":"2024-04-12T11:49:35.096725+0000","last_peered":"2024-04-12T11:49:35.096725+0000","last_clean":"2024-04-12T11:49:35.096725+0000","last_became_active":"2024-04-12T11:45:52.984190+0000","last_became_peered":"2024-04-12T11:45:52.984190+0000","last_unstale":"2024-04-12T11:49:35.096725+0000","last_undegraded":"2024-04-12T11:49:35.096725+0000","last_fullsized":"2024-04-12T11:49:35.096725+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:52:19.144894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.434823+0000","last_change":"2024-04-12T11:48:45.349940+0000","last_active":"2024-04-12T11:49:49.434823+0000","last_peered":"2024-04-12T11:49:49.434823+0000","last_clean":"2024-04-12T11:49:49.434823+0000","last_became_active":"2024-04-12T11:48:45.349603+0000","last_became_peered":"2024-04-12T11:48:45.349603+0000","last_unstale":"2024-04-12T11:49:49.434823+0000","last_undegraded":"2024-04-12T11:49:49.434823+0000","last_fullsized":"2024-04-12T11:49:49.434823+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:04:27.503011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":83,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.093204+0000","last_change":"2024-04-12T11:48:45.357689+0000","last_active":"2024-04-12T11:49:35.093204+0000","last_peered":"2024-04-12T11:49:35.093204+0000","last_clean":"2024-04-12T11:49:35.093204+0000","last_became_active":"2024-04-12T11:48:45.349500+0000","last_became_peered":"2024-04-12T11:48:45.349500+0000","last_unstale":"2024-04-12T11:49:35.093204+0000","last_undegraded":"2024-04-12T11:49:35.093204+0000","last_fullsized":"2024-04-12T11:49:35.093204+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:54:58.532082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1897,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.449337+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:49:49.449337+0000","last_peered":"2024-04-12T11:49:49.449337+0000","last_clean":"2024-04-12T11:49:49.449337+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:49:49.449337+0000","last_undegraded":"2024-04-12T11:49:49.449337+0000","last_fullsized":"2024-04-12T11:49:49.449337+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1851,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368684+0000","last_change":"2024-04-12T11:45:53.024626+0000","last_active":"2024-04-12T11:49:49.368684+0000","last_peered":"2024-04-12T11:49:49.368684+0000","last_clean":"2024-04-12T11:49:49.368684+0000","last_became_active":"2024-04-12T11:45:52.976276+0000","last_became_peered":"2024-04-12T11:45:52.976276+0000","last_unstale":"2024-04-12T11:49:49.368684+0000","last_undegraded":"2024-04-12T11:49:49.368684+0000","last_fullsized":"2024-04-12T11:49:49.368684+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:49.929210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1810,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.367900+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:49:49.367900+0000","last_peered":"2024-04-12T11:49:49.367900+0000","last_clean":"2024-04-12T11:49:49.367900+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:49:49.367900+0000","last_undegraded":"2024-04-12T11:49:49.367900+0000","last_fullsized":"2024-04-12T11:49:49.367900+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":53,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092592+0000","last_change":"2024-04-12T11:45:53.035604+0000","last_active":"2024-04-12T11:49:35.092592+0000","last_peered":"2024-04-12T11:49:35.092592+0000","last_clean":"2024-04-12T11:49:35.092592+0000","last_became_active":"2024-04-12T11:45:52.984722+0000","last_became_peered":"2024-04-12T11:45:52.984722+0000","last_unstale":"2024-04-12T11:49:35.092592+0000","last_undegraded":"2024-04-12T11:49:35.092592+0000","last_fullsized":"2024-04-12T11:49:35.092592+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:30.674395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.436469+0000","last_change":"2024-04-12T11:48:45.353499+0000","last_active":"2024-04-12T11:49:49.436469+0000","last_peered":"2024-04-12T11:49:49.436469+0000","last_clean":"2024-04-12T11:49:49.436469+0000","last_became_active":"2024-04-12T11:48:45.350015+0000","last_became_peered":"2024-04-12T11:48:45.350015+0000","last_unstale":"2024-04-12T11:49:49.436469+0000","last_undegraded":"2024-04-12T11:49:49.436469+0000","last_fullsized":"2024-04-12T11:49:49.436469+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:24:17.358128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1934,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.367879+0000","last_change":"2024-04-12T11:48:45.351398+0000","last_active":"2024-04-12T11:49:49.367879+0000","last_peered":"2024-04-12T11:49:49.367879+0000","last_clean":"2024-04-12T11:49:49.367879+0000","last_became_active":"2024-04-12T11:48:45.350886+0000","last_became_peered":"2024-04-12T11:48:45.350886+0000","last_unstale":"2024-04-12T11:49:49.367879+0000","last_undegraded":"2024-04-12T11:49:49.367879+0000","last_fullsized":"2024-04-12T11:49:49.367879+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:02.835400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1252,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.096742+0000","last_change":"2024-04-12T11:48:45.350844+0000","last_active":"2024-04-12T11:49:35.096742+0000","last_peered":"2024-04-12T11:49:35.096742+0000","last_clean":"2024-04-12T11:49:35.096742+0000","last_became_active":"2024-04-12T11:48:45.350024+0000","last_became_peered":"2024-04-12T11:48:45.350024+0000","last_unstale":"2024-04-12T11:49:35.096742+0000","last_undegraded":"2024-04-12T11:49:35.096742+0000","last_fullsized":"2024-04-12T11:49:35.096742+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:44:47.030599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":47,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092779+0000","last_change":"2024-04-12T11:45:53.006808+0000","last_active":"2024-04-12T11:49:35.092779+0000","last_peered":"2024-04-12T11:49:35.092779+0000","last_clean":"2024-04-12T11:49:35.092779+0000","last_became_active":"2024-04-12T11:45:53.006480+0000","last_became_peered":"2024-04-12T11:45:53.006480+0000","last_unstale":"2024-04-12T11:49:35.092779+0000","last_undegraded":"2024-04-12T11:49:35.092779+0000","last_fullsized":"2024-04-12T11:49:35.092779+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:47.685096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1935,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368361+0000","last_change":"2024-04-12T11:48:45.352862+0000","last_active":"2024-04-12T11:49:49.368361+0000","last_peered":"2024-04-12T11:49:49.368361+0000","last_clean":"2024-04-12T11:49:49.368361+0000","last_became_active":"2024-04-12T11:48:45.352485+0000","last_became_peered":"2024-04-12T11:48:45.352485+0000","last_unstale":"2024-04-12T11:49:49.368361+0000","last_undegraded":"2024-04-12T11:49:49.368361+0000","last_fullsized":"2024-04-12T11:49:49.368361+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:11:20.916845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1201,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.087975+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:49:35.087975+0000","last_peered":"2024-04-12T11:49:35.087975+0000","last_clean":"2024-04-12T11:49:35.087975+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:49:35.087975+0000","last_undegraded":"2024-04-12T11:49:35.087975+0000","last_fullsized":"2024-04-12T11:49:35.087975+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.433793+0000","last_change":"2024-04-12T11:48:45.361653+0000","last_active":"2024-04-12T11:49:49.433793+0000","last_peered":"2024-04-12T11:49:49.433793+0000","last_clean":"2024-04-12T11:49:49.433793+0000","last_became_active":"2024-04-12T11:48:45.361334+0000","last_became_peered":"2024-04-12T11:48:45.361334+0000","last_unstale":"2024-04-12T11:49:49.433793+0000","last_undegraded":"2024-04-12T11:49:49.433793+0000","last_fullsized":"2024-04-12T11:49:49.433793+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:58:25.548034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"1245'8288","reported_seq":8337,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:51.386932+0000","last_change":"2024-04-12T11:47:31.337856+0000","last_active":"2024-04-12T11:49:51.386932+0000","last_peered":"2024-04-12T11:49:51.386932+0000","last_clean":"2024-04-12T11:49:51.386932+0000","last_became_active":"2024-04-12T11:47:31.337539+0000","last_became_peered":"2024-04-12T11:47:31.337539+0000","last_unstale":"2024-04-12T11:49:51.386932+0000","last_undegraded":"2024-04-12T11:49:51.386932+0000","last_fullsized":"2024-04-12T11:49:51.386932+0000","mapping_epoch":1226,"log_start":"1230'1600","ondisk_log_start":"1230'1600","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6688,"log_dups_size":0,"ondisk_log_size":6688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:15.596706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358744064,"num_objects":5474,"num_object_clones":0,"num_object_copies":16422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8288,"num_write_kb":440384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":51,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092414+0000","last_change":"2024-04-12T11:45:53.028640+0000","last_active":"2024-04-12T11:49:35.092414+0000","last_peered":"2024-04-12T11:49:35.092414+0000","last_clean":"2024-04-12T11:49:35.092414+0000","last_became_active":"2024-04-12T11:45:52.983099+0000","last_became_peered":"2024-04-12T11:45:52.983099+0000","last_unstale":"2024-04-12T11:49:35.092414+0000","last_undegraded":"2024-04-12T11:49:35.092414+0000","last_fullsized":"2024-04-12T11:49:35.092414+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:43:01.897014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1203,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.429705+0000","last_change":"2024-04-12T11:45:52.993160+0000","last_active":"2024-04-12T11:49:49.429705+0000","last_peered":"2024-04-12T11:49:49.429705+0000","last_clean":"2024-04-12T11:49:49.429705+0000","last_became_active":"2024-04-12T11:45:52.989310+0000","last_became_peered":"2024-04-12T11:45:52.989310+0000","last_unstale":"2024-04-12T11:49:49.429705+0000","last_undegraded":"2024-04-12T11:49:49.429705+0000","last_fullsized":"2024-04-12T11:49:49.429705+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:58:47.362374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1245'8507","reported_seq":8560,"reported_epoch":1245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:54.963638+0000","last_change":"2024-04-12T11:48:46.804539+0000","last_active":"2024-04-12T11:49:54.963638+0000","last_peered":"2024-04-12T11:49:54.963638+0000","last_clean":"2024-04-12T11:48:44.288786+0000","last_became_active":"2024-04-12T11:48:46.787738+0000","last_became_peered":"2024-04-12T11:48:46.787738+0000","last_unstale":"2024-04-12T11:49:54.963638+0000","last_undegraded":"2024-04-12T11:48:46.374623+0000","last_fullsized":"2024-04-12T11:48:46.337433+0000","mapping_epoch":1235,"log_start":"1230'600","ondisk_log_start":"1230'600","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351862784,"num_objects":5369,"num_object_clones":0,"num_object_copies":16107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5831,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8507,"num_write_kb":444032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":4262},{"shards":"1,4","objects":-462}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":51,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092277+0000","last_change":"2024-04-12T11:45:53.031759+0000","last_active":"2024-04-12T11:49:35.092277+0000","last_peered":"2024-04-12T11:49:35.092277+0000","last_clean":"2024-04-12T11:49:35.092277+0000","last_became_active":"2024-04-12T11:45:52.984545+0000","last_became_peered":"2024-04-12T11:45:52.984545+0000","last_unstale":"2024-04-12T11:49:35.092277+0000","last_undegraded":"2024-04-12T11:49:35.092277+0000","last_fullsized":"2024-04-12T11:49:35.092277+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:16:10.421992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1857,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.369371+0000","last_change":"2024-04-12T11:48:45.351213+0000","last_active":"2024-04-12T11:49:49.369371+0000","last_peered":"2024-04-12T11:49:49.369371+0000","last_clean":"2024-04-12T11:49:49.369371+0000","last_became_active":"2024-04-12T11:48:45.350772+0000","last_became_peered":"2024-04-12T11:48:45.350772+0000","last_unstale":"2024-04-12T11:49:49.369371+0000","last_undegraded":"2024-04-12T11:49:49.369371+0000","last_fullsized":"2024-04-12T11:49:49.369371+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:17:48.831705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1245'8259","reported_seq":8295,"reported_epoch":1245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:53.039844+0000","last_change":"2024-04-12T11:48:46.932869+0000","last_active":"2024-04-12T11:49:53.039844+0000","last_peered":"2024-04-12T11:49:53.039844+0000","last_clean":"2024-04-12T11:48:44.337474+0000","last_became_active":"2024-04-12T11:48:46.916781+0000","last_became_peered":"2024-04-12T11:48:46.916781+0000","last_unstale":"2024-04-12T11:49:53.039844+0000","last_undegraded":"2024-04-12T11:48:46.385475+0000","last_fullsized":"2024-04-12T11:48:46.335429+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8259,"log_dups_size":0,"ondisk_log_size":8259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:02:58.896126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347013120,"num_objects":5295,"num_object_clones":0,"num_object_copies":15885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5745,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8259,"num_write_kb":433728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":4263},{"shards":"5,7","objects":-450}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":51,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.093130+0000","last_change":"2024-04-12T11:45:53.169686+0000","last_active":"2024-04-12T11:49:35.093130+0000","last_peered":"2024-04-12T11:49:35.093130+0000","last_clean":"2024-04-12T11:49:35.093130+0000","last_became_active":"2024-04-12T11:45:53.021021+0000","last_became_peered":"2024-04-12T11:45:53.021021+0000","last_unstale":"2024-04-12T11:49:35.093130+0000","last_undegraded":"2024-04-12T11:49:35.093130+0000","last_fullsized":"2024-04-12T11:49:35.093130+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:48:23.506812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1220,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.087928+0000","last_change":"2024-04-12T11:45:52.979601+0000","last_active":"2024-04-12T11:49:35.087928+0000","last_peered":"2024-04-12T11:49:35.087928+0000","last_clean":"2024-04-12T11:49:35.087928+0000","last_became_active":"2024-04-12T11:45:52.979307+0000","last_became_peered":"2024-04-12T11:45:52.979307+0000","last_unstale":"2024-04-12T11:49:35.087928+0000","last_undegraded":"2024-04-12T11:49:35.087928+0000","last_fullsized":"2024-04-12T11:49:35.087928+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:21.367107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1245'8428","reported_seq":8454,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:52.972012+0000","last_change":"2024-04-12T11:47:31.345659+0000","last_active":"2024-04-12T11:49:52.972012+0000","last_peered":"2024-04-12T11:49:52.972012+0000","last_clean":"2024-04-12T11:49:52.972012+0000","last_became_active":"2024-04-12T11:47:31.345313+0000","last_became_peered":"2024-04-12T11:47:31.345313+0000","last_unstale":"2024-04-12T11:49:52.972012+0000","last_undegraded":"2024-04-12T11:49:52.972012+0000","last_fullsized":"2024-04-12T11:49:52.972012+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8428,"log_dups_size":0,"ondisk_log_size":8428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:03:10.526704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354156544,"num_objects":5404,"num_object_clones":0,"num_object_copies":16212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8428,"num_write_kb":442624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1236,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.096350+0000","last_change":"2024-04-12T11:48:45.344961+0000","last_active":"2024-04-12T11:49:35.096350+0000","last_peered":"2024-04-12T11:49:35.096350+0000","last_clean":"2024-04-12T11:49:35.096350+0000","last_became_active":"2024-04-12T11:48:45.344344+0000","last_became_peered":"2024-04-12T11:48:45.344344+0000","last_unstale":"2024-04-12T11:49:35.096350+0000","last_undegraded":"2024-04-12T11:49:35.096350+0000","last_fullsized":"2024-04-12T11:49:35.096350+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:47:55.659541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1909,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.367994+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:49:49.367994+0000","last_peered":"2024-04-12T11:49:49.367994+0000","last_clean":"2024-04-12T11:49:49.367994+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:49:49.367994+0000","last_undegraded":"2024-04-12T11:49:49.367994+0000","last_fullsized":"2024-04-12T11:49:49.367994+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"249'161","reported_seq":1562,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092898+0000","last_change":"2024-04-12T11:49:07.188158+0000","last_active":"2024-04-12T11:49:35.092898+0000","last_peered":"2024-04-12T11:49:35.092898+0000","last_clean":"2024-04-12T11:49:35.092898+0000","last_became_active":"2024-04-12T11:49:07.187847+0000","last_became_peered":"2024-04-12T11:49:07.187847+0000","last_unstale":"2024-04-12T11:49:35.092898+0000","last_undegraded":"2024-04-12T11:49:35.092898+0000","last_fullsized":"2024-04-12T11:49:35.092898+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.1","version":"1245'8331","reported_seq":12772,"reported_epoch":1245,"state":"active+recovering+remapped","last_fresh":"2024-04-12T11:49:55.315433+0000","last_change":"2024-04-12T11:48:46.724770+0000","last_active":"2024-04-12T11:49:55.315433+0000","last_peered":"2024-04-12T11:49:55.315433+0000","last_clean":"2024-04-12T11:48:44.315992+0000","last_became_active":"2024-04-12T11:48:46.712972+0000","last_became_peered":"2024-04-12T11:48:46.712972+0000","last_unstale":"2024-04-12T11:49:55.315433+0000","last_undegraded":"2024-04-12T11:49:55.315433+0000","last_fullsized":"2024-04-12T11:49:55.315433+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8331,"log_dups_size":0,"ondisk_log_size":8331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344653824,"num_objects":5259,"num_object_clones":0,"num_object_copies":15777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3566,"num_objects_unfound":0,"num_objects_dirty":5259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8331,"num_write_kb":434880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2198,"num_bytes_recovered":144048128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[4,6,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":1693}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":35,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.430811+0000","last_change":"2024-04-12T11:48:45.359254+0000","last_active":"2024-04-12T11:49:49.430811+0000","last_peered":"2024-04-12T11:49:49.430811+0000","last_clean":"2024-04-12T11:49:49.430811+0000","last_became_active":"2024-04-12T11:48:45.348788+0000","last_became_peered":"2024-04-12T11:48:45.348788+0000","last_unstale":"2024-04-12T11:49:49.430811+0000","last_undegraded":"2024-04-12T11:49:49.430811+0000","last_fullsized":"2024-04-12T11:49:49.430811+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:15:04.501653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1847,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368246+0000","last_change":"2024-04-12T11:48:45.351839+0000","last_active":"2024-04-12T11:49:49.368246+0000","last_peered":"2024-04-12T11:49:49.368246+0000","last_clean":"2024-04-12T11:49:49.368246+0000","last_became_active":"2024-04-12T11:48:45.351276+0000","last_became_peered":"2024-04-12T11:48:45.351276+0000","last_unstale":"2024-04-12T11:49:49.368246+0000","last_undegraded":"2024-04-12T11:49:49.368246+0000","last_fullsized":"2024-04-12T11:49:49.368246+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:23:03.106998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"249'161","reported_seq":1562,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.091547+0000","last_change":"2024-04-12T11:49:07.233588+0000","last_active":"2024-04-12T11:49:35.091547+0000","last_peered":"2024-04-12T11:49:35.091547+0000","last_clean":"2024-04-12T11:49:35.091547+0000","last_became_active":"2024-04-12T11:49:07.233149+0000","last_became_peered":"2024-04-12T11:49:07.233149+0000","last_unstale":"2024-04-12T11:49:35.091547+0000","last_undegraded":"2024-04-12T11:49:35.091547+0000","last_fullsized":"2024-04-12T11:49:35.091547+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.0","version":"1245'8362","reported_seq":8412,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:54.854269+0000","last_change":"2024-04-12T11:47:31.345847+0000","last_active":"2024-04-12T11:49:54.854269+0000","last_peered":"2024-04-12T11:49:54.854269+0000","last_clean":"2024-04-12T11:49:54.854269+0000","last_became_active":"2024-04-12T11:47:31.345534+0000","last_became_peered":"2024-04-12T11:47:31.345534+0000","last_unstale":"2024-04-12T11:49:54.854269+0000","last_undegraded":"2024-04-12T11:49:54.854269+0000","last_fullsized":"2024-04-12T11:49:54.854269+0000","mapping_epoch":1226,"log_start":"1230'400","ondisk_log_start":"1230'400","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7962,"log_dups_size":0,"ondisk_log_size":7962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:32:26.224355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349437952,"num_objects":5332,"num_object_clones":0,"num_object_copies":15996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8362,"num_write_kb":438208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":849,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.367999+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:49:49.367999+0000","last_peered":"2024-04-12T11:49:49.367999+0000","last_clean":"2024-04-12T11:49:49.367999+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:49:49.367999+0000","last_undegraded":"2024-04-12T11:49:49.367999+0000","last_fullsized":"2024-04-12T11:49:49.367999+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1845,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368144+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:49:49.368144+0000","last_peered":"2024-04-12T11:49:49.368144+0000","last_clean":"2024-04-12T11:49:49.368144+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:49:49.368144+0000","last_undegraded":"2024-04-12T11:49:49.368144+0000","last_fullsized":"2024-04-12T11:49:49.368144+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1245'8069","reported_seq":8105,"reported_epoch":1245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:52.303477+0000","last_change":"2024-04-12T11:48:46.758392+0000","last_active":"2024-04-12T11:49:52.303477+0000","last_peered":"2024-04-12T11:49:52.303477+0000","last_clean":"2024-04-12T11:48:44.327141+0000","last_became_active":"2024-04-12T11:48:46.750040+0000","last_became_peered":"2024-04-12T11:48:46.750040+0000","last_unstale":"2024-04-12T11:49:52.303477+0000","last_undegraded":"2024-04-12T11:48:46.373778+0000","last_fullsized":"2024-04-12T11:48:46.339427+0000","mapping_epoch":1235,"log_start":"1230'2112","ondisk_log_start":"1230'2112","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5957,"log_dups_size":0,"ondisk_log_size":5957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:54:20.338800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347930624,"num_objects":5309,"num_object_clones":0,"num_object_copies":15927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5634,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8069,"num_write_kb":428096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,1],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":4254},{"shards":"1,7","objects":-325}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1844,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.369447+0000","last_change":"2024-04-12T11:45:52.976139+0000","last_active":"2024-04-12T11:49:49.369447+0000","last_peered":"2024-04-12T11:49:49.369447+0000","last_clean":"2024-04-12T11:49:49.369447+0000","last_became_active":"2024-04-12T11:45:52.975803+0000","last_became_peered":"2024-04-12T11:45:52.975803+0000","last_unstale":"2024-04-12T11:49:49.369447+0000","last_undegraded":"2024-04-12T11:49:49.369447+0000","last_fullsized":"2024-04-12T11:49:49.369447+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:46.618143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1810,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368610+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:49:49.368610+0000","last_peered":"2024-04-12T11:49:49.368610+0000","last_clean":"2024-04-12T11:49:49.368610+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:49:49.368610+0000","last_undegraded":"2024-04-12T11:49:49.368610+0000","last_fullsized":"2024-04-12T11:49:49.368610+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"249'161","reported_seq":1562,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092576+0000","last_change":"2024-04-12T11:49:07.186288+0000","last_active":"2024-04-12T11:49:35.092576+0000","last_peered":"2024-04-12T11:49:35.092576+0000","last_clean":"2024-04-12T11:49:35.092576+0000","last_became_active":"2024-04-12T11:49:07.185683+0000","last_became_peered":"2024-04-12T11:49:07.185683+0000","last_unstale":"2024-04-12T11:49:35.092576+0000","last_undegraded":"2024-04-12T11:49:35.092576+0000","last_fullsized":"2024-04-12T11:49:35.092576+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2","version":"1245'8288","reported_seq":8340,"reported_epoch":1245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:54.894739+0000","last_change":"2024-04-12T11:48:46.765801+0000","last_active":"2024-04-12T11:49:54.894739+0000","last_peered":"2024-04-12T11:49:54.894739+0000","last_clean":"2024-04-12T11:48:44.392445+0000","last_became_active":"2024-04-12T11:48:46.751638+0000","last_became_peered":"2024-04-12T11:48:46.751638+0000","last_unstale":"2024-04-12T11:49:54.894739+0000","last_undegraded":"2024-04-12T11:48:46.375433+0000","last_fullsized":"2024-04-12T11:48:46.336636+0000","mapping_epoch":1235,"log_start":"1230'300","ondisk_log_start":"1230'300","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7988,"log_dups_size":0,"ondisk_log_size":7988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346423296,"num_objects":5286,"num_object_clones":0,"num_object_copies":15858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5646,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8288,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2","objects":-360},{"shards":"1,2,7","objects":4145}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.435503+0000","last_change":"2024-04-12T11:48:45.351852+0000","last_active":"2024-04-12T11:49:49.435503+0000","last_peered":"2024-04-12T11:49:49.435503+0000","last_clean":"2024-04-12T11:49:49.435503+0000","last_became_active":"2024-04-12T11:48:45.350286+0000","last_became_peered":"2024-04-12T11:48:45.350286+0000","last_unstale":"2024-04-12T11:49:49.435503+0000","last_undegraded":"2024-04-12T11:49:49.435503+0000","last_fullsized":"2024-04-12T11:49:49.435503+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:51:26.610188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1937,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.438452+0000","last_change":"2024-04-12T11:48:45.361378+0000","last_active":"2024-04-12T11:49:49.438452+0000","last_peered":"2024-04-12T11:49:49.438452+0000","last_clean":"2024-04-12T11:49:49.438452+0000","last_became_active":"2024-04-12T11:48:45.360917+0000","last_became_peered":"2024-04-12T11:48:45.360917+0000","last_unstale":"2024-04-12T11:49:49.438452+0000","last_undegraded":"2024-04-12T11:49:49.438452+0000","last_fullsized":"2024-04-12T11:49:49.438452+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:46:07.334663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"249'161","reported_seq":1562,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.091964+0000","last_change":"2024-04-12T11:49:07.185970+0000","last_active":"2024-04-12T11:49:35.091964+0000","last_peered":"2024-04-12T11:49:35.091964+0000","last_clean":"2024-04-12T11:49:35.091964+0000","last_became_active":"2024-04-12T11:49:07.185643+0000","last_became_peered":"2024-04-12T11:49:07.185643+0000","last_unstale":"2024-04-12T11:49:35.091964+0000","last_undegraded":"2024-04-12T11:49:35.091964+0000","last_fullsized":"2024-04-12T11:49:35.091964+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.f","version":"1245'8235","reported_seq":8273,"reported_epoch":1245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:52.996396+0000","last_change":"2024-04-12T11:48:46.932836+0000","last_active":"2024-04-12T11:49:52.996396+0000","last_peered":"2024-04-12T11:49:52.996396+0000","last_clean":"2024-04-12T11:48:44.299270+0000","last_became_active":"2024-04-12T11:48:46.917768+0000","last_became_peered":"2024-04-12T11:48:46.917768+0000","last_unstale":"2024-04-12T11:49:52.996396+0000","last_undegraded":"2024-04-12T11:48:46.478160+0000","last_fullsized":"2024-04-12T11:48:46.335134+0000","mapping_epoch":1235,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8235,"log_dups_size":0,"ondisk_log_size":8235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:00:52.272820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351207424,"num_objects":5359,"num_object_clones":0,"num_object_copies":16077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5721,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8235,"num_write_kb":435008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":4283},{"shards":"1,5","objects":-362}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1921,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.442684+0000","last_change":"2024-04-12T11:45:52.980673+0000","last_active":"2024-04-12T11:49:49.442684+0000","last_peered":"2024-04-12T11:49:49.442684+0000","last_clean":"2024-04-12T11:49:49.442684+0000","last_became_active":"2024-04-12T11:45:52.978978+0000","last_became_peered":"2024-04-12T11:45:52.978978+0000","last_unstale":"2024-04-12T11:49:49.442684+0000","last_undegraded":"2024-04-12T11:49:49.442684+0000","last_fullsized":"2024-04-12T11:49:49.442684+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:32:10.303905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1252,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.096818+0000","last_change":"2024-04-12T11:48:45.419824+0000","last_active":"2024-04-12T11:49:35.096818+0000","last_peered":"2024-04-12T11:49:35.096818+0000","last_clean":"2024-04-12T11:49:35.096818+0000","last_became_active":"2024-04-12T11:48:45.419375+0000","last_became_peered":"2024-04-12T11:48:45.419375+0000","last_unstale":"2024-04-12T11:49:35.096818+0000","last_undegraded":"2024-04-12T11:49:35.096818+0000","last_fullsized":"2024-04-12T11:49:35.096818+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:19:30.300350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1900,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.447871+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:49:49.447871+0000","last_peered":"2024-04-12T11:49:49.447871+0000","last_clean":"2024-04-12T11:49:49.447871+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:49:49.447871+0000","last_undegraded":"2024-04-12T11:49:49.447871+0000","last_fullsized":"2024-04-12T11:49:49.447871+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"249'161","reported_seq":1561,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092704+0000","last_change":"2024-04-12T11:49:07.180601+0000","last_active":"2024-04-12T11:49:35.092704+0000","last_peered":"2024-04-12T11:49:35.092704+0000","last_clean":"2024-04-12T11:49:35.092704+0000","last_became_active":"2024-04-12T11:49:07.180271+0000","last_became_peered":"2024-04-12T11:49:07.180271+0000","last_unstale":"2024-04-12T11:49:35.092704+0000","last_undegraded":"2024-04-12T11:49:35.092704+0000","last_fullsized":"2024-04-12T11:49:35.092704+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184891,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.9","version":"1245'8068","reported_seq":8118,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:51.297670+0000","last_change":"2024-04-12T11:47:31.337684+0000","last_active":"2024-04-12T11:49:51.297670+0000","last_peered":"2024-04-12T11:49:51.297670+0000","last_clean":"2024-04-12T11:49:51.297670+0000","last_became_active":"2024-04-12T11:47:31.337391+0000","last_became_peered":"2024-04-12T11:47:31.337391+0000","last_unstale":"2024-04-12T11:49:51.297670+0000","last_undegraded":"2024-04-12T11:49:51.297670+0000","last_fullsized":"2024-04-12T11:49:51.297670+0000","mapping_epoch":1226,"log_start":"1230'1400","ondisk_log_start":"1230'1400","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6668,"log_dups_size":0,"ondisk_log_size":6668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:26:49.292658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355074048,"num_objects":5418,"num_object_clones":0,"num_object_copies":16254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8068,"num_write_kb":431552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1206,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.096583+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:49:35.096583+0000","last_peered":"2024-04-12T11:49:35.096583+0000","last_clean":"2024-04-12T11:49:35.096583+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:49:35.096583+0000","last_undegraded":"2024-04-12T11:49:35.096583+0000","last_fullsized":"2024-04-12T11:49:35.096583+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1832,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.367781+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:49:49.367781+0000","last_peered":"2024-04-12T11:49:49.367781+0000","last_clean":"2024-04-12T11:49:49.367781+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:49:49.367781+0000","last_undegraded":"2024-04-12T11:49:49.367781+0000","last_fullsized":"2024-04-12T11:49:49.367781+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":144,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368581+0000","last_change":"2024-04-12T11:48:45.418653+0000","last_active":"2024-04-12T11:49:49.368581+0000","last_peered":"2024-04-12T11:49:49.368581+0000","last_clean":"2024-04-12T11:49:49.368581+0000","last_became_active":"2024-04-12T11:48:45.418274+0000","last_became_peered":"2024-04-12T11:48:45.418274+0000","last_unstale":"2024-04-12T11:49:49.368581+0000","last_undegraded":"2024-04-12T11:49:49.368581+0000","last_fullsized":"2024-04-12T11:49:49.368581+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:39:24.012667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"249'161","reported_seq":1562,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092784+0000","last_change":"2024-04-12T11:49:07.185964+0000","last_active":"2024-04-12T11:49:35.092784+0000","last_peered":"2024-04-12T11:49:35.092784+0000","last_clean":"2024-04-12T11:49:35.092784+0000","last_became_active":"2024-04-12T11:49:07.185523+0000","last_became_peered":"2024-04-12T11:49:07.185523+0000","last_unstale":"2024-04-12T11:49:35.092784+0000","last_undegraded":"2024-04-12T11:49:35.092784+0000","last_fullsized":"2024-04-12T11:49:35.092784+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184891,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.a","version":"1245'8225","reported_seq":8254,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:53.017747+0000","last_change":"2024-04-12T11:47:31.341656+0000","last_active":"2024-04-12T11:49:53.017747+0000","last_peered":"2024-04-12T11:49:53.017747+0000","last_clean":"2024-04-12T11:49:53.017747+0000","last_became_active":"2024-04-12T11:47:31.341335+0000","last_became_peered":"2024-04-12T11:47:31.341335+0000","last_unstale":"2024-04-12T11:49:53.017747+0000","last_undegraded":"2024-04-12T11:49:53.017747+0000","last_fullsized":"2024-04-12T11:49:53.017747+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8225,"log_dups_size":0,"ondisk_log_size":8225,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:16:26.010127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355008512,"num_objects":5417,"num_object_clones":0,"num_object_copies":16251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8225,"num_write_kb":436544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1897,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.433682+0000","last_change":"2024-04-12T11:48:45.417421+0000","last_active":"2024-04-12T11:49:49.433682+0000","last_peered":"2024-04-12T11:49:49.433682+0000","last_clean":"2024-04-12T11:49:49.433682+0000","last_became_active":"2024-04-12T11:48:45.417046+0000","last_became_peered":"2024-04-12T11:48:45.417046+0000","last_unstale":"2024-04-12T11:49:49.433682+0000","last_undegraded":"2024-04-12T11:49:49.433682+0000","last_fullsized":"2024-04-12T11:49:49.433682+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:17:29.993647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1900,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.438959+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:49:49.438959+0000","last_peered":"2024-04-12T11:49:49.438959+0000","last_clean":"2024-04-12T11:49:49.438959+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:49:49.438959+0000","last_undegraded":"2024-04-12T11:49:49.438959+0000","last_fullsized":"2024-04-12T11:49:49.438959+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1902,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.441152+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:49:49.441152+0000","last_peered":"2024-04-12T11:49:49.441152+0000","last_clean":"2024-04-12T11:49:49.441152+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:49:49.441152+0000","last_undegraded":"2024-04-12T11:49:49.441152+0000","last_fullsized":"2024-04-12T11:49:49.441152+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1245'8251","reported_seq":8303,"reported_epoch":1245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-04-12T11:49:51.279888+0000","last_change":"2024-04-12T11:48:46.811757+0000","last_active":"2024-04-12T11:49:51.279888+0000","last_peered":"2024-04-12T11:49:51.279888+0000","last_clean":"2024-04-12T11:48:44.316482+0000","last_became_active":"2024-04-12T11:48:46.789368+0000","last_became_peered":"2024-04-12T11:48:46.789368+0000","last_unstale":"2024-04-12T11:49:51.279888+0000","last_undegraded":"2024-04-12T11:48:46.382926+0000","last_fullsized":"2024-04-12T11:48:46.336397+0000","mapping_epoch":1235,"log_start":"1230'1500","ondisk_log_start":"1230'1500","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6751,"log_dups_size":0,"ondisk_log_size":6751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356450304,"num_objects":5439,"num_object_clones":0,"num_object_copies":16317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5726,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8251,"num_write_kb":438080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":-287},{"shards":"0,5,7","objects":4320}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1233,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.096706+0000","last_change":"2024-04-12T11:45:52.986504+0000","last_active":"2024-04-12T11:49:35.096706+0000","last_peered":"2024-04-12T11:49:35.096706+0000","last_clean":"2024-04-12T11:49:35.096706+0000","last_became_active":"2024-04-12T11:45:52.984346+0000","last_became_peered":"2024-04-12T11:45:52.984346+0000","last_unstale":"2024-04-12T11:49:35.096706+0000","last_undegraded":"2024-04-12T11:49:35.096706+0000","last_fullsized":"2024-04-12T11:49:35.096706+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:26.859024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1562,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.091996+0000","last_change":"2024-04-12T11:49:07.218311+0000","last_active":"2024-04-12T11:49:35.091996+0000","last_peered":"2024-04-12T11:49:35.091996+0000","last_clean":"2024-04-12T11:49:35.091996+0000","last_became_active":"2024-04-12T11:49:07.217921+0000","last_became_peered":"2024-04-12T11:49:07.217921+0000","last_unstale":"2024-04-12T11:49:35.091996+0000","last_undegraded":"2024-04-12T11:49:35.091996+0000","last_fullsized":"2024-04-12T11:49:35.091996+0000","mapping_epoch":1210,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.434888+0000","last_change":"2024-04-12T11:48:45.352874+0000","last_active":"2024-04-12T11:49:49.434888+0000","last_peered":"2024-04-12T11:49:49.434888+0000","last_clean":"2024-04-12T11:49:49.434888+0000","last_became_active":"2024-04-12T11:48:45.352350+0000","last_became_peered":"2024-04-12T11:48:45.352350+0000","last_unstale":"2024-04-12T11:49:49.434888+0000","last_undegraded":"2024-04-12T11:49:49.434888+0000","last_fullsized":"2024-04-12T11:49:49.434888+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:44:46.925487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1212,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.088182+0000","last_change":"2024-04-12T11:45:53.016616+0000","last_active":"2024-04-12T11:49:35.088182+0000","last_peered":"2024-04-12T11:49:35.088182+0000","last_clean":"2024-04-12T11:49:35.088182+0000","last_became_active":"2024-04-12T11:45:53.016292+0000","last_became_peered":"2024-04-12T11:45:53.016292+0000","last_unstale":"2024-04-12T11:49:35.088182+0000","last_undegraded":"2024-04-12T11:49:35.088182+0000","last_fullsized":"2024-04-12T11:49:35.088182+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:53:29.382293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"249'161","reported_seq":1562,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092153+0000","last_change":"2024-04-12T11:49:07.187065+0000","last_active":"2024-04-12T11:49:35.092153+0000","last_peered":"2024-04-12T11:49:35.092153+0000","last_clean":"2024-04-12T11:49:35.092153+0000","last_became_active":"2024-04-12T11:49:07.186776+0000","last_became_peered":"2024-04-12T11:49:07.186776+0000","last_unstale":"2024-04-12T11:49:35.092153+0000","last_undegraded":"2024-04-12T11:49:35.092153+0000","last_fullsized":"2024-04-12T11:49:35.092153+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"1245'7978","reported_seq":8028,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:51.369092+0000","last_change":"2024-04-12T11:47:31.337491+0000","last_active":"2024-04-12T11:49:51.369092+0000","last_peered":"2024-04-12T11:49:51.369092+0000","last_clean":"2024-04-12T11:49:51.369092+0000","last_became_active":"2024-04-12T11:47:31.337137+0000","last_became_peered":"2024-04-12T11:47:31.337137+0000","last_unstale":"2024-04-12T11:49:51.369092+0000","last_undegraded":"2024-04-12T11:49:51.369092+0000","last_fullsized":"2024-04-12T11:49:51.369092+0000","mapping_epoch":1226,"log_start":"1230'1300","ondisk_log_start":"1230'1300","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6678,"log_dups_size":0,"ondisk_log_size":6678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:29:28.254379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349700096,"num_objects":5336,"num_object_clones":0,"num_object_copies":16008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7978,"num_write_kb":426048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":47,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.091841+0000","last_change":"2024-04-12T11:45:52.987902+0000","last_active":"2024-04-12T11:49:35.091841+0000","last_peered":"2024-04-12T11:49:35.091841+0000","last_clean":"2024-04-12T11:49:35.091841+0000","last_became_active":"2024-04-12T11:45:52.980281+0000","last_became_peered":"2024-04-12T11:45:52.980281+0000","last_unstale":"2024-04-12T11:49:35.091841+0000","last_undegraded":"2024-04-12T11:49:35.091841+0000","last_fullsized":"2024-04-12T11:49:35.091841+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:07:12.203048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1934,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.367849+0000","last_change":"2024-04-12T11:48:45.351359+0000","last_active":"2024-04-12T11:49:49.367849+0000","last_peered":"2024-04-12T11:49:49.367849+0000","last_clean":"2024-04-12T11:49:49.367849+0000","last_became_active":"2024-04-12T11:48:45.350809+0000","last_became_peered":"2024-04-12T11:48:45.350809+0000","last_unstale":"2024-04-12T11:49:49.367849+0000","last_undegraded":"2024-04-12T11:49:49.367849+0000","last_fullsized":"2024-04-12T11:49:49.367849+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:58:36.906714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1909,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368453+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:49:49.368453+0000","last_peered":"2024-04-12T11:49:49.368453+0000","last_clean":"2024-04-12T11:49:49.368453+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:49:49.368453+0000","last_undegraded":"2024-04-12T11:49:49.368453+0000","last_fullsized":"2024-04-12T11:49:49.368453+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"249'161","reported_seq":1562,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092356+0000","last_change":"2024-04-12T11:49:07.186438+0000","last_active":"2024-04-12T11:49:35.092356+0000","last_peered":"2024-04-12T11:49:35.092356+0000","last_clean":"2024-04-12T11:49:35.092356+0000","last_became_active":"2024-04-12T11:49:07.185633+0000","last_became_peered":"2024-04-12T11:49:07.185633+0000","last_unstale":"2024-04-12T11:49:35.092356+0000","last_undegraded":"2024-04-12T11:49:35.092356+0000","last_fullsized":"2024-04-12T11:49:35.092356+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.e","version":"1245'8225","reported_seq":8252,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:52.254128+0000","last_change":"2024-04-12T11:47:31.338497+0000","last_active":"2024-04-12T11:49:52.254128+0000","last_peered":"2024-04-12T11:49:52.254128+0000","last_clean":"2024-04-12T11:49:52.254128+0000","last_became_active":"2024-04-12T11:47:31.338160+0000","last_became_peered":"2024-04-12T11:47:31.338160+0000","last_unstale":"2024-04-12T11:49:52.254128+0000","last_undegraded":"2024-04-12T11:49:52.254128+0000","last_fullsized":"2024-04-12T11:49:52.254128+0000","mapping_epoch":1226,"log_start":"1230'2280","ondisk_log_start":"1230'2280","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5945,"log_dups_size":0,"ondisk_log_size":5945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:22:37.585891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350945280,"num_objects":5355,"num_object_clones":0,"num_object_copies":16065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8225,"num_write_kb":434560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":47,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.093169+0000","last_change":"2024-04-12T11:45:53.005585+0000","last_active":"2024-04-12T11:49:35.093169+0000","last_peered":"2024-04-12T11:49:35.093169+0000","last_clean":"2024-04-12T11:49:35.093169+0000","last_became_active":"2024-04-12T11:45:52.987880+0000","last_became_peered":"2024-04-12T11:45:52.987880+0000","last_unstale":"2024-04-12T11:49:35.093169+0000","last_undegraded":"2024-04-12T11:49:35.093169+0000","last_fullsized":"2024-04-12T11:49:35.093169+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:09.819861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1900,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.442594+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:49:49.442594+0000","last_peered":"2024-04-12T11:49:49.442594+0000","last_clean":"2024-04-12T11:49:49.442594+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:49:49.442594+0000","last_undegraded":"2024-04-12T11:49:49.442594+0000","last_fullsized":"2024-04-12T11:49:49.442594+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1905,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368281+0000","last_change":"2024-04-12T11:45:52.982299+0000","last_active":"2024-04-12T11:49:49.368281+0000","last_peered":"2024-04-12T11:49:49.368281+0000","last_clean":"2024-04-12T11:49:49.368281+0000","last_became_active":"2024-04-12T11:45:52.981942+0000","last_became_peered":"2024-04-12T11:45:52.981942+0000","last_unstale":"2024-04-12T11:49:49.368281+0000","last_undegraded":"2024-04-12T11:49:49.368281+0000","last_fullsized":"2024-04-12T11:49:49.368281+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:18:13.711200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"249'161","reported_seq":1562,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092963+0000","last_change":"2024-04-12T11:49:07.187534+0000","last_active":"2024-04-12T11:49:35.092963+0000","last_peered":"2024-04-12T11:49:35.092963+0000","last_clean":"2024-04-12T11:49:35.092963+0000","last_became_active":"2024-04-12T11:49:07.187286+0000","last_became_peered":"2024-04-12T11:49:07.187286+0000","last_unstale":"2024-04-12T11:49:35.092963+0000","last_undegraded":"2024-04-12T11:49:35.092963+0000","last_fullsized":"2024-04-12T11:49:35.092963+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"1245'8254","reported_seq":8284,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:52.305625+0000","last_change":"2024-04-12T11:47:31.339988+0000","last_active":"2024-04-12T11:49:52.305625+0000","last_peered":"2024-04-12T11:49:52.305625+0000","last_clean":"2024-04-12T11:49:52.305625+0000","last_became_active":"2024-04-12T11:47:31.339650+0000","last_became_peered":"2024-04-12T11:47:31.339650+0000","last_unstale":"2024-04-12T11:49:52.305625+0000","last_undegraded":"2024-04-12T11:49:52.305625+0000","last_fullsized":"2024-04-12T11:49:52.305625+0000","mapping_epoch":1226,"log_start":"1230'2368","ondisk_log_start":"1230'2368","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5886,"log_dups_size":0,"ondisk_log_size":5886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:02:57.059787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355532824,"num_objects":5426,"num_object_clones":0,"num_object_copies":16278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5426,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8254,"num_write_kb":437697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":35,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.433237+0000","last_change":"2024-04-12T11:48:45.361376+0000","last_active":"2024-04-12T11:49:49.433237+0000","last_peered":"2024-04-12T11:49:49.433237+0000","last_clean":"2024-04-12T11:49:49.433237+0000","last_became_active":"2024-04-12T11:48:45.354494+0000","last_became_peered":"2024-04-12T11:48:45.354494+0000","last_unstale":"2024-04-12T11:49:49.433237+0000","last_undegraded":"2024-04-12T11:49:49.433237+0000","last_fullsized":"2024-04-12T11:49:49.433237+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:42:08.500124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1899,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.441221+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:49:49.441221+0000","last_peered":"2024-04-12T11:49:49.441221+0000","last_clean":"2024-04-12T11:49:49.441221+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:49:49.441221+0000","last_undegraded":"2024-04-12T11:49:49.441221+0000","last_fullsized":"2024-04-12T11:49:49.441221+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.435402+0000","last_change":"2024-04-12T11:48:45.349034+0000","last_active":"2024-04-12T11:49:49.435402+0000","last_peered":"2024-04-12T11:49:49.435402+0000","last_clean":"2024-04-12T11:49:49.435402+0000","last_became_active":"2024-04-12T11:48:45.348579+0000","last_became_peered":"2024-04-12T11:48:45.348579+0000","last_unstale":"2024-04-12T11:49:49.435402+0000","last_undegraded":"2024-04-12T11:49:49.435402+0000","last_fullsized":"2024-04-12T11:49:49.435402+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:57.578035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"249'161","reported_seq":1562,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.092080+0000","last_change":"2024-04-12T11:49:07.199005+0000","last_active":"2024-04-12T11:49:35.092080+0000","last_peered":"2024-04-12T11:49:35.092080+0000","last_clean":"2024-04-12T11:49:35.092080+0000","last_became_active":"2024-04-12T11:49:07.198633+0000","last_became_peered":"2024-04-12T11:49:07.198633+0000","last_unstale":"2024-04-12T11:49:35.092080+0000","last_undegraded":"2024-04-12T11:49:35.092080+0000","last_fullsized":"2024-04-12T11:49:35.092080+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"1245'8024","reported_seq":8076,"reported_epoch":1245,"state":"active+recovery_wait+remapped","last_fresh":"2024-04-12T11:49:51.338095+0000","last_change":"2024-04-12T11:48:46.816570+0000","last_active":"2024-04-12T11:49:51.338095+0000","last_peered":"2024-04-12T11:49:51.338095+0000","last_clean":"2024-04-12T11:48:44.312876+0000","last_became_active":"2024-04-12T11:48:46.789195+0000","last_became_peered":"2024-04-12T11:48:46.789195+0000","last_unstale":"2024-04-12T11:49:51.338095+0000","last_undegraded":"2024-04-12T11:49:51.338095+0000","last_fullsized":"2024-04-12T11:49:51.338095+0000","mapping_epoch":1235,"log_start":"1230'1300","ondisk_log_start":"1230'1300","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6724,"log_dups_size":0,"ondisk_log_size":6724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346030080,"num_objects":5280,"num_object_clones":0,"num_object_copies":15840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5611,"num_objects_unfound":0,"num_objects_dirty":5280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8024,"num_write_kb":425728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[{"shards":"0,4,7","objects":-331}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1245,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.088313+0000","last_change":"2024-04-12T11:45:52.979745+0000","last_active":"2024-04-12T11:49:35.088313+0000","last_peered":"2024-04-12T11:49:35.088313+0000","last_clean":"2024-04-12T11:49:35.088313+0000","last_became_active":"2024-04-12T11:45:52.978936+0000","last_became_peered":"2024-04-12T11:45:52.978936+0000","last_unstale":"2024-04-12T11:49:35.088313+0000","last_undegraded":"2024-04-12T11:49:35.088313+0000","last_fullsized":"2024-04-12T11:49:35.088313+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:22.710430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1912,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368136+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:49:49.368136+0000","last_peered":"2024-04-12T11:49:49.368136+0000","last_clean":"2024-04-12T11:49:49.368136+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:49:49.368136+0000","last_undegraded":"2024-04-12T11:49:49.368136+0000","last_fullsized":"2024-04-12T11:49:49.368136+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1819,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368522+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:49:49.368522+0000","last_peered":"2024-04-12T11:49:49.368522+0000","last_clean":"2024-04-12T11:49:49.368522+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:49:49.368522+0000","last_undegraded":"2024-04-12T11:49:49.368522+0000","last_fullsized":"2024-04-12T11:49:49.368522+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1914,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368165+0000","last_change":"2024-04-12T11:45:52.979689+0000","last_active":"2024-04-12T11:49:49.368165+0000","last_peered":"2024-04-12T11:49:49.368165+0000","last_clean":"2024-04-12T11:49:49.368165+0000","last_became_active":"2024-04-12T11:45:52.979286+0000","last_became_peered":"2024-04-12T11:45:52.979286+0000","last_unstale":"2024-04-12T11:49:49.368165+0000","last_undegraded":"2024-04-12T11:49:49.368165+0000","last_fullsized":"2024-04-12T11:49:49.368165+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:10.453472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":172,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.096422+0000","last_change":"2024-04-12T11:48:45.344748+0000","last_active":"2024-04-12T11:49:35.096422+0000","last_peered":"2024-04-12T11:49:35.096422+0000","last_clean":"2024-04-12T11:49:35.096422+0000","last_became_active":"2024-04-12T11:48:45.344290+0000","last_became_peered":"2024-04-12T11:48:45.344290+0000","last_unstale":"2024-04-12T11:49:35.096422+0000","last_undegraded":"2024-04-12T11:49:35.096422+0000","last_fullsized":"2024-04-12T11:49:35.096422+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:13.440125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1215,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.096269+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:49:35.096269+0000","last_peered":"2024-04-12T11:49:35.096269+0000","last_clean":"2024-04-12T11:49:35.096269+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:49:35.096269+0000","last_undegraded":"2024-04-12T11:49:35.096269+0000","last_fullsized":"2024-04-12T11:49:35.096269+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1870,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368815+0000","last_change":"2024-04-12T11:48:45.419685+0000","last_active":"2024-04-12T11:49:49.368815+0000","last_peered":"2024-04-12T11:49:49.368815+0000","last_clean":"2024-04-12T11:49:49.368815+0000","last_became_active":"2024-04-12T11:48:45.419344+0000","last_became_peered":"2024-04-12T11:48:45.419344+0000","last_unstale":"2024-04-12T11:49:49.368815+0000","last_undegraded":"2024-04-12T11:49:49.368815+0000","last_fullsized":"2024-04-12T11:49:49.368815+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:24:01.383355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1200,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-04-12T11:49:35.096895+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:49:35.096895+0000","last_peered":"2024-04-12T11:49:35.096895+0000","last_clean":"2024-04-12T11:49:35.096895+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:49:35.096895+0000","last_undegraded":"2024-04-12T11:49:35.096895+0000","last_fullsized":"2024-04-12T11:49:35.096895+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1861,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.369106+0000","last_change":"2024-04-12T11:48:45.351004+0000","last_active":"2024-04-12T11:49:49.369106+0000","last_peered":"2024-04-12T11:49:49.369106+0000","last_clean":"2024-04-12T11:49:49.369106+0000","last_became_active":"2024-04-12T11:48:45.350567+0000","last_became_peered":"2024-04-12T11:48:45.350567+0000","last_unstale":"2024-04-12T11:49:49.369106+0000","last_undegraded":"2024-04-12T11:49:49.369106+0000","last_fullsized":"2024-04-12T11:49:49.369106+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:48:43.488302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.429506+0000","last_change":"2024-04-12T11:48:45.418133+0000","last_active":"2024-04-12T11:49:49.429506+0000","last_peered":"2024-04-12T11:49:49.429506+0000","last_clean":"2024-04-12T11:49:49.429506+0000","last_became_active":"2024-04-12T11:48:45.417690+0000","last_became_peered":"2024-04-12T11:48:45.417690+0000","last_unstale":"2024-04-12T11:49:49.429506+0000","last_undegraded":"2024-04-12T11:49:49.429506+0000","last_fullsized":"2024-04-12T11:49:49.429506+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:09:21.940487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":31,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.435452+0000","last_change":"2024-04-12T11:48:45.418845+0000","last_active":"2024-04-12T11:49:49.435452+0000","last_peered":"2024-04-12T11:49:49.435452+0000","last_clean":"2024-04-12T11:49:49.435452+0000","last_became_active":"2024-04-12T11:48:45.417891+0000","last_became_peered":"2024-04-12T11:48:45.417891+0000","last_unstale":"2024-04-12T11:49:49.435452+0000","last_undegraded":"2024-04-12T11:49:49.435452+0000","last_fullsized":"2024-04-12T11:49:49.435452+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:12:05.184852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1943,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.368179+0000","last_change":"2024-04-12T11:43:26.370162+0000","last_active":"2024-04-12T11:49:49.368179+0000","last_peered":"2024-04-12T11:49:49.368179+0000","last_clean":"2024-04-12T11:49:49.368179+0000","last_became_active":"2024-04-12T11:43:26.352240+0000","last_became_peered":"2024-04-12T11:43:26.352240+0000","last_unstale":"2024-04-12T11:49:49.368179+0000","last_undegraded":"2024-04-12T11:49:49.368179+0000","last_fullsized":"2024-04-12T11:49:49.368179+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:34:09.364519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1809,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-04-12T11:49:49.367664+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:49:49.367664+0000","last_peered":"2024-04-12T11:49:49.367664+0000","last_clean":"2024-04-12T11:49:49.367664+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:49:49.367664+0000","last_undegraded":"2024-04-12T11:49:49.367664+0000","last_fullsized":"2024-04-12T11:49:49.367664+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":11724,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5620170776,"num_objects":85758,"num_object_clones":0,"num_object_copies":257274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34303,"num_objects_misplaced":9177,"num_objects_unfound":0,"num_objects_dirty":85758,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131792,"num_write_kb":6961537,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2199,"num_bytes_recovered":144113664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17548390400,"data_stored":17548378184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116632,"ondisk_log_size":116632,"up":48,"acting":42,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234450,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4413524,"kb_used_data":3827084,"kb_used_omap":1,"kb_used_meta":586430,"kb_avail":89331628,"statfs":{"total":95995035648,"available":91475587072,"internally_reserved":0,"allocated":3918934016,"data_stored":3916528178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":600504779},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496087,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447628,"kb_used_data":1041664,"kb_used_omap":1,"kb_used_meta":405950,"kb_avail":92297524,"statfs":{"total":95995035648,"available":94512664576,"internally_reserved":0,"allocated":1066663936,"data_stored":1064270798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":415693256},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536397,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3625460,"kb_used_data":3075692,"kb_used_omap":30,"kb_used_meta":549729,"kb_avail":90119692,"statfs":{"total":95995035648,"available":92282564608,"internally_reserved":0,"allocated":3149508608,"data_stored":3147130034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":562922818},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3,2,0,2,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190480,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2723260,"kb_used_data":2105332,"kb_used_omap":43,"kb_used_meta":617876,"kb_avail":91021892,"statfs":{"total":95995035648,"available":93206417408,"internally_reserved":0,"allocated":2155859968,"data_stored":2153474326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":632705067},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849609,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2428084,"kb_used_data":1760172,"kb_used_omap":36,"kb_used_meta":667803,"kb_avail":91317068,"statfs":{"total":95995035648,"available":93508677632,"internally_reserved":0,"allocated":1802416128,"data_stored":1800037564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":683830452},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704961,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1162040,"kb_used_data":598572,"kb_used_omap":17,"kb_used_meta":563438,"kb_avail":92583112,"statfs":{"total":95995035648,"available":94805106688,"internally_reserved":0,"allocated":612937728,"data_stored":610559220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":576961015},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,7],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018174,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1839688,"kb_used_data":1360512,"kb_used_omap":42,"kb_used_meta":479125,"kb_avail":91905464,"statfs":{"total":95995035648,"available":94111195136,"internally_reserved":0,"allocated":1393164288,"data_stored":1390770112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":490624824},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,5,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757723,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4112060,"kb_used_data":3473076,"kb_used_omap":1,"kb_used_meta":638910,"kb_avail":89633092,"statfs":{"total":95995035648,"available":91784286208,"internally_reserved":0,"allocated":3556429824,"data_stored":3554043698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":654244294},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2148073472,"data_stored":2148073472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3127382016,"data_stored":3127377944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1792409600,"data_stored":1792409600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":606273536,"data_stored":606273536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1376583680,"data_stored":1376583680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3532918784,"data_stored":3532914712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1056309248,"data_stored":1056309248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3908440064,"data_stored":3908435992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:49:57.790 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-04-12T11:49:57.791 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool set unique_pool_4 pg_num 26 2024-04-12T11:49:57.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:57.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:57.910 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:57.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:58.010 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:58.035 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:58.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:58.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:58.137 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:58.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:58.174 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:58.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:58.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:49:58.322 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:58.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:58.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:49:58.424 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:58.459 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:58.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:58.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:58.492 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:49:58.561 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:58.590 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/257346 objects degraded (13.330%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:49:58.590 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/257346 objects degraded (13.330%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:49:58.598 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:58.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:58.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:49:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/257346 objects degraded (13.330%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:49:58.730 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:58.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:58.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:49:58.829 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:135285 9.2 9:4e12d665:::benchmark_data_smithi029_222585_object26508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1245)", 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.766213+0000", 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.059341610000000003, 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.059389852, 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:58.830 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 135285 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766213+0000", 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766213+0000", 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766214+0000", 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2900000000000003e-07 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766217+0000", 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6449999999999998e-06 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766228+0000", 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0754000000000001e-05 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766234+0000", 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5100000000000004e-06 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.785579+0000", 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019345087 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.785754+0000", 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017456399999999999 2024-04-12T11:49:58.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.785987+0000", 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023330900000000001 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801069+0000", 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015081603000000001 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35445.0:135295 9.2 e1245/1235)", 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.816592+0000", 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0089621709999999997, 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0090979019999999997, 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.816592+0000", 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.835 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.816592+0000", 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.816593+0000", 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.093e-06 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.816599+0000", 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8119999999999996e-06 2024-04-12T11:49:58.836 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.816611+0000", 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1511e-05 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.816618+0000", 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2930000000000003e-06 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:135286 9.2 9:4a503ceb:::benchmark_data_smithi029_222585_object26509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1245)", 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.766243+0000", 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.059311153999999998, 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.059502113000000002, 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 135286 2024-04-12T11:49:58.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766243+0000", 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766243+0000", 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766244+0000", 2024-04-12T11:49:58.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0999999999999999e-07 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766247+0000", 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.977e-06 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766256+0000", 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6369999999999994e-06 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.766262+0000", 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.789e-06 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.794923+0000", 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.028660959 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.795329+0000", 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00040559100000000001 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.795708+0000", 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000379761 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35445.0:135286 9.2 e1245/1235)", 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.818297+0000", 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0072577739999999998, 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075190420000000001, 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.842 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.818297+0000", 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.818297+0000", 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.818298+0000", 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.122e-06 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.818304+0000", 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3579999999999999e-06 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.818316+0000", 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2312e-05 2024-04-12T11:49:58.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.818325+0000", 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1920000000000005e-06 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35445.0:135295 9.2 e1245/1235)", 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.819781+0000", 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0057734350000000004, 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060820809999999996, 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.845 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.819781+0000", 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.819781+0000", 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.819782+0000", 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4199999999999995e-07 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.819786+0000", 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6410000000000001e-06 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.819796+0000", 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8900000000000002e-06 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.819804+0000", 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2180000000000002e-06 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:135317 9.4 9:21a93d52:::benchmark_data_smithi029_222585_object26540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1245)", 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.820499+0000", 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0050559589999999996, 2024-04-12T11:49:58.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054164479999999999, 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 135317 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.820499+0000", 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.820499+0000", 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.820500+0000", 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6750000000000001e-06 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.820506+0000", 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3159999999999996e-06 2024-04-12T11:49:58.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.820521+0000", 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5620999999999998e-05 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.820529+0000", 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9400000000000002e-06 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.820584+0000", 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4559999999999999e-05 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.821028+0000", 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000444003 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4", 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.821572+0000", 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00054446300000000002 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.852 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:135319 9.0 9:01344fb2:::benchmark_data_smithi029_222585_object26542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1245)", 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.822493+0000", 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0030613709999999998, 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034972530000000001, 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 135319 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.822493+0000", 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.822493+0000", 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.822494+0000", 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.204e-06 2024-04-12T11:49:58.854 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.822500+0000", 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0249999999999999e-06 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.822516+0000", 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5648e-05 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.822525+0000", 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8529999999999998e-06 2024-04-12T11:49:58.855 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.822577+0000", 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2284000000000003e-05 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.822714+0000", 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013689999999999999 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.822829+0000", 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115414 2024-04-12T11:49:58.856 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35445.0:135321 9.a e1245/1226)", 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.825349+0000", 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.000205787, 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00071046100000000001, 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.825349+0000", 2024-04-12T11:49:58.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:58.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.825349+0000", 2024-04-12T11:49:58.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.825350+0000", 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4410000000000001e-06 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.825363+0000", 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3191e-05 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.825385+0000", 2024-04-12T11:49:58.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1586e-05 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.825395+0000", 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0019e-05 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35445.0:135289 9.a e1245/1226 9:51939e08:::benchmark_data_smithi029_222585_object26512:head v 1245'8435, mlcod=1245'8432)", 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.772974+0000", 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.052580264000000002, 2024-04-12T11:49:58.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.053139263999999999, 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.772974+0000", 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.772974+0000", 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.772975+0000", 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1519999999999999e-06 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.772987+0000", 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1989e-05 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.772997+0000", 2024-04-12T11:49:58.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7580000000000004e-06 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.773006+0000", 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4440000000000008e-06 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.778064+0000", 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050583879999999996 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:58.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.778083+0000", 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9185999999999999e-05 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:135295 9.2 9:4f5a3a60:::benchmark_data_smithi029_222585_object26518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1245)", 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.781253+0000", 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.044301014, 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.044930787, 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:58.865 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 135295 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.781253+0000", 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.781253+0000", 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.781255+0000", 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.553e-06 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.781261+0000", 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1079999999999998e-06 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.781275+0000", 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3847e-05 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.781283+0000", 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1939999999999998e-06 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801106+0000", 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019822861000000001 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806543+0000", 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054365680000000001 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806693+0000", 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000150006 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35445.0:135278 9.2 e1245/1235)", 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.801138+0000", 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.024416192, 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025147559999999999, 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.869 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801138+0000", 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801138+0000", 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801145+0000", 2024-04-12T11:49:58.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.438e-06 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801151+0000", 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3280000000000002e-06 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801165+0000", 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3534e-05 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801175+0000", 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0774999999999999e-05 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:135305 9.4 9:24c0773a:::benchmark_data_smithi029_222585_object26528:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1245)", 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.801547+0000", 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.02400745, 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024791242000000002, 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 135305 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801547+0000", 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801547+0000", 2024-04-12T11:49:58.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801549+0000", 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5489999999999999e-06 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801554+0000", 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1159999999999998e-06 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801566+0000", 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2303999999999999e-05 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801574+0000", 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4889999999999995e-06 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806448+0000", 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004873835 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806606+0000", 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015784600000000001 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4", 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806763+0000", 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015646599999999999 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:58.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.819534+0000", 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012771485000000001 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.819984+0000", 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00044978200000000002 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:135278 9.2 9:440f4901:::benchmark_data_smithi029_222585_object26501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1245)", 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.750571+0000", 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.074983011000000002, 2024-04-12T11:49:58.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.075867922000000004, 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 135278 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.750571+0000", 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.750571+0000", 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.750573+0000", 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.905e-06 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.750580+0000", 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0400000000000004e-06 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.750595+0000", 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4409999999999999e-05 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.750603+0000", 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5739999999999996e-06 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.772981+0000", 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022377340999999999 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.773160+0000", 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017897799999999999 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.773329+0000", 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016922500000000001 2024-04-12T11:49:58.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.785496+0000", 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012166613999999999 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.785542+0000", 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6801000000000001e-05 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.882 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35445.0:135304 9.a e1245/1226)", 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.801888+0000", 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.023666155000000001, 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024641107999999998, 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801888+0000", 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801888+0000", 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801889+0000", 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.13e-07 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801896+0000", 2024-04-12T11:49:58.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.131e-06 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801904+0000", 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.524e-06 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.801911+0000", 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4359999999999999e-06 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35445.0:135286 9.2 e1245/1235)", 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.804515+0000", 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.021039948999999999, 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022067923, 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.804515+0000", 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.804515+0000", 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.804516+0000", 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9800000000000002e-07 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.804521+0000", 2024-04-12T11:49:58.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9359999999999996e-06 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.804534+0000", 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2176e-05 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.804542+0000", 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8529999999999998e-06 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.889 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35445.0:135285 9.2 e1245/1235)", 2024-04-12T11:49:58.889 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:49:58.806318+0000", 2024-04-12T11:49:58.889 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.019236118, 2024-04-12T11:49:58.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020321247000000001, 2024-04-12T11:49:58.889 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:49:58.889 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:49:58.889 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T11:49:58.889 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:49:58.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806318+0000", 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806318+0000", 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806319+0000", 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0100000000000001e-06 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806327+0000", 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.233e-06 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806341+0000", 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4594e-05 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:49:58.806351+0000", 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0139999999999999e-05 2024-04-12T11:49:58.891 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.892 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:49:58.892 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.892 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:49:58.892 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2024-04-12T11:49:58.892 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 16 2024-04-12T11:49:58.892 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:49:58.934 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:58.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:59.036 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:59.073 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:59.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:59.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:59.175 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:59.206 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:59.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:59.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:49:59.308 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:49:59.348 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:59.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:59.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:49:59.450 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:59.494 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:59.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:59.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:59.601 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:49:59.634 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:59.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:59.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:49:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:49:59 smithi029 ceph-mon[93650]: pgmap v1978: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 34303/252693 objects degraded (13.575%); 9177/252693 objects misplaced (3.632%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T11:49:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:49:59 smithi029 ceph-mon[97787]: pgmap v1978: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 34303/252693 objects degraded (13.575%); 9177/252693 objects misplaced (3.632%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T11:49:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:49:59 smithi087 ceph-mon[89014]: pgmap v1978: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 34303/252693 objects degraded (13.575%); 9177/252693 objects misplaced (3.632%); 2.3 MiB/s, 36 objects/s recovering 2024-04-12T11:49:59.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:49:59.782 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:49:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:59.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:49:59.884 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:49:59.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:49:59.914 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:49:59.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:00.015 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:00.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-04-12T11:50:00.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:00.066 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:00.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:00.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:00.221 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:00.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:00.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:00.323 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:00.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:00.353 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:00.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:00.455 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:00.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:00.488 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:00.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:00.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:00.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:00.621 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:00.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:00.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[97787]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 34303/252693 objects degraded (13.575%), 6 pgs degraded, 6 pgs undersized 2024-04-12T11:50:00.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[97787]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-04-12T11:50:00.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[97787]: [WRN] PG_DEGRADED: Degraded data redundancy: 34303/252693 objects degraded (13.575%), 6 pgs degraded, 6 pgs undersized 2024-04-12T11:50:00.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[97787]: pg 9.2 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2024-04-12T11:50:00.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[97787]: pg 9.3 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2024-04-12T11:50:00.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[97787]: pg 9.4 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2024-04-12T11:50:00.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[97787]: pg 9.7 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2024-04-12T11:50:00.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[97787]: pg 9.8 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2024-04-12T11:50:00.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[97787]: pg 9.f is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,1] 2024-04-12T11:50:00.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[93650]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 34303/252693 objects degraded (13.575%), 6 pgs degraded, 6 pgs undersized 2024-04-12T11:50:00.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[93650]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-04-12T11:50:00.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[93650]: [WRN] PG_DEGRADED: Degraded data redundancy: 34303/252693 objects degraded (13.575%), 6 pgs degraded, 6 pgs undersized 2024-04-12T11:50:00.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[93650]: pg 9.2 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2024-04-12T11:50:00.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[93650]: pg 9.3 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2024-04-12T11:50:00.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[93650]: pg 9.4 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2024-04-12T11:50:00.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[93650]: pg 9.7 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2024-04-12T11:50:00.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[93650]: pg 9.8 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2024-04-12T11:50:00.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:00 smithi029 ceph-mon[93650]: pg 9.f is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,1] 2024-04-12T11:50:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:00 smithi087 ceph-mon[89014]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 34303/252693 objects degraded (13.575%), 6 pgs degraded, 6 pgs undersized 2024-04-12T11:50:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:00 smithi087 ceph-mon[89014]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-04-12T11:50:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:00 smithi087 ceph-mon[89014]: [WRN] PG_DEGRADED: Degraded data redundancy: 34303/252693 objects degraded (13.575%), 6 pgs degraded, 6 pgs undersized 2024-04-12T11:50:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:00 smithi087 ceph-mon[89014]: pg 9.2 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2024-04-12T11:50:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:00 smithi087 ceph-mon[89014]: pg 9.3 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,1] 2024-04-12T11:50:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:00 smithi087 ceph-mon[89014]: pg 9.4 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2024-04-12T11:50:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:00 smithi087 ceph-mon[89014]: pg 9.7 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2024-04-12T11:50:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:00 smithi087 ceph-mon[89014]: pg 9.8 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,5] 2024-04-12T11:50:00.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:00 smithi087 ceph-mon[89014]: pg 9.f is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,1] 2024-04-12T11:50:00.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:00.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:00.781 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:00.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:00.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:00.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:00.941 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:00.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:01.042 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:01.071 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:01.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:01.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:01.115 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:01.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:01.200 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:01.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:01.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:01.246 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:50:01.298 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:50:01.302 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:01.332 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:01.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:01.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:01.335 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:50:01.385 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:50:01.426 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:50:01.435 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:01.465 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:01.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:01.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:01.467 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:50:01.506 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:01 smithi029 ceph-mon[97787]: pgmap v1979: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 34303/250452 objects degraded (13.696%); 9177/250452 objects misplaced (3.664%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:50:01.506 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:01 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3428674663' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:50:01.507 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:50:01.507 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:01 smithi029 ceph-mon[93650]: pgmap v1979: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 34303/250452 objects degraded (13.696%); 9177/250452 objects misplaced (3.664%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:50:01.507 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:01 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3428674663' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:50:01.542 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:50:01.550 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:50:01.550 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:50:01.550 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:50:01.550 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:50:01.551 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:50:01.551 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:50:01.551 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:50:01.551 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:50:01.567 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:01.597 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:01.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:01.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:01.699 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:01 smithi087 ceph-mon[89014]: pgmap v1979: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 34303/250452 objects degraded (13.696%); 9177/250452 objects misplaced (3.664%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:50:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:01 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3428674663' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:50:01.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:01.732 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:01.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:01.809 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:01.832 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:01.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:01.881 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:01.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:01.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:02.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:02.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:02.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:02.157 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:02.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:02.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:02.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:02.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:02.302 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:02.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:02.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:02.451 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:02.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:02.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:02.554 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:02.601 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:02.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:02.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:02.703 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:50:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:50:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:50:02.744 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:02.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:02.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:02.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:02.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:02.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:02.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:02.983 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:03.021 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:03.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:03.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:03.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:03.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:03.174 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:03.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:03.275 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:03.320 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:03.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:03.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:03.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:03.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:03.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:03.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:03.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:03.601 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:03.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:03.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:03.700 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:03 smithi029 ceph-mon[93650]: pgmap v1980: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 34303/245412 objects degraded (13.978%); 9003/245412 objects misplaced (3.669%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T11:50:03.700 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/245412 objects degraded (13.978%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:03.700 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:03 smithi029 ceph-mon[97787]: pgmap v1980: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 34303/245412 objects degraded (13.978%); 9003/245412 objects misplaced (3.669%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T11:50:03.700 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/245412 objects degraded (13.978%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:03.702 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:03 smithi087 ceph-mon[89014]: pgmap v1980: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 34303/245412 objects degraded (13.978%); 9003/245412 objects misplaced (3.669%); 2.8 MiB/s, 45 objects/s recovering 2024-04-12T11:50:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/245412 objects degraded (13.978%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:03.751 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:03.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:03.852 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:03.890 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:03.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:03.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:03.992 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:04.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:04.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:04.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:04.120 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:04.155 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:04.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:04.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:04.256 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:04.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:04.290 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:04.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:04.391 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:04.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:04.427 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:04.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:04.475 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:50:04.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:04.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:04.572 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:04.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:04.647 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:04 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4108912326' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:50:04.647 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:04 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4108912326' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:50:04.674 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:04 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4108912326' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:50:04.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:04.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:04.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:04.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:04.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:04.887 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:04.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:04.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:05.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:05.030 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:05.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:05.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:05.175 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:05.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:05.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:05.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:50:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:50:04] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2024-04-12T11:50:05.277 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:05.315 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:05.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:05.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:05.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:05.460 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:05.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:05.561 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:05.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:05.600 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:05.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:05.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:50:05.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:05 smithi029 ceph-mon[97787]: pgmap v1981: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 34303/241431 objects degraded (14.208%); 9003/241431 objects misplaced (3.729%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:50:05.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:05 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:50:05.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:05 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4108912326' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:50:05.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:05 smithi029 ceph-mon[97787]: osdmap e1246: 8 total, 8 up, 8 in 2024-04-12T11:50:05.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:05 smithi029 ceph-mon[93650]: pgmap v1981: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 34303/241431 objects degraded (14.208%); 9003/241431 objects misplaced (3.729%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:50:05.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:05 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:50:05.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:05 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4108912326' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:50:05.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:05 smithi029 ceph-mon[93650]: osdmap e1246: 8 total, 8 up, 8 in 2024-04-12T11:50:05.702 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:05 smithi087 ceph-mon[89014]: pgmap v1981: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 34303/241431 objects degraded (14.208%); 9003/241431 objects misplaced (3.729%); 1.9 MiB/s, 30 objects/s recovering 2024-04-12T11:50:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:05 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:50:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:05 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4108912326' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:50:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:05 smithi087 ceph-mon[89014]: osdmap e1246: 8 total, 8 up, 8 in 2024-04-12T11:50:05.737 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:05.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:05.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:05.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:05.906 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:05.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:06.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:06.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:06.044 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:06.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:06.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:06.180 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:06.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:06.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:06.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:06.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:06.325 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:06.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:06.426 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:06.454 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:06.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:06.555 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:06.588 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:06.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:06.663 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:06.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:06.690 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:06.718 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:06.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:06.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:06.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:06.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:06.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:06.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:06.965 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:07.003 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:07.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:07.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:07.158 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:07.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:07.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:07.260 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:07.287 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:07.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:07.389 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:07.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:07.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:07.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:07.521 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:07.564 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:07.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:07.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:07.666 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:07 smithi087 ceph-mon[89014]: pgmap v1983: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 34303/239187 objects degraded (14.341%); 8827/239187 objects misplaced (3.690%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:50:07.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:07.720 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:07.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:07.729 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:07 smithi029 ceph-mon[97787]: pgmap v1983: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 34303/239187 objects degraded (14.341%); 8827/239187 objects misplaced (3.690%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:50:07.730 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:07 smithi029 ceph-mon[93650]: pgmap v1983: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 34303/239187 objects degraded (14.341%); 8827/239187 objects misplaced (3.690%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:50:07.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:07.826 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:50:07.826 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:50:07.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:50:07.865 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:07.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:07.967 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:07.995 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:07.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:07.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:08.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:08.111 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:08.124 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:08.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:08.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:08.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:08.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:08.271 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:08.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:08.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:08.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:08.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:08.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:08.495 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 9 pg_num to 26 2024-04-12T11:50:08.515 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:08 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3459596248' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:50:08.515 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:08 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:50:08.515 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/239187 objects degraded (14.341%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:08.515 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:08 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3459596248' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:50:08.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:08 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:50:08.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/239187 objects degraded (14.341%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:08.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:08.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:08.554 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:08.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:08.655 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:08.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:08.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:08.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:08.730 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-04-12T11:50:08.788 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:08.819 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:08.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:08.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:08.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:08.950 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:08.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:08 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3459596248' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:50:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:08 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2024-04-12T11:50:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/239187 objects degraded (14.341%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:09.053 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:09.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:09.112 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:09.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:09.213 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:09.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:09.260 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:09.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:09.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:09.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:09.399 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:09.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:09.500 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:09.543 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:09.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:09.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:09.645 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:09.675 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:09.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:09.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:09 smithi029 ceph-mon[97787]: pgmap v1984: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 34303/234477 objects degraded (14.630%); 8827/234477 objects misplaced (3.765%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:50:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:09 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-04-12T11:50:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:09 smithi029 ceph-mon[97787]: osdmap e1247: 8 total, 8 up, 8 in 2024-04-12T11:50:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:09 smithi029 ceph-mon[93650]: pgmap v1984: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 34303/234477 objects degraded (14.630%); 8827/234477 objects misplaced (3.765%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:50:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:09 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-04-12T11:50:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:09 smithi029 ceph-mon[93650]: osdmap e1247: 8 total, 8 up, 8 in 2024-04-12T11:50:09.777 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:09.820 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:09.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:09.921 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:09.962 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:09.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:09.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:09 smithi087 ceph-mon[89014]: pgmap v1984: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 34303/234477 objects degraded (14.630%); 8827/234477 objects misplaced (3.765%); 2.2 MiB/s, 34 objects/s recovering 2024-04-12T11:50:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:09 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-04-12T11:50:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:09 smithi087 ceph-mon[89014]: osdmap e1247: 8 total, 8 up, 8 in 2024-04-12T11:50:10.064 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:10.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:10.105 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:10.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:10.205 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:10.236 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:10.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:10.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:10.339 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:10.372 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:10.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:10.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:10.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:10.518 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:10.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:10.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:10.620 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:10.680 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:10.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:10.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:10.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:10 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2024-04-12T11:50:10.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:10 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2024-04-12T11:50:10.781 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:10.818 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:10.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:10.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:10.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:10.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:10.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:10.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:10 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2024-04-12T11:50:11.056 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:11.080 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:11.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:11.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:11.184 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:11.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:11.209 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:11.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:11.311 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:11.336 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:11.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:11.357 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:11.362 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:11.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:11.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:11.463 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:11.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:11.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:11.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:11.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:11.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:11.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:11 smithi029 ceph-mon[93650]: pgmap v1986: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 34303/232110 objects degraded (14.779%); 8827/232110 objects misplaced (3.803%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T11:50:11.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:11 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-04-12T11:50:11.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:11 smithi029 ceph-mon[93650]: osdmap e1248: 8 total, 8 up, 8 in 2024-04-12T11:50:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:11 smithi029 ceph-mon[97787]: pgmap v1986: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 34303/232110 objects degraded (14.779%); 8827/232110 objects misplaced (3.803%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T11:50:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:11 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-04-12T11:50:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:11 smithi029 ceph-mon[97787]: osdmap e1248: 8 total, 8 up, 8 in 2024-04-12T11:50:11.726 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:11.750 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:11.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:11.851 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:11.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:11.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:11 smithi087 ceph-mon[89014]: pgmap v1986: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 34303/232110 objects degraded (14.779%); 8827/232110 objects misplaced (3.803%); 1.4 MiB/s, 21 objects/s recovering 2024-04-12T11:50:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:11 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-04-12T11:50:11.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:11 smithi087 ceph-mon[89014]: osdmap e1248: 8 total, 8 up, 8 in 2024-04-12T11:50:11.968 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:50:11.978 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:11.991 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:50:12.002 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:12.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:12.014 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:50:12.044 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:50:12.065 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:50:12.085 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:50:12.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:12.106 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:50:12.128 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:50:12.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:50:12.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:50:12.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:50:12.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:50:12.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:50:12.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:50:12.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:50:12.130 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:50:12.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:12.132 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:12.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:12.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:12.257 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:12.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:12.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:12.359 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:12.383 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:12.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:12.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:12.485 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:12.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:12.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:12.622 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:12.657 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:12.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:12.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:50:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:50:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:50:12.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:12 smithi087 ceph-mon[89014]: osdmap e1249: 8 total, 8 up, 8 in 2024-04-12T11:50:12.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:12 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3729498899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:50:12.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:12 smithi087 ceph-mon[89014]: pgmap v1989: 109 pgs: 8 unknown, 1 active, 1 active+recovery_wait+degraded, 2 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 106 MiB/s wr, 2.85k op/s; 34303/249165 objects degraded (13.767%); 8608/249165 objects misplaced (3.455%); 17 MiB/s, 264 objects/s recovering 2024-04-12T11:50:12.758 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:12.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:12 smithi029 ceph-mon[97787]: osdmap e1249: 8 total, 8 up, 8 in 2024-04-12T11:50:12.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:12 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3729498899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:50:12.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:12 smithi029 ceph-mon[97787]: pgmap v1989: 109 pgs: 8 unknown, 1 active, 1 active+recovery_wait+degraded, 2 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 106 MiB/s wr, 2.85k op/s; 34303/249165 objects degraded (13.767%); 8608/249165 objects misplaced (3.455%); 17 MiB/s, 264 objects/s recovering 2024-04-12T11:50:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:12 smithi029 ceph-mon[93650]: osdmap e1249: 8 total, 8 up, 8 in 2024-04-12T11:50:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:12 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3729498899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:50:12.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:12 smithi029 ceph-mon[93650]: pgmap v1989: 109 pgs: 8 unknown, 1 active, 1 active+recovery_wait+degraded, 2 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 106 MiB/s wr, 2.85k op/s; 34303/249165 objects degraded (13.767%); 8608/249165 objects misplaced (3.455%); 17 MiB/s, 264 objects/s recovering 2024-04-12T11:50:12.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:12.783 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:12.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:12.884 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:12.908 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:12.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:12.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:13.010 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:13.036 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:13.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:13.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:13.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:13.170 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:13.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:13.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:13.272 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:13.299 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:13.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:13.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:13.401 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:13.428 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:13.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:13.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:13.530 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:13.557 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:13.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:13.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:13.682 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:13.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:13.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:13.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:13 smithi029 ceph-mon[97787]: osdmap e1250: 8 total, 8 up, 8 in 2024-04-12T11:50:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 34303/249165 objects degraded (13.767%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:13 smithi029 ceph-mon[93650]: osdmap e1250: 8 total, 8 up, 8 in 2024-04-12T11:50:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 34303/249165 objects degraded (13.767%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:13.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:13.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:13.811 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:13.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:13.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:13.937 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:13.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:13 smithi087 ceph-mon[89014]: osdmap e1250: 8 total, 8 up, 8 in 2024-04-12T11:50:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 34303/249165 objects degraded (13.767%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:14.039 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:14.064 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:14.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:14.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:14.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:14.190 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:14.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:14.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:14.292 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:14.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:14.320 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:14.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:14.421 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:14.449 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:14.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:14.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:14.513 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:50:14.513 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:50:14.513 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-04-12T11:50:14.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd test-reweight-by-utilization 110 0.05 2024-04-12T11:50:14.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:14.579 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:14.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:14.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:14.705 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:14.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:14.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:14.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:14 smithi029 ceph-mon[97787]: osdmap e1251: 8 total, 8 up, 8 in 2024-04-12T11:50:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:14 smithi029 ceph-mon[97787]: pgmap v1992: 109 pgs: 1 peering, 5 unknown, 1 active, 4 active+recovery_wait+degraded, 2 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 160 MiB/s wr, 3.88k op/s; 31478/246876 objects degraded (12.751%); 8608/246876 objects misplaced (3.487%); 25 MiB/s, 398 objects/s recovering 2024-04-12T11:50:14.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:14 smithi029 ceph-mon[93650]: osdmap e1251: 8 total, 8 up, 8 in 2024-04-12T11:50:14.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:14 smithi029 ceph-mon[93650]: pgmap v1992: 109 pgs: 1 peering, 5 unknown, 1 active, 4 active+recovery_wait+degraded, 2 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 160 MiB/s wr, 3.88k op/s; 31478/246876 objects degraded (12.751%); 8608/246876 objects misplaced (3.487%); 25 MiB/s, 398 objects/s recovering 2024-04-12T11:50:14.807 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:14.836 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:14.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:14.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:14.938 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:14.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:14.968 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:14.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:14.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:14 smithi087 ceph-mon[89014]: osdmap e1251: 8 total, 8 up, 8 in 2024-04-12T11:50:14.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:14 smithi087 ceph-mon[89014]: pgmap v1992: 109 pgs: 1 peering, 5 unknown, 1 active, 4 active+recovery_wait+degraded, 2 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 160 MiB/s wr, 3.88k op/s; 31478/246876 objects degraded (12.751%); 8608/246876 objects misplaced (3.487%); 25 MiB/s, 398 objects/s recovering 2024-04-12T11:50:15.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:15.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:15.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:15.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:15.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:15.223 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:15.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:15.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:15.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:50:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:50:14] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2024-04-12T11:50:15.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:15.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:15.352 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:15.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:15.454 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:15.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:15.483 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:15.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:15.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:15.611 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:15.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:15.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:15.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:16.034 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:16.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:16.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:16.136 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:16.162 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:16.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:16.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:16.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:16.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:16.288 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:16.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:16.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:16.432 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:16.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:16.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:16.534 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:16.564 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:16.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:16.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:16.666 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:16.691 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:16.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:16.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:16.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:16.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:16.818 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:16.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:16.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:16.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:16.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:16.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:17.056 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:17.097 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:17.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:17.199 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:17.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:17.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:17.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:17.328 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:17.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:17.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:17.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:17.459 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:17 smithi087 ceph-mon[89014]: pgmap v1993: 109 pgs: 1 peering, 2 unknown, 2 active+recovery_wait+remapped, 1 active, 6 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.89k op/s; 31465/239679 objects degraded (13.128%); 8602/239679 objects misplaced (3.589%); 721 KiB/s, 11 objects/s recovering 2024-04-12T11:50:17.496 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:17.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:17.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:17.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:17 smithi029 ceph-mon[97787]: pgmap v1993: 109 pgs: 1 peering, 2 unknown, 2 active+recovery_wait+remapped, 1 active, 6 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.89k op/s; 31465/239679 objects degraded (13.128%); 8602/239679 objects misplaced (3.589%); 721 KiB/s, 11 objects/s recovering 2024-04-12T11:50:17.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:17 smithi029 ceph-mon[93650]: pgmap v1993: 109 pgs: 1 peering, 2 unknown, 2 active+recovery_wait+remapped, 1 active, 6 active+recovery_wait+degraded, 1 active+recovering+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.89k op/s; 31465/239679 objects degraded (13.128%); 8602/239679 objects misplaced (3.589%); 721 KiB/s, 11 objects/s recovering 2024-04-12T11:50:17.598 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:17.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:17.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:17.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:17.754 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:17.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:17.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:17.856 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:17.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:17.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:17.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:17.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:18.007 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:18.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:18.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:18.109 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:18.136 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:18.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:18.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:18.238 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:18.265 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:18.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:18.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:18.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:18.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:18.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:18.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:18.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 31465/239679 objects degraded (13.128%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:18.496 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:18.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 31465/239679 objects degraded (13.128%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:18.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 31465/239679 objects degraded (13.128%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:50:18.521 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:18.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:18.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:18.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:18.655 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:18.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:18.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:18.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:18.794 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:18.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:18.897 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:18.922 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:18.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:18.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:19.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:19.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:19.060 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:19.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:19.161 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:19.197 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:19.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:19.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:19.299 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:19.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:19.323 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:19.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:19.424 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:19.451 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:19.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:19.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:19 smithi087 ceph-mon[89014]: pgmap v1994: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34199/224223 objects degraded (15.252%); 8326/224223 objects misplaced (3.713%) 2024-04-12T11:50:19.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:19 smithi029 ceph-mon[93650]: pgmap v1994: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34199/224223 objects degraded (15.252%); 8326/224223 objects misplaced (3.713%) 2024-04-12T11:50:19.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:19 smithi029 ceph-mon[97787]: pgmap v1994: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34199/224223 objects degraded (15.252%); 8326/224223 objects misplaced (3.713%) 2024-04-12T11:50:19.553 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:19.581 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:19.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:19.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:19.683 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:19.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:19.710 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:19.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:19.811 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:19.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:19.851 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:19.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:19.952 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:19.978 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:19.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:19.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:20.079 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:20.110 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:20.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:20.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:20.212 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:20.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:50:20.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:20.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:20 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:20.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:20.237 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:20.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:20.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:20.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:20.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:20.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:20.504 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:20.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:20.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:50:20.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:20.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:20 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:20.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:50:20.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:20.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:20 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:20.606 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:20.635 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:20.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:20.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:20.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:20.761 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:20.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:20.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:20.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:20.906 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:20.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:21.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:21.042 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:21.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:21.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:21.143 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:21.168 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:21.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:21.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:21.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:21 smithi087 ceph-mon[89014]: pgmap v1995: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34136/223524 objects degraded (15.272%); 8326/223524 objects misplaced (3.725%) 2024-04-12T11:50:21.271 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:21.296 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:21.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:21.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:21.398 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:21.428 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:21.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:21 smithi029 ceph-mon[93650]: pgmap v1995: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34136/223524 objects degraded (15.272%); 8326/223524 objects misplaced (3.725%) 2024-04-12T11:50:21.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:21 smithi029 ceph-mon[97787]: pgmap v1995: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34136/223524 objects degraded (15.272%); 8326/223524 objects misplaced (3.725%) 2024-04-12T11:50:21.530 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:21.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:21.557 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:21.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:21.658 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:21.686 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:21.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:21.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:21.823 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:21.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:21.924 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:21.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:21.949 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:21.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:22.050 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:22.079 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:22.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:22.181 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:22.208 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:22.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:22.311 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:22.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:22.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:22.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:22.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:22.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:22.468 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:50:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:50:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:50:22.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:22.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:22.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:22.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:22.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:22.575 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:22.609 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:22.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:22.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:22.711 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:22.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:22.748 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:22.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:22.849 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:22.878 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:22.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:22.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:22.979 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:23.008 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:23.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:23.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:23.110 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:23.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:23.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:23.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:23.296 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:23.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:23.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:23.330 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:23 smithi087 ceph-mon[89014]: pgmap v1996: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33852/220851 objects degraded (15.328%); 8182/220851 objects misplaced (3.705%) 2024-04-12T11:50:23.330 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 33852/220851 objects degraded (15.328%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:23.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:23.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:23.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:23.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:23.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:23 smithi029 ceph-mon[97787]: pgmap v1996: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33852/220851 objects degraded (15.328%); 8182/220851 objects misplaced (3.705%) 2024-04-12T11:50:23.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 33852/220851 objects degraded (15.328%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:23.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:23 smithi029 ceph-mon[93650]: pgmap v1996: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33852/220851 objects degraded (15.328%); 8182/220851 objects misplaced (3.705%) 2024-04-12T11:50:23.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 33852/220851 objects degraded (15.328%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:23.523 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:23.550 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:23.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:23.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:23.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:23.690 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:23.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:23.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:23.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:23.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:23.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:23.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:23.931 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:23.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:23.961 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:23.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:24.062 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:24.098 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:24.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:24.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:24.201 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:24.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:24.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:24.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:24.328 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:24.354 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:24.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:24.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:24.456 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:24.492 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:24.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:24.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:24.594 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:24.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:24.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:24.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:24.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:24.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:24.763 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:24.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:24.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:24.911 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:24.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:25.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:25.055 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:25.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:25.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:25.083 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:50:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:50:24] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2024-04-12T11:50:25.157 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:25.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:25.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:25.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:25.286 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:25.314 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:25.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:25.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:25.417 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:25.441 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:25.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:25.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:25.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:25 smithi087 ceph-mon[89014]: pgmap v1997: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33640/218709 objects degraded (15.381%); 8182/218709 objects misplaced (3.741%) 2024-04-12T11:50:25.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:25.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:25.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:25 smithi029 ceph-mon[97787]: pgmap v1997: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33640/218709 objects degraded (15.381%); 8182/218709 objects misplaced (3.741%) 2024-04-12T11:50:25.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:25.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:25.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:25 smithi029 ceph-mon[93650]: pgmap v1997: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33640/218709 objects degraded (15.381%); 8182/218709 objects misplaced (3.741%) 2024-04-12T11:50:25.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:25.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:25.544 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:25.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:25.569 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:25.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:25.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:25.707 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:25.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:25.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:25.809 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:25.838 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:25.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:25.939 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:25.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:25.983 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:25.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:26.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:26.116 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:26.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:26.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:26.218 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:26.254 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:26.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:26.358 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:26.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:26.385 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:26.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:26.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:26.509 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:26.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:26.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:26.611 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:26.636 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:26.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:26.738 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:26.775 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:26.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:26.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:50:26.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:26.930 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:26.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:26.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:27.032 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:27.097 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:27.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:27.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:27.107 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:27 smithi029 ceph-mon[93650]: pgmap v1998: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33534/217749 objects degraded (15.400%); 8182/217749 objects misplaced (3.758%) 2024-04-12T11:50:27.107 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:27 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:27.107 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:27 smithi029 ceph-mon[97787]: pgmap v1998: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33534/217749 objects degraded (15.400%); 8182/217749 objects misplaced (3.758%) 2024-04-12T11:50:27.199 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:27.226 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:27.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:27.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:27.328 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:27.353 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:27.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:27.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:27.455 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:27.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:27 smithi087 ceph-mon[89014]: pgmap v1998: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 33534/217749 objects degraded (15.400%); 8182/217749 objects misplaced (3.758%) 2024-04-12T11:50:27.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:27 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:27.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:27.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:27.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:27 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:27.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:27.625 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:27.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:27.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:27.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:27.751 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:27.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:27.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:27.853 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:27.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:27.878 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:27.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:27.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:28.017 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:28.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:28.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:28.123 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:28.162 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:28.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:28.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:28.263 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:28.292 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:28.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:28.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:28.418 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:28.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:28 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 33534/217749 objects degraded (15.400%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:28.470 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:28 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:28.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 33534/217749 objects degraded (15.400%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:28.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:28 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:50:28.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 33534/217749 objects degraded (15.400%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:28.521 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:28.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:28.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:28.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:28.650 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:28.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:28.675 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:28.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:28.777 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:28.802 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:28.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:28.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:28.903 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:28.927 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:28.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:28.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:29.028 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:29.064 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:29.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:29.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:29.126 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:29.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:29.218 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:29.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:29.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:29.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:29 smithi029 ceph-mon[97787]: pgmap v1999: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 33214/214953 objects degraded (15.452%); 8007/214953 objects misplaced (3.725%); 8.6 MiB/s, 137 objects/s recovering 2024-04-12T11:50:29.233 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:29 smithi029 ceph-mon[93650]: pgmap v1999: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 33214/214953 objects degraded (15.452%); 8007/214953 objects misplaced (3.725%); 8.6 MiB/s, 137 objects/s recovering 2024-04-12T11:50:29.251 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:50:29.251 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:50:29.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:121579 8.3 8:c342c08f:::benchmark_data_smithi029_194418_object15269:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:44.401168+0000", 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 344.84533072900001, 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.007615543, 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.252 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.253 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.253 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 121579 2024-04-12T11:50:29.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.253 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.253 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401168+0000", 2024-04-12T11:50:29.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401168+0000", 2024-04-12T11:50:29.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401170+0000", 2024-04-12T11:50:29.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.499e-06 2024-04-12T11:50:29.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401175+0000", 2024-04-12T11:50:29.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8189999999999999e-06 2024-04-12T11:50:29.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401186+0000", 2024-04-12T11:50:29.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1049999999999999e-05 2024-04-12T11:50:29.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.401193+0000", 2024-04-12T11:50:29.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.182e-06 2024-04-12T11:50:29.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.608442+0000", 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20724941099999999 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.608454+0000", 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1792e-05 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.408699+0000", 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.800244972 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.408784+0000", 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4818e-05 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:121596 8.13 8:c8d875f1:::benchmark_data_smithi029_194418_object15286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:44.643200+0000", 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 344.60329881199999, 2024-04-12T11:50:29.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.79399299700000003, 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 121596 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643200+0000", 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643200+0000", 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643202+0000", 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.655e-06 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643210+0000", 2024-04-12T11:50:29.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4759999999999998e-06 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643229+0000", 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8681e-05 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643242+0000", 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3291e-05 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643348+0000", 2024-04-12T11:50:29.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105769 2024-04-12T11:50:29.262 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.262 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:50:29.262 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.643370+0000", 2024-04-12T11:50:29.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2418000000000001e-05 2024-04-12T11:50:29.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.429928+0000", 2024-04-12T11:50:29.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78655749699999999 2024-04-12T11:50:29.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.430037+0000", 2024-04-12T11:50:29.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109009 2024-04-12T11:50:29.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:50:29.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.430234+0000", 2024-04-12T11:50:29.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019738399999999999 2024-04-12T11:50:29.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.431275+0000", 2024-04-12T11:50:29.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001040944 2024-04-12T11:50:29.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.431487+0000", 2024-04-12T11:50:29.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000211296 2024-04-12T11:50:29.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.437078+0000", 2024-04-12T11:50:29.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00559181 2024-04-12T11:50:29.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.267 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.437144+0000", 2024-04-12T11:50:29.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5567000000000004e-05 2024-04-12T11:50:29.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.437193+0000", 2024-04-12T11:50:29.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9200000000000003e-05 2024-04-12T11:50:29.268 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.268 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.269 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.269 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160456 8.7 8:e0bb330f:::benchmark_data_smithi029_194418_object54146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:50:29.269 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.342918+0000", 2024-04-12T11:50:29.269 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 290.903581422, 2024-04-12T11:50:29.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37251305099999998, 2024-04-12T11:50:29.269 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.270 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.270 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.270 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.270 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.270 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.270 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160456 2024-04-12T11:50:29.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.270 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342918+0000", 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342918+0000", 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342919+0000", 2024-04-12T11:50:29.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4580000000000001e-06 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342924+0000", 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8860000000000003e-06 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342936+0000", 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2337000000000001e-05 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342944+0000", 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6059999999999997e-06 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343025+0000", 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1656999999999994e-05 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343181+0000", 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015508300000000001 2024-04-12T11:50:29.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343604+0000", 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00042354900000000001 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346834+0000", 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032303229999999998 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347546+0000", 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000711369 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347894+0000", 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034823999999999999 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715256+0000", 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36736171200000001 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715308+0000", 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1749999999999997e-05 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715431+0000", 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012308100000000001 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160461 8.3 8:c31b08d7:::benchmark_data_smithi029_194418_object54151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.343478+0000", 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 290.90302070899997, 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.371828979, 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160461 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343478+0000", 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343478+0000", 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343479+0000", 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2399999999999997e-07 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343483+0000", 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3039999999999998e-06 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343494+0000", 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0451999999999999e-05 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343501+0000", 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6140000000000003e-06 2024-04-12T11:50:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.344049+0000", 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00054763100000000003 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.344163+0000", 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113725 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.344543+0000", 2024-04-12T11:50:29.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00038017299999999998 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715034+0000", 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37049053999999998 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715054+0000", 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0163999999999999e-05 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715190+0000", 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000135964 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715252+0000", 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2842000000000006e-05 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715307+0000", 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4746000000000001e-05 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160465 8.b 8:dae09cce:::benchmark_data_smithi029_194418_object54155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.346986+0000", 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 290.89951262599999, 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37148354900000002, 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160465 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346986+0000", 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346986+0000", 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346988+0000", 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3859999999999999e-06 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346993+0000", 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.716e-06 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347005+0000", 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1119e-05 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347012+0000", 2024-04-12T11:50:29.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.695e-06 2024-04-12T11:50:29.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347065+0000", 2024-04-12T11:50:29.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2281999999999999e-05 2024-04-12T11:50:29.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347173+0000", 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108682 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,4", 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347304+0000", 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131259 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715066+0000", 2024-04-12T11:50:29.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36776108400000002 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715465+0000", 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00039961 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.718246+0000", 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027803720000000001 2024-04-12T11:50:29.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.718291+0000", 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5025999999999999e-05 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.718351+0000", 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0430000000000002e-05 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.718470+0000", 2024-04-12T11:50:29.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118888 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPushReply(8.b 1206/1204 [PushReplyOp(8:db105a17:::benchmark_data_smithi029_194418_object67346:head)])", 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.347683+0000", 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 290.89881602100002, 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36772148199999999, 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.291 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347683+0000", 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347683+0000", 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347683+0000", 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9999999999999999e-07 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347687+0000", 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3500000000000001e-06 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347694+0000", 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0890000000000002e-06 2024-04-12T11:50:29.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347701+0000", 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1199999999999996e-06 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715160+0000", 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36745903600000002 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715404+0000", 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00024458699999999999 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:160461 8.3 e1206/1199 ondisk, result = 0)", 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.350060+0000", 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 290.89643896799998, 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36500701200000002, 2024-04-12T11:50:29.295 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350060+0000", 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350060+0000", 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350061+0000", 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.097e-06 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350066+0000", 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8600000000000001e-06 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350077+0000", 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0832000000000001e-05 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350084+0000", 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7190000000000001e-06 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715014+0000", 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36492922900000002 2024-04-12T11:50:29.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715026+0000", 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2103000000000001e-05 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715067+0000", 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1171999999999998e-05 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.299 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.300 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160468 8.13 8:c940b4f5:::benchmark_data_smithi029_194418_object54158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:50:29.300 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.350298+0000", 2024-04-12T11:50:29.300 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 290.89620084299997, 2024-04-12T11:50:29.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.41454260900000001, 2024-04-12T11:50:29.300 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.300 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.300 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.300 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160468 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350298+0000", 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350298+0000", 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350299+0000", 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.195e-06 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.303 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350304+0000", 2024-04-12T11:50:29.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9860000000000002e-06 2024-04-12T11:50:29.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715096+0000", 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36479172900000001 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715110+0000", 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3563999999999999e-05 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715435+0000", 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032556700000000002 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715576+0000", 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014090700000000001 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.716039+0000", 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00046330200000000002 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.759398+0000", 2024-04-12T11:50:29.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043358830000000001 2024-04-12T11:50:29.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.759416+0000", 2024-04-12T11:50:29.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7728999999999999e-05 2024-04-12T11:50:29.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.764532+0000", 2024-04-12T11:50:29.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051162480000000003 2024-04-12T11:50:29.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.764800+0000", 2024-04-12T11:50:29.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000267896 2024-04-12T11:50:29.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.764841+0000", 2024-04-12T11:50:29.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0655999999999998e-05 2024-04-12T11:50:29.309 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.309 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.309 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.309 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:166686 8.10 8:0f70ea2e:::benchmark_data_smithi029_194418_object60376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1207)", 2024-04-12T11:50:29.309 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:48.383130+0000", 2024-04-12T11:50:29.310 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 280.86336862899998, 2024-04-12T11:50:29.310 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37085964300000002, 2024-04-12T11:50:29.310 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.310 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.311 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.311 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.311 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.312 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.312 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166686 2024-04-12T11:50:29.312 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.312 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.312 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.312 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383130+0000", 2024-04-12T11:50:29.313 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.313 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.313 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.313 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383130+0000", 2024-04-12T11:50:29.314 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.314 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.314 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.314 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383132+0000", 2024-04-12T11:50:29.314 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3570000000000001e-06 2024-04-12T11:50:29.314 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.314 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.315 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383137+0000", 2024-04-12T11:50:29.315 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7089999999999999e-06 2024-04-12T11:50:29.315 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.315 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.315 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.315 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383157+0000", 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9262e-05 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383165+0000", 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9009999999999994e-06 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383213+0000", 2024-04-12T11:50:29.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8167000000000001e-05 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383342+0000", 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000128885 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4,5", 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383718+0000", 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00037679800000000002 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753747+0000", 2024-04-12T11:50:29.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37002887800000001 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753778+0000", 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0664000000000003e-05 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753828+0000", 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9713999999999999e-05 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753870+0000", 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2629000000000001e-05 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753912+0000", 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1334000000000001e-05 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753990+0000", 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8344999999999996e-05 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:166687 8.3 8:c4dc7983:::benchmark_data_smithi029_194418_object60377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1207)", 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:48.383448+0000", 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 280.86305067799998, 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37355421100000002, 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166687 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383448+0000", 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383448+0000", 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383449+0000", 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0400000000000005e-07 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383454+0000", 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8760000000000004e-06 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383465+0000", 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0628000000000001e-05 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383472+0000", 2024-04-12T11:50:29.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.413e-06 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383528+0000", 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6011999999999998e-05 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383646+0000", 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011756699999999999 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.384094+0000", 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00044867000000000003 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753559+0000", 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36946446300000002 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753688+0000", 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000129333 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.756946+0000", 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003258169 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.756976+0000", 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0133000000000001e-05 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757002+0000", 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6043000000000001e-05 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:166689 8.13 8:cf77213d:::benchmark_data_smithi029_194418_object60379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1207)", 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:48.385913+0000", 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 280.860585915, 2024-04-12T11:50:29.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37121580300000001, 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166689 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.385913+0000", 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.385913+0000", 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.385914+0000", 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4020000000000001e-06 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.385920+0000", 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4199999999999998e-06 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753345+0000", 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36742511 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753368+0000", 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2959000000000002e-05 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753536+0000", 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000168536 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753680+0000", 2024-04-12T11:50:29.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014355999999999999 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.754100+0000", 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041976999999999997 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.756970+0000", 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028701680000000002 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757009+0000", 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8596000000000001e-05 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757052+0000", 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3241999999999997e-05 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.332 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757095+0000", 2024-04-12T11:50:29.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3022000000000002e-05 2024-04-12T11:50:29.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.333 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757129+0000", 2024-04-12T11:50:29.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4017999999999999e-05 2024-04-12T11:50:29.333 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.333 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.333 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:166687 8.3 e1207/1199 ondisk, result = 0)", 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:48.388674+0000", 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 280.85782491200001, 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.364894616, 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.388674+0000", 2024-04-12T11:50:29.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.388674+0000", 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.388675+0000", 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9100000000000002e-07 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.388681+0000", 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5409999999999997e-06 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753344+0000", 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36466264300000001 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753368+0000", 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3822e-05 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753543+0000", 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017491100000000001 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753551+0000", 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5550000000000003e-06 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753569+0000", 2024-04-12T11:50:29.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7252999999999999e-05 2024-04-12T11:50:29.338 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.338 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.338 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.338 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:167527 8.b e1208/1207 8:d8b3b4b5:::benchmark_data_smithi029_194418_object61217:head v 1208'10757, mlcod=1208'10755)", 2024-04-12T11:50:29.338 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.368934+0000", 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 278.87756528300002, 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37717215500000001, 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368934+0000", 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368934+0000", 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368935+0000", 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.048e-06 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368941+0000", 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4930000000000003e-06 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368952+0000", 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0958000000000001e-05 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368959+0000", 2024-04-12T11:50:29.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6449999999999999e-06 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369095+0000", 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000136475 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369115+0000", 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9879000000000001e-05 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.375105+0000", 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059899319999999999 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.746106+0000", 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37100072499999998 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.343 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167531 8.f 8:f56e1a80:::benchmark_data_smithi029_194418_object61221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.369039+0000", 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 278.87745956600003, 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.45084892500000001, 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.344 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167531 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369039+0000", 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369039+0000", 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369040+0000", 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0920000000000001e-06 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369044+0000", 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4199999999999999e-06 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369051+0000", 2024-04-12T11:50:29.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5800000000000003e-06 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369057+0000", 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8510000000000004e-06 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370219+0000", 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011614049999999999 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370344+0000", 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000124948 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,4", 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370468+0000", 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012388500000000001 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752078+0000", 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38161004300000001 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752119+0000", 2024-04-12T11:50:29.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1168000000000003e-05 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752172+0000", 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3448000000000001e-05 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819711+0000", 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067538835000000005 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819764+0000", 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3220000000000002e-05 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819888+0000", 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012402999999999999 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167533 8.f 8:f5e9f8d9:::benchmark_data_smithi029_194418_object61223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.369313+0000", 2024-04-12T11:50:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 278.87718628800002, 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.45089991499999998, 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167533 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369313+0000", 2024-04-12T11:50:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369313+0000", 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369314+0000", 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9800000000000002e-07 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369318+0000", 2024-04-12T11:50:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.685e-06 2024-04-12T11:50:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369329+0000", 2024-04-12T11:50:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0509000000000001e-05 2024-04-12T11:50:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.353 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369336+0000", 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5850000000000002e-06 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374380+0000", 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050431699999999996 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374496+0000", 2024-04-12T11:50:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011610100000000001 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,4", 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374620+0000", 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012398500000000001 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752288+0000", 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37766854799999999 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752332+0000", 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3488000000000001e-05 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752374+0000", 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1845999999999999e-05 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820056+0000", 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067682592 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820100+0000", 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3464999999999998e-05 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820213+0000", 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011294300000000001 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167535 8.0 8:023edd32:::benchmark_data_smithi029_194418_object61225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:50:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 278.87715125099999, 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38064882700000002, 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167535 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:50:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9999999999999998e-07 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369350+0000", 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.351e-06 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369356+0000", 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.874e-06 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369362+0000", 2024-04-12T11:50:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1639999999999998e-06 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369478+0000", 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115281 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369611+0000", 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000132879 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4,5", 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370066+0000", 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000455045 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.746687+0000", 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37662145200000002 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749353+0000", 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026659259999999999 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749762+0000", 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000408537 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749809+0000", 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7679000000000002e-05 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749868+0000", 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8362999999999997e-05 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749997+0000", 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012887599999999999 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167536 8.17 8:effa3b55:::benchmark_data_smithi029_194418_object61226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.372734+0000", 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 278.87376508400001, 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.447314869, 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:50:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167536 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372734+0000", 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372734+0000", 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372735+0000", 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.53e-06 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372740+0000", 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4349999999999999e-06 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372752+0000", 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2604e-05 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372761+0000", 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4349999999999997e-06 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.373764+0000", 2024-04-12T11:50:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001003309 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.373898+0000", 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0001338 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374260+0000", 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00036174299999999998 2024-04-12T11:50:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752035+0000", 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37777566800000001 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752211+0000", 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000175685 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752248+0000", 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6981999999999999e-05 2024-04-12T11:50:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819903+0000", 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067654598999999996 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819947+0000", 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3894999999999999e-05 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820049+0000", 2024-04-12T11:50:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102184 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:167531 8.f e1208/1204 ondisk, result = 0)", 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.376542+0000", 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 278.86995745000002, 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37563801400000002, 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376542+0000", 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376542+0000", 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376543+0000", 2024-04-12T11:50:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.229e-06 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376548+0000", 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2159999999999997e-06 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749229+0000", 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.372680911 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749236+0000", 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6140000000000003e-06 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752159+0000", 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029229360000000001 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752167+0000", 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1720000000000001e-06 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752180+0000", 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2935999999999999e-05 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:167536 8.17 e1208/1204 ondisk, result = 0)", 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 278.86930231899998, 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37484725600000002, 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7100000000000001e-07 2024-04-12T11:50:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377202+0000", 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8250000000000004e-06 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.745962+0000", 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36876029599999999 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.745974+0000", 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1501000000000001e-05 2024-04-12T11:50:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752020+0000", 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006045847 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752028+0000", 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4959999999999996e-06 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752044+0000", 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.562e-05 2024-04-12T11:50:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:167531 8.f e1208/1204 ondisk, result = 0)", 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.379123+0000", 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 278.867376036, 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37296465699999998, 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:50:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379123+0000", 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379123+0000", 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379124+0000", 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6599999999999996e-07 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:50:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379128+0000", 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2030000000000002e-06 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.748158+0000", 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.369030319 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.748173+0000", 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4620999999999999e-05 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752064+0000", 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038913590000000001 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752071+0000", 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9770000000000003e-06 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752088+0000", 2024-04-12T11:50:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6512e-05 2024-04-12T11:50:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:50:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:50:29.385 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:50:29.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:29.387 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:29.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:29 smithi087 ceph-mon[89014]: pgmap v1999: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 33214/214953 objects degraded (15.452%); 8007/214953 objects misplaced (3.725%); 8.6 MiB/s, 137 objects/s recovering 2024-04-12T11:50:29.488 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:29.513 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:29.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:29.614 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:29.638 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:29.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:29.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:29.740 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:29.767 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:29.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:29.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:29.869 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:29.894 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:29.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:29.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:30.030 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:30.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:30.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:30.118 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:50:30.132 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:30.157 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:30.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:30.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:30.259 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:30.298 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:30.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:30.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:30.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:30.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:30.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:30.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:30.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:30 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2585145851' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:50:30.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:30 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:50:30.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:30 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2585145851' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:50:30.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:30 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:50:30.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:30 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2585145851' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:50:30.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:30 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:50:30.541 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:30.566 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:30.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:30.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:30.668 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:30.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:30.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:30.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:30.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:30.829 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:30.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:30.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:30.931 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:30.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:30.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:30.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:31.060 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:31.085 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:31.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:31.187 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:31.212 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:31.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:31.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:31.314 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:31.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:31.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:31.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:31.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:31 smithi087 ceph-mon[89014]: pgmap v2000: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 33117/213723 objects degraded (15.495%); 8007/213723 objects misplaced (3.746%); 7.3 MiB/s, 116 objects/s recovering 2024-04-12T11:50:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:31 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:50:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:31 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:50:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:31 smithi087 ceph-mon[89014]: osdmap e1252: 8 total, 8 up, 8 in 2024-04-12T11:50:31.487 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:31.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:31.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:31.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:31 smithi029 ceph-mon[97787]: pgmap v2000: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 33117/213723 objects degraded (15.495%); 8007/213723 objects misplaced (3.746%); 7.3 MiB/s, 116 objects/s recovering 2024-04-12T11:50:31.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:31 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:50:31.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:31 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:50:31.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:31 smithi029 ceph-mon[97787]: osdmap e1252: 8 total, 8 up, 8 in 2024-04-12T11:50:31.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:31 smithi029 ceph-mon[93650]: pgmap v2000: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 33117/213723 objects degraded (15.495%); 8007/213723 objects misplaced (3.746%); 7.3 MiB/s, 116 objects/s recovering 2024-04-12T11:50:31.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:31 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:50:31.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:31 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:50:31.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:31 smithi029 ceph-mon[93650]: osdmap e1252: 8 total, 8 up, 8 in 2024-04-12T11:50:31.589 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:31.628 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:31.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:31.730 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:31.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:31.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:31.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:31.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:31.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:31.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:31.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:32.022 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:32.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:32.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:32.124 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:32.148 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:32.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:32.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:32.249 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:32.287 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:32.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:32.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:32.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:32.431 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:32.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:32.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:32.533 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:32.559 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:32.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:32.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:32.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:32.685 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:32.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:50:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:50:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:50:32.785 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:32.809 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:32.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:32.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:32.911 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:32.936 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:32.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:32.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:33.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:33.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:33.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:33.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:33.178 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:33.202 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:33.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:33.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:33.304 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:33.336 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:33.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:33.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:33.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:33 smithi087 ceph-mon[89014]: pgmap v2002: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 32817/211137 objects degraded (15.543%); 7830/211137 objects misplaced (3.708%); 8.7 MiB/s, 138 objects/s recovering 2024-04-12T11:50:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 32817/211137 objects degraded (15.543%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:33.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:33.468 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:33.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:33.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:33 smithi029 ceph-mon[97787]: pgmap v2002: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 32817/211137 objects degraded (15.543%); 7830/211137 objects misplaced (3.708%); 8.7 MiB/s, 138 objects/s recovering 2024-04-12T11:50:33.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 32817/211137 objects degraded (15.543%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:33 smithi029 ceph-mon[93650]: pgmap v2002: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 32817/211137 objects degraded (15.543%); 7830/211137 objects misplaced (3.708%); 8.7 MiB/s, 138 objects/s recovering 2024-04-12T11:50:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 32817/211137 objects degraded (15.543%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:33.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:33.612 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:33.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:33.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:33.714 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:33.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:33.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:33.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:33.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:33.876 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:33.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:33.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:33.978 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:34.004 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:34.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:34.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:34.106 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:34.132 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:34.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:34.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:34.234 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:34.260 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:34.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:34.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:34.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:34.387 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:34.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:34.488 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:34.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:34.513 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:34.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:34.614 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:34.638 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:34.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:34.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:34.740 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:34.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:34.771 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:34.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:34.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:34.904 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:34.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:35.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:35.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:35.044 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:35.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:35.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:35.180 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:35.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:35.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:35 smithi029 ceph-mon[93650]: pgmap v2003: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 32612/209022 objects degraded (15.602%); 7830/209022 objects misplaced (3.746%); 8.6 MiB/s, 137 objects/s recovering 2024-04-12T11:50:35.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:35 smithi029 ceph-mon[97787]: pgmap v2003: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 32612/209022 objects degraded (15.602%); 7830/209022 objects misplaced (3.746%); 8.6 MiB/s, 137 objects/s recovering 2024-04-12T11:50:35.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:50:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:50:34] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2024-04-12T11:50:35.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:35.315 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:35.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:35.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:35.416 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:35.442 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:35.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:35.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:35.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:35 smithi087 ceph-mon[89014]: pgmap v2003: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 32612/209022 objects degraded (15.602%); 7830/209022 objects misplaced (3.746%); 8.6 MiB/s, 137 objects/s recovering 2024-04-12T11:50:35.544 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:35.569 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:35.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:35.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:35.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:35.704 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:35.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:35.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:35.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:35.842 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:35.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:35.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:35.944 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:35.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:35.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:35.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:36.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:36.096 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:36.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:36.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:36.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:36.243 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:36.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:36.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:36.346 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:36.371 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:36.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:36.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:36.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:36.516 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:36.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:36.618 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:36.646 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:36.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:36.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:36.748 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:36.775 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:36.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:36.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:36.878 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:36.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:36.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:36.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:36.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-04-12T11:50:37.004 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:37.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:37.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:37.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:37.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:37.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:37.194 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:37.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:37.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:37 smithi029 ceph-mon[93650]: pgmap v2004: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 32494/208056 objects degraded (15.618%); 7830/208056 objects misplaced (3.763%); 8.7 MiB/s, 138 objects/s recovering 2024-04-12T11:50:37.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:37.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:37 smithi029 ceph-mon[97787]: pgmap v2004: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 32494/208056 objects degraded (15.618%); 7830/208056 objects misplaced (3.763%); 8.7 MiB/s, 138 objects/s recovering 2024-04-12T11:50:37.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:37.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:37.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:37.328 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:37.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:37.429 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:37.457 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:37.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:37.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:37.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:37 smithi087 ceph-mon[89014]: pgmap v2004: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 32494/208056 objects degraded (15.618%); 7830/208056 objects misplaced (3.763%); 8.7 MiB/s, 138 objects/s recovering 2024-04-12T11:50:37.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:37.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:37.599 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:37.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:37.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:37.701 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:37.741 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:37.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:37.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:37.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:50:37.843 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:37.868 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:37.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:37.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:37.970 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:38.026 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:38.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:38.068 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:38.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:38.136 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 32494/208056 objects degraded (15.618%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:38.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:38.213 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:38.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:38.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:38.315 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:38.345 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:38.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:38.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 32494/208056 objects degraded (15.618%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:38.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 32494/208056 objects degraded (15.618%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:38.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:38.576 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:38.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:38.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:38.678 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:38.704 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:38.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:38.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:38.806 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:38.833 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:38.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:38.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:38.935 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:38.959 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:38.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:38.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:39.061 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:39.117 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:39.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:39.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:39.219 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:39.246 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:39.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:39.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:39.272 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:39 smithi029 ceph-mon[97787]: pgmap v2005: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 32175/204870 objects degraded (15.705%); 7667/204870 objects misplaced (3.742%); 8.6 MiB/s, 138 objects/s recovering 2024-04-12T11:50:39.272 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:39 smithi029 ceph-mon[93650]: pgmap v2005: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 32175/204870 objects degraded (15.705%); 7667/204870 objects misplaced (3.742%); 8.6 MiB/s, 138 objects/s recovering 2024-04-12T11:50:39.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:39.353 INFO:teuthology.orchestra.run.smithi029.stdout:moved 11 / 327 (3.36391%) 2024-04-12T11:50:39.353 INFO:teuthology.orchestra.run.smithi029.stdout:avg 40.875 2024-04-12T11:50:39.353 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 8.34322 -> 8.59415 (expected baseline 5.98044) 2024-04-12T11:50:39.353 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 24 -> 25 pgs (0.587156 -> 0.611621 * mean) 2024-04-12T11:50:39.353 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.7 with 54 -> 54 pgs (1.3211 -> 1.3211 * mean) 2024-04-12T11:50:39.354 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:50:39.354 INFO:teuthology.orchestra.run.smithi029.stdout:oload 110 2024-04-12T11:50:39.354 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 0.05 2024-04-12T11:50:39.354 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2024-04-12T11:50:39.354 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0255 2024-04-12T11:50:39.354 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0280 2024-04-12T11:50:39.354 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 weight 1.0000 -> 0.9500 2024-04-12T11:50:39.354 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 weight 1.0000 -> 0.9500 2024-04-12T11:50:39.354 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 weight 1.0000 -> 0.9500 2024-04-12T11:50:39.354 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 weight 0.8770 -> 0.9224 2024-04-12T11:50:39.360 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2024-04-12T11:50:39.390 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:39.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:39.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:39.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:39 smithi087 ceph-mon[89014]: pgmap v2005: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 32175/204870 objects degraded (15.705%); 7667/204870 objects misplaced (3.742%); 8.6 MiB/s, 138 objects/s recovering 2024-04-12T11:50:39.491 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:39.519 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:39.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:39.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:39.621 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:39.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:39.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:39.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:39.711 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:50:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:50:39.348+0000 7f4c1f740700 -1 mgr.server reply reply (4) Interrupted system call no change 2024-04-12T11:50:39.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:39.792 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:39.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:39.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:39.894 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:39.921 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:39.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:39.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:40.023 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:40.061 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:40.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:40.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:40.163 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:40.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:40.190 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:40.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:40.297 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:40.337 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:40.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:40.439 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:40.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:40.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:40.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:40 smithi087 ceph-mon[89014]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:50:40.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:40 smithi029 ceph-mon[97787]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:50:40.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:40 smithi029 ceph-mon[93650]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:50:40.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:40.604 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:40.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:40.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:40.707 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:40.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:40.733 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:40.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:40.834 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:40.858 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:40.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:40.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:40.960 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:40.985 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:40.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:40.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:41.087 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:41.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:41.125 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:41.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:41.226 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:41.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:41.251 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:41.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:41.352 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:41.380 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:41.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:41.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:41.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:41 smithi087 ceph-mon[89014]: pgmap v2006: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 32044/203439 objects degraded (15.751%); 7667/203439 objects misplaced (3.769%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:50:41.482 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:41.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:41.509 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:41.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:41.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:41 smithi029 ceph-mon[97787]: pgmap v2006: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 32044/203439 objects degraded (15.751%); 7667/203439 objects misplaced (3.769%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:50:41.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:41 smithi029 ceph-mon[93650]: pgmap v2006: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 32044/203439 objects degraded (15.751%); 7667/203439 objects misplaced (3.769%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:50:41.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:41.646 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:41.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:41.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:41.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:41.782 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:41.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:41.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:41.884 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:41.919 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:41.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:41.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:42.021 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:42.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:42.047 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:42.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:42.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:42.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:42.183 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:42.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:42.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:42.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:42.320 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:42.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:42.422 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:42.449 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:42.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:42.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:42.552 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:42.579 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:42.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:42.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:42.683 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:42.713 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:42.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:42.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:50:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:50:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:50:42.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:42.854 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:42.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:42.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:42.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:42.985 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:42.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:43.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:43.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:43.114 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:43.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:43.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:43.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:43.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:43.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:43.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:43.389 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:43.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:43.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:43.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:43 smithi087 ceph-mon[89014]: pgmap v2007: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 360 op/s; 31673/200202 objects degraded (15.821%); 7493/200202 objects misplaced (3.743%); 9.7 MiB/s, 155 objects/s recovering 2024-04-12T11:50:43.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 31673/200202 objects degraded (15.821%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:43.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:43.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:43 smithi029 ceph-mon[93650]: pgmap v2007: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 360 op/s; 31673/200202 objects degraded (15.821%); 7493/200202 objects misplaced (3.743%); 9.7 MiB/s, 155 objects/s recovering 2024-04-12T11:50:43.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 31673/200202 objects degraded (15.821%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:43.521 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:43 smithi029 ceph-mon[97787]: pgmap v2007: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 360 op/s; 31673/200202 objects degraded (15.821%); 7493/200202 objects misplaced (3.743%); 9.7 MiB/s, 155 objects/s recovering 2024-04-12T11:50:43.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 31673/200202 objects degraded (15.821%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:43.557 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:43.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:43.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:43.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:43.689 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:43.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:43.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:43.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:43.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:43.834 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:43.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:43.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:43.965 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:43.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:43.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:44.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:44.104 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:44.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:44.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:44.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:44.236 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:44.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:44.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:44.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:44.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:44.363 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:44.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:44.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:44.501 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:44.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:44.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:44.607 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:44.651 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:44.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:44.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:44.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:44.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:44.792 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:44.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:44.893 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:44.919 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:44.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:44.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:45.021 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:45.058 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:45.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:45.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:45.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:45.198 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:45.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:45.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:45.203 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:50:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:50:44] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2024-04-12T11:50:45.301 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:45.334 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:45.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:45.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:45.364 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:45.395 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-04-12T11:50:45.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 3.0 2024-04-12T11:50:45.401 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:45.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:45.459 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:45 smithi029 ceph-mon[93650]: pgmap v2008: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 31451/197784 objects degraded (15.902%); 7493/197784 objects misplaced (3.788%); 8.9 MiB/s, 142 objects/s recovering 2024-04-12T11:50:45.459 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:45 smithi029 ceph-mon[97787]: pgmap v2008: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 31451/197784 objects degraded (15.902%); 7493/197784 objects misplaced (3.788%); 8.9 MiB/s, 142 objects/s recovering 2024-04-12T11:50:45.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:45 smithi087 ceph-mon[89014]: pgmap v2008: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 370 op/s; 31451/197784 objects degraded (15.902%); 7493/197784 objects misplaced (3.788%); 8.9 MiB/s, 142 objects/s recovering 2024-04-12T11:50:45.486 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:45.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:45.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:45.588 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:45.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:45.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:45.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:45.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:45.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:45.753 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:45.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:45.854 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:45.879 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:45.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:45.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:45.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:46.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:46.025 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:46.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:46.126 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:46.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:46.177 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:46.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:46.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:46.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:46.319 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:46.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:46.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:46.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:46.446 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:46.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:46.547 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:46.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:46.585 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:46.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:46.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:46.726 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:46.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:46.827 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:46.853 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:46.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:46.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:46.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:46.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:46.980 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:46.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:47.081 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:47.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:47.107 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:47.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:47.208 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:47.234 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:47.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:47.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:47.337 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:47.363 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:47.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:47.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:47.465 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:47.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:47 smithi087 ceph-mon[89014]: pgmap v2009: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 31327/196656 objects degraded (15.930%); 7493/196656 objects misplaced (3.810%); 8.5 MiB/s, 135 objects/s recovering 2024-04-12T11:50:47.472 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:47 smithi029 ceph-mon[97787]: pgmap v2009: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 31327/196656 objects degraded (15.930%); 7493/196656 objects misplaced (3.810%); 8.5 MiB/s, 135 objects/s recovering 2024-04-12T11:50:47.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:47 smithi029 ceph-mon[93650]: pgmap v2009: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 31327/196656 objects degraded (15.930%); 7493/196656 objects misplaced (3.810%); 8.5 MiB/s, 135 objects/s recovering 2024-04-12T11:50:47.489 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:47.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:47.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:47.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:47.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:47.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:47.725 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:47.749 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:47.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:47.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:47.852 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:47.887 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:47.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:47.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:48.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:48.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:48.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:48.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:48.140 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:50:48.155 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:48.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:48.189 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:50:48.231 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:50:48.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:48.275 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:50:48.294 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:48.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:48.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:48.308 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:50:48.348 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:50:48.377 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:50:48.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:48.410 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:50:48.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 31327/196656 objects degraded (15.930%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:48.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:48 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1729245713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:50:48.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 31327/196656 objects degraded (15.930%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:48.411 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:48 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1729245713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:50:48.416 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:50:48.416 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:50:48.417 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:50:48.417 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:50:48.417 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:50:48.417 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:50:48.417 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:50:48.418 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:50:48.436 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:48.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:48.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:48.538 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:48.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:48.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:48.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:48.665 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:48.690 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:48.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:48.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 31327/196656 objects degraded (15.930%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:48 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1729245713' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:50:48.791 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:48.820 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:48.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:48.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:48.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:48.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:48.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:49.056 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:49.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:49.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:49.194 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:49.233 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:49.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:49.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:49.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:49.374 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:49.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:49.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:49.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:49.493 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:49 smithi029 ceph-mon[93650]: pgmap v2010: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 30954/193482 objects degraded (15.998%); 7289/193482 objects misplaced (3.767%); 11 MiB/s, 173 objects/s recovering 2024-04-12T11:50:49.493 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:49 smithi029 ceph-mon[97787]: pgmap v2010: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 30954/193482 objects degraded (15.998%); 7289/193482 objects misplaced (3.767%); 11 MiB/s, 173 objects/s recovering 2024-04-12T11:50:49.510 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:49.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:49.612 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:49.649 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:49.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:49.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:49 smithi087 ceph-mon[89014]: pgmap v2010: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 30954/193482 objects degraded (15.998%); 7289/193482 objects misplaced (3.767%); 11 MiB/s, 173 objects/s recovering 2024-04-12T11:50:49.750 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:49.785 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:49.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:49.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:49.887 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:49.915 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:49.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:49.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:50.016 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:50.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:50.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:50.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:50.154 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:50.177 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:50.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:50.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:50.279 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:50.317 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:50.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:50.419 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:50.446 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:50.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:50.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:50.548 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:50.574 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:50.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:50.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:50.676 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:50.701 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:50.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:50.802 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:50.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:50.837 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:50.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:50.939 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:50.965 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:50.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:50.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:51.066 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:51.090 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:51.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:51.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:51.192 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:51.201 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:50:51.215 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:51.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:51.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:51.317 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:51.341 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:51.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:51.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:51.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:51.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:51.493 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:51.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:51.506 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:51 smithi029 ceph-mon[97787]: pgmap v2011: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 355 op/s; 30849/192072 objects degraded (16.061%); 7289/192072 objects misplaced (3.795%); 8.9 MiB/s, 142 objects/s recovering 2024-04-12T11:50:51.506 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:51 smithi029 ceph-mon[93650]: pgmap v2011: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 355 op/s; 30849/192072 objects degraded (16.061%); 7289/192072 objects misplaced (3.795%); 8.9 MiB/s, 142 objects/s recovering 2024-04-12T11:50:51.594 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:51.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:51.627 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:51.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:51 smithi087 ceph-mon[89014]: pgmap v2011: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 355 op/s; 30849/192072 objects degraded (16.061%); 7289/192072 objects misplaced (3.795%); 8.9 MiB/s, 142 objects/s recovering 2024-04-12T11:50:51.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:51.752 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:51.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:51.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:51.854 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:51.882 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:51.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:51.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:51.984 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:52.008 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:52.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:52.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:52.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:52.138 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:52.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:52.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:52.274 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:52.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:52.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:52.399 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:52.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:52.500 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:52.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:52.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:52.525 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:52.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:52.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:52.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:50:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:50:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:50:52.618 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:50:52.627 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:52.653 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:52.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:52.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:52.755 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:52.788 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:52.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:52.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:52.890 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:52.916 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:52.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:52.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:53.018 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:53.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:53.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:53.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:53.183 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:53.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:53.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:53.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:53.308 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:53.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:53.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:53.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:53.442 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:53.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:53.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:53 smithi029 ceph-mon[97787]: pgmap v2012: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 30494/188967 objects degraded (16.137%); 7079/188967 objects misplaced (3.746%); 11 MiB/s, 178 objects/s recovering 2024-04-12T11:50:53.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 30494/188967 objects degraded (16.137%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:53 smithi029 ceph-mon[93650]: pgmap v2012: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 30494/188967 objects degraded (16.137%); 7079/188967 objects misplaced (3.746%); 11 MiB/s, 178 objects/s recovering 2024-04-12T11:50:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 30494/188967 objects degraded (16.137%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:53.544 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:53.573 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:53.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:53.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:53.675 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:53.701 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:53.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:53.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:53 smithi087 ceph-mon[89014]: pgmap v2012: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 30494/188967 objects degraded (16.137%); 7079/188967 objects misplaced (3.746%); 11 MiB/s, 178 objects/s recovering 2024-04-12T11:50:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 30494/188967 objects degraded (16.137%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:53.804 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:53.831 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:53.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:53.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:53.933 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:53.966 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:53.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:53.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:54.068 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:54.094 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:54.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:54.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:54.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:54.229 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:54.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:54.330 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:54.356 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:54.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:54.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:54.458 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:54.482 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:54.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:54.584 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:54.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:54.610 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:54.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:54.710 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:54.738 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:54.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:54.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:54.839 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:54.862 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:54.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:54.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:54.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:54.988 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:54.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:54.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:55.053 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:55.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:55.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:55.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:55.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:55.233 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:55.236 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:50:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:50:54] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2024-04-12T11:50:55.259 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:55.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:55.362 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:55.386 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:55.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:55.488 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:55.513 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:55.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:55.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:55.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:55 smithi029 ceph-mon[97787]: pgmap v2013: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 30269/186579 objects degraded (16.223%); 7079/186579 objects misplaced (3.794%); 9.5 MiB/s, 151 objects/s recovering 2024-04-12T11:50:55.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:55 smithi029 ceph-mon[93650]: pgmap v2013: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 30269/186579 objects degraded (16.223%); 7079/186579 objects misplaced (3.794%); 9.5 MiB/s, 151 objects/s recovering 2024-04-12T11:50:55.615 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:55.640 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:55.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:55.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:55 smithi087 ceph-mon[89014]: pgmap v2013: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 30269/186579 objects degraded (16.223%); 7079/186579 objects misplaced (3.794%); 9.5 MiB/s, 151 objects/s recovering 2024-04-12T11:50:55.742 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:55.766 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:55.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:55.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:55.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:55.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:55.925 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:55.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:56.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:56.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:56.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:56.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:56.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:56.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:56.206 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:56.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:56.263 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:50:56.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:56.333 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:56.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:56.434 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:56.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:56.464 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:56 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1937645889' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:50:56.465 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:56 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:50:56.465 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:56 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1937645889' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:50:56.465 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:56 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:50:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:56.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:56.566 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:56.576 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:50:56.576 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:50:56.593 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:56.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:56.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:56.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:56.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:56 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1937645889' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:50:56.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:56 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:50:56.738 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:56.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:56.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:56.840 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:56.867 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:56.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:56.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:56.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:57.006 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:57.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:57.031 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:50:57.107 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:57.133 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:57.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:57.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:57.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:57.260 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:57.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:57.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:57.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:57.399 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:57.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:57.500 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:57.503 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:57 smithi029 ceph-mon[97787]: pgmap v2014: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 30159/185592 objects degraded (16.250%); 7079/185592 objects misplaced (3.814%); 8.9 MiB/s, 141 objects/s recovering 2024-04-12T11:50:57.503 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:57 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:50:57.503 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:57 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:50:57.503 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:57 smithi029 ceph-mon[97787]: osdmap e1253: 8 total, 8 up, 8 in 2024-04-12T11:50:57.504 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:57 smithi029 ceph-mon[93650]: pgmap v2014: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 30159/185592 objects degraded (16.250%); 7079/185592 objects misplaced (3.814%); 8.9 MiB/s, 141 objects/s recovering 2024-04-12T11:50:57.504 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:57 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:50:57.504 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:57 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:50:57.504 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:57 smithi029 ceph-mon[93650]: osdmap e1253: 8 total, 8 up, 8 in 2024-04-12T11:50:57.528 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:57.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:57.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:57.630 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:57.656 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:57.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:57 smithi087 ceph-mon[89014]: pgmap v2014: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 338 op/s; 30159/185592 objects degraded (16.250%); 7079/185592 objects misplaced (3.814%); 8.9 MiB/s, 141 objects/s recovering 2024-04-12T11:50:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:57 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:50:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:57 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:50:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:57 smithi087 ceph-mon[89014]: osdmap e1253: 8 total, 8 up, 8 in 2024-04-12T11:50:57.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:57.789 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:57.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:57.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:57.924 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:57.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:57.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:58.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:50:58.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:58.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:50:58.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:58.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:58.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:58.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:58.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:58.349 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:58.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:58.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:50:58.451 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:58.481 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:58.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:58.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:58.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 30159/185592 objects degraded (16.250%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:58.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 30159/185592 objects degraded (16.250%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:58.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:58.629 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:58.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 30159/185592 objects degraded (16.250%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:50:58.730 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:58.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:58.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:58.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:58.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:50:58.894 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:58.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:58.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:50:58.996 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:50:59.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:59.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:59.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:50:59.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:59.149 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:59.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:59.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:59.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:59.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:59.277 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:59.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:59.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:50:59.416 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:59.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:59.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:50:59.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:50:59.521 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:50:59 smithi029 ceph-mon[97787]: pgmap v2016: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 29837/182793 objects degraded (16.323%); 6871/182793 objects misplaced (3.759%); 9.6 MiB/s, 153 objects/s recovering 2024-04-12T11:50:59.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:50:59 smithi029 ceph-mon[93650]: pgmap v2016: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 29837/182793 objects degraded (16.323%); 6871/182793 objects misplaced (3.759%); 9.6 MiB/s, 153 objects/s recovering 2024-04-12T11:50:59.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:59.560 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:59.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:50:59.661 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:50:59.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:59.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:59.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:50:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:50:59 smithi087 ceph-mon[89014]: pgmap v2016: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 29837/182793 objects degraded (16.323%); 6871/182793 objects misplaced (3.759%); 9.6 MiB/s, 153 objects/s recovering 2024-04-12T11:50:59.790 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:50:59.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:59.816 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:50:59.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:50:59.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:50:59.942 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:50:59.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:50:59.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:00.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:00.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:00.084 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:00.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:00.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:51:00.185 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:00.211 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:00.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:00.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:00.312 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:00.335 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:00.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:00.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:00.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:00.465 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:00.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:00.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:00.567 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:00.591 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:00.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:00.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:00.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:00.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:00.720 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:00.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:00.821 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:00.850 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:00.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:00.952 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:00.981 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:00.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:00.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:01.084 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:01.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:01.116 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:01.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:01.217 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:01.241 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:01.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:01.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:01.343 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:01.369 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:01.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:01.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:01.471 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:01.496 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:01.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:01.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:01.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:01 smithi029 ceph-mon[97787]: pgmap v2017: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 29724/181461 objects degraded (16.380%); 6871/181461 objects misplaced (3.786%); 9.6 MiB/s, 154 objects/s recovering 2024-04-12T11:51:01.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:01 smithi029 ceph-mon[93650]: pgmap v2017: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 29724/181461 objects degraded (16.380%); 6871/181461 objects misplaced (3.786%); 9.6 MiB/s, 154 objects/s recovering 2024-04-12T11:51:01.598 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:01.625 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:01.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:01 smithi087 ceph-mon[89014]: pgmap v2017: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 29724/181461 objects degraded (16.380%); 6871/181461 objects misplaced (3.786%); 9.6 MiB/s, 154 objects/s recovering 2024-04-12T11:51:01.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:01.752 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:01.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:01.854 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:01.890 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:01.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:01.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:01.992 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:02.016 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:02.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:02.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:02.151 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:02.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:02.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:02.280 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:02.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:02.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:02.381 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:02.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:02.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:02.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:02.507 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:02.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:02.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:02.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:02.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:02.672 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:02.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:02.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:51:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:51:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:51:02.774 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:02.798 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:02.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:02.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:02.929 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:02.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:02.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:03.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:03.074 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:03.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:03.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:03.176 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:03.211 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:03.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:03.313 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:03.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:03.364 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:03.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:03.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:03.492 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:03.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:03.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:03 smithi029 ceph-mon[97787]: pgmap v2018: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 359 op/s; 29368/178188 objects degraded (16.481%); 6639/178188 objects misplaced (3.726%); 9.8 MiB/s, 156 objects/s recovering 2024-04-12T11:51:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 29368/178188 objects degraded (16.481%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:03 smithi029 ceph-mon[93650]: pgmap v2018: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 359 op/s; 29368/178188 objects degraded (16.481%); 6639/178188 objects misplaced (3.726%); 9.8 MiB/s, 156 objects/s recovering 2024-04-12T11:51:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 29368/178188 objects degraded (16.481%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:03.594 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:03.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:03.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:03.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:03 smithi087 ceph-mon[89014]: pgmap v2018: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 359 op/s; 29368/178188 objects degraded (16.481%); 6639/178188 objects misplaced (3.726%); 9.8 MiB/s, 156 objects/s recovering 2024-04-12T11:51:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 29368/178188 objects degraded (16.481%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:03.723 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:03.748 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:03.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:03.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:03.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:03.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:03.888 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:03.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:03.989 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:04.014 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:04.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:04.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:04.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:04.167 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:04.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:04.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:04.270 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:04.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:04.309 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:04.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:04.410 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:04.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:04.438 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:04.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:04.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:04.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:04.576 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:04.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:04.677 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:04.703 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:04.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:04.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:04.805 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:04.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:04.839 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:04.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:04.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:04.977 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:04.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:04.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:05.079 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:05.114 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:05.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:05.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:05.216 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:05.247 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:05.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:05.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:51:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:51:04] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2024-04-12T11:51:05.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:05.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:05.383 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:05.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:05.479 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-04-12T11:51:05.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:05.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:05 smithi029 ceph-mon[97787]: pgmap v2019: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 29142/175566 objects degraded (16.599%); 6639/175566 objects misplaced (3.781%); 9.8 MiB/s, 156 objects/s recovering 2024-04-12T11:51:05.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:05 smithi029 ceph-mon[93650]: pgmap v2019: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 29142/175566 objects degraded (16.599%); 6639/175566 objects misplaced (3.781%); 9.8 MiB/s, 156 objects/s recovering 2024-04-12T11:51:05.563 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:05.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:05.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:05 smithi087 ceph-mon[89014]: pgmap v2019: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 29142/175566 objects degraded (16.599%); 6639/175566 objects misplaced (3.781%); 9.8 MiB/s, 156 objects/s recovering 2024-04-12T11:51:05.753 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:05.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:05.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:05.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:05.881 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:05.906 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:05.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:05.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:06.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:06.040 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:06.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:06.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:06.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:06.183 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:06.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:06.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:06.285 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:06.313 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:06.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:06.414 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:06.439 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:06.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:06.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:06.541 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:06.569 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:06.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:06.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:06.710 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:06.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:06.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:06.811 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:06.838 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:06.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:06.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:06.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:06.981 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:06.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:06.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:07.083 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:07.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:07.110 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:07.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:07.211 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:07.236 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:07.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:07.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:07.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:07.362 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:07.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:07.464 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:07.488 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:07.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:07.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:07.576 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:07 smithi029 ceph-mon[97787]: pgmap v2020: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 29014/174387 objects degraded (16.638%); 6639/174387 objects misplaced (3.807%); 9.9 MiB/s, 158 objects/s recovering 2024-04-12T11:51:07.576 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:07.576 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:07.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:07 smithi029 ceph-mon[93650]: pgmap v2020: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 29014/174387 objects degraded (16.638%); 6639/174387 objects misplaced (3.807%); 9.9 MiB/s, 158 objects/s recovering 2024-04-12T11:51:07.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:07.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:07.590 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:07.618 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:07.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:07.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:07 smithi087 ceph-mon[89014]: pgmap v2020: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 29014/174387 objects degraded (16.638%); 6639/174387 objects misplaced (3.807%); 9.9 MiB/s, 158 objects/s recovering 2024-04-12T11:51:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:07.721 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:07.762 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:07.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:07.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:07.864 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:07.904 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:07.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:07.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:08.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:08.048 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:08.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:08.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:08.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:08.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:08.174 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:08.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:08.275 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:08.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:08.311 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:08.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:08.325 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:51:08 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:51:08.155+0000 7f4c1f740700 -1 mgr.server reply reply (1) Operation not permitted no change 2024-04-12T11:51:08.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:08.454 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:08.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:08.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:08.556 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:08.580 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:08.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:08.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:08.682 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:08.707 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:08.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:08.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 29014/174387 objects degraded (16.638%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 29014/174387 objects degraded (16.638%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 29014/174387 objects degraded (16.638%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:08.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:08.844 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:08.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:08.947 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:08.971 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:08.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:08.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:09.073 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:09.099 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:09.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:09.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:09.201 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:09.238 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:09.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:09.340 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:09.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:09.365 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:09.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:09.465 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:09.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:09.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:09.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:09.592 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:09.628 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:09.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:09.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:09 smithi087 ceph-mon[89014]: pgmap v2021: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 28664/171219 objects degraded (16.741%); 6424/171219 objects misplaced (3.752%); 11 MiB/s, 182 objects/s recovering 2024-04-12T11:51:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:09 smithi087 ceph-mon[89014]: from='client.45485 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:51:09.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:09.767 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:09.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:09.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:09 smithi029 ceph-mon[97787]: pgmap v2021: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 28664/171219 objects degraded (16.741%); 6424/171219 objects misplaced (3.752%); 11 MiB/s, 182 objects/s recovering 2024-04-12T11:51:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:09 smithi029 ceph-mon[97787]: from='client.45485 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:51:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:09 smithi029 ceph-mon[93650]: pgmap v2021: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 28664/171219 objects degraded (16.741%); 6424/171219 objects misplaced (3.752%); 11 MiB/s, 182 objects/s recovering 2024-04-12T11:51:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:09 smithi029 ceph-mon[93650]: from='client.45485 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:51:09.869 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:09.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:09.896 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:09.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:09.997 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:10.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:10.022 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:10.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:10.123 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:10.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:10.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:10.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:10.253 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 327 (0%) 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:avg 40.875 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 8.34322 -> 8.34322 (expected baseline 5.98044) 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 24 -> 24 pgs (0.587156 -> 0.587156 * mean) 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.7 with 54 -> 54 pgs (1.3211 -> 1.3211 * mean) 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 3 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0231 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.2313 2024-04-12T11:51:10.255 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 weight 0.8770 -> 0.8877 2024-04-12T11:51:10.256 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2024-04-12T11:51:10.278 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:10.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:10.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:10.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:10.410 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:10.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:10.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:10.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:10.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:10.575 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:10.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:10.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:10.715 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:10.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:10.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:10.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:10.850 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:10.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:10.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:10.952 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:10.986 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:10.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:10.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:11.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:11.103 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:11.131 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-04-12T11:51:11.131 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 0.05 2024-04-12T11:51:11.148 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:11.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:11.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:11.251 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:11.296 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:11.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:11.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:11.434 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:11.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:11.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:11 smithi029 ceph-mon[97787]: pgmap v2022: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 28548/169755 objects degraded (16.817%); 6424/169755 objects misplaced (3.784%); 9.1 MiB/s, 144 objects/s recovering 2024-04-12T11:51:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:11 smithi029 ceph-mon[93650]: pgmap v2022: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 28548/169755 objects degraded (16.817%); 6424/169755 objects misplaced (3.784%); 9.1 MiB/s, 144 objects/s recovering 2024-04-12T11:51:11.536 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:11.572 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:11.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:11.674 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:11.701 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:11.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:11.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:11 smithi087 ceph-mon[89014]: pgmap v2022: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 28548/169755 objects degraded (16.817%); 6424/169755 objects misplaced (3.784%); 9.1 MiB/s, 144 objects/s recovering 2024-04-12T11:51:11.803 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:11.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:11.839 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:11.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:11.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:11.975 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:11.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:11.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:12.077 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:12.101 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:12.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:12.203 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:12.228 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:12.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:12.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:12.330 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:12.356 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:12.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:12.457 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:12.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:12.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:12.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:12.601 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:12.629 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:12.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:12.671 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:51:12.708 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:51:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:51:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:51:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:51:12.730 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:12.743 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:51:12.763 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:12.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:12.780 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:51:12.816 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:51:12.854 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:51:12.865 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:12.882 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:51:12.889 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:12.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:12.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:12.918 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:51:12.921 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:51:12.921 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:51:12.921 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:51:12.921 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:51:12.921 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:51:12.922 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:51:12.922 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:51:12.922 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:51:12.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:13.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:13.030 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:13.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:13.132 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:13.158 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:13.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:13.259 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:13.286 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:13.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:13.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:13.388 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:13.415 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:13.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:13.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:13.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:13.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:13.555 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:13.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:13.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:13.664 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:13 smithi029 ceph-mon[97787]: pgmap v2023: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 396 op/s; 28421/167193 objects degraded (16.999%); 6209/167193 objects misplaced (3.714%); 10 MiB/s, 163 objects/s recovering 2024-04-12T11:51:13.665 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:13 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1858031103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:51:13.665 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 28421/167193 objects degraded (16.999%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:13.665 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:13 smithi029 ceph-mon[93650]: pgmap v2023: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 396 op/s; 28421/167193 objects degraded (16.999%); 6209/167193 objects misplaced (3.714%); 10 MiB/s, 163 objects/s recovering 2024-04-12T11:51:13.665 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:13 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1858031103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:51:13.665 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 28421/167193 objects degraded (16.999%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:13.703 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:13.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:13.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:13 smithi087 ceph-mon[89014]: pgmap v2023: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 396 op/s; 28421/167193 objects degraded (16.999%); 6209/167193 objects misplaced (3.714%); 10 MiB/s, 163 objects/s recovering 2024-04-12T11:51:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:13 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1858031103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:51:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 28421/167193 objects degraded (16.999%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:13.805 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:13.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:13.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:13.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:13.908 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:13.943 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:13.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:13.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:13.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:14.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:14.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:14.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:14.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:14.197 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:14.230 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:14.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:14.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:14.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:14.362 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:14.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:14.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:14.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:14.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:14.505 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:14.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:14.606 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:14.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:14.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:14.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:14.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:14.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:14.782 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:14.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:14.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:14.910 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:14.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:14.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:15.012 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:15.036 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:15.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:15.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:15.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:15.164 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:15.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:15.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:15.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:15.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:51:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:51:14] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2024-04-12T11:51:15.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:15.296 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:15.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:15.354 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:51:15.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:15.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:15.430 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:15.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:15.531 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:15.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:15.556 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:15.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:15.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:15.697 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:15.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:15.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:15 smithi087 ceph-mon[89014]: pgmap v2024: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 27939/163965 objects degraded (17.040%); 6209/163965 objects misplaced (3.787%); 9.7 MiB/s, 154 objects/s recovering 2024-04-12T11:51:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:15 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3774946078' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:51:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:15 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:51:15.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:15 smithi029 ceph-mon[97787]: pgmap v2024: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 27939/163965 objects degraded (17.040%); 6209/163965 objects misplaced (3.787%); 9.7 MiB/s, 154 objects/s recovering 2024-04-12T11:51:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:15 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3774946078' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:51:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:15 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:51:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:15 smithi029 ceph-mon[93650]: pgmap v2024: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 27939/163965 objects degraded (17.040%); 6209/163965 objects misplaced (3.787%); 9.7 MiB/s, 154 objects/s recovering 2024-04-12T11:51:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:15 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3774946078' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:51:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:15 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:51:15.798 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:15.822 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:15.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:15.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:15.923 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:15.958 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:15.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:16.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:16.109 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:16.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:16.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:16.211 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:16.250 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:16.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:16.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:16.352 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:16.376 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:16.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:16.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:16.465 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:16 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:51:16.465 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:16 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:51:16.465 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:16 smithi029 ceph-mon[93650]: osdmap e1254: 8 total, 8 up, 8 in 2024-04-12T11:51:16.465 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:16 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:51:16.465 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:16 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:51:16.465 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:16 smithi029 ceph-mon[97787]: osdmap e1254: 8 total, 8 up, 8 in 2024-04-12T11:51:16.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:16.513 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:16.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:16.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:16.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:16.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:16.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:16.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:16 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:51:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:16 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:51:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:16 smithi087 ceph-mon[89014]: osdmap e1254: 8 total, 8 up, 8 in 2024-04-12T11:51:16.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:16.782 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:16.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:16.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:16.907 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:16.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:16.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:17.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:17.043 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:17.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:17.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:17.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:17.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:17.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:17.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:17.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:17.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:17.354 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:17.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:17.455 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:17.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:17.500 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:17.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:17.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:17 smithi029 ceph-mon[97787]: pgmap v2026: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 27814/162903 objects degraded (17.074%); 6209/162903 objects misplaced (3.811%); 10 MiB/s, 162 objects/s recovering 2024-04-12T11:51:17.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:17 smithi029 ceph-mon[93650]: pgmap v2026: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 27814/162903 objects degraded (17.074%); 6209/162903 objects misplaced (3.811%); 10 MiB/s, 162 objects/s recovering 2024-04-12T11:51:17.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:51:17.601 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:17.624 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:17.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:17.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:17 smithi087 ceph-mon[89014]: pgmap v2026: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 27814/162903 objects degraded (17.074%); 6209/162903 objects misplaced (3.811%); 10 MiB/s, 162 objects/s recovering 2024-04-12T11:51:17.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:17.754 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:17.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:17.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:17.856 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:17.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:17.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:17.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:17.987 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:18.017 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:18.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:18.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:18.119 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:18.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:18.149 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:18.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:18.250 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:18.277 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:18.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:18.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:18.379 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:18.423 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:18.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:18.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:18.498 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:51:18.525 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:18.551 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:18.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:18.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:18.654 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:18.707 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:18.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:18.808 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:18.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:18.859 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:18.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:18.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:18 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:18.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:18 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:18.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:18 smithi029 ceph-mon[93650]: pgmap v2027: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 27420/159459 objects degraded (17.196%); 5929/159459 objects misplaced (3.718%); 11 MiB/s, 173 objects/s recovering 2024-04-12T11:51:18.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 27814/162903 objects degraded (17.074%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:18.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:18 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:18.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:18 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:18.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:18 smithi029 ceph-mon[97787]: pgmap v2027: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 27420/159459 objects degraded (17.196%); 5929/159459 objects misplaced (3.718%); 11 MiB/s, 173 objects/s recovering 2024-04-12T11:51:18.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 27814/162903 objects degraded (17.074%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:18.960 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:19.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:19.008 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:19.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:19.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:19.135 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:51:19.136 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:51:19.136 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:51:19.136 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:51:19.136 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:51:19.140 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:19.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:19.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:18 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:18 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:18 smithi087 ceph-mon[89014]: pgmap v2027: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 27420/159459 objects degraded (17.196%); 5929/159459 objects misplaced (3.718%); 11 MiB/s, 173 objects/s recovering 2024-04-12T11:51:19.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 27814/162903 objects degraded (17.074%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:19.242 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:19.266 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:19.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:19.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:19.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:19.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:19.410 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:19.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:19.511 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:19.540 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:19.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:19.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:19.643 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:19.667 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:19.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:19.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:19.770 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:19.795 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:19.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:19.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:19.897 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:19.940 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:19.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:19.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:20.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:51:20.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:51:20.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:20.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:51:20.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:51:20.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:20.041 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:20.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:20.083 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:20.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:20.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-04-12T11:51:20.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:20.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:51:20.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:51:20.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:20.230 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:20.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:20.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:20.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:20.356 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:20.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:20.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:20.481 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:20.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:20.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:20.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:20.622 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:20.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:20.715 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 327 (0%) 2024-04-12T11:51:20.715 INFO:teuthology.orchestra.run.smithi029.stdout:avg 40.875 2024-04-12T11:51:20.715 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 8.34322 -> 8.34322 (expected baseline 5.98044) 2024-04-12T11:51:20.715 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 24 -> 24 pgs (0.587156 -> 0.587156 * mean) 2024-04-12T11:51:20.715 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.7 with 54 -> 54 pgs (1.3211 -> 1.3211 * mean) 2024-04-12T11:51:20.716 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:51:20.716 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2024-04-12T11:51:20.716 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 0.05 2024-04-12T11:51:20.716 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2024-04-12T11:51:20.716 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0222 2024-04-12T11:51:20.716 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.2222 2024-04-12T11:51:20.716 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2024-04-12T11:51:20.723 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:20.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:20.752 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:20.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:20.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:20.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:20.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:20.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:20.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:21.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:21.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:21.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:20 smithi029 ceph-mon[97787]: pgmap v2028: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 27271/157896 objects degraded (17.271%); 5929/157896 objects misplaced (3.755%); 11 MiB/s, 176 objects/s recovering 2024-04-12T11:51:21.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:20 smithi029 ceph-mon[93650]: pgmap v2028: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 27271/157896 objects degraded (17.271%); 5929/157896 objects misplaced (3.755%); 11 MiB/s, 176 objects/s recovering 2024-04-12T11:51:21.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:21.156 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:21.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:21.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:21.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:20 smithi087 ceph-mon[89014]: pgmap v2028: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 27271/157896 objects degraded (17.271%); 5929/157896 objects misplaced (3.755%); 11 MiB/s, 176 objects/s recovering 2024-04-12T11:51:21.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:21.293 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:21.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:21.395 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:21.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:21.422 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:21.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:21.523 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:21.557 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:21.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:21.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:21.687 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:21.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:21.789 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:21.827 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:21.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:21.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:21.929 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:21.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:21.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:21.992 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:22.060 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:22.063 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:22.085 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:22.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:22.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:22.187 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:22.224 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:22.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:22.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:22.326 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:22.351 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:22.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:22.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:22 smithi029 ceph-mon[97787]: from='client.35610 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:51:22.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:22.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:22.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:22 smithi029 ceph-mon[93650]: from='client.35610 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:51:22.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:22.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:22.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:22.489 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:22.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:22.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:22.591 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:22.616 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:22.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:22.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:51:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:51:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:51:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:22 smithi087 ceph-mon[89014]: from='client.35610 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:51:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:51:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:22.718 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:22.741 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:22.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:22.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:22.843 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:22.867 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:22.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:22.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:22.969 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:23.008 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:23.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:23.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:23.109 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:23.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:23.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:23.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:23.208 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-04-12T11:51:23.208 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight-by-utilization 1000 1.0 2024-04-12T11:51:23.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:23.268 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:23.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:23.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:23.370 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:23.397 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:23.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:23.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:23.498 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:23.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:23 smithi029 ceph-mon[97787]: pgmap v2029: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 27147/155220 objects degraded (17.489%); 5705/155220 objects misplaced (3.675%); 11 MiB/s, 177 objects/s recovering 2024-04-12T11:51:23.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 27147/155220 objects degraded (17.489%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:23 smithi029 ceph-mon[93650]: pgmap v2029: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 27147/155220 objects degraded (17.489%); 5705/155220 objects misplaced (3.675%); 11 MiB/s, 177 objects/s recovering 2024-04-12T11:51:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 27147/155220 objects degraded (17.489%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:23.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:23.536 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:23.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:23.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:23.690 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:23.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:23.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:23 smithi087 ceph-mon[89014]: pgmap v2029: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 27147/155220 objects degraded (17.489%); 5705/155220 objects misplaced (3.675%); 11 MiB/s, 177 objects/s recovering 2024-04-12T11:51:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 27147/155220 objects degraded (17.489%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:23.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:23.825 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:23.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:23.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:23.927 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:23.955 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:23.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:24.056 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:24.089 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:24.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:24.191 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:24.216 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:24.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:24.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:24.318 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:24.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:24.344 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:24.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:24.445 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:24.472 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:24.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:24.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:24.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:24.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:24.612 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:24.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:24.713 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:24.738 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:24.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:24.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:24.839 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:24.867 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:24.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:24.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:24.969 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:25.003 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:25.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:25.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:25.106 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:25.144 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:25.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:25.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:25.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:51:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:51:24] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2024-04-12T11:51:25.246 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:25.273 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:25.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:25.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:25.375 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:25.411 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:25.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:25.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:25.456 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:25 smithi029 ceph-mon[93650]: pgmap v2030: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 26638/151722 objects degraded (17.557%); 5705/151722 objects misplaced (3.760%); 11 MiB/s, 180 objects/s recovering 2024-04-12T11:51:25.457 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:25 smithi029 ceph-mon[97787]: pgmap v2030: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 26638/151722 objects degraded (17.557%); 5705/151722 objects misplaced (3.760%); 11 MiB/s, 180 objects/s recovering 2024-04-12T11:51:25.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:25.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:25.562 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:25.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:25.594 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:25.600 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:25.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:25.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:25.711 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:25.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:25 smithi087 ceph-mon[89014]: pgmap v2030: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 26638/151722 objects degraded (17.557%); 5705/151722 objects misplaced (3.760%); 11 MiB/s, 180 objects/s recovering 2024-04-12T11:51:25.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:25.872 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:25.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:25.973 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:26.012 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:26.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:26.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:26.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:26.150 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:26.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:26.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:26.277 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:26.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:26.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:26.368 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:51:26.379 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:26.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:26.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:26.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:26.507 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:26.533 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:26.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:26.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:26.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:26.672 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:26.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:26.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:26 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2918831464' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:51:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:26 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:51:26.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:26 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2918831464' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:51:26.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:26 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:51:26.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:26 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2918831464' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:51:26.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:26 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:51:26.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:26.809 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:26.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:26.911 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:26.954 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:26.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:26.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:27.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:27.082 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:27.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:27.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:27.184 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:27.243 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:27.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:27.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:27.346 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:27.372 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:27.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:27.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:27.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:27.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:27.500 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:27.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:27.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:27 smithi029 ceph-mon[97787]: pgmap v2031: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 387 op/s; 26502/150474 objects degraded (17.612%); 5705/150474 objects misplaced (3.791%); 11 MiB/s, 169 objects/s recovering 2024-04-12T11:51:27.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:27 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:51:27.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:27 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:51:27.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:27 smithi029 ceph-mon[97787]: osdmap e1255: 8 total, 8 up, 8 in 2024-04-12T11:51:27.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:27 smithi029 ceph-mon[93650]: pgmap v2031: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 387 op/s; 26502/150474 objects degraded (17.612%); 5705/150474 objects misplaced (3.791%); 11 MiB/s, 169 objects/s recovering 2024-04-12T11:51:27.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:27 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:51:27.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:27 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:51:27.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:27 smithi029 ceph-mon[93650]: osdmap e1255: 8 total, 8 up, 8 in 2024-04-12T11:51:27.602 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:27.642 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:27.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:27.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:27 smithi087 ceph-mon[89014]: pgmap v2031: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 387 op/s; 26502/150474 objects degraded (17.612%); 5705/150474 objects misplaced (3.791%); 11 MiB/s, 169 objects/s recovering 2024-04-12T11:51:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:27 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:51:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:27 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:51:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:27 smithi087 ceph-mon[89014]: osdmap e1255: 8 total, 8 up, 8 in 2024-04-12T11:51:27.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:27.771 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:27.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:27.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:27.873 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:27.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:27.912 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:27.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:28.013 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:28.054 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:28.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:28.155 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:28.190 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:28.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:28.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:28.292 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:28.315 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:28.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:28.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:28.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:28.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:28.458 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:28.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:28.559 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:28.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:28.597 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:28.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:28.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 26502/150474 objects degraded (17.612%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:28.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 26502/150474 objects degraded (17.612%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:28.698 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:28.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 26502/150474 objects degraded (17.612%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:28.727 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:28.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:28.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:28.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:28.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:28.869 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:28.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:28.970 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:28.997 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:28.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:28.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:29.099 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:29.125 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:29.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:29.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:29.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:29.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:29.252 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:29.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:29.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:29.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:29.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:29.488 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:29.515 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:29.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:29.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:29.617 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:29.643 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:29.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:29 smithi087 ceph-mon[89014]: pgmap v2033: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 26116/146925 objects degraded (17.775%); 5410/146925 objects misplaced (3.682%); 11 MiB/s, 182 objects/s recovering 2024-04-12T11:51:29.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:29 smithi029 ceph-mon[93650]: pgmap v2033: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 26116/146925 objects degraded (17.775%); 5410/146925 objects misplaced (3.682%); 11 MiB/s, 182 objects/s recovering 2024-04-12T11:51:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:29 smithi029 ceph-mon[97787]: pgmap v2033: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 26116/146925 objects degraded (17.775%); 5410/146925 objects misplaced (3.682%); 11 MiB/s, 182 objects/s recovering 2024-04-12T11:51:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:29.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:29.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:29.871 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:29.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:29.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:29.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:29.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:30.034 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:30.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:30.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:30.136 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:30.172 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:30.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:30.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:30.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:30.309 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:30.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:30.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:30.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:30.449 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:30.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:30.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:30.576 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:30.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:30.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:30.678 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:30.704 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:30.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:30.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:30.807 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:30.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:30.838 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:30.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:30.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:30.985 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:30.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:30.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:31.087 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:31.135 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:31.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:31.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:31.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:31.275 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:31.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:31.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:31.377 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:31.410 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:31.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:31.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:31.512 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:31.542 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:31.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:31.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:31.644 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:31.671 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:31.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:31 smithi087 ceph-mon[89014]: pgmap v2034: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 25978/145263 objects degraded (17.883%); 5410/145263 objects misplaced (3.724%); 11 MiB/s, 181 objects/s recovering 2024-04-12T11:51:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:31 smithi029 ceph-mon[97787]: pgmap v2034: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 25978/145263 objects degraded (17.883%); 5410/145263 objects misplaced (3.724%); 11 MiB/s, 181 objects/s recovering 2024-04-12T11:51:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:31 smithi029 ceph-mon[93650]: pgmap v2034: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 25978/145263 objects degraded (17.883%); 5410/145263 objects misplaced (3.724%); 11 MiB/s, 181 objects/s recovering 2024-04-12T11:51:31.773 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:31.807 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:31.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:31.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:31.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:31.951 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:31.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:31.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:32.054 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:32.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:32.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:32.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:32.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:32.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:32.220 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:32.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:32.321 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:32.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:32.347 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:32.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:32.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:32.478 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:32.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:32.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:32.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:32 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4245093154' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:51:32.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:32 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4245093154' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:51:32.579 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:32.611 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:32.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:32.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:32.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:32.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:32 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4245093154' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:51:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:51:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:51:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:51:32.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:32.750 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:32.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:32.851 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:32.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:32.879 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:32.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:32.980 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:33.005 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:33.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:33.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:33.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:33.138 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:33.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:33.240 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:33.265 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:33.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:33.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:33.367 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:33.393 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:33.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:33.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:33.495 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:33.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:33 smithi029 ceph-mon[97787]: pgmap v2035: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 25826/142458 objects degraded (18.129%); 5169/142458 objects misplaced (3.628%); 12 MiB/s, 185 objects/s recovering 2024-04-12T11:51:33.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 25826/142458 objects degraded (18.129%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:33 smithi029 ceph-mon[93650]: pgmap v2035: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 25826/142458 objects degraded (18.129%); 5169/142458 objects misplaced (3.628%); 12 MiB/s, 185 objects/s recovering 2024-04-12T11:51:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 25826/142458 objects degraded (18.129%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:33.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:33.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:33.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:33.622 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:33.651 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:33.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:33.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:33 smithi087 ceph-mon[89014]: pgmap v2035: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 25826/142458 objects degraded (18.129%); 5169/142458 objects misplaced (3.628%); 12 MiB/s, 185 objects/s recovering 2024-04-12T11:51:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 25826/142458 objects degraded (18.129%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:33.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:33.799 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:33.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:33.900 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:33.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:33.942 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:33.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:34.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:34.084 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:34.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:34.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:34.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:34.229 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:34.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:34.330 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:34.356 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:34.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:34.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:34.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:34.486 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:34.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:34.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:34.588 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:34.616 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:34.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:34.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:34.718 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:34.743 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:34.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:34.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:34.846 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:34.873 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:34.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:34.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:34.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:35.010 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:35.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:35.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:35.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:35.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:35.155 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:35.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:35.256 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:51:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:51:34] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2024-04-12T11:51:35.283 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:35.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:35.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:35.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:35.424 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:35.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:35.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:35.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:35.561 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:35.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:35.662 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:35.689 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:35.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:35.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:35.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:35 smithi087 ceph-mon[89014]: pgmap v2036: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 25342/139182 objects degraded (18.208%); 5169/139182 objects misplaced (3.714%); 12 MiB/s, 184 objects/s recovering 2024-04-12T11:51:35.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:35 smithi029 ceph-mon[97787]: pgmap v2036: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 25342/139182 objects degraded (18.208%); 5169/139182 objects misplaced (3.714%); 12 MiB/s, 184 objects/s recovering 2024-04-12T11:51:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:35 smithi029 ceph-mon[93650]: pgmap v2036: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 25342/139182 objects degraded (18.208%); 5169/139182 objects misplaced (3.714%); 12 MiB/s, 184 objects/s recovering 2024-04-12T11:51:35.791 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:35.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:35.816 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:35.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:35.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:35.954 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:35.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:35.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:36.056 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:36.084 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:36.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:36.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:36.186 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:36.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:36.210 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:36.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:36.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:36.335 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:36.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:36.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:36.437 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:36.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:36.462 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:36.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:36.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:36.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:36.602 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:36.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:36.703 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:36.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:36.727 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:36.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:36.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:36.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:36.828 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:36.866 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:36.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:36.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:36.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:37.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:37.005 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:37.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:37.106 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:37.132 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:37.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:37.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:37.258 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:37.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:37.359 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:37.385 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:37.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:37.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:37.487 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:37.514 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:37.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:37.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:37.616 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:37.639 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:37.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:37.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:37.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:37 smithi029 ceph-mon[93650]: pgmap v2037: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 25212/138015 objects degraded (18.268%); 5169/138015 objects misplaced (3.745%); 11 MiB/s, 183 objects/s recovering 2024-04-12T11:51:37.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:37 smithi029 ceph-mon[97787]: pgmap v2037: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 25212/138015 objects degraded (18.268%); 5169/138015 objects misplaced (3.745%); 11 MiB/s, 183 objects/s recovering 2024-04-12T11:51:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:37 smithi087 ceph-mon[89014]: pgmap v2037: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 25212/138015 objects degraded (18.268%); 5169/138015 objects misplaced (3.745%); 11 MiB/s, 183 objects/s recovering 2024-04-12T11:51:37.741 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:37.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:37.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:37.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:37.881 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:37.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:37.911 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:37.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:38.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:38.035 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:38.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:38.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:38.137 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:38.172 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:38.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:38.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:38.275 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:38.302 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:38.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:38.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:38.404 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:38.430 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:38.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:38.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:38.532 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:38.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:38.558 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:38.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:38.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:38.696 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:38.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 25212/138015 objects degraded (18.268%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 25212/138015 objects degraded (18.268%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:38.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 25212/138015 objects degraded (18.268%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:38.798 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:38.825 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:38.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:38.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:38.927 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:38.967 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:38.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:38.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:39.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:39.100 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:39.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:39.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:39.225 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:39.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:39.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:39.327 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:39.351 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:39.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:39.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:39.452 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:39.477 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:39.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:39.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:39.579 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:39.610 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:39.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:39.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:39.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:39 smithi087 ceph-mon[89014]: pgmap v2038: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 24801/134277 objects degraded (18.470%); 4874/134277 objects misplaced (3.630%); 12 MiB/s, 216 objects/s recovering 2024-04-12T11:51:39.747 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:39.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:39.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:39 smithi029 ceph-mon[93650]: pgmap v2038: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 24801/134277 objects degraded (18.470%); 4874/134277 objects misplaced (3.630%); 12 MiB/s, 216 objects/s recovering 2024-04-12T11:51:39.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:39 smithi029 ceph-mon[97787]: pgmap v2038: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 24801/134277 objects degraded (18.470%); 4874/134277 objects misplaced (3.630%); 12 MiB/s, 216 objects/s recovering 2024-04-12T11:51:39.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:39.872 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:39.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:39.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:39.974 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:39.999 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:40.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:40.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:40.101 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:40.134 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:40.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:40.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:40.236 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:40.264 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:40.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:40.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:40.366 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:40.390 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:40.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:40.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:40.492 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:40.515 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:40.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:40.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:40.651 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:40.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:40.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:40.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:40.795 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:40.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:40.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:40.897 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:40.925 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:40.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:40.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:41.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:41.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:41.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:41.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:41.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:41.176 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:41.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:41.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:41.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:41.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:41.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:41.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:41.411 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:41.443 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:41.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:41.545 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:41.576 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:41.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:41.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:41.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:41.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:41 smithi029 ceph-mon[93650]: pgmap v2039: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 24666/132699 objects degraded (18.588%); 4874/132699 objects misplaced (3.673%); 9.1 MiB/s, 165 objects/s recovering 2024-04-12T11:51:41.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:41 smithi029 ceph-mon[97787]: pgmap v2039: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 24666/132699 objects degraded (18.588%); 4874/132699 objects misplaced (3.673%); 9.1 MiB/s, 165 objects/s recovering 2024-04-12T11:51:41.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:41.707 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:41.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:41 smithi087 ceph-mon[89014]: pgmap v2039: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 24666/132699 objects degraded (18.588%); 4874/132699 objects misplaced (3.673%); 9.1 MiB/s, 165 objects/s recovering 2024-04-12T11:51:41.808 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:41.845 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:41.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:41.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:41.947 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:41.985 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:41.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:42.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:42.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:42.119 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:42.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:42.220 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:42.246 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:42.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:42.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:42.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:42.385 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:42.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:42.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:42.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:42.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:42.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:42.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:42.646 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:42.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:51:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:51:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:51:42.748 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:42.783 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:42.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:42.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:42.885 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:42.917 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:42.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:42.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:43.020 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:43.052 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:43.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:43.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:43.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:43.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:43.190 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:43.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:43.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:43.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:43.326 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:43.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:43.427 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:43.454 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:43.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:43.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:43.556 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:43.584 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:43.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:43.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:43.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:43.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:43 smithi087 ceph-mon[89014]: pgmap v2040: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 424 op/s; 24513/129966 objects degraded (18.861%); 4646/129966 objects misplaced (3.575%); 9.2 MiB/s, 185 objects/s recovering 2024-04-12T11:51:43.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 24513/129966 objects degraded (18.861%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:43.721 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:43.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:43.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:43 smithi029 ceph-mon[93650]: pgmap v2040: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 424 op/s; 24513/129966 objects degraded (18.861%); 4646/129966 objects misplaced (3.575%); 9.2 MiB/s, 185 objects/s recovering 2024-04-12T11:51:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 24513/129966 objects degraded (18.861%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:43 smithi029 ceph-mon[97787]: pgmap v2040: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 424 op/s; 24513/129966 objects degraded (18.861%); 4646/129966 objects misplaced (3.575%); 9.2 MiB/s, 185 objects/s recovering 2024-04-12T11:51:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 24513/129966 objects degraded (18.861%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:43.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:43.848 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:43.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:43.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:43.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:43.975 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:43.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:43.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:44.077 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:44.117 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:44.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:44.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:44.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:44.248 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:44.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:44.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:44.390 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:44.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:44.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:44.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:44.519 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:44.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:44.620 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:44.649 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:44.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:44.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:44.751 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:44.789 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:44.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:44.891 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:44.931 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:44.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:45.032 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:45.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:45.056 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:45.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:45.157 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:45.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:45.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:45.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:51:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:51:44] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2024-04-12T11:51:45.286 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:45.312 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:45.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:45.414 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:45.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:45.444 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:45.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:45.545 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:45.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:45.585 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:45.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:45.686 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:45.711 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:45.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:45.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:45 smithi087 ceph-mon[89014]: pgmap v2041: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 24013/126465 objects degraded (18.988%); 4646/126465 objects misplaced (3.674%); 9.7 MiB/s, 194 objects/s recovering 2024-04-12T11:51:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:45 smithi029 ceph-mon[97787]: pgmap v2041: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 24013/126465 objects degraded (18.988%); 4646/126465 objects misplaced (3.674%); 9.7 MiB/s, 194 objects/s recovering 2024-04-12T11:51:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:45 smithi029 ceph-mon[93650]: pgmap v2041: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 24013/126465 objects degraded (18.988%); 4646/126465 objects misplaced (3.674%); 9.7 MiB/s, 194 objects/s recovering 2024-04-12T11:51:45.813 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:45.846 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:45.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:45.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:45.948 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:45.972 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:45.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:45.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:46.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:46.100 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:46.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:46.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:46.235 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:46.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:46.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:46.337 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:46.362 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:46.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:46.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:46.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:46.503 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:46.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:46.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:46.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:46.640 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:46.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:46.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:46.742 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:46.769 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:46.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:46.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:46.870 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:46.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:46.906 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:46.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:47.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:47.048 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:47.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:47.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:47.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:47.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:47.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:47.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:47.331 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:47.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:47.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:47.432 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:47.468 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:47.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:47.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:47.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:47.606 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:47.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:47.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:47.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:47 smithi087 ceph-mon[89014]: pgmap v2042: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 23903/125442 objects degraded (19.055%); 4646/125442 objects misplaced (3.704%); 7.8 MiB/s, 162 objects/s recovering 2024-04-12T11:51:47.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:47.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:47.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:47.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:47 smithi029 ceph-mon[93650]: pgmap v2042: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 23903/125442 objects degraded (19.055%); 4646/125442 objects misplaced (3.704%); 7.8 MiB/s, 162 objects/s recovering 2024-04-12T11:51:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:47 smithi029 ceph-mon[97787]: pgmap v2042: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 381 op/s; 23903/125442 objects degraded (19.055%); 4646/125442 objects misplaced (3.704%); 7.8 MiB/s, 162 objects/s recovering 2024-04-12T11:51:47.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:47.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:47.881 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:47.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:47.907 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:51:47.907 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:51:47.907 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:51:47.907 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:51:47.907 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:51:47.907 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:51:47.907 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:51:47.907 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:51:47.908 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:51:47.908 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:51:47.908 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:51:47.908 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:51:47.908 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:51:47.908 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:51:47.908 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:51:47.908 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:51:47.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:48.019 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:48.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:48.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:48.149 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:48.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:48.251 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:48.283 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:48.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:48.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:48.344 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:51:48.385 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:48.410 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:48.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:48.512 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:48.537 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:48.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:48.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:48.682 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:48.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:48.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:48.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 23903/125442 objects degraded (19.055%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:48.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 23903/125442 objects degraded (19.055%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 23903/125442 objects degraded (19.055%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:48.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:48.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:48.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:48.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:48.943 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:48.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:48.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:48.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:49.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:49.096 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:49.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:49.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:49.198 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:49.229 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:49.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:49.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:49.331 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:49.355 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:49.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:49.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:49.457 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:49.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:49.495 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:49.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:49.596 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:49.624 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:49.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:49 smithi087 ceph-mon[89014]: pgmap v2043: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 23474/121686 objects degraded (19.291%); 4247/121686 objects misplaced (3.490%); 9.3 MiB/s, 221 objects/s recovering 2024-04-12T11:51:49.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:49.762 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:49.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:49.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:49.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:49 smithi029 ceph-mon[97787]: pgmap v2043: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 23474/121686 objects degraded (19.291%); 4247/121686 objects misplaced (3.490%); 9.3 MiB/s, 221 objects/s recovering 2024-04-12T11:51:49.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:49 smithi029 ceph-mon[93650]: pgmap v2043: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 23474/121686 objects degraded (19.291%); 4247/121686 objects misplaced (3.490%); 9.3 MiB/s, 221 objects/s recovering 2024-04-12T11:51:49.863 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:49.890 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:49.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:49.992 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:50.022 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:50.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:50.124 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:50.153 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:50.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:50.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:50.255 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:50.282 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:50.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:50.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:50.384 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:50.412 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:50.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:50.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:50.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:50.538 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:50.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:50.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:50.639 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:50.672 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:50.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:50.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:50.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:51:50.773 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:50.798 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:50.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:50.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:50.899 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:50.924 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:50.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:50.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:51.026 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:51.069 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:51.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:51.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:51.175 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:51.209 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:51.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:51.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:51.311 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:51.338 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:51.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:51.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:51.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:51.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:51.483 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:51.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:51.519 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:51.584 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:51.615 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:51.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:51.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:51.716 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:51 smithi087 ceph-mon[89014]: pgmap v2044: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 399 op/s; 23326/119883 objects degraded (19.457%); 4247/119883 objects misplaced (3.543%); 7.7 MiB/s, 175 objects/s recovering 2024-04-12T11:51:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:51.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:51 smithi029 ceph-mon[97787]: pgmap v2044: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 399 op/s; 23326/119883 objects degraded (19.457%); 4247/119883 objects misplaced (3.543%); 7.7 MiB/s, 175 objects/s recovering 2024-04-12T11:51:51.735 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:51.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:51 smithi029 ceph-mon[93650]: pgmap v2044: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 399 op/s; 23326/119883 objects degraded (19.457%); 4247/119883 objects misplaced (3.543%); 7.7 MiB/s, 175 objects/s recovering 2024-04-12T11:51:51.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:51:51.753 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:51.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:51.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:51.855 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:51.887 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:51.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:51.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:51.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:52.026 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:52.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:52.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:52.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:52.155 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:52.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:52.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:52.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:52.300 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:52.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:52.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:52.449 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:52.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:52.550 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:52.573 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:52.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:52.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:52.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:51:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:51:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:51:52.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:52.711 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:52.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:52.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:52.814 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:52.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:52.853 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:52.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:52.954 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:52.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:52.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:52.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:53.084 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:53.107 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:53.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:53.209 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:53.237 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:53.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:53.339 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:53.362 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:53.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:53.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:53.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:53.504 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:53.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:53.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:53.655 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:53.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:53.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:53 smithi087 ceph-mon[89014]: pgmap v2045: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 23162/116436 objects degraded (19.892%); 3820/116436 objects misplaced (3.281%); 7.8 MiB/s, 213 objects/s recovering 2024-04-12T11:51:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 23162/116436 objects degraded (19.892%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:53.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:53 smithi029 ceph-mon[97787]: pgmap v2045: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 23162/116436 objects degraded (19.892%); 3820/116436 objects misplaced (3.281%); 7.8 MiB/s, 213 objects/s recovering 2024-04-12T11:51:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 23162/116436 objects degraded (19.892%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:53 smithi029 ceph-mon[93650]: pgmap v2045: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 23162/116436 objects degraded (19.892%); 3820/116436 objects misplaced (3.281%); 7.8 MiB/s, 213 objects/s recovering 2024-04-12T11:51:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 23162/116436 objects degraded (19.892%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:53.798 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:53.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:53.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:53.900 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:53.926 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:53.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:53.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:54.032 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:54.068 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:54.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:54.170 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:54.196 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:54.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:54.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:54.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:54.322 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:54.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:54.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:54.424 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:54.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:54.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:54.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:54.477 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:51:54.477 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:51:54.564 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:54.594 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:54.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:54.696 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:54.722 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:54.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:54.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:54.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:54.866 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:54.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:54.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:54.969 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:54.998 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:54.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:55.099 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:55.124 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:55.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:55.167 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:51:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:51:54] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2024-04-12T11:51:55.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:55.270 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:55.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:55.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:55.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:55.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:55.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:55.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:55.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:55.548 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:55.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:55.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:55.649 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:55.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:55.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:55.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:55 smithi087 ceph-mon[89014]: pgmap v2046: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 22492/111999 objects degraded (20.082%); 3820/111999 objects misplaced (3.411%); 11 MiB/s, 237 objects/s recovering 2024-04-12T11:51:55.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:55 smithi029 ceph-mon[97787]: pgmap v2046: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 22492/111999 objects degraded (20.082%); 3820/111999 objects misplaced (3.411%); 11 MiB/s, 237 objects/s recovering 2024-04-12T11:51:55.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:55 smithi029 ceph-mon[93650]: pgmap v2046: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 22492/111999 objects degraded (20.082%); 3820/111999 objects misplaced (3.411%); 11 MiB/s, 237 objects/s recovering 2024-04-12T11:51:55.776 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 327 (0%) 2024-04-12T11:51:55.776 INFO:teuthology.orchestra.run.smithi029.stdout:avg 40.875 2024-04-12T11:51:55.776 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 8.34322 -> 8.34322 (expected baseline 5.98044) 2024-04-12T11:51:55.776 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 24 -> 24 pgs (0.587156 -> 0.587156 * mean) 2024-04-12T11:51:55.776 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.7 with 54 -> 54 pgs (1.3211 -> 1.3211 * mean) 2024-04-12T11:51:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:51:55.777 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2024-04-12T11:51:55.777 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 1 2024-04-12T11:51:55.777 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2024-04-12T11:51:55.777 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0189 2024-04-12T11:51:55.777 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.1892 2024-04-12T11:51:55.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:55.786 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2024-04-12T11:51:55.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:55.818 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:55.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:55.919 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:55.945 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:55.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:56.047 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:56.083 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:56.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:56.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:56.185 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:56.213 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:56.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:56.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:56.315 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:56.348 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:56.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:56.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:56.408 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-04-12T11:51:56.450 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:56.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:56.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:56.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:56.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:56.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:56.618 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:56.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:56.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:56.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:56.761 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:56.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:56.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:56.897 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:56.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:56.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:56.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:57.061 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:57.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:57.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:57.163 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:57.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:57.191 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:57.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:57.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:57.322 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:57.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:57.425 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:57.451 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:57.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:57.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:57.553 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:57.558 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:57 smithi029 ceph-mon[93650]: from='client.35625 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:51:57.558 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:57 smithi029 ceph-mon[93650]: pgmap v2047: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 22324/110616 objects degraded (20.182%); 3820/110616 objects misplaced (3.453%); 8.8 MiB/s, 209 objects/s recovering 2024-04-12T11:51:57.558 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:57 smithi029 ceph-mon[97787]: from='client.35625 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:51:57.559 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:57 smithi029 ceph-mon[97787]: pgmap v2047: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 22324/110616 objects degraded (20.182%); 3820/110616 objects misplaced (3.453%); 8.8 MiB/s, 209 objects/s recovering 2024-04-12T11:51:57.559 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-04-12T11:51:57.559 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight-by-utilization 1000 0.05 2024-04-12T11:51:57.598 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:57.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:57.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:57 smithi087 ceph-mon[89014]: from='client.35625 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:51:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:57 smithi087 ceph-mon[89014]: pgmap v2047: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 22324/110616 objects degraded (20.182%); 3820/110616 objects misplaced (3.453%); 8.8 MiB/s, 209 objects/s recovering 2024-04-12T11:51:57.727 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:57.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:57.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:57.828 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:51:57.856 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:57.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:57.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:51:57.958 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:57.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:57.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:57.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:51:58.089 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:51:58.128 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:58.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:58.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:51:58.230 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:58.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:58.267 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:58.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:58.368 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:58.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:58.409 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:58.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:58.510 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:58.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:58.550 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:58.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:58.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:51:58.693 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:58.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:58.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:51:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 22324/110616 objects degraded (20.182%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:58.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 22324/110616 objects degraded (20.182%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:58.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 22324/110616 objects degraded (20.182%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:51:58.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:58.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:58.840 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:58.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:58.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:51:58.996 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:58.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:58.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:51:59.080 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:59.081 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:59.098 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:59.123 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:59.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:59.170 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:51:59.225 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:59.251 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:59.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:59.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:59.353 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:59.381 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:59.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:59.483 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:51:59.512 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:59.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:51:59.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:51:59.646 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:51:59.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:59.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:51:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:51:59 smithi087 ceph-mon[89014]: pgmap v2048: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 21848/106062 objects degraded (20.599%); 3354/106062 objects misplaced (3.162%); 11 MiB/s, 279 objects/s recovering 2024-04-12T11:51:59.748 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:51:59.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:51:59 smithi029 ceph-mon[97787]: pgmap v2048: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 21848/106062 objects degraded (20.599%); 3354/106062 objects misplaced (3.162%); 11 MiB/s, 279 objects/s recovering 2024-04-12T11:51:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:51:59 smithi029 ceph-mon[93650]: pgmap v2048: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 21848/106062 objects degraded (20.599%); 3354/106062 objects misplaced (3.162%); 11 MiB/s, 279 objects/s recovering 2024-04-12T11:51:59.778 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:59.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:51:59.880 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:51:59.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:51:59.922 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:51:59.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:00.023 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:00.052 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:00.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:00.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:00.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:00.189 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:00.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:00.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:00.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:00.331 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:00.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:00.432 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:00.476 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:00.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:00.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:00.489 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:52:00.579 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:00.620 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:00.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:00.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:00 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/112186077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:52:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:00 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:52:00.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:00.757 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:00.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:00.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:00.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:00 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/112186077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:52:00.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:00 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:52:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:00 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/112186077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:52:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:00 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:52:00.858 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:00.884 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:00.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:00.986 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:01.024 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:01.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:01.126 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:01.160 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:01.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:01.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:01.262 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:01.300 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:01.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:01.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:01.401 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:01.434 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:01.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:01.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:01.536 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:01.562 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:01.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:01.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:01.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:01.692 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:01.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:01.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:01.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:01 smithi029 ceph-mon[97787]: pgmap v2049: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 21684/104154 objects degraded (20.819%); 3354/104154 objects misplaced (3.220%); 9.3 MiB/s, 223 objects/s recovering 2024-04-12T11:52:01.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:01 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:52:01.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:01 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:52:01.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:01 smithi029 ceph-mon[97787]: osdmap e1256: 8 total, 8 up, 8 in 2024-04-12T11:52:01.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:01 smithi029 ceph-mon[93650]: pgmap v2049: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 21684/104154 objects degraded (20.819%); 3354/104154 objects misplaced (3.220%); 9.3 MiB/s, 223 objects/s recovering 2024-04-12T11:52:01.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:01 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:52:01.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:01 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:52:01.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:01 smithi029 ceph-mon[93650]: osdmap e1256: 8 total, 8 up, 8 in 2024-04-12T11:52:01.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:01.823 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:01.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:01.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:01.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:01.951 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:01.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:01.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:01 smithi087 ceph-mon[89014]: pgmap v2049: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 21684/104154 objects degraded (20.819%); 3354/104154 objects misplaced (3.220%); 9.3 MiB/s, 223 objects/s recovering 2024-04-12T11:52:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:01 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:52:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:01 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:52:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:01 smithi087 ceph-mon[89014]: osdmap e1256: 8 total, 8 up, 8 in 2024-04-12T11:52:02.054 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:02.077 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:02.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:02.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:02.179 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:02.202 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:02.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:02.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:02.306 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:02.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:02.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:02.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:02.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:02.455 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:02.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:02.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:02.478 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:02 smithi029 ceph-mon[93650]: osdmap e1257: 8 total, 8 up, 8 in 2024-04-12T11:52:02.557 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:02.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:02.591 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:02.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:02.692 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:02.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:02 smithi087 ceph-mon[89014]: osdmap e1257: 8 total, 8 up, 8 in 2024-04-12T11:52:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:52:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:52:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:52:02.730 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:02.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:02.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:02.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:02 smithi029 ceph-mon[97787]: osdmap e1257: 8 total, 8 up, 8 in 2024-04-12T11:52:02.832 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:02.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:02.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:02.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:02.962 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:03.003 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:03.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:03.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:03.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:03.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:03.139 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:03.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:03.240 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:03.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:03.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:03.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:03.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:03.413 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:03.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:03.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:03.514 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:03.545 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:03.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:03.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:03.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:03.687 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:03.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:03.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:03 smithi029 ceph-mon[93650]: pgmap v2052: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 396 op/s; 21497/102477 objects degraded (20.977%); 3354/102477 objects misplaced (3.273%); 7.8 MiB/s, 182 objects/s recovering 2024-04-12T11:52:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:03 smithi029 ceph-mon[93650]: osdmap e1258: 8 total, 8 up, 8 in 2024-04-12T11:52:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 21497/102477 objects degraded (20.977%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:03 smithi029 ceph-mon[97787]: pgmap v2052: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 396 op/s; 21497/102477 objects degraded (20.977%); 3354/102477 objects misplaced (3.273%); 7.8 MiB/s, 182 objects/s recovering 2024-04-12T11:52:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:03 smithi029 ceph-mon[97787]: osdmap e1258: 8 total, 8 up, 8 in 2024-04-12T11:52:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 21497/102477 objects degraded (20.977%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:03.789 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:03.820 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:03.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:03.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:03.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:03.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:03.955 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:03.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:03 smithi087 ceph-mon[89014]: pgmap v2052: 109 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 94 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 396 op/s; 21497/102477 objects degraded (20.977%); 3354/102477 objects misplaced (3.273%); 7.8 MiB/s, 182 objects/s recovering 2024-04-12T11:52:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:03 smithi087 ceph-mon[89014]: osdmap e1258: 8 total, 8 up, 8 in 2024-04-12T11:52:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 21497/102477 objects degraded (20.977%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:04.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:04.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:04.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:04.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:04.198 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:04.232 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:04.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:04.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:04.333 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:04.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:04.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:04.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:04.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:04.508 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:04.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:04.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:04.610 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:04.639 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:04.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:04.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:04.701 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:52:04.741 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:04.754 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:52:04.759 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:52:04.770 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:04.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:04.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:04.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:04.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:04.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:05.004 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:05.030 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:05.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:05.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:05.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:05.166 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:05.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:05.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:05.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:52:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:52:04] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2024-04-12T11:52:05.273 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:05.314 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:05.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:05.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:05.416 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:05.449 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:05.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:05.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:05.550 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:05.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:05.589 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:05.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:05.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:05.731 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:05.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:05.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:05.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:05 smithi029 ceph-mon[97787]: pgmap v2054: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 19939/96234 objects degraded (20.719%); 3857/96234 objects misplaced (4.008%); 11 MiB/s, 233 objects/s recovering 2024-04-12T11:52:05.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:05 smithi029 ceph-mon[93650]: pgmap v2054: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 19939/96234 objects degraded (20.719%); 3857/96234 objects misplaced (4.008%); 11 MiB/s, 233 objects/s recovering 2024-04-12T11:52:05.834 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:05.876 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:05.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:05.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:05.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:05 smithi087 ceph-mon[89014]: pgmap v2054: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 19939/96234 objects degraded (20.719%); 3857/96234 objects misplaced (4.008%); 11 MiB/s, 233 objects/s recovering 2024-04-12T11:52:05.979 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:06.011 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:06.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:06.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:06.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:06.146 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:06.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:06.248 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:06.276 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:06.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:06.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:06.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:06.418 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:06.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:06.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:06.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:06.558 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:06.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:06.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:06.660 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:06.695 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:06.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:06.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:06.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:06.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 327 (0%) 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout:avg 40.875 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 8.34322 -> 8.34322 (expected baseline 5.98044) 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 24 -> 24 pgs (0.587156 -> 0.587156 * mean) 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.7 with 54 -> 54 pgs (1.3211 -> 1.3211 * mean) 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 0.05 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0175 2024-04-12T11:52:06.768 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.1752 2024-04-12T11:52:06.770 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-04-12T11:52:06.775 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2024-04-12T11:52:06.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:06.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:06.866 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:06.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:06.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:06.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:07.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:07.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:07.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:07.155 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:07.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:07.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:07.257 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:07.286 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:07.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:07.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:07.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:07.429 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:07.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:07.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:07.531 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:07.575 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:07.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:07.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:07.719 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:07.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:07.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:07.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:07 smithi029 ceph-mon[97787]: pgmap v2055: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 19770/94710 objects degraded (20.874%); 3857/94710 objects misplaced (4.072%); 11 MiB/s, 234 objects/s recovering 2024-04-12T11:52:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:07 smithi029 ceph-mon[93650]: pgmap v2055: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 19770/94710 objects degraded (20.874%); 3857/94710 objects misplaced (4.072%); 11 MiB/s, 234 objects/s recovering 2024-04-12T11:52:07.821 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:07.860 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:07.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:07.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:07.961 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:07.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:07 smithi087 ceph-mon[89014]: pgmap v2055: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 19770/94710 objects degraded (20.874%); 3857/94710 objects misplaced (4.072%); 11 MiB/s, 234 objects/s recovering 2024-04-12T11:52:07.997 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:07.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:08.099 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:08.127 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:08.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:08.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:08.203 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:08.228 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:08.242 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:08.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:08.254 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:08.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:08.355 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:08.386 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:08.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:08.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:08.488 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:08.514 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:08.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:08.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:08.616 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:08.648 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:08.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:08.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:08.750 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:08.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:08 smithi029 ceph-mon[97787]: from='client.35634 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:52:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 19770/94710 objects degraded (20.874%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:08 smithi029 ceph-mon[93650]: from='client.35634 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:52:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 19770/94710 objects degraded (20.874%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:08.776 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:08.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:08.878 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:08.906 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:08.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:08.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:08 smithi087 ceph-mon[89014]: from='client.35634 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T11:52:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 19770/94710 objects degraded (20.874%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:09.008 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:09.040 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:09.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:09.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:09.142 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:09.179 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:09.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:09.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:09.281 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:09.326 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:09.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:09.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:09.466 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:09.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:09.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:09.568 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:09.598 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:09.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:09.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:09.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:09.728 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:09 smithi029 ceph-mon[97787]: pgmap v2056: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 19135/89076 objects degraded (21.482%); 3857/89076 objects misplaced (4.330%); 13 MiB/s, 244 objects/s recovering 2024-04-12T11:52:09.728 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:09 smithi029 ceph-mon[93650]: pgmap v2056: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 19135/89076 objects degraded (21.482%); 3857/89076 objects misplaced (4.330%); 13 MiB/s, 244 objects/s recovering 2024-04-12T11:52:09.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:09.738 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:09.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:09.839 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:09.856 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:52:09.865 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:09.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:09.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:09.890 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:52:09.928 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:52:09.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:09 smithi087 ceph-mon[89014]: pgmap v2056: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 19135/89076 objects degraded (21.482%); 3857/89076 objects misplaced (4.330%); 13 MiB/s, 244 objects/s recovering 2024-04-12T11:52:09.975 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:52:10.006 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:10.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:10.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:10.013 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:52:10.048 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:52:10.082 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:52:10.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:10.121 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:52:10.127 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:52:10.127 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:52:10.127 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:52:10.127 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:52:10.127 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:52:10.127 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:52:10.127 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:52:10.127 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:52:10.147 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:10.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:10.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:10.249 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:10.288 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:10.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:10.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:10.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:10.434 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:10.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:10.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:10.535 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:10.568 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:10.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:10.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:10.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:10.712 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:10.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:10.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:10.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/456542642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:10.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/456542642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:10.813 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:10.852 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:10.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:10.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:10.954 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/456542642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:10.991 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:10.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:10.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:11.092 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:11.121 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:11.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:11.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:11.223 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:11.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:11.288 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:11.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:11.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:11.428 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:11.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:11.529 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:11.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:11.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:11.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:11.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:11.701 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:11.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:11.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:11.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:11 smithi029 ceph-mon[97787]: pgmap v2057: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 18938/86646 objects degraded (21.857%); 3715/86646 objects misplaced (4.288%); 14 MiB/s, 255 objects/s recovering 2024-04-12T11:52:11.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:11 smithi029 ceph-mon[93650]: pgmap v2057: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 18938/86646 objects degraded (21.857%); 3715/86646 objects misplaced (4.288%); 14 MiB/s, 255 objects/s recovering 2024-04-12T11:52:11.803 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:11.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:11.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:11.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:11.943 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:11 smithi087 ceph-mon[89014]: pgmap v2057: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 18938/86646 objects degraded (21.857%); 3715/86646 objects misplaced (4.288%); 14 MiB/s, 255 objects/s recovering 2024-04-12T11:52:11.970 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:11.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:12.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:12.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:12.111 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:12.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:12.212 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:12.255 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:12.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:12.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:12.357 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:12.381 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:12.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:12.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:12.483 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:12.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:12.518 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:12.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:12.619 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:12.651 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:12.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:12.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:12.664 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:52:12.664 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:52:12.664 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-04-12T11:52:12.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T11:52:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:52:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:52:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:52:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:12 smithi087 ceph-mon[89014]: pgmap v2058: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 18743/84810 objects degraded (22.100%); 3715/84810 objects misplaced (4.380%); 13 MiB/s, 238 objects/s recovering 2024-04-12T11:52:12.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:12.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:12.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:12.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:12.908 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:12.931 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:12.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:12.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:13.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:12 smithi029 ceph-mon[93650]: pgmap v2058: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 18743/84810 objects degraded (22.100%); 3715/84810 objects misplaced (4.380%); 13 MiB/s, 238 objects/s recovering 2024-04-12T11:52:13.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:12 smithi029 ceph-mon[97787]: pgmap v2058: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 18743/84810 objects degraded (22.100%); 3715/84810 objects misplaced (4.380%); 13 MiB/s, 238 objects/s recovering 2024-04-12T11:52:13.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:13.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:13.068 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:13.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:13.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:13.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:13.213 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:13.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:13.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:52:13.314 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:13.374 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:13.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:13.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:13.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:13.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:13.520 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:13.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:13.621 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:13.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:13.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:13.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:13.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:13.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 18743/84810 objects degraded (22.100%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:13.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 18743/84810 objects degraded (22.100%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:13.790 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:13.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:13.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:13.892 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:13.915 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:13.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:13.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 18743/84810 objects degraded (22.100%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:14.017 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:14.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:14.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:14.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:14.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:14.192 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:14.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:14.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:14.294 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:14.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:14.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:14.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:14.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:14.447 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:14.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:14.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:14.549 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:14.579 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:14.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:14.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:14.681 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:14.710 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:14.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:14.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:14.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:14.847 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:14.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:14.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:14.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:14 smithi029 ceph-mon[97787]: pgmap v2059: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 17988/76368 objects degraded (23.554%); 3498/76368 objects misplaced (4.580%); 16 MiB/s, 276 objects/s recovering 2024-04-12T11:52:14.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:14 smithi029 ceph-mon[93650]: pgmap v2059: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 17988/76368 objects degraded (23.554%); 3498/76368 objects misplaced (4.580%); 16 MiB/s, 276 objects/s recovering 2024-04-12T11:52:14.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:14.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:14 smithi087 ceph-mon[89014]: pgmap v2059: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 17988/76368 objects degraded (23.554%); 3498/76368 objects misplaced (4.580%); 16 MiB/s, 276 objects/s recovering 2024-04-12T11:52:14.983 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:14.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:15.084 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:15.118 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:15.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:15.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:15.220 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:15.250 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:15.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:15.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:15.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:52:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:52:14] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2024-04-12T11:52:15.352 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:15.376 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:15.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:15.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:15.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:15.502 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:15.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:15.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:15.604 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:15.638 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:15.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:15.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:15.740 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:15.772 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:15.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:15.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:15.874 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:15.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:15.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:15.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:15.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:16.026 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:16.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:16.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:16.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:16.163 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:16.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:16.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:16.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:16.298 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:16.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:16.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:16.400 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:16.430 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:16.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:16.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:16.556 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:16.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:16.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:16.657 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:16.681 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:16.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:16.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:16.783 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:16.823 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:16.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:16.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:16.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:16.956 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:16.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:16.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:17.058 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:17.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:17.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:17.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:17.196 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:17.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:17.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:17.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:17.328 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:17.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:17.361 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:17.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:17.381 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:17 smithi029 ceph-mon[93650]: pgmap v2060: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 17775/74457 objects degraded (23.873%); 3498/74457 objects misplaced (4.698%); 13 MiB/s, 210 objects/s recovering 2024-04-12T11:52:17.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:17 smithi029 ceph-mon[97787]: pgmap v2060: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 17775/74457 objects degraded (23.873%); 3498/74457 objects misplaced (4.698%); 13 MiB/s, 210 objects/s recovering 2024-04-12T11:52:17.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:17 smithi087 ceph-mon[89014]: pgmap v2060: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 17775/74457 objects degraded (23.873%); 3498/74457 objects misplaced (4.698%); 13 MiB/s, 210 objects/s recovering 2024-04-12T11:52:17.514 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:17.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:17.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:17.616 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:17.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:17.656 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:17.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:17.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:17.809 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:17.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:17.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:17.911 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:17.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:17.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:17.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:18.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:18.092 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:18.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:18.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:18.193 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:18.236 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:18.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:18.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:18.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 17775/74457 objects degraded (23.873%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:18.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 17775/74457 objects degraded (23.873%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:18.341 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:18.390 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:18.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:18.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:18.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 17775/74457 objects degraded (23.873%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:18.478 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:18.482 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:18.492 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:18.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:18.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:18.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:18.567 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:52:18.567 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:52:18.567 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:52:18.568 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:52:18.568 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:52:18.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:18.648 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:18.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:18.750 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:18.784 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:18.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:18.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:18.924 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:18.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:18.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:19.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:19.055 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:19.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:19.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:19.145 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:52:19.157 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:19.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:19.205 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:19.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:19.226 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:19 smithi029 ceph-mon[97787]: pgmap v2061: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 17500/70479 objects degraded (24.830%); 3498/70479 objects misplaced (4.963%); 14 MiB/s, 219 objects/s recovering 2024-04-12T11:52:19.227 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:19 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3080636143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:52:19.227 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:19 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:52:19.227 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:52:19.228 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:19 smithi029 ceph-mon[93650]: pgmap v2061: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 17500/70479 objects degraded (24.830%); 3498/70479 objects misplaced (4.963%); 14 MiB/s, 219 objects/s recovering 2024-04-12T11:52:19.228 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:19 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3080636143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:52:19.228 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:19 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:52:19.228 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:52:19.306 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:19.335 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:19.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:19.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:19.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:19 smithi087 ceph-mon[89014]: pgmap v2061: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 17500/70479 objects degraded (24.830%); 3498/70479 objects misplaced (4.963%); 14 MiB/s, 219 objects/s recovering 2024-04-12T11:52:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:19 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3080636143' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:52:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:19 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:52:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:52:19.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:19.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:19.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:19.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:19.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:19.625 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:19.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:19.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:19.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:19.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:19.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:19.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:19.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:19.918 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:19.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:20.019 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:20.047 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:20.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:20.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:20.182 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:20.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:20.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:20.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:20.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:20.327 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:20.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:20.428 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:20.466 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:20.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:20 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:52:20.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:20 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:52:20.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:20 smithi087 ceph-mon[89014]: osdmap e1259: 8 total, 8 up, 8 in 2024-04-12T11:52:20.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:20.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:20.501 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:20 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:52:20.502 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:20 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:52:20.502 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:20 smithi029 ceph-mon[97787]: osdmap e1259: 8 total, 8 up, 8 in 2024-04-12T11:52:20.502 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:20 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:52:20.502 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:20 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:52:20.502 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:20 smithi029 ceph-mon[93650]: osdmap e1259: 8 total, 8 up, 8 in 2024-04-12T11:52:20.569 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:20.596 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:20.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:20.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:20.697 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:20.744 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:20.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:20.846 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:20.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:20.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:20.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:20.982 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:21.011 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:21.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:21.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:21.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:21.149 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:21.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:21.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:21.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:21.286 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:21.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:21.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:21.388 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:21.429 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:21.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:21.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:21 smithi087 ceph-mon[89014]: pgmap v2063: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 17004/66381 objects degraded (25.616%); 3314/66381 objects misplaced (4.992%); 15 MiB/s, 234 objects/s recovering 2024-04-12T11:52:21.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:21 smithi029 ceph-mon[93650]: pgmap v2063: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 17004/66381 objects degraded (25.616%); 3314/66381 objects misplaced (4.992%); 15 MiB/s, 234 objects/s recovering 2024-04-12T11:52:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:21 smithi029 ceph-mon[97787]: pgmap v2063: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 17004/66381 objects degraded (25.616%); 3314/66381 objects misplaced (4.992%); 15 MiB/s, 234 objects/s recovering 2024-04-12T11:52:21.531 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:21.555 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:21.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:21.656 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:21.710 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:21.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:21.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:21.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:21.848 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:21.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:21.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:21.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:21.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:21.985 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:21.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:22.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:22.109 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:22.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:22.211 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:22.243 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:22.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:22.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:22.345 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:22.384 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:22.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:22.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:22.509 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:22.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:22.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:22.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:22.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:22.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:22.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:22.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:52:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:52:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:52:22.749 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:22.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:22.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:22.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:22.776 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:22.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:22.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:22.878 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:22.907 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:22.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:22.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:23.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:23.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:23.045 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:23.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:23.146 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:23.182 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:23.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:23.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:23.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:23.307 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:23.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:23.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:23.409 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:23.433 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:23.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:23.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:23.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:23 smithi087 ceph-mon[89014]: pgmap v2064: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 17004/65742 objects degraded (25.865%); 3314/65742 objects misplaced (5.041%); 13 MiB/s, 214 objects/s recovering 2024-04-12T11:52:23.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 17004/65742 objects degraded (25.865%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:23.534 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:23.571 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:23.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:23.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:23.700 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:23.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:23.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:23 smithi029 ceph-mon[93650]: pgmap v2064: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 17004/65742 objects degraded (25.865%); 3314/65742 objects misplaced (5.041%); 13 MiB/s, 214 objects/s recovering 2024-04-12T11:52:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 17004/65742 objects degraded (25.865%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:23.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:23 smithi029 ceph-mon[97787]: pgmap v2064: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 17004/65742 objects degraded (25.865%); 3314/65742 objects misplaced (5.041%); 13 MiB/s, 214 objects/s recovering 2024-04-12T11:52:23.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 17004/65742 objects degraded (25.865%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:23.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:23.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:23.825 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:23.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:23.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:23.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:23.966 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:23.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:24.067 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:24.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:24.113 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:24.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:24.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:24.249 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:24.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:24.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:24.351 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:24.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:24.378 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:24.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:24.479 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:24.521 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:24.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:24.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:24.646 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:24.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:24.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:24.748 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:24.772 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:24.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:24.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:24.873 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:24.909 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:24.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:24.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:25.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:25.039 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:25.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:25.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:25.142 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:25.165 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:25.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:25.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:25.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:25.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:52:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:52:24] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2024-04-12T11:52:25.290 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:25.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:25.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:25.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:25.433 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:25.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:25.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:25.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:25 smithi087 ceph-mon[89014]: pgmap v2065: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 16008/55551 objects degraded (28.817%); 3098/55551 objects misplaced (5.577%); 15 MiB/s, 237 objects/s recovering 2024-04-12T11:52:25.534 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:25.551 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:25 smithi029 ceph-mon[97787]: pgmap v2065: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 16008/55551 objects degraded (28.817%); 3098/55551 objects misplaced (5.577%); 15 MiB/s, 237 objects/s recovering 2024-04-12T11:52:25.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:25 smithi029 ceph-mon[93650]: pgmap v2065: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 16008/55551 objects degraded (28.817%); 3098/55551 objects misplaced (5.577%); 15 MiB/s, 237 objects/s recovering 2024-04-12T11:52:25.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:25.573 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:25.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:25.674 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:25.710 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:25.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:25.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:25.812 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:25.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:25.852 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:25.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:25.952 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:25.997 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:25.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:25.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:26.099 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:26.144 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:26.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:26.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:26.288 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:26.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:26.390 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:26.430 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:26.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:26.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:26.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:26.573 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:26.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:26.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:26.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:26.718 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:26.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:26.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:26.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:26.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:26.858 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:26.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:26.959 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:26.991 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:26.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:26.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:27.092 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:27.121 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:27.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:27.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:27.222 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:27.252 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:27.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:27.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:27.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:26 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:27.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:26 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:27.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:26 smithi029 ceph-mon[93650]: pgmap v2066: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 16008/54981 objects degraded (29.116%); 3098/54981 objects misplaced (5.635%); 14 MiB/s, 216 objects/s recovering 2024-04-12T11:52:27.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:26 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:27.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:26 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:27.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:26 smithi029 ceph-mon[97787]: pgmap v2066: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 16008/54981 objects degraded (29.116%); 3098/54981 objects misplaced (5.635%); 14 MiB/s, 216 objects/s recovering 2024-04-12T11:52:27.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:27.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:27.393 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:27.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:27.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:26 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:27.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:26 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:27.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:26 smithi087 ceph-mon[89014]: pgmap v2066: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 16008/54981 objects degraded (29.116%); 3098/54981 objects misplaced (5.635%); 14 MiB/s, 216 objects/s recovering 2024-04-12T11:52:27.495 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:27.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:27.540 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:27.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:27.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:27.671 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:27.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:27.772 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:27.804 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:27.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:27.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:27.906 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:27.939 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:27.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:27.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:28.041 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:28.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:28.077 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:28.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:28.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:28.216 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:28.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:28.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-04-12T11:52:28.302 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16008/54981 objects degraded (29.116%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:28.303 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16008/54981 objects degraded (29.116%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:28.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:28.355 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:28.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:28.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:28.457 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16008/54981 objects degraded (29.116%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:28.512 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:28.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:28.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:28.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:28.632 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:52:28.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:28.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:28.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:28.675 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:52:28.710 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:52:28.751 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:52:28.768 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:28.788 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:52:28.799 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:28.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:28.824 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:52:28.859 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:52:28.893 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:52:28.896 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:52:28.896 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:52:28.896 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:52:28.896 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:52:28.897 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:52:28.897 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:52:28.897 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:52:28.897 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:52:28.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:28.933 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:28.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:28.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:29.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:29.083 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:29.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:29.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:29.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:52:29.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:29 smithi029 ceph-mon[93650]: pgmap v2067: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 15841/49518 objects degraded (31.990%); 3098/49518 objects misplaced (6.256%); 13 MiB/s, 205 objects/s recovering 2024-04-12T11:52:29.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:29 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1860942038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:29.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:29 smithi029 ceph-mon[97787]: pgmap v2067: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 15841/49518 objects degraded (31.990%); 3098/49518 objects misplaced (6.256%); 13 MiB/s, 205 objects/s recovering 2024-04-12T11:52:29.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:29 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1860942038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:29.185 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:29.222 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:29.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:29.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:29.323 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:29.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:29.387 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:29.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:29 smithi087 ceph-mon[89014]: pgmap v2067: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 15841/49518 objects degraded (31.990%); 3098/49518 objects misplaced (6.256%); 13 MiB/s, 205 objects/s recovering 2024-04-12T11:52:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:29 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1860942038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:29.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:29.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:29.537 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:29.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:29.638 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:29.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:29.668 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:29.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:29.769 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:29.803 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:29.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:29.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:29.905 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:29.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:29.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:29.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:30.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:30.081 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:30.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:30.183 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:30.213 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:30.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:30.315 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:30.347 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:30.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:30.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:30.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:30.482 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:30.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:30.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:30.584 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:30.614 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:30.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:30.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:30.715 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:30.750 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:30.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:30.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:30.852 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:30.876 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:30.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:30.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:30.978 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:31.014 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:31.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:31.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:31.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:31.157 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:31.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:31.258 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:31.296 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:31.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:31.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:31.422 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:31.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:31.463 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:31.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:31 smithi087 ceph-mon[89014]: pgmap v2068: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 925 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 15643/44430 objects degraded (35.208%); 3029/44430 objects misplaced (6.817%); 9.1 MiB/s, 150 objects/s recovering 2024-04-12T11:52:31.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:31 smithi029 ceph-mon[97787]: pgmap v2068: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 925 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 15643/44430 objects degraded (35.208%); 3029/44430 objects misplaced (6.817%); 9.1 MiB/s, 150 objects/s recovering 2024-04-12T11:52:31.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:31 smithi029 ceph-mon[93650]: pgmap v2068: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 925 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 15643/44430 objects degraded (35.208%); 3029/44430 objects misplaced (6.817%); 9.1 MiB/s, 150 objects/s recovering 2024-04-12T11:52:31.564 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:31.594 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:31.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:31.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:31.695 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:31.723 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:31.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:31.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:31.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:31.848 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:31.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:31.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:31.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:31.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:52:31.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:31.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:31.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:32.083 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T11:52:32.116 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:32.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:32.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:32.119 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T11:52:32.218 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:32.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:32.255 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:32.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:32.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:32.388 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:32.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:32.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:32.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:32.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:32.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:32.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:32.636 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:32.670 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:32.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:32.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:52:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:52:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:52:32.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:32.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:32.812 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:32.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:32.913 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:32.950 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:32.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:33.052 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:33.078 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:33.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:33.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:33.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:33.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:33.214 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:33.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:33.315 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:33.351 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:33.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:33.452 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:33.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:33 smithi087 ceph-mon[89014]: from='client.55198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:52:33.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:33 smithi087 ceph-mon[89014]: pgmap v2069: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 907 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 15643/43599 objects degraded (35.879%); 3029/43599 objects misplaced (6.947%); 8.3 MiB/s, 137 objects/s recovering 2024-04-12T11:52:33.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15643/43599 objects degraded (35.879%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:33.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:33.492 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:33.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:33.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:33 smithi029 ceph-mon[97787]: from='client.55198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:52:33.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:33 smithi029 ceph-mon[97787]: pgmap v2069: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 907 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 15643/43599 objects degraded (35.879%); 3029/43599 objects misplaced (6.947%); 8.3 MiB/s, 137 objects/s recovering 2024-04-12T11:52:33.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15643/43599 objects degraded (35.879%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:33 smithi029 ceph-mon[93650]: from='client.55198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T11:52:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:33 smithi029 ceph-mon[93650]: pgmap v2069: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 907 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 15643/43599 objects degraded (35.879%); 3029/43599 objects misplaced (6.947%); 8.3 MiB/s, 137 objects/s recovering 2024-04-12T11:52:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15643/43599 objects degraded (35.879%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:33.592 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:33.627 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:33.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:33.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:33.728 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:33.760 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:33.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:33.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:33.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:33.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:33.899 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:33.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:34.000 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:34.037 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:34.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:34.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:34.139 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:34.175 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:34.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:34.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:34.239 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2068,"stamp":"2024-04-12T11:52:30.055462+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":969545706,"num_objects":14810,"num_object_clones":0,"num_object_copies":44430,"num_objects_missing_on_primary":4257,"num_objects_missing":4257,"num_objects_degraded":15643,"num_objects_misplaced":3029,"num_objects_unfound":0,"num_objects_dirty":14810,"num_whiteouts":0,"num_read":487,"num_read_kb":913,"num_write":203064,"num_write_kb":6963983,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27045,"num_bytes_recovered":1631598156,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":231781,"ondisk_log_size":231781,"up":327,"acting":326,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":340,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10333688,"kb_used_data":5263160,"kb_used_omap":175,"kb_used_meta":5070032,"kb_avail":739627528,"statfs":{"total":767960285184,"available":757378588672,"internally_reserved":0,"allocated":5389475840,"data_stored":5369511522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":180055,"internal_metadata":5191712937},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,3,7,2,7,2,5,2,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-479526912,"num_objects":-7317,"num_object_clones":0,"num_object_copies":-21951,"num_objects_missing_on_primary":-563,"num_objects_missing":-563,"num_objects_degraded":-1361,"num_objects_misplaced":-285,"num_objects_unfound":0,"num_objects_dirty":-7317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7317,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1648,"num_bytes_recovered":104333312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.925265"},"pg_stats":[{"pgid":"9.10","version":"1259'10803","reported_seq":10889,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:27.571330+0000","last_change":"2024-04-12T11:50:11.556041+0000","last_active":"2024-04-12T11:52:27.571330+0000","last_peered":"2024-04-12T11:52:27.571330+0000","last_clean":"2024-04-12T11:52:27.571330+0000","last_became_active":"2024-04-12T11:50:11.542185+0000","last_became_peered":"2024-04-12T11:50:11.542185+0000","last_unstale":"2024-04-12T11:52:27.571330+0000","last_undegraded":"2024-04-12T11:52:27.571330+0000","last_fullsized":"2024-04-12T11:52:27.571330+0000","mapping_epoch":1248,"log_start":"1230'1000","ondisk_log_start":"1230'1000","created":1248,"last_epoch_clean":1249,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":9803,"log_dups_size":0,"ondisk_log_size":9803,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-12T11:52:32.478203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32374784,"num_objects":494,"num_object_clones":0,"num_object_copies":1482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6353,"num_write_kb":219104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":1875,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.129315+0000","last_change":"2024-04-12T11:23:36.642787+0000","last_active":"2024-04-12T11:52:19.129315+0000","last_peered":"2024-04-12T11:52:19.129315+0000","last_clean":"2024-04-12T11:52:19.129315+0000","last_became_active":"2024-04-12T11:23:36.641336+0000","last_became_peered":"2024-04-12T11:23:36.641336+0000","last_unstale":"2024-04-12T11:52:19.129315+0000","last_undegraded":"2024-04-12T11:52:19.129315+0000","last_fullsized":"2024-04-12T11:52:19.129315+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:32:12.724529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.140743+0000","last_change":"2024-04-12T11:48:45.352231+0000","last_active":"2024-04-12T11:52:19.140743+0000","last_peered":"2024-04-12T11:52:19.140743+0000","last_clean":"2024-04-12T11:52:19.140743+0000","last_became_active":"2024-04-12T11:48:45.351324+0000","last_became_peered":"2024-04-12T11:48:45.351324+0000","last_unstale":"2024-04-12T11:52:19.140743+0000","last_undegraded":"2024-04-12T11:52:19.140743+0000","last_fullsized":"2024-04-12T11:52:19.140743+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:53:01.615486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.13","version":"1259'10620","reported_seq":14203,"reported_epoch":1259,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:52:27.341488+0000","last_change":"2024-04-12T11:50:11.624865+0000","last_active":"2024-04-12T11:52:27.341488+0000","last_peered":"2024-04-12T11:52:27.341488+0000","last_clean":"2024-04-12T11:48:44.327141+0000","last_became_active":"2024-04-12T11:50:11.610037+0000","last_became_peered":"2024-04-12T11:50:11.610037+0000","last_unstale":"2024-04-12T11:52:27.341488+0000","last_undegraded":"2024-04-12T11:50:11.570250+0000","last_fullsized":"2024-04-12T11:52:27.341488+0000","mapping_epoch":1248,"log_start":"1231'2812","ondisk_log_start":"1231'2812","created":1248,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7808,"log_dups_size":0,"ondisk_log_size":7808,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29949952,"num_objects":457,"num_object_clones":0,"num_object_copies":1371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1055,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6232,"num_write_kb":214048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1763,"num_bytes_recovered":115474432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,3,7","objects":457}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1959,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.134100+0000","last_change":"2024-04-12T11:48:45.353042+0000","last_active":"2024-04-12T11:52:19.134100+0000","last_peered":"2024-04-12T11:52:19.134100+0000","last_clean":"2024-04-12T11:52:19.134100+0000","last_became_active":"2024-04-12T11:48:45.352721+0000","last_became_peered":"2024-04-12T11:48:45.352721+0000","last_unstale":"2024-04-12T11:52:19.134100+0000","last_undegraded":"2024-04-12T11:52:19.134100+0000","last_fullsized":"2024-04-12T11:52:19.134100+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:00:32.520262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1980,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.145967+0000","last_change":"2024-04-12T11:48:45.353306+0000","last_active":"2024-04-12T11:52:19.145967+0000","last_peered":"2024-04-12T11:52:19.145967+0000","last_clean":"2024-04-12T11:52:19.145967+0000","last_became_active":"2024-04-12T11:48:45.352065+0000","last_became_peered":"2024-04-12T11:48:45.352065+0000","last_unstale":"2024-04-12T11:52:19.145967+0000","last_undegraded":"2024-04-12T11:52:19.145967+0000","last_fullsized":"2024-04-12T11:52:19.145967+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:26:36.201291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.12","version":"1259'10740","reported_seq":14337,"reported_epoch":1259,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:52:28.141010+0000","last_change":"2024-04-12T11:50:13.268320+0000","last_active":"2024-04-12T11:52:28.141010+0000","last_peered":"2024-04-12T11:52:28.141010+0000","last_clean":"2024-04-12T11:48:44.392445+0000","last_became_active":"2024-04-12T11:50:12.559301+0000","last_became_peered":"2024-04-12T11:50:12.559301+0000","last_unstale":"2024-04-12T11:52:28.141010+0000","last_undegraded":"2024-04-12T11:50:12.509110+0000","last_fullsized":"2024-04-12T11:52:28.141010+0000","mapping_epoch":1248,"log_start":"1230'900","ondisk_log_start":"1230'900","created":1248,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":9840,"log_dups_size":0,"ondisk_log_size":9840,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31981568,"num_objects":488,"num_object_clones":0,"num_object_copies":1464,"num_objects_missing_on_primary":1040,"num_objects_missing":1040,"num_objects_degraded":1040,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6299,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1756,"num_bytes_recovered":115081216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,3","objects":488}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.140605+0000","last_change":"2024-04-12T11:48:45.351604+0000","last_active":"2024-04-12T11:52:19.140605+0000","last_peered":"2024-04-12T11:52:19.140605+0000","last_clean":"2024-04-12T11:52:19.140605+0000","last_became_active":"2024-04-12T11:48:45.349968+0000","last_became_peered":"2024-04-12T11:48:45.349968+0000","last_unstale":"2024-04-12T11:52:19.140605+0000","last_undegraded":"2024-04-12T11:52:19.140605+0000","last_fullsized":"2024-04-12T11:52:19.140605+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:56:42.419931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1209,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.553033+0000","last_change":"2024-04-12T11:22:19.616723+0000","last_active":"2024-04-12T11:52:02.553033+0000","last_peered":"2024-04-12T11:52:02.553033+0000","last_clean":"2024-04-12T11:52:02.553033+0000","last_became_active":"2024-04-12T11:22:19.616371+0000","last_became_peered":"2024-04-12T11:22:19.616371+0000","last_unstale":"2024-04-12T11:52:02.553033+0000","last_undegraded":"2024-04-12T11:52:02.553033+0000","last_fullsized":"2024-04-12T11:52:02.553033+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:34:03.486200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1950,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.146435+0000","last_change":"2024-04-12T11:48:45.351162+0000","last_active":"2024-04-12T11:52:19.146435+0000","last_peered":"2024-04-12T11:52:19.146435+0000","last_clean":"2024-04-12T11:52:19.146435+0000","last_became_active":"2024-04-12T11:48:45.350817+0000","last_became_peered":"2024-04-12T11:48:45.350817+0000","last_unstale":"2024-04-12T11:52:19.146435+0000","last_undegraded":"2024-04-12T11:52:19.146435+0000","last_fullsized":"2024-04-12T11:52:19.146435+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:59.627216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.143329+0000","last_change":"2024-04-12T11:48:45.420195+0000","last_active":"2024-04-12T11:52:19.143329+0000","last_peered":"2024-04-12T11:52:19.143329+0000","last_clean":"2024-04-12T11:52:19.143329+0000","last_became_active":"2024-04-12T11:48:45.419810+0000","last_became_peered":"2024-04-12T11:48:45.419810+0000","last_unstale":"2024-04-12T11:52:19.143329+0000","last_undegraded":"2024-04-12T11:52:19.143329+0000","last_fullsized":"2024-04-12T11:52:19.143329+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:33:11.234641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1241,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.486017+0000","last_change":"2024-04-12T11:45:52.986340+0000","last_active":"2024-04-12T11:52:02.486017+0000","last_peered":"2024-04-12T11:52:02.486017+0000","last_clean":"2024-04-12T11:52:02.486017+0000","last_became_active":"2024-04-12T11:45:52.984190+0000","last_became_peered":"2024-04-12T11:45:52.984190+0000","last_unstale":"2024-04-12T11:52:02.486017+0000","last_undegraded":"2024-04-12T11:52:02.486017+0000","last_fullsized":"2024-04-12T11:52:02.486017+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:52:19.144894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.139347+0000","last_change":"2024-04-12T11:48:45.349940+0000","last_active":"2024-04-12T11:52:19.139347+0000","last_peered":"2024-04-12T11:52:19.139347+0000","last_clean":"2024-04-12T11:52:19.139347+0000","last_became_active":"2024-04-12T11:48:45.349603+0000","last_became_peered":"2024-04-12T11:48:45.349603+0000","last_unstale":"2024-04-12T11:52:19.139347+0000","last_undegraded":"2024-04-12T11:52:19.139347+0000","last_fullsized":"2024-04-12T11:52:19.139347+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:04:27.503011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":98,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.483164+0000","last_change":"2024-04-12T11:48:45.357689+0000","last_active":"2024-04-12T11:52:02.483164+0000","last_peered":"2024-04-12T11:52:02.483164+0000","last_clean":"2024-04-12T11:52:02.483164+0000","last_became_active":"2024-04-12T11:48:45.349500+0000","last_became_peered":"2024-04-12T11:48:45.349500+0000","last_unstale":"2024-04-12T11:52:02.483164+0000","last_undegraded":"2024-04-12T11:52:02.483164+0000","last_fullsized":"2024-04-12T11:52:02.483164+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:54:58.532082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1926,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.135602+0000","last_change":"2024-04-12T11:22:40.580378+0000","last_active":"2024-04-12T11:52:19.135602+0000","last_peered":"2024-04-12T11:52:19.135602+0000","last_clean":"2024-04-12T11:52:19.135602+0000","last_became_active":"2024-04-12T11:22:40.580061+0000","last_became_peered":"2024-04-12T11:22:40.580061+0000","last_unstale":"2024-04-12T11:52:19.135602+0000","last_undegraded":"2024-04-12T11:52:19.135602+0000","last_fullsized":"2024-04-12T11:52:19.135602+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:25:46.211354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":1880,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.130483+0000","last_change":"2024-04-12T11:45:53.024626+0000","last_active":"2024-04-12T11:52:19.130483+0000","last_peered":"2024-04-12T11:52:19.130483+0000","last_clean":"2024-04-12T11:52:19.130483+0000","last_became_active":"2024-04-12T11:45:52.976276+0000","last_became_peered":"2024-04-12T11:45:52.976276+0000","last_unstale":"2024-04-12T11:52:19.130483+0000","last_undegraded":"2024-04-12T11:52:19.130483+0000","last_fullsized":"2024-04-12T11:52:19.130483+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:49.929210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1839,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.130031+0000","last_change":"2024-04-12T11:23:36.652564+0000","last_active":"2024-04-12T11:52:19.130031+0000","last_peered":"2024-04-12T11:52:19.130031+0000","last_clean":"2024-04-12T11:52:19.130031+0000","last_became_active":"2024-04-12T11:23:36.650121+0000","last_became_peered":"2024-04-12T11:23:36.650121+0000","last_unstale":"2024-04-12T11:52:19.130031+0000","last_undegraded":"2024-04-12T11:52:19.130031+0000","last_fullsized":"2024-04-12T11:52:19.130031+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:04.627924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.19","version":"1259'10691","reported_seq":10782,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:27.554238+0000","last_change":"2024-04-12T11:52:26.464120+0000","last_active":"2024-04-12T11:52:27.554238+0000","last_peered":"2024-04-12T11:52:27.554238+0000","last_clean":"2024-04-12T11:52:27.554238+0000","last_became_active":"2024-04-12T11:50:11.545583+0000","last_became_peered":"2024-04-12T11:50:11.545583+0000","last_unstale":"2024-04-12T11:52:27.554238+0000","last_undegraded":"2024-04-12T11:52:27.554238+0000","last_fullsized":"2024-04-12T11:52:27.554238+0000","mapping_epoch":1248,"log_start":"1230'2100","ondisk_log_start":"1230'2100","created":1248,"last_epoch_clean":1249,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8591,"log_dups_size":0,"ondisk_log_size":8591,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-12T11:52:31.407558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29425664,"num_objects":449,"num_object_clones":0,"num_object_copies":1347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6294,"num_write_kb":215776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":68,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.483518+0000","last_change":"2024-04-12T11:45:53.035604+0000","last_active":"2024-04-12T11:52:02.483518+0000","last_peered":"2024-04-12T11:52:02.483518+0000","last_clean":"2024-04-12T11:52:02.483518+0000","last_became_active":"2024-04-12T11:45:52.984722+0000","last_became_peered":"2024-04-12T11:45:52.984722+0000","last_unstale":"2024-04-12T11:52:02.483518+0000","last_undegraded":"2024-04-12T11:52:02.483518+0000","last_fullsized":"2024-04-12T11:52:02.483518+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:30.674395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.146467+0000","last_change":"2024-04-12T11:48:45.353499+0000","last_active":"2024-04-12T11:52:19.146467+0000","last_peered":"2024-04-12T11:52:19.146467+0000","last_clean":"2024-04-12T11:52:19.146467+0000","last_became_active":"2024-04-12T11:48:45.350015+0000","last_became_peered":"2024-04-12T11:48:45.350015+0000","last_unstale":"2024-04-12T11:52:19.146467+0000","last_undegraded":"2024-04-12T11:52:19.146467+0000","last_fullsized":"2024-04-12T11:52:19.146467+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:24:17.358128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.18","version":"1259'10853","reported_seq":14461,"reported_epoch":1259,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:52:28.000629+0000","last_change":"2024-04-12T11:50:15.429131+0000","last_active":"2024-04-12T11:52:28.000629+0000","last_peered":"2024-04-12T11:52:28.000629+0000","last_clean":"2024-04-12T11:48:44.316482+0000","last_became_active":"2024-04-12T11:50:14.727511+0000","last_became_peered":"2024-04-12T11:50:14.727511+0000","last_unstale":"2024-04-12T11:52:28.000629+0000","last_undegraded":"2024-04-12T11:50:12.526604+0000","last_fullsized":"2024-04-12T11:52:28.000629+0000","mapping_epoch":1248,"log_start":"1230'2300","ondisk_log_start":"1230'2300","created":1248,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8553,"log_dups_size":0,"ondisk_log_size":8553,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32571392,"num_objects":497,"num_object_clones":0,"num_object_copies":1491,"num_objects_missing_on_primary":1114,"num_objects_missing":1114,"num_objects_degraded":1114,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6348,"num_write_kb":219040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1765,"num_bytes_recovered":115671040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,3,5","objects":497}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1964,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.146879+0000","last_change":"2024-04-12T11:48:45.351398+0000","last_active":"2024-04-12T11:52:19.146879+0000","last_peered":"2024-04-12T11:52:19.146879+0000","last_clean":"2024-04-12T11:52:19.146879+0000","last_became_active":"2024-04-12T11:48:45.350886+0000","last_became_peered":"2024-04-12T11:48:45.350886+0000","last_unstale":"2024-04-12T11:52:19.146879+0000","last_undegraded":"2024-04-12T11:52:19.146879+0000","last_fullsized":"2024-04-12T11:52:19.146879+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:10:02.835400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1267,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.485920+0000","last_change":"2024-04-12T11:48:45.350844+0000","last_active":"2024-04-12T11:52:02.485920+0000","last_peered":"2024-04-12T11:52:02.485920+0000","last_clean":"2024-04-12T11:52:02.485920+0000","last_became_active":"2024-04-12T11:48:45.350024+0000","last_became_peered":"2024-04-12T11:48:45.350024+0000","last_unstale":"2024-04-12T11:52:02.485920+0000","last_undegraded":"2024-04-12T11:52:02.485920+0000","last_fullsized":"2024-04-12T11:52:02.485920+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:44:47.030599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":62,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482468+0000","last_change":"2024-04-12T11:45:53.006808+0000","last_active":"2024-04-12T11:52:02.482468+0000","last_peered":"2024-04-12T11:52:02.482468+0000","last_clean":"2024-04-12T11:52:02.482468+0000","last_became_active":"2024-04-12T11:45:53.006480+0000","last_became_peered":"2024-04-12T11:45:53.006480+0000","last_unstale":"2024-04-12T11:52:02.482468+0000","last_undegraded":"2024-04-12T11:52:02.482468+0000","last_fullsized":"2024-04-12T11:52:02.482468+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:11:47.685096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1964,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.146054+0000","last_change":"2024-04-12T11:48:45.352862+0000","last_active":"2024-04-12T11:52:19.146054+0000","last_peered":"2024-04-12T11:52:19.146054+0000","last_clean":"2024-04-12T11:52:19.146054+0000","last_became_active":"2024-04-12T11:48:45.352485+0000","last_became_peered":"2024-04-12T11:48:45.352485+0000","last_unstale":"2024-04-12T11:52:19.146054+0000","last_undegraded":"2024-04-12T11:52:19.146054+0000","last_fullsized":"2024-04-12T11:52:19.146054+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:11:20.916845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1216,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.554022+0000","last_change":"2024-04-12T11:21:59.594292+0000","last_active":"2024-04-12T11:52:02.554022+0000","last_peered":"2024-04-12T11:52:02.554022+0000","last_clean":"2024-04-12T11:52:02.554022+0000","last_became_active":"2024-04-12T11:21:59.593815+0000","last_became_peered":"2024-04-12T11:21:59.593815+0000","last_unstale":"2024-04-12T11:52:02.554022+0000","last_undegraded":"2024-04-12T11:52:02.554022+0000","last_fullsized":"2024-04-12T11:52:02.554022+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":515,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:24:10.850314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.142525+0000","last_change":"2024-04-12T11:48:45.361653+0000","last_active":"2024-04-12T11:52:19.142525+0000","last_peered":"2024-04-12T11:52:19.142525+0000","last_clean":"2024-04-12T11:52:19.142525+0000","last_became_active":"2024-04-12T11:48:45.361334+0000","last_became_peered":"2024-04-12T11:48:45.361334+0000","last_unstale":"2024-04-12T11:52:19.142525+0000","last_undegraded":"2024-04-12T11:52:19.142525+0000","last_fullsized":"2024-04-12T11:52:19.142525+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:58:25.548034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"1259'10997","reported_seq":11092,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:27.441077+0000","last_change":"2024-04-12T11:52:23.542449+0000","last_active":"2024-04-12T11:52:27.441077+0000","last_peered":"2024-04-12T11:52:27.441077+0000","last_clean":"2024-04-12T11:52:27.441077+0000","last_became_active":"2024-04-12T11:50:11.541855+0000","last_became_peered":"2024-04-12T11:50:11.541855+0000","last_unstale":"2024-04-12T11:52:27.441077+0000","last_undegraded":"2024-04-12T11:52:27.441077+0000","last_fullsized":"2024-04-12T11:52:27.441077+0000","mapping_epoch":1226,"log_start":"1230'2300","ondisk_log_start":"1230'2300","created":1226,"last_epoch_clean":1256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8697,"log_dups_size":0,"ondisk_log_size":8697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-12T11:52:28.434735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26378240,"num_objects":402,"num_object_clones":0,"num_object_copies":1206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6478,"num_write_kb":220192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":66,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482895+0000","last_change":"2024-04-12T11:45:53.028640+0000","last_active":"2024-04-12T11:52:02.482895+0000","last_peered":"2024-04-12T11:52:02.482895+0000","last_clean":"2024-04-12T11:52:02.482895+0000","last_became_active":"2024-04-12T11:45:52.983099+0000","last_became_peered":"2024-04-12T11:45:52.983099+0000","last_unstale":"2024-04-12T11:52:02.482895+0000","last_undegraded":"2024-04-12T11:52:02.482895+0000","last_fullsized":"2024-04-12T11:52:02.482895+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:43:01.897014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1218,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:03.126524+0000","last_change":"2024-04-12T11:45:52.993160+0000","last_active":"2024-04-12T11:52:03.126524+0000","last_peered":"2024-04-12T11:52:03.126524+0000","last_clean":"2024-04-12T11:52:03.126524+0000","last_became_active":"2024-04-12T11:45:52.989310+0000","last_became_peered":"2024-04-12T11:45:52.989310+0000","last_unstale":"2024-04-12T11:52:03.126524+0000","last_undegraded":"2024-04-12T11:52:03.126524+0000","last_fullsized":"2024-04-12T11:52:03.126524+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:58:47.362374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1259'11013","reported_seq":14857,"reported_epoch":1259,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:52:27.606197+0000","last_change":"2024-04-12T11:50:12.561068+0000","last_active":"2024-04-12T11:52:27.606197+0000","last_peered":"2024-04-12T11:52:27.606197+0000","last_clean":"2024-04-12T11:48:44.288786+0000","last_became_active":"2024-04-12T11:50:12.516374+0000","last_became_peered":"2024-04-12T11:50:12.516374+0000","last_unstale":"2024-04-12T11:52:27.606197+0000","last_undegraded":"2024-04-12T11:50:12.505594+0000","last_fullsized":"2024-04-12T11:52:27.606197+0000","mapping_epoch":1249,"log_start":"1230'1100","ondisk_log_start":"1230'1100","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":9913,"log_dups_size":0,"ondisk_log_size":9913,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29425664,"num_objects":449,"num_object_clones":0,"num_object_copies":1347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1037,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6489,"num_write_kb":222016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1879,"num_bytes_recovered":123076608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"1,3,4","objects":449}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":66,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482843+0000","last_change":"2024-04-12T11:45:53.031759+0000","last_active":"2024-04-12T11:52:02.482843+0000","last_peered":"2024-04-12T11:52:02.482843+0000","last_clean":"2024-04-12T11:52:02.482843+0000","last_became_active":"2024-04-12T11:45:52.984545+0000","last_became_peered":"2024-04-12T11:45:52.984545+0000","last_unstale":"2024-04-12T11:52:02.482843+0000","last_undegraded":"2024-04-12T11:52:02.482843+0000","last_fullsized":"2024-04-12T11:52:02.482843+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:16:10.421992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1886,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.130803+0000","last_change":"2024-04-12T11:48:45.351213+0000","last_active":"2024-04-12T11:52:19.130803+0000","last_peered":"2024-04-12T11:52:19.130803+0000","last_clean":"2024-04-12T11:52:19.130803+0000","last_became_active":"2024-04-12T11:48:45.350772+0000","last_became_peered":"2024-04-12T11:48:45.350772+0000","last_unstale":"2024-04-12T11:52:19.130803+0000","last_undegraded":"2024-04-12T11:52:19.130803+0000","last_fullsized":"2024-04-12T11:52:19.130803+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:17:48.831705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1259'10753","reported_seq":14371,"reported_epoch":1259,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:52:28.119237+0000","last_change":"2024-04-12T11:50:12.554070+0000","last_active":"2024-04-12T11:52:28.119237+0000","last_peered":"2024-04-12T11:52:28.119237+0000","last_clean":"2024-04-12T11:48:44.337474+0000","last_became_active":"2024-04-12T11:50:12.522681+0000","last_became_peered":"2024-04-12T11:50:12.522681+0000","last_unstale":"2024-04-12T11:52:28.119237+0000","last_undegraded":"2024-04-12T11:50:12.510378+0000","last_fullsized":"2024-04-12T11:52:28.119237+0000","mapping_epoch":1249,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":10753,"log_dups_size":0,"ondisk_log_size":10753,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30867456,"num_objects":471,"num_object_clones":0,"num_object_copies":1413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1046,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6306,"num_write_kb":216864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1779,"num_bytes_recovered":116523008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"3,5,7","objects":471}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":66,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482979+0000","last_change":"2024-04-12T11:45:53.169686+0000","last_active":"2024-04-12T11:52:02.482979+0000","last_peered":"2024-04-12T11:52:02.482979+0000","last_clean":"2024-04-12T11:52:02.482979+0000","last_became_active":"2024-04-12T11:45:53.021021+0000","last_became_peered":"2024-04-12T11:45:53.021021+0000","last_unstale":"2024-04-12T11:52:02.482979+0000","last_undegraded":"2024-04-12T11:52:02.482979+0000","last_fullsized":"2024-04-12T11:52:02.482979+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:48:23.506812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1236,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:07.245059+0000","last_change":"2024-04-12T11:45:52.979601+0000","last_active":"2024-04-12T11:52:07.245059+0000","last_peered":"2024-04-12T11:52:07.245059+0000","last_clean":"2024-04-12T11:52:07.245059+0000","last_became_active":"2024-04-12T11:45:52.979307+0000","last_became_peered":"2024-04-12T11:45:52.979307+0000","last_unstale":"2024-04-12T11:52:07.245059+0000","last_undegraded":"2024-04-12T11:52:07.245059+0000","last_fullsized":"2024-04-12T11:52:07.245059+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:21.367107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1259'11061","reported_seq":11113,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:28.108736+0000","last_change":"2024-04-12T11:50:11.562418+0000","last_active":"2024-04-12T11:52:28.108736+0000","last_peered":"2024-04-12T11:52:28.108736+0000","last_clean":"2024-04-12T11:52:28.108736+0000","last_became_active":"2024-04-12T11:50:11.548931+0000","last_became_peered":"2024-04-12T11:50:11.548931+0000","last_unstale":"2024-04-12T11:52:28.108736+0000","last_undegraded":"2024-04-12T11:52:28.108736+0000","last_fullsized":"2024-04-12T11:52:28.108736+0000","mapping_epoch":1226,"log_start":"0'0","ondisk_log_start":"0'0","created":1226,"last_epoch_clean":1249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":11061,"log_dups_size":0,"ondisk_log_size":11061,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28082176,"num_objects":428,"num_object_clones":0,"num_object_copies":1284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6487,"num_write_kb":221312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1251,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.485802+0000","last_change":"2024-04-12T11:48:45.344961+0000","last_active":"2024-04-12T11:52:02.485802+0000","last_peered":"2024-04-12T11:52:02.485802+0000","last_clean":"2024-04-12T11:52:02.485802+0000","last_became_active":"2024-04-12T11:48:45.344344+0000","last_became_peered":"2024-04-12T11:48:45.344344+0000","last_unstale":"2024-04-12T11:52:02.485802+0000","last_undegraded":"2024-04-12T11:52:02.485802+0000","last_fullsized":"2024-04-12T11:52:02.485802+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:47:55.659541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1938,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.147377+0000","last_change":"2024-04-12T11:23:36.646373+0000","last_active":"2024-04-12T11:52:19.147377+0000","last_peered":"2024-04-12T11:52:19.147377+0000","last_clean":"2024-04-12T11:52:19.147377+0000","last_became_active":"2024-04-12T11:23:36.645761+0000","last_became_peered":"2024-04-12T11:23:36.645761+0000","last_unstale":"2024-04-12T11:52:19.147377+0000","last_undegraded":"2024-04-12T11:52:19.147377+0000","last_fullsized":"2024-04-12T11:52:19.147377+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:35:57.569786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"249'161","reported_seq":1577,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482713+0000","last_change":"2024-04-12T11:49:07.188158+0000","last_active":"2024-04-12T11:52:02.482713+0000","last_peered":"2024-04-12T11:52:02.482713+0000","last_clean":"2024-04-12T11:52:02.482713+0000","last_became_active":"2024-04-12T11:49:07.187847+0000","last_became_peered":"2024-04-12T11:49:07.187847+0000","last_unstale":"2024-04-12T11:52:02.482713+0000","last_undegraded":"2024-04-12T11:52:02.482713+0000","last_fullsized":"2024-04-12T11:52:02.482713+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.1","version":"1259'10747","reported_seq":16296,"reported_epoch":1259,"state":"active+recovery_wait+remapped","last_fresh":"2024-04-12T11:52:25.087724+0000","last_change":"2024-04-12T11:50:11.552108+0000","last_active":"2024-04-12T11:52:25.087724+0000","last_peered":"2024-04-12T11:52:25.087724+0000","last_clean":"2024-04-12T11:48:44.315992+0000","last_became_active":"2024-04-12T11:50:11.528798+0000","last_became_peered":"2024-04-12T11:50:11.528798+0000","last_unstale":"2024-04-12T11:52:25.087724+0000","last_undegraded":"2024-04-12T11:52:25.087724+0000","last_fullsized":"2024-04-12T11:52:25.087724+0000","mapping_epoch":1235,"log_start":"1230'500","ondisk_log_start":"1230'500","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":10247,"log_dups_size":0,"ondisk_log_size":10247,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33554432,"num_objects":512,"num_object_clones":0,"num_object_copies":1536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1510,"num_objects_unfound":0,"num_objects_dirty":512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6283,"num_write_kb":217440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1367,"num_bytes_recovered":89587712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[4,6,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":-998}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":64,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.138945+0000","last_change":"2024-04-12T11:48:45.359254+0000","last_active":"2024-04-12T11:52:19.138945+0000","last_peered":"2024-04-12T11:52:19.138945+0000","last_clean":"2024-04-12T11:52:19.138945+0000","last_became_active":"2024-04-12T11:48:45.348788+0000","last_became_peered":"2024-04-12T11:48:45.348788+0000","last_unstale":"2024-04-12T11:52:19.138945+0000","last_undegraded":"2024-04-12T11:52:19.138945+0000","last_fullsized":"2024-04-12T11:52:19.138945+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:15:04.501653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1876,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.129376+0000","last_change":"2024-04-12T11:48:45.351839+0000","last_active":"2024-04-12T11:52:19.129376+0000","last_peered":"2024-04-12T11:52:19.129376+0000","last_clean":"2024-04-12T11:52:19.129376+0000","last_became_active":"2024-04-12T11:48:45.351276+0000","last_became_peered":"2024-04-12T11:48:45.351276+0000","last_unstale":"2024-04-12T11:52:19.129376+0000","last_undegraded":"2024-04-12T11:52:19.129376+0000","last_fullsized":"2024-04-12T11:52:19.129376+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:23:03.106998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"249'161","reported_seq":1577,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482156+0000","last_change":"2024-04-12T11:49:07.233588+0000","last_active":"2024-04-12T11:52:02.482156+0000","last_peered":"2024-04-12T11:52:02.482156+0000","last_clean":"2024-04-12T11:52:02.482156+0000","last_became_active":"2024-04-12T11:49:07.233149+0000","last_became_peered":"2024-04-12T11:49:07.233149+0000","last_unstale":"2024-04-12T11:52:02.482156+0000","last_undegraded":"2024-04-12T11:52:02.482156+0000","last_fullsized":"2024-04-12T11:52:02.482156+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.0","version":"1259'10828","reported_seq":10915,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:27.594784+0000","last_change":"2024-04-12T11:50:11.541856+0000","last_active":"2024-04-12T11:52:27.594784+0000","last_peered":"2024-04-12T11:52:27.594784+0000","last_clean":"2024-04-12T11:52:27.594784+0000","last_became_active":"2024-04-12T11:50:11.530971+0000","last_became_peered":"2024-04-12T11:50:11.530971+0000","last_unstale":"2024-04-12T11:52:27.594784+0000","last_undegraded":"2024-04-12T11:52:27.594784+0000","last_fullsized":"2024-04-12T11:52:27.594784+0000","mapping_epoch":1226,"log_start":"1230'1000","ondisk_log_start":"1230'1000","created":1226,"last_epoch_clean":1249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":9828,"log_dups_size":0,"ondisk_log_size":9828,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-12T11:52:32.478133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30736384,"num_objects":469,"num_object_clones":0,"num_object_copies":1407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6378,"num_write_kb":219104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":879,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.129396+0000","last_change":"2024-04-12T11:34:59.853152+0000","last_active":"2024-04-12T11:52:19.129396+0000","last_peered":"2024-04-12T11:52:19.129396+0000","last_clean":"2024-04-12T11:52:19.129396+0000","last_became_active":"2024-04-12T11:34:59.852783+0000","last_became_peered":"2024-04-12T11:34:59.852783+0000","last_unstale":"2024-04-12T11:52:19.129396+0000","last_undegraded":"2024-04-12T11:52:19.129396+0000","last_fullsized":"2024-04-12T11:52:19.129396+0000","mapping_epoch":1111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:39:20.648910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1874,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.129772+0000","last_change":"2024-04-12T11:23:36.651736+0000","last_active":"2024-04-12T11:52:19.129772+0000","last_peered":"2024-04-12T11:52:19.129772+0000","last_clean":"2024-04-12T11:52:19.129772+0000","last_became_active":"2024-04-12T11:23:36.650872+0000","last_became_peered":"2024-04-12T11:23:36.650872+0000","last_unstale":"2024-04-12T11:52:19.129772+0000","last_undegraded":"2024-04-12T11:52:19.129772+0000","last_fullsized":"2024-04-12T11:52:19.129772+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:28:11.754124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1259'10622","reported_seq":14251,"reported_epoch":1259,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:52:27.362037+0000","last_change":"2024-04-12T11:50:12.567741+0000","last_active":"2024-04-12T11:52:27.362037+0000","last_peered":"2024-04-12T11:52:27.362037+0000","last_clean":"2024-04-12T11:48:44.327141+0000","last_became_active":"2024-04-12T11:50:12.555465+0000","last_became_peered":"2024-04-12T11:50:12.555465+0000","last_unstale":"2024-04-12T11:52:27.362037+0000","last_undegraded":"2024-04-12T11:50:12.515788+0000","last_fullsized":"2024-04-12T11:52:27.362037+0000","mapping_epoch":1249,"log_start":"1231'2812","ondisk_log_start":"1231'2812","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7810,"log_dups_size":0,"ondisk_log_size":7810,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29818880,"num_objects":455,"num_object_clones":0,"num_object_copies":1365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1031,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6234,"num_write_kb":214048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1785,"num_bytes_recovered":116981760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,3,7","objects":455}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1873,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.130838+0000","last_change":"2024-04-12T11:45:52.976139+0000","last_active":"2024-04-12T11:52:19.130838+0000","last_peered":"2024-04-12T11:52:19.130838+0000","last_clean":"2024-04-12T11:52:19.130838+0000","last_became_active":"2024-04-12T11:45:52.975803+0000","last_became_peered":"2024-04-12T11:45:52.975803+0000","last_unstale":"2024-04-12T11:52:19.130838+0000","last_undegraded":"2024-04-12T11:52:19.130838+0000","last_fullsized":"2024-04-12T11:52:19.130838+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:46.618143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1839,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.129885+0000","last_change":"2024-04-12T11:23:36.646615+0000","last_active":"2024-04-12T11:52:19.129885+0000","last_peered":"2024-04-12T11:52:19.129885+0000","last_clean":"2024-04-12T11:52:19.129885+0000","last_became_active":"2024-04-12T11:23:36.643763+0000","last_became_peered":"2024-04-12T11:23:36.643763+0000","last_unstale":"2024-04-12T11:52:19.129885+0000","last_undegraded":"2024-04-12T11:52:19.129885+0000","last_fullsized":"2024-04-12T11:52:19.129885+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:41:26.269139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"249'161","reported_seq":1577,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482593+0000","last_change":"2024-04-12T11:49:07.186288+0000","last_active":"2024-04-12T11:52:02.482593+0000","last_peered":"2024-04-12T11:52:02.482593+0000","last_clean":"2024-04-12T11:52:02.482593+0000","last_became_active":"2024-04-12T11:49:07.185683+0000","last_became_peered":"2024-04-12T11:49:07.185683+0000","last_unstale":"2024-04-12T11:52:02.482593+0000","last_undegraded":"2024-04-12T11:52:02.482593+0000","last_fullsized":"2024-04-12T11:52:02.482593+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.2","version":"1259'10761","reported_seq":14438,"reported_epoch":1259,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:52:28.104173+0000","last_change":"2024-04-12T11:50:13.408534+0000","last_active":"2024-04-12T11:52:28.104173+0000","last_peered":"2024-04-12T11:52:28.104173+0000","last_clean":"2024-04-12T11:48:44.392445+0000","last_became_active":"2024-04-12T11:50:12.553470+0000","last_became_peered":"2024-04-12T11:50:12.553470+0000","last_unstale":"2024-04-12T11:52:28.104173+0000","last_undegraded":"2024-04-12T11:50:12.510324+0000","last_fullsized":"2024-04-12T11:52:28.104173+0000","mapping_epoch":1249,"log_start":"1230'900","ondisk_log_start":"1230'900","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":9861,"log_dups_size":0,"ondisk_log_size":9861,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30605312,"num_objects":467,"num_object_clones":0,"num_object_copies":1401,"num_objects_missing_on_primary":1054,"num_objects_missing":1054,"num_objects_degraded":1054,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6320,"num_write_kb":217184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1796,"num_bytes_recovered":117637120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,3","objects":467}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.143436+0000","last_change":"2024-04-12T11:48:45.351852+0000","last_active":"2024-04-12T11:52:19.143436+0000","last_peered":"2024-04-12T11:52:19.143436+0000","last_clean":"2024-04-12T11:52:19.143436+0000","last_became_active":"2024-04-12T11:48:45.350286+0000","last_became_peered":"2024-04-12T11:48:45.350286+0000","last_unstale":"2024-04-12T11:52:19.143436+0000","last_undegraded":"2024-04-12T11:52:19.143436+0000","last_fullsized":"2024-04-12T11:52:19.143436+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:51:26.610188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1966,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.137654+0000","last_change":"2024-04-12T11:48:45.361378+0000","last_active":"2024-04-12T11:52:19.137654+0000","last_peered":"2024-04-12T11:52:19.137654+0000","last_clean":"2024-04-12T11:52:19.137654+0000","last_became_active":"2024-04-12T11:48:45.360917+0000","last_became_peered":"2024-04-12T11:48:45.360917+0000","last_unstale":"2024-04-12T11:52:19.137654+0000","last_undegraded":"2024-04-12T11:52:19.137654+0000","last_fullsized":"2024-04-12T11:52:19.137654+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:46:07.334663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"249'161","reported_seq":1577,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.483130+0000","last_change":"2024-04-12T11:49:07.185970+0000","last_active":"2024-04-12T11:52:02.483130+0000","last_peered":"2024-04-12T11:52:02.483130+0000","last_clean":"2024-04-12T11:52:02.483130+0000","last_became_active":"2024-04-12T11:49:07.185643+0000","last_became_peered":"2024-04-12T11:49:07.185643+0000","last_unstale":"2024-04-12T11:52:02.483130+0000","last_undegraded":"2024-04-12T11:52:02.483130+0000","last_fullsized":"2024-04-12T11:52:02.483130+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.f","version":"1259'12689","reported_seq":14034,"reported_epoch":1259,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-04-12T11:52:28.146497+0000","last_change":"2024-04-12T11:52:01.268505+0000","last_active":"2024-04-12T11:52:28.146497+0000","last_peered":"2024-04-12T11:52:28.146497+0000","last_clean":"2024-04-12T11:48:44.299270+0000","last_became_active":"2024-04-12T11:48:46.917768+0000","last_became_peered":"2024-04-12T11:48:46.917768+0000","last_unstale":"2024-04-12T11:52:28.146497+0000","last_undegraded":"2024-04-12T11:48:46.478160+0000","last_fullsized":"2024-04-12T11:48:46.335134+0000","mapping_epoch":1235,"log_start":"1232'4779","ondisk_log_start":"1232'4779","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7910,"log_dups_size":0,"ondisk_log_size":7910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:00:52.272820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59310080,"num_objects":905,"num_object_clones":0,"num_object_copies":2715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5075,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12689,"num_write_kb":435008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":647,"num_bytes_recovered":38731776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,5","objects":-4170}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1950,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.135500+0000","last_change":"2024-04-12T11:45:52.980673+0000","last_active":"2024-04-12T11:52:19.135500+0000","last_peered":"2024-04-12T11:52:19.135500+0000","last_clean":"2024-04-12T11:52:19.135500+0000","last_became_active":"2024-04-12T11:45:52.978978+0000","last_became_peered":"2024-04-12T11:45:52.978978+0000","last_unstale":"2024-04-12T11:52:19.135500+0000","last_undegraded":"2024-04-12T11:52:19.135500+0000","last_fullsized":"2024-04-12T11:52:19.135500+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:32:10.303905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1267,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.485898+0000","last_change":"2024-04-12T11:48:45.419824+0000","last_active":"2024-04-12T11:52:02.485898+0000","last_peered":"2024-04-12T11:52:02.485898+0000","last_clean":"2024-04-12T11:52:02.485898+0000","last_became_active":"2024-04-12T11:48:45.419375+0000","last_became_peered":"2024-04-12T11:48:45.419375+0000","last_unstale":"2024-04-12T11:52:02.485898+0000","last_undegraded":"2024-04-12T11:52:02.485898+0000","last_fullsized":"2024-04-12T11:52:02.485898+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:19:30.300350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1929,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.135554+0000","last_change":"2024-04-12T11:23:36.646757+0000","last_active":"2024-04-12T11:52:19.135554+0000","last_peered":"2024-04-12T11:52:19.135554+0000","last_clean":"2024-04-12T11:52:19.135554+0000","last_became_active":"2024-04-12T11:23:36.646372+0000","last_became_peered":"2024-04-12T11:23:36.646372+0000","last_unstale":"2024-04-12T11:52:19.135554+0000","last_undegraded":"2024-04-12T11:52:19.135554+0000","last_fullsized":"2024-04-12T11:52:19.135554+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:04.547672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"249'161","reported_seq":1576,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482384+0000","last_change":"2024-04-12T11:49:07.180601+0000","last_active":"2024-04-12T11:52:02.482384+0000","last_peered":"2024-04-12T11:52:02.482384+0000","last_clean":"2024-04-12T11:52:02.482384+0000","last_became_active":"2024-04-12T11:49:07.180271+0000","last_became_peered":"2024-04-12T11:49:07.180271+0000","last_unstale":"2024-04-12T11:52:02.482384+0000","last_undegraded":"2024-04-12T11:52:02.482384+0000","last_fullsized":"2024-04-12T11:52:02.482384+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184891,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.9","version":"1259'10667","reported_seq":10759,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:27.595305+0000","last_change":"2024-04-12T11:52:25.481881+0000","last_active":"2024-04-12T11:52:27.595305+0000","last_peered":"2024-04-12T11:52:27.595305+0000","last_clean":"2024-04-12T11:52:27.595305+0000","last_became_active":"2024-04-12T11:50:11.529949+0000","last_became_peered":"2024-04-12T11:50:11.529949+0000","last_unstale":"2024-04-12T11:52:27.595305+0000","last_undegraded":"2024-04-12T11:52:27.595305+0000","last_fullsized":"2024-04-12T11:52:27.595305+0000","mapping_epoch":1226,"log_start":"1230'2100","ondisk_log_start":"1230'2100","created":1226,"last_epoch_clean":1249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8567,"log_dups_size":0,"ondisk_log_size":8567,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-12T11:52:30.442954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30998528,"num_objects":473,"num_object_clones":0,"num_object_copies":1419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6270,"num_write_kb":215776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1221,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.485655+0000","last_change":"2024-04-12T11:22:19.620067+0000","last_active":"2024-04-12T11:52:02.485655+0000","last_peered":"2024-04-12T11:52:02.485655+0000","last_clean":"2024-04-12T11:52:02.485655+0000","last_became_active":"2024-04-12T11:22:19.619735+0000","last_became_peered":"2024-04-12T11:22:19.619735+0000","last_unstale":"2024-04-12T11:52:02.485655+0000","last_undegraded":"2024-04-12T11:52:02.485655+0000","last_fullsized":"2024-04-12T11:52:02.485655+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:02:29.026957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1861,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.129402+0000","last_change":"2024-04-12T11:23:36.652867+0000","last_active":"2024-04-12T11:52:19.129402+0000","last_peered":"2024-04-12T11:52:19.129402+0000","last_clean":"2024-04-12T11:52:19.129402+0000","last_became_active":"2024-04-12T11:23:36.650907+0000","last_became_peered":"2024-04-12T11:23:36.650907+0000","last_unstale":"2024-04-12T11:52:19.129402+0000","last_undegraded":"2024-04-12T11:52:19.129402+0000","last_fullsized":"2024-04-12T11:52:19.129402+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:38:20.258773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":173,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.145632+0000","last_change":"2024-04-12T11:48:45.418653+0000","last_active":"2024-04-12T11:52:19.145632+0000","last_peered":"2024-04-12T11:52:19.145632+0000","last_clean":"2024-04-12T11:52:19.145632+0000","last_became_active":"2024-04-12T11:48:45.418274+0000","last_became_peered":"2024-04-12T11:48:45.418274+0000","last_unstale":"2024-04-12T11:52:19.145632+0000","last_undegraded":"2024-04-12T11:52:19.145632+0000","last_fullsized":"2024-04-12T11:52:19.145632+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:39:24.012667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"249'161","reported_seq":1577,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482477+0000","last_change":"2024-04-12T11:49:07.185964+0000","last_active":"2024-04-12T11:52:02.482477+0000","last_peered":"2024-04-12T11:52:02.482477+0000","last_clean":"2024-04-12T11:52:02.482477+0000","last_became_active":"2024-04-12T11:49:07.185523+0000","last_became_peered":"2024-04-12T11:49:07.185523+0000","last_unstale":"2024-04-12T11:52:02.482477+0000","last_undegraded":"2024-04-12T11:52:02.482477+0000","last_fullsized":"2024-04-12T11:52:02.482477+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184891,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.a","version":"1259'12766","reported_seq":12810,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:28.130238+0000","last_change":"2024-04-12T11:47:31.341656+0000","last_active":"2024-04-12T11:52:28.130238+0000","last_peered":"2024-04-12T11:52:28.130238+0000","last_clean":"2024-04-12T11:52:28.130238+0000","last_became_active":"2024-04-12T11:47:31.341335+0000","last_became_peered":"2024-04-12T11:47:31.341335+0000","last_unstale":"2024-04-12T11:52:28.130238+0000","last_undegraded":"2024-04-12T11:52:28.130238+0000","last_fullsized":"2024-04-12T11:52:28.130238+0000","mapping_epoch":1226,"log_start":"1232'4796","ondisk_log_start":"1232'4796","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":7970,"log_dups_size":0,"ondisk_log_size":7970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:16:26.010127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57409536,"num_objects":876,"num_object_clones":0,"num_object_copies":2628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12766,"num_write_kb":436544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1926,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.151856+0000","last_change":"2024-04-12T11:48:45.417421+0000","last_active":"2024-04-12T11:52:19.151856+0000","last_peered":"2024-04-12T11:52:19.151856+0000","last_clean":"2024-04-12T11:52:19.151856+0000","last_became_active":"2024-04-12T11:48:45.417046+0000","last_became_peered":"2024-04-12T11:48:45.417046+0000","last_unstale":"2024-04-12T11:52:19.151856+0000","last_undegraded":"2024-04-12T11:52:19.151856+0000","last_fullsized":"2024-04-12T11:52:19.151856+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:17:29.993647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1930,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.138918+0000","last_change":"2024-04-12T11:22:40.580411+0000","last_active":"2024-04-12T11:52:19.138918+0000","last_peered":"2024-04-12T11:52:19.138918+0000","last_clean":"2024-04-12T11:52:19.138918+0000","last_became_active":"2024-04-12T11:22:40.580031+0000","last_became_peered":"2024-04-12T11:22:40.580031+0000","last_unstale":"2024-04-12T11:52:19.138918+0000","last_undegraded":"2024-04-12T11:52:19.138918+0000","last_fullsized":"2024-04-12T11:52:19.138918+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:42.882599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1931,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.135973+0000","last_change":"2024-04-12T11:22:19.610785+0000","last_active":"2024-04-12T11:52:19.135973+0000","last_peered":"2024-04-12T11:52:19.135973+0000","last_clean":"2024-04-12T11:52:19.135973+0000","last_became_active":"2024-04-12T11:22:19.610479+0000","last_became_peered":"2024-04-12T11:22:19.610479+0000","last_unstale":"2024-04-12T11:52:19.135973+0000","last_undegraded":"2024-04-12T11:52:19.135973+0000","last_fullsized":"2024-04-12T11:52:19.135973+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:31:49.391442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1259'10904","reported_seq":14578,"reported_epoch":1259,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:52:28.116608+0000","last_change":"2024-04-12T11:50:15.453471+0000","last_active":"2024-04-12T11:52:28.116608+0000","last_peered":"2024-04-12T11:52:28.116608+0000","last_clean":"2024-04-12T11:48:44.316482+0000","last_became_active":"2024-04-12T11:50:15.430643+0000","last_became_peered":"2024-04-12T11:50:15.430643+0000","last_unstale":"2024-04-12T11:52:28.116608+0000","last_undegraded":"2024-04-12T11:50:14.001707+0000","last_fullsized":"2024-04-12T11:52:28.116608+0000","mapping_epoch":1249,"log_start":"1230'2300","ondisk_log_start":"1230'2300","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8604,"log_dups_size":0,"ondisk_log_size":8604,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29229056,"num_objects":446,"num_object_clones":0,"num_object_copies":1338,"num_objects_missing_on_primary":1049,"num_objects_missing":1049,"num_objects_degraded":1049,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6399,"num_write_kb":219040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1798,"num_bytes_recovered":117702656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,3,5","objects":446}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1248,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.485798+0000","last_change":"2024-04-12T11:45:52.986504+0000","last_active":"2024-04-12T11:52:02.485798+0000","last_peered":"2024-04-12T11:52:02.485798+0000","last_clean":"2024-04-12T11:52:02.485798+0000","last_became_active":"2024-04-12T11:45:52.984346+0000","last_became_peered":"2024-04-12T11:45:52.984346+0000","last_unstale":"2024-04-12T11:52:02.485798+0000","last_undegraded":"2024-04-12T11:52:02.485798+0000","last_fullsized":"2024-04-12T11:52:02.485798+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:26.859024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1578,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482380+0000","last_change":"2024-04-12T11:49:07.218311+0000","last_active":"2024-04-12T11:52:02.482380+0000","last_peered":"2024-04-12T11:52:02.482380+0000","last_clean":"2024-04-12T11:52:02.482380+0000","last_became_active":"2024-04-12T11:49:07.217921+0000","last_became_peered":"2024-04-12T11:49:07.217921+0000","last_unstale":"2024-04-12T11:52:02.482380+0000","last_undegraded":"2024-04-12T11:52:02.482380+0000","last_fullsized":"2024-04-12T11:52:02.482380+0000","mapping_epoch":1210,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.152707+0000","last_change":"2024-04-12T11:48:45.352874+0000","last_active":"2024-04-12T11:52:19.152707+0000","last_peered":"2024-04-12T11:52:19.152707+0000","last_clean":"2024-04-12T11:52:19.152707+0000","last_became_active":"2024-04-12T11:48:45.352350+0000","last_became_peered":"2024-04-12T11:48:45.352350+0000","last_unstale":"2024-04-12T11:52:19.152707+0000","last_undegraded":"2024-04-12T11:52:19.152707+0000","last_fullsized":"2024-04-12T11:52:19.152707+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:44:46.925487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1227,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.552789+0000","last_change":"2024-04-12T11:45:53.016616+0000","last_active":"2024-04-12T11:52:02.552789+0000","last_peered":"2024-04-12T11:52:02.552789+0000","last_clean":"2024-04-12T11:52:02.552789+0000","last_became_active":"2024-04-12T11:45:53.016292+0000","last_became_peered":"2024-04-12T11:45:53.016292+0000","last_unstale":"2024-04-12T11:52:02.552789+0000","last_undegraded":"2024-04-12T11:52:02.552789+0000","last_fullsized":"2024-04-12T11:52:02.552789+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:53:29.382293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"249'161","reported_seq":1577,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482608+0000","last_change":"2024-04-12T11:49:07.187065+0000","last_active":"2024-04-12T11:52:02.482608+0000","last_peered":"2024-04-12T11:52:02.482608+0000","last_clean":"2024-04-12T11:52:02.482608+0000","last_became_active":"2024-04-12T11:49:07.186776+0000","last_became_peered":"2024-04-12T11:49:07.186776+0000","last_unstale":"2024-04-12T11:52:02.482608+0000","last_undegraded":"2024-04-12T11:52:02.482608+0000","last_fullsized":"2024-04-12T11:52:02.482608+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"1259'12386","reported_seq":12467,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:27.587009+0000","last_change":"2024-04-12T11:47:31.337491+0000","last_active":"2024-04-12T11:52:27.587009+0000","last_peered":"2024-04-12T11:52:27.587009+0000","last_clean":"2024-04-12T11:52:27.587009+0000","last_became_active":"2024-04-12T11:47:31.337137+0000","last_became_peered":"2024-04-12T11:47:31.337137+0000","last_unstale":"2024-04-12T11:52:27.587009+0000","last_undegraded":"2024-04-12T11:52:27.587009+0000","last_fullsized":"2024-04-12T11:52:27.587009+0000","mapping_epoch":1226,"log_start":"1237'6076","ondisk_log_start":"1237'6076","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6310,"log_dups_size":0,"ondisk_log_size":6310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:48:34.429072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60817408,"num_objects":928,"num_object_clones":0,"num_object_copies":2784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12386,"num_write_kb":426048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":63,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482293+0000","last_change":"2024-04-12T11:45:52.987902+0000","last_active":"2024-04-12T11:52:02.482293+0000","last_peered":"2024-04-12T11:52:02.482293+0000","last_clean":"2024-04-12T11:52:02.482293+0000","last_became_active":"2024-04-12T11:45:52.980281+0000","last_became_peered":"2024-04-12T11:45:52.980281+0000","last_unstale":"2024-04-12T11:52:02.482293+0000","last_undegraded":"2024-04-12T11:52:02.482293+0000","last_fullsized":"2024-04-12T11:52:02.482293+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:07:12.203048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1963,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.145485+0000","last_change":"2024-04-12T11:48:45.351359+0000","last_active":"2024-04-12T11:52:19.145485+0000","last_peered":"2024-04-12T11:52:19.145485+0000","last_clean":"2024-04-12T11:52:19.145485+0000","last_became_active":"2024-04-12T11:48:45.350809+0000","last_became_peered":"2024-04-12T11:48:45.350809+0000","last_unstale":"2024-04-12T11:52:19.145485+0000","last_undegraded":"2024-04-12T11:52:19.145485+0000","last_fullsized":"2024-04-12T11:52:19.145485+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:58:36.906714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1939,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.146146+0000","last_change":"2024-04-12T11:22:40.582674+0000","last_active":"2024-04-12T11:52:19.146146+0000","last_peered":"2024-04-12T11:52:19.146146+0000","last_clean":"2024-04-12T11:52:19.146146+0000","last_became_active":"2024-04-12T11:22:40.582356+0000","last_became_peered":"2024-04-12T11:22:40.582356+0000","last_unstale":"2024-04-12T11:52:19.146146+0000","last_undegraded":"2024-04-12T11:52:19.146146+0000","last_fullsized":"2024-04-12T11:52:19.146146+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:05:55.443374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"249'161","reported_seq":1577,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482920+0000","last_change":"2024-04-12T11:49:07.186438+0000","last_active":"2024-04-12T11:52:02.482920+0000","last_peered":"2024-04-12T11:52:02.482920+0000","last_clean":"2024-04-12T11:52:02.482920+0000","last_became_active":"2024-04-12T11:49:07.185633+0000","last_became_peered":"2024-04-12T11:49:07.185633+0000","last_unstale":"2024-04-12T11:52:02.482920+0000","last_undegraded":"2024-04-12T11:52:02.482920+0000","last_fullsized":"2024-04-12T11:52:02.482920+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.e","version":"1259'12615","reported_seq":12659,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:27.342544+0000","last_change":"2024-04-12T11:47:31.338497+0000","last_active":"2024-04-12T11:52:27.342544+0000","last_peered":"2024-04-12T11:52:27.342544+0000","last_clean":"2024-04-12T11:52:27.342544+0000","last_became_active":"2024-04-12T11:47:31.338160+0000","last_became_peered":"2024-04-12T11:47:31.338160+0000","last_unstale":"2024-04-12T11:52:27.342544+0000","last_undegraded":"2024-04-12T11:52:27.342544+0000","last_fullsized":"2024-04-12T11:52:27.342544+0000","mapping_epoch":1226,"log_start":"1242'7347","ondisk_log_start":"1242'7347","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:22:37.585891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63242240,"num_objects":965,"num_object_clones":0,"num_object_copies":2895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12615,"num_write_kb":434560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:12.371726+0000","last_change":"2024-04-12T11:45:53.005585+0000","last_active":"2024-04-12T11:52:12.371726+0000","last_peered":"2024-04-12T11:52:12.371726+0000","last_clean":"2024-04-12T11:52:12.371726+0000","last_became_active":"2024-04-12T11:45:52.987880+0000","last_became_peered":"2024-04-12T11:45:52.987880+0000","last_unstale":"2024-04-12T11:52:12.371726+0000","last_undegraded":"2024-04-12T11:52:12.371726+0000","last_fullsized":"2024-04-12T11:52:12.371726+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:09.819861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1929,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.135399+0000","last_change":"2024-04-12T11:22:40.581026+0000","last_active":"2024-04-12T11:52:19.135399+0000","last_peered":"2024-04-12T11:52:19.135399+0000","last_clean":"2024-04-12T11:52:19.135399+0000","last_became_active":"2024-04-12T11:22:40.580719+0000","last_became_peered":"2024-04-12T11:22:40.580719+0000","last_unstale":"2024-04-12T11:52:19.135399+0000","last_undegraded":"2024-04-12T11:52:19.135399+0000","last_fullsized":"2024-04-12T11:52:19.135399+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:37:29.097874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1934,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.147637+0000","last_change":"2024-04-12T11:45:52.982299+0000","last_active":"2024-04-12T11:52:19.147637+0000","last_peered":"2024-04-12T11:52:19.147637+0000","last_clean":"2024-04-12T11:52:19.147637+0000","last_became_active":"2024-04-12T11:45:52.981942+0000","last_became_peered":"2024-04-12T11:45:52.981942+0000","last_unstale":"2024-04-12T11:52:19.147637+0000","last_undegraded":"2024-04-12T11:52:19.147637+0000","last_fullsized":"2024-04-12T11:52:19.147637+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:18:13.711200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"249'161","reported_seq":1577,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482802+0000","last_change":"2024-04-12T11:49:07.187534+0000","last_active":"2024-04-12T11:52:02.482802+0000","last_peered":"2024-04-12T11:52:02.482802+0000","last_clean":"2024-04-12T11:52:02.482802+0000","last_became_active":"2024-04-12T11:49:07.187286+0000","last_became_peered":"2024-04-12T11:49:07.187286+0000","last_unstale":"2024-04-12T11:52:02.482802+0000","last_undegraded":"2024-04-12T11:52:02.482802+0000","last_fullsized":"2024-04-12T11:52:02.482802+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.d","version":"1259'12749","reported_seq":12794,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:27.352880+0000","last_change":"2024-04-12T11:47:31.339988+0000","last_active":"2024-04-12T11:52:27.352880+0000","last_peered":"2024-04-12T11:52:27.352880+0000","last_clean":"2024-04-12T11:52:27.352880+0000","last_became_active":"2024-04-12T11:47:31.339650+0000","last_became_peered":"2024-04-12T11:47:31.339650+0000","last_unstale":"2024-04-12T11:52:27.352880+0000","last_undegraded":"2024-04-12T11:52:27.352880+0000","last_fullsized":"2024-04-12T11:52:27.352880+0000","mapping_epoch":1226,"log_start":"1243'7494","ondisk_log_start":"1243'7494","created":1226,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":5255,"log_dups_size":0,"ondisk_log_size":5255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:02:57.059787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60948504,"num_objects":931,"num_object_clones":0,"num_object_copies":2793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":931,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12749,"num_write_kb":437697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":64,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.142623+0000","last_change":"2024-04-12T11:48:45.361376+0000","last_active":"2024-04-12T11:52:19.142623+0000","last_peered":"2024-04-12T11:52:19.142623+0000","last_clean":"2024-04-12T11:52:19.142623+0000","last_became_active":"2024-04-12T11:48:45.354494+0000","last_became_peered":"2024-04-12T11:48:45.354494+0000","last_unstale":"2024-04-12T11:52:19.142623+0000","last_undegraded":"2024-04-12T11:52:19.142623+0000","last_fullsized":"2024-04-12T11:52:19.142623+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:42:08.500124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1928,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.137920+0000","last_change":"2024-04-12T11:23:36.641436+0000","last_active":"2024-04-12T11:52:19.137920+0000","last_peered":"2024-04-12T11:52:19.137920+0000","last_clean":"2024-04-12T11:52:19.137920+0000","last_became_active":"2024-04-12T11:23:36.640814+0000","last_became_peered":"2024-04-12T11:23:36.640814+0000","last_unstale":"2024-04-12T11:52:19.137920+0000","last_undegraded":"2024-04-12T11:52:19.137920+0000","last_fullsized":"2024-04-12T11:52:19.137920+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:42:29.283281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.140185+0000","last_change":"2024-04-12T11:48:45.349034+0000","last_active":"2024-04-12T11:52:19.140185+0000","last_peered":"2024-04-12T11:52:19.140185+0000","last_clean":"2024-04-12T11:52:19.140185+0000","last_became_active":"2024-04-12T11:48:45.348579+0000","last_became_peered":"2024-04-12T11:48:45.348579+0000","last_unstale":"2024-04-12T11:52:19.140185+0000","last_undegraded":"2024-04-12T11:52:19.140185+0000","last_fullsized":"2024-04-12T11:52:19.140185+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:30:57.578035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"249'161","reported_seq":1577,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.482500+0000","last_change":"2024-04-12T11:49:07.199005+0000","last_active":"2024-04-12T11:52:02.482500+0000","last_peered":"2024-04-12T11:52:02.482500+0000","last_clean":"2024-04-12T11:52:02.482500+0000","last_became_active":"2024-04-12T11:49:07.198633+0000","last_became_peered":"2024-04-12T11:49:07.198633+0000","last_unstale":"2024-04-12T11:52:02.482500+0000","last_undegraded":"2024-04-12T11:52:02.482500+0000","last_fullsized":"2024-04-12T11:52:02.482500+0000","mapping_epoch":1240,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:09:21.719506+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":184890,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"1259'12445","reported_seq":23760,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:28.146096+0000","last_change":"2024-04-12T11:52:02.533715+0000","last_active":"2024-04-12T11:52:28.146096+0000","last_peered":"2024-04-12T11:52:28.146096+0000","last_clean":"2024-04-12T11:52:28.146096+0000","last_became_active":"2024-04-12T11:52:02.483230+0000","last_became_peered":"2024-04-12T11:52:02.483230+0000","last_unstale":"2024-04-12T11:52:28.146096+0000","last_undegraded":"2024-04-12T11:52:28.146096+0000","last_fullsized":"2024-04-12T11:52:28.146096+0000","mapping_epoch":1257,"log_start":"1236'5709","ondisk_log_start":"1236'5709","created":1226,"last_epoch_clean":1258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:12:45.401478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56295424,"num_objects":859,"num_object_clones":0,"num_object_copies":2577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12445,"num_write_kb":425728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5611,"num_bytes_recovered":231473152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1260,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.553095+0000","last_change":"2024-04-12T11:45:52.979745+0000","last_active":"2024-04-12T11:52:02.553095+0000","last_peered":"2024-04-12T11:52:02.553095+0000","last_clean":"2024-04-12T11:52:02.553095+0000","last_became_active":"2024-04-12T11:45:52.978936+0000","last_became_peered":"2024-04-12T11:45:52.978936+0000","last_unstale":"2024-04-12T11:52:02.553095+0000","last_undegraded":"2024-04-12T11:52:02.553095+0000","last_fullsized":"2024-04-12T11:52:02.553095+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:22.710430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":1942,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.145864+0000","last_change":"2024-04-12T11:23:36.639754+0000","last_active":"2024-04-12T11:52:19.145864+0000","last_peered":"2024-04-12T11:52:19.145864+0000","last_clean":"2024-04-12T11:52:19.145864+0000","last_became_active":"2024-04-12T11:23:36.639352+0000","last_became_peered":"2024-04-12T11:23:36.639352+0000","last_unstale":"2024-04-12T11:52:19.145864+0000","last_undegraded":"2024-04-12T11:52:19.145864+0000","last_fullsized":"2024-04-12T11:52:19.145864+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:19:22.846688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1848,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.130687+0000","last_change":"2024-04-12T11:23:36.647227+0000","last_active":"2024-04-12T11:52:19.130687+0000","last_peered":"2024-04-12T11:52:19.130687+0000","last_clean":"2024-04-12T11:52:19.130687+0000","last_became_active":"2024-04-12T11:23:36.645993+0000","last_became_peered":"2024-04-12T11:23:36.645993+0000","last_unstale":"2024-04-12T11:52:19.130687+0000","last_undegraded":"2024-04-12T11:52:19.130687+0000","last_fullsized":"2024-04-12T11:52:19.130687+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:44:02.882374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.11","version":"1259'10750","reported_seq":16295,"reported_epoch":1259,"state":"active+recovery_wait+remapped","last_fresh":"2024-04-12T11:52:25.296856+0000","last_change":"2024-04-12T11:50:13.533188+0000","last_active":"2024-04-12T11:52:25.296856+0000","last_peered":"2024-04-12T11:52:25.296856+0000","last_clean":"2024-04-12T11:48:44.315992+0000","last_became_active":"2024-04-12T11:50:13.518460+0000","last_became_peered":"2024-04-12T11:50:13.518460+0000","last_unstale":"2024-04-12T11:52:25.296856+0000","last_undegraded":"2024-04-12T11:52:25.296856+0000","last_fullsized":"2024-04-12T11:52:25.296856+0000","mapping_epoch":1250,"log_start":"1230'500","ondisk_log_start":"1230'500","created":1248,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":10250,"log_dups_size":0,"ondisk_log_size":10250,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33357824,"num_objects":509,"num_object_clones":0,"num_object_copies":1527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1519,"num_objects_unfound":0,"num_objects_dirty":509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6286,"num_write_kb":217440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1367,"num_bytes_recovered":89587712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[4,6,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":-1010}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":1943,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.147533+0000","last_change":"2024-04-12T11:45:52.979689+0000","last_active":"2024-04-12T11:52:19.147533+0000","last_peered":"2024-04-12T11:52:19.147533+0000","last_clean":"2024-04-12T11:52:19.147533+0000","last_became_active":"2024-04-12T11:45:52.979286+0000","last_became_peered":"2024-04-12T11:45:52.979286+0000","last_unstale":"2024-04-12T11:52:19.147533+0000","last_undegraded":"2024-04-12T11:52:19.147533+0000","last_fullsized":"2024-04-12T11:52:19.147533+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:10.453472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":187,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.485840+0000","last_change":"2024-04-12T11:48:45.344748+0000","last_active":"2024-04-12T11:52:02.485840+0000","last_peered":"2024-04-12T11:52:02.485840+0000","last_clean":"2024-04-12T11:52:02.485840+0000","last_became_active":"2024-04-12T11:48:45.344290+0000","last_became_peered":"2024-04-12T11:48:45.344290+0000","last_unstale":"2024-04-12T11:52:02.485840+0000","last_undegraded":"2024-04-12T11:52:02.485840+0000","last_fullsized":"2024-04-12T11:52:02.485840+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:13.440125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.16","version":"1259'11054","reported_seq":11105,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:28.049070+0000","last_change":"2024-04-12T11:50:11.577866+0000","last_active":"2024-04-12T11:52:28.049070+0000","last_peered":"2024-04-12T11:52:28.049070+0000","last_clean":"2024-04-12T11:52:28.049070+0000","last_became_active":"2024-04-12T11:50:11.565578+0000","last_became_peered":"2024-04-12T11:50:11.565578+0000","last_unstale":"2024-04-12T11:52:28.049070+0000","last_undegraded":"2024-04-12T11:52:28.049070+0000","last_fullsized":"2024-04-12T11:52:28.049070+0000","mapping_epoch":1248,"log_start":"0'0","ondisk_log_start":"0'0","created":1248,"last_epoch_clean":1249,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":11054,"log_dups_size":0,"ondisk_log_size":11054,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28540928,"num_objects":436,"num_object_clones":0,"num_object_copies":1308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6481,"num_write_kb":221312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":1230,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.485755+0000","last_change":"2024-04-12T11:23:36.640856+0000","last_active":"2024-04-12T11:52:02.485755+0000","last_peered":"2024-04-12T11:52:02.485755+0000","last_clean":"2024-04-12T11:52:02.485755+0000","last_became_active":"2024-04-12T11:23:36.640459+0000","last_became_peered":"2024-04-12T11:23:36.640459+0000","last_unstale":"2024-04-12T11:52:02.485755+0000","last_undegraded":"2024-04-12T11:52:02.485755+0000","last_fullsized":"2024-04-12T11:52:02.485755+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:27:08.738704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1899,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.130191+0000","last_change":"2024-04-12T11:48:45.419685+0000","last_active":"2024-04-12T11:52:19.130191+0000","last_peered":"2024-04-12T11:52:19.130191+0000","last_clean":"2024-04-12T11:52:19.130191+0000","last_became_active":"2024-04-12T11:48:45.419344+0000","last_became_peered":"2024-04-12T11:48:45.419344+0000","last_unstale":"2024-04-12T11:52:19.130191+0000","last_undegraded":"2024-04-12T11:52:19.130191+0000","last_fullsized":"2024-04-12T11:52:19.130191+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:24:01.383355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.17","version":"1259'10831","reported_seq":14589,"reported_epoch":1259,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:52:28.114099+0000","last_change":"2024-04-12T11:50:11.636315+0000","last_active":"2024-04-12T11:52:28.114099+0000","last_peered":"2024-04-12T11:52:28.114099+0000","last_clean":"2024-04-12T11:48:44.337474+0000","last_became_active":"2024-04-12T11:50:11.624315+0000","last_became_peered":"2024-04-12T11:50:11.624315+0000","last_unstale":"2024-04-12T11:52:28.114099+0000","last_undegraded":"2024-04-12T11:50:11.584165+0000","last_fullsized":"2024-04-12T11:52:28.114099+0000","mapping_epoch":1248,"log_start":"0'0","ondisk_log_start":"0'0","created":1248,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":10831,"log_dups_size":0,"ondisk_log_size":10831,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25755648,"num_objects":393,"num_object_clones":0,"num_object_copies":1179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6384,"num_write_kb":216864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1850,"num_bytes_recovered":121241600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"3,5,7","objects":393}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":1215,"reported_epoch":1258,"state":"active+clean","last_fresh":"2024-04-12T11:52:02.485839+0000","last_change":"2024-04-12T11:23:36.645502+0000","last_active":"2024-04-12T11:52:02.485839+0000","last_peered":"2024-04-12T11:52:02.485839+0000","last_clean":"2024-04-12T11:52:02.485839+0000","last_became_active":"2024-04-12T11:23:36.645122+0000","last_became_peered":"2024-04-12T11:23:36.645122+0000","last_unstale":"2024-04-12T11:52:02.485839+0000","last_undegraded":"2024-04-12T11:52:02.485839+0000","last_fullsized":"2024-04-12T11:52:02.485839+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:22:52.342854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1890,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.129604+0000","last_change":"2024-04-12T11:48:45.351004+0000","last_active":"2024-04-12T11:52:19.129604+0000","last_peered":"2024-04-12T11:52:19.129604+0000","last_clean":"2024-04-12T11:52:19.129604+0000","last_became_active":"2024-04-12T11:48:45.350567+0000","last_became_peered":"2024-04-12T11:48:45.350567+0000","last_unstale":"2024-04-12T11:52:19.129604+0000","last_undegraded":"2024-04-12T11:52:19.129604+0000","last_fullsized":"2024-04-12T11:52:19.129604+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:48:43.488302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"1259'10958","reported_seq":14730,"reported_epoch":1259,"state":"active+recovery_wait+degraded","last_fresh":"2024-04-12T11:52:27.578260+0000","last_change":"2024-04-12T11:50:11.669276+0000","last_active":"2024-04-12T11:52:27.578260+0000","last_peered":"2024-04-12T11:52:27.578260+0000","last_clean":"2024-04-12T11:48:44.288786+0000","last_became_active":"2024-04-12T11:50:11.654833+0000","last_became_peered":"2024-04-12T11:50:11.654833+0000","last_unstale":"2024-04-12T11:52:27.578260+0000","last_undegraded":"2024-04-12T11:50:11.606428+0000","last_fullsized":"2024-04-12T11:52:27.578260+0000","mapping_epoch":1248,"log_start":"1230'1100","ondisk_log_start":"1230'1100","created":1248,"last_epoch_clean":1227,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":9858,"log_dups_size":0,"ondisk_log_size":9858,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33030144,"num_objects":504,"num_object_clones":0,"num_object_copies":1512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1072,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6434,"num_write_kb":222016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1843,"num_bytes_recovered":120782848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"1,3,4","objects":504}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.140741+0000","last_change":"2024-04-12T11:48:45.418133+0000","last_active":"2024-04-12T11:52:19.140741+0000","last_peered":"2024-04-12T11:52:19.140741+0000","last_clean":"2024-04-12T11:52:19.140741+0000","last_became_active":"2024-04-12T11:48:45.417690+0000","last_became_peered":"2024-04-12T11:48:45.417690+0000","last_unstale":"2024-04-12T11:52:19.140741+0000","last_undegraded":"2024-04-12T11:52:19.140741+0000","last_fullsized":"2024-04-12T11:52:19.140741+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:09:21.940487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":60,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.139257+0000","last_change":"2024-04-12T11:48:45.418845+0000","last_active":"2024-04-12T11:52:19.139257+0000","last_peered":"2024-04-12T11:52:19.139257+0000","last_clean":"2024-04-12T11:52:19.139257+0000","last_became_active":"2024-04-12T11:48:45.417891+0000","last_became_peered":"2024-04-12T11:48:45.417891+0000","last_unstale":"2024-04-12T11:52:19.139257+0000","last_undegraded":"2024-04-12T11:52:19.139257+0000","last_fullsized":"2024-04-12T11:52:19.139257+0000","mapping_epoch":1234,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:12:05.184852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.15","version":"1259'10877","reported_seq":10966,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:27.543681+0000","last_change":"2024-04-12T11:52:22.438616+0000","last_active":"2024-04-12T11:52:27.543681+0000","last_peered":"2024-04-12T11:52:27.543681+0000","last_clean":"2024-04-12T11:52:27.543681+0000","last_became_active":"2024-04-12T11:50:11.556771+0000","last_became_peered":"2024-04-12T11:50:11.556771+0000","last_unstale":"2024-04-12T11:52:27.543681+0000","last_undegraded":"2024-04-12T11:52:27.543681+0000","last_fullsized":"2024-04-12T11:52:27.543681+0000","mapping_epoch":1248,"log_start":"1230'2300","ondisk_log_start":"1230'2300","created":1248,"last_epoch_clean":1249,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:47:30.319080+0000","last_clean_scrub_stamp":"2024-04-12T11:47:30.319080+0000","objects_scrubbed":0,"log_size":8577,"log_dups_size":0,"ondisk_log_size":8577,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34242560,"num_objects":523,"num_object_clones":0,"num_object_copies":1569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6359,"num_write_kb":220192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":1972,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.146532+0000","last_change":"2024-04-12T11:43:26.370162+0000","last_active":"2024-04-12T11:52:19.146532+0000","last_peered":"2024-04-12T11:52:19.146532+0000","last_clean":"2024-04-12T11:52:19.146532+0000","last_became_active":"2024-04-12T11:43:26.352240+0000","last_became_peered":"2024-04-12T11:43:26.352240+0000","last_unstale":"2024-04-12T11:52:19.146532+0000","last_undegraded":"2024-04-12T11:52:19.146532+0000","last_fullsized":"2024-04-12T11:52:19.146532+0000","mapping_epoch":1190,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:34:09.364519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1838,"reported_epoch":1259,"state":"active+clean","last_fresh":"2024-04-12T11:52:19.128919+0000","last_change":"2024-04-12T11:23:36.637907+0000","last_active":"2024-04-12T11:52:19.128919+0000","last_peered":"2024-04-12T11:52:19.128919+0000","last_clean":"2024-04-12T11:52:19.128919+0000","last_became_active":"2024-04-12T11:23:36.637554+0000","last_became_peered":"2024-04-12T11:23:36.637554+0000","last_unstale":"2024-04-12T11:52:19.128919+0000","last_undegraded":"2024-04-12T11:52:19.128919+0000","last_fullsized":"2024-04-12T11:52:19.128919+0000","mapping_epoch":611,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:06:43.577660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":11724,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25048,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2033792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":4},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":968949784,"num_objects":14786,"num_object_clones":0,"num_object_copies":44358,"num_objects_missing_on_primary":4257,"num_objects_missing":4257,"num_objects_degraded":15643,"num_objects_misplaced":3029,"num_objects_unfound":0,"num_objects_dirty":14786,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202764,"num_write_kb":6961537,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27007,"num_bytes_recovered":1629552640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5260783616,"data_stored":5260771400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":229955,"ondisk_log_size":229955,"up":78,"acting":77,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234481,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1948596,"kb_used_data":1299756,"kb_used_omap":1,"kb_used_meta":648766,"kb_avail":91796556,"statfs":{"total":95995035648,"available":93999673344,"internally_reserved":0,"allocated":1330950144,"data_stored":1328436337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":664336843},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496118,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":638888,"kb_used_data":192736,"kb_used_omap":1,"kb_used_meta":446142,"kb_avail":93106264,"statfs":{"total":95995035648,"available":95340814336,"internally_reserved":0,"allocated":197361664,"data_stored":194860557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":456849864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536428,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1138132,"kb_used_data":550860,"kb_used_omap":30,"kb_used_meta":587233,"kb_avail":92607020,"statfs":{"total":95995035648,"available":94829588480,"internally_reserved":0,"allocated":564080640,"data_stored":561594097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":601326914},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190512,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1296284,"kb_used_data":624212,"kb_used_omap":43,"kb_used_meta":672020,"kb_avail":92448868,"statfs":{"total":95995035648,"available":94667640832,"internally_reserved":0,"allocated":639193088,"data_stored":636699477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":688148523},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849639,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1652820,"kb_used_data":948876,"kb_used_omap":36,"kb_used_meta":703899,"kb_avail":92092332,"statfs":{"total":95995035648,"available":94302547968,"internally_reserved":0,"allocated":971649024,"data_stored":969162491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":720792756},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":590,"seq":2534030704992,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1244308,"kb_used_data":476748,"kb_used_omap":17,"kb_used_meta":767534,"kb_avail":92500844,"statfs":{"total":95995035648,"available":94720864256,"internally_reserved":0,"allocated":488189952,"data_stored":485703475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17929,"internal_metadata":785955319},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018204,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":836584,"kb_used_data":281056,"kb_used_omap":42,"kb_used_meta":555477,"kb_avail":92908568,"statfs":{"total":95995035648,"available":95138373632,"internally_reserved":0,"allocated":287801344,"data_stored":285299199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":568809272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757754,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1578076,"kb_used_data":888916,"kb_used_omap":1,"kb_used_meta":688958,"kb_avail":92167076,"statfs":{"total":95995035648,"available":94379085824,"internally_reserved":0,"allocated":910249984,"data_stored":907755889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":705493446},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,5,2,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5032,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":627834880,"data_stored":627834880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":551620608,"data_stored":551616536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":961216512,"data_stored":961216512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":471007232,"data_stored":471007232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":276496384,"data_stored":276496384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":892407808,"data_stored":892403736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":187891712,"data_stored":187891712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1292308480,"data_stored":1292304408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T11:52:34.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-04-12T11:52:34.245 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:34.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:34.345 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:34.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:34.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:34.447 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:34.482 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:34.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:34.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:34.584 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:34.618 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:34.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:34.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:34.720 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:34.757 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:34.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:34.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:34.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:34.894 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:34.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:34.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:34.996 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:35.032 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:35.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:35.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:35.134 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:35.144 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:35.147 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:52:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:52:34] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2024-04-12T11:52:35.166 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:35.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:35.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:35.300 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:35.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:35.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:35.401 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:35.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:35 smithi029 ceph-mon[97787]: pgmap v2070: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 644 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 15530/30948 objects degraded (50.181%); 3029/30948 objects misplaced (9.787%); 8.3 MiB/s, 146 objects/s recovering 2024-04-12T11:52:35.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:35 smithi029 ceph-mon[93650]: pgmap v2070: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 644 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 15530/30948 objects degraded (50.181%); 3029/30948 objects misplaced (9.787%); 8.3 MiB/s, 146 objects/s recovering 2024-04-12T11:52:35.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:35.442 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:35.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:35.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:35 smithi087 ceph-mon[89014]: pgmap v2070: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 644 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 966 op/s; 15530/30948 objects degraded (50.181%); 3029/30948 objects misplaced (9.787%); 8.3 MiB/s, 146 objects/s recovering 2024-04-12T11:52:35.542 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:35.574 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:35.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:35.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:35.714 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:35.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:35.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:35.815 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:35.847 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:35.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:35.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:35.948 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:35.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:35.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:36.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:36.126 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:36.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:36.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:36.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:36.262 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:36.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:36.364 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:36.391 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:36.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:36.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:36.493 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:36.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:36.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:36.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:36.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:36.669 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:36.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:36.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:36.770 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:36.816 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:36.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:36.917 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:36.956 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:36.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:36.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:37.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:37.099 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:37.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:37.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:37.200 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:37.232 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:37.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:37.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:37.333 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:37.357 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:37.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:37.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:37.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:37 smithi087 ceph-mon[89014]: pgmap v2071: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 629 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15530/30237 objects degraded (51.361%); 3029/30237 objects misplaced (10.018%); 2.0 MiB/s, 45 objects/s recovering 2024-04-12T11:52:37.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:37.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:37.502 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:37.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:37.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:37 smithi029 ceph-mon[97787]: pgmap v2071: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 629 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15530/30237 objects degraded (51.361%); 3029/30237 objects misplaced (10.018%); 2.0 MiB/s, 45 objects/s recovering 2024-04-12T11:52:37.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:37 smithi029 ceph-mon[93650]: pgmap v2071: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 629 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 15530/30237 objects degraded (51.361%); 3029/30237 objects misplaced (10.018%); 2.0 MiB/s, 45 objects/s recovering 2024-04-12T11:52:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:37.602 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:37.632 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:37.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:37.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:37.664 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:37.734 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:37.773 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:37.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:37.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:37.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:37.913 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:37.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:37.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:38.015 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:38.063 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:38.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:38.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:38.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:38.202 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:38.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:38.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:38.304 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:38.361 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:38.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:38.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15530/30237 objects degraded (51.361%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:38.372 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15530/30237 objects degraded (51.361%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:38.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:38.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15530/30237 objects degraded (51.361%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:38.504 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:38.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:38.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:38.606 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:38.638 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:38.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:38.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:38.740 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:38.775 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:38.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:38.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:38.877 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:38.910 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:38.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:39.012 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:39.043 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:39.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:39.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:39.144 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:39.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:39.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:39.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:39.204 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:39 smithi029 ceph-mon[97787]: pgmap v2072: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 484 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 15530/23274 objects degraded (66.727%); 3029/23274 objects misplaced (13.015%); 2.0 MiB/s, 45 objects/s recovering 2024-04-12T11:52:39.205 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:39 smithi029 ceph-mon[93650]: pgmap v2072: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 484 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 15530/23274 objects degraded (66.727%); 3029/23274 objects misplaced (13.015%); 2.0 MiB/s, 45 objects/s recovering 2024-04-12T11:52:39.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:39.332 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:39.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:39.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:39.434 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:39.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:39 smithi087 ceph-mon[89014]: pgmap v2072: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 484 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 15530/23274 objects degraded (66.727%); 3029/23274 objects misplaced (13.015%); 2.0 MiB/s, 45 objects/s recovering 2024-04-12T11:52:39.470 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:39.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:39.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:39.571 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:39.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:39.611 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:39.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:39.712 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:39.745 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:39.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:39.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:39.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:39.890 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:39.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:39.991 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:40.026 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:40.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:40.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:40.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:40.163 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:40.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:40.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:40.194 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:52:40.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:40.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:40.300 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:40.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:40.401 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:40.436 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:40.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:40.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:40 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1865289809' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:52:40.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:40 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:52:40.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:40 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1865289809' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:52:40.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:40 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:52:40.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:40 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1865289809' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:52:40.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:40 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:52:40.538 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:40.577 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:40.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:40.678 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:40.714 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:40.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:40.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:40.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:40.853 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:40.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:40.955 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:40.992 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:40.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:40.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:41.094 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:41.142 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:41.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:41.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:41.244 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:41.258 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:41 smithi029 ceph-mon[97787]: pgmap v2073: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 363 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 15446/17448 objects degraded (88.526%); 3029/17448 objects misplaced (17.360%); 1.1 MiB/s, 38 objects/s recovering 2024-04-12T11:52:41.258 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:41 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:52:41.258 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:41 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:52:41.258 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:41 smithi029 ceph-mon[97787]: osdmap e1260: 8 total, 8 up, 8 in 2024-04-12T11:52:41.258 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:41 smithi029 ceph-mon[93650]: pgmap v2073: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 363 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 15446/17448 objects degraded (88.526%); 3029/17448 objects misplaced (17.360%); 1.1 MiB/s, 38 objects/s recovering 2024-04-12T11:52:41.259 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:41 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:52:41.259 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:41 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:52:41.259 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:41 smithi029 ceph-mon[93650]: osdmap e1260: 8 total, 8 up, 8 in 2024-04-12T11:52:41.279 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:41.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:41.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:41.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:41.428 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:41.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:41.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:41.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:41 smithi087 ceph-mon[89014]: pgmap v2073: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 363 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 15446/17448 objects degraded (88.526%); 3029/17448 objects misplaced (17.360%); 1.1 MiB/s, 38 objects/s recovering 2024-04-12T11:52:41.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:41 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:52:41.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:41 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:52:41.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:41 smithi087 ceph-mon[89014]: osdmap e1260: 8 total, 8 up, 8 in 2024-04-12T11:52:41.489 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:41.531 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:41.562 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:41.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:41.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:41.567 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:52:41.567 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:52:41.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:41.706 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:41.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:41.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:41.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:41.839 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:41.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:41.940 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:41.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:41.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:42.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:42.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:42.109 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:42.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:42.211 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:42.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:42.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:42.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:42.354 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:52:42.362 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:42.397 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:52:42.397 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:42.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:42.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:42.436 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:52:42.477 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:52:42.499 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:42.521 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:52:42.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:42.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:42.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:42.561 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:52:42.603 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:52:42.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:42.646 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:52:42.652 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:52:42.652 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:52:42.652 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:52:42.652 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:52:42.652 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:52:42.652 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:52:42.652 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:52:42.652 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:52:42.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:42.681 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:42.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:52:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:52:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:52:42.782 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:42.806 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:42.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:42.907 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:42.945 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:42.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:42.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:43.047 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:43.082 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:43.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:43.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:43.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:43.219 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:43.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:43.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:43.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:52:43.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:43 smithi029 ceph-mon[93650]: pgmap v2075: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 344 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 15446/16566 objects degraded (93.239%); 3029/16566 objects misplaced (18.284%); 0 B/s, 19 objects/s recovering 2024-04-12T11:52:43.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:43 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/442279606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:43.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15446/16566 objects degraded (93.239%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:43 smithi029 ceph-mon[97787]: pgmap v2075: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 344 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 15446/16566 objects degraded (93.239%); 3029/16566 objects misplaced (18.284%); 0 B/s, 19 objects/s recovering 2024-04-12T11:52:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:43 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/442279606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15446/16566 objects degraded (93.239%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:43.321 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:43.354 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:43.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:43.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:43.422 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-04-12T11:52:43.456 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:43.457 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:43 smithi087 ceph-mon[89014]: pgmap v2075: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 344 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 15446/16566 objects degraded (93.239%); 3029/16566 objects misplaced (18.284%); 0 B/s, 19 objects/s recovering 2024-04-12T11:52:43.457 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:43 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/442279606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:43.457 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15446/16566 objects degraded (93.239%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:43.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:43.543 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:43.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:43.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:43.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:43.688 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:43.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:43.748 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:52:43.789 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:43.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:43.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:43.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:43.931 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:43.961 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:43.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:43.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:43.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 108774 objects 2024-04-12T11:52:43.965 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :208.975 2024-04-12T11:52:44.063 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:44.086 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:44.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:44.188 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:44.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:44.212 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:44.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:44.313 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:44.337 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:44.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:44.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:44.439 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:44.464 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:44.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:44.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:44.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:44.591 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:44.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:44.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:44.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:44.717 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:44.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:44.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:44.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:44.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:44.843 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:44.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:44.944 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:44.968 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:44.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:44.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:45.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:52:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:52:44] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.43.0" 2024-04-12T11:52:45.070 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:45.094 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:45.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:45.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:45.219 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:45.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:45.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:45.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:45.345 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:45.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:45.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:45.383 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:45.447 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:45.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:45 smithi087 ceph-mon[89014]: pgmap v2076: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 73 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 15346/3564 objects degraded (430.584%); 3029/3564 objects misplaced (84.989%); 0 B/s, 18 objects/s recovering 2024-04-12T11:52:45.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:45.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:45.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:45.504 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:45 smithi029 ceph-mon[93650]: pgmap v2076: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 73 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 15346/3564 objects degraded (430.584%); 3029/3564 objects misplaced (84.989%); 0 B/s, 18 objects/s recovering 2024-04-12T11:52:45.505 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:45 smithi029 ceph-mon[97787]: pgmap v2076: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 73 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 15346/3564 objects degraded (430.584%); 3029/3564 objects misplaced (84.989%); 0 B/s, 18 objects/s recovering 2024-04-12T11:52:45.573 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:45.596 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:45.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:45.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:45.698 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:45.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:45.723 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:45.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:45.824 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:45.847 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:45.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:45.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:45.949 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:45.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:45.974 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:45.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:46.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:46.101 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:46.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:46.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:46.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:52:46.203 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:46.228 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:46.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:46.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:46.329 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:46.353 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:46.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:46.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:46.455 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:46.478 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:46.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:46.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:46.580 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:46.604 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:46.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:46.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:46.631 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:52:46.631 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:52:46.631 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T11:52:46.632 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:52:46.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:46.730 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:46.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:46.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:46.832 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:46.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:46.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:46.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:46.961 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:46.985 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:46.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:46.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:47.086 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:47.111 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:47.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:47.213 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:47.238 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:47.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:47.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:47.339 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:47.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:47.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:47.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:47.374 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:47 smithi029 ceph-mon[93650]: pgmap v2077: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 61 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 15346/2961 objects degraded (518.271%); 3029/2961 objects misplaced (102.297%); 0 B/s, 18 objects/s recovering 2024-04-12T11:52:47.374 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:47 smithi029 ceph-mon[97787]: pgmap v2077: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 61 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 15346/2961 objects degraded (518.271%); 3029/2961 objects misplaced (102.297%); 0 B/s, 18 objects/s recovering 2024-04-12T11:52:47.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:47.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:47 smithi087 ceph-mon[89014]: pgmap v2077: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 61 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 15346/2961 objects degraded (518.271%); 3029/2961 objects misplaced (102.297%); 0 B/s, 18 objects/s recovering 2024-04-12T11:52:47.491 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:47.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:47.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:47.506 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-04-12T11:52:47.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:47.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:47.618 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:47.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:47.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:47.742 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:47.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:47.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:47.788 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:52:47.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:47.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:47.869 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:47.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:47.913 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 11:52:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=cleanup t=2024-04-12T11:52:47.555075183Z level=info msg="Completed cleanup jobs" duration=20.805545ms 2024-04-12T11:52:47.970 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:47.995 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:47.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:47.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:48.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:48.119 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:48.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:48.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:48.218 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 1 pgp_num to 11 2024-04-12T11:52:48.221 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:48.247 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:48.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:48.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:48.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:48.373 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:48.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:48.464 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:48 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3962389483' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2024-04-12T11:52:48.465 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:48 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2024-04-12T11:52:48.465 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15346/2961 objects degraded (518.271%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:48.474 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:48.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:48.498 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:48.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:48.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:48 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3962389483' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2024-04-12T11:52:48.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:48 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2024-04-12T11:52:48.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15346/2961 objects degraded (518.271%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:48.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:48 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3962389483' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2024-04-12T11:52:48.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:48 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2024-04-12T11:52:48.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15346/2961 objects degraded (518.271%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:48.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:48.624 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:48.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:48.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:48.749 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:48.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:48.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:48.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:48.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:48.874 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:48.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:48.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:49.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:49.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:49.103 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:49.127 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:49.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:49.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:49.229 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:49.254 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:49.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:49.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:49.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:49.380 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:49.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:49.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:49.442 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:49 smithi087 ceph-mon[89014]: pgmap v2078: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 28 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 15346/1395 objects degraded (1100.072%); 3029/1395 objects misplaced (217.133%); 0 B/s, 18 objects/s recovering 2024-04-12T11:52:49.442 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:49 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-04-12T11:52:49.442 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:49 smithi087 ceph-mon[89014]: osdmap e1261: 8 total, 8 up, 8 in 2024-04-12T11:52:49.481 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:49.505 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:49.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:49.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:49 smithi029 ceph-mon[93650]: pgmap v2078: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 28 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 15346/1395 objects degraded (1100.072%); 3029/1395 objects misplaced (217.133%); 0 B/s, 18 objects/s recovering 2024-04-12T11:52:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:49 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-04-12T11:52:49.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:49 smithi029 ceph-mon[93650]: osdmap e1261: 8 total, 8 up, 8 in 2024-04-12T11:52:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:49 smithi029 ceph-mon[97787]: pgmap v2078: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 28 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 15346/1395 objects degraded (1100.072%); 3029/1395 objects misplaced (217.133%); 0 B/s, 18 objects/s recovering 2024-04-12T11:52:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:49 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-04-12T11:52:49.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:49 smithi029 ceph-mon[97787]: osdmap e1261: 8 total, 8 up, 8 in 2024-04-12T11:52:49.607 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:49.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:49.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:49.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:49.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:49.744 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:52:49.744 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:52:49.745 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:52:49.745 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:52:49.745 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:52:49.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:49.758 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:49.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:49.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:49.884 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:49.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:49.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:49.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:50.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:50.011 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:50.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:50.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:50.135 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:50.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:50.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:50.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:50.263 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:50.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:50.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:50.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:50.388 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:50.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:50.412 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-04-12T11:52:50.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:50.494 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-04-12T11:52:50.494 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-04-12T11:52:50.512 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:50.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:50.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:50.608 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:50.614 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:50.625 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:52:50.638 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:50.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:50.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:50.739 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:50.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:50.764 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:50.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:50.865 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:50.888 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:50.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:50.990 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:51.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:51.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:51.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:51.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:51.140 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:51.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:51.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:51.241 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:51.265 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:51.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:51.366 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:51.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:51.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:51.454 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:51 smithi029 ceph-mon[97787]: pgmap v2080: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 14885/612 objects degraded (2432.190%); 3029/612 objects misplaced (494.935%); 0 B/s, 56 objects/s recovering 2024-04-12T11:52:51.454 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:51 smithi029 ceph-mon[93650]: pgmap v2080: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 14885/612 objects degraded (2432.190%); 3029/612 objects misplaced (494.935%); 0 B/s, 56 objects/s recovering 2024-04-12T11:52:51.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:51 smithi087 ceph-mon[89014]: pgmap v2080: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 14885/612 objects degraded (2432.190%); 3029/612 objects misplaced (494.935%); 0 B/s, 56 objects/s recovering 2024-04-12T11:52:51.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:51.516 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:51.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:51.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:51.618 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:51.641 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:51.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:51.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:51.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:51.767 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:51.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:51.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:51.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:51.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:51.893 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:51.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:51.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:52.018 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:52.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:52.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:52.119 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:52.142 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:52.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:52.244 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:52.268 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:52.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:52.369 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:52.393 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:52.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:52.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:52.495 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:52.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:52.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:52.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:52.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:52.644 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:52.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:52.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:52.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:52.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:52:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:52:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:52:52.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:52.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:52.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:52.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:52.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:52.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:52:52.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:52.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:52.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:52.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:52.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:52.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:52.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:52.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:52.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:52.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:53.023 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:53.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:53.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:53.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:53.151 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:53.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:53.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:53.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:53.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:53.277 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:53.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:53.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:53.403 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:53.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:53.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:53.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:53 smithi029 ceph-mon[97787]: pgmap v2081: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 14885/612 objects degraded (2432.190%); 3029/612 objects misplaced (494.935%); 0 B/s, 56 objects/s recovering 2024-04-12T11:52:53.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 14885/612 objects degraded (2432.190%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:53.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:52:53.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:52:53.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:53 smithi029 ceph-mon[93650]: pgmap v2081: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 14885/612 objects degraded (2432.190%); 3029/612 objects misplaced (494.935%); 0 B/s, 56 objects/s recovering 2024-04-12T11:52:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 14885/612 objects degraded (2432.190%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:52:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:52:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:53.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:53.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:53.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:53.629 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:53.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:53.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:53.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:53.669 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:53 smithi087 ceph-mon[89014]: pgmap v2081: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 14885/612 objects degraded (2432.190%); 3029/612 objects misplaced (494.935%); 0 B/s, 56 objects/s recovering 2024-04-12T11:52:53.669 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 14885/612 objects degraded (2432.190%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:53.669 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:52:53.669 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:52:53.669 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:52:53.755 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:53.778 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:53.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:53.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:53.880 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:53.904 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:53.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:53.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:53.978 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:52:53.979 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:52:53.979 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:52:53.991 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:52:54.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:54.007 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:54.013 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:52:54.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:54.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:54.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:54.040 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:52:54.063 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:52:54.083 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:52:54.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:52:54.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:52:54.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:52:54.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:52:54.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:52:54.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:52:54.085 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:52:54.085 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:52:54.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:54.157 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:54.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:54.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:54.258 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:54.282 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:54.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:54.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:54.361 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:52:54.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:54.408 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:54.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:54.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:54.510 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:54.533 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:54.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:54.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:54.635 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:54.659 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:54.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:54.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:54.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:54 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4144317247' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:52:54.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:54 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3246052712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:54.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:54.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:54 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4144317247' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:52:54.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:54 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3246052712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:54.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:54 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4144317247' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:52:54.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:54 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3246052712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:52:54.785 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:54.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:54.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:54.887 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:52:54.910 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:54.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:52:55.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:55.035 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:55.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:55.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:55.136 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:55.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:55.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:55.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:55.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:52:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:52:54] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2024-04-12T11:52:55.284 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:55.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:55.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:55.386 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:55.409 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:55.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:55.495 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:52:55.496 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:52:55.496 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-04-12T11:52:55.496 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-12T11:52:55.496 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.5 2024-04-12T11:52:55.511 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:55.564 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:55.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:55.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:55.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:55 smithi029 ceph-mon[97787]: pgmap v2082: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14263/612 objects degraded (2330.556%); 3029/612 objects misplaced (494.935%); 0 B/s, 108 objects/s recovering 2024-04-12T11:52:55.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:55 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:52:55.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:55 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4144317247' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:52:55.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:55 smithi029 ceph-mon[97787]: osdmap e1262: 8 total, 8 up, 8 in 2024-04-12T11:52:55.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:55 smithi029 ceph-mon[93650]: pgmap v2082: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14263/612 objects degraded (2330.556%); 3029/612 objects misplaced (494.935%); 0 B/s, 108 objects/s recovering 2024-04-12T11:52:55.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:55 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:52:55.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:55 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4144317247' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:52:55.662 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:55 smithi029 ceph-mon[93650]: osdmap e1262: 8 total, 8 up, 8 in 2024-04-12T11:52:55.666 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:55.689 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:55.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:55.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:55.717 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:52:55 smithi087 systemd[1]: Stopping Ceph osd.5 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:52:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:55 smithi087 ceph-mon[89014]: pgmap v2082: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14263/612 objects degraded (2330.556%); 3029/612 objects misplaced (494.935%); 0 B/s, 108 objects/s recovering 2024-04-12T11:52:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:55 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:52:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:55 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4144317247' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:52:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:55 smithi087 ceph-mon[89014]: osdmap e1262: 8 total, 8 up, 8 in 2024-04-12T11:52:55.791 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:55.814 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:55.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:55.916 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:55.939 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:55.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:55.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:56.041 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:56.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:56.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:56.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:56.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:56.192 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:56.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:56.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:56.293 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:56.317 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:56.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:56.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:56.418 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:56.442 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:56.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:56.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:56.544 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:56.569 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:56.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:56.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:56.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:56.694 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:56.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:56.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:56.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:56.819 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:56.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:56.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:56.921 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:56.944 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:56.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:57.046 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:57.070 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:57.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:57.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:57.171 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:57.194 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:57.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:57.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:57.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:57.320 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:57.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:57.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:57.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:57.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:57.447 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:57.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:57.547 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:57.571 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:57.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:57.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:57.696 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:57.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:57 smithi087 ceph-mon[89014]: pgmap v2084: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 40 op/s; 14263/612 objects degraded (2330.556%); 3029/612 objects misplaced (494.935%); 0 B/s, 135 objects/s recovering 2024-04-12T11:52:57.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:57 smithi029 ceph-mon[93650]: pgmap v2084: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 40 op/s; 14263/612 objects degraded (2330.556%); 3029/612 objects misplaced (494.935%); 0 B/s, 135 objects/s recovering 2024-04-12T11:52:57.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:57 smithi029 ceph-mon[97787]: pgmap v2084: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 40 op/s; 14263/612 objects degraded (2330.556%); 3029/612 objects misplaced (494.935%); 0 B/s, 135 objects/s recovering 2024-04-12T11:52:57.798 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:57.822 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:57.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:57.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:57.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:57.947 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:57.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:57.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:58.048 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:58.053 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:52:58.071 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:58.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:58.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:58.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:58.199 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:58.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:58.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:58.300 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:58.323 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:58.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:58.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:58.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:58.449 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:58.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:58.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:58.550 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:52:58.575 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:58.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:58.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:52:58.607 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:52:58.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 14263/612 objects degraded (2330.556%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:58.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 14263/612 objects degraded (2330.556%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:58.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:58.701 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:58.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 14263/612 objects degraded (2330.556%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:52:58.803 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:58.827 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:58.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:58.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:58.929 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:52:58.952 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:58.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:58.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:52:59.054 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:59.077 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:59.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:59.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:59.179 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:52:59.203 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:59.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:52:59.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:52:59.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:59.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:59.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:52:59.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:59.436 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_4' removed 2024-04-12T11:52:59.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:59.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:52:59.558 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:59.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:59.582 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:59.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:59.677 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:59 smithi087 ceph-mon[89014]: pgmap v2085: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14263/612 objects degraded (2330.556%); 3029/612 objects misplaced (494.935%); 0 B/s, 63 objects/s recovering 2024-04-12T11:52:59.677 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:52:59 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2500908645' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:52:59.684 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:52:59.707 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:52:59.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:59.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:52:59.730 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:52:59.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:59 smithi029 ceph-mon[97787]: pgmap v2085: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14263/612 objects degraded (2330.556%); 3029/612 objects misplaced (494.935%); 0 B/s, 63 objects/s recovering 2024-04-12T11:52:59.764 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:52:59 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2500908645' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:52:59.764 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:59 smithi029 ceph-mon[93650]: pgmap v2085: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 10 active+recovery_wait+degraded, 96 active+clean; 12 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14263/612 objects degraded (2330.556%); 3029/612 objects misplaced (494.935%); 0 B/s, 63 objects/s recovering 2024-04-12T11:52:59.764 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:52:59 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2500908645' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:52:59.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:52:59.833 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:59.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:59.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:52:59.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:52:59.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:52:59.959 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:52:59.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:00.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:00.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:00.084 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:00.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:00.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:00.208 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:00.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:00.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:00.309 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:00.333 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:00.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:00.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:00.435 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:00.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:00.470 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:00.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:00.571 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:00.595 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:00.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:00.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:00.697 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:00 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2500908645' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:53:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:00 smithi087 ceph-mon[89014]: osdmap e1263: 8 total, 8 up, 8 in 2024-04-12T11:53:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:00 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2024-04-12T11:53:00.717 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[43892]: 2024-04-12T11:53:00.500+0000 7f1654be4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:53:00.717 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[43892]: 2024-04-12T11:53:00.500+0000 7f1654be4700 -1 osd.5 1264 *** Got signal Terminated *** 2024-04-12T11:53:00.717 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[43892]: 2024-04-12T11:53:00.500+0000 7f1654be4700 -1 osd.5 1264 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T11:53:00.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:00.724 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:00.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:00.769 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:53:00.769 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:53:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:00 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2500908645' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:53:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:00 smithi029 ceph-mon[93650]: osdmap e1263: 8 total, 8 up, 8 in 2024-04-12T11:53:00.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:00 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2024-04-12T11:53:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:00 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2500908645' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:53:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:00 smithi029 ceph-mon[97787]: osdmap e1263: 8 total, 8 up, 8 in 2024-04-12T11:53:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:00 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2024-04-12T11:53:00.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:00.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:00.849 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:00.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:00.950 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:00.974 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:00.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:00.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:01.076 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:01.099 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:01.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:01.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:01.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:01.203 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-12T11:53:01.204 INFO:tasks.radosbench:Beginning radosbench... 2024-04-12T11:53:01.204 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-04-12T11:53:01.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-04-12T11:53:01.224 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:01.227 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:01.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:01.329 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:01.352 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:01.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:01.419 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:01 smithi087 ceph-mon[89014]: pgmap v2087: 83 pgs: 83 active+clean; 582 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-04-12T11:53:01.419 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:01 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14263/612 objects degraded (2330.556%), 11 pgs degraded, 1 pg undersized) 2024-04-12T11:53:01.419 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:01 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-04-12T11:53:01.419 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:01 smithi087 ceph-mon[89014]: osdmap e1264: 8 total, 8 up, 8 in 2024-04-12T11:53:01.419 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:01 smithi087 ceph-mon[89014]: osd.5 marked itself down and dead 2024-04-12T11:53:01.440 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-04-12T11:53:01.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:01.477 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:01.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:01.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:01.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:01 smithi029 ceph-mon[93650]: pgmap v2087: 83 pgs: 83 active+clean; 582 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-04-12T11:53:01.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:01 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14263/612 objects degraded (2330.556%), 11 pgs degraded, 1 pg undersized) 2024-04-12T11:53:01.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:01 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-04-12T11:53:01.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:01 smithi029 ceph-mon[93650]: osdmap e1264: 8 total, 8 up, 8 in 2024-04-12T11:53:01.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:01 smithi029 ceph-mon[93650]: osd.5 marked itself down and dead 2024-04-12T11:53:01.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:01 smithi029 ceph-mon[97787]: pgmap v2087: 83 pgs: 83 active+clean; 582 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-04-12T11:53:01.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:01 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14263/612 objects degraded (2330.556%), 11 pgs degraded, 1 pg undersized) 2024-04-12T11:53:01.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:01 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-04-12T11:53:01.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:01 smithi029 ceph-mon[97787]: osdmap e1264: 8 total, 8 up, 8 in 2024-04-12T11:53:01.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:01 smithi029 ceph-mon[97787]: osd.5 marked itself down and dead 2024-04-12T11:53:01.580 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:01.605 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:01.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:01.645 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:53:01.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:01.730 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:01.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:01.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:01.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:01.855 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:01.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:01.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:01.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:01.980 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:01.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:02.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:02.105 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:02.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:02.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:02.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:02.229 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:02.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:02.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:02.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:02.354 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:02.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:02.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:02.456 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:02.482 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:02.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:02.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:02.583 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:02.593 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:53:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:53:02.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:53:02.593 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:02 smithi087 podman[195158]: 2024-04-12 11:53:02.317812072 +0000 UTC m=+2.570971762 container died 573e2f90e4da8a206479578c632d0a26e1768af84f8e97c84e54792feafe50b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=reef-d540eba, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T11:53:02.593 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:02 smithi087 ceph-mon[89014]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:53:02.593 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:02 smithi087 ceph-mon[89014]: osdmap e1265: 8 total, 7 up, 8 in 2024-04-12T11:53:02.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:02.610 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:02.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:02.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:02 smithi029 ceph-mon[97787]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:53:02.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:02 smithi029 ceph-mon[97787]: osdmap e1265: 8 total, 7 up, 8 in 2024-04-12T11:53:02.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:02 smithi029 ceph-mon[93650]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:53:02.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:02 smithi029 ceph-mon[93650]: osdmap e1265: 8 total, 7 up, 8 in 2024-04-12T11:53:02.711 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:02.734 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:02.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:02.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:02.835 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:02.859 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:02.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:02.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:02.920 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:02 smithi087 podman[195158]: 2024-04-12 11:53:02.692604939 +0000 UTC m=+2.945764636 container cleanup 573e2f90e4da8a206479578c632d0a26e1768af84f8e97c84e54792feafe50b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-d540eba, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True) 2024-04-12T11:53:02.920 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:02 smithi087 bash[195158]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5 2024-04-12T11:53:02.960 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:02.986 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:02.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:03.088 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:03.111 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:03.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:03.180 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:02 smithi087 podman[195379]: 2024-04-12 11:53:02.918968932 +0000 UTC m=+0.769941937 container remove 573e2f90e4da8a206479578c632d0a26e1768af84f8e97c84e54792feafe50b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240403, RELEASE=reef-d540eba, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True) 2024-04-12T11:53:03.212 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:03.236 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:03.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:03.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:03.337 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:03.361 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:03.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:03.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:03.463 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:03.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:03 smithi087 ceph-mon[89014]: pgmap v2090: 83 pgs: 10 stale+active+clean, 1 peering, 72 active+clean; 530 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-04-12T11:53:03.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:03 smithi087 ceph-mon[89014]: osdmap e1266: 8 total, 7 up, 8 in 2024-04-12T11:53:03.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:03 smithi087 ceph-mon[89014]: osdmap e1267: 8 total, 7 up, 8 in 2024-04-12T11:53:03.488 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:03.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:03.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:03.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:03.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:03.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:03.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:03.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:03.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:03.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:03.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:03 smithi029 ceph-mon[93650]: pgmap v2090: 83 pgs: 10 stale+active+clean, 1 peering, 72 active+clean; 530 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-04-12T11:53:03.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:03 smithi029 ceph-mon[93650]: osdmap e1266: 8 total, 7 up, 8 in 2024-04-12T11:53:03.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:03 smithi029 ceph-mon[93650]: osdmap e1267: 8 total, 7 up, 8 in 2024-04-12T11:53:03.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:03 smithi029 ceph-mon[97787]: pgmap v2090: 83 pgs: 10 stale+active+clean, 1 peering, 72 active+clean; 530 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-04-12T11:53:03.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:03 smithi029 ceph-mon[97787]: osdmap e1266: 8 total, 7 up, 8 in 2024-04-12T11:53:03.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:03 smithi029 ceph-mon[97787]: osdmap e1267: 8 total, 7 up, 8 in 2024-04-12T11:53:03.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:03.864 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:03.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:03.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:03.966 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:03.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:03.991 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:03.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:04.050 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:53:04.050 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:53:04.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:04.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:04.118 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:04.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:04.150 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:53:04.161 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:04.170 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:53:04.195 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:53:04.215 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:53:04.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:04.236 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:53:04.242 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:04.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:04.255 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:53:04.275 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:53:04.275 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:53:04.275 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:53:04.275 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:53:04.275 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:53:04.275 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:53:04.275 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-04-12T11:53:04.276 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:53:04.276 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:53:04.344 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:04.368 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:04.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:04.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:04.470 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:04.494 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:04.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:04.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:04.596 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:04.619 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:04.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:04.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:04.691 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-04-12T11:53:04.721 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:04.744 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:04.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:04.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:04.846 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:04.872 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:04.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:04.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:04.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:04.997 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:04.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:04.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:05.077 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:53:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:53:04] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2024-04-12T11:53:05.088 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:53:05.098 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:05.121 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:05.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:05.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:05.191 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:05 smithi087 ceph-mon[89014]: pgmap v2093: 83 pgs: 14 active+undersized, 2 active+undersized+degraded, 4 active+undersized+remapped, 5 remapped+peering, 2 stale+active+clean, 3 peering, 53 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3/72 objects degraded (4.167%); 74 KiB/s, 0 objects/s recovering 2024-04-12T11:53:05.192 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:05 smithi087 ceph-mon[89014]: osdmap e1268: 8 total, 7 up, 8 in 2024-04-12T11:53:05.192 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:05 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1744160028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:05.192 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:05 smithi087 ceph-mon[89014]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-12T11:53:05.192 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:05 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:05.192 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:05 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1116296384' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:53:05.192 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:05 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:53:05.192 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:04 smithi087 podman[195704]: 2024-04-12 11:53:04.834146071 +0000 UTC m=+0.018957332 image pull bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3 2024-04-12T11:53:05.192 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:04 smithi087 podman[195704]: 2024-04-12 11:53:04.93818341 +0000 UTC m=+0.122994668 container create e1dd8fbf7418ce853d0eef4e261a87d83e0e637d2f77038b816fb4fa76d20899 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-d540eba, GIT_CLEAN=True) 2024-04-12T11:53:05.192 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:05 smithi087 podman[195704]: 2024-04-12 11:53:05.191061293 +0000 UTC m=+0.375872551 container init e1dd8fbf7418ce853d0eef4e261a87d83e0e637d2f77038b816fb4fa76d20899 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=reef-d540eba, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-12T11:53:05.224 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:05.247 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:05.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:05.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:05.333 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:53:05.349 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:05.363 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[97787]: pgmap v2093: 83 pgs: 14 active+undersized, 2 active+undersized+degraded, 4 active+undersized+remapped, 5 remapped+peering, 2 stale+active+clean, 3 peering, 53 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3/72 objects degraded (4.167%); 74 KiB/s, 0 objects/s recovering 2024-04-12T11:53:05.363 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[97787]: osdmap e1268: 8 total, 7 up, 8 in 2024-04-12T11:53:05.363 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1744160028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:05.363 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[97787]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-12T11:53:05.363 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:05.363 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1116296384' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:53:05.363 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:53:05.363 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[93650]: pgmap v2093: 83 pgs: 14 active+undersized, 2 active+undersized+degraded, 4 active+undersized+remapped, 5 remapped+peering, 2 stale+active+clean, 3 peering, 53 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 3/72 objects degraded (4.167%); 74 KiB/s, 0 objects/s recovering 2024-04-12T11:53:05.364 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[93650]: osdmap e1268: 8 total, 7 up, 8 in 2024-04-12T11:53:05.364 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1744160028' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:05.364 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[93650]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-12T11:53:05.364 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:05.364 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1116296384' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:53:05.364 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:05 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:53:05.372 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:05.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:05.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:05.456 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:05 smithi087 podman[195704]: 2024-04-12 11:53:05.194717486 +0000 UTC m=+0.379528748 container start e1dd8fbf7418ce853d0eef4e261a87d83e0e637d2f77038b816fb4fa76d20899 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, RELEASE=reef-d540eba, GIT_CLEAN=True, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T11:53:05.456 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:05 smithi087 podman[195704]: 2024-04-12 11:53:05.29087664 +0000 UTC m=+0.475687899 container attach e1dd8fbf7418ce853d0eef4e261a87d83e0e637d2f77038b816fb4fa76d20899 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=reef-d540eba, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-12T11:53:05.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:05.498 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:05.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:05.502 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-04-12T11:53:05.503 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-04-12T11:53:05.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:05.624 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:05.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:05.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:05.725 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:05.749 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:05.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:05.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:05.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:05.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:05.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:05.999 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:06.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:06.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:06.101 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:06.126 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:06.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:06.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:06.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:06 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:53:06.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:06 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:53:06.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:06 smithi029 ceph-mon[93650]: osdmap e1269: 8 total, 7 up, 8 in 2024-04-12T11:53:06.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:06 smithi029 ceph-mon[93650]: 1.6 scrub starts 2024-04-12T11:53:06.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:06 smithi029 ceph-mon[93650]: 1.6 scrub ok 2024-04-12T11:53:06.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:06 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:53:06.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:06 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:53:06.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:06 smithi029 ceph-mon[97787]: osdmap e1269: 8 total, 7 up, 8 in 2024-04-12T11:53:06.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:06 smithi029 ceph-mon[97787]: 1.6 scrub starts 2024-04-12T11:53:06.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:06 smithi029 ceph-mon[97787]: 1.6 scrub ok 2024-04-12T11:53:06.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:06.251 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:06.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:06.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:06.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:06.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:06.377 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:06.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:06.457 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:06 smithi087 podman[195704]: 2024-04-12 11:53:06.152674485 +0000 UTC m=+1.337485748 container died e1dd8fbf7418ce853d0eef4e261a87d83e0e637d2f77038b816fb4fa76d20899 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-deactivate, RELEASE=reef-d540eba, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T11:53:06.458 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:06 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:53:06.458 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:06 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:53:06.458 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:06 smithi087 ceph-mon[89014]: osdmap e1269: 8 total, 7 up, 8 in 2024-04-12T11:53:06.458 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:06 smithi087 ceph-mon[89014]: 1.6 scrub starts 2024-04-12T11:53:06.458 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:06 smithi087 ceph-mon[89014]: 1.6 scrub ok 2024-04-12T11:53:06.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:06.501 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:06.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:06.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:06.603 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:06.627 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:06.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:06.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:06.752 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:06.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:06.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:06.854 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:06.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:06.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:06.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:06.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:07.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:07.002 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:07.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:07.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:07.127 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:07.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:07.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:07.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:07.251 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:07.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:07.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:07.294 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:53:07.295 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:53:07.295 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:53:07.295 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:53:07.295 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:53:07.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:07.376 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:07.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:07.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:07.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:07.502 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:07.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:07.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:07.503 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:53:07.604 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:07.630 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:07.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:07.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:07.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:07 smithi087 ceph-mon[89014]: 1.1 scrub starts 2024-04-12T11:53:07.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:07 smithi087 ceph-mon[89014]: 1.1 scrub ok 2024-04-12T11:53:07.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:07 smithi087 ceph-mon[89014]: pgmap v2096: 83 pgs: 16 active+undersized, 4 active+undersized+degraded, 3 active+undersized+remapped, 5 remapped+peering, 3 peering, 52 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 16 op/s; 5/72 objects degraded (6.944%); 119 KiB/s, 0 objects/s recovering 2024-04-12T11:53:07.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:07.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:07.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:07 smithi087 ceph-mon[89014]: 1.0 scrub starts 2024-04-12T11:53:07.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:07 smithi087 ceph-mon[89014]: 1.0 scrub ok 2024-04-12T11:53:07.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[93650]: 1.1 scrub starts 2024-04-12T11:53:07.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[93650]: 1.1 scrub ok 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[93650]: pgmap v2096: 83 pgs: 16 active+undersized, 4 active+undersized+degraded, 3 active+undersized+remapped, 5 remapped+peering, 3 peering, 52 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 16 op/s; 5/72 objects degraded (6.944%); 119 KiB/s, 0 objects/s recovering 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[93650]: 1.0 scrub starts 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[93650]: 1.0 scrub ok 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[97787]: 1.1 scrub starts 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[97787]: 1.1 scrub ok 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[97787]: pgmap v2096: 83 pgs: 16 active+undersized, 4 active+undersized+degraded, 3 active+undersized+remapped, 5 remapped+peering, 3 peering, 52 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 16 op/s; 5/72 objects degraded (6.944%); 119 KiB/s, 0 objects/s recovering 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:07.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[97787]: 1.0 scrub starts 2024-04-12T11:53:07.725 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:07 smithi029 ceph-mon[97787]: 1.0 scrub ok 2024-04-12T11:53:07.732 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:07.755 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:07.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:07.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:07.856 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:07.881 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:07.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:07.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:07.941 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-04-12T11:53:07.982 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:08.009 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:08.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:08.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:08.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:08.122 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:08.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:08.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:08.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:08.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:53:08.236 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:08.259 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:08.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:08.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:08.336 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_5' created 2024-04-12T11:53:08.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:08.376 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.5.service' 2024-04-12T11:53:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:08.385 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:08.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:08.440 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T11:53:08.440 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-12T11:53:08.481 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:53:08.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:08.510 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:08.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:08.603 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:08 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2605276301' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2024-04-12T11:53:08.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:08.635 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:08.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:08.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:08.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:08.760 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:08.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:08.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:08.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:08 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2605276301' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2024-04-12T11:53:08.766 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:08 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2605276301' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2024-04-12T11:53:08.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:08.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:08.887 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:08.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:08.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:09.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:09.012 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:09.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:09.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:09.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-04-12T11:53:09.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:09.137 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:09.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:09.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:09.262 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:09.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:09.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:09.364 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:09.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:09 smithi029 ceph-mon[97787]: pgmap v2097: 83 pgs: 1 active+clean+remapped, 17 active+undersized, 5 active+undersized+degraded, 3 remapped+peering, 1 peering, 56 active+clean; 425 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/69 objects degraded (13.043%) 2024-04-12T11:53:09.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2605276301' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-04-12T11:53:09.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:09 smithi029 ceph-mon[97787]: osdmap e1270: 8 total, 7 up, 8 in 2024-04-12T11:53:09.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:09 smithi029 ceph-mon[97787]: 1.9 scrub starts 2024-04-12T11:53:09.382 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:09 smithi029 ceph-mon[97787]: 1.9 scrub ok 2024-04-12T11:53:09.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:09 smithi029 ceph-mon[93650]: pgmap v2097: 83 pgs: 1 active+clean+remapped, 17 active+undersized, 5 active+undersized+degraded, 3 remapped+peering, 1 peering, 56 active+clean; 425 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/69 objects degraded (13.043%) 2024-04-12T11:53:09.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2605276301' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-04-12T11:53:09.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:09 smithi029 ceph-mon[93650]: osdmap e1270: 8 total, 7 up, 8 in 2024-04-12T11:53:09.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:09 smithi029 ceph-mon[93650]: 1.9 scrub starts 2024-04-12T11:53:09.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:09 smithi029 ceph-mon[93650]: 1.9 scrub ok 2024-04-12T11:53:09.389 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:09.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:09 smithi087 ceph-mon[89014]: pgmap v2097: 83 pgs: 1 active+clean+remapped, 17 active+undersized, 5 active+undersized+degraded, 3 remapped+peering, 1 peering, 56 active+clean; 425 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/69 objects degraded (13.043%) 2024-04-12T11:53:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2605276301' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-04-12T11:53:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:09 smithi087 ceph-mon[89014]: osdmap e1270: 8 total, 7 up, 8 in 2024-04-12T11:53:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:09 smithi087 ceph-mon[89014]: 1.9 scrub starts 2024-04-12T11:53:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:09 smithi087 ceph-mon[89014]: 1.9 scrub ok 2024-04-12T11:53:09.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:09.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:09.515 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:09.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:09.616 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:09.640 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:09.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:09.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:09.742 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:09.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:09.769 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:09.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:09.870 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:09.893 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:09.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:09.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:09.995 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:10.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:10.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:10.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:10.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:10.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:10.145 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:10.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:10.163 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:10.181 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:10.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:10.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:10.271 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:10.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:10.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:10.397 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:10.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:10.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:10.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:10 smithi029 ceph-mon[97787]: 1.7 scrub starts 2024-04-12T11:53:10.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:10 smithi029 ceph-mon[97787]: 1.7 scrub ok 2024-04-12T11:53:10.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:10 smithi029 ceph-mon[97787]: osdmap e1271: 8 total, 7 up, 8 in 2024-04-12T11:53:10.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:10 smithi029 ceph-mon[97787]: 1.3 scrub starts 2024-04-12T11:53:10.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:10 smithi029 ceph-mon[97787]: 1.3 scrub ok 2024-04-12T11:53:10.406 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:10 smithi029 ceph-mon[93650]: 1.7 scrub starts 2024-04-12T11:53:10.406 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:10 smithi029 ceph-mon[93650]: 1.7 scrub ok 2024-04-12T11:53:10.406 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:10 smithi029 ceph-mon[93650]: osdmap e1271: 8 total, 7 up, 8 in 2024-04-12T11:53:10.406 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:10 smithi029 ceph-mon[93650]: 1.3 scrub starts 2024-04-12T11:53:10.407 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:10 smithi029 ceph-mon[93650]: 1.3 scrub ok 2024-04-12T11:53:10.426 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86170 8.5 8:aeba37fe:::benchmark_data_smithi029_194418_object86169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.136001+0000", 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 555.28740660100004, 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3913194349999998, 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.427 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86170 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.136001+0000", 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.136001+0000", 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.428 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.135999+0000", 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.136037+0000", 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8074000000000002e-05 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.136056+0000", 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8858000000000001e-05 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.429 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.136066+0000", 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7210000000000006e-06 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.136305+0000", 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000238884 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.136631+0000", 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00032643 2024-04-12T11:53:10.430 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 4,7", 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.137406+0000", 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00077492900000000005 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.150122+0000", 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012716085 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.150134+0000", 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1464000000000001e-05 2024-04-12T11:53:10.431 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141783+0000", 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.99164903400000004 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141797+0000", 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4102e-05 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.468240+0000", 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.326443517 2024-04-12T11:53:10.432 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.469025+0000", 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00078502300000000004 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.527209+0000", 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.058183704000000003 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.527320+0000", 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000111113 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.433 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86196 8.5 8:ab261e6d:::benchmark_data_smithi029_194418_object86195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.174188+0000", 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 555.24921925199999, 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3658279970000002, 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86196 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.434 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.174188+0000", 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.174188+0000", 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.435 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.436 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.174187+0000", 2024-04-12T11:53:10.436 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:10.436 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.436 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.436 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.436 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.174242+0000", 2024-04-12T11:53:10.436 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4882000000000001e-05 2024-04-12T11:53:10.437 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.437 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.437 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.437 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.174254+0000", 2024-04-12T11:53:10.437 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2357e-05 2024-04-12T11:53:10.437 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.437 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.438 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.438 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.174261+0000", 2024-04-12T11:53:10.438 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3640000000000004e-06 2024-04-12T11:53:10.438 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.438 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.438 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.438 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.174315+0000", 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4675000000000001e-05 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.174328+0000", 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2354000000000001e-05 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141824+0000", 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.96749681899999995 2024-04-12T11:53:10.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.440 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.440 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.440 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141832+0000", 2024-04-12T11:53:10.440 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6720000000000004e-06 2024-04-12T11:53:10.440 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.440 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.440 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.527362+0000", 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3855303409999999 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.527477+0000", 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000114756 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.527651+0000", 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017354600000000001 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.441 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.536787+0000", 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0091358840000000004 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.536796+0000", 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6069999999999997e-06 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.536906+0000", 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000109453 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.442 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.536952+0000", 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6513e-05 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.540016+0000", 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030638380000000002 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:86206 8.5 8:a0abdb3f:::benchmark_data_smithi029_194418_object86205:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:43:55.195565+0000", 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 555.22784200599995, 2024-04-12T11:53:10.443 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3528878139999998, 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 86206 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.195565+0000", 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.444 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.195565+0000", 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.195564+0000", 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.195636+0000", 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1936999999999999e-05 2024-04-12T11:53:10.445 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.195648+0000", 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1901e-05 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.195654+0000", 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7800000000000003e-06 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.195693+0000", 2024-04-12T11:53:10.446 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8587000000000002e-05 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:55.195701+0000", 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8520000000000004e-06 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141855+0000", 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.94615448499999999 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.447 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:56.141864+0000", 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7080000000000003e-06 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.529951+0000", 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3880864939999999 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.530106+0000", 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015497199999999999 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.530324+0000", 2024-04-12T11:53:10.448 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021896 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.536805+0000", 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0064800659999999996 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.536846+0000", 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0988999999999997e-05 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.544488+0000", 2024-04-12T11:53:10.449 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0076427609999999997 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.544551+0000", 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2716000000000001e-05 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:43:57.548453+0000", 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0039021070000000001 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.450 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121588 8.15 8:ae52c813:::benchmark_data_smithi029_194418_object15278:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:44.398969+0000", 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 506.02443800899999, 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1841509750000001, 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121588 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.451 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.398969+0000", 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.398969+0000", 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.398967+0000", 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.452 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.398978+0000", 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1060000000000001e-05 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.398995+0000", 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6637999999999999e-05 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.399004+0000", 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2369999999999998e-06 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.453 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.652826+0000", 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.25382115999999999 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:44.652837+0000", 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1807000000000001e-05 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.577083+0000", 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.92424541400000004 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.454 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for missing object", 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.577234+0000", 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015152699999999999 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.583065+0000", 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0058303119999999998 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.583120+0000", 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5695000000000001e-05 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.456 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121609 8.5 8:a731e9c9:::benchmark_data_smithi029_194418_object15299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:45.499061+0000", 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 504.92434677699998, 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.96776600800000001, 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121609 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.457 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499061+0000", 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499061+0000", 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499059+0000", 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:53:10.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499068+0000", 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1859999999999998e-06 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499104+0000", 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6201000000000001e-05 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.499116+0000", 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2283000000000001e-05 2024-04-12T11:53:10.459 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.577384+0000", 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.078268022000000007 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.577394+0000", 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7680000000000003e-06 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.429200+0000", 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.85180545699999999 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.460 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.429846+0000", 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00064660800000000001 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.457423+0000", 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.027576255000000001 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.457526+0000", 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000103182 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.461 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.457854+0000", 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00032832800000000002 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.460022+0000", 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002168424 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.464885+0000", 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0048626219999999996 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.462 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.464890+0000", 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1290000000000004e-06 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.464942+0000", 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1445e-05 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.466827+0000", 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001884887 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121622 8.5 8:a3300873:::benchmark_data_smithi029_194418_object15312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:45.517945+0000", 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 504.90546246100001, 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.002112503, 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121622 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.464 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.517945+0000", 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.517945+0000", 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.517944+0000", 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:10.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.517951+0000", 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.323e-06 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.517961+0000", 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.842e-06 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.466 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.517967+0000", 2024-04-12T11:53:10.467 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3280000000000002e-06 2024-04-12T11:53:10.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.467 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.467 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.467 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.586082+0000", 2024-04-12T11:53:10.467 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.068114571999999998 2024-04-12T11:53:10.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.467 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.467 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.467 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.586094+0000", 2024-04-12T11:53:10.468 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2444e-05 2024-04-12T11:53:10.468 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.468 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.468 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.468 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.459325+0000", 2024-04-12T11:53:10.468 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.87323073500000004 2024-04-12T11:53:10.468 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.468 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.468 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.469 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.459982+0000", 2024-04-12T11:53:10.469 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000656702 2024-04-12T11:53:10.469 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.469 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.469 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.469 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.482451+0000", 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.022468717999999999 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.482527+0000", 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6451000000000004e-05 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.482828+0000", 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00030059100000000001 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.470 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.490497+0000", 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0076697370000000003 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.518606+0000", 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.028108815999999998 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.518651+0000", 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4515999999999999e-05 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.471 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.518703+0000", 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2389999999999998e-05 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.520057+0000", 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001354294 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121625 8.5 8:a397433e:::benchmark_data_smithi029_194418_object15315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:45.520288+0000", 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 504.903119737, 2024-04-12T11:53:10.472 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0008609449999999, 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121625 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.520288+0000", 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.473 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.520288+0000", 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.520286+0000", 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.520294+0000", 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9240000000000007e-06 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.474 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.520310+0000", 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6762e-05 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.520319+0000", 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4249999999999998e-06 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.586119+0000", 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.065799982000000007 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.586128+0000", 2024-04-12T11:53:10.475 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3870000000000001e-06 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.460043+0000", 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.87391524099999995 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.460691+0000", 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000647771 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.487002+0000", 2024-04-12T11:53:10.476 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.026311246 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.487074+0000", 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2019000000000003e-05 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.487395+0000", 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00032063600000000001 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.494635+0000", 2024-04-12T11:53:10.477 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0072395760000000002 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.520102+0000", 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.025467190000000001 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.520134+0000", 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.2648999999999999e-05 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.478 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.520166+0000", 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.1785000000000001e-05 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.521149+0000", 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00098228700000000009 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121631 8.5 8:a5c49cc7:::benchmark_data_smithi029_194418_object15321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:45.528748+0000", 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 504.89465949800001, 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.993251985, 2024-04-12T11:53:10.479 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121631 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.528748+0000", 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.528748+0000", 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.528746+0000", 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.528757+0000", 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1260000000000001e-05 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.528770+0000", 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.293e-05 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.528780+0000", 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.662e-06 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.586153+0000", 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.057372487 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.586163+0000", 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0241000000000001e-05 2024-04-12T11:53:10.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.460729+0000", 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.87456588999999996 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.461556+0000", 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00082724899999999995 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.490524+0000", 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.028968587000000001 2024-04-12T11:53:10.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.490586+0000", 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1326000000000002e-05 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.490839+0000", 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00025361399999999999 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.501114+0000", 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.010274994000000001 2024-04-12T11:53:10.484 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.521200+0000", 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.020085166000000002 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.521252+0000", 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2880000000000002e-05 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.521283+0000", 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.0960000000000002e-05 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.522000+0000", 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00071635899999999996 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121639 8.5 8:a5783c3f:::benchmark_data_smithi029_194418_object15329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:45.540988+0000", 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 504.88241951600003, 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.982182428, 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.486 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121639 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.540988+0000", 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.540988+0000", 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.540987+0000", 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.540992+0000", 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5480000000000003e-06 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.541001+0000", 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0240000000000003e-06 2024-04-12T11:53:10.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.541006+0000", 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3210000000000004e-06 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.592162+0000", 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.051155659999999999 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.592174+0000", 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2033e-05 2024-04-12T11:53:10.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.461593+0000", 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.86941926199999997 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.462211+0000", 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000618354 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.494663+0000", 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.032451488000000001 2024-04-12T11:53:10.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.494734+0000", 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1155000000000006e-05 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.495003+0000", 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000268705 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.504280+0000", 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0092772570000000006 2024-04-12T11:53:10.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.522053+0000", 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.017773088999999999 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.522099+0000", 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6402000000000001e-05 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.522152+0000", 2024-04-12T11:53:10.492 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2342000000000001e-05 2024-04-12T11:53:10.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.493 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.493 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.523170+0000", 2024-04-12T11:53:10.493 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0010185420000000001 2024-04-12T11:53:10.493 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.493 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.494 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.494 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.494 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121645 8.5 8:a596297c:::benchmark_data_smithi029_194418_object15335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-04-12T11:53:10.494 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:45.565051+0000", 2024-04-12T11:53:10.494 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 504.85835675300001, 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.95924810199999999, 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121645 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.565051+0000", 2024-04-12T11:53:10.495 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.496 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.496 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.496 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.565051+0000", 2024-04-12T11:53:10.496 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.565049+0000", 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.565055+0000", 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8950000000000003e-06 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.565065+0000", 2024-04-12T11:53:10.497 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0128e-05 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.565071+0000", 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3899999999999998e-06 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.592199+0000", 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.027128016000000001 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.498 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.592209+0000", 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4550000000000001e-06 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.467513+0000", 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.87530425300000003 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.468172+0000", 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00065887499999999996 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.499 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.499197+0000", 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.031024764999999999 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.499308+0000", 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000111591 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.499656+0000", 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00034720299999999998 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.508474+0000", 2024-04-12T11:53:10.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0088186059999999997 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.523224+0000", 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014749620999999999 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.523268+0000", 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4087e-05 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.523315+0000", 2024-04-12T11:53:10.501 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7500000000000003e-05 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.524299+0000", 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00098324799999999993 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121667 8.5 8:a66aaab1:::benchmark_data_smithi029_194418_object15357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:45.605271+0000", 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 504.81813675799998, 2024-04-12T11:53:10.502 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.92003602799999995, 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121667 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.605271+0000", 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.503 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.605271+0000", 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.605269+0000", 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.605276+0000", 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5239999999999997e-06 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.605287+0000", 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1755e-05 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.605295+0000", 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8099999999999998e-06 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.605347+0000", 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1854999999999997e-05 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.605360+0000", 2024-04-12T11:53:10.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2573e-05 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.468231+0000", 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.86287154600000004 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.468794+0000", 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00056256999999999998 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.506 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.504311+0000", 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.035516991999999997 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.504379+0000", 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8478000000000004e-05 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.504682+0000", 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000302801 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.507 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.514597+0000", 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0099154059999999999 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.524342+0000", 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0097450880000000007 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.524372+0000", 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.9958e-05 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.508 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.524404+0000", 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.1316000000000003e-05 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.525307+0000", 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00090286399999999997 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121699 8.5 8:a40e07c9:::benchmark_data_smithi029_194418_object15389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:45.679253+0000", 2024-04-12T11:53:10.509 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 504.74415383600001, 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.84733915000000004, 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121699 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.679253+0000", 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.510 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.679253+0000", 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.679252+0000", 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.679260+0000", 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6739999999999997e-06 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.679275+0000", 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5458e-05 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.679286+0000", 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.095e-05 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.679351+0000", 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4739999999999993e-05 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.679366+0000", 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5299999999999999e-05 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.468831+0000", 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.78946522299999999 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.469439+0000", 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00060774799999999997 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.513 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.508502+0000", 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.039062576000000002 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.508561+0000", 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8928000000000002e-05 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.508886+0000", 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00032523900000000001 2024-04-12T11:53:10.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.518564+0000", 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0096786960000000005 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.525382+0000", 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.006817104 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.525432+0000", 2024-04-12T11:53:10.515 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0701999999999997e-05 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.525464+0000", 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.1693000000000003e-05 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.526593+0000", 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0011286429999999999 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121720 8.5 8:a584644b:::benchmark_data_smithi029_194418_object15410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:45.743277+0000", 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 504.68013031499999, 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.78521415800000005, 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121720 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.743277+0000", 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.743277+0000", 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.743275+0000", 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:53:10.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.743285+0000", 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0040999999999999e-05 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.743302+0000", 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.647e-05 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.519 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.743311+0000", 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9769999999999995e-06 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.743376+0000", 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4468999999999995e-05 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.743391+0000", 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5189e-05 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.469472+0000", 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.72608092000000002 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.470499+0000", 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001027032 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.512631+0000", 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.042131455999999998 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.521 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.512733+0000", 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000102268 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.513075+0000", 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000341706 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.518615+0000", 2024-04-12T11:53:10.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0055403140000000002 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.526653+0000", 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.008038211 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.526872+0000", 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021889699999999999 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.526943+0000", 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0850000000000001e-05 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.528491+0000", 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0015483560000000001 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:121732 8.5 8:a438b53e:::benchmark_data_smithi029_194418_object15422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:45.802551+0000", 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 504.62085639899999, 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.72773870799999996, 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.524 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 121732 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.802551+0000", 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.802551+0000", 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.802550+0000", 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.802557+0000", 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9639999999999997e-06 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.802569+0000", 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2754e-05 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.802577+0000", 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5809999999999999e-06 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.802633+0000", 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6252999999999998e-05 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:45.802647+0000", 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4248e-05 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.470819+0000", 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.66817129399999997 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.471426+0000", 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00060724900000000003 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.516746+0000", 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.045320014999999998 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.516839+0000", 2024-04-12T11:53:10.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3463999999999998e-05 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.517177+0000", 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00033786299999999999 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.521162+0000", 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0039845490000000004 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.528551+0000", 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0073892580000000001 2024-04-12T11:53:10.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.528636+0000", 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4839000000000005e-05 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.528689+0000", 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2864000000000001e-05 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:46.530290+0000", 2024-04-12T11:53:10.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001600852 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:131756 8.15 8:a9de84f8:::benchmark_data_smithi029_194418_object25446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:57.668284+0000", 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 492.75512368, 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.47398037700000001, 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.531 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 131756 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.668284+0000", 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.668284+0000", 2024-04-12T11:53:10.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.668283+0000", 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.668287+0000", 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9949999999999999e-06 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.668294+0000", 2024-04-12T11:53:10.533 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2439999999999996e-06 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.668300+0000", 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6389999999999997e-06 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.686952+0000", 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.018652275999999999 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.687626+0000", 2024-04-12T11:53:10.534 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00067399799999999998 2024-04-12T11:53:10.535 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.535 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.535 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.535 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.050547+0000", 2024-04-12T11:53:10.535 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.36292082399999998 2024-04-12T11:53:10.535 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.535 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.535 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.535 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.050634+0000", 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6929999999999996e-05 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.050898+0000", 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000264357 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.053147+0000", 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0022488080000000001 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.053154+0000", 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4839999999999998e-06 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.140514+0000", 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.087359501000000006 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.140561+0000", 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7581999999999999e-05 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.142264+0000", 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001702589 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:131765 8.5 8:a256403d:::benchmark_data_smithi029_194418_object25455:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:53:10.539 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:57.676550+0000", 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 492.74685758300001, 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.46666330099999997, 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 131765 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.676550+0000", 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.676550+0000", 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.676548+0000", 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.676558+0000", 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0171e-05 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.676576+0000", 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.755e-05 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.676587+0000", 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1199999999999999e-05 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.687710+0000", 2024-04-12T11:53:10.542 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.011122965 2024-04-12T11:53:10.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.543 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.543 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.543 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.688243+0000", 2024-04-12T11:53:10.543 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00053320799999999997 2024-04-12T11:53:10.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.543 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.543 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.543 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.050562+0000", 2024-04-12T11:53:10.544 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.36231949400000002 2024-04-12T11:53:10.544 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.544 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.544 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.050670+0000", 2024-04-12T11:53:10.544 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010719599999999999 2024-04-12T11:53:10.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.545 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.545 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.050901+0000", 2024-04-12T11:53:10.545 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00023182000000000001 2024-04-12T11:53:10.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.545 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.545 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.053175+0000", 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0022730879999999999 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.053224+0000", 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9186e-05 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.141930+0000", 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.088705883999999999 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.141965+0000", 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4909000000000001e-05 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.143213+0000", 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0012484880000000001 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:131773 8.5 8:a1add9dc:::benchmark_data_smithi029_194418_object25463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:57.694257+0000", 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 492.72915053100002, 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.44759690200000002, 2024-04-12T11:53:10.547 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 131773 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.694257+0000", 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.694257+0000", 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.694255+0000", 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.694266+0000", 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0324e-05 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.694282+0000", 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6469000000000001e-05 2024-04-12T11:53:10.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.694292+0000", 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7089999999999997e-06 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.694333+0000", 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1508000000000003e-05 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.694815+0000", 2024-04-12T11:53:10.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00048118800000000002 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.048659+0000", 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.35384449000000001 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.048742+0000", 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3048999999999997e-05 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.049059+0000", 2024-04-12T11:53:10.551 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00031723199999999997 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.053107+0000", 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0040481709999999997 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.053160+0000", 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2175000000000001e-05 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.140524+0000", 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.087363986000000005 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.140568+0000", 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4382000000000003e-05 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.141854+0000", 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0012856689999999999 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:131775 8.15 8:ac5e2498:::benchmark_data_smithi029_194418_object25465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1202)", 2024-04-12T11:53:10.553 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:44:57.697177+0000", 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 492.72623049200001, 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.44184642200000002, 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 131775 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.697177+0000", 2024-04-12T11:53:10.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.697177+0000", 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.697176+0000", 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.555 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.697181+0000", 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.524e-06 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.697196+0000", 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4567e-05 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.697203+0000", 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0890000000000002e-06 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.697315+0000", 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000111496 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:57.697974+0000", 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00065938599999999998 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.048617+0000", 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.35064339500000002 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.557 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.048707+0000", 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9291999999999998e-05 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.048998+0000", 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000291212 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.052267+0000", 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0032696309999999998 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.052308+0000", 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0213000000000003e-05 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.137252+0000", 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.084944710000000007 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.137307+0000", 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4455e-05 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:44:58.139023+0000", 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0017164109999999999 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:168298 8.5 8:a0dcacb7:::benchmark_data_smithi029_194418_object61988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:52.386903+0000", 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 438.03650453300003, 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8128649829999999, 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.560 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 168298 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.386903+0000", 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.386903+0000", 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.386901+0000", 2024-04-12T11:53:10.561 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.386911+0000", 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2799999999999992e-06 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.386924+0000", 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3112000000000001e-05 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.562 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.386932+0000", 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8089999999999999e-06 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.391081+0000", 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0041484429999999999 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.391091+0000", 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6730000000000004e-06 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.564 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.564 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.974683+0000", 2024-04-12T11:53:10.564 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.58359260800000001 2024-04-12T11:53:10.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.564 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.974695+0000", 2024-04-12T11:53:10.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1331e-05 2024-04-12T11:53:10.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.565 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.188312+0000", 2024-04-12T11:53:10.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2136177130000001 2024-04-12T11:53:10.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.566 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.566 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.188485+0000", 2024-04-12T11:53:10.566 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017314699999999999 2024-04-12T11:53:10.566 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.566 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.567 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 4,7", 2024-04-12T11:53:10.567 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.188985+0000", 2024-04-12T11:53:10.567 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00049925100000000004 2024-04-12T11:53:10.567 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.567 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.567 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.567 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.194990+0000", 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0060054160000000004 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.195641+0000", 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00065135400000000004 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.198839+0000", 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0031979130000000001 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.198888+0000", 2024-04-12T11:53:10.568 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8442999999999997e-05 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.199768+0000", 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00087997499999999996 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35217.0:168316 8.15 8:ab88cbe1:::benchmark_data_smithi029_194418_object62006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:52.422888+0000", 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 438.00051945500002, 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.776048659, 2024-04-12T11:53:10.569 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:10.570 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:10.570 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:10.570 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35217", 2024-04-12T11:53:10.570 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:53:10.570 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 168316 2024-04-12T11:53:10.570 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.570 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:10.570 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.571 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:10.571 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.422888+0000", 2024-04-12T11:53:10.571 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.571 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.571 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.571 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:10.571 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.422888+0000", 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.422886+0000", 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.422894+0000", 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2640000000000008e-06 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:10.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.422912+0000", 2024-04-12T11:53:10.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7566000000000001e-05 2024-04-12T11:53:10.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.573 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:10.573 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.422923+0000", 2024-04-12T11:53:10.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0739e-05 2024-04-12T11:53:10.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.422995+0000", 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1797000000000004e-05 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.423004+0000", 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.5789999999999993e-06 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.574 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.974674+0000", 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.55167002200000004 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:52.974689+0000", 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5092000000000001e-05 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.192241+0000", 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.217551815 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.192462+0000", 2024-04-12T11:53:10.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00022064799999999999 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4,7", 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.192873+0000", 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00041103599999999999 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.195760+0000", 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0028872820000000001 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.195914+0000", 2024-04-12T11:53:10.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015404899999999999 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.196057+0000", 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014326700000000001 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.198754+0000", 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0026970679999999999 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.198811+0000", 2024-04-12T11:53:10.577 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6422999999999998e-05 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:54.198937+0000", 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012569999999999999 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:10.578 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:53:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:10.604 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:10.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:10.704 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:10 smithi087 ceph-mon[89014]: 1.7 scrub starts 2024-04-12T11:53:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:10 smithi087 ceph-mon[89014]: 1.7 scrub ok 2024-04-12T11:53:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:10 smithi087 ceph-mon[89014]: osdmap e1271: 8 total, 7 up, 8 in 2024-04-12T11:53:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:10 smithi087 ceph-mon[89014]: 1.3 scrub starts 2024-04-12T11:53:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:10 smithi087 ceph-mon[89014]: 1.3 scrub ok 2024-04-12T11:53:10.728 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:10.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:10.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:10.830 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:10.855 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:10.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:10.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:10.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:10.980 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:10.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:11.082 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:11.106 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:11.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:11.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:11.200 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-04-12T11:53:11.207 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:11.265 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:11.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:11.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:11.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:11.392 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:11.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:11.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:11.402 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:53:11.407 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:53:11.421 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:53:11.441 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:53:11.461 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:53:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: pgmap v2100: 99 pgs: 2 creating+peering, 14 unknown, 4 active+clean+remapped, 17 active+undersized, 5 active+undersized+degraded, 57 active+clean; 425 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/69 objects degraded (13.043%) 2024-04-12T11:53:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: 1.4 scrub starts 2024-04-12T11:53:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: 1.4 scrub ok 2024-04-12T11:53:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:53:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T11:53:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: osdmap e1272: 8 total, 7 up, 8 in 2024-04-12T11:53:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: 1.8 scrub starts 2024-04-12T11:53:11.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: 1.8 scrub ok 2024-04-12T11:53:11.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: 1.5 scrub starts 2024-04-12T11:53:11.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: 1.5 scrub ok 2024-04-12T11:53:11.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: 1.2 scrub starts 2024-04-12T11:53:11.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:11 smithi087 ceph-mon[89014]: 1.2 scrub ok 2024-04-12T11:53:11.481 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:53:11.494 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:11.500 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:53:11.517 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:11.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:11.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:11.522 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:53:11.522 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:53:11.522 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:53:11.522 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:53:11.522 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:53:11.522 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:53:11.522 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-04-12T11:53:11.522 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:53:11.522 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: pgmap v2100: 99 pgs: 2 creating+peering, 14 unknown, 4 active+clean+remapped, 17 active+undersized, 5 active+undersized+degraded, 57 active+clean; 425 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/69 objects degraded (13.043%) 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: 1.4 scrub starts 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: 1.4 scrub ok 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: osdmap e1272: 8 total, 7 up, 8 in 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: 1.8 scrub starts 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: 1.8 scrub ok 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: 1.5 scrub starts 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: 1.5 scrub ok 2024-04-12T11:53:11.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: 1.2 scrub starts 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[97787]: 1.2 scrub ok 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: pgmap v2100: 99 pgs: 2 creating+peering, 14 unknown, 4 active+clean+remapped, 17 active+undersized, 5 active+undersized+degraded, 57 active+clean; 425 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/69 objects degraded (13.043%) 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: 1.4 scrub starts 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: 1.4 scrub ok 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: osdmap e1272: 8 total, 7 up, 8 in 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: 1.8 scrub starts 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: 1.8 scrub ok 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: 1.5 scrub starts 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: 1.5 scrub ok 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: 1.2 scrub starts 2024-04-12T11:53:11.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:11 smithi029 ceph-mon[93650]: 1.2 scrub ok 2024-04-12T11:53:11.619 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:11.642 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:11.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:11.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:11.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:11.769 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:11.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:11.870 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:11.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:11.895 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:11.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:11.996 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:12.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:12.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:12.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:12.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:12.143 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:12.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:12.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:12.245 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:12.268 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:12.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:12.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:12.370 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:12.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:12.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:12.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:12.496 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:12.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:12 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2720727884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:12.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:12 smithi029 ceph-mon[93650]: 1.a scrub starts 2024-04-12T11:53:12.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:12 smithi029 ceph-mon[93650]: 1.a scrub ok 2024-04-12T11:53:12.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:12 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2720727884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:12.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:12 smithi029 ceph-mon[97787]: 1.a scrub starts 2024-04-12T11:53:12.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:12 smithi029 ceph-mon[97787]: 1.a scrub ok 2024-04-12T11:53:12.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:12.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:12.622 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:12.643 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:53:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:53:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:53:12.643 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:12 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2720727884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:12.643 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:12 smithi087 ceph-mon[89014]: 1.a scrub starts 2024-04-12T11:53:12.643 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:12 smithi087 ceph-mon[89014]: 1.a scrub ok 2024-04-12T11:53:12.646 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:12.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:12.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:12.748 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:12.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:12.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:12.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:12.873 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:12.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:12.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:12.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:12.969 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-04-12T11:53:12.969 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-04-12T11:53:12.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:13.023 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:13.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:13.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:13.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:13.149 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:13.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:13.162 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:53:13.162 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:53:13.250 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:13.274 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:13.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:13.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:13.376 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:13.400 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:13.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:13.441 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-04-12T11:53:13.441 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:53:13.441 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-04-12T11:53:13.441 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-04-12T11:53:13.441 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-04-12T11:53:13.442 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-04-12T11:53:13.501 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:13.525 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:13.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:13.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:13.627 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:13.652 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:13.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:13.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:13 smithi087 ceph-mon[89014]: pgmap v2102: 99 pgs: 2 creating+peering, 11 unknown, 4 active+clean+remapped, 17 active+undersized, 5 active+undersized+degraded, 60 active+clean; 425 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/69 objects degraded (13.043%) 2024-04-12T11:53:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9/69 objects degraded (13.043%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:13.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:13.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:13 smithi029 ceph-mon[93650]: pgmap v2102: 99 pgs: 2 creating+peering, 11 unknown, 4 active+clean+remapped, 17 active+undersized, 5 active+undersized+degraded, 60 active+clean; 425 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/69 objects degraded (13.043%) 2024-04-12T11:53:13.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9/69 objects degraded (13.043%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:13.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:13 smithi029 ceph-mon[97787]: pgmap v2102: 99 pgs: 2 creating+peering, 11 unknown, 4 active+clean+remapped, 17 active+undersized, 5 active+undersized+degraded, 60 active+clean; 425 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/69 objects degraded (13.043%) 2024-04-12T11:53:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9/69 objects degraded (13.043%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:13.778 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:13.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:13.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:13.880 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:13.905 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:13.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:13.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:13.983 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-04-12T11:53:14.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:14.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:14.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:14.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:14.087 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:53:14.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:14.158 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:14.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:14.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:14.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:14.284 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:14.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:14.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:14.386 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:14.409 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:14.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:14.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:14.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:14.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:14.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:14.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:14.636 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:14.660 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:14.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:14.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:14.786 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:14.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:14.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:14.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:14.911 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:14.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:14.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:14.970 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:53:15.013 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:15.038 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:15.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:15.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:15.140 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:15.166 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:15.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:15.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:15.188 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:15.267 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:53:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:53:14] "GET /metrics HTTP/1.1" 200 34178 "" "Prometheus/2.43.0" 2024-04-12T11:53:15.291 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:15.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:15.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:15.371 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-04-12T11:53:15.371 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:53:15.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:15.417 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:15.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:15.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:15.419 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:15 smithi087 ceph-mon[89014]: pgmap v2103: 99 pgs: 4 active+clean+remapped, 22 active+undersized, 5 active+undersized+degraded, 68 active+clean; 634 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/72 objects degraded (12.500%); 2/72 objects misplaced (2.778%) 2024-04-12T11:53:15.419 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:15 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4022510693' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:15.419 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:15 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:15.419 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:15 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4265028844' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:53:15.419 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:15 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:53:15.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:15.544 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:15.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:15.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:15.633 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:53:15.645 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:15.661 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:53:15.661 INFO:teuthology.orchestra.run.smithi087.stdout: "success": "" 2024-04-12T11:53:15.661 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:53:15.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:15.671 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:15.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:15 smithi029 ceph-mon[93650]: pgmap v2103: 99 pgs: 4 active+clean+remapped, 22 active+undersized, 5 active+undersized+degraded, 68 active+clean; 634 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/72 objects degraded (12.500%); 2/72 objects misplaced (2.778%) 2024-04-12T11:53:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:15 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4022510693' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:15 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:15 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4265028844' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:53:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:15 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:53:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:15 smithi029 ceph-mon[97787]: pgmap v2103: 99 pgs: 4 active+clean+remapped, 22 active+undersized, 5 active+undersized+degraded, 68 active+clean; 634 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/72 objects degraded (12.500%); 2/72 objects misplaced (2.778%) 2024-04-12T11:53:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:15 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4022510693' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:15 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:15 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4265028844' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:53:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:15 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:53:15.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:15.795 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:15.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:15.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:15.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:15.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:15.921 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:15.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:16.022 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:16.046 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:16.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:16.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:16.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:16.172 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:16.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:16.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:16.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:16.298 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:16.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:16.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:16.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:16.428 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:16.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:16.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:16 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:53:16.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:16 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:53:16.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:16 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:53:16.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:16 smithi087 ceph-mon[89014]: osdmap e1273: 8 total, 7 up, 8 in 2024-04-12T11:53:16.529 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:16.553 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:16.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:16.654 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:16.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:16 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:53:16.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:16 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:53:16.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:16 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:53:16.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:16 smithi029 ceph-mon[97787]: osdmap e1273: 8 total, 7 up, 8 in 2024-04-12T11:53:16.658 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:16 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:53:16.658 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:16 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:53:16.659 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:16 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:53:16.659 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:16 smithi029 ceph-mon[93650]: osdmap e1273: 8 total, 7 up, 8 in 2024-04-12T11:53:16.678 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:16.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:16.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:16.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:16.805 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:16.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:16.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:16.907 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:16.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:16.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:17.034 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:17.060 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:17.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:17.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:17.161 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:17.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:17.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:17.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:17.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:17.314 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:17.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:17.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:17.415 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:17.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:17 smithi029 ceph-mon[97787]: pgmap v2105: 99 pgs: 4 active+clean+remapped, 22 active+undersized, 5 active+undersized+degraded, 68 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/72 objects degraded (12.500%); 2/72 objects misplaced (2.778%) 2024-04-12T11:53:17.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:17 smithi029 ceph-mon[97787]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:53:17.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:17 smithi029 ceph-mon[93650]: pgmap v2105: 99 pgs: 4 active+clean+remapped, 22 active+undersized, 5 active+undersized+degraded, 68 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/72 objects degraded (12.500%); 2/72 objects misplaced (2.778%) 2024-04-12T11:53:17.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:17 smithi029 ceph-mon[93650]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:53:17.441 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:17.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:17 smithi087 ceph-mon[89014]: pgmap v2105: 99 pgs: 4 active+clean+remapped, 22 active+undersized, 5 active+undersized+degraded, 68 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/72 objects degraded (12.500%); 2/72 objects misplaced (2.778%) 2024-04-12T11:53:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:17 smithi087 ceph-mon[89014]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:53:17.543 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:17.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:17.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:17.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:17.669 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:17.681 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-04-12T11:53:17.683 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-12T11:53:17.684 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-12T11:53:17.691 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:17.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:17.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:17.731 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2024-04-12T11:53:17.731 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-12T11:53:17.731 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_256330 2024-04-12T11:53:17.731 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:53:17.731 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2024-04-12T11:53:17.793 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:17.829 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:17.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:17.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:17.905 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:53:17.906 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:53:17.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:17.976 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:17.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:17.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:18.062 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:53:18.078 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:18.111 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:53:18.116 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:18.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:18.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:18.151 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:53:18.192 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:53:18.218 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:18.236 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:53:18.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:18.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:18.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:18.277 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:53:18.315 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:53:18.315 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:53:18.315 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:53:18.316 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:53:18.316 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:53:18.316 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:53:18.316 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-04-12T11:53:18.316 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:53:18.316 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:53:18.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:18.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:18.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:18.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:18.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:18.541 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:18.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:18.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:18.589 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:18 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/696006227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:18.589 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9/72 objects degraded (12.500%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:18.631 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-04-12T11:53:18.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:18.680 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:18.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:18.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:18.731 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3773 3757 234.8 234.812 0.00555652 0.00424647 2024-04-12T11:53:18.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:18 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/696006227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:18.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9/72 objects degraded (12.500%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:18.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:18 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/696006227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:18.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9/72 objects degraded (12.500%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:18.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:18.819 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:18.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:18.921 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:18.961 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:18.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:18.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:18.974 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:53:19.063 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:19.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:19.105 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:19.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:19.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:19.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:19.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:19.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:19.345 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:19.383 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:19.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:19.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-04-12T11:53:19.480 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-04-12T11:53:19.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:19.522 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:19.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:19.624 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:19.659 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:19.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:19.710 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:19 smithi087 ceph-mon[89014]: pgmap v2106: 99 pgs: 4 active+clean+remapped, 22 active+undersized, 5 active+undersized+degraded, 68 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/72 objects degraded (12.500%); 2/72 objects misplaced (2.778%) 2024-04-12T11:53:19.731 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 7345 7329 229.014 223.25 0.00216929 0.00435829 2024-04-12T11:53:19.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:19.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:19 smithi029 ceph-mon[93650]: pgmap v2106: 99 pgs: 4 active+clean+remapped, 22 active+undersized, 5 active+undersized+degraded, 68 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/72 objects degraded (12.500%); 2/72 objects misplaced (2.778%) 2024-04-12T11:53:19.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:19 smithi029 ceph-mon[97787]: pgmap v2106: 99 pgs: 4 active+clean+remapped, 22 active+undersized, 5 active+undersized+degraded, 68 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9/72 objects degraded (12.500%); 2/72 objects misplaced (2.778%) 2024-04-12T11:53:19.800 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:19.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:19.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:19.902 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:19.929 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:19.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:19.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:20.030 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:20.061 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:20.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:20.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:20.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:20.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:20.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:20.301 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:20.340 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:20.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:20.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:20.434 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:53:20.441 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:20.477 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:20.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:20.578 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:20.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:20.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:20.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:20.715 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:20.731 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 10474 10458 217.843 195.562 0.00512485 0.00458183 2024-04-12T11:53:20.749 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:20.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:20.810 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:20.851 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:20.887 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:20.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:20.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:20.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:21.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:21.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:21.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:21.177 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:21.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:21.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:21.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:21.323 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:21.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:21.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:21.426 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:21.461 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:21.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:21.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:21.482 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:53:21.541 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:21 smithi029 ceph-mon[97787]: pgmap v2107: 99 pgs: 4 active+clean+remapped, 19 active+undersized, 8 active+undersized+degraded, 68 active+clean; 50 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 4.9 MiB/s wr, 63 op/s; 600/2445 objects degraded (24.540%); 2/2445 objects misplaced (0.082%) 2024-04-12T11:53:21.541 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:21.541 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:21.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:21 smithi029 ceph-mon[93650]: pgmap v2107: 99 pgs: 4 active+clean+remapped, 19 active+undersized, 8 active+undersized+degraded, 68 active+clean; 50 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 4.9 MiB/s wr, 63 op/s; 600/2445 objects degraded (24.540%); 2/2445 objects misplaced (0.082%) 2024-04-12T11:53:21.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:21.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:21.563 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:21.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:21 smithi087 ceph-mon[89014]: pgmap v2107: 99 pgs: 4 active+clean+remapped, 19 active+undersized, 8 active+undersized+degraded, 68 active+clean; 50 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 4.9 MiB/s wr, 63 op/s; 600/2445 objects degraded (24.540%); 2/2445 objects misplaced (0.082%) 2024-04-12T11:53:21.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:21.577 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:21.591 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:21.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:21.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:21.654 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(8.d 1211/1209 [PushOp(8:b0760182:::benchmark_data_smithi029_194418_object88037:head, version: 1195'5537, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b0760182:::benchmark_data_smithi029_194418_object88037:head@1195'5537, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:57.138220+0000", 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 444.51210822799999, 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.22547447800000001, 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.655 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138220+0000", 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138220+0000", 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138213+0000", 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999933 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138258+0000", 2024-04-12T11:53:21.656 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4864999999999998e-05 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138278+0000", 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9587000000000001e-05 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.138294+0000", 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6532999999999999e-05 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.360093+0000", 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.221798943 2024-04-12T11:53:21.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.360105+0000", 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1331e-05 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:57.363694+0000", 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0035898330000000002 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(8.d 1211/1209 [PushOp(8:b9eded05:::benchmark_data_smithi029_194418_object88441:head, version: 1195'5559, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b9eded05:::benchmark_data_smithi029_194418_object88441:head@1195'5559, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T11:53:21.658 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:58.071527+0000", 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 443.57880115299997, 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27444023299999998, 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071527+0000", 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071527+0000", 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.659 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071526+0000", 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071565+0000", 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9232e-05 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071591+0000", 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5477e-05 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.660 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.071609+0000", 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7731999999999999e-05 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.341174+0000", 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.26956538899999999 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.341186+0000", 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2300000000000001e-05 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:58.345967+0000", 2024-04-12T11:53:21.661 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0047811040000000004 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(8.d 1211/1209 [PushOp(8:b10db278:::benchmark_data_smithi029_194418_object88780:head, version: 1195'5581, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:b10db278:::benchmark_data_smithi029_194418_object88780:head@1195'5581, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:45:59.765684+0000", 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 441.884644541, 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.316417805, 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.662 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765684+0000", 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765684+0000", 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765682+0000", 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765726+0000", 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3307999999999999e-05 2024-04-12T11:53:21.663 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765751+0000", 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5627e-05 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:45:59.765769+0000", 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7890999999999999e-05 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.664 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.076685+0000", 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.31091553999999999 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.076700+0000", 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5271e-05 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:46:00.082101+0000", 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.005401622 2024-04-12T11:53:21.665 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:108891 9.3 9:c60e3d25:::benchmark_data_smithi029_222585_object114:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1242)", 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:49:15.146417+0000", 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 246.50391096199999, 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.17430869600000001, 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 108891 2024-04-12T11:53:21.666 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146417+0000", 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146417+0000", 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146415+0000", 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:53:21.667 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146424+0000", 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1929999999999999e-06 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146440+0000", 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5299999999999999e-05 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.146451+0000", 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1250000000000001e-05 2024-04-12T11:53:21.668 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.160678+0000", 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014226518 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.160865+0000", 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018702000000000001 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.161061+0000", 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000196367 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.669 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.164151+0000", 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030902199999999999 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.304438+0000", 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.14028655100000001 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.316514+0000", 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.012076304 2024-04-12T11:53:21.670 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.316573+0000", 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9247999999999999e-05 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:15.320726+0000", 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0041527200000000004 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:129613 9.3 9:c6cf49d4:::benchmark_data_smithi029_222585_object20836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:49:49.271673+0000", 2024-04-12T11:53:21.671 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 212.378654885, 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.175389875, 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 129613 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271673+0000", 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.672 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271673+0000", 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271673+0000", 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271676+0000", 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.8870000000000001e-06 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.673 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271683+0000", 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1300000000000003e-06 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271688+0000", 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0130000000000001e-06 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271731+0000", 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3322000000000003e-05 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271881+0000", 2024-04-12T11:53:21.674 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000149419 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.272035+0000", 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000154726 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.277195+0000", 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0051592620000000004 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.281954+0000", 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0047598600000000003 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.675 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.443501+0000", 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.16154670400000001 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.443561+0000", 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9360000000000001e-05 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.447063+0000", 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003502668 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.676 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:129614 9.3 9:ce735544:::benchmark_data_smithi029_222585_object20837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1244)", 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:49:49.271826+0000", 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 212.378501989, 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.17772210499999999, 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 129614 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271826+0000", 2024-04-12T11:53:21.677 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271826+0000", 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271826+0000", 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271829+0000", 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.597e-06 2024-04-12T11:53:21.678 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271837+0000", 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6020000000000002e-06 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.271843+0000", 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6670000000000001e-06 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.274657+0000", 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0028145039999999998 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.679 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.274783+0000", 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000125699 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.274928+0000", 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014504200000000001 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.277339+0000", 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0024109079999999998 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.285389+0000", 2024-04-12T11:53:21.680 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0080502609999999995 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.447135+0000", 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.16174655099999999 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.447197+0000", 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1599000000000004e-05 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:49:49.449548+0000", 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002351255 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.681 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:143341 9.3 9:c70bbc5a:::benchmark_data_smithi029_222585_object34564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:50:17.572936+0000", 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 184.07739188100001, 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.17730494799999999, 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 143341 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.682 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.572936+0000", 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.572936+0000", 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.572935+0000", 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.683 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.572942+0000", 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3640000000000001e-06 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.572955+0000", 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3001e-05 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.572964+0000", 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3159999999999997e-06 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.573015+0000", 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1748e-05 2024-04-12T11:53:21.684 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.573665+0000", 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000649947 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.743978+0000", 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.17031252199999999 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.744091+0000", 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000113232 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.685 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.744495+0000", 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00040393699999999999 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.748072+0000", 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003577434 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.748116+0000", 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3485000000000001e-05 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.686 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.749297+0000", 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0011807639999999999 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.749379+0000", 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2616999999999999e-05 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.750241+0000", 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00086200600000000001 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.687 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.688 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:143342 9.3 9:c4b68419:::benchmark_data_smithi029_222585_object34565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-04-12T11:53:21.688 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:50:17.574897+0000", 2024-04-12T11:53:21.688 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 184.07543110099999, 2024-04-12T11:53:21.688 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.18303891899999999, 2024-04-12T11:53:21.688 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.688 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.688 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.688 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.688 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 143342 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.574897+0000", 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.574897+0000", 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.689 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.574896+0000", 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.574903+0000", 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6579999999999996e-06 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.574913+0000", 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0448999999999999e-05 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.574919+0000", 2024-04-12T11:53:21.690 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5879999999999997e-06 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.574954+0000", 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4604000000000003e-05 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.575401+0000", 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00044745700000000001 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.746899+0000", 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.171497497 2024-04-12T11:53:21.691 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.746992+0000", 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3673999999999998e-05 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.747239+0000", 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00024690299999999998 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.748443+0000", 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001204225 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.692 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.750285+0000", 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0018420419999999999 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.756680+0000", 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0063949050000000002 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.756779+0000", 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8361000000000002e-05 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.693 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:17.757936+0000", 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0011573480000000001 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:144365 9.3 9:c37e2b5a:::benchmark_data_smithi029_222585_object35588:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:50:20.635813+0000", 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 181.01451487099999, 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.17279238799999999, 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.694 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 144365 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.635813+0000", 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.635813+0000", 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.695 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.635812+0000", 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.635818+0000", 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.674e-06 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.635831+0000", 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2296999999999999e-05 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.696 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.635838+0000", 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0690000000000004e-06 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.637150+0000", 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001312031 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.637613+0000", 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00046368899999999998 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.642790+0000", 2024-04-12T11:53:21.697 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0051766479999999998 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.642905+0000", 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011522400000000001 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.643293+0000", 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000387904 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.645480+0000", 2024-04-12T11:53:21.698 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0021863830000000001 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.646323+0000", 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00084326699999999997 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.807089+0000", 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.16076570500000001 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.807128+0000", 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9209999999999999e-05 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.699 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:20.808606+0000", 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0014779439999999999 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:147961 9.d 9:baf086e5:::benchmark_data_smithi029_222585_object39184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:50:31.822103+0000", 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 169.82822501499999, 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34741429299999999, 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.700 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 147961 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.822103+0000", 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.822103+0000", 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.701 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.822102+0000", 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.822109+0000", 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7530000000000004e-06 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.822122+0000", 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2799999999999999e-05 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.702 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.822128+0000", 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8780000000000003e-06 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.822164+0000", 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6866999999999998e-05 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.822265+0000", 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000100182 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,5", 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.822345+0000", 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0238e-05 2024-04-12T11:53:21.703 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.824619+0000", 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0022744150000000001 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:31.824910+0000", 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00029075299999999999 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:32.169450+0000", 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34453972900000002 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.704 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:32.169488+0000", 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8225000000000001e-05 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:50:32.169518+0000", 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.9587999999999999e-05 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:193138 9.3 9:c24ab900:::benchmark_data_smithi029_222585_object84361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:52:13.798469+0000", 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 67.851858815, 2024-04-12T11:53:21.705 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.19314917200000001, 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 193138 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.798469+0000", 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.706 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.798469+0000", 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.798469+0000", 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.798473+0000", 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.692e-06 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.798480+0000", 2024-04-12T11:53:21.707 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.554e-06 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.798486+0000", 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6439999999999997e-06 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.799214+0000", 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00072858700000000005 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.799810+0000", 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00059522299999999998 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.708 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.877500+0000", 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.077690040000000002 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.877589+0000", 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9580999999999995e-05 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.878079+0000", 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00049017500000000003 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.709 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.884129+0000", 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0060493709999999996 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.887431+0000", 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0033026409999999998 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.991125+0000", 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.103694113 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.991176+0000", 2024-04-12T11:53:21.710 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0547999999999998e-05 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.991619+0000", 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00044257600000000003 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:193141 9.13 9:c9722d17:::benchmark_data_smithi029_222585_object84364:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:52:13.799620+0000", 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 67.850708373000003, 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.19631125199999999, 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.711 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.712 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.712 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.712 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.712 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 193141 2024-04-12T11:53:21.712 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.712 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.712 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.712 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:21.713 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.799620+0000", 2024-04-12T11:53:21.713 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.713 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.713 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.713 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.713 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.799620+0000", 2024-04-12T11:53:21.713 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.713 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.713 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.799619+0000", 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.799623+0000", 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3880000000000002e-06 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.799634+0000", 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0091999999999999e-05 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.714 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.799640+0000", 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7449999999999998e-06 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.799681+0000", 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1115000000000002e-05 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.800135+0000", 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00045405199999999998 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.715 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.885352+0000", 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.085216997000000003 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.885441+0000", 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8614000000000002e-05 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.885794+0000", 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00035260499999999998 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.716 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.887688+0000", 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0018938819999999999 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.984972+0000", 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.097284226000000001 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.995088+0000", 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.010115964 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.995129+0000", 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1149000000000002e-05 2024-04-12T11:53:21.717 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.995931+0000", 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00080221100000000005 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:193150 9.13 9:caa9fba0:::benchmark_data_smithi029_222585_object84373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:52:13.814070+0000", 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 67.836258552999993, 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.180982535, 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.718 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 193150 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.814070+0000", 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.814070+0000", 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.719 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.814068+0000", 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.814075+0000", 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3520000000000003e-06 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.814086+0000", 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1583000000000001e-05 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.720 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.814093+0000", 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9959999999999996e-06 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.814143+0000", 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0139000000000002e-05 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.814699+0000", 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00055574800000000001 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.882370+0000", 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.067670469999999996 2024-04-12T11:53:21.721 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.882447+0000", 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6923999999999997e-05 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.882850+0000", 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000403366 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.887416+0000", 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0045656830000000001 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.722 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.887681+0000", 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000265028 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.994597+0000", 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.106916627 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.994636+0000", 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8890999999999997e-05 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.723 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:13.995052+0000", 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00041602400000000003 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:193369 9.e 9:7d022a85:::benchmark_data_smithi029_222585_object84592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:52:14.254554+0000", 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 67.395774208999995, 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.18461133699999999, 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.724 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 193369 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.254554+0000", 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.254554+0000", 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.254552+0000", 2024-04-12T11:53:21.725 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.254559+0000", 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7660000000000001e-06 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.254572+0000", 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2571999999999999e-05 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.254578+0000", 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.63e-06 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.726 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.254625+0000", 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6448e-05 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.254766+0000", 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014146300000000001 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.254903+0000", 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000136694 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.727 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.257998+0000", 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003095315 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.258158+0000", 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015924099999999999 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.439077+0000", 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.18091939900000001 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.439129+0000", 2024-04-12T11:53:21.728 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2287000000000003e-05 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:14.439165+0000", 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6057999999999999e-05 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:194309 9.13 9:c91302b8:::benchmark_data_smithi029_222585_object85532:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:52:15.842804+0000", 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 65.807524154000006, 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.172204677, 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.729 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 194309 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.842804+0000", 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.842804+0000", 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.730 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.842803+0000", 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.842810+0000", 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9399999999999996e-06 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.842820+0000", 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7140000000000005e-06 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.731 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.842826+0000", 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3990000000000001e-06 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.842875+0000", 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8392e-05 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.843591+0000", 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00071646999999999998 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.848119+0000", 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0045279919999999998 2024-04-12T11:53:21.732 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.848211+0000", 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1766000000000002e-05 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.848652+0000", 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00044073700000000002 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.851034+0000", 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023825560000000001 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.733 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:15.857036+0000", 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.006001386 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.014613+0000", 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.15757702800000001 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.014650+0000", 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.7539000000000001e-05 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.734 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.015009+0000", 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00035856199999999999 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:194688 9.13 9:cc72f1a4:::benchmark_data_smithi029_222585_object85911:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:52:16.477559+0000", 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 65.172769423000005, 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.244486856, 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.735 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 194688 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.477559+0000", 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.477559+0000", 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.477557+0000", 2024-04-12T11:53:21.736 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.477566+0000", 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4980000000000006e-06 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.477579+0000", 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3182e-05 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.477586+0000", 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3509999999999996e-06 2024-04-12T11:53:21.737 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.477649+0000", 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3003000000000006e-05 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.478415+0000", 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00076603900000000004 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.634917+0000", 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.15650135200000001 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.738 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.635018+0000", 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000101766 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.635468+0000", 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00044953499999999998 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.640596+0000", 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0051282690000000004 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.739 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.643146+0000", 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0025497279999999998 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.721672+0000", 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.078526422999999998 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.721707+0000", 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4931999999999997e-05 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.722046+0000", 2024-04-12T11:53:21.740 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00033848899999999998 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:194690 9.13 9:c9a4734e:::benchmark_data_smithi029_222585_object85913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:52:16.479444+0000", 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 65.170883860999993, 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.25587864799999999, 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 194690 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.741 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.479444+0000", 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.479444+0000", 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.479443+0000", 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:21.742 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.479448+0000", 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.25e-06 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.479457+0000", 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8009999999999998e-06 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.479463+0000", 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9869999999999996e-06 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.743 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.479636+0000", 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000172134 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.480280+0000", 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00064445099999999999 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.638246+0000", 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.15796623500000001 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.638336+0000", 2024-04-12T11:53:21.744 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.9215999999999994e-05 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.638753+0000", 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00041715399999999998 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.641393+0000", 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0026407959999999999 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.643190+0000", 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0017968529999999999 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.745 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.734895+0000", 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.091704424000000007 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.734944+0000", 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9330999999999999e-05 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:16.735323+0000", 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000378959 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.746 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:199497 9.3 9:c6399bf2:::benchmark_data_smithi029_222585_object90720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1259)", 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:52:23.219200+0000", 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 58.431128055999999, 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.274727944, 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 199497 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.747 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.748 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.219200+0000", 2024-04-12T11:53:21.748 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 13458 13442 209.956 186.5 0.00648965 0.0047564 2024-04-12T11:53:21.748 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.219200+0000", 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.219199+0000", 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.219204+0000", 2024-04-12T11:53:21.749 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4229999999999998e-06 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.219211+0000", 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2599999999999999e-06 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.219221+0000", 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9240000000000002e-06 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.750 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.219259+0000", 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.7676000000000003e-05 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.219811+0000", 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000552821 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.227471+0000", 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0076593190000000004 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.227556+0000", 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5748999999999995e-05 2024-04-12T11:53:21.751 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.227988+0000", 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00043193099999999998 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.235131+0000", 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0071428300000000002 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.235140+0000", 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3590000000000008e-06 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.752 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.493633+0000", 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.25849366699999998 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.493684+0000", 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0929e-05 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.493928+0000", 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00024398000000000001 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.753 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:199502 9.13 9:ccc6d8cf:::benchmark_data_smithi029_222585_object90725:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1259)", 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:52:23.224011+0000", 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 58.426317554000001, 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27319553400000002, 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 199502 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.754 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.224011+0000", 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.224011+0000", 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.224009+0000", 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.224018+0000", 2024-04-12T11:53:21.755 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8429999999999999e-06 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.224033+0000", 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5379999999999998e-05 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.224043+0000", 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7790000000000007e-06 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.224096+0000", 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3103999999999998e-05 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.756 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.224633+0000", 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00053715999999999996 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.235378+0000", 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.01074458 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.235458+0000", 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0563999999999996e-05 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.757 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.235856+0000", 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00039772700000000001 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.237335+0000", 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001478638 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.237836+0000", 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00050123599999999996 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.758 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.496895+0000", 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.259059439 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.496945+0000", 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9529e-05 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.497206+0000", 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00026125699999999998 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.759 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35445.0:199506 9.3 9:c526f1a2:::benchmark_data_smithi029_222585_object90729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1259)", 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:52:23.230242+0000", 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 58.420085817, 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.27440294999999998, 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35445", 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 199506 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.230242+0000", 2024-04-12T11:53:21.760 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.230242+0000", 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.230241+0000", 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.230248+0000", 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1810000000000004e-06 2024-04-12T11:53:21.761 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.234907+0000", 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.004659318 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.234921+0000", 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3431999999999999e-05 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.235163+0000", 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000242434 2024-04-12T11:53:21.762 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for degraded object", 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.235810+0000", 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00064714899999999997 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.497283+0000", 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.26147298499999999 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.497341+0000", 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7892000000000001e-05 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.763 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,3", 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.497788+0000", 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00044684999999999999 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.500610+0000", 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0028215610000000002 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.504008+0000", 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0033983849999999999 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:53:21.764 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.504359+0000", 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00035129300000000003 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.504413+0000", 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3332999999999999e-05 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:52:23.504645+0000", 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000232833 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:53:21.765 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:53:21.789 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:21.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:21.839 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-04-12T11:53:21.839 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:53:21.839 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 4, 5, 6, 7, 3] 2024-04-12T11:53:21.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd out 5 2024-04-12T11:53:21.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:21.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:21.934 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:21.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:22.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:22.078 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:22.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:22.179 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:22.222 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:22.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:22.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:22.324 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:22.358 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:22.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:22.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:22.460 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:22.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:22.498 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:22.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:22.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:22.638 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:53:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:53:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:53:22.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:22.639 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:22.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:22.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-04-12T11:53:22.733 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 16421 16405 204.998 185.188 0.00533936 0.00487163 2024-04-12T11:53:22.740 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:22.817 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:22.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:22.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:22.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:22.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:22.966 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:22.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:23.067 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:23.095 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:23.100 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:23.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:23.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:23.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:23.250 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:23.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:23.351 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:23.387 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:23.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:23.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:23.489 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:23.515 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:23 smithi029 ceph-mon[97787]: pgmap v2108: 99 pgs: 4 active+clean+remapped, 19 active+undersized, 8 active+undersized+degraded, 68 active+clean; 161 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 16 MiB/s wr, 239 op/s; 600/7788 objects degraded (7.704%); 2/7788 objects misplaced (0.026%) 2024-04-12T11:53:23.515 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 600/7788 objects degraded (7.704%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:23.515 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:23 smithi029 ceph-mon[93650]: pgmap v2108: 99 pgs: 4 active+clean+remapped, 19 active+undersized, 8 active+undersized+degraded, 68 active+clean; 161 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 16 MiB/s wr, 239 op/s; 600/7788 objects degraded (7.704%); 2/7788 objects misplaced (0.026%) 2024-04-12T11:53:23.515 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 600/7788 objects degraded (7.704%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:23.522 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:23.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:23.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:23.624 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:23.629 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:23.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:23.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:23 smithi087 ceph-mon[89014]: pgmap v2108: 99 pgs: 4 active+clean+remapped, 19 active+undersized, 8 active+undersized+degraded, 68 active+clean; 161 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 16 MiB/s wr, 239 op/s; 600/7788 objects degraded (7.704%); 2/7788 objects misplaced (0.026%) 2024-04-12T11:53:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 600/7788 objects degraded (7.704%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:23.733 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 18919 18903 196.845 156.125 0.00716718 0.00507482 2024-04-12T11:53:23.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:23.788 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:23.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:23.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:23.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:23.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:23.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:24.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:24.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:24.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:24.124 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:24.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:24.220 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:24.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:24.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:24.322 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:24.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:24.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:24.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:24.415 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:53:24.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:24.481 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:24.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:24.582 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:24.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:24.606 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:24.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:24.707 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:24 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2119179502' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:53:24.733 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 21196 21180 189.054 142.312 0.00410355 0.00528417 2024-04-12T11:53:24.744 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:24.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:24.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:24.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:24 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2119179502' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:53:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:24 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2119179502' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:53:24.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:24.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:24.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:24.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:25.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:25.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:25.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:25.111 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:25.145 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:25.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:25.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:25.247 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:53:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:53:24] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2024-04-12T11:53:25.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:25.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:25.373 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:25.409 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:25.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:25.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:25.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:25.549 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:25.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:25.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:25.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:25.690 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:25.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:25.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:25 smithi087 ceph-mon[89014]: pgmap v2109: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 948 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s; 5406/45531 objects degraded (11.873%); 2/45531 objects misplaced (0.004%) 2024-04-12T11:53:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:25 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:53:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:25 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2119179502' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:53:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:25 smithi087 ceph-mon[89014]: osdmap e1274: 8 total, 7 up, 8 in 2024-04-12T11:53:25.733 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 23377 23361 182.46 136.312 0.00324011 0.00547464 2024-04-12T11:53:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:25 smithi029 ceph-mon[93650]: pgmap v2109: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 948 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s; 5406/45531 objects degraded (11.873%); 2/45531 objects misplaced (0.004%) 2024-04-12T11:53:25.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:25 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:53:25.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:25 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2119179502' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:53:25.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:25 smithi029 ceph-mon[93650]: osdmap e1274: 8 total, 7 up, 8 in 2024-04-12T11:53:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:25 smithi029 ceph-mon[97787]: pgmap v2109: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 948 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s; 5406/45531 objects degraded (11.873%); 2/45531 objects misplaced (0.004%) 2024-04-12T11:53:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:25 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:53:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:25 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2119179502' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:53:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:25 smithi029 ceph-mon[97787]: osdmap e1274: 8 total, 7 up, 8 in 2024-04-12T11:53:25.791 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:25.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:25.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:25.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:25.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:25.944 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:25.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:25.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:26.046 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:26.081 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:26.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:26.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:26.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:26.206 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:26.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:26.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:26.308 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:26.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:26.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:26.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:26.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:26.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:26.489 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:26.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:26.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:26.614 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:26.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:26.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:26.716 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:26.733 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 24669 24653 171.16 80.75 0.00217847 0.00572457 2024-04-12T11:53:26.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:26.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:26.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:26.840 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:26.870 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:26.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:26.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:27.009 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:27.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:27.110 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:27.146 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:27.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:27.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:27.271 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:27.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:27.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:27.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:27.396 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:27.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:27.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:27.497 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:27.532 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:27.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:27.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:27.634 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:27.674 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:27.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:27.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:27 smithi087 ceph-mon[89014]: pgmap v2111: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 5406/52689 objects degraded (10.260%); 2/52689 objects misplaced (0.004%) 2024-04-12T11:53:27.734 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 15 26053 26038 162.701 86.5625 0.00359178 0.00614068 2024-04-12T11:53:27.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:27 smithi029 ceph-mon[97787]: pgmap v2111: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 5406/52689 objects degraded (10.260%); 2/52689 objects misplaced (0.004%) 2024-04-12T11:53:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:27 smithi029 ceph-mon[93650]: pgmap v2111: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 5406/52689 objects degraded (10.260%); 2/52689 objects misplaced (0.004%) 2024-04-12T11:53:27.776 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:27.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:27.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:27.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:27.916 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:27.939 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:27.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:27.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:28.041 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:28.064 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:28.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:28.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:28.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:28.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:28.189 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:28.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:28.290 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:28.323 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:28.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:28.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:28.424 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:28.447 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:28.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:28.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:28.549 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:28.572 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:28.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:28.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:28.674 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:28.697 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:28.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:28.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5406/52689 objects degraded (10.260%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:28.733 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 26726 26710 151.729 42 0.00994106 0.00658599 2024-04-12T11:53:28.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5406/52689 objects degraded (10.260%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5406/52689 objects degraded (10.260%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:28.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:28.839 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:28.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:28.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:28.940 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:28.964 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:28.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:28.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:29.065 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:29.088 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:29.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:29.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:29.190 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:29.215 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:29.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:29.317 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:29.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:29.355 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:29.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:29.456 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:29.495 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:29.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:29.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:29.597 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:29.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:29.636 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:29.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:29 smithi087 ceph-mon[89014]: pgmap v2112: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s; 6728/69768 objects degraded (9.643%); 2/69768 objects misplaced (0.003%) 2024-04-12T11:53:29.734 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 28380 28364 147.699 103.375 0.00170818 0.00671653 2024-04-12T11:53:29.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:29.760 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:29.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:29.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:29 smithi029 ceph-mon[93650]: pgmap v2112: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s; 6728/69768 objects degraded (9.643%); 2/69768 objects misplaced (0.003%) 2024-04-12T11:53:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:29 smithi029 ceph-mon[97787]: pgmap v2112: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s; 6728/69768 objects degraded (9.643%); 2/69768 objects misplaced (0.003%) 2024-04-12T11:53:29.862 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:29.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:29.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:29.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:29.987 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:30.011 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:30.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:30.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:30.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:30.139 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:30.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:30.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:30.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:30.281 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:30.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:30.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:30.383 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:30.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:30.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:30.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:30.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:30.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:30.547 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:30.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:30.648 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:30.671 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:30.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:30.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:30.734 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 29452 29436 141.491 67 0.00677901 0.0070619 2024-04-12T11:53:30.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:30.809 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:30.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:30.911 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:30.946 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:30.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:30.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:31.048 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:31.088 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:31.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:31.189 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:31.228 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:31.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:31.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:31.330 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:31.354 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:31.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:31.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:31.456 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:31.497 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:31.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:31.599 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:31.632 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:31.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:31.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:31 smithi087 ceph-mon[89014]: pgmap v2113: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s; 8251/75837 objects degraded (10.880%); 2/75837 objects misplaced (0.003%) 2024-04-12T11:53:31.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:31.734 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 31268 31252 139.491 113.5 0.00172404 0.0071093 2024-04-12T11:53:31.757 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:31.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:31.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:31 smithi029 ceph-mon[93650]: pgmap v2113: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s; 8251/75837 objects degraded (10.880%); 2/75837 objects misplaced (0.003%) 2024-04-12T11:53:31.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:31 smithi029 ceph-mon[97787]: pgmap v2113: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s; 8251/75837 objects degraded (10.880%); 2/75837 objects misplaced (0.003%) 2024-04-12T11:53:31.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:31.883 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:31.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:31.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:31.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:32.008 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:32.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:32.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:32.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:32.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:32.134 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:32.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:32.235 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:32.274 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:32.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:32.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:32.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:32.411 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:32.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:32.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:32.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:32.553 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:32.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:32.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:32.655 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:32.678 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:32.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:32.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:53:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:53:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:53:32.734 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 32529 32513 135.446 78.8125 0.00242642 0.00737197 2024-04-12T11:53:32.780 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:32.804 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:32.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:32.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:32.905 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:32.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:32.946 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:32.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:33.047 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:33.088 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:33.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:33.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:33.190 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:33.226 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:33.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:33.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:33.328 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:33.351 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:33.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:33.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:33.453 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:33.478 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:33.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:33.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:33.580 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:33.604 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:33.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:33.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:33.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:33 smithi087 ceph-mon[89014]: pgmap v2114: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s; 8251/79512 objects degraded (10.377%); 2/79512 objects misplaced (0.003%) 2024-04-12T11:53:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 8251/79512 objects degraded (10.377%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:33.734 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 33717 33701 131.621 74.25 0.00754083 0.0075915 2024-04-12T11:53:33.744 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:33.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:33.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:33 smithi029 ceph-mon[97787]: pgmap v2114: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s; 8251/79512 objects degraded (10.377%); 2/79512 objects misplaced (0.003%) 2024-04-12T11:53:33.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 8251/79512 objects degraded (10.377%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:33 smithi029 ceph-mon[93650]: pgmap v2114: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s; 8251/79512 objects degraded (10.377%); 2/79512 objects misplaced (0.003%) 2024-04-12T11:53:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 8251/79512 objects degraded (10.377%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:33.847 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:33.870 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:33.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:33.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:33.972 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:33.995 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:33.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:33.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:34.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:34.129 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:34.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:34.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:34.231 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:34.271 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:34.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:34.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:34.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:34.409 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:34.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:34.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:34.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:34.534 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:34.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:34.636 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:34.659 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:34.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:34.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:34.734 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 34934 34918 128.353 76.0625 0.00200796 0.00777045 2024-04-12T11:53:34.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:34.792 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:34.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:34.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:34.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:34.923 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:34.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:34.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:35.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:35.048 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:35.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:35.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:35.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:35.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:35.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:53:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:53:34] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2024-04-12T11:53:35.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:35.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:35.332 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:35.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:35.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:35.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:35.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:35.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:35.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:35.597 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:35.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:35.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:35.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:35 smithi087 ceph-mon[89014]: pgmap v2115: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 10260/95502 objects degraded (10.743%); 2/95502 objects misplaced (0.002%) 2024-04-12T11:53:35.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:35.732 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:35.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:35.734 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 36335 36319 126.086 87.5625 0.0021478 0.00791761 2024-04-12T11:53:35.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:35 smithi029 ceph-mon[93650]: pgmap v2115: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 10260/95502 objects degraded (10.743%); 2/95502 objects misplaced (0.002%) 2024-04-12T11:53:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:35 smithi029 ceph-mon[97787]: pgmap v2115: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 10260/95502 objects degraded (10.743%); 2/95502 objects misplaced (0.002%) 2024-04-12T11:53:35.833 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:35.872 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:35.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:35.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:35.973 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:36.012 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:36.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:36.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:36.114 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:36.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:36.138 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:36.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:36.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:36.279 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:36.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:36.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:36.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:36.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:36.419 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:36.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:36.520 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:36.555 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:36.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:36.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:36.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:36.694 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:36.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:36.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:36.734 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 38120 38104 125.321 111.562 0.00420356 0.00797324 2024-04-12T11:53:36.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:36.836 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:36.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:36.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:36.938 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:36.966 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:36.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:36.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:37.068 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:37.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:37.101 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:37.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:37.201 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:37.233 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:37.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:37.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:37.372 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:37.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:37.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:37.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:37.508 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:37.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:37.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:37.610 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:37.642 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:37.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:37.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:37 smithi087 ceph-mon[89014]: pgmap v2116: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 10260/98853 objects degraded (10.379%); 2/98853 objects misplaced (0.002%) 2024-04-12T11:53:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:37.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:37.734 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:53:37.734435+0000 min lat: 0.00116361 max lat: 0.517271 avg lat: 0.00798416 2024-04-12T11:53:37.735 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:53:37.735 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 40078 40062 125.173 122.375 0.00501592 0.00798416 2024-04-12T11:53:37.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:37.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:37 smithi029 ceph-mon[97787]: pgmap v2116: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 10260/98853 objects degraded (10.379%); 2/98853 objects misplaced (0.002%) 2024-04-12T11:53:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:37 smithi029 ceph-mon[93650]: pgmap v2116: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 10260/98853 objects degraded (10.379%); 2/98853 objects misplaced (0.002%) 2024-04-12T11:53:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:37.783 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:37.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:37.884 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:37.920 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:37.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:38.022 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:38.059 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:38.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:38.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:38.162 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:38.196 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:38.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:38.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:38.297 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:38.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:38.338 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:38.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:38.396 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 10260/98853 objects degraded (10.379%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:38.396 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 10260/98853 objects degraded (10.379%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:38.439 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:38.477 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:38.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:38.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:38.579 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:38.621 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:38.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:38.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 10260/98853 objects degraded (10.379%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:38.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:38.735 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 42136 42120 125.336 128.625 0.00819446 0.00797336 2024-04-12T11:53:38.764 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:38.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:38.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:38.865 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:38.900 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:38.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:38.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:39.001 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:39.035 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:39.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:39.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:39.136 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:39.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:39.177 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:39.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:39.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:39.317 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:39.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:39.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:39.419 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:39.453 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:39.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:39.554 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:39.600 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:39.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:39.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:39.667 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:39 smithi029 ceph-mon[93650]: pgmap v2117: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s; 11234/111576 objects degraded (10.068%); 2/111576 objects misplaced (0.002%) 2024-04-12T11:53:39.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:39 smithi029 ceph-mon[97787]: pgmap v2117: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s; 11234/111576 objects degraded (10.068%); 2/111576 objects misplaced (0.002%) 2024-04-12T11:53:39.702 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:39 smithi087 ceph-mon[89014]: pgmap v2117: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s; 11234/111576 objects degraded (10.068%); 2/111576 objects misplaced (0.002%) 2024-04-12T11:53:39.735 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 44216 44200 125.547 130 0.0058596 0.00796008 2024-04-12T11:53:39.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:39.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:39.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:39.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:39.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:39.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:39.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:39.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:40.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:40.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:40.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:40.116 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:40.147 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:40.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:40.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:40.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:40.272 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:40.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:40.373 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:40.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:40.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:40.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:40.544 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:40.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:40.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:40.646 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:40.684 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:40.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:40.735 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 46092 46076 125.186 117.25 0.00841076 0.00798188 2024-04-12T11:53:40.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:40.825 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:40.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:40.927 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:40.965 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:40.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:40.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:41.066 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:41.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:41.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:41.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:41.203 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:41.243 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:41.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:41.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:41.345 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:41.378 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:41.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:41.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:41.480 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:41.511 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:41.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:41.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:41.569 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:53:41.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:41 smithi029 ceph-mon[97787]: pgmap v2118: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 12809/122778 objects degraded (10.433%); 2/122778 objects misplaced (0.002%) 2024-04-12T11:53:41.606 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:41 smithi029 ceph-mon[93650]: pgmap v2118: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 12809/122778 objects degraded (10.433%); 2/122778 objects misplaced (0.002%) 2024-04-12T11:53:41.613 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:41.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:41.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:41.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:41 smithi087 ceph-mon[89014]: pgmap v2118: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s; 12809/122778 objects degraded (10.433%); 2/122778 objects misplaced (0.002%) 2024-04-12T11:53:41.735 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 47964 47948 124.844 117 0.00648444 0.00800498 2024-04-12T11:53:41.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:41.787 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:41.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:41.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:41.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:41.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:41.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:41.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:42.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:42.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:42.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:42.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:42.211 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:42.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:42.312 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:42.346 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:42.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:42.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:42.448 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:42.480 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:42.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:42.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:42.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:42.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:42.633 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:42.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:53:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:53:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:53:42.735 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:42.735 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 49715 49699 124.228 109.438 0.0213945 0.00804386 2024-04-12T11:53:42.791 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:42.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:42.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:42.847 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:53:42.890 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:53:42.893 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:42.933 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:53:42.941 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:42.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:42.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:42.977 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:53:43.018 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:53:43.043 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:43.064 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:53:43.068 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:43.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:43.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:43.091 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:53:43.091 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:53:43.092 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:53:43.092 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:53:43.092 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:53:43.092 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:53:43.092 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-04-12T11:53:43.092 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:53:43.092 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:53:43.169 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:43.203 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:43.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:43.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:43.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:43.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:43.332 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:43.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:43.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:43.468 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:43.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:43.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:43.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:43.606 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:43.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:43.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:43.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:43 smithi029 ceph-mon[93650]: pgmap v2119: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 12809/122778 objects degraded (10.433%); 2/122778 objects misplaced (0.002%) 2024-04-12T11:53:43.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:43 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1090447115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:43.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 12809/122778 objects degraded (10.433%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:43.652 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:43 smithi029 ceph-mon[97787]: pgmap v2119: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 12809/122778 objects degraded (10.433%); 2/122778 objects misplaced (0.002%) 2024-04-12T11:53:43.652 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:43 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1090447115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:43.652 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 12809/122778 objects degraded (10.433%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:43.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:43 smithi087 ceph-mon[89014]: pgmap v2119: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 12809/122778 objects degraded (10.433%); 2/122778 objects misplaced (0.002%) 2024-04-12T11:53:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:43 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1090447115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 12809/122778 objects degraded (10.433%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:43.735 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 51355 51339 123.392 102.5 0.0069939 0.00809922 2024-04-12T11:53:43.741 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:43.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:43.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:43.843 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:43.878 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:43.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:43.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:43.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:44.023 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:44.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:44.124 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:44.131 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:53:44.131 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:53:44.156 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:44.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:44.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:44.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:44.297 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:44.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:44.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:44.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:44.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:44.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:44.540 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:44.570 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:44.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:44.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:44.672 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:44.708 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:44.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:44.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:44.735 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 52997 52981 122.622 102.625 0.0061225 0.00814953 2024-04-12T11:53:44.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:44.849 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:44.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:44.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:44.865 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:44.952 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:44.984 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:44.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:44.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:45.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:45.124 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:45.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:45.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:45.226 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:45.260 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:45.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:45.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:45.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T11:53:44.897+0000 7fae24212700 -1 mon.a@0(leader).osd e1274 definitely_dead 0 2024-04-12T11:53:45.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:53:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:53:44] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.43.0" 2024-04-12T11:53:45.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:45.367 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.5. 2024-04-12T11:53:45.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:45.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:45.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:45.470 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-04-12T11:53:45.471 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-04-12T11:53:45.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:45.527 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:45.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:45.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:45.629 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:45.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:45.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:45.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:45 smithi087 ceph-mon[89014]: pgmap v2120: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s; 15752/146400 objects degraded (10.760%); 2/146400 objects misplaced (0.001%) 2024-04-12T11:53:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:45 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2507795707' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2024-04-12T11:53:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:45 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2024-04-12T11:53:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:45 smithi087 ceph-mon[89014]: Client client.admin marked osd.5 out, after it was down for 43 seconds 2024-04-12T11:53:45.735 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 54155 54139 120.828 72.375 0.00235914 0.00816984 2024-04-12T11:53:45.755 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:45 smithi029 ceph-mon[97787]: pgmap v2120: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s; 15752/146400 objects degraded (10.760%); 2/146400 objects misplaced (0.001%) 2024-04-12T11:53:45.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:45 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2507795707' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2024-04-12T11:53:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:45 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2024-04-12T11:53:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:45 smithi029 ceph-mon[97787]: Client client.admin marked osd.5 out, after it was down for 43 seconds 2024-04-12T11:53:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:45 smithi029 ceph-mon[93650]: pgmap v2120: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s; 15752/146400 objects degraded (10.760%); 2/146400 objects misplaced (0.001%) 2024-04-12T11:53:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:45 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2507795707' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2024-04-12T11:53:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:45 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2024-04-12T11:53:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:45 smithi029 ceph-mon[93650]: Client client.admin marked osd.5 out, after it was down for 43 seconds 2024-04-12T11:53:45.778 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:45.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:45.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:45.879 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:45.905 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:45.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:45.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:46.006 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:46.030 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:46.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:46.132 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:46.155 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:46.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:46.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:46.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:46.283 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:46.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:46.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:46.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:46.410 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:46.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:46.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:46.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-04-12T11:53:46.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:46.572 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:46.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:46.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:46.674 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:46.698 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:46.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:46 smithi087 ceph-mon[89014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:53:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:46 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-04-12T11:53:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:46 smithi087 ceph-mon[89014]: osdmap e1275: 8 total, 7 up, 7 in 2024-04-12T11:53:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:46 smithi087 ceph-mon[89014]: osdmap e1276: 8 total, 7 up, 7 in 2024-04-12T11:53:46.736 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 54155 54139 116.661 0 - 0.00816984 2024-04-12T11:53:46.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:46 smithi029 ceph-mon[93650]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:53:46.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:46 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-04-12T11:53:46.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:46 smithi029 ceph-mon[93650]: osdmap e1275: 8 total, 7 up, 7 in 2024-04-12T11:53:46.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:46 smithi029 ceph-mon[93650]: osdmap e1276: 8 total, 7 up, 7 in 2024-04-12T11:53:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:46 smithi029 ceph-mon[97787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T11:53:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:46 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-04-12T11:53:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:46 smithi029 ceph-mon[97787]: osdmap e1275: 8 total, 7 up, 7 in 2024-04-12T11:53:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:46 smithi029 ceph-mon[97787]: osdmap e1276: 8 total, 7 up, 7 in 2024-04-12T11:53:46.800 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:46.823 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:46.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:46.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:46.948 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:46.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:47.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:47.073 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:47.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:47.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:47.175 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:47.198 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:47.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:47.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:47.300 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:47.325 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:47.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:47.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:47.427 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:47.452 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:47.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:47.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:53:47.510 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:47 smithi029 ceph-mon[93650]: pgmap v2122: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 15752/151254 objects degraded (10.414%); 2/151254 objects misplaced (0.001%) 2024-04-12T11:53:47.511 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:47 smithi029 ceph-mon[93650]: osdmap e1277: 8 total, 7 up, 7 in 2024-04-12T11:53:47.511 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:47 smithi029 ceph-mon[97787]: pgmap v2122: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 15752/151254 objects degraded (10.414%); 2/151254 objects misplaced (0.001%) 2024-04-12T11:53:47.511 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:47 smithi029 ceph-mon[97787]: osdmap e1277: 8 total, 7 up, 7 in 2024-04-12T11:53:47.554 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:47.601 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:47.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:47.702 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:47 smithi087 ceph-mon[89014]: pgmap v2122: 99 pgs: 4 active+clean+remapped, 17 active+undersized, 10 active+undersized+degraded, 68 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 15752/151254 objects degraded (10.414%); 2/151254 objects misplaced (0.001%) 2024-04-12T11:53:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:47 smithi087 ceph-mon[89014]: osdmap e1277: 8 total, 7 up, 7 in 2024-04-12T11:53:47.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:47.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:47.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:47.738 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 54401 54385 113.28 7.6875 0.00856069 0.0088219 2024-04-12T11:53:47.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:47.870 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:47.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:47.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:47.971 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:48.004 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:48.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:48.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:48.053 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:48.106 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:48.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:48.154 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:48.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:48.255 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:48.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:48.291 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:48.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:48.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:48.431 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:48.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:48.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:48.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15752/151254 objects degraded (10.414%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:48.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:48 smithi029 ceph-mon[97787]: osdmap e1278: 8 total, 7 up, 7 in 2024-04-12T11:53:48.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15752/151254 objects degraded (10.414%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:48.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:48 smithi029 ceph-mon[93650]: osdmap e1278: 8 total, 7 up, 7 in 2024-04-12T11:53:48.532 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:48.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:48.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:48.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:48.665 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:48.694 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:48.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:48.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15752/151254 objects degraded (10.414%), 10 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:48 smithi087 ceph-mon[89014]: osdmap e1278: 8 total, 7 up, 7 in 2024-04-12T11:53:48.737 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 55513 55497 111.868 69.5 0.0134133 0.00893036 2024-04-12T11:53:48.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:48.860 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:48.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:48.911 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:48.962 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:48.994 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:48.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:48.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:49.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:49.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:49.145 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:49.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:49.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:49.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:49.287 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:49.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:49.388 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:49.420 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:49.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:49.483 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:49 smithi029 ceph-mon[93650]: pgmap v2125: 99 pgs: 1 remapped+peering, 3 active+clean+remapped, 6 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+undersized+degraded, 82 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 128 MiB/s wr, 2.04k op/s; 12850/159582 objects degraded (8.052%); 2/159582 objects misplaced (0.001%); 108 B/s, 1 objects/s recovering 2024-04-12T11:53:49.483 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:49 smithi029 ceph-mon[93650]: osdmap e1279: 8 total, 7 up, 7 in 2024-04-12T11:53:49.483 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:49 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3937763678' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:49.483 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:49 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:49.484 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:49 smithi029 ceph-mon[97787]: pgmap v2125: 99 pgs: 1 remapped+peering, 3 active+clean+remapped, 6 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+undersized+degraded, 82 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 128 MiB/s wr, 2.04k op/s; 12850/159582 objects degraded (8.052%); 2/159582 objects misplaced (0.001%); 108 B/s, 1 objects/s recovering 2024-04-12T11:53:49.484 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:49 smithi029 ceph-mon[97787]: osdmap e1279: 8 total, 7 up, 7 in 2024-04-12T11:53:49.484 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:49 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3937763678' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:49.484 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:49 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:49.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:49.557 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:49.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:49.658 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:49.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:49.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:49.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:49 smithi087 ceph-mon[89014]: pgmap v2125: 99 pgs: 1 remapped+peering, 3 active+clean+remapped, 6 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+undersized+degraded, 82 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 128 MiB/s wr, 2.04k op/s; 12850/159582 objects degraded (8.052%); 2/159582 objects misplaced (0.001%); 108 B/s, 1 objects/s recovering 2024-04-12T11:53:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:49 smithi087 ceph-mon[89014]: osdmap e1279: 8 total, 7 up, 7 in 2024-04-12T11:53:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:49 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3937763678' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:49 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:53:49.737 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 56552 56536 110.401 64.9375 0.0196132 0.00905206 2024-04-12T11:53:49.794 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:49.837 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:49.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:49.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:49.939 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:49.981 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:49.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:49.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:50.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:50.122 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:50.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:50.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:50.123 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:53:50.224 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:50.259 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:50.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:50.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:50.361 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:50.393 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:50.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:50.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:50.495 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:50.520 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:50.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:50.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:50.622 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:50.651 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:50.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:50.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:50 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:53:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:50 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:53:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:50 smithi087 ceph-mon[89014]: osdmap e1280: 8 total, 7 up, 7 in 2024-04-12T11:53:50.737 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 57509 57493 108.869 59.8125 0.0136516 0.00917932 2024-04-12T11:53:50.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:50.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:50 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:53:50.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:50 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:53:50.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:50 smithi029 ceph-mon[97787]: osdmap e1280: 8 total, 7 up, 7 in 2024-04-12T11:53:50.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:50 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:53:50.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:50 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:53:50.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:50 smithi029 ceph-mon[93650]: osdmap e1280: 8 total, 7 up, 7 in 2024-04-12T11:53:50.796 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:50.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:50.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:50.897 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:50.939 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:50.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:50.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:51.041 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:51.086 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:51.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:51.188 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:51.227 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:51.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:51.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:51.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:51.372 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:51.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:51.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:51.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:51.504 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:51.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:51.606 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:51.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:51.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:51.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:51 smithi087 ceph-mon[89014]: pgmap v2128: 99 pgs: 3 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 90 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 13385/164919 objects degraded (8.116%); 704 KiB/s, 11 objects/s recovering 2024-04-12T11:53:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:51.737 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 58620 58604 107.709 69.4375 0.0376993 0.00927835 2024-04-12T11:53:51.767 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:51 smithi029 ceph-mon[97787]: pgmap v2128: 99 pgs: 3 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 90 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 13385/164919 objects degraded (8.116%); 704 KiB/s, 11 objects/s recovering 2024-04-12T11:53:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:51 smithi029 ceph-mon[93650]: pgmap v2128: 99 pgs: 3 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 90 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 13385/164919 objects degraded (8.116%); 704 KiB/s, 11 objects/s recovering 2024-04-12T11:53:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:53:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:53:51.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:53:51 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:53:51.338+0000 7f4bde257700 -1 mgr get_metadata_python Requested missing service osd.5 2024-04-12T11:53:51.795 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:51.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:51.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:51.897 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:51.934 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:51.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:51.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:51.993 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 3] out_osds: [5] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2024-04-12T11:53:51.994 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:53:51.994 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2024-04-12T11:53:51.994 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2024-04-12T11:53:51.994 DEBUG:teuthology.orchestra.run.smithi087:osd.5> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.5.service 2024-04-12T11:53:51.996 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl start ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.5 2024-04-12T11:53:52.036 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:52.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:52.078 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:52.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:52.179 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:52.221 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:52.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:52.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:52.316 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:52 smithi087 systemd[1]: Starting Ceph osd.5 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:53:52.323 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:52.360 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:52.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:52.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:52.461 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:52.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:52.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:52.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:52.568 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:53:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:53:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:53:52.568 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:52 smithi087 podman[199032]: 2024-04-12 11:53:52.29156785 +0000 UTC m=+0.029677484 image pull bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3 2024-04-12T11:53:52.569 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:52 smithi087 podman[199032]: 2024-04-12 11:53:52.420289988 +0000 UTC m=+0.158399596 container create b0e1e82f7e7cd5fee104f1d8e6780a2d4eca4d07fcb5a1b2dd9712aee2e997cc (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, RELEASE=reef-d540eba, CEPH_POINT_RELEASE=) 2024-04-12T11:53:52.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:52.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:52.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:52.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:52.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:52.737 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 59707 59691 106.572 67.9375 0.0219183 0.00937675 2024-04-12T11:53:52.774 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:52.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:52.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:52.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:52.917 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:52.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:52.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:52.940 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:53:52.967 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:52 smithi087 podman[199032]: 2024-04-12 11:53:52.665106289 +0000 UTC m=+0.403215906 container init b0e1e82f7e7cd5fee104f1d8e6780a2d4eca4d07fcb5a1b2dd9712aee2e997cc (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=reef-d540eba, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-12T11:53:52.967 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:52 smithi087 podman[199032]: 2024-04-12 11:53:52.67487565 +0000 UTC m=+0.412985259 container start b0e1e82f7e7cd5fee104f1d8e6780a2d4eca4d07fcb5a1b2dd9712aee2e997cc (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=reef-d540eba, GIT_CLEAN=True) 2024-04-12T11:53:52.967 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:52 smithi087 podman[199032]: 2024-04-12 11:53:52.773225586 +0000 UTC m=+0.511335189 container attach b0e1e82f7e7cd5fee104f1d8e6780a2d4eca4d07fcb5a1b2dd9712aee2e997cc (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=reef-d540eba, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403) 2024-04-12T11:53:52.981 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:53:53.019 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:53.020 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:53:53.058 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:53:53.064 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:53.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:53.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:53.104 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:53:53.136 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:53:53.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:53.176 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:53:53.176 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:53:53.176 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:53:53.177 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:53:53.177 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:53:53.177 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:53:53.177 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-04-12T11:53:53.177 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:53:53.177 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:53:53.208 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:53.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:53.310 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:53.346 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:53.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:53.446 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:53.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:53.486 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:53.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:53.587 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:53.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:53.624 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:53.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:53.713 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:53 smithi029 ceph-mon[93650]: pgmap v2130: 99 pgs: 3 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 90 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 688 op/s; 13385/164919 objects degraded (8.116%); 596 KiB/s, 8 objects/s recovering 2024-04-12T11:53:53.713 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:53 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1562874445' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:53.713 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13385/164919 objects degraded (8.116%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:53.713 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:53:53.713 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:53 smithi029 ceph-mon[97787]: pgmap v2130: 99 pgs: 3 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 90 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 688 op/s; 13385/164919 objects degraded (8.116%); 596 KiB/s, 8 objects/s recovering 2024-04-12T11:53:53.713 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:53 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1562874445' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:53.713 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13385/164919 objects degraded (8.116%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:53.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate[199062]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate[199062]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate[199062]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate[199062]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate[199062]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 bash[199032]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 bash[199032]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 bash[199032]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 bash[199032]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 bash[199032]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-12T11:53:53.718 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 bash[199032]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T11:53:53.719 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 bash[199032]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-12T11:53:53.719 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate[199062]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T11:53:53.719 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate[199062]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-12T11:53:53.719 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:53 smithi087 podman[199272]: 2024-04-12 11:53:53.55249768 +0000 UTC m=+0.055231559 container died b0e1e82f7e7cd5fee104f1d8e6780a2d4eca4d07fcb5a1b2dd9712aee2e997cc (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, ceph=True, RELEASE=reef-d540eba) 2024-04-12T11:53:53.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-mon[89014]: pgmap v2130: 99 pgs: 3 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 90 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 688 op/s; 13385/164919 objects degraded (8.116%); 596 KiB/s, 8 objects/s recovering 2024-04-12T11:53:53.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1562874445' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:53:53.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13385/164919 objects degraded (8.116%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:53.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:53:53.725 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:53.740 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 60631 60615 105.208 57.75 0.0137597 0.00949573 2024-04-12T11:53:53.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:53.762 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:53.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:53.864 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:53.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:53.916 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:53.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:54.018 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:54.078 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:54.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:54.180 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:54.226 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:54.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:54.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:54.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:53:54.328 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:54.363 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:54.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:54.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:54.465 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:54.494 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:54.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:54.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:54.596 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:54.602 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-04-12T11:53:54.602 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-04-12T11:53:54.625 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:54.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:54.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:54.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:54.741 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 61478 61462 103.796 52.9375 0.0131982 0.00962823 2024-04-12T11:53:54.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:54.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:54.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:54.873 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:54.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:54.936 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:54.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:55.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:55.078 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:55.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:55.167 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:53:55.179 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:55.213 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:53:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:53:54] "GET /metrics HTTP/1.1" 200 35749 "" "Prometheus/2.43.0" 2024-04-12T11:53:55.214 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:53:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:53:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:53:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:53:55.214 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:53:55.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:55.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:55.346 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:55.353 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:55 smithi087 podman[199272]: 2024-04-12 11:53:55.065798453 +0000 UTC m=+1.568532326 container remove b0e1e82f7e7cd5fee104f1d8e6780a2d4eca4d07fcb5a1b2dd9712aee2e997cc (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=reef-d540eba, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403) 2024-04-12T11:53:55.353 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:55 smithi087 podman[199448]: 2024-04-12 11:53:55.256370356 +0000 UTC m=+0.025003928 image pull bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3 2024-04-12T11:53:55.387 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:55.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:55.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:55.489 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:55.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:55 smithi029 ceph-mon[93650]: pgmap v2131: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s; 16525/177411 objects degraded (9.315%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:53:55.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:55 smithi029 ceph-mon[97787]: pgmap v2131: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s; 16525/177411 objects degraded (9.315%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:53:55.538 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:55.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:55.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:55.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:55.684 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:55.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:55.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:55 smithi087 ceph-mon[89014]: pgmap v2131: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s; 16525/177411 objects degraded (9.315%); 3.8 MiB/s, 60 objects/s recovering 2024-04-12T11:53:55.740 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 62382 62366 102.551 56.5 0.00878019 0.00974511 2024-04-12T11:53:55.787 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:55.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:55.834 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:55.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:55.936 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:55.972 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:55.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:56.073 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:56.116 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:56.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:56.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:56.218 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:56.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:56.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:56.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:56.361 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:56.393 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:56.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:56.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:56.494 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:53:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:56.530 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:56.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:53:56.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:53:56.631 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:56.690 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:56.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:56.717 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:56 smithi087 podman[199448]: 2024-04-12 11:53:56.397804578 +0000 UTC m=+1.166438127 container create 081b534f5a76daffb3c3618bbdbc8fa96af3199d5d4bc9c0c45192ddf84f21f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, ceph=True, RELEASE=reef-d540eba, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-12T11:53:56.740 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 63264 63248 101.335 55.125 0.0777056 0.00986135 2024-04-12T11:53:56.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:56.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:56.856 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:56.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:56.957 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:56.994 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:56.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:56.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:57.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:57.125 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-04-12T11:53:57.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:57.134 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:57.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:57.163 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:57 smithi087 podman[199448]: 2024-04-12 11:53:57.017981424 +0000 UTC m=+1.786614989 container init 081b534f5a76daffb3c3618bbdbc8fa96af3199d5d4bc9c0c45192ddf84f21f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=reef-d540eba, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-12T11:53:57.163 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:57 smithi087 podman[199448]: 2024-04-12 11:53:57.023848802 +0000 UTC m=+1.792482358 container start 081b534f5a76daffb3c3618bbdbc8fa96af3199d5d4bc9c0c45192ddf84f21f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=reef-d540eba, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-12T11:53:57.163 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:57 smithi087 ceph-osd[199532]: -- 172.21.15.87:0/3399876042 <== mon.1 v2:172.21.15.87:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55baf42531e0 con 0x55baf4223000 2024-04-12T11:53:57.163 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:57 smithi087 bash[199448]: 081b534f5a76daffb3c3618bbdbc8fa96af3199d5d4bc9c0c45192ddf84f21f4 2024-04-12T11:53:57.163 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:57 smithi087 systemd[1]: Started Ceph osd.5 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T11:53:57.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:57.265 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:57.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:57.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:57.367 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:57.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:57.397 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:57.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:57.498 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:57.527 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:57.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:57.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:57.626 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:57 smithi029 ceph-mon[97787]: pgmap v2132: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s; 16525/180303 objects degraded (9.165%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T11:53:57.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:57 smithi029 ceph-mon[93650]: pgmap v2132: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s; 16525/180303 objects degraded (9.165%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T11:53:57.629 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:57.661 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:57.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:57.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:57.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-04-12T11:53:57.717 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:53:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T11:53:57.577+0000 7fbd489087c0 -1 Falling back to public interface 2024-04-12T11:53:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:57 smithi087 ceph-mon[89014]: pgmap v2132: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s; 16525/180303 objects degraded (9.165%); 2.9 MiB/s, 45 objects/s recovering 2024-04-12T11:53:57.733 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:53:57.741 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:53:57.741162+0000 min lat: 0.00116361 max lat: 2.15979 avg lat: 0.00996748 2024-04-12T11:53:57.741 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:53:57.741 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 15 64199 64184 100.263 58.5 0.0127654 0.00996748 2024-04-12T11:53:57.763 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:57.792 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:57.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:57.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:57.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:57.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:57.944 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:57.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:58.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:58.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:58.097 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:58.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:58.198 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:58.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:58.235 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:58.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:58.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:58.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:58.360 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:58.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:58.461 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:53:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16525/180303 objects degraded (9.165%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:58.500 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:58.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:53:58.602 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:58.635 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:58.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:58.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:58.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:58.741 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 65015 64999 99.0595 50.9375 0.0893799 0.0100709 2024-04-12T11:53:58.762 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:58.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:58.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:58.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16525/180303 objects degraded (9.165%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:58.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16525/180303 objects degraded (9.165%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:53:58.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:58.905 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:58.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:58.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:59.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:59.044 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:59.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:59.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:59.146 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:59.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:59.174 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:59.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:59.275 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:53:59.309 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:59.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:59.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:53:59.411 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:53:59.447 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:59.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:59.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:53:59.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:53:59.586 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:59.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:59.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:53:59.688 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:53:59.716 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:53:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:59.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:53:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:53:59 smithi087 ceph-mon[89014]: pgmap v2133: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 934 op/s; 16338/188043 objects degraded (8.688%); 3.6 MiB/s, 57 objects/s recovering 2024-04-12T11:53:59.741 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 65756 65740 97.8037 46.3125 0.00605226 0.0102141 2024-04-12T11:53:59.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:53:59 smithi029 ceph-mon[97787]: pgmap v2133: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 934 op/s; 16338/188043 objects degraded (8.688%); 3.6 MiB/s, 57 objects/s recovering 2024-04-12T11:53:59.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:53:59 smithi029 ceph-mon[93650]: pgmap v2133: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 58 MiB/s wr, 934 op/s; 16338/188043 objects degraded (8.688%); 3.6 MiB/s, 57 objects/s recovering 2024-04-12T11:53:59.818 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:53:59.861 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:53:59.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:53:59.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:53:59.963 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:00.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:00.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:00.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:00.087 INFO:teuthology.orchestra.run.smithi087.stderr:no valid command found; 10 closest matches: 2024-04-12T11:54:00.087 INFO:teuthology.orchestra.run.smithi087.stderr:0 2024-04-12T11:54:00.088 INFO:teuthology.orchestra.run.smithi087.stderr:1 2024-04-12T11:54:00.088 INFO:teuthology.orchestra.run.smithi087.stderr:2 2024-04-12T11:54:00.088 INFO:teuthology.orchestra.run.smithi087.stderr:abort 2024-04-12T11:54:00.088 INFO:teuthology.orchestra.run.smithi087.stderr:assert 2024-04-12T11:54:00.088 INFO:teuthology.orchestra.run.smithi087.stderr:bluefs debug_inject_read_zeros 2024-04-12T11:54:00.088 INFO:teuthology.orchestra.run.smithi087.stderr:bluefs files list 2024-04-12T11:54:00.088 INFO:teuthology.orchestra.run.smithi087.stderr:bluefs stats 2024-04-12T11:54:00.088 INFO:teuthology.orchestra.run.smithi087.stderr:bluestore bluefs device info [] 2024-04-12T11:54:00.088 INFO:teuthology.orchestra.run.smithi087.stderr:config diff 2024-04-12T11:54:00.088 INFO:teuthology.orchestra.run.smithi087.stderr:admin_socket: invalid command 2024-04-12T11:54:00.101 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:00.133 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:00.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:00.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:00.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:00.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:00.278 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:00.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:00.379 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:00.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:00.423 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:00.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:00.524 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:00.548 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:00.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:00.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:00.650 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:00.686 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:00.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:00.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:00.741 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 66724 66708 96.9362 60.5 0.00961853 0.0103102 2024-04-12T11:54:00.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:00.829 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:00.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:00.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:00.936 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:00.960 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:00.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:01.062 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:01.109 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:01.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:01.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:01.210 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:01.250 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:01.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:01.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:01.352 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:01.375 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:01.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:01.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:01.477 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:01.500 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:01.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:01.545 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-12T11:54:01.545 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2024-04-12T11:54:01.602 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:01.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:01.628 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:01.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:01 smithi087 ceph-mon[89014]: pgmap v2134: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 958 op/s; 16182/193689 objects degraded (8.355%); 4.2 MiB/s, 66 objects/s recovering 2024-04-12T11:54:01.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:01.742 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 67378 67362 95.6622 40.875 0.0115697 0.0104482 2024-04-12T11:54:01.765 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:01.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:01.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:01.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:01 smithi029 ceph-mon[97787]: pgmap v2134: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 958 op/s; 16182/193689 objects degraded (8.355%); 4.2 MiB/s, 66 objects/s recovering 2024-04-12T11:54:01.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:01 smithi029 ceph-mon[93650]: pgmap v2134: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 958 op/s; 16182/193689 objects degraded (8.355%); 4.2 MiB/s, 66 objects/s recovering 2024-04-12T11:54:01.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:01.926 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:01.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:02.028 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:02.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:02.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:02.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:02.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:02.197 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:02.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:02.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:02.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:02.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:02.337 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:02.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:02.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:02.477 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:02.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:02.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:02.579 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:02.608 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:02.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:02.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:02.710 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:54:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:54:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:54:02.742 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 68195 68179 94.6703 51.0625 0.00836606 0.0105577 2024-04-12T11:54:02.748 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:02.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:02.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:02.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:02.874 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:02.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:02.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:02.975 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:03.018 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:03.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:03.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:03.119 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:03.161 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:03.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:03.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:03.266 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:03.289 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:03.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:03.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:03.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:03.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:03.430 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:03.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:03.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:03 smithi087 ceph-mon[89014]: pgmap v2135: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 800 op/s; 16182/193689 objects degraded (8.355%); 3.5 MiB/s, 55 objects/s recovering 2024-04-12T11:54:03.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16182/193689 objects degraded (8.355%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:03.530 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:03.576 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:03.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:03.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:03.678 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:03.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:03.722 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:03.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:03.742 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 68953 68937 93.6422 47.375 0.0148223 0.0106731 2024-04-12T11:54:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:03 smithi029 ceph-mon[93650]: pgmap v2135: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 800 op/s; 16182/193689 objects degraded (8.355%); 3.5 MiB/s, 55 objects/s recovering 2024-04-12T11:54:03.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16182/193689 objects degraded (8.355%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:03 smithi029 ceph-mon[97787]: pgmap v2135: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 800 op/s; 16182/193689 objects degraded (8.355%); 3.5 MiB/s, 55 objects/s recovering 2024-04-12T11:54:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16182/193689 objects degraded (8.355%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:03.823 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:03.852 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:03.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:03.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:03.954 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:03.977 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:03.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:04.079 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:04.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:04.118 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:04.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:04.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:04.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:04.260 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:04.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:04.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:04.397 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:04.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:04.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:04.499 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:04.523 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:04.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:04.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:04.625 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:04.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:04.673 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:04.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:04.742 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 69545 69529 92.4372 37 0.00798593 0.0108122 2024-04-12T11:54:04.774 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:04.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:04.821 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:04.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:04.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:04.961 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:04.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:04.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:05.063 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:05.087 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:05.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:05.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:05.188 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:05.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:05.226 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:05.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:54:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:54:04] "GET /metrics HTTP/1.1" 200 35749 "" "Prometheus/2.43.0" 2024-04-12T11:54:05.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:05.364 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:05.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:05.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:05.508 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:05.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:05.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:05.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:05.650 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:05.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:05.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:05 smithi087 ceph-mon[89014]: pgmap v2136: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 15816/203409 objects degraded (7.775%); 5.4 MiB/s, 86 objects/s recovering 2024-04-12T11:54:05.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:05 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:05.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:05 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:05.742 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 70345 70329 91.5531 50 0.0185963 0.0109147 2024-04-12T11:54:05.751 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:05.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:05 smithi029 ceph-mon[97787]: pgmap v2136: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 15816/203409 objects degraded (7.775%); 5.4 MiB/s, 86 objects/s recovering 2024-04-12T11:54:05.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:05.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:05.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:05 smithi029 ceph-mon[93650]: pgmap v2136: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 15816/203409 objects degraded (7.775%); 5.4 MiB/s, 86 objects/s recovering 2024-04-12T11:54:05.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:05.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:05.775 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:05.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:05.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:05.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:05.901 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:05.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:06.002 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:06.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:06.042 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:06.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:06.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:06.181 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:06.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:06.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:06.324 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:06.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:06.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:06.426 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:06.450 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:06.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:06.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:06.549 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-04-12T11:54:06.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:06.576 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:06.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:06.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:06.678 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:06.706 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:06.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:06.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:06.742 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 70827 70811 90.2996 30.125 0.00766342 0.0110672 2024-04-12T11:54:06.768 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:54:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T11:54:06.328+0000 7f4bde257700 -1 mgr get_metadata_python Requested missing service osd.5 2024-04-12T11:54:06.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:06.851 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:06.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:06.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:06.952 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:06.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:06.994 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:06.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:07.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:07.118 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:07.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:07.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:07.166 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:07.166 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:07.166 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:06 smithi087 ceph-mon[89014]: pgmap v2137: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 49 MiB/s wr, 780 op/s; 15816/205512 objects degraded (7.696%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:54:07.166 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:07.220 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:07.250 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:07.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:07.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:07.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:07.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:07.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:06 smithi029 ceph-mon[93650]: pgmap v2137: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 49 MiB/s wr, 780 op/s; 15816/205512 objects degraded (7.696%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:54:07.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:07.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:07.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:07.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:06 smithi029 ceph-mon[97787]: pgmap v2137: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 49 MiB/s wr, 780 op/s; 15816/205512 objects degraded (7.696%); 3.7 MiB/s, 59 objects/s recovering 2024-04-12T11:54:07.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:07.352 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:07.392 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:07.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:07.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:07.494 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:07.530 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:07.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:07.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:07.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:07.669 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:07.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:07.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:07.710 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:54:07.743 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 71654 71638 89.5273 51.6875 0.00402837 0.0111489 2024-04-12T11:54:07.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:07.810 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:07.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:07.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:07.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:07.956 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:07.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:07.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:08.058 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:08.081 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:08.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:08.182 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:08.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:08.208 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:08.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:08.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15816/205512 objects degraded (7.696%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:08.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15816/205512 objects degraded (7.696%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:08.309 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:08.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:08.349 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:08.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:08.450 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:08.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15816/205512 objects degraded (7.696%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:08.490 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:08.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:08.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:08.592 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:08.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:08.626 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:08.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:08.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:08.742 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 72476 72460 88.7792 51.375 0.0296682 0.0112573 2024-04-12T11:54:08.766 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:08.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:08.868 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:08.892 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:08.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:08.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:08.993 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:09.017 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:09.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:09.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:09.103 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:54:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T11:54:08.793+0000 7fbd489087c0 -1 osd.5 1265 log_to_monitors true 2024-04-12T11:54:09.119 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:09.151 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:09.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:09.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:09.289 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:09.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:09.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:09.381 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 11:54:09 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T11:54:09.187+0000 7fbd39d62700 -1 osd.5 1265 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T11:54:09.382 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:09 smithi087 ceph-mon[89014]: pgmap v2138: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 861 op/s; 15642/211344 objects degraded (7.401%); 4.6 MiB/s, 73 objects/s recovering 2024-04-12T11:54:09.382 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:09 smithi087 ceph-mon[89014]: from='osd.5 [v2:172.21.15.87:6808/3350310311,v1:172.21.15.87:6809/3350310311]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:54:09.382 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:09 smithi087 ceph-mon[89014]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:54:09.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:09.414 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:09.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:09.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:09.515 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:09 smithi029 ceph-mon[93650]: pgmap v2138: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 861 op/s; 15642/211344 objects degraded (7.401%); 4.6 MiB/s, 73 objects/s recovering 2024-04-12T11:54:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:09 smithi029 ceph-mon[93650]: from='osd.5 [v2:172.21.15.87:6808/3350310311,v1:172.21.15.87:6809/3350310311]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:54:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:09 smithi029 ceph-mon[93650]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:54:09.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:09 smithi029 ceph-mon[97787]: pgmap v2138: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 861 op/s; 15642/211344 objects degraded (7.401%); 4.6 MiB/s, 73 objects/s recovering 2024-04-12T11:54:09.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:09 smithi029 ceph-mon[97787]: from='osd.5 [v2:172.21.15.87:6808/3350310311,v1:172.21.15.87:6809/3350310311]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:54:09.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:09 smithi029 ceph-mon[97787]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T11:54:09.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:09.545 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:09.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:09.646 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:09.687 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:09.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:09.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:09.743 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 73061 73045 87.7751 36.5625 0.0166459 0.0113874 2024-04-12T11:54:09.789 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:09.828 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:09.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:09.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:09.930 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:09.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:09.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:10.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:10.114 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:10.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:10.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:10.216 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:10.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:10 smithi087 ceph-mon[89014]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T11:54:10.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:10 smithi087 ceph-mon[89014]: osdmap e1281: 8 total, 7 up, 7 in 2024-04-12T11:54:10.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:10 smithi087 ceph-mon[89014]: from='osd.5 [v2:172.21.15.87:6808/3350310311,v1:172.21.15.87:6809/3350310311]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:54:10.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:10 smithi087 ceph-mon[89014]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:54:10.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:10 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:54:10.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:10.258 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:10.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:10.359 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:10.399 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:10.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:10.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:10.500 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:10.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:10 smithi029 ceph-mon[97787]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T11:54:10.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:10 smithi029 ceph-mon[97787]: osdmap e1281: 8 total, 7 up, 7 in 2024-04-12T11:54:10.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:10 smithi029 ceph-mon[97787]: from='osd.5 [v2:172.21.15.87:6808/3350310311,v1:172.21.15.87:6809/3350310311]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:54:10.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:10 smithi029 ceph-mon[97787]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:54:10.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:10 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:54:10.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:10 smithi029 ceph-mon[93650]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T11:54:10.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:10 smithi029 ceph-mon[93650]: osdmap e1281: 8 total, 7 up, 7 in 2024-04-12T11:54:10.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:10 smithi029 ceph-mon[93650]: from='osd.5 [v2:172.21.15.87:6808/3350310311,v1:172.21.15.87:6809/3350310311]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:54:10.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:10 smithi029 ceph-mon[93650]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-12T11:54:10.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:10 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:54:10.540 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:10.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:10.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:10.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:10.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:10.685 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:10.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:10.743 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 73958 73942 87.1767 56.0625 0.0112971 0.0114652 2024-04-12T11:54:10.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:10.812 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:10.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:10.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:10.913 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:10.950 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:10.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:10.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:11.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:11.089 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:11.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:11.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:11.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:11.214 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:11.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:11.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:11.316 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:11.352 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:11.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:11.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:11.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:11 smithi087 ceph-mon[89014]: pgmap v2140: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 745 op/s; 15499/216072 objects degraded (7.173%); 4.3 MiB/s, 68 objects/s recovering 2024-04-12T11:54:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:11 smithi087 ceph-mon[89014]: osd.5 [v2:172.21.15.87:6808/3350310311,v1:172.21.15.87:6809/3350310311] boot 2024-04-12T11:54:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:11 smithi087 ceph-mon[89014]: osdmap e1282: 8 total, 8 up, 7 in 2024-04-12T11:54:11.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:11 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:54:11.497 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:11.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:11.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:11 smithi029 ceph-mon[97787]: pgmap v2140: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 745 op/s; 15499/216072 objects degraded (7.173%); 4.3 MiB/s, 68 objects/s recovering 2024-04-12T11:54:11.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:11 smithi029 ceph-mon[97787]: osd.5 [v2:172.21.15.87:6808/3350310311,v1:172.21.15.87:6809/3350310311] boot 2024-04-12T11:54:11.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:11 smithi029 ceph-mon[97787]: osdmap e1282: 8 total, 8 up, 7 in 2024-04-12T11:54:11.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:11 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:54:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:11 smithi029 ceph-mon[93650]: pgmap v2140: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 745 op/s; 15499/216072 objects degraded (7.173%); 4.3 MiB/s, 68 objects/s recovering 2024-04-12T11:54:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:11 smithi029 ceph-mon[93650]: osd.5 [v2:172.21.15.87:6808/3350310311,v1:172.21.15.87:6809/3350310311] boot 2024-04-12T11:54:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:11 smithi029 ceph-mon[93650]: osdmap e1282: 8 total, 8 up, 7 in 2024-04-12T11:54:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:11 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T11:54:11.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:11.642 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:11.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:11.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:11.711 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:54:11.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:11.748 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 74770 74754 86.4969 50.75 0.030508 0.0115548 2024-04-12T11:54:11.779 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:11.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:11.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:11.904 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:11.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:11.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:12.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:12.029 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:12.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:12.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:12.131 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:12.170 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:12.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:12.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:12.258 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:54:12.258 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:54:12.258 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T11:54:12.258 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:54:12.272 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:12.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:12.296 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:12.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:12.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:12.420 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:12.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:12.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:12.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:12.561 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:12.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:12.662 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:12.698 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:12.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:12.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:54:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:54:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:54:12.746 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 75168 75152 85.3766 24.875 0.0105441 0.0117072 2024-04-12T11:54:12.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:12.842 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:12.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:12.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:12.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:12.967 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:12.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:12.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:13.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:13.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:13.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:13.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:13.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:13.223 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:13.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:13.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:13.325 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:13.349 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:13.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:13.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:13.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:13.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:13 smithi087 ceph-mon[89014]: pgmap v2142: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 15499/216072 objects degraded (7.173%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:54:13.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15499/216072 objects degraded (7.173%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:13.474 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:13.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:13.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:13.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:13 smithi029 ceph-mon[97787]: pgmap v2142: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 15499/216072 objects degraded (7.173%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:54:13.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15499/216072 objects degraded (7.173%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:13.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:13 smithi029 ceph-mon[93650]: pgmap v2142: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 15499/216072 objects degraded (7.173%); 2.5 MiB/s, 39 objects/s recovering 2024-04-12T11:54:13.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15499/216072 objects degraded (7.173%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:13.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:13.599 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:13.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:13.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:13.701 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:13.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:13.725 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:13.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:13.746 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 75543 75527 84.2707 23.4375 0.0107521 0.0117465 2024-04-12T11:54:13.826 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:13.854 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:13.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:13.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:13.955 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:13.986 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:13.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:14.088 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:14.123 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:14.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:14.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:14.225 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:14.251 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:14.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:14.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:14.352 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:14.379 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:14.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:14.481 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:14.505 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:14.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:14.607 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:14.630 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:14.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:14.732 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:14.746 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 76049 76033 83.3473 31.625 0.0277663 0.0119917 2024-04-12T11:54:14.766 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:14.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:14.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:14.868 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:14.892 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:14.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:14.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:14.993 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:15.017 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:15.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:15.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:15.119 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:15.144 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:15.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:15.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:15.189 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:54:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:54:14] "GET /metrics HTTP/1.1" 200 35746 "" "Prometheus/2.43.0" 2024-04-12T11:54:15.247 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:15.286 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:15.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:15.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:15.388 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:15.416 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:15 smithi087 ceph-mon[89014]: pgmap v2143: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 15084/224607 objects degraded (6.716%); 5.7 MiB/s, 91 objects/s recovering 2024-04-12T11:54:15.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:15.424 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:15.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:15.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:15 smithi029 ceph-mon[97787]: pgmap v2143: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 15084/224607 objects degraded (6.716%); 5.7 MiB/s, 91 objects/s recovering 2024-04-12T11:54:15.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:15 smithi029 ceph-mon[93650]: pgmap v2143: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 15084/224607 objects degraded (6.716%); 5.7 MiB/s, 91 objects/s recovering 2024-04-12T11:54:15.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:15.550 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:15.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:15.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:15.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:15.675 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:15.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:15.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:15.746 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 76468 76452 82.3619 26.1875 0.134654 0.0121069 2024-04-12T11:54:15.777 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:15.811 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:15.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:15.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:15.913 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:15.951 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:15.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:15.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:16.053 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:16.091 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:16.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:16.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:16.193 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:16.216 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:16.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:16.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:16.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:16.353 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:16.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:16.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:16.455 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:16.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:16.491 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:16.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:16.592 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:16.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:16.621 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:16.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:16.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:16.746 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 77124 77108 81.6609 41 0.147275 0.0122297 2024-04-12T11:54:16.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:16.762 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:16.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:16.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:16.910 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:16.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:16.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:17.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:17.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:17.049 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:17.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:17.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:17.192 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:17.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:17.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:17.293 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:17.338 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:17.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:17.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:17 smithi087 ceph-mon[89014]: pgmap v2144: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 39 MiB/s wr, 623 op/s; 15084/226317 objects degraded (6.665%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T11:54:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:17 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:17.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:17 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:17.470 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:17.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:17.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:17.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:17 smithi029 ceph-mon[93650]: pgmap v2144: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 39 MiB/s wr, 623 op/s; 15084/226317 objects degraded (6.665%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T11:54:17.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:17 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:17.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:17 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:17.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:17 smithi029 ceph-mon[97787]: pgmap v2144: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 39 MiB/s wr, 623 op/s; 15084/226317 objects degraded (6.665%); 4.4 MiB/s, 69 objects/s recovering 2024-04-12T11:54:17.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:17 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:17.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:17 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:17.572 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:17.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:17.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:17.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:17.717 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:17.747 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:54:17.746696+0000 min lat: 0.00116361 max lat: 2.15979 avg lat: 0.012298 2024-04-12T11:54:17.747 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:54:17.747 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 78055 78039 81.2696 58.1875 0.0170915 0.012298 2024-04-12T11:54:17.752 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:17.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:17.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:17.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:17.888 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:17.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:17.990 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:18.022 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:18.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:18.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:18.123 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:18.171 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:18.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:18.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:18.217 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 15084/226317 objects degraded (6.665%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:18.219 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 15084/226317 objects degraded (6.665%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:18.273 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:18.311 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:18.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:18.350 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:54:18.412 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:18.441 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:18.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:18.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:18.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 15084/226317 objects degraded (6.665%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:18.542 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:18.575 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:18.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:18.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:18.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:18.717 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:18.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:18.747 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 78969 78953 80.8738 57.125 0.0345854 0.0123589 2024-04-12T11:54:18.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:18.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:18.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:18.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:18.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:19.007 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:19.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:19.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:19.109 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:19.232 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:19.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:19.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:19.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:19 smithi029 ceph-mon[97787]: pgmap v2145: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 14838/230994 objects degraded (6.424%); 4.6 MiB/s, 73 objects/s recovering 2024-04-12T11:54:19.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:19 smithi029 ceph-mon[93650]: pgmap v2145: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 14838/230994 objects degraded (6.424%); 4.6 MiB/s, 73 objects/s recovering 2024-04-12T11:54:19.334 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:19.363 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:19.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:19.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:19.465 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:19 smithi087 ceph-mon[89014]: pgmap v2145: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 632 op/s; 14838/230994 objects degraded (6.424%); 4.6 MiB/s, 73 objects/s recovering 2024-04-12T11:54:19.494 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:19.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:19.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:19.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:19.626 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:19.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:19.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:19.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:19.747 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 79850 79834 80.4575 55.0625 0.0106063 0.0124232 2024-04-12T11:54:19.760 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:19.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:19.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:19.861 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:19.890 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:19.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:19.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:19.991 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:20.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:20.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:20.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:20.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:20.147 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:54:20.150 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:20.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:20.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:20.252 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:20.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:20 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1810805627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:54:20.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:20 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:54:20.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:20 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1810805627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:54:20.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:20 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:54:20.294 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:20.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:20.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:20.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:20.441 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:20.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:20.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:20.452 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 3] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:54:20.452 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:54:20.453 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-04-12T11:54:20.453 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-04-12T11:54:20.453 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-04-12T11:54:20.453 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-04-12T11:54:20.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:20 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1810805627' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:54:20.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:20 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:54:20.542 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:20.576 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:20.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:20.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:20.678 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:20.710 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:20.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:20.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:20.747 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 80678 80662 80.0018 51.75 0.0173555 0.0124935 2024-04-12T11:54:20.811 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:20.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:20.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:20.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:21.006 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:21.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:21.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:21.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:21.148 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:21.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:21.207 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:54:21.249 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:21.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:21 smithi029 ceph-mon[97787]: pgmap v2146: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 656 op/s; 14644/235764 objects degraded (6.211%); 5.3 MiB/s, 85 objects/s recovering 2024-04-12T11:54:21.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:21 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:54:21.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:21 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:54:21.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:21 smithi029 ceph-mon[97787]: osdmap e1283: 8 total, 8 up, 7 in 2024-04-12T11:54:21.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:21 smithi029 ceph-mon[93650]: pgmap v2146: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 656 op/s; 14644/235764 objects degraded (6.211%); 5.3 MiB/s, 85 objects/s recovering 2024-04-12T11:54:21.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:21 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:54:21.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:21 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:54:21.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:21 smithi029 ceph-mon[93650]: osdmap e1283: 8 total, 8 up, 7 in 2024-04-12T11:54:21.282 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:21.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:21.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:21.384 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:21.425 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:21.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:21 smithi087 ceph-mon[89014]: pgmap v2146: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 656 op/s; 14644/235764 objects degraded (6.211%); 5.3 MiB/s, 85 objects/s recovering 2024-04-12T11:54:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:21 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:54:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:21 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:54:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:21 smithi087 ceph-mon[89014]: osdmap e1283: 8 total, 8 up, 7 in 2024-04-12T11:54:21.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:21.567 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:21.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:21.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:21.668 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:21.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:21.711 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:21.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:21.749 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 81485 81469 79.5373 50.4375 0.0379338 0.0125664 2024-04-12T11:54:21.811 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:21.842 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:21.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:21.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:21.944 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:21.973 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:21.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:21.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:22.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:22.109 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:22.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:22.211 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:22.241 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:22.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:22.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:22.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:22.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:22.342 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:22.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:22.377 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:22.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:22.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:22.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:54:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:54:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:54:22.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:22.514 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:22.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:22.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:22.616 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:22.648 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:22.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:22.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:22.749 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 82386 82370 79.1799 56.3125 0.0141167 0.0126242 2024-04-12T11:54:22.750 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:22.777 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:22.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:22.880 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:22.921 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:22.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:22.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:23.023 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:23.062 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:23.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:23.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:23.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:23.202 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:23.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:23.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:23.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:23.310 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:23 smithi087 ceph-mon[89014]: pgmap v2148: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 656 op/s; 14644/235764 objects degraded (6.211%); 5.3 MiB/s, 85 objects/s recovering 2024-04-12T11:54:23.310 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 14644/235764 objects degraded (6.211%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:23.350 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:23.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:23.445 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:54:23.452 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:23.463 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:23 smithi029 ceph-mon[97787]: pgmap v2148: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 656 op/s; 14644/235764 objects degraded (6.211%); 5.3 MiB/s, 85 objects/s recovering 2024-04-12T11:54:23.463 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 14644/235764 objects degraded (6.211%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:23.464 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:23 smithi029 ceph-mon[93650]: pgmap v2148: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 656 op/s; 14644/235764 objects degraded (6.211%); 5.3 MiB/s, 85 objects/s recovering 2024-04-12T11:54:23.464 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 14644/235764 objects degraded (6.211%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:23.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:23.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:23.516 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:54:23.516 INFO:teuthology.orchestra.run.smithi087.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-04-12T11:54:23.516 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:54:23.591 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:23.626 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:23.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:23.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:23.728 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:23.749 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 83261 83245 78.8088 54.6875 0.0367304 0.0126828 2024-04-12T11:54:23.778 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:23.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:23.879 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:23.923 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:23.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:23.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:24.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:24.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:24.091 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:24.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:24.192 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:24.237 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:24.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:24.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:24.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:24.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:24.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:24.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:24.414 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:54:24.455 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:54:24.488 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:24.496 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:54:24.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:24.526 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:24.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:24.532 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:54:24.574 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:54:24.615 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:54:24.627 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:24.658 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:54:24.667 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:24.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:24.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:24.699 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:54:24.715 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:54:24.715 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:54:24.715 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:54:24.715 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:54:24.715 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:54:24.715 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:54:24.715 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:54:24.715 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:54:24.749 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 84049 84033 78.3677 49.25 0.0257627 0.0127539 2024-04-12T11:54:24.769 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:24.803 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:24.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:24.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:24.905 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:24.936 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:24.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:24.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:25.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:25.079 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:25.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:25.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:25.210 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:25.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:25.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:25.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:25 smithi029 ceph-mon[97787]: pgmap v2149: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 720 op/s; 14216/246243 objects degraded (5.773%); 5.4 MiB/s, 86 objects/s recovering 2024-04-12T11:54:25.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:25 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4198635250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:54:25.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:25 smithi029 ceph-mon[93650]: pgmap v2149: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 720 op/s; 14216/246243 objects degraded (5.773%); 5.4 MiB/s, 86 objects/s recovering 2024-04-12T11:54:25.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:25 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4198635250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:54:25.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:54:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:54:24] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-04-12T11:54:25.311 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:25.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:25.351 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:25.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:25.398 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:121589 8.6 8:671b29b8:::benchmark_data_smithi029_194418_object15279:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1199)", 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:44:44.398957+0000", 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 579.87060963700003, 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0599646630000001, 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 121589 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398957+0000", 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398957+0000", 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398959+0000", 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9599999999999999e-06 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398969+0000", 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6709999999999994e-06 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398985+0000", 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5784999999999999e-05 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.398997+0000", 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2522e-05 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.657919+0000", 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25892183699999999 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:44.657929+0000", 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5419999999999994e-06 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.391598+0000", 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73366960800000003 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:45.391612+0000", 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4151000000000001e-05 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.427659+0000", 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.036046614 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2024-04-12T11:54:25.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.427796+0000", 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013716499999999999 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.458097+0000", 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030300494000000001 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:44:46.458922+0000", 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00082531399999999995 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168251 8.c 8:3d814e3d:::benchmark_data_smithi029_194418_object61941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.323733+0000", 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 511.94583361299999, 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.010112119, 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168251 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323733+0000", 2024-04-12T11:54:25.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323733+0000", 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323735+0000", 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4589999999999999e-06 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323741+0000", 2024-04-12T11:54:25.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0070000000000003e-06 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323757+0000", 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5869999999999999e-05 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.323765+0000", 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0649999999999999e-06 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.360710+0000", 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036944880999999999 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.363394+0000", 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002683983 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.378917+0000", 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015522948 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.379027+0000", 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011003899999999999 2024-04-12T11:54:25.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4,5", 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.379202+0000", 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000175716 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394056+0000", 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014854192 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394068+0000", 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1245e-05 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.967696+0000", 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.57362859499999996 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.967709+0000", 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2986000000000001e-05 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.314657+0000", 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.346947624 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.314840+0000", 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018301 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.333767+0000", 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018927557000000001 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.333845+0000", 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7941999999999993e-05 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.411 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168281 8.16 8:69d01196:::benchmark_data_smithi029_194418_object61971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.354640+0000", 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 511.91492727100001, 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8219508370000002, 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168281 2024-04-12T11:54:25.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354640+0000", 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354640+0000", 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354641+0000", 2024-04-12T11:54:25.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3090000000000001e-06 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354645+0000", 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4440000000000001e-06 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354656+0000", 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0263e-05 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354664+0000", 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9300000000000003e-06 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.354959+0000", 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000295481 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.355057+0000", 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7807999999999996e-05 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.357030+0000", 2024-04-12T11:54:25.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019734489999999999 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.364390+0000", 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073593649999999997 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388599+0000", 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024208994000000001 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388612+0000", 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3397000000000001e-05 2024-04-12T11:54:25.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973262+0000", 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.58464973899999995 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973272+0000", 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0592e-05 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176509+0000", 2024-04-12T11:54:25.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2032365409999999 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176590+0000", 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1525000000000003e-05 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168294 8.c 8:33ff478c:::benchmark_data_smithi029_194418_object61984:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.387000+0000", 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 511.88256651199998, 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9560116860000001, 2024-04-12T11:54:25.418 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168294 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387000+0000", 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387000+0000", 2024-04-12T11:54:25.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387002+0000", 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3769999999999999e-06 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387006+0000", 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.206e-06 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387016+0000", 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6930000000000002e-06 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387023+0000", 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8180000000000004e-06 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394098+0000", 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0070740580000000003 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.394105+0000", 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2269999999999996e-06 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.968399+0000", 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.57429384699999997 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.968408+0000", 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2350000000000005e-06 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.318138+0000", 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3497297449999999 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.318280+0000", 2024-04-12T11:54:25.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000142356 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5,7", 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.318409+0000", 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000128713 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.330729+0000", 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012320516 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.330916+0000", 2024-04-12T11:54:25.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018688 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.331041+0000", 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012471800000000001 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.341131+0000", 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010089964 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.341357+0000", 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000226734 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.343012+0000", 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016545990000000001 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168295 8.6 8:63357634:::benchmark_data_smithi029_194418_object61985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.387036+0000", 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 511.88253084000002, 2024-04-12T11:54:25.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.799481667, 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168295 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387036+0000", 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387036+0000", 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387037+0000", 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1799999999999995e-07 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387039+0000", 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6240000000000002e-06 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387046+0000", 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.888e-06 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.387053+0000", 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5780000000000004e-06 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388702+0000", 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001649347 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.388711+0000", 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8720000000000008e-06 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973114+0000", 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.58440336500000001 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973125+0000", 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0274e-05 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.165152+0000", 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.192027036 2024-04-12T11:54:25.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.165316+0000", 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000164561 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.165826+0000", 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000509495 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170021+0000", 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041956379999999998 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.173298+0000", 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003276251 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.185462+0000", 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012164071 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.185532+0000", 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0222999999999995e-05 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.186518+0000", 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00098592600000000003 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168309 8.16 8:68816e76:::benchmark_data_smithi029_194418_object61999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.404119+0000", 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 511.865448168, 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.786614266, 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.432 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168309 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404119+0000", 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404119+0000", 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404120+0000", 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.393e-06 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404126+0000", 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9630000000000003e-06 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404169+0000", 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2865999999999999e-05 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404179+0000", 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0002e-05 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404244+0000", 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4998000000000003e-05 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.404255+0000", 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0845e-05 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973300+0000", 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56904551800000003 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973305+0000", 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8539999999999996e-06 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176619+0000", 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2033139099999999 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176694+0000", 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4870999999999995e-05 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6,7", 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.177067+0000", 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000373023 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.181264+0000", 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041970030000000004 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.183957+0000", 2024-04-12T11:54:25.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026927610000000001 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.184058+0000", 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100765 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190488+0000", 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0064309559999999998 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190552+0000", 2024-04-12T11:54:25.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3929000000000004e-05 2024-04-12T11:54:25.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190733+0000", 2024-04-12T11:54:25.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000180609 2024-04-12T11:54:25.441 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.441 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.441 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.441 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168311 8.6 8:65642b03:::benchmark_data_smithi029_194418_object62001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:54:25.441 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.408072+0000", 2024-04-12T11:54:25.441 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 511.86149518100001, 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.782363267, 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168311 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408072+0000", 2024-04-12T11:54:25.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408072+0000", 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408073+0000", 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4950000000000001e-06 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408080+0000", 2024-04-12T11:54:25.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6250000000000001e-06 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408097+0000", 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7031999999999999e-05 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408107+0000", 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8470000000000008e-06 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408190+0000", 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3578000000000005e-05 2024-04-12T11:54:25.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.408206+0000", 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5577e-05 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973171+0000", 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56496518699999998 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973182+0000", 2024-04-12T11:54:25.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0550000000000001e-05 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170067+0000", 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1968853310000001 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170241+0000", 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017409500000000001 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2024-04-12T11:54:25.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.170963+0000", 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00072238800000000002 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176537+0000", 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055731390000000004 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.176553+0000", 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6648999999999999e-05 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.188471+0000", 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011917366 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.188530+0000", 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9120000000000003e-05 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190435+0000", 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001905288 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168312 8.6 8:679ba278:::benchmark_data_smithi029_194418_object62002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.409075+0000", 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 511.86049205900002, 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.782290435, 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168312 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409075+0000", 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409075+0000", 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409076+0000", 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2200000000000002e-07 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409080+0000", 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4240000000000003e-06 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409092+0000", 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1416e-05 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409100+0000", 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7260000000000008e-06 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409233+0000", 2024-04-12T11:54:25.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013282999999999999 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.409245+0000", 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2078e-05 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973213+0000", 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56396750699999998 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973222+0000", 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1900000000000001e-06 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.173473+0000", 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.200250587 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.173613+0000", 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000140218 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.174102+0000", 2024-04-12T11:54:25.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00048906700000000004 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.178907+0000", 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048052770000000002 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.181255+0000", 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023480789999999999 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190540+0000", 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092848930000000007 2024-04-12T11:54:25.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190596+0000", 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6042999999999999e-05 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.191365+0000", 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00076917800000000003 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:168313 8.16 8:6d52a699:::benchmark_data_smithi029_194418_object62003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1209)", 2024-04-12T11:54:25.455 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:52.410304+0000", 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 511.85926315699999, 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.780861611, 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 168313 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410304+0000", 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410304+0000", 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410305+0000", 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7499999999999998e-07 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410310+0000", 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7209999999999999e-06 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410327+0000", 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6297999999999999e-05 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410336+0000", 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0359999999999995e-06 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410401+0000", 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5167999999999997e-05 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.410413+0000", 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1912999999999999e-05 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973335+0000", 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56292207400000005 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:52.973345+0000", 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.91e-06 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.178899+0000", 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2055542189999999 2024-04-12T11:54:25.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.178983+0000", 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4301999999999993e-05 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6,7", 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.179305+0000", 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032191099999999997 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.181339+0000", 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020332520000000001 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.184110+0000", 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027713640000000001 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.184229+0000", 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011907499999999999 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.190999+0000", 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067697510000000001 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.191053+0000", 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3785000000000001e-05 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:54.191165+0000", 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011285699999999999 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28344 10.6 10:6e372a0a:::benchmark_data_smithi029_256330_object28343:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.636497+0000", 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 54.633069906999999, 2024-04-12T11:54:25.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.514864867, 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28344 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636497+0000", 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636497+0000", 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636498+0000", 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.031e-06 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636529+0000", 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0688999999999997e-05 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636540+0000", 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0917000000000001e-05 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636547+0000", 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4250000000000001e-06 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640097+0000", 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035499939999999999 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640239+0000", 2024-04-12T11:54:25.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000142113 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4", 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640380+0000", 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000141113 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.641764+0000", 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013837739999999999 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.150979+0000", 2024-04-12T11:54:25.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50921531600000003 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151037+0000", 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7224e-05 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151362+0000", 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032527100000000002 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28359 10.8 10:1699d62b:::benchmark_data_smithi029_256330_object28358:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.640282+0000", 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 54.629284519000002, 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51126328099999996, 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28359 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640282+0000", 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640282+0000", 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640284+0000", 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2550000000000001e-06 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640310+0000", 2024-04-12T11:54:25.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6261000000000001e-05 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640321+0000", 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1199999999999999e-05 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640330+0000", 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7010000000000002e-06 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640392+0000", 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2129000000000001e-05 2024-04-12T11:54:25.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640525+0000", 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013261800000000001 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640649+0000", 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012463700000000001 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.641965+0000", 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001315433 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.642936+0000", 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00097142699999999997 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151453+0000", 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50851709099999998 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151504+0000", 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0624000000000002e-05 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151546+0000", 2024-04-12T11:54:25.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1904999999999999e-05 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28362 10.8 10:1e680b68:::benchmark_data_smithi029_256330_object28361:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.643369+0000", 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 54.626198260000002, 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51099498499999996, 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.473 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28362 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643369+0000", 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643369+0000", 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643369+0000", 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2399999999999998e-07 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643391+0000", 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1288999999999999e-05 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643399+0000", 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6379999999999993e-06 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643406+0000", 2024-04-12T11:54:25.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7789999999999998e-06 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.645541+0000", 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002134698 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.645668+0000", 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000127813 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.645850+0000", 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018152399999999999 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152136+0000", 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50628628399999998 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152428+0000", 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000291236 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154207+0000", 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017794110000000001 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154317+0000", 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110235 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154364+0000", 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6453999999999999e-05 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28367 10.8 10:172b0297:::benchmark_data_smithi029_256330_object28366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.643647+0000", 2024-04-12T11:54:25.478 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 54.625920121999997, 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51514234299999995, 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28367 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643647+0000", 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643647+0000", 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643648+0000", 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6000000000000002e-07 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643669+0000", 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1191000000000001e-05 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643679+0000", 2024-04-12T11:54:25.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0607e-05 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643691+0000", 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1769e-05 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152468+0000", 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50877640800000001 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152609+0000", 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000141058 2024-04-12T11:54:25.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152760+0000", 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015116700000000001 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158543+0000", 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0057828669999999997 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158586+0000", 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3633000000000001e-05 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158683+0000", 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6847000000000003e-05 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158742+0000", 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8697999999999999e-05 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158789+0000", 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7237999999999997e-05 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28370 10.6 10:6134ad43:::benchmark_data_smithi029_256330_object28369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.644366+0000", 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 54.625200530999997, 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51561275900000003, 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.484 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28370 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644366+0000", 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644366+0000", 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644367+0000", 2024-04-12T11:54:25.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1699999999999997e-07 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644431+0000", 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3511999999999998e-05 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644443+0000", 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2517000000000001e-05 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644451+0000", 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7989999999999994e-06 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151408+0000", 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50695730000000006 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151604+0000", 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019551200000000001 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4", 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151805+0000", 2024-04-12T11:54:25.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000201466 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.153704+0000", 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001898409 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.159834+0000", 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061304890000000003 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.159891+0000", 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6725000000000003e-05 2024-04-12T11:54:25.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.159979+0000", 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8213000000000004e-05 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28372 10.8 10:100b9aa0:::benchmark_data_smithi029_256330_object28371:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.644503+0000", 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 54.625064102000003, 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.514388289, 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.489 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28372 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644503+0000", 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644503+0000", 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644504+0000", 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5600000000000005e-07 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644527+0000", 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3509000000000001e-05 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644536+0000", 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0799999999999995e-06 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644543+0000", 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8029999999999999e-06 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154400+0000", 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50985725800000004 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154537+0000", 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013657400000000001 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154983+0000", 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00044639700000000002 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158625+0000", 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036415060000000001 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158663+0000", 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8380999999999997e-05 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158800+0000", 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000137427 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158848+0000", 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7664000000000003e-05 2024-04-12T11:54:25.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158891+0000", 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2933999999999999e-05 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:75508 10.8 10:12b3b9c8:::benchmark_data_smithi029_256330_object75507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:13.182801+0000", 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 11.086765629, 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73462143300000005, 2024-04-12T11:54:25.495 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75508 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182801+0000", 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182801+0000", 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182803+0000", 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.516e-06 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182825+0000", 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.251e-05 2024-04-12T11:54:25.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182838+0000", 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2595000000000001e-05 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182846+0000", 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3699999999999995e-06 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182937+0000", 2024-04-12T11:54:25.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0354999999999998e-05 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.183087+0000", 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015046300000000001 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.183241+0000", 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000153988 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.188275+0000", 2024-04-12T11:54:25.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050336510000000001 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.190168+0000", 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001893385 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.917315+0000", 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.72714688500000002 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.917373+0000", 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7748999999999999e-05 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.917423+0000", 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9966000000000002e-05 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:75532 10.8 10:16d45b92:::benchmark_data_smithi029_256330_object75531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:13.208605+0000", 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 11.060961523, 2024-04-12T11:54:25.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71678397999999999, 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75532 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208605+0000", 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208605+0000", 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208607+0000", 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5069999999999999e-06 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208629+0000", 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2036e-05 2024-04-12T11:54:25.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208644+0000", 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4888e-05 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208653+0000", 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6050000000000004e-06 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208707+0000", 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3154e-05 2024-04-12T11:54:25.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208821+0000", 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011414200000000001 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208925+0000", 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010425900000000001 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.213198+0000", 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042726819999999999 2024-04-12T11:54:25.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.213287+0000", 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9775e-05 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925086+0000", 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71179846400000002 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925163+0000", 2024-04-12T11:54:25.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7373999999999995e-05 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925389+0000", 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022609400000000001 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:75534 10.8 10:1c9cded7:::benchmark_data_smithi029_256330_object75533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:13.209813+0000", 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 11.059754331000001, 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71577829400000004, 2024-04-12T11:54:25.507 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75534 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209813+0000", 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209813+0000", 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209814+0000", 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0899999999999999e-06 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209860+0000", 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6703999999999999e-05 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209872+0000", 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1678999999999999e-05 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209880+0000", 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3990000000000004e-06 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.210963+0000", 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001083028 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.211126+0000", 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000162342 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.211246+0000", 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000120622 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.213352+0000", 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002105909 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.214394+0000", 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010415229999999999 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925433+0000", 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71103895299999997 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925549+0000", 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115869 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925591+0000", 2024-04-12T11:54:25.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2175999999999997e-05 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:76046 10.6 10:685cd136:::benchmark_data_smithi029_256330_object76045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:14.743676+0000", 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 9.5258908429999991, 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36810407000000001, 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76046 2024-04-12T11:54:25.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743676+0000", 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743676+0000", 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743677+0000", 2024-04-12T11:54:25.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2929999999999999e-06 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743699+0000", 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1786000000000001e-05 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743711+0000", 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1473e-05 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743718+0000", 2024-04-12T11:54:25.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2599999999999999e-06 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743792+0000", 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4210999999999998e-05 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743968+0000", 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017591500000000001 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.744157+0000", 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018945600000000001 2024-04-12T11:54:25.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.746515+0000", 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023574239999999999 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.754397+0000", 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078825560000000006 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.108850+0000", 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.35445250900000003 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.108912+0000", 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1970999999999993e-05 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.111780+0000", 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028682159999999998 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:76053 10.6 10:6bdceeb5:::benchmark_data_smithi029_256330_object76052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:14.750992+0000", 2024-04-12T11:54:25.518 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 9.5185751399999994, 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36981416700000003, 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76053 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.750992+0000", 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.750992+0000", 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.750993+0000", 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4980000000000001e-06 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751017+0000", 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3402000000000001e-05 2024-04-12T11:54:25.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751030+0000", 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2982999999999999e-05 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751037+0000", 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7160000000000003e-06 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751157+0000", 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011981000000000001 2024-04-12T11:54:25.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751349+0000", 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000191807 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751649+0000", 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00029982300000000001 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.754558+0000", 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029093610000000001 2024-04-12T11:54:25.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.756360+0000", 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001801932 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.118087+0000", 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36172679800000002 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.118158+0000", 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1054000000000001e-05 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:54:25.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:54:25.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.120806+0000", 2024-04-12T11:54:25.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002647983 2024-04-12T11:54:25.524 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.524 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.524 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.524 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:54:25.524 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:54:25.524 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:54:25.524 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:25.525 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:25 smithi087 ceph-mon[89014]: pgmap v2149: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 720 op/s; 14216/246243 objects degraded (5.773%); 5.4 MiB/s, 86 objects/s recovering 2024-04-12T11:54:25.525 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:25 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4198635250' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:54:25.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:25.565 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:25.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:25.666 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:25.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:25.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:25.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:25.750 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 84988 84972 78.0782 58.6875 0.0285284 0.0128027 2024-04-12T11:54:25.805 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:25.847 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:25.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:25.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:25.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:25.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:25.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:25.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:26.084 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:26.115 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:26.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:26.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:26.267 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:26.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:26.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:26.369 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:26.399 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:26.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:26.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:26.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-04-12T11:54:26.501 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:26.585 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:26.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:26.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:26.726 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:26.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:26.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:26.751 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 85878 85862 77.7522 55.625 0.0127808 0.0128554 2024-04-12T11:54:26.828 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:26.864 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:26.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:26.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:26.966 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:27.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:27.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:27.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:27.161 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:27.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:27.263 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:27.295 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:27.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:27.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:27.397 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:27.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:27.431 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:27.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:27.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:27 smithi087 ceph-mon[89014]: pgmap v2150: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 746 op/s; 14216/248712 objects degraded (5.716%); 5.4 MiB/s, 86 objects/s recovering 2024-04-12T11:54:27.487 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:27 smithi029 ceph-mon[93650]: pgmap v2150: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 746 op/s; 14216/248712 objects degraded (5.716%); 5.4 MiB/s, 86 objects/s recovering 2024-04-12T11:54:27.487 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:27 smithi029 ceph-mon[97787]: pgmap v2150: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 746 op/s; 14216/248712 objects degraded (5.716%); 5.4 MiB/s, 86 objects/s recovering 2024-04-12T11:54:27.532 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:27.538 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:54:27.558 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:27.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:27.660 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:27.718 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:27.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:27.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:27.750 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 86765 86749 77.4334 55.4375 0.00660275 0.0129081 2024-04-12T11:54:27.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:27.880 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:27.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:27.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:27.982 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:28.012 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:28.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:28.114 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:28.160 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:28.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:28.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:28.291 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:28.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:28.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:28.440 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:28.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:28.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:28.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 14216/248712 objects degraded (5.716%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:28.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 14216/248712 objects degraded (5.716%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:28.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 14216/248712 objects degraded (5.716%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:28.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:28.578 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:28.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:28.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:28.616 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:54:28.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:28.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:28.726 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:28.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:28.751 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 87677 87661 77.1455 57 0.0198489 0.0129567 2024-04-12T11:54:28.828 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:28.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:28.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:28.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:28.978 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:29.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:29.045 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:29.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:29.146 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:29.175 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:29.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:29.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:29.247 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 3] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:54:29.247 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:54:29.247 INFO:tasks.thrashosds.thrasher:Reweighting osd 6 to 0.792020106353153 2024-04-12T11:54:29.247 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 6 0.792020106353153 2024-04-12T11:54:29.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:29.339 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:29.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:29.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:29.450 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:29.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:29 smithi087 ceph-mon[89014]: pgmap v2151: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 13956/256119 objects degraded (5.449%); 5.5 MiB/s, 88 objects/s recovering 2024-04-12T11:54:29.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:29.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:29.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:29.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:29 smithi029 ceph-mon[93650]: pgmap v2151: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 13956/256119 objects degraded (5.449%); 5.5 MiB/s, 88 objects/s recovering 2024-04-12T11:54:29.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:29 smithi029 ceph-mon[97787]: pgmap v2151: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s; 13956/256119 objects degraded (5.449%); 5.5 MiB/s, 88 objects/s recovering 2024-04-12T11:54:29.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:54:29.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:29.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:29.670 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:29.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:29.755 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 88529 88513 76.8118 53.25 0.0162957 0.0130127 2024-04-12T11:54:29.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:29.829 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:29.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:29.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:29.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:29.973 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:29.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:29.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:30.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:30.104 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:30.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:30.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:30.205 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:30.241 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:30.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:30.343 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:30.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:30.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:30.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:30.488 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:30.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:30.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:30.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:30.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:30.656 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:30.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:30.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:30.752 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 89414 89398 76.5173 55.3125 0.00810588 0.0130625 2024-04-12T11:54:30.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:30.809 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:30.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:30.911 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:30.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:30.950 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:30.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:31.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:31.080 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:31.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:31.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:31.181 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:31.222 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:31.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:31.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:31.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:31.362 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:31.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:31.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:31.464 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:31.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:31 smithi087 ceph-mon[89014]: pgmap v2152: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s; 13753/262017 objects degraded (5.249%); 5.6 MiB/s, 89 objects/s recovering 2024-04-12T11:54:31.480 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:31 smithi029 ceph-mon[93650]: pgmap v2152: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s; 13753/262017 objects degraded (5.249%); 5.6 MiB/s, 89 objects/s recovering 2024-04-12T11:54:31.480 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:31 smithi029 ceph-mon[97787]: pgmap v2152: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s; 13753/262017 objects degraded (5.249%); 5.6 MiB/s, 89 objects/s recovering 2024-04-12T11:54:31.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:31.497 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:31.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:31.598 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:31.626 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:31.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:31.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:31.728 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:31.753 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 90290 90274 76.2223 54.75 0.0291949 0.013114 2024-04-12T11:54:31.782 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:31.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:31.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:31.913 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:31.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:31.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:32.015 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:32.044 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:32.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:32.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:32.187 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:32 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1231781246' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:54:32.187 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:32 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:54:32.188 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:54:32.204 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:32.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:32.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:32.306 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:32.345 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:32.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:32.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:32.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:32.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:32 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1231781246' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:54:32.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:32 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:54:32.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:54:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:54:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:54:32.486 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:32.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:32.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:32 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1231781246' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:54:32.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:32 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:54:32.588 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:32.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:32.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:32.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:32.733 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:32.754 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 91094 91078 75.8751 50.25 0.0146205 0.0131739 2024-04-12T11:54:32.765 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:32.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:32.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:32.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:32.937 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:32.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:32.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:33.038 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:33.068 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:33.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:33.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:33.170 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:33.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:33.220 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:33.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:33.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:33.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:33.371 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:33.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:33 smithi087 ceph-mon[89014]: pgmap v2153: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 731 op/s; 13753/262017 objects degraded (5.249%); 4.7 MiB/s, 74 objects/s recovering 2024-04-12T11:54:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:33 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:54:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:33 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:54:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:33 smithi087 ceph-mon[89014]: osdmap e1284: 8 total, 8 up, 7 in 2024-04-12T11:54:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:33 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:33 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13753/262017 objects degraded (5.249%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:33.472 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:33.501 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:33.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:33.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[93650]: pgmap v2153: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 731 op/s; 13753/262017 objects degraded (5.249%); 4.7 MiB/s, 74 objects/s recovering 2024-04-12T11:54:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:54:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:54:33.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[93650]: osdmap e1284: 8 total, 8 up, 7 in 2024-04-12T11:54:33.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:33.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:33.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13753/262017 objects degraded (5.249%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:33.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[97787]: pgmap v2153: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 731 op/s; 13753/262017 objects degraded (5.249%); 4.7 MiB/s, 74 objects/s recovering 2024-04-12T11:54:33.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:54:33.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:54:33.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[97787]: osdmap e1284: 8 total, 8 up, 7 in 2024-04-12T11:54:33.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:33.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:33.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13753/262017 objects degraded (5.249%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:33.602 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:33.607 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:54:33.607 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:54:33.615 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:54:33.617 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:54:33.631 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:33.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:33.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:33.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:33.757 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 91832 91816 75.4826 46.125 0.019178 0.0132419 2024-04-12T11:54:33.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:33.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:33.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:33.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:33.926 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:33.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:34.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:34.059 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:34.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:34.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:34.161 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:34.188 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:34.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:34.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:34.290 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:34.323 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:34.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:34.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:34.424 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:34.475 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:34.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:34.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:34.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:34.620 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:34.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:34.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:34.722 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:34.754 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:34.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:34.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:34.759 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 92632 92616 75.1481 50 0.0106823 0.0133003 2024-04-12T11:54:34.856 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:34.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:34.896 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:34.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:34.997 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:35.030 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:35.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:35.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:35.131 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:35.167 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:35.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:35.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:35.190 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:54:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:54:34] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-04-12T11:54:35.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:35.308 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:35.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:35.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:35.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:35.449 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:35.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:35.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:35.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:35 smithi087 ceph-mon[89014]: pgmap v2155: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 867 op/s; 13266/272262 objects degraded (4.873%); 5.9 MiB/s, 94 objects/s recovering 2024-04-12T11:54:35.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:35 smithi029 ceph-mon[93650]: pgmap v2155: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 867 op/s; 13266/272262 objects degraded (4.873%); 5.9 MiB/s, 94 objects/s recovering 2024-04-12T11:54:35.521 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:35 smithi029 ceph-mon[97787]: pgmap v2155: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.5 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 867 op/s; 13266/272262 objects degraded (4.873%); 5.9 MiB/s, 94 objects/s recovering 2024-04-12T11:54:35.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:35.584 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:35.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:35.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:35.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:35.726 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:35.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:35.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:35.759 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 93466 93450 74.8529 52.125 0.0217086 0.0133536 2024-04-12T11:54:35.828 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:35.866 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:35.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:35.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:35.968 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:35.998 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:35.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:36.099 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:36.129 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:36.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:36.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:36.232 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:36.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:36.256 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:36.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:36.357 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:36.387 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:36.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:36.483 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:54:36.489 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:36.522 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:36.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:36.624 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:36.671 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:36.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:36.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:36.761 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 94204 94188 74.4876 46.125 0.0143631 0.0134196 2024-04-12T11:54:36.773 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:36.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:36.810 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:36.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:36.911 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:36.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:36.968 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:36.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:37.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:37.096 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:37.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:37.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:37.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:37.222 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:37.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:37.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:37.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:37.367 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:37.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:37.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:37.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:37.528 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:37.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:37.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:37.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:37.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:37.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:37 smithi087 ceph-mon[89014]: pgmap v2156: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 13266/274677 objects degraded (4.830%); 5.9 MiB/s, 94 objects/s recovering 2024-04-12T11:54:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:37.762 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:54:37.761236+0000 min lat: 0.00116361 max lat: 2.15979 avg lat: 0.0134877 2024-04-12T11:54:37.762 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:54:37.762 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 94913 94897 74.1105 44.3125 0.0163592 0.0134877 2024-04-12T11:54:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:37 smithi029 ceph-mon[97787]: pgmap v2156: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 13266/274677 objects degraded (4.830%); 5.9 MiB/s, 94 objects/s recovering 2024-04-12T11:54:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:37 smithi029 ceph-mon[93650]: pgmap v2156: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s; 13266/274677 objects degraded (4.830%); 5.9 MiB/s, 94 objects/s recovering 2024-04-12T11:54:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:54:37.783 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:37.832 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:37.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:37.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:37.975 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:37.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:37.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:38.077 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:38.105 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:38.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:38.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:38.207 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:38.249 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:38.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:38.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:38.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:38.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:38.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:38.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:38.521 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:38.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:38.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:38.622 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:38.649 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:38.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:38.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13266/274677 objects degraded (4.830%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:38.751 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:38.761 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 95616 95600 73.738 43.9375 0.0108376 0.0135558 2024-04-12T11:54:38.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13266/274677 objects degraded (4.830%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:38.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13266/274677 objects degraded (4.830%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:38.785 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:38.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:38.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:38.887 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:38.919 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:38.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:38.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:39.020 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:39.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:39.046 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:39.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:39.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:39.175 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:39.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:39.277 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:39.314 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:39.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:39.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:39.415 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:39.444 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:39.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:39.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:39.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-04-12T11:54:39.546 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:39.579 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:39.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:39.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:39.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:39 smithi087 ceph-mon[89014]: pgmap v2157: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 52 MiB/s wr, 838 op/s; 13004/281268 objects degraded (4.623%); 5.9 MiB/s, 95 objects/s recovering 2024-04-12T11:54:39.737 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:39.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:39.761 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 96263 96247 73.332 40.4375 0.00494156 0.0136248 2024-04-12T11:54:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:39 smithi029 ceph-mon[93650]: pgmap v2157: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 52 MiB/s wr, 838 op/s; 13004/281268 objects degraded (4.623%); 5.9 MiB/s, 95 objects/s recovering 2024-04-12T11:54:39.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:39 smithi029 ceph-mon[97787]: pgmap v2157: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 52 MiB/s wr, 838 op/s; 13004/281268 objects degraded (4.623%); 5.9 MiB/s, 95 objects/s recovering 2024-04-12T11:54:39.839 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:39.869 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:39.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:39.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:39.972 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:40.004 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:40.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:40.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:40.106 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:40.132 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:40.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:40.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:40.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:40.275 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:40.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:40.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:40.377 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:40.410 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:40.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:40.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:40.512 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:40.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:40.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:40.637 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:40.666 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:40.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:40.762 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 96953 96937 72.9681 43.125 0.0155566 0.0136987 2024-04-12T11:54:40.768 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:40.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:40.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:40.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:40.942 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:40.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:40.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:41.044 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:41.067 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:41.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:41.169 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:41.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:41.192 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:41.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:41.294 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:41.331 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:41.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:41.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:41.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:41.466 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:41.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:41.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:41.606 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:41.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:41.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:41.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:41 smithi087 ceph-mon[89014]: pgmap v2158: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s; 12737/286143 objects degraded (4.451%); 6.3 MiB/s, 101 objects/s recovering 2024-04-12T11:54:41.731 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:41.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:41.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:41.761 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 97568 97552 72.5571 38.4375 0.00957164 0.0137625 2024-04-12T11:54:41.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:41 smithi029 ceph-mon[93650]: pgmap v2158: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s; 12737/286143 objects degraded (4.451%); 6.3 MiB/s, 101 objects/s recovering 2024-04-12T11:54:41.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:41 smithi029 ceph-mon[97787]: pgmap v2158: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s; 12737/286143 objects degraded (4.451%); 6.3 MiB/s, 101 objects/s recovering 2024-04-12T11:54:41.833 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:41.857 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:41.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:41.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:41.959 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:41.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:41.989 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:41.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:42.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:42.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:42.128 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:42.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:42.229 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:42.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:42.252 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:42.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:42.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:42.385 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:42.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:42.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:42.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:42.523 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:42.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:42.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:42.625 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:42.656 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:42.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:42.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:54:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:54:42.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:54:42.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:42.763 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 98167 98151 72.1428 37.4375 0.0123191 0.0138552 2024-04-12T11:54:42.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:42.805 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:42.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:42.906 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:42.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:42.930 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:42.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:43.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:43.060 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:43.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:43.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:43.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:43.197 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:43.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:43.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:43.299 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:43.334 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:43.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:43.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:43.478 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:43.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:43.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:43.579 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:43.608 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:43.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:43.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:43.710 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:43 smithi087 ceph-mon[89014]: pgmap v2159: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s; 12737/286143 objects degraded (4.451%); 6.3 MiB/s, 101 objects/s recovering 2024-04-12T11:54:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 12737/286143 objects degraded (4.451%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:43.754 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:43.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:43.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:43.763 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 98831 98815 71.7865 41.5 0.0190957 0.0139247 2024-04-12T11:54:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:43 smithi029 ceph-mon[97787]: pgmap v2159: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s; 12737/286143 objects degraded (4.451%); 6.3 MiB/s, 101 objects/s recovering 2024-04-12T11:54:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 12737/286143 objects degraded (4.451%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:43 smithi029 ceph-mon[93650]: pgmap v2159: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.8 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s; 12737/286143 objects degraded (4.451%); 6.3 MiB/s, 101 objects/s recovering 2024-04-12T11:54:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 12737/286143 objects degraded (4.451%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:43.856 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:43.889 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:43.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:43.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:43.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:44.034 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:44.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:44.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:44.136 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:44.160 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:44.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:44.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:44.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:44.286 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:44.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:44.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:44.388 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:44.411 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:44.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:44.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:44.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:44.560 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:44.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:44.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:44.662 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:44.717 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:44.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:44.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:44.763 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 99418 99402 71.3832 36.6875 0.0421831 0.0140018 2024-04-12T11:54:44.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:44.857 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:44.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:44.959 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:44.992 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:44.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:44.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:45.094 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:45.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:45.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:45.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:45.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:45.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:54:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:54:44] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.43.0" 2024-04-12T11:54:45.273 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:45.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:45.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:45.374 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:45.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:45.407 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:45.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:45.508 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:45.545 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:45.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:45.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:45.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:45.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:45.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:45.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:45 smithi087 ceph-mon[89014]: pgmap v2160: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s; 12208/293841 objects degraded (4.155%); 8.1 MiB/s, 129 objects/s recovering 2024-04-12T11:54:45.763 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 100275 100259 71.1807 53.5625 0.00948225 0.0140436 2024-04-12T11:54:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:45 smithi029 ceph-mon[97787]: pgmap v2160: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s; 12208/293841 objects degraded (4.155%); 8.1 MiB/s, 129 objects/s recovering 2024-04-12T11:54:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:45 smithi029 ceph-mon[93650]: pgmap v2160: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s; 12208/293841 objects degraded (4.155%); 8.1 MiB/s, 129 objects/s recovering 2024-04-12T11:54:45.784 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:45.813 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:45.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:45.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:45.915 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:45.945 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:45.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:45.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:46.047 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:46.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:46.083 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:46.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:46.184 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:46.208 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:46.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:46.309 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:46.358 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:46.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:46.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:46.460 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:46.504 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:46.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:46.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:46.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:46.636 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:46.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:46.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:46.738 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:46.763 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 101042 101026 70.9195 47.9375 0.0138667 0.0140945 2024-04-12T11:54:46.770 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:46.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:46.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:46.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:46.921 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:46.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:47.023 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:47.046 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:47.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:47.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:47.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:47.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:47.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:47.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:47.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:47.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:47.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:47.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:47.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:47.457 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:47.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:47.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:47.558 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:47.591 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:47.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:47.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:47.693 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:47 smithi087 ceph-mon[89014]: pgmap v2161: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 12208/295704 objects degraded (4.128%); 5.5 MiB/s, 88 objects/s recovering 2024-04-12T11:54:47.730 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:47.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:47.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:47.759 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:47 smithi029 ceph-mon[97787]: pgmap v2161: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 12208/295704 objects degraded (4.128%); 5.5 MiB/s, 88 objects/s recovering 2024-04-12T11:54:47.759 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:47 smithi029 ceph-mon[93650]: pgmap v2161: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 12208/295704 objects degraded (4.128%); 5.5 MiB/s, 88 objects/s recovering 2024-04-12T11:54:47.763 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0081 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 101763 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 70.6624 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 44.5259 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 234.812 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1130 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 712.414 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3757 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.014147 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0359801 2024-04-12T11:54:47.764 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 2.15979 2024-04-12T11:54:47.765 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00116361 2024-04-12T11:54:47.771 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2024-04-12T11:54:47.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:47.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:47.855 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:47.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:47.956 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:47.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:47.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:47.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:48.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:48.120 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:48.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:48.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:48.221 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:48.260 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:48.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:48.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:48.362 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:48.394 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:48.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:48.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:48.496 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:48.527 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:48.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:48.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:48.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:48.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:48.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 12208/295704 objects degraded (4.128%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:48.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:48.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 12208/295704 objects degraded (4.128%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:48.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 12208/295704 objects degraded (4.128%), 5 pgs degraded (PG_DEGRADED) 2024-04-12T11:54:48.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:48.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:48.881 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:48.919 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:48.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:48.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:49.021 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:49.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:49.058 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:49.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:49.159 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:49.190 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:49.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:49.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:49.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:49.335 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:49.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:49.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:49.437 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:49.460 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:49.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:49.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:49.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:49.603 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:49.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:49.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:49 smithi087 ceph-mon[89014]: pgmap v2162: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 44 MiB/s wr, 697 op/s; 11783/299802 objects degraded (3.930%); 7.7 MiB/s, 123 objects/s recovering 2024-04-12T11:54:49.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:49.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:49.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:49 smithi029 ceph-mon[93650]: pgmap v2162: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 44 MiB/s wr, 697 op/s; 11783/299802 objects degraded (3.930%); 7.7 MiB/s, 123 objects/s recovering 2024-04-12T11:54:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:49 smithi029 ceph-mon[97787]: pgmap v2162: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 44 MiB/s wr, 697 op/s; 11783/299802 objects degraded (3.930%); 7.7 MiB/s, 123 objects/s recovering 2024-04-12T11:54:49.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:49.888 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:49.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:49.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:49.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:50.012 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:50.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:50.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:50.114 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:50.139 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:50.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:50.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:50.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:50.278 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:50.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:50.380 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:50.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:50.414 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:50.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:50.515 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:50.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:50.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:50.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:50.649 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:50.683 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:50.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:50.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:50.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:50.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:50.826 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:50.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:50.927 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:50.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:50.958 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:50.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:51.059 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:51.083 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:51.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:51.185 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:51.209 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:51.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:51.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:51.311 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:51.335 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:51.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:51.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:51.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:51.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:51.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:51.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:51.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:51.607 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:51.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:51.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:51.709 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:51 smithi087 ceph-mon[89014]: pgmap v2163: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 85 B/s rd, 41 MiB/s wr, 706 op/s; 11488/303480 objects degraded (3.785%); 7.9 MiB/s, 126 objects/s recovering 2024-04-12T11:54:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:51.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:51.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:51.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:51.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:51 smithi029 ceph-mon[97787]: pgmap v2163: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 85 B/s rd, 41 MiB/s wr, 706 op/s; 11488/303480 objects degraded (3.785%); 7.9 MiB/s, 126 objects/s recovering 2024-04-12T11:54:51.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:51 smithi029 ceph-mon[93650]: pgmap v2163: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 85 B/s rd, 41 MiB/s wr, 706 op/s; 11488/303480 objects degraded (3.785%); 7.9 MiB/s, 126 objects/s recovering 2024-04-12T11:54:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:54:51.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:51.876 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:51.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:51.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:51.978 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:52.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:52.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:52.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:52.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:52.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:52.155 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:52.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:52.256 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:52.297 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:52.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:52.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:52.399 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:52.428 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:52.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:52.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:52.530 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:52.559 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:52.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:52.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:52.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:54:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:54:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:54:52.660 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:52.684 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:52.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:52.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:52.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:52.810 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:52.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:52.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:52.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:52.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:52.937 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:52.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:53.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:53.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:53.062 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:53.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:53.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:53.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:53.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:53.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:53.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:53.312 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:53.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:53.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:53.414 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:53.450 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:53.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:53.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:53.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:53.591 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:53.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:53.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:53.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:53 smithi087 ceph-mon[89014]: pgmap v2164: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 85 B/s rd, 33 MiB/s wr, 571 op/s; 11488/303480 objects degraded (3.785%); 6.5 MiB/s, 104 objects/s recovering 2024-04-12T11:54:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:53 smithi087 ceph-mon[89014]: osdmap e1285: 8 total, 8 up, 7 in 2024-04-12T11:54:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11488/303480 objects degraded (3.785%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T11:54:53.737 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:53.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:53 smithi029 ceph-mon[97787]: pgmap v2164: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 85 B/s rd, 33 MiB/s wr, 571 op/s; 11488/303480 objects degraded (3.785%); 6.5 MiB/s, 104 objects/s recovering 2024-04-12T11:54:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:53 smithi029 ceph-mon[97787]: osdmap e1285: 8 total, 8 up, 7 in 2024-04-12T11:54:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11488/303480 objects degraded (3.785%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T11:54:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:53 smithi029 ceph-mon[93650]: pgmap v2164: 99 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.2 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 85 B/s rd, 33 MiB/s wr, 571 op/s; 11488/303480 objects degraded (3.785%); 6.5 MiB/s, 104 objects/s recovering 2024-04-12T11:54:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:53 smithi029 ceph-mon[93650]: osdmap e1285: 8 total, 8 up, 7 in 2024-04-12T11:54:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11488/303480 objects degraded (3.785%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-12T11:54:53.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:53.880 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:53.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:53.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:54.017 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:54.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:54.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:54.119 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:54.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:54.150 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:54.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:54.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:54.283 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:54.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:54.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:54.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:54.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:54.428 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:54.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:54.435 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:54 smithi029 ceph-mon[97787]: osdmap e1286: 8 total, 8 up, 7 in 2024-04-12T11:54:54.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:54 smithi029 ceph-mon[93650]: osdmap e1286: 8 total, 8 up, 7 in 2024-04-12T11:54:54.530 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:54.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:54.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:54.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:54.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:54.709 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:54.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:54.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:54 smithi087 ceph-mon[89014]: osdmap e1286: 8 total, 8 up, 7 in 2024-04-12T11:54:54.810 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:54.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:54.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:54.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:54.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:54.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:54.992 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:54.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:55.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:55.129 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:55.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:55.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:55.155 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:54:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:54:54] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.43.0" 2024-04-12T11:54:55.231 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:55.263 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:55.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:55.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:55.365 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:55.403 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:54:55.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:55.413 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:55.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:55.443 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:54:55.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:55 smithi029 ceph-mon[97787]: pgmap v2167: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 25 MiB/s wr, 786 op/s; 10427/296154 objects degraded (3.521%); 14 MiB/s, 222 objects/s recovering 2024-04-12T11:54:55.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:55 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3157169799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:54:55.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:55 smithi029 ceph-mon[93650]: pgmap v2167: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 25 MiB/s wr, 786 op/s; 10427/296154 objects degraded (3.521%); 14 MiB/s, 222 objects/s recovering 2024-04-12T11:54:55.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:55 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3157169799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:54:55.484 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:54:55.513 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:55.522 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:54:55.549 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:55.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:55.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:55.567 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:54:55.603 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:54:55.641 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:54:55.651 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:55.683 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:54:55.687 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:55.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:55.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:55.690 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:54:55.690 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:54:55.690 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:54:55.690 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:54:55.690 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:54:55.690 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:54:55.690 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:54:55.690 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:54:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:55 smithi087 ceph-mon[89014]: pgmap v2167: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 127 B/s rd, 25 MiB/s wr, 786 op/s; 10427/296154 objects degraded (3.521%); 14 MiB/s, 222 objects/s recovering 2024-04-12T11:54:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:55 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3157169799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:54:55.789 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:55.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:55.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:55.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:55.932 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:55.964 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:55.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:55.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:56.066 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:56.098 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:56.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:56.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:56.200 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:56.233 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:56.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:56.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:56.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:56.378 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:56.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:56.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:56.480 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:56.508 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:56.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:56.609 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:56.644 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:56.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:56.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:56.746 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:56.773 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:56.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:56.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:56.874 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:56.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:56.936 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:56.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:57.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:57.064 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:57.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:57.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:57.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:57.212 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:57.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:57.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:57.313 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:54:57.343 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:57.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:57.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:54:57.445 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:57.473 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:57.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:57.507 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:54:57.511 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:54:57.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:54:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:57.645 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:57.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:54:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:57 smithi029 ceph-mon[93650]: pgmap v2168: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 127 B/s rd, 14 MiB/s wr, 687 op/s; 10427/294417 objects degraded (3.542%); 11 MiB/s, 169 objects/s recovering 2024-04-12T11:54:57.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:57 smithi029 ceph-mon[97787]: pgmap v2168: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 127 B/s rd, 14 MiB/s wr, 687 op/s; 10427/294417 objects degraded (3.542%); 11 MiB/s, 169 objects/s recovering 2024-04-12T11:54:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:57 smithi087 ceph-mon[89014]: pgmap v2168: 99 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 127 B/s rd, 14 MiB/s wr, 687 op/s; 10427/294417 objects degraded (3.542%); 11 MiB/s, 169 objects/s recovering 2024-04-12T11:54:57.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:57.788 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:57.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:57.889 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:57.915 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:57.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:57.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:58.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:58.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:58.071 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:58.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:58.172 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:58.207 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:58.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:58.309 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:58.354 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:58.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:58.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:58.425 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.6 to 0.79202 (cac1) 2024-04-12T11:54:58.456 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:58.502 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:58.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:58.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:58.604 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:58.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:54:58.629 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:58.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:58.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:58 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3987958264' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.792020106353153}]: dispatch 2024-04-12T11:54:58.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:58 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.792020106353153}]: dispatch 2024-04-12T11:54:58.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 10427/294417 objects degraded (3.542%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T11:54:58.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:58 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3987958264' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.792020106353153}]: dispatch 2024-04-12T11:54:58.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:58 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.792020106353153}]: dispatch 2024-04-12T11:54:58.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 10427/294417 objects degraded (3.542%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T11:54:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:58 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3987958264' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.792020106353153}]: dispatch 2024-04-12T11:54:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:58 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.792020106353153}]: dispatch 2024-04-12T11:54:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 10427/294417 objects degraded (3.542%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T11:54:58.731 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:58.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:58.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:58.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:58.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:54:58.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:58.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:58.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:54:58.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:54:59.008 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:59.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:59.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:54:59.109 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:59.133 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:59.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:59.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:59.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:59.259 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:59.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:59.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:59.361 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:54:59.384 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:59.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:59.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:54:59.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:54:59.510 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:59.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:59.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:54:59.612 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:59.635 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:59.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:59.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:59 smithi087 ceph-mon[89014]: pgmap v2169: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 720 KiB/s wr, 571 op/s; 10427/290295 objects degraded (3.592%); 8.3 MiB/s, 132 objects/s recovering 2024-04-12T11:54:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:59 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.792020106353153}]': finished 2024-04-12T11:54:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:59 smithi087 ceph-mon[89014]: osdmap e1287: 8 total, 8 up, 7 in 2024-04-12T11:54:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:54:59 smithi087 ceph-mon[89014]: osdmap e1288: 8 total, 8 up, 7 in 2024-04-12T11:54:59.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:54:59.760 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:54:59.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:59.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:54:59.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:59 smithi029 ceph-mon[97787]: pgmap v2169: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 720 KiB/s wr, 571 op/s; 10427/290295 objects degraded (3.592%); 8.3 MiB/s, 132 objects/s recovering 2024-04-12T11:54:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:59 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.792020106353153}]': finished 2024-04-12T11:54:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:59 smithi029 ceph-mon[97787]: osdmap e1287: 8 total, 8 up, 7 in 2024-04-12T11:54:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:54:59 smithi029 ceph-mon[97787]: osdmap e1288: 8 total, 8 up, 7 in 2024-04-12T11:54:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:59 smithi029 ceph-mon[93650]: pgmap v2169: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 720 KiB/s wr, 571 op/s; 10427/290295 objects degraded (3.592%); 8.3 MiB/s, 132 objects/s recovering 2024-04-12T11:54:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:59 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.792020106353153}]': finished 2024-04-12T11:54:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:59 smithi029 ceph-mon[93650]: osdmap e1287: 8 total, 8 up, 7 in 2024-04-12T11:54:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:54:59 smithi029 ceph-mon[93650]: osdmap e1288: 8 total, 8 up, 7 in 2024-04-12T11:54:59.862 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:54:59.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:54:59.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:54:59.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:54:59.987 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:00.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:00.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:00.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:00.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:00.178 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:00.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:00.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:00.280 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:00.303 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:00.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:00.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:00.405 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:00.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:00.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:00.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:00.574 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:00.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:00.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:00.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:00.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:00.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:00.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:00.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:00.856 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:00.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:00.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:00.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:00.997 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:00.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:00.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:01.099 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:01.151 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:01.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:01.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:01.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:01.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:01.301 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:01.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:01.401 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:01.461 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:01.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:01.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:01.563 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:01.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:01.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:01.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:01.701 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:01 smithi087 ceph-mon[89014]: pgmap v2172: 99 pgs: 3 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 733 op/s; 9995/283533 objects degraded (3.525%); 9.5 MiB/s, 151 objects/s recovering 2024-04-12T11:55:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:01 smithi087 ceph-mon[89014]: osdmap e1289: 8 total, 8 up, 7 in 2024-04-12T11:55:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:01 smithi087 ceph-mon[89014]: 10.e starting backfill to osd.3 from (0'0,0'0] MAX to 1286'6997 2024-04-12T11:55:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:01 smithi087 ceph-mon[89014]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 1286'6631 2024-04-12T11:55:01.724 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:01 smithi029 ceph-mon[93650]: pgmap v2172: 99 pgs: 3 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 733 op/s; 9995/283533 objects degraded (3.525%); 9.5 MiB/s, 151 objects/s recovering 2024-04-12T11:55:01.724 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:01 smithi029 ceph-mon[93650]: osdmap e1289: 8 total, 8 up, 7 in 2024-04-12T11:55:01.724 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:01 smithi029 ceph-mon[93650]: 10.e starting backfill to osd.3 from (0'0,0'0] MAX to 1286'6997 2024-04-12T11:55:01.724 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:01 smithi029 ceph-mon[93650]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 1286'6631 2024-04-12T11:55:01.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:01 smithi029 ceph-mon[97787]: pgmap v2172: 99 pgs: 3 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 733 op/s; 9995/283533 objects degraded (3.525%); 9.5 MiB/s, 151 objects/s recovering 2024-04-12T11:55:01.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:01 smithi029 ceph-mon[97787]: osdmap e1289: 8 total, 8 up, 7 in 2024-04-12T11:55:01.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:01 smithi029 ceph-mon[97787]: 10.e starting backfill to osd.3 from (0'0,0'0] MAX to 1286'6997 2024-04-12T11:55:01.725 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:01 smithi029 ceph-mon[97787]: 10.d starting backfill to osd.3 from (0'0,0'0] MAX to 1286'6631 2024-04-12T11:55:01.750 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:01.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:01.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:01.852 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:01.880 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:01.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:01.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:01.982 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:02.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:02.014 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:02.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:02.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:02.155 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:02.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:02.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:02.257 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:02.287 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:02.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:02.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:02.389 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:02.423 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:02.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:02.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:02.525 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:02.553 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:02.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:02.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:02.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:02.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:02.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:02.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:02 smithi087 ceph-mon[89014]: osdmap e1290: 8 total, 8 up, 7 in 2024-04-12T11:55:02.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:55:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:55:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:55:02.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:02 smithi029 ceph-mon[97787]: osdmap e1290: 8 total, 8 up, 7 in 2024-04-12T11:55:02.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:02 smithi029 ceph-mon[93650]: osdmap e1290: 8 total, 8 up, 7 in 2024-04-12T11:55:02.804 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:02.831 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:02.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:02.933 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:02.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:02.982 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:02.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:03.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:03.118 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:55:03.118 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:55:03.118 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:55:03.118 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:55:03.119 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:55:03.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:03.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:03.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:03.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:03.275 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:03.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:03.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:03.376 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:03.407 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:03.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:03.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:03.508 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:03.550 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:03.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:03.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:03.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:03.690 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:03.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:03.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:03 smithi087 ceph-mon[89014]: pgmap v2175: 99 pgs: 3 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 9995/283533 objects degraded (3.525%); 6.7 MiB/s, 107 objects/s recovering 2024-04-12T11:55:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:03 smithi087 ceph-mon[89014]: osdmap e1291: 8 total, 8 up, 7 in 2024-04-12T11:55:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9995/283533 objects degraded (3.525%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:03.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:03 smithi029 ceph-mon[93650]: pgmap v2175: 99 pgs: 3 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 9995/283533 objects degraded (3.525%); 6.7 MiB/s, 107 objects/s recovering 2024-04-12T11:55:03.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:03 smithi029 ceph-mon[93650]: osdmap e1291: 8 total, 8 up, 7 in 2024-04-12T11:55:03.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9995/283533 objects degraded (3.525%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:03.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:03 smithi029 ceph-mon[97787]: pgmap v2175: 99 pgs: 3 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 9995/283533 objects degraded (3.525%); 6.7 MiB/s, 107 objects/s recovering 2024-04-12T11:55:03.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:03 smithi029 ceph-mon[97787]: osdmap e1291: 8 total, 8 up, 7 in 2024-04-12T11:55:03.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9995/283533 objects degraded (3.525%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:03.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:03.823 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:03.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:03.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:03.925 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:03.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:03.957 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:03.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:04.058 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:04.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:04.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:04.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:04.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:04.239 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:04.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:04.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:04.341 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:04.376 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:04.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:04.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:04.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:04.498 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-04-12T11:55:04.528 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:04.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:04.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:04.629 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:04.664 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:04.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:04.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:04.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:04.845 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:04.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:04.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:04.947 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:05.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:05.001 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:05.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:05.023 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:55:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:55:04] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.43.0" 2024-04-12T11:55:05.102 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:05.148 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:05.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:05.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:05.250 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:05.279 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:05.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:05.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:05.346 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:55:05.381 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:05.410 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:05.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:05.424 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:55:05.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:05.568 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:05.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:05.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:05.670 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:05.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:05.699 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:05.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:05 smithi087 ceph-mon[89014]: pgmap v2177: 99 pgs: 3 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 9662/277047 objects degraded (3.487%); 4.4 MiB/s, 70 objects/s recovering 2024-04-12T11:55:05.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:05 smithi029 ceph-mon[93650]: pgmap v2177: 99 pgs: 3 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 9662/277047 objects degraded (3.487%); 4.4 MiB/s, 70 objects/s recovering 2024-04-12T11:55:05.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:05 smithi029 ceph-mon[97787]: pgmap v2177: 99 pgs: 3 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.6 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 9662/277047 objects degraded (3.487%); 4.4 MiB/s, 70 objects/s recovering 2024-04-12T11:55:05.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:05.854 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:05.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:05.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:05.955 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:06.005 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:06.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:06.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:06.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:06.153 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:06.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:06.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:06.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:06.301 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:06.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:06.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:06.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:06.440 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:06.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:06.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:06.465 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:55:06.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:06.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:06.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:06.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:06.684 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:06.715 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:06.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:06.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2091770375' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:55:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:06.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2091770375' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:55:06.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:06.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2091770375' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:55:06.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:06.772 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 6, 7, 3] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:55:06.773 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:55:06.773 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 4, 6, 7, 3] 2024-04-12T11:55:06.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd out 1 2024-04-12T11:55:06.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:06.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:06.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:06.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:06.996 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:06.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:06.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:07.098 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:07.127 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:07.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:07.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:07.228 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:07.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:07.259 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:07.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:07.359 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:07.392 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:07.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:07.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:07.494 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:07.519 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:07.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:07.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:07.622 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:07.671 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:07.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:07.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:07 smithi087 ceph-mon[89014]: pgmap v2178: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 9662/274860 objects degraded (3.515%); 10771/274860 objects misplaced (3.919%); 9.3 MiB/s, 149 objects/s recovering 2024-04-12T11:55:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:07 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:55:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:07 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2091770375' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:55:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:07 smithi087 ceph-mon[89014]: osdmap e1292: 8 total, 8 up, 7 in 2024-04-12T11:55:07.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:07 smithi029 ceph-mon[97787]: pgmap v2178: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 9662/274860 objects degraded (3.515%); 10771/274860 objects misplaced (3.919%); 9.3 MiB/s, 149 objects/s recovering 2024-04-12T11:55:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:07 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:55:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:07 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2091770375' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:55:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:07 smithi029 ceph-mon[97787]: osdmap e1292: 8 total, 8 up, 7 in 2024-04-12T11:55:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:07 smithi029 ceph-mon[93650]: pgmap v2178: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 9662/274860 objects degraded (3.515%); 10771/274860 objects misplaced (3.919%); 9.3 MiB/s, 149 objects/s recovering 2024-04-12T11:55:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:07 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:55:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:07 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2091770375' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:55:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:07 smithi029 ceph-mon[93650]: osdmap e1292: 8 total, 8 up, 7 in 2024-04-12T11:55:07.773 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:07.804 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:07.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:07.906 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:07.937 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:07.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:07.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:08.039 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:08.069 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:08.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:08.171 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:08.201 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:08.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:08.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:08.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:08.342 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:08.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:08.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:08.443 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:08.476 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:08.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:08.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:08.578 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:08.611 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:08.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:08.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:08.713 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9662/274860 objects degraded (3.515%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:08.747 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:08.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:08.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9662/274860 objects degraded (3.515%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9662/274860 objects degraded (3.515%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:08.849 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:08.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:08.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:08.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:08.987 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:09.022 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:09.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:09.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:09.123 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:09.157 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:09.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:09.259 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:09.303 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:09.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:09.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:09.405 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:09.436 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:09.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:09.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:09.538 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:09.581 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:09.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:09.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:09.683 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:09 smithi087 ceph-mon[89014]: pgmap v2180: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 9662/270312 objects degraded (3.574%); 10771/270312 objects misplaced (3.985%); 8.4 MiB/s, 133 objects/s recovering 2024-04-12T11:55:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:55:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:55:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:09.719 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:09.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:09.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:09.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[97787]: pgmap v2180: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 9662/270312 objects degraded (3.574%); 10771/270312 objects misplaced (3.985%); 8.4 MiB/s, 133 objects/s recovering 2024-04-12T11:55:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:55:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:55:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[93650]: pgmap v2180: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 9662/270312 objects degraded (3.574%); 10771/270312 objects misplaced (3.985%); 8.4 MiB/s, 133 objects/s recovering 2024-04-12T11:55:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:55:09.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:55:09.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:09.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:09.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:09.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:09.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:09.965 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:09.994 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:09.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:09.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:10.095 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:10.125 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:10.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:10.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:10.226 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:10.252 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:10.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:10.354 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:10.383 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:10.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:10.485 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:10.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:10.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:10.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:10.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:10.658 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:10.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:10.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:10.760 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:10.788 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:10.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:10.889 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:10.937 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:10.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:11.039 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:11.068 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:11.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:11.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:11.117 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160456 8.7 8:e0bb330f:::benchmark_data_smithi029_194418_object54146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.342918+0000", 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 572.76928046499995, 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37251305099999998, 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160456 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342918+0000", 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342918+0000", 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342919+0000", 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4580000000000001e-06 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342924+0000", 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8860000000000003e-06 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342936+0000", 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2337000000000001e-05 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.342944+0000", 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6059999999999997e-06 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343025+0000", 2024-04-12T11:55:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1656999999999994e-05 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343181+0000", 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015508300000000001 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343604+0000", 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00042354900000000001 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.346834+0000", 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032303229999999998 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347546+0000", 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000711369 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.347894+0000", 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034823999999999999 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715256+0000", 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36736171200000001 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715308+0000", 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1749999999999997e-05 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715431+0000", 2024-04-12T11:55:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012308100000000001 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160461 8.3 8:c31b08d7:::benchmark_data_smithi029_194418_object54151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.343478+0000", 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 572.76871975200004, 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.371828979, 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160461 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343478+0000", 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343478+0000", 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343479+0000", 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2399999999999997e-07 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343483+0000", 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3039999999999998e-06 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343494+0000", 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0451999999999999e-05 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.343501+0000", 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6140000000000003e-06 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.344049+0000", 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00054763100000000003 2024-04-12T11:55:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.344163+0000", 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113725 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.344543+0000", 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00038017299999999998 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715034+0000", 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37049053999999998 2024-04-12T11:55:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715054+0000", 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0163999999999999e-05 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715190+0000", 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000135964 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715252+0000", 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2842000000000006e-05 2024-04-12T11:55:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715307+0000", 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4746000000000001e-05 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:160468 8.13 8:c940b4f5:::benchmark_data_smithi029_194418_object54158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1206)", 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:38.350298+0000", 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 572.76189988600004, 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.41454260900000001, 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 160468 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350298+0000", 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350298+0000", 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350299+0000", 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.195e-06 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.350304+0000", 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9860000000000002e-06 2024-04-12T11:55:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715096+0000", 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36479172900000001 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715110+0000", 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3563999999999999e-05 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715435+0000", 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032556700000000002 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.715576+0000", 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014090700000000001 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.716039+0000", 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00046330200000000002 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.759398+0000", 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043358830000000001 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.759416+0000", 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7728999999999999e-05 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.764532+0000", 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051162480000000003 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.764800+0000", 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000267896 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:38.764841+0000", 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0655999999999998e-05 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:166687 8.3 8:c4dc7983:::benchmark_data_smithi029_194418_object60377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1207)", 2024-04-12T11:55:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:48.383448+0000", 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 562.72874972099999, 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37355421100000002, 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166687 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383448+0000", 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383448+0000", 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383449+0000", 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0400000000000005e-07 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383454+0000", 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8760000000000004e-06 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383465+0000", 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0628000000000001e-05 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383472+0000", 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.413e-06 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383528+0000", 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6011999999999998e-05 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.383646+0000", 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011756699999999999 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.384094+0000", 2024-04-12T11:55:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00044867000000000003 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753559+0000", 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36946446300000002 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.753688+0000", 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000129333 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.756946+0000", 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003258169 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.756976+0000", 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0133000000000001e-05 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:48.757002+0000", 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6043000000000001e-05 2024-04-12T11:55:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35217.0:167527 8.b e1208/1207 8:d8b3b4b5:::benchmark_data_smithi029_194418_object61217:head v 1208'10757, mlcod=1208'10755)", 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.368934+0000", 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 560.74326432600003, 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37717215500000001, 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368934+0000", 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368934+0000", 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368935+0000", 2024-04-12T11:55:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.048e-06 2024-04-12T11:55:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368941+0000", 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4930000000000003e-06 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368952+0000", 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0958000000000001e-05 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.368959+0000", 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6449999999999999e-06 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369095+0000", 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000136475 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369115+0000", 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9879000000000001e-05 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.375105+0000", 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059899319999999999 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.746106+0000", 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37100072499999998 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167531 8.f 8:f56e1a80:::benchmark_data_smithi029_194418_object61221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.369039+0000", 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 560.74315860900003, 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.45084892500000001, 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167531 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369039+0000", 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369039+0000", 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369040+0000", 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0920000000000001e-06 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369044+0000", 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4199999999999999e-06 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369051+0000", 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5800000000000003e-06 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369057+0000", 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8510000000000004e-06 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370219+0000", 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011614049999999999 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370344+0000", 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000124948 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,4", 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370468+0000", 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012388500000000001 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752078+0000", 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38161004300000001 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752119+0000", 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1168000000000003e-05 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752172+0000", 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3448000000000001e-05 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819711+0000", 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067538835000000005 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819764+0000", 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3220000000000002e-05 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819888+0000", 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012402999999999999 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167533 8.f 8:f5e9f8d9:::benchmark_data_smithi029_194418_object61223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:55:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.369313+0000", 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 560.74288533100002, 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.45089991499999998, 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167533 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369313+0000", 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369313+0000", 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369314+0000", 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9800000000000002e-07 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369318+0000", 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.685e-06 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369329+0000", 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0509000000000001e-05 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369336+0000", 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5850000000000002e-06 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374380+0000", 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050431699999999996 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374496+0000", 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011610100000000001 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,4", 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374620+0000", 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012398500000000001 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752288+0000", 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37766854799999999 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752332+0000", 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3488000000000001e-05 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752374+0000", 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1845999999999999e-05 2024-04-12T11:55:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820056+0000", 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067682592 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820100+0000", 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3464999999999998e-05 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820213+0000", 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011294300000000001 2024-04-12T11:55:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167535 8.0 8:023edd32:::benchmark_data_smithi029_194418_object61225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 560.74285029400005, 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38064882700000002, 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:55:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167535 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369348+0000", 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9999999999999998e-07 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369350+0000", 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.351e-06 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369356+0000", 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.874e-06 2024-04-12T11:55:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369362+0000", 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1639999999999998e-06 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369478+0000", 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115281 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.369611+0000", 2024-04-12T11:55:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000132879 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4,5", 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.370066+0000", 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000455045 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.746687+0000", 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37662145200000002 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749353+0000", 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026659259999999999 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749762+0000", 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000408537 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749809+0000", 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7679000000000002e-05 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749868+0000", 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8362999999999997e-05 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749997+0000", 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012887599999999999 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35217.0:167536 8.17 8:effa3b55:::benchmark_data_smithi029_194418_object61226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1208)", 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.372734+0000", 2024-04-12T11:55:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 560.73946412700002, 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.447314869, 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35217", 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1541330537", 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 167536 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372734+0000", 2024-04-12T11:55:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372734+0000", 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372735+0000", 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.53e-06 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372740+0000", 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4349999999999999e-06 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372752+0000", 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2604e-05 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.372761+0000", 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4349999999999997e-06 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.373764+0000", 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001003309 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.373898+0000", 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0001338 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.374260+0000", 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00036174299999999998 2024-04-12T11:55:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752035+0000", 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37777566800000001 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752211+0000", 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000175685 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752248+0000", 2024-04-12T11:55:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6981999999999999e-05 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819903+0000", 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067654598999999996 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.819947+0000", 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3894999999999999e-05 2024-04-12T11:55:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.820049+0000", 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102184 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:167531 8.f e1208/1204 ondisk, result = 0)", 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.376542+0000", 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 560.73565649299996, 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37563801400000002, 2024-04-12T11:55:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376542+0000", 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376542+0000", 2024-04-12T11:55:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376543+0000", 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.229e-06 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.376548+0000", 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2159999999999997e-06 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749229+0000", 2024-04-12T11:55:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.372680911 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.749236+0000", 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6140000000000003e-06 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752159+0000", 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029229360000000001 2024-04-12T11:55:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752167+0000", 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1720000000000001e-06 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752180+0000", 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2935999999999999e-05 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:167536 8.17 e1208/1204 ondisk, result = 0)", 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 560.73500136200005, 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37484725600000002, 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377197+0000", 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7100000000000001e-07 2024-04-12T11:55:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.377202+0000", 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8250000000000004e-06 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.745962+0000", 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36876029599999999 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.745974+0000", 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1501000000000001e-05 2024-04-12T11:55:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752020+0000", 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006045847 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752028+0000", 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4959999999999996e-06 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752044+0000", 2024-04-12T11:55:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.562e-05 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35217.0:167531 8.f e1208/1204 ondisk, result = 0)", 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:45:50.379123+0000", 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 560.73307507899995, 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37296465699999998, 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379123+0000", 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379123+0000", 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379124+0000", 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6599999999999996e-07 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.379128+0000", 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2030000000000002e-06 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.748158+0000", 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.369030319 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.748173+0000", 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4620999999999999e-05 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752064+0000", 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038913590000000001 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752071+0000", 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9770000000000003e-06 2024-04-12T11:55:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:45:50.752088+0000", 2024-04-12T11:55:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6512e-05 2024-04-12T11:55:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:147939 9.14 9:2c77ffe8:::benchmark_data_smithi029_222585_object39162:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-04-12T11:55:11.188 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:50:31.794259+0000", 2024-04-12T11:55:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 279.31793871799999, 2024-04-12T11:55:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38859126199999999, 2024-04-12T11:55:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:55:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 147939 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.794259+0000", 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.794259+0000", 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.794262+0000", 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6630000000000002e-06 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.794271+0000", 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5620000000000004e-06 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.794286+0000", 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5218000000000001e-05 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.794297+0000", 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0958000000000001e-05 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.794372+0000", 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4958999999999998e-05 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.794833+0000", 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00046157199999999999 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.820407+0000", 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025574237 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.165696+0000", 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34528857400000001 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4", 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.166092+0000", 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00039635499999999998 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.177352+0000", 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011259348000000001 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.177526+0000", 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017402800000000001 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.181067+0000", 2024-04-12T11:55:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035412899999999999 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.181134+0000", 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7430999999999999e-05 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.182851+0000", 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017160669999999999 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:147946 9.14 9:2b66249a:::benchmark_data_smithi029_222585_object39169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:50:31.803938+0000", 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 279.30825968099998, 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.40482041000000002, 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 147946 2024-04-12T11:55:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.803938+0000", 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.803938+0000", 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.803941+0000", 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2129999999999998e-06 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.803947+0000", 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9310000000000002e-06 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.803960+0000", 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.294e-05 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.803970+0000", 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3160000000000003e-06 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.804275+0000", 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00030482000000000003 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.804774+0000", 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00049990099999999997 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.175077+0000", 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37030259300000001 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.175182+0000", 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000104743 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4", 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.175531+0000", 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034966400000000002 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.183920+0000", 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0083881270000000004 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.183938+0000", 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8487000000000001e-05 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.206689+0000", 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022750947000000001 2024-04-12T11:55:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.206751+0000", 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1629000000000002e-05 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.208759+0000", 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020080990000000002 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35445.0:147953 9.4 9:27e7fc9e:::benchmark_data_smithi029_222585_object39176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:50:31.812474+0000", 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 279.29972394700002, 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39875056399999997, 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35445", 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2830382204", 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 147953 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.812474+0000", 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.812474+0000", 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.812476+0000", 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.601e-06 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.812481+0000", 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5629999999999999e-06 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.812493+0000", 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1378e-05 2024-04-12T11:55:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.812501+0000", 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.191e-06 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.813482+0000", 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00098071399999999993 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:31.813973+0000", 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00049130400000000002 2024-04-12T11:55:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.177390+0000", 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36341673899999999 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.177521+0000", 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013185000000000001 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4", 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.178129+0000", 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00060798700000000005 2024-04-12T11:55:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.182912+0000", 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047822100000000003 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.183945+0000", 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010337790000000001 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.208500+0000", 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024555079 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.208917+0000", 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041675000000000002 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:50:32.211225+0000", 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023074189999999998 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28369 10.2 10:4ae5b963:::benchmark_data_smithi029_256330_object28368:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.644209+0000", 2024-04-12T11:55:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 101.46798897399999, 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51128793500000003, 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28369 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644209+0000", 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644209+0000", 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644211+0000", 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7349999999999999e-06 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644234+0000", 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3073000000000001e-05 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644247+0000", 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3263e-05 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644255+0000", 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.729e-06 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644312+0000", 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7408999999999999e-05 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644494+0000", 2024-04-12T11:55:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018137200000000001 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644630+0000", 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000136556 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.646188+0000", 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001557952 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.647249+0000", 2024-04-12T11:55:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010610400000000001 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154196+0000", 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.506946542 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.155434+0000", 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001237953 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.155497+0000", 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3311000000000004e-05 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:33405 10.2 10:4829f386:::benchmark_data_smithi029_256330_object33404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:33.223479+0000", 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 97.888719453999997, 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38356202099999998, 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 33405 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223479+0000", 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223479+0000", 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223480+0000", 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.875e-06 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223507+0000", 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6846999999999999e-05 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223522+0000", 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4944e-05 2024-04-12T11:55:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223531+0000", 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1419999999999999e-06 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223593+0000", 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1894000000000001e-05 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223728+0000", 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013449299999999999 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223858+0000", 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000129813 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.225383+0000", 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001525857 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.225544+0000", 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000160722 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.606916+0000", 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38137180199999998 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.606985+0000", 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9036999999999996e-05 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.607041+0000", 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5594999999999999e-05 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35727.0:76041 10.0 e1282/1270 10:043b3560:::benchmark_data_smithi029_256330_object76040:head v 1282'4759, mlcod=1282'4757)", 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:14.739382+0000", 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 56.372815952000003, 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37150275999999999, 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739382+0000", 2024-04-12T11:55:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739382+0000", 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739383+0000", 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.003e-06 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739418+0000", 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5002000000000001e-05 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739429+0000", 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1209999999999999e-05 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739436+0000", 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0589999999999997e-06 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.099758+0000", 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36032216 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.099788+0000", 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9048e-05 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.110718+0000", 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010930949000000001 2024-04-12T11:55:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.110885+0000", 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000166329 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:114603 10.2 10:48272e51:::benchmark_data_smithi029_256330_object12837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1292)", 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:55:06.847593+0000", 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 4.2646046330000003, 2024-04-12T11:55:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.053728673999999997, 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 114603 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.847593+0000", 2024-04-12T11:55:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.847593+0000", 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.847595+0000", 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.314e-06 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.847599+0000", 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7310000000000001e-06 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.847614+0000", 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4936999999999999e-05 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.847622+0000", 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1919999999999999e-06 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.852443+0000", 2024-04-12T11:55:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048214640000000001 2024-04-12T11:55:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.852752+0000", 2024-04-12T11:55:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000308688 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.853086+0000", 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033411500000000001 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.857727+0000", 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046410729999999999 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.859081+0000", 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00135414 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.901166+0000", 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.042084840999999998 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.901270+0000", 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103751 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:06.901322+0000", 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2428e-05 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:115114 10.2 10:4dbc8722:::benchmark_data_smithi029_256330_object13348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1292)", 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:55:07.512769+0000", 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.5994292789999998, 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.060439846999999998, 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:55:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 115114 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.512769+0000", 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.512769+0000", 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.512770+0000", 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.751e-06 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.512777+0000", 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5910000000000001e-06 2024-04-12T11:55:11.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.512802+0000", 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4646000000000001e-05 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.512811+0000", 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6779999999999995e-06 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.522281+0000", 2024-04-12T11:55:11.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0094691559999999994 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.522850+0000", 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00056968199999999996 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.523731+0000", 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00088110200000000004 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.528426+0000", 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046948830000000004 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.531392+0000", 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029654450000000001 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.572805+0000", 2024-04-12T11:55:11.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.041413068999999997 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.573014+0000", 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020917200000000001 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:55:07.573209+0000", 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019467200000000001 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.239 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:55:11.240 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:55:11.240 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:55:11.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:11.268 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:11.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:11.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:11.416 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:11.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:11.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:11.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:11.546 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:11.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:11.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:55:11.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:11.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:11.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:11.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:11.764 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:11 smithi029 ceph-mon[97787]: pgmap v2181: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 856 op/s; 9493/262968 objects degraded (3.610%); 9709/262968 objects misplaced (3.692%); 13 MiB/s, 206 objects/s recovering 2024-04-12T11:55:11.764 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:11 smithi029 ceph-mon[93650]: pgmap v2181: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 856 op/s; 9493/262968 objects degraded (3.610%); 9709/262968 objects misplaced (3.692%); 13 MiB/s, 206 objects/s recovering 2024-04-12T11:55:11.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:11.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:11.832 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:11.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:11.933 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:11 smithi087 ceph-mon[89014]: pgmap v2181: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 856 op/s; 9493/262968 objects degraded (3.610%); 9709/262968 objects misplaced (3.692%); 13 MiB/s, 206 objects/s recovering 2024-04-12T11:55:11.975 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:11.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:11.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:12.077 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:12.126 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:12.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:12.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:12.228 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:12.251 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:12.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:12.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:12.353 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:12.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:12.379 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:12.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:12.479 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:12.508 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:12.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:12.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:12.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:12.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:12.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:12.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:12.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:12 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2350765134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:55:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:55:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:55:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:55:12.750 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:12.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:12 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2350765134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:55:12.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:12 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2350765134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:55:12.784 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:12.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:12.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:12.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:12.909 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:12.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:12.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:13.010 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:13.036 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:13.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:13.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:13.138 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:13.178 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:13.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:13.279 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:13.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:13.321 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:13.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:13.410 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:55:13.410 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:55:13.411 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:55:13.411 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:55:13.411 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:55:13.411 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:55:13.411 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:55:13.411 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:55:13.411 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:55:13.411 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:55:13.411 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:55:13.411 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:55:13.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:55:13.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:55:13.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:55:13.412 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:55:13.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:13.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:13.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:13.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:13.611 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:13.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:13.713 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:13.740 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:13.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:13.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:13 smithi029 ceph-mon[93650]: pgmap v2182: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 9493/262968 objects degraded (3.610%); 9709/262968 objects misplaced (3.692%); 11 MiB/s, 170 objects/s recovering 2024-04-12T11:55:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9493/262968 objects degraded (3.610%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:13 smithi029 ceph-mon[97787]: pgmap v2182: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 9493/262968 objects degraded (3.610%); 9709/262968 objects misplaced (3.692%); 11 MiB/s, 170 objects/s recovering 2024-04-12T11:55:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9493/262968 objects degraded (3.610%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:13.842 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:13.874 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:13.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:13.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:13 smithi087 ceph-mon[89014]: pgmap v2182: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 9493/262968 objects degraded (3.610%); 9709/262968 objects misplaced (3.692%); 11 MiB/s, 170 objects/s recovering 2024-04-12T11:55:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9493/262968 objects degraded (3.610%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:13.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:14.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:14.007 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:14.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:14.108 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:14.139 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:14.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:14.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:14.241 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:14.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:14.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:14.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:14.373 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:14.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:14.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:14.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:14.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:14.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:14.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:14.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:14.638 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:14.668 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:14.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:14.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:14.770 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:14.811 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:14.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:14.913 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:14.942 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:14.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:14.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:15.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:15.067 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:15.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:15.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:15.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:15.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:15.192 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:15.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:15.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:55:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:55:14] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.43.0" 2024-04-12T11:55:15.293 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:15.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:15.324 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:15.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:15.425 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:15.454 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:15.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:15.556 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:15.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:15.582 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:15.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:15.683 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:15.706 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:15.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:15.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:15 smithi029 ceph-mon[97787]: pgmap v2183: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 9319/253416 objects degraded (3.677%); 9709/253416 objects misplaced (3.831%); 9.3 MiB/s, 149 objects/s recovering 2024-04-12T11:55:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:15 smithi029 ceph-mon[93650]: pgmap v2183: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 9319/253416 objects degraded (3.677%); 9709/253416 objects misplaced (3.831%); 9.3 MiB/s, 149 objects/s recovering 2024-04-12T11:55:15.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:15.839 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:15.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:15.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:15.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:15 smithi087 ceph-mon[89014]: pgmap v2183: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 9319/253416 objects degraded (3.677%); 9709/253416 objects misplaced (3.831%); 9.3 MiB/s, 149 objects/s recovering 2024-04-12T11:55:15.979 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:15.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:15.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:16.081 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:16.110 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:16.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:16.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:16.211 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:16.239 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:16.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:16.341 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:16.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:16.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:16.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:16.465 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:16.496 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:16.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:16.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:16.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:16.627 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:16.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:16.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:16.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:16.754 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:16.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:16.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:16.856 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:16.891 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:16.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:16.992 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:17.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:17.035 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:17.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:17.136 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:17.182 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:17.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:17.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:17.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:17.327 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:17.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:17.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:17.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:17.464 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:17.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:17.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:17.566 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:17.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:17.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:17.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:17.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:17.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:17.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:17.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:17.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:17 smithi029 ceph-mon[93650]: pgmap v2184: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 9319/251352 objects degraded (3.708%); 8580/251352 objects misplaced (3.414%); 10 MiB/s, 165 objects/s recovering 2024-04-12T11:55:17.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:17 smithi029 ceph-mon[97787]: pgmap v2184: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 9319/251352 objects degraded (3.708%); 8580/251352 objects misplaced (3.414%); 10 MiB/s, 165 objects/s recovering 2024-04-12T11:55:17.841 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:17.873 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:17.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:17.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:17 smithi087 ceph-mon[89014]: pgmap v2184: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 9319/251352 objects degraded (3.708%); 8580/251352 objects misplaced (3.414%); 10 MiB/s, 165 objects/s recovering 2024-04-12T11:55:17.975 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:18.016 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:18.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:18.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:18.164 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:18.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:18.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:18.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:18.299 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:18.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:18.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:18.400 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:18.424 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:18.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:18.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:18.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:18.550 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:18.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:18.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:18.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:18.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:18.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:18.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:18.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9319/251352 objects degraded (3.708%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:18.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9319/251352 objects degraded (3.708%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:18.793 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:18.816 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:18.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:18.918 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:18.942 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:18.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:18.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9319/251352 objects degraded (3.708%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:19.044 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:19.075 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:19.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:19.176 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:19.205 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:19.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:19.307 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:19.330 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:19.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:19.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:19.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:19.455 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:19.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:19.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:19.557 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:19.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:19.587 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:19.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:19.688 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:19.727 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:19.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:19.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:19.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:19 smithi029 ceph-mon[93650]: pgmap v2185: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 9319/247548 objects degraded (3.765%); 8580/247548 objects misplaced (3.466%); 8.9 MiB/s, 141 objects/s recovering 2024-04-12T11:55:19.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:19 smithi029 ceph-mon[97787]: pgmap v2185: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 9319/247548 objects degraded (3.765%); 8580/247548 objects misplaced (3.466%); 8.9 MiB/s, 141 objects/s recovering 2024-04-12T11:55:19.828 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:19.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:19.858 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:19.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:19.959 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:19 smithi087 ceph-mon[89014]: pgmap v2185: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 5.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 9319/247548 objects degraded (3.765%); 8580/247548 objects misplaced (3.466%); 8.9 MiB/s, 141 objects/s recovering 2024-04-12T11:55:19.995 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:19.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:19.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:20.097 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:20.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:20.134 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:20.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:20.235 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:20.277 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:20.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:20.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:20.379 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:20.413 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:20.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:20.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:20.515 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:20.561 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:20.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:20.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:20.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:20.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:20.699 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:20.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:20.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:20.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:20.839 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:20.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:20.940 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:20.964 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:20.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:20.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:21.066 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:21.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:21.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:21.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:21.197 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:21.235 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:21.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:21.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:21.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:21.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:21.361 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:21.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:21.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:21.487 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:21.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:21.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:21.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:21.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:21.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:21.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:21.723 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:21.747 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:21.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:21.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:21.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:21 smithi029 ceph-mon[97787]: pgmap v2186: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 9159/241599 objects degraded (3.791%); 7677/241599 objects misplaced (3.178%); 13 MiB/s, 203 objects/s recovering 2024-04-12T11:55:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:21 smithi029 ceph-mon[93650]: pgmap v2186: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 9159/241599 objects degraded (3.791%); 7677/241599 objects misplaced (3.178%); 13 MiB/s, 203 objects/s recovering 2024-04-12T11:55:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:21.849 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:21.878 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:21.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:21.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:21 smithi087 ceph-mon[89014]: pgmap v2186: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 9159/241599 objects degraded (3.791%); 7677/241599 objects misplaced (3.178%); 13 MiB/s, 203 objects/s recovering 2024-04-12T11:55:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:55:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:21.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:22.020 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:22.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:22.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:22.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:22.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:22.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:22.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:22.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:22.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:22.285 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:22.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:22.386 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:22.418 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:22.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:22.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:22.554 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:22.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:22.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:22.655 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:22.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:22.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:22.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:55:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:55:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:55:22.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:22.839 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:22.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:22.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:22.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:22.986 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:22.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:22.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:23.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:23.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:23.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:23.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:23.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:23.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:23.255 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:23.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:23.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:23.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:23.394 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:23.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:23.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:23.536 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:23.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:23.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:23.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:23.661 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:23.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:23.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:23.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:23.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:23 smithi029 ceph-mon[93650]: pgmap v2187: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 9159/241599 objects degraded (3.791%); 7677/241599 objects misplaced (3.178%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:55:23.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9159/241599 objects degraded (3.791%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:23.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:23 smithi029 ceph-mon[97787]: pgmap v2187: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 9159/241599 objects degraded (3.791%); 7677/241599 objects misplaced (3.178%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:55:23.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9159/241599 objects degraded (3.791%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:23.786 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:23.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:23.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:23.888 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:23.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:23.925 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:23.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:23 smithi087 ceph-mon[89014]: pgmap v2187: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.9 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 9159/241599 objects degraded (3.791%); 7677/241599 objects misplaced (3.178%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:55:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9159/241599 objects degraded (3.791%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:24.026 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:24.050 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:24.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:24.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:24.152 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:24.187 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:24.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:24.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:24.325 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:24.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:24.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:24.426 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:24.461 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:24.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:24.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:24.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:24.604 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:24.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:24.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:24.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:24.748 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:24.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:24.849 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:24.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:24.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:24.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:25.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:55:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:55:24] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.43.0" 2024-04-12T11:55:25.019 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:25.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:25.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:25.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:25.151 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:25.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:25.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:25.253 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:25.283 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:25.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:25.383 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:25.410 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:25.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:25.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:25.511 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:25.534 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:25.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:25.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:25.636 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:25.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:25.670 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:25.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:25.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:25 smithi029 ceph-mon[93650]: pgmap v2188: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 9159/236058 objects degraded (3.880%); 7677/236058 objects misplaced (3.252%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:55:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:25 smithi029 ceph-mon[97787]: pgmap v2188: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 9159/236058 objects degraded (3.880%); 7677/236058 objects misplaced (3.252%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:55:25.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:25.816 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:25.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:25.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:25.918 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:25.952 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:25.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:25.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:25.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:25 smithi087 ceph-mon[89014]: pgmap v2188: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 747 op/s; 9159/236058 objects degraded (3.880%); 7677/236058 objects misplaced (3.252%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:55:26.054 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:26.090 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:26.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:26.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:26.192 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:26.236 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:26.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:26.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:26.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:26.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:26.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:26.487 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:26.525 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:26.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:26.627 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:26.657 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:26.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:26.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:26.759 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:26.783 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:26.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:26.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:26.885 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:26.915 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:26.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:26.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:27.016 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:27.053 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:27.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:27.155 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:27.198 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:27.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:27.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:27.300 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:27.326 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:27.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:27.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:27.427 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:27.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:27.451 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:27.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:27.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:27.596 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:27.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:27.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:27.698 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:27.729 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:27.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:27 smithi029 ceph-mon[97787]: pgmap v2189: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 8985/231666 objects degraded (3.878%); 6656/231666 objects misplaced (2.873%); 13 MiB/s, 201 objects/s recovering 2024-04-12T11:55:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:27 smithi029 ceph-mon[93650]: pgmap v2189: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 8985/231666 objects degraded (3.878%); 6656/231666 objects misplaced (2.873%); 13 MiB/s, 201 objects/s recovering 2024-04-12T11:55:27.830 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:27.854 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:27.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:27.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:27.955 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:27.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:27 smithi087 ceph-mon[89014]: pgmap v2189: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 8985/231666 objects degraded (3.878%); 6656/231666 objects misplaced (2.873%); 13 MiB/s, 201 objects/s recovering 2024-04-12T11:55:27.995 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:27.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:27.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:28.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:28.144 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:28.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:28.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:28.245 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:28.288 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:28.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:28.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:28.390 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:28.412 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:28.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:28.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:28.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:28.537 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:28.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:28.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:28.638 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:28.678 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:28.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:28.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 8985/231666 objects degraded (3.878%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:28.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 8985/231666 objects degraded (3.878%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:28.780 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:28.819 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:28.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:28.921 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:28.955 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:28.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 8985/231666 objects degraded (3.878%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:29.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:29.091 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:29.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:29.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:29.193 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:29.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:29.217 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:29.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:29.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:29.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:29.358 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:29.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:29.459 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:29.501 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:29.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:29.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:29.602 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:29.645 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:29.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:29.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:29.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:29 smithi029 ceph-mon[93650]: pgmap v2190: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 8985/227877 objects degraded (3.943%); 6656/227877 objects misplaced (2.921%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:55:29.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:29 smithi029 ceph-mon[97787]: pgmap v2190: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 8985/227877 objects degraded (3.943%); 6656/227877 objects misplaced (2.921%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:55:29.769 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:29.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:29.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:29.871 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:29.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:29.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:29.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:29 smithi087 ceph-mon[89014]: pgmap v2190: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 8985/227877 objects degraded (3.943%); 6656/227877 objects misplaced (2.921%); 8.8 MiB/s, 141 objects/s recovering 2024-04-12T11:55:29.999 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:30.026 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:30.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:30.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:30.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:30.172 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:30.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:30.273 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:30.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:30.309 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:30.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:30.409 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:30.433 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:30.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:30.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:30.534 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:30.558 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:30.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:30.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:30.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:30.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:30.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:30.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:30.822 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:30.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:30.923 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:30.948 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:30.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:31.049 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:31.082 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:31.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:31.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:31.186 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:31.219 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:31.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:31.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:31.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:31.356 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:31.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:31.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:31.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:31.502 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:31.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:31.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:31.604 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:31.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:31.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:31.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:31.735 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:31.759 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:31.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:31.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:31.860 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:31.892 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:31.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:31.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:31.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:31 smithi087 ceph-mon[89014]: pgmap v2191: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 8810/222246 objects degraded (3.964%); 5878/222246 objects misplaced (2.645%); 12 MiB/s, 199 objects/s recovering 2024-04-12T11:55:31.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:31 smithi029 ceph-mon[93650]: pgmap v2191: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 8810/222246 objects degraded (3.964%); 5878/222246 objects misplaced (2.645%); 12 MiB/s, 199 objects/s recovering 2024-04-12T11:55:32.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:31 smithi029 ceph-mon[97787]: pgmap v2191: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 8810/222246 objects degraded (3.964%); 5878/222246 objects misplaced (2.645%); 12 MiB/s, 199 objects/s recovering 2024-04-12T11:55:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:32.034 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:32.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:32.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:32.171 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:32.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:32.273 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:32.296 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:32.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:32.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:32.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:32.443 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:32.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:32.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:32.545 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:32.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:32.576 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:32.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:32.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:55:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:55:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:55:32.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:32.719 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:32.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:32.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:32.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:32.849 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:32.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:32.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:32.981 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:32.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:33.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:33.119 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:33.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:33.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:33.221 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:33.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:33.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:33.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:33.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:33.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:33.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:33.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:33.491 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:33.515 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:33.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:33.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:33.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:33.654 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:33.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:33.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:33.755 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:33.792 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:33.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:33.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:33.893 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:33.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:33.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:33.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:33 smithi029 ceph-mon[97787]: pgmap v2192: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 8810/222246 objects degraded (3.964%); 5878/222246 objects misplaced (2.645%); 8.4 MiB/s, 133 objects/s recovering 2024-04-12T11:55:33.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 8810/222246 objects degraded (3.964%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:33.954 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:33 smithi029 ceph-mon[93650]: pgmap v2192: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 8810/222246 objects degraded (3.964%); 5878/222246 objects misplaced (2.645%); 8.4 MiB/s, 133 objects/s recovering 2024-04-12T11:55:33.954 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 8810/222246 objects degraded (3.964%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:33 smithi087 ceph-mon[89014]: pgmap v2192: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 8810/222246 objects degraded (3.964%); 5878/222246 objects misplaced (2.645%); 8.4 MiB/s, 133 objects/s recovering 2024-04-12T11:55:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 8810/222246 objects degraded (3.964%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:34.035 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:34.066 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:34.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:34.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:34.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:34.212 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:34.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:34.313 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:34.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:34.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:34.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:34.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:34.483 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:34.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:34.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:34.585 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:34.612 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:34.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:34.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:34.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:34.763 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:34.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:34.864 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:34.907 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:34.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:34.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:35.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:35.049 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:35.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:35.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:35.151 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:35.194 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:35.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:35.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:55:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:55:34] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.43.0" 2024-04-12T11:55:35.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:35.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:35.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:35.431 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:35.461 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:35.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:35.562 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:35.589 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:35.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:35.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:35.691 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:35.722 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:35.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:35.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:35.823 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:35.858 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:35.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:35.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:35.960 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:35 smithi087 ceph-mon[89014]: pgmap v2193: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 8810/217014 objects degraded (4.060%); 5878/217014 objects misplaced (2.709%); 8.4 MiB/s, 133 objects/s recovering 2024-04-12T11:55:36.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:36.006 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:36.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:36.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:35 smithi029 ceph-mon[97787]: pgmap v2193: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 8810/217014 objects degraded (4.060%); 5878/217014 objects misplaced (2.709%); 8.4 MiB/s, 133 objects/s recovering 2024-04-12T11:55:36.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:35 smithi029 ceph-mon[93650]: pgmap v2193: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 682 op/s; 8810/217014 objects degraded (4.060%); 5878/217014 objects misplaced (2.709%); 8.4 MiB/s, 133 objects/s recovering 2024-04-12T11:55:36.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:36.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:36.148 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:36.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:36.248 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:36.278 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:36.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:36.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:36.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:36.426 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:36.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:36.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:36.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:36.573 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:36.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:36.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:36.674 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:36.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:36.703 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:36.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:36.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:36.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:36.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:36.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:36.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:36.985 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:36.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:36.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:37.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:37.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:37.087 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:37.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:37.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:37.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-04-12T11:55:37.232 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:37.264 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:37.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:37.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:37.366 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:37.396 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:37.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:37.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:37.409 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:55:37.498 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:37.527 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:37.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:37.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:37.630 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:37.660 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:37.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:37.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:37.735 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:55:37.738 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:37 smithi029 ceph-mon[93650]: pgmap v2194: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 8589/212667 objects degraded (4.039%); 4931/212667 objects misplaced (2.319%); 13 MiB/s, 207 objects/s recovering 2024-04-12T11:55:37.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:37 smithi029 ceph-mon[97787]: pgmap v2194: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 8589/212667 objects degraded (4.039%); 4931/212667 objects misplaced (2.319%); 13 MiB/s, 207 objects/s recovering 2024-04-12T11:55:37.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:37.814 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:37.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:37.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:37.916 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:37.965 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:37.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:37.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:37 smithi087 ceph-mon[89014]: pgmap v2194: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 8589/212667 objects degraded (4.039%); 4931/212667 objects misplaced (2.319%); 13 MiB/s, 207 objects/s recovering 2024-04-12T11:55:38.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:38.107 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:38.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:38.208 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:38.236 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:38.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:38.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:38.337 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:38.368 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:38.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:38.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:38.469 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:38.512 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:38.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:38.614 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:38.643 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:38.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:38.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:38.774 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:38.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:38.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:38.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:38.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:38.922 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:38.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:38.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 8589/212667 objects degraded (4.039%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:39.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 8589/212667 objects degraded (4.039%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:39.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 8589/212667 objects degraded (4.039%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:39.023 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:39.046 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:39.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:39.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:39.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:39.178 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:39.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:39.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:39.280 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:39.310 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:39.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:39.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:39.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:39.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:55:39.453 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:39.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:39.555 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:39.626 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:39.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:39.728 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:39.756 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:39.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:39.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:39 smithi029 ceph-mon[93650]: pgmap v2195: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 8589/208182 objects degraded (4.126%); 4931/208182 objects misplaced (2.369%); 8.3 MiB/s, 133 objects/s recovering 2024-04-12T11:55:39.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:39 smithi029 ceph-mon[97787]: pgmap v2195: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 8589/208182 objects degraded (4.126%); 4931/208182 objects misplaced (2.369%); 8.3 MiB/s, 133 objects/s recovering 2024-04-12T11:55:39.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:39.897 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:39.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:39.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:39 smithi087 ceph-mon[89014]: pgmap v2195: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 8589/208182 objects degraded (4.126%); 4931/208182 objects misplaced (2.369%); 8.3 MiB/s, 133 objects/s recovering 2024-04-12T11:55:39.983 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:55:39.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:40.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:40.073 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:40.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:40.174 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:40.234 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:40.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:40.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:40.376 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:40.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:40.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:40.477 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:40.504 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:40.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:40.606 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:40.655 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:40.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:40.756 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:40.798 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:40.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:40.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:40.900 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:40.927 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:40.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:41.030 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:41.055 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:41.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:41.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:41.157 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:41.181 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:41.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:41.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:41.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:41.308 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:41.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:41.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:41.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:41.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:41.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:41.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:41.541 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:41.568 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:41.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:41.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:41.670 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:41.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:41 smithi029 ceph-mon[97787]: pgmap v2196: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 8379/200871 objects degraded (4.171%); 4063/200871 objects misplaced (2.023%); 13 MiB/s, 203 objects/s recovering 2024-04-12T11:55:41.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:41 smithi029 ceph-mon[97787]: osdmap e1293: 8 total, 8 up, 7 in 2024-04-12T11:55:41.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:41 smithi029 ceph-mon[93650]: pgmap v2196: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 8379/200871 objects degraded (4.171%); 4063/200871 objects misplaced (2.023%); 13 MiB/s, 203 objects/s recovering 2024-04-12T11:55:41.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:41 smithi029 ceph-mon[93650]: osdmap e1293: 8 total, 8 up, 7 in 2024-04-12T11:55:41.699 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:41.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:41.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:41.802 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:41.834 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:41.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:41.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:41.936 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:41.944 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:55:41.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:41 smithi087 ceph-mon[89014]: pgmap v2196: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 8379/200871 objects degraded (4.171%); 4063/200871 objects misplaced (2.023%); 13 MiB/s, 203 objects/s recovering 2024-04-12T11:55:41.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:41 smithi087 ceph-mon[89014]: osdmap e1293: 8 total, 8 up, 7 in 2024-04-12T11:55:41.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:41.968 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:41.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:42.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:42.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:42.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:42.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:42.204 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:42.234 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:42.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:42.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:42.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:42.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:42.384 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:42.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:42.485 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:42.512 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:42.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:42.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:42.575 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:55:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:55:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:55:42.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:42.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:42.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:42.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:42.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:42.794 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:42.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:42.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:42.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:42.940 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:42.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:42.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:42 smithi087 ceph-mon[89014]: osdmap e1294: 8 total, 8 up, 7 in 2024-04-12T11:55:42.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:42 smithi087 ceph-mon[89014]: pgmap v2199: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 8379/200871 objects degraded (4.171%); 4063/200871 objects misplaced (2.023%); 14 MiB/s, 216 objects/s recovering 2024-04-12T11:55:43.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:42 smithi029 ceph-mon[97787]: osdmap e1294: 8 total, 8 up, 7 in 2024-04-12T11:55:43.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:42 smithi029 ceph-mon[97787]: pgmap v2199: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 8379/200871 objects degraded (4.171%); 4063/200871 objects misplaced (2.023%); 14 MiB/s, 216 objects/s recovering 2024-04-12T11:55:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:42 smithi029 ceph-mon[93650]: osdmap e1294: 8 total, 8 up, 7 in 2024-04-12T11:55:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:42 smithi029 ceph-mon[93650]: pgmap v2199: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 8379/200871 objects degraded (4.171%); 4063/200871 objects misplaced (2.023%); 14 MiB/s, 216 objects/s recovering 2024-04-12T11:55:43.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:42 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T11:55:42.745+0000 7fae24212700 -1 mon.a@0(leader).osd e1294 definitely_dead 0 2024-04-12T11:55:43.041 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:43.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:43.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:43.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:43.178 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:43.210 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:43.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:43.312 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:43.345 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:43.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:43.446 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:43.476 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:43.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:43.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:43.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:43.607 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.1. 2024-04-12T11:55:43.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:43.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:43.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:43.724 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:43.749 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:43.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:43.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:43.851 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:43.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:43.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:43.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:43.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:43 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4274452953' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2024-04-12T11:55:43.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:43 smithi029 ceph-mon[93650]: Client client.admin marked osd.1 out, while it was still marked up 2024-04-12T11:55:43.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 8379/200871 objects degraded (4.171%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:43.881 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:43 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4274452953' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2024-04-12T11:55:43.881 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:43 smithi029 ceph-mon[97787]: Client client.admin marked osd.1 out, while it was still marked up 2024-04-12T11:55:43.881 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 8379/200871 objects degraded (4.171%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:43.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:43 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4274452953' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2024-04-12T11:55:43.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:43 smithi087 ceph-mon[89014]: Client client.admin marked osd.1 out, while it was still marked up 2024-04-12T11:55:43.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 8379/200871 objects degraded (4.171%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:43.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:44.002 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:44.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:44.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:44.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:44.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:44.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:44.234 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:44.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:44.259 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:44.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:44.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:44.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:44.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:44.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:44.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:44.513 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:44.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:44.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:44.655 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:44.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:44.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:44.757 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:44.782 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:44.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:44.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:44.884 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:44.908 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:44.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:44.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:44.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:44 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4274452953' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-04-12T11:55:44.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:44 smithi029 ceph-mon[97787]: osdmap e1295: 8 total, 8 up, 6 in 2024-04-12T11:55:44.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:44 smithi029 ceph-mon[97787]: pgmap v2201: 99 pgs: 1 remapped+peering, 7 peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 5036/195021 objects degraded (2.582%); 4063/195021 objects misplaced (2.083%); 8.8 MiB/s, 140 objects/s recovering 2024-04-12T11:55:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:44 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4274452953' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-04-12T11:55:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:44 smithi029 ceph-mon[93650]: osdmap e1295: 8 total, 8 up, 6 in 2024-04-12T11:55:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:44 smithi029 ceph-mon[93650]: pgmap v2201: 99 pgs: 1 remapped+peering, 7 peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 5036/195021 objects degraded (2.582%); 4063/195021 objects misplaced (2.083%); 8.8 MiB/s, 140 objects/s recovering 2024-04-12T11:55:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:44 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4274452953' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-04-12T11:55:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:44 smithi087 ceph-mon[89014]: osdmap e1295: 8 total, 8 up, 6 in 2024-04-12T11:55:44.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:44 smithi087 ceph-mon[89014]: pgmap v2201: 99 pgs: 1 remapped+peering, 7 peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 5036/195021 objects degraded (2.582%); 4063/195021 objects misplaced (2.083%); 8.8 MiB/s, 140 objects/s recovering 2024-04-12T11:55:45.010 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:45.037 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:45.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:45.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:45.139 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:45.164 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:45.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:45.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:45.169 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:55:45.169 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:55:45.181 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:55:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:55:44] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.43.0" 2024-04-12T11:55:45.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:45.291 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:45.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:45.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:45.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:45.417 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:45.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:45.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:45.544 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:45.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:45.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:45.606 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:55:45.646 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:45.684 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:45.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:45.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:45.786 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:45.811 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:45.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:45.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:45.885 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:45 smithi029 ceph-mon[93650]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-12T11:55:45.885 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:45 smithi029 ceph-mon[93650]: osdmap e1296: 8 total, 8 up, 6 in 2024-04-12T11:55:45.885 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:45 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/731595526' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:55:45.885 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:45 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:55:45.885 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:45 smithi029 ceph-mon[97787]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-12T11:55:45.885 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:45 smithi029 ceph-mon[97787]: osdmap e1296: 8 total, 8 up, 6 in 2024-04-12T11:55:45.886 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:45 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/731595526' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:55:45.886 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:45 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:55:45.913 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:45.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:45.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:45.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:45 smithi087 ceph-mon[89014]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-12T11:55:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:45 smithi087 ceph-mon[89014]: osdmap e1296: 8 total, 8 up, 6 in 2024-04-12T11:55:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:45 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/731595526' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:55:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:45 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:55:46.040 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:46.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:46.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:46.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:46.206 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:46.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:46.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:46.307 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:46.349 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:46.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:46.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:46.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:46.491 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:46.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:46.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:46.592 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:46.632 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:46.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:46.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:46.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:46.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:46.772 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:46.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:46.873 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:46.905 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:46.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:46.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:46 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:55:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:46 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:55:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:46 smithi087 ceph-mon[89014]: 10.2 starting backfill to osd.3 from (0'0,0'0] MAX to 1294'8787 2024-04-12T11:55:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:46 smithi087 ceph-mon[89014]: osdmap e1297: 8 total, 8 up, 6 in 2024-04-12T11:55:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:46 smithi087 ceph-mon[89014]: 10.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1294'8742 2024-04-12T11:55:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:46 smithi087 ceph-mon[89014]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 1294'8911 2024-04-12T11:55:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:46 smithi087 ceph-mon[89014]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 1294'8808 2024-04-12T11:55:46.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:46 smithi087 ceph-mon[89014]: pgmap v2204: 99 pgs: 3 remapped+peering, 22 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 712 op/s; 4625/191313 objects degraded (2.418%); 2438/191313 objects misplaced (1.274%); 26 MiB/s, 414 objects/s recovering 2024-04-12T11:55:47.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:47.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:55:47.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:55:47.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[93650]: 10.2 starting backfill to osd.3 from (0'0,0'0] MAX to 1294'8787 2024-04-12T11:55:47.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[93650]: osdmap e1297: 8 total, 8 up, 6 in 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[93650]: 10.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1294'8742 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[93650]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 1294'8911 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[93650]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 1294'8808 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[93650]: pgmap v2204: 99 pgs: 3 remapped+peering, 22 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 712 op/s; 4625/191313 objects degraded (2.418%); 2438/191313 objects misplaced (1.274%); 26 MiB/s, 414 objects/s recovering 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[97787]: 10.2 starting backfill to osd.3 from (0'0,0'0] MAX to 1294'8787 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[97787]: osdmap e1297: 8 total, 8 up, 6 in 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[97787]: 10.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1294'8742 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[97787]: 10.b starting backfill to osd.4 from (0'0,0'0] MAX to 1294'8911 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[97787]: 10.0 starting backfill to osd.7 from (0'0,0'0] MAX to 1294'8808 2024-04-12T11:55:47.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:46 smithi029 ceph-mon[97787]: pgmap v2204: 99 pgs: 3 remapped+peering, 22 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 712 op/s; 4625/191313 objects degraded (2.418%); 2438/191313 objects misplaced (1.274%); 26 MiB/s, 414 objects/s recovering 2024-04-12T11:55:47.056 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:47.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:47.157 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:47.187 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:47.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:47.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:47.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:47.324 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:47.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:47.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:47.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:47.462 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:47.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:47.564 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:47.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:47.603 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:47.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:47.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:47.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:47.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:47.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:47 smithi029 ceph-mon[97787]: osdmap e1298: 8 total, 8 up, 6 in 2024-04-12T11:55:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:47 smithi029 ceph-mon[97787]: osdmap e1299: 8 total, 8 up, 6 in 2024-04-12T11:55:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:47 smithi029 ceph-mon[93650]: osdmap e1298: 8 total, 8 up, 6 in 2024-04-12T11:55:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:47 smithi029 ceph-mon[93650]: osdmap e1299: 8 total, 8 up, 6 in 2024-04-12T11:55:47.847 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:47.882 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:47.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:47.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:47 smithi087 ceph-mon[89014]: osdmap e1298: 8 total, 8 up, 6 in 2024-04-12T11:55:47.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:47 smithi087 ceph-mon[89014]: osdmap e1299: 8 total, 8 up, 6 in 2024-04-12T11:55:47.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:48.020 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:48.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:48.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-04-12T11:55:48.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:48.198 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:48.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:48.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:48.299 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:48.328 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:48.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:48.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:48.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:48.483 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:48.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:48.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:48.585 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:48.617 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:48.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:48.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:48.759 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:48.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:48.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:48.904 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:48.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:48.964 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 4625/191313 objects degraded (2.418%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:48.964 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:48 smithi029 ceph-mon[97787]: pgmap v2207: 99 pgs: 1 active+recovery_wait+remapped, 3 remapped+peering, 22 peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 507 op/s; 4625/189123 objects degraded (2.445%); 10302/189123 objects misplaced (5.447%); 30 MiB/s, 473 objects/s recovering 2024-04-12T11:55:48.965 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 4625/191313 objects degraded (2.418%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:48.965 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:48 smithi029 ceph-mon[93650]: pgmap v2207: 99 pgs: 1 active+recovery_wait+remapped, 3 remapped+peering, 22 peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 507 op/s; 4625/189123 objects degraded (2.445%); 10302/189123 objects misplaced (5.447%); 30 MiB/s, 473 objects/s recovering 2024-04-12T11:55:48.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 4625/191313 objects degraded (2.418%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:48.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:48 smithi087 ceph-mon[89014]: pgmap v2207: 99 pgs: 1 active+recovery_wait+remapped, 3 remapped+peering, 22 peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 507 op/s; 4625/189123 objects degraded (2.445%); 10302/189123 objects misplaced (5.447%); 30 MiB/s, 473 objects/s recovering 2024-04-12T11:55:49.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:49.048 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:49.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:49.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:49.150 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:49.179 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:49.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:49.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:49.221 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:55:49.256 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:55:49.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:49.304 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:55:49.308 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:55:49.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:49.333 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:49.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:49.353 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:55:49.394 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:55:49.434 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:49.435 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:55:49.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:49.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:49.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:49.478 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:55:49.515 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:55:49.523 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:55:49.524 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:55:49.524 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:55:49.524 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:55:49.524 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:55:49.524 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:55:49.524 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:55:49.524 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:55:49.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:49.605 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:49.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:49.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:49.707 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:49.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:49.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:49.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:49 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4254230603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:55:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:49 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4254230603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:55:49.846 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:49.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:49.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:49.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:49.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:49 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4254230603' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:55:49.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:50.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:50.034 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:50.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:50.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:50.183 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:50.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:50.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:50.285 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:50.320 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:50.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:50.421 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:50.457 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:50.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:50.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:50.559 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:50.601 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:50.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:50.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:50.703 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:50.731 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:50.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:50.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:50.833 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:50.875 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:50.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:50.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:50.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:55:50.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:50 smithi029 ceph-mon[97787]: pgmap v2208: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 572 op/s; 7361/183480 objects degraded (4.012%); 13481/183480 objects misplaced (7.347%); 36 MiB/s, 568 objects/s recovering 2024-04-12T11:55:50.954 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:50 smithi029 ceph-mon[93650]: pgmap v2208: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 572 op/s; 7361/183480 objects degraded (4.012%); 13481/183480 objects misplaced (7.347%); 36 MiB/s, 568 objects/s recovering 2024-04-12T11:55:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:50 smithi087 ceph-mon[89014]: pgmap v2208: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 572 op/s; 7361/183480 objects degraded (4.012%); 13481/183480 objects misplaced (7.347%); 36 MiB/s, 568 objects/s recovering 2024-04-12T11:55:50.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:51.006 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:51.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:51.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:51.107 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:51.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:51.140 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:51.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:51.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:51.283 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:51.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:51.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:51.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:51.432 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:51.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:51.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:51.534 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:51.565 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:51.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:51.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:51.666 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:51.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:51.701 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:51.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:51.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:51.840 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:51.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:51.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:51.942 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:51 smithi087 ceph-mon[89014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2024-04-12T11:55:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:51.974 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:51.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:51.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:51 smithi029 ceph-mon[93650]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2024-04-12T11:55:52.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:52.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:51 smithi029 ceph-mon[97787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2024-04-12T11:55:52.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:55:52.076 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:52.113 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:52.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:52.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:52.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:52.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:52.257 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:52.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:52.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:52.397 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:52.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:52.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:52.498 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:52.505 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 6, 7, 3] out_osds: [5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:55:52.505 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:55:52.505 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-04-12T11:55:52.505 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-04-12T11:55:52.505 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-04-12T11:55:52.505 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-04-12T11:55:52.532 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:52.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:52.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:52.565 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:55:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:55:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:55:52.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:52.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:52.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:52.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:52.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:52.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:52 smithi029 ceph-mon[93650]: pgmap v2209: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 399 op/s; 7361/183480 objects degraded (4.012%); 13481/183480 objects misplaced (7.347%); 16 MiB/s, 255 objects/s recovering 2024-04-12T11:55:52.811 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:52 smithi029 ceph-mon[97787]: pgmap v2209: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 399 op/s; 7361/183480 objects degraded (4.012%); 13481/183480 objects misplaced (7.347%); 16 MiB/s, 255 objects/s recovering 2024-04-12T11:55:52.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:52.832 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:52.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:52.837 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:55:52.883 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:55:52.932 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:55:52.932 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:55:52.932 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:55:52.932 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:55:52.932 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:55:52.933 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:52.966 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:52 smithi087 ceph-mon[89014]: pgmap v2209: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 399 op/s; 7361/183480 objects degraded (4.012%); 13481/183480 objects misplaced (7.347%); 16 MiB/s, 255 objects/s recovering 2024-04-12T11:55:52.996 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:52.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:52.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:53.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:53.138 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:53.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:53.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:53.240 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:53.280 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:53.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:53.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:53.382 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:53.417 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:53.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:53.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:53.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:53.521 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:55:53.549 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:53.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:53.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:53.651 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:53.682 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:53.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:53.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:53.784 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:53.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:53.815 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:53.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:53.916 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 7361/183480 objects degraded (4.012%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:53.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:53.975 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:53.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:54.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 7361/183480 objects degraded (4.012%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:54.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 7361/183480 objects degraded (4.012%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:54.076 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:54.111 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:54.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:54.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:54.213 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:54.246 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:54.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:54.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:54.348 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:54.382 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:54.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:54.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:54.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:54.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:54.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:54.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:54.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:54.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:54.667 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:54.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:54.768 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:54.773 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:55:54.773 INFO:teuthology.orchestra.run.smithi087.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-04-12T11:55:54.773 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:55:54.812 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:54.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:54.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:54.822 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:54 smithi087 ceph-mon[89014]: pgmap v2210: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 7099/177207 objects degraded (4.006%); 12980/177207 objects misplaced (7.325%); 15 MiB/s, 241 objects/s recovering 2024-04-12T11:55:54.914 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:54.953 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:54.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:55.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:54 smithi029 ceph-mon[97787]: pgmap v2210: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 7099/177207 objects degraded (4.006%); 12980/177207 objects misplaced (7.325%); 15 MiB/s, 241 objects/s recovering 2024-04-12T11:55:55.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:54 smithi029 ceph-mon[93650]: pgmap v2210: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 7099/177207 objects degraded (4.006%); 12980/177207 objects misplaced (7.325%); 15 MiB/s, 241 objects/s recovering 2024-04-12T11:55:55.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:55:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:55:54] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.43.0" 2024-04-12T11:55:55.056 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:55.090 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:55.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:55.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:55.191 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:55.223 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:55.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:55.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:55.259 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-04-12T11:55:55.305 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:55:55.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:55.393 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:55.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:55.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:55.494 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:55.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:55.534 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:55.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:55.636 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:55.689 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:55.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:55.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:55.791 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:55.819 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:55.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:55.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:55.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:55.953 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:55.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:56.055 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:56.098 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:56.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:56.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:56.200 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:56.230 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:56.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:56.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:56.331 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:56.361 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:56.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:56.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:56.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:56.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:56.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:56.601 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:56.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:56.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:56.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:56.734 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:56.768 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:56.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:56.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:56.869 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:56.899 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:56.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:56.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:57.003 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:57.032 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:57.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:57.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:57.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:57.177 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:57.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:57.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:57.208 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:55:57.279 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:57.307 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:57.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:57.408 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:57.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:57.445 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:57.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:57.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:57 smithi087 ceph-mon[89014]: pgmap v2211: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 716 op/s; 6790/170838 objects degraded (3.975%); 12110/170838 objects misplaced (7.089%); 18 MiB/s, 289 objects/s recovering 2024-04-12T11:55:57.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:57 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2877357765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:55:57.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:57 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:55:57.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:57 smithi029 ceph-mon[97787]: pgmap v2211: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 716 op/s; 6790/170838 objects degraded (3.975%); 12110/170838 objects misplaced (7.089%); 18 MiB/s, 289 objects/s recovering 2024-04-12T11:55:57.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:57 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2877357765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:55:57.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:57 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:55:57.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:57 smithi029 ceph-mon[93650]: pgmap v2211: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 716 op/s; 6790/170838 objects degraded (3.975%); 12110/170838 objects misplaced (7.089%); 18 MiB/s, 289 objects/s recovering 2024-04-12T11:55:57.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:57 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2877357765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:55:57.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:57 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:55:57.546 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:57.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:57.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:57.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:57.683 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:57.717 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:57.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:57.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:57.818 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:57.848 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:57.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:57.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:57.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:57.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:57.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:55:58.074 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:58.119 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:58.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:58.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:58.221 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:55:58.251 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:58.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:58.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:55:58.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:58.378 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:58.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:58.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:58 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:55:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:58 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:55:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:58 smithi087 ceph-mon[89014]: osdmap e1300: 8 total, 8 up, 6 in 2024-04-12T11:55:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 6790/170838 objects degraded (3.975%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:58.479 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:58.515 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:58.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:58.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:58 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:55:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:58 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:55:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:58 smithi029 ceph-mon[97787]: osdmap e1300: 8 total, 8 up, 6 in 2024-04-12T11:55:58.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 6790/170838 objects degraded (3.975%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:58.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:58 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:55:58.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:58 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:55:58.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:58 smithi029 ceph-mon[93650]: osdmap e1300: 8 total, 8 up, 6 in 2024-04-12T11:55:58.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 6790/170838 objects degraded (3.975%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:55:58.617 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:55:58.656 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:58.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:58.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:55:58.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:58.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:58.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:58.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:58.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:55:58.933 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:58.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:55:59.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:55:59.071 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:59.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:55:59.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:55:59.216 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:59.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:55:59.317 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:55:59.360 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:59.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:59.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:55:59.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:59.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:55:59 smithi087 ceph-mon[89014]: pgmap v2213: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 731 op/s; 6790/167163 objects degraded (4.062%); 11623/167163 objects misplaced (6.953%); 17 MiB/s, 275 objects/s recovering 2024-04-12T11:55:59.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:59.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:59.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:59.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:55:59 smithi029 ceph-mon[93650]: pgmap v2213: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 731 op/s; 6790/167163 objects degraded (4.062%); 11623/167163 objects misplaced (6.953%); 17 MiB/s, 275 objects/s recovering 2024-04-12T11:55:59.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:55:59 smithi029 ceph-mon[97787]: pgmap v2213: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 731 op/s; 6790/167163 objects degraded (4.062%); 11623/167163 objects misplaced (6.953%); 17 MiB/s, 275 objects/s recovering 2024-04-12T11:55:59.602 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:59.625 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:59.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:59.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:55:59.765 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:55:59.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:59.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:55:59.867 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:55:59.906 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:55:59.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:55:59.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:00.008 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:00.033 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:00.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:00.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:00.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:00.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:00.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:00.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:00.283 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:00.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:00.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:00.427 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:00.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:00.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:00.573 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:00.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:00.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:00.583 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 6, 7, 3] out_osds: [5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:56:00.583 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:56:00.583 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-04-12T11:56:00.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-04-12T11:56:00.675 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:00.704 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:00.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:00.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:00.805 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:00.854 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:00.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:00.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:00.989 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:00.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:00.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:01.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:01.113 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:01.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:01.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:01.215 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:01.245 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:01.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:01.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:01.346 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:01.370 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:01.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:01.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:01.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:01.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:01 smithi029 ceph-mon[97787]: pgmap v2214: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 813 op/s; 6210/159075 objects degraded (3.904%); 10789/159075 objects misplaced (6.782%); 11 MiB/s, 218 objects/s recovering 2024-04-12T11:56:01.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:01 smithi029 ceph-mon[93650]: pgmap v2214: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 813 op/s; 6210/159075 objects degraded (3.904%); 10789/159075 objects misplaced (6.782%); 11 MiB/s, 218 objects/s recovering 2024-04-12T11:56:01.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:01.533 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:01.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:01.634 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:01.668 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:01.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:01.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:01 smithi087 ceph-mon[89014]: pgmap v2214: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 813 op/s; 6210/159075 objects degraded (3.904%); 10789/159075 objects misplaced (6.782%); 11 MiB/s, 218 objects/s recovering 2024-04-12T11:56:01.769 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:01.794 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:01.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:01.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:01.896 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:01.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:01.919 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:01.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:02.020 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:02.045 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:02.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:02.147 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:02.170 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:02.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:02.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:02.272 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:02.295 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:02.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:02.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:02.425 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:02.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:02.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:02.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:02 smithi029 ceph-mon[93650]: osdmap e1301: 8 total, 8 up, 6 in 2024-04-12T11:56:02.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:02 smithi029 ceph-mon[97787]: osdmap e1301: 8 total, 8 up, 6 in 2024-04-12T11:56:02.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:02.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:02.570 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:02.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:02.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:02.696 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:02.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:02.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:02 smithi087 ceph-mon[89014]: osdmap e1301: 8 total, 8 up, 6 in 2024-04-12T11:56:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:56:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:56:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:56:02.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:02.821 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:02.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:02.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:02.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:02.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:02.947 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:02.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:03.048 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:03.074 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:03.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:03.175 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:03.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:03.200 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:03.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:03.301 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:03.326 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:03.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:03.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:03.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:03.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:03.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:03.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:03 smithi029 ceph-mon[93650]: pgmap v2216: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 755 op/s; 6210/159075 objects degraded (3.904%); 10789/159075 objects misplaced (6.782%); 12 MiB/s, 240 objects/s recovering 2024-04-12T11:56:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:03 smithi029 ceph-mon[93650]: osdmap e1302: 8 total, 8 up, 6 in 2024-04-12T11:56:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 6210/159075 objects degraded (3.904%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:56:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:03 smithi029 ceph-mon[97787]: pgmap v2216: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 755 op/s; 6210/159075 objects degraded (3.904%); 10789/159075 objects misplaced (6.782%); 12 MiB/s, 240 objects/s recovering 2024-04-12T11:56:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:03 smithi029 ceph-mon[97787]: osdmap e1302: 8 total, 8 up, 6 in 2024-04-12T11:56:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 6210/159075 objects degraded (3.904%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:56:03.558 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:03.588 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:03.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:03.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:03.690 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:03 smithi087 ceph-mon[89014]: pgmap v2216: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 755 op/s; 6210/159075 objects degraded (3.904%); 10789/159075 objects misplaced (6.782%); 12 MiB/s, 240 objects/s recovering 2024-04-12T11:56:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:03 smithi087 ceph-mon[89014]: osdmap e1302: 8 total, 8 up, 6 in 2024-04-12T11:56:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 6210/159075 objects degraded (3.904%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T11:56:03.722 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:03.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:03.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:03.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:03.848 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:03.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:03.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:03.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:03.979 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:03.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:03.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:04.081 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:04.105 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:04.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:04.206 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:04.236 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:04.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:04.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:04.337 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:04.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:04.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:04.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:04.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:04.489 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:04.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:04.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:04 smithi029 ceph-mon[93650]: osdmap e1303: 8 total, 8 up, 6 in 2024-04-12T11:56:04.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:04 smithi029 ceph-mon[97787]: osdmap e1303: 8 total, 8 up, 6 in 2024-04-12T11:56:04.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:04.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:04.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:04.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:04.716 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:04.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:04 smithi087 ceph-mon[89014]: osdmap e1303: 8 total, 8 up, 6 in 2024-04-12T11:56:04.762 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:04.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:04.864 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:04.892 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:04.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:04.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:04.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:05.034 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:05.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:05.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:05.136 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:05.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:05.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:05.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:05.253 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:56:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:56:04] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.43.0" 2024-04-12T11:56:05.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:05.308 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:05.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:05.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:05.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:05.447 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:05.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:05.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:05.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:05 smithi029 ceph-mon[93650]: pgmap v2219: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 6061/155028 objects degraded (3.910%); 10423/155028 objects misplaced (6.723%); 9.2 MiB/s, 207 objects/s recovering 2024-04-12T11:56:05.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:05 smithi029 ceph-mon[97787]: pgmap v2219: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 6061/155028 objects degraded (3.910%); 10423/155028 objects misplaced (6.723%); 9.2 MiB/s, 207 objects/s recovering 2024-04-12T11:56:05.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:05.590 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:05.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:05.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:05.692 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:05.715 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:05.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:05.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:05 smithi087 ceph-mon[89014]: pgmap v2219: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 3.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 6061/155028 objects degraded (3.910%); 10423/155028 objects misplaced (6.723%); 9.2 MiB/s, 207 objects/s recovering 2024-04-12T11:56:05.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:05.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:05.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:05.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:05.943 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:05.974 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:05.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:05.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:06.076 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:06.110 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:06.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:06.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:06.212 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:06.236 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:06.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:06.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:06.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:06.367 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:06.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:06.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:06.469 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:06.504 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:06.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:06.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:06.606 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:06.631 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:06.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:06.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:06.733 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:06.764 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:06.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:06.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:06.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:06.904 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:06.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:06.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:07.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:07.046 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:07.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:07.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:07.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:07.184 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:07.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:07.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:07.286 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:07.309 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:07.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:07.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:07.411 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:07.434 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:07.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:07.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:07.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:07 smithi029 ceph-mon[93650]: pgmap v2220: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 411 op/s; 5747/151668 objects degraded (3.789%); 9266/151668 objects misplaced (6.109%); 14 MiB/s, 246 objects/s recovering 2024-04-12T11:56:07.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:07.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:07 smithi029 ceph-mon[97787]: pgmap v2220: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 411 op/s; 5747/151668 objects degraded (3.789%); 9266/151668 objects misplaced (6.109%); 14 MiB/s, 246 objects/s recovering 2024-04-12T11:56:07.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:07.535 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:07.559 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:07.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:07.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:07.698 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:07.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:07.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:07 smithi087 ceph-mon[89014]: pgmap v2220: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 3.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 411 op/s; 5747/151668 objects degraded (3.789%); 9266/151668 objects misplaced (6.109%); 14 MiB/s, 246 objects/s recovering 2024-04-12T11:56:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:07.800 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:07.832 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:07.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:07.934 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:07.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:07.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:08.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:08.096 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:08.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:08.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:08.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:08.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:08.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:08.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:08.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:08.378 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:08.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:08.480 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:08.505 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:08.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:08.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5747/151668 objects degraded (3.789%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:08.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5747/151668 objects degraded (3.789%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:08.606 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:08.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:08.630 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:08.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5747/151668 objects degraded (3.789%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:08.732 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:08.759 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:08.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:08.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:08.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:08.901 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:08.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:09.003 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:09.026 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:09.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:09.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:09.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:09.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:09.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:09.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:09.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:09.302 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:09.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:09.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:09.443 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:09.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:09.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:09.544 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:09.568 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:09.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:09.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:09.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:09.695 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:09.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:09.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:09 smithi087 ceph-mon[89014]: pgmap v2221: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5747/148905 objects degraded (3.860%); 8915/148905 objects misplaced (5.987%); 12 MiB/s, 214 objects/s recovering 2024-04-12T11:56:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:09 smithi029 ceph-mon[93650]: pgmap v2221: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5747/148905 objects degraded (3.860%); 8915/148905 objects misplaced (5.987%); 12 MiB/s, 214 objects/s recovering 2024-04-12T11:56:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:09 smithi029 ceph-mon[97787]: pgmap v2221: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 3.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5747/148905 objects degraded (3.860%); 8915/148905 objects misplaced (5.987%); 12 MiB/s, 214 objects/s recovering 2024-04-12T11:56:09.797 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:09.823 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:09.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:09.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:09.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:09.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:09.974 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:09.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:10.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:10.101 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:10.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:10.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:10.203 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:10.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:10.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:10.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:10.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:10.373 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:10.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:10.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:10.511 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:10.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:10.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:10.613 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:10.641 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:10.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:10.644 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:10 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:56:10.743 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:10.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:10 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:56:10.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:10 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:56:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:10.788 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:10.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:10.889 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:10.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:10.936 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:10.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:11.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:11.070 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:11.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:11.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:11.172 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:11.212 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:11.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:11.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:11.314 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:11.354 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:11.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:11.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:11.456 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:11.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:11.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:11.580 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:11.613 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:11.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:11.713 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:11 smithi087 ceph-mon[89014]: pgmap v2222: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 5502/142701 objects degraded (3.856%); 8312/142701 objects misplaced (5.825%); 16 MiB/s, 276 objects/s recovering 2024-04-12T11:56:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:11.750 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:11.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:11.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:11 smithi029 ceph-mon[97787]: pgmap v2222: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 5502/142701 objects degraded (3.856%); 8312/142701 objects misplaced (5.825%); 16 MiB/s, 276 objects/s recovering 2024-04-12T11:56:11.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:11 smithi029 ceph-mon[93650]: pgmap v2222: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 5502/142701 objects degraded (3.856%); 8312/142701 objects misplaced (5.825%); 16 MiB/s, 276 objects/s recovering 2024-04-12T11:56:11.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:11.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:11.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:11.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:11.979 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:12.002 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:12.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:12.104 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:12.131 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:12.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:12.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:12.233 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:12.265 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:12.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:12.367 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:12.408 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:12.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:12.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:12.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:56:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:56:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:56:12.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:12.550 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:12.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:12.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:12.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:12.693 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:12.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:12.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:12.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:12.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:12.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:12.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:12.932 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:12.965 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:12.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:12.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:13.067 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:13.090 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:13.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:13.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:13.192 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:13.216 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:13.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:13.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:13.317 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:13.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:13.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:13.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:13.444 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:13.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:13.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:13.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:13.577 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:13.610 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:13.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:13.712 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:13 smithi087 ceph-mon[89014]: pgmap v2223: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 5502/142701 objects degraded (3.856%); 8312/142701 objects misplaced (5.825%); 13 MiB/s, 223 objects/s recovering 2024-04-12T11:56:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5502/142701 objects degraded (3.856%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:13.748 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:13.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:13.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:13.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:13 smithi029 ceph-mon[93650]: pgmap v2223: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 5502/142701 objects degraded (3.856%); 8312/142701 objects misplaced (5.825%); 13 MiB/s, 223 objects/s recovering 2024-04-12T11:56:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5502/142701 objects degraded (3.856%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:13 smithi029 ceph-mon[97787]: pgmap v2223: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 5502/142701 objects degraded (3.856%); 8312/142701 objects misplaced (5.825%); 13 MiB/s, 223 objects/s recovering 2024-04-12T11:56:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5502/142701 objects degraded (3.856%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:13.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:13.889 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:13.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:13.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:13.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:14.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:14.015 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:14.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:14.116 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:14.148 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:14.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:14.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:14.250 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:14.283 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:14.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:14.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:14.385 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:14.408 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:14.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:14.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:14.510 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:14.547 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:14.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:14.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:14.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:14.689 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:14.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:14.791 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:14.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:14.833 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:14.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:14.934 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:14.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:14.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:14.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:15.071 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:15.112 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:15.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:15.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:15.141 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:56:15.158 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:56:15.214 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:15.251 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:15.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:15.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:56:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:56:14] "GET /metrics HTTP/1.1" 200 36623 "" "Prometheus/2.43.0" 2024-04-12T11:56:15.353 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:15.376 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:15.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:15.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:15.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:15.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:15.516 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:15.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:15.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:15.654 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:15.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:15 smithi087 ceph-mon[89014]: pgmap v2224: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 5502/138318 objects degraded (3.978%); 7762/138318 objects misplaced (5.612%); 12 MiB/s, 202 objects/s recovering 2024-04-12T11:56:15.756 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:15.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:15 smithi029 ceph-mon[97787]: pgmap v2224: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 5502/138318 objects degraded (3.978%); 7762/138318 objects misplaced (5.612%); 12 MiB/s, 202 objects/s recovering 2024-04-12T11:56:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:15 smithi029 ceph-mon[93650]: pgmap v2224: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.8 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 550 op/s; 5502/138318 objects degraded (3.978%); 7762/138318 objects misplaced (5.612%); 12 MiB/s, 202 objects/s recovering 2024-04-12T11:56:15.793 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:15.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:15.895 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:15.931 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:15.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:15.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:16.033 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:16.057 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:16.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:16.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:16.182 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:16.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:16.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:16.283 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:16.314 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:16.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:16.416 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:16.458 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:16.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:16.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:16.560 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:16.583 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:16.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:16.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:16.685 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:16.708 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:16.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:16.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:16 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:56:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:16 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:56:16.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:16 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:56:16.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:16 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:56:16.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:16.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:56:16.858 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:16.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:16.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:16.960 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:16 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:56:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:16 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:56:16.986 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:16.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:17.087 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:17.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:17.131 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:17.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:17.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:17.271 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:17.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:17.373 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:17.408 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:17.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:17.510 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:17.548 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:17.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:17.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:17.650 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:17.679 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:17.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:17.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:17.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:17.817 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:17.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:17.919 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:17.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:17.947 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:17.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:17 smithi087 ceph-mon[89014]: pgmap v2225: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 604 op/s; 5371/133266 objects degraded (4.030%); 7040/133266 objects misplaced (5.283%); 13 MiB/s, 234 objects/s recovering 2024-04-12T11:56:18.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:17 smithi029 ceph-mon[93650]: pgmap v2225: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 604 op/s; 5371/133266 objects degraded (4.030%); 7040/133266 objects misplaced (5.283%); 13 MiB/s, 234 objects/s recovering 2024-04-12T11:56:18.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:17 smithi029 ceph-mon[97787]: pgmap v2225: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 604 op/s; 5371/133266 objects degraded (4.030%); 7040/133266 objects misplaced (5.283%); 13 MiB/s, 234 objects/s recovering 2024-04-12T11:56:18.048 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:18.071 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:18.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:18.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:18.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:18.206 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:18.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:18.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:18.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:18.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:18.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:18.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:18.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:18.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:18.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:18.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:18.572 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:18.610 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:18.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:18.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:18.698 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:56:18.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:18.745 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:56:18.772 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:18.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:18.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:18.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5371/133266 objects degraded (4.030%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:18.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5371/133266 objects degraded (4.030%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:18.813 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:56:18.860 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:56:18.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:18.902 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:18.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:18.915 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:56:18.960 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:56:18.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5371/133266 objects degraded (4.030%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:18.999 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:56:19.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:19.035 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:56:19.040 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:56:19.040 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:56:19.040 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:56:19.040 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:56:19.040 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:56:19.041 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:56:19.041 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:56:19.041 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:56:19.051 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:19.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:19.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:19.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:19.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:19.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:19.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:19.301 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:19.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:19.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:19.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:19.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:19.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:19.466 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:19.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:19.567 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:19.594 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:19.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:19.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:19.696 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:19.738 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:19.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:19.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:19.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:19.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:19.884 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:19.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:19 smithi087 ceph-mon[89014]: pgmap v2226: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 605 op/s; 5371/129846 objects degraded (4.136%); 6578/129846 objects misplaced (5.066%); 6.9 MiB/s, 123 objects/s recovering 2024-04-12T11:56:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:19 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1242034429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:56:19.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:20.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:20.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:20.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:20.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:19 smithi029 ceph-mon[93650]: pgmap v2226: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 605 op/s; 5371/129846 objects degraded (4.136%); 6578/129846 objects misplaced (5.066%); 6.9 MiB/s, 123 objects/s recovering 2024-04-12T11:56:20.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:19 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1242034429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:56:20.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:19 smithi029 ceph-mon[97787]: pgmap v2226: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 605 op/s; 5371/129846 objects degraded (4.136%); 6578/129846 objects misplaced (5.066%); 6.9 MiB/s, 123 objects/s recovering 2024-04-12T11:56:20.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:19 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1242034429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:56:20.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:20.162 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:20.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:20.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:20.264 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:20.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:20.311 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:20.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:20.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:20.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:20.460 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:20.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:20.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:20.605 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:20.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:20.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:20.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:20.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:20.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:20.888 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:20.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:20.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:20.990 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:21.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:21.028 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:21.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:21.129 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:21.159 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:21.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:21.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:21.309 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:21.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:21.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:21.458 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:21.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:21.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:21.560 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:21.593 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:21.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:21.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:21.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:21.734 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:21.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:21.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:21.836 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:21.866 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:21.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:21.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:21.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:21 smithi029 ceph-mon[97787]: pgmap v2227: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 734 op/s; 5258/122451 objects degraded (4.294%); 5622/122451 objects misplaced (4.591%); 11 MiB/s, 191 objects/s recovering 2024-04-12T11:56:21.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:21.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:21 smithi029 ceph-mon[93650]: pgmap v2227: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 734 op/s; 5258/122451 objects degraded (4.294%); 5622/122451 objects misplaced (4.591%); 11 MiB/s, 191 objects/s recovering 2024-04-12T11:56:21.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:21 smithi087 ceph-mon[89014]: pgmap v2227: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 734 op/s; 5258/122451 objects degraded (4.294%); 5622/122451 objects misplaced (4.591%); 11 MiB/s, 191 objects/s recovering 2024-04-12T11:56:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:21.968 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:21.991 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:21.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:21.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:22.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:22.118 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:22.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:22.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:22.219 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:22.244 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:22.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:22.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:22.346 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:22.371 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:22.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:22.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:22.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:22.498 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:22.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:22.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:22.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:22.621 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:56:22.635 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:22.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:22.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:22 smithi087 ceph-mon[89014]: osdmap e1304: 8 total, 8 up, 6 in 2024-04-12T11:56:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:56:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:56:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:56:22.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:22 smithi029 ceph-mon[93650]: osdmap e1304: 8 total, 8 up, 6 in 2024-04-12T11:56:22.735 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:22 smithi029 ceph-mon[97787]: osdmap e1304: 8 total, 8 up, 6 in 2024-04-12T11:56:22.737 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:22.765 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:22.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:22.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:22.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:22.946 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:22.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:22.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:22.965 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:56:22.965 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:56:22.977 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:56:23.057 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:23.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:23.093 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:23.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:23.194 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:23.224 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:23.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:23.326 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:23.358 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:23.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:no valid command found; 10 closest matches: 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:0 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:1 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:2 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:abort 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:assert 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:bench [] [] [] [] 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs debug_inject_read_zeros 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs files list 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs stats 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:bluestore allocator dump block 2024-04-12T11:56:23.372 INFO:teuthology.orchestra.run.smithi029.stderr:Error EINVAL: invalid command 2024-04-12T11:56:23.460 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:23.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:23.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:23.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:23.601 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:23.630 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:23.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:23.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:23.731 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:23.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:23.768 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:23.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:23.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:23 smithi029 ceph-mon[97787]: pgmap v2229: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 5258/122451 objects degraded (4.294%); 5622/122451 objects misplaced (4.591%); 8.3 MiB/s, 156 objects/s recovering 2024-04-12T11:56:23.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:23 smithi029 ceph-mon[97787]: osdmap e1305: 8 total, 8 up, 6 in 2024-04-12T11:56:23.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 5258/122451 objects degraded (4.294%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:23 smithi029 ceph-mon[93650]: pgmap v2229: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 5258/122451 objects degraded (4.294%); 5622/122451 objects misplaced (4.591%); 8.3 MiB/s, 156 objects/s recovering 2024-04-12T11:56:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:23 smithi029 ceph-mon[93650]: osdmap e1305: 8 total, 8 up, 6 in 2024-04-12T11:56:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 5258/122451 objects degraded (4.294%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:23.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:23.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:23.909 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:23.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:23 smithi087 ceph-mon[89014]: pgmap v2229: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 5258/122451 objects degraded (4.294%); 5622/122451 objects misplaced (4.591%); 8.3 MiB/s, 156 objects/s recovering 2024-04-12T11:56:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:23 smithi087 ceph-mon[89014]: osdmap e1305: 8 total, 8 up, 6 in 2024-04-12T11:56:23.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 5258/122451 objects degraded (4.294%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:24.010 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:24.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:24.059 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:24.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:24.160 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:24.197 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:24.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:24.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:24.299 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:24.328 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:24.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:24.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:24.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:24.478 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:24.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:24.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:24.580 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:24.621 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:24.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:24.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:24.724 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:24.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:24.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:24.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-04-12T11:56:24.857 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:24.884 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:24.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:24.962 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:56:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:56:24] "GET /metrics HTTP/1.1" 200 36623 "" "Prometheus/2.43.0" 2024-04-12T11:56:24.986 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:25.015 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:25.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:25.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:25.171 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:25.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:25.273 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:25.306 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:25.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:25.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:25.407 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:25.458 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:25.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:25.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:25.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:25.607 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:25.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:25.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:25.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:25.746 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:25.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:25.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:25.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:25.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:25.902 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:25.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:25.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:25 smithi087 ceph-mon[89014]: pgmap v2231: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 686 op/s; 5182/116793 objects degraded (4.437%); 4994/116793 objects misplaced (4.276%); 5.5 MiB/s, 101 objects/s recovering 2024-04-12T11:56:26.002 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:26.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:25 smithi029 ceph-mon[97787]: pgmap v2231: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 686 op/s; 5182/116793 objects degraded (4.437%); 4994/116793 objects misplaced (4.276%); 5.5 MiB/s, 101 objects/s recovering 2024-04-12T11:56:26.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:25 smithi029 ceph-mon[93650]: pgmap v2231: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 94 active+clean; 2.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 686 op/s; 5182/116793 objects degraded (4.437%); 4994/116793 objects misplaced (4.276%); 5.5 MiB/s, 101 objects/s recovering 2024-04-12T11:56:26.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:26.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:26.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:26.133 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:26.167 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:26.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:26.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:26.269 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:26.301 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:26.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:26.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:26.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:26.441 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:26.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:26.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:26.543 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:26.571 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:26.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:26.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-12T11:56:26.639 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-04-12T11:56:26.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-04-12T11:56:26.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:26.742 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:26.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:26.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:26.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:26.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:26.888 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:26.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:26.989 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:27.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:27.023 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:27.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:27.124 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:27.133 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:56:27.146 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:56:27.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:27.230 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:27.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:27.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:27.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:27.366 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:27.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:27.467 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:27.493 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:27.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:27.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:27.574 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:27 smithi029 ceph-mon[97787]: pgmap v2232: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 772 op/s; 4947/111300 objects degraded (4.445%); 4875/111300 objects misplaced (4.380%); 6.4 MiB/s, 146 objects/s recovering 2024-04-12T11:56:27.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:27 smithi029 ceph-mon[93650]: pgmap v2232: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 772 op/s; 4947/111300 objects degraded (4.445%); 4875/111300 objects misplaced (4.380%); 6.4 MiB/s, 146 objects/s recovering 2024-04-12T11:56:27.595 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:27.642 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:27.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:27.743 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:27.772 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:27.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:27.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:27.874 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:27.914 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:27.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:27.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:27 smithi087 ceph-mon[89014]: pgmap v2232: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 772 op/s; 4947/111300 objects degraded (4.445%); 4875/111300 objects misplaced (4.380%); 6.4 MiB/s, 146 objects/s recovering 2024-04-12T11:56:28.016 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:28.044 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:28.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:28.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:28.146 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:28.216 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:28.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:28.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:28.319 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:28.349 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:28.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:28.451 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:28.477 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:28.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:28.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:28.579 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:28.606 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:28.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:28.708 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:28.734 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:28.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:28.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:28.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 4947/111300 objects degraded (4.445%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 4947/111300 objects degraded (4.445%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:28.836 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:28.879 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:28.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:28.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:28.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 4947/111300 objects degraded (4.445%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:28.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:29.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:29.019 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:29.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:29.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:29.156 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:29.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:29.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:29.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:29.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:29.302 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:29.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:29.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:29.447 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:29.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:29.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:29.549 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:29.585 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:29.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:29.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:29.602 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:56:29.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:29.722 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:29.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:29.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:29.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:29 smithi029 ceph-mon[93650]: pgmap v2233: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 630 op/s; 4947/107310 objects degraded (4.610%); 4319/107310 objects misplaced (4.025%); 944 KiB/s, 44 objects/s recovering 2024-04-12T11:56:29.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:29 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2810104804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:56:29.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:29 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:56:29.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:29 smithi029 ceph-mon[97787]: pgmap v2233: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 630 op/s; 4947/107310 objects degraded (4.610%); 4319/107310 objects misplaced (4.025%); 944 KiB/s, 44 objects/s recovering 2024-04-12T11:56:29.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:29 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2810104804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:56:29.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:29 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:56:29.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:29.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:29.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:29.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:29.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:29 smithi087 ceph-mon[89014]: pgmap v2233: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 630 op/s; 4947/107310 objects degraded (4.610%); 4319/107310 objects misplaced (4.025%); 944 KiB/s, 44 objects/s recovering 2024-04-12T11:56:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:29 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2810104804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:56:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:29 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:56:29.994 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:29.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:30.096 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:30.125 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:30.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:30.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:30.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:30.258 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:30.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:30.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:30.288 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:56:30.359 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:30.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:30.415 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:30.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:30.516 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:30.546 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:30.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:30.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:30.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:30.689 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:30.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:30.732 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:30 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:56:30.732 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:30 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:56:30.732 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:30 smithi029 ceph-mon[93650]: osdmap e1306: 8 total, 8 up, 6 in 2024-04-12T11:56:30.732 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:30 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:56:30.732 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:30 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:56:30.732 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:30 smithi029 ceph-mon[97787]: osdmap e1306: 8 total, 8 up, 6 in 2024-04-12T11:56:30.791 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:30.820 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:30.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:30.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:30.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:30 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:56:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:30 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:56:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:30 smithi087 ceph-mon[89014]: osdmap e1306: 8 total, 8 up, 6 in 2024-04-12T11:56:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:30.968 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:30.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:31.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:31.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:31.115 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:31.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:31.216 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:31.249 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:31.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:31.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:31.350 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:31.384 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:31.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:31.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:31.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:31.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:31.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:31.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:31.629 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:31.670 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:31.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:31 smithi029 ceph-mon[93650]: pgmap v2235: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4545/97938 objects degraded (4.641%); 4319/97938 objects misplaced (4.410%); 944 KiB/s, 63 objects/s recovering 2024-04-12T11:56:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:31 smithi029 ceph-mon[97787]: pgmap v2235: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4545/97938 objects degraded (4.641%); 4319/97938 objects misplaced (4.410%); 944 KiB/s, 63 objects/s recovering 2024-04-12T11:56:31.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:31.816 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:31.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:31.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:31.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:31.966 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:31.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:31 smithi087 ceph-mon[89014]: pgmap v2235: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4545/97938 objects degraded (4.641%); 4319/97938 objects misplaced (4.410%); 944 KiB/s, 63 objects/s recovering 2024-04-12T11:56:32.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:32.104 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:32.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:32.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:32.205 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:32.241 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:32.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:32.343 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:32.377 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:32.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:32.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:32.479 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:32.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:32.522 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:32.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:32.623 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:32.637 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:56:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:56:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:56:32.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:32.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:32.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:32.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:32.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:32.811 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:32.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:32.912 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:32.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:32 smithi029 ceph-mon[97787]: pgmap v2236: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 4545/97938 objects degraded (4.641%); 4319/97938 objects misplaced (4.410%); 787 KiB/s, 52 objects/s recovering 2024-04-12T11:56:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:32 smithi029 ceph-mon[93650]: pgmap v2236: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 4545/97938 objects degraded (4.641%); 4319/97938 objects misplaced (4.410%); 787 KiB/s, 52 objects/s recovering 2024-04-12T11:56:32.940 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:32.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:32.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:32 smithi087 ceph-mon[89014]: pgmap v2236: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 4545/97938 objects degraded (4.641%); 4319/97938 objects misplaced (4.410%); 787 KiB/s, 52 objects/s recovering 2024-04-12T11:56:33.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:33.078 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:33.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:33.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:33.180 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:33.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:33.269 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:33.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:33.370 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:33.401 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:33.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:33.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:33.503 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:33.518 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:56:33.518 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:56:33.538 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:33.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:33.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:33.640 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:33.673 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:33.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:33.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:33.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 4545/97938 objects degraded (4.641%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 4545/97938 objects degraded (4.641%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:33.774 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:33.805 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:33.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:33.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:33.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:56:33.907 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:33.935 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:33.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:33.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 4545/97938 objects degraded (4.641%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:34.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:34.067 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:34.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:34.147 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:56:34.147 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:56:34.148 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:56:34.149 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:56:34.149 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:56:34.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:34.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:34.213 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:34.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:34.314 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:34.358 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:34.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:34.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:34.459 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:34.501 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:34.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:34.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:34.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:34.641 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:34.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:34.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:34.743 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:34.776 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:34.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:34.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:34.878 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:34.913 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:34 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2614493697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:56:34.913 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:34 smithi029 ceph-mon[93650]: pgmap v2237: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 4282/91194 objects degraded (4.695%); 3740/91194 objects misplaced (4.101%); 755 KiB/s, 50 objects/s recovering 2024-04-12T11:56:34.914 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:34 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2614493697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:56:34.914 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:34 smithi029 ceph-mon[97787]: pgmap v2237: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 4282/91194 objects degraded (4.695%); 3740/91194 objects misplaced (4.101%); 755 KiB/s, 50 objects/s recovering 2024-04-12T11:56:34.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:34.925 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:34.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:34 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2614493697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:56:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:34 smithi087 ceph-mon[89014]: pgmap v2237: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 4282/91194 objects degraded (4.695%); 3740/91194 objects misplaced (4.101%); 755 KiB/s, 50 objects/s recovering 2024-04-12T11:56:35.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:35.055 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:35.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:35.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:35.157 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:35.191 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:35.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:35.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:35.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:56:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:56:34] "GET /metrics HTTP/1.1" 200 36623 "" "Prometheus/2.43.0" 2024-04-12T11:56:35.292 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:35.322 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:35.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:35.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:35.424 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:35.453 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:35.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:35.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:35.555 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:35.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:35.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:35.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:35.688 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:35.719 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:35.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:35.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:35.821 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:35.880 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:35.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:35.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:35.982 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:36.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:36.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:36.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:36.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:36.165 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:36.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:36.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:36.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:36.301 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:36.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:36.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:36.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-04-12T11:56:36.403 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:36.432 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:36.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:36.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:36.534 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:36.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:36.583 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:36.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:36.684 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:36.728 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:36.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:36.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:36.857 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:36.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:36.959 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:36.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:36.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:36.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:37.018 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:56:37.089 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:37.130 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:37.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:37.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:37.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:37.271 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:37.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:37.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:37.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:37.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:37.497 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:37.528 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:37.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:56:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:37.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:37.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:37.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:37.696 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:37.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:37 smithi087 ceph-mon[89014]: pgmap v2238: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.7 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 900 op/s; 4126/84264 objects degraded (4.897%); 3740/84264 objects misplaced (4.438%); 0 B/s, 30 objects/s recovering 2024-04-12T11:56:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:56:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:37 smithi029 ceph-mon[93650]: pgmap v2238: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.7 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 900 op/s; 4126/84264 objects degraded (4.897%); 3740/84264 objects misplaced (4.438%); 0 B/s, 30 objects/s recovering 2024-04-12T11:56:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:56:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:37 smithi029 ceph-mon[97787]: pgmap v2238: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.7 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 900 op/s; 4126/84264 objects degraded (4.897%); 3740/84264 objects misplaced (4.438%); 0 B/s, 30 objects/s recovering 2024-04-12T11:56:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:56:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:37.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:37.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:37.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:37.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:37.942 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:37.983 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:37.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:37.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:38.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:38.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:38.127 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:38.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:38.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:38.251 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:38.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:38.353 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:38.388 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:38.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:38.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:38.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:38.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:38.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:38.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:38.629 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:38.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:38.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 4126/84264 objects degraded (4.897%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:38.755 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:38.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 4126/84264 objects degraded (4.897%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:38.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 4126/84264 objects degraded (4.897%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:38.790 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:38.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:38.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:38.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:38.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:38.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:38.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:39.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:39.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:39.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:39.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:39.196 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:39.219 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:39.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:39.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:39.321 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:39.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:39.345 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:39.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:39.446 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:39.481 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:39.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:39.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:39.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:39.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:39.620 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:39.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:39 smithi087 ceph-mon[89014]: pgmap v2239: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.6 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 903 op/s; 4126/80199 objects degraded (5.145%); 3174/80199 objects misplaced (3.958%); 0 B/s, 30 objects/s recovering 2024-04-12T11:56:39.720 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:39.764 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:39.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:39.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:39 smithi029 ceph-mon[97787]: pgmap v2239: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.6 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 903 op/s; 4126/80199 objects degraded (5.145%); 3174/80199 objects misplaced (3.958%); 0 B/s, 30 objects/s recovering 2024-04-12T11:56:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:39 smithi029 ceph-mon[93650]: pgmap v2239: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.6 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 903 op/s; 4126/80199 objects degraded (5.145%); 3174/80199 objects misplaced (3.958%); 0 B/s, 30 objects/s recovering 2024-04-12T11:56:39.866 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:39.890 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:39.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:39.991 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:40.015 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:40.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:40.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:40.117 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:40.148 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:40.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:40.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:40.249 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:40.283 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:40.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:40.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:40.384 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:40.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:40.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:40.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:40.521 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:40.544 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:40.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:40.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:40.645 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:40.669 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:40.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:40.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:40.771 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:40.804 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:40.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:40.905 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:40.937 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:40.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:41.039 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:41.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:41.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:41.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:41.174 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:41.222 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:41.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:41.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:41.324 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:41.349 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:41.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:41.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:41.452 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:41.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:41.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:41.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:41.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:41.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:41.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:41.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:41 smithi087 ceph-mon[89014]: pgmap v2240: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 937 op/s; 3696/71592 objects degraded (5.163%); 3174/71592 objects misplaced (4.433%); 0 B/s, 28 objects/s recovering 2024-04-12T11:56:41.725 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:41.757 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:41.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:41.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:41.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:41 smithi029 ceph-mon[97787]: pgmap v2240: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 937 op/s; 3696/71592 objects degraded (5.163%); 3174/71592 objects misplaced (4.433%); 0 B/s, 28 objects/s recovering 2024-04-12T11:56:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:41 smithi029 ceph-mon[93650]: pgmap v2240: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 937 op/s; 3696/71592 objects degraded (5.163%); 3174/71592 objects misplaced (4.433%); 0 B/s, 28 objects/s recovering 2024-04-12T11:56:41.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:41.888 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:41.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:41.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:41.990 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:42.032 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:42.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:42.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:42.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:42.158 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:42.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:42.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:42.260 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:42.283 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:42.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:42.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:42.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:42.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:42.427 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:42.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:42.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:42.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:42.569 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:42.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:42.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:42.693 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:42.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:42.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:56:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:56:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:56:42.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:42.837 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:42.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:42.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:42.938 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:42.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:42.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:42.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:43.071 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:43.094 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:43.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:43.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:43.196 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:43.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:43.241 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:43.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:43.342 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:43.374 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:43.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:43.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:43.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:43.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:43.519 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:43.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:43.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:43.650 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:43.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:43.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:43 smithi087 ceph-mon[89014]: pgmap v2241: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 731 op/s; 3696/71592 objects degraded (5.163%); 3174/71592 objects misplaced (4.433%); 0 B/s, 25 objects/s recovering 2024-04-12T11:56:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 3696/71592 objects degraded (5.163%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:43.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:43 smithi029 ceph-mon[93650]: pgmap v2241: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 731 op/s; 3696/71592 objects degraded (5.163%); 3174/71592 objects misplaced (4.433%); 0 B/s, 25 objects/s recovering 2024-04-12T11:56:43.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 3696/71592 objects degraded (5.163%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:43 smithi029 ceph-mon[97787]: pgmap v2241: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 731 op/s; 3696/71592 objects degraded (5.163%); 3174/71592 objects misplaced (4.433%); 0 B/s, 25 objects/s recovering 2024-04-12T11:56:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 3696/71592 objects degraded (5.163%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:43.775 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:43.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:43.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:43.877 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:43.906 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:43.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:44.008 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:44.039 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:44.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:44.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:44.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:44.165 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:44.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:44.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:44.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:44.291 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:44.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:44.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:44.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:44.424 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:44.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:44.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:44.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:44.554 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:44.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:44.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:44.679 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:44.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:44.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:44.781 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:44.815 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:44.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:44.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:44.917 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:44.949 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:44.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:44.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:45.051 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:45.075 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:45.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:45.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:45.178 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:45.213 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:45.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:45.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:45.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:56:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:56:44] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.43.0" 2024-04-12T11:56:45.314 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:45.356 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:45.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:45.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:45.485 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:45.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:45.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:45.610 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:45.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:45.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:45.711 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:45 smithi087 ceph-mon[89014]: pgmap v2242: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.3 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 890 op/s; 3446/65877 objects degraded (5.231%); 2680/65877 objects misplaced (4.068%); 0 B/s, 25 objects/s recovering 2024-04-12T11:56:45.740 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:45.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:45.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:45 smithi029 ceph-mon[97787]: pgmap v2242: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.3 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 890 op/s; 3446/65877 objects degraded (5.231%); 2680/65877 objects misplaced (4.068%); 0 B/s, 25 objects/s recovering 2024-04-12T11:56:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:45 smithi029 ceph-mon[93650]: pgmap v2242: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.3 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 890 op/s; 3446/65877 objects degraded (5.231%); 2680/65877 objects misplaced (4.068%); 0 B/s, 25 objects/s recovering 2024-04-12T11:56:45.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:45.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:45.886 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:45.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:45.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:46.013 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:46.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:46.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:46.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:46.141 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:46.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:46.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:46.242 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:46.267 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:46.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:46.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:46.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:46.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:46.411 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:46.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:46.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:46.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:46.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:46.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:46.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:46.679 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:46.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:46.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:46.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:46.826 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:46.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:46.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:46.928 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:46.951 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:46.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:46.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:47.053 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:47.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:47.095 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:47.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:47.196 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:47.228 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:47.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:47.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:47.330 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:47.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:47.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:47.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:47.472 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:47.506 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:47.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:47.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:47.608 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:47.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:47.639 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:47.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:47 smithi087 ceph-mon[89014]: pgmap v2243: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 860 op/s; 3324/60183 objects degraded (5.523%); 2680/60183 objects misplaced (4.453%); 0 B/s, 35 objects/s recovering 2024-04-12T11:56:47.740 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:47.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:47 smithi029 ceph-mon[93650]: pgmap v2243: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 860 op/s; 3324/60183 objects degraded (5.523%); 2680/60183 objects misplaced (4.453%); 0 B/s, 35 objects/s recovering 2024-04-12T11:56:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:47 smithi029 ceph-mon[97787]: pgmap v2243: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 860 op/s; 3324/60183 objects degraded (5.523%); 2680/60183 objects misplaced (4.453%); 0 B/s, 35 objects/s recovering 2024-04-12T11:56:47.776 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:47.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:47.878 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:47.901 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:47.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:47.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:48.003 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:48.045 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:48.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:48.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:48.205 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:48.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:48.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:48.306 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:48.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:48.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:48.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:48.432 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:48.465 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:48.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:48.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:48.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:48.613 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:48.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:48.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:48.715 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:48.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 3324/60183 objects degraded (5.523%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:48.751 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:48.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:48.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:48.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 3324/60183 objects degraded (5.523%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:48.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 3324/60183 objects degraded (5.523%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:48.852 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:48.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:48.880 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:48.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:48.981 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:49.015 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:49.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:49.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:49.117 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:49.142 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:49.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:49.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:49.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:49.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:49.289 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:49.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:49.390 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:49.421 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:49.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:49.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:49.563 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:49.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:49.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:49.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:49.703 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:49.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:49.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:49 smithi087 ceph-mon[89014]: pgmap v2244: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 751 op/s; 3324/57195 objects degraded (5.812%); 2282/57195 objects misplaced (3.990%); 0 B/s, 22 objects/s recovering 2024-04-12T11:56:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:49 smithi029 ceph-mon[93650]: pgmap v2244: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 751 op/s; 3324/57195 objects degraded (5.812%); 2282/57195 objects misplaced (3.990%); 0 B/s, 22 objects/s recovering 2024-04-12T11:56:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:49 smithi029 ceph-mon[97787]: pgmap v2244: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 751 op/s; 3324/57195 objects degraded (5.812%); 2282/57195 objects misplaced (3.990%); 0 B/s, 22 objects/s recovering 2024-04-12T11:56:49.805 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:49.834 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:49.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:49.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:49.936 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:49.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:49.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:49.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:50.074 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:50.117 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:50.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:50.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:50.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:50.257 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:50.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:50.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:50.359 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:50.388 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:50.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:50.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:50.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:50.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:50.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:50.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:50.638 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:50.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:50.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:50.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:50.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:50.821 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:50.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:50.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:50.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:50.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:50.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:50.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:51.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:51.099 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:51.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:51.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:51.244 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:51.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:51.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:51.347 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:51.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:51.375 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:51.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:51.477 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:51.518 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:51.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:51.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:51.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:51.648 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:51.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:51.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:51 smithi029 ceph-mon[97787]: pgmap v2245: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.0 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 2966/49761 objects degraded (5.960%); 2282/49761 objects misplaced (4.586%); 0 B/s, 33 objects/s recovering 2024-04-12T11:56:51.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:51.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:51 smithi029 ceph-mon[93650]: pgmap v2245: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.0 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 2966/49761 objects degraded (5.960%); 2282/49761 objects misplaced (4.586%); 0 B/s, 33 objects/s recovering 2024-04-12T11:56:51.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:51 smithi087 ceph-mon[89014]: pgmap v2245: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.0 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 2966/49761 objects degraded (5.960%); 2282/49761 objects misplaced (4.586%); 0 B/s, 33 objects/s recovering 2024-04-12T11:56:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:56:51.750 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:51.776 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:51.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:51.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:51.878 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:51.905 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:51.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:52.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:52.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:52.045 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:52.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:52.146 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:52.178 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:52.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:52.279 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:52.302 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:52.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:52.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:52.406 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:52.439 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:52.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:52.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:52.541 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:52.586 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:52.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:52.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:52.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:56:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:56:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:56:52.687 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:52.717 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:52.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:52.819 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:52.851 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:52.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:52.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:52.953 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:52.995 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:52.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:52.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:53.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:53.120 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:53.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:53.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:53.222 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:53.246 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:53.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:53.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:53.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:53.374 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:53.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:53.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:53.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:53.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:53.516 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:53.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:53.617 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:53.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:53.647 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:53.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:53 smithi087 ceph-mon[89014]: pgmap v2246: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.0 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 2966/49761 objects degraded (5.960%); 2282/49761 objects misplaced (4.586%); 0 B/s, 20 objects/s recovering 2024-04-12T11:56:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 2966/49761 objects degraded (5.960%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:53.748 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:53 smithi029 ceph-mon[93650]: pgmap v2246: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.0 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 2966/49761 objects degraded (5.960%); 2282/49761 objects misplaced (4.586%); 0 B/s, 20 objects/s recovering 2024-04-12T11:56:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 2966/49761 objects degraded (5.960%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:53 smithi029 ceph-mon[97787]: pgmap v2246: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 1.0 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 606 op/s; 2966/49761 objects degraded (5.960%); 2282/49761 objects misplaced (4.586%); 0 B/s, 20 objects/s recovering 2024-04-12T11:56:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 2966/49761 objects degraded (5.960%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:53.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:53.785 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:53.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:53.886 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:53.921 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:53.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:53.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:54.023 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:54.046 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:54.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:54.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:54.148 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:54.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:54.189 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:54.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:54.290 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:54.322 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:54.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:54.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:54.424 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:54.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:54.451 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:54.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:54.551 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:54.575 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:54.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:54.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:54.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:54.722 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:54.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:54.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:54.824 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:54.854 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:54.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:54.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:54.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:54.995 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:54.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:54.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:55.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:56:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:56:54] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.43.0" 2024-04-12T11:56:55.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:55.132 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:55.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:55.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:55.234 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:55.263 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:55.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:55.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:55.365 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:55.400 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:55.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:55.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:55.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:55.540 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:55.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:55.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:55.642 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:55.675 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:55.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:55.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:55 smithi087 ceph-mon[89014]: pgmap v2247: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 906 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 779 op/s; 2716/43533 objects degraded (6.239%); 1700/43533 objects misplaced (3.905%); 0 B/s, 20 objects/s recovering 2024-04-12T11:56:55.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:55 smithi029 ceph-mon[93650]: pgmap v2247: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 906 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 779 op/s; 2716/43533 objects degraded (6.239%); 1700/43533 objects misplaced (3.905%); 0 B/s, 20 objects/s recovering 2024-04-12T11:56:55.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:55 smithi029 ceph-mon[97787]: pgmap v2247: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 906 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 779 op/s; 2716/43533 objects degraded (6.239%); 1700/43533 objects misplaced (3.905%); 0 B/s, 20 objects/s recovering 2024-04-12T11:56:55.778 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:55.821 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:55.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:55.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:55.953 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:55.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:56.055 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:56.083 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:56.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:56.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:56.117 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:56:56.118 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:56:56.184 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:56.214 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:56.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:56.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:56.267 INFO:teuthology.orchestra.run.smithi029.stderr:no valid command found; 10 closest matches: 2024-04-12T11:56:56.267 INFO:teuthology.orchestra.run.smithi029.stderr:0 2024-04-12T11:56:56.267 INFO:teuthology.orchestra.run.smithi029.stderr:1 2024-04-12T11:56:56.268 INFO:teuthology.orchestra.run.smithi029.stderr:2 2024-04-12T11:56:56.268 INFO:teuthology.orchestra.run.smithi029.stderr:abort 2024-04-12T11:56:56.268 INFO:teuthology.orchestra.run.smithi029.stderr:assert 2024-04-12T11:56:56.268 INFO:teuthology.orchestra.run.smithi029.stderr:bench [] [] [] [] 2024-04-12T11:56:56.268 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs debug_inject_read_zeros 2024-04-12T11:56:56.268 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs files list 2024-04-12T11:56:56.268 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs stats 2024-04-12T11:56:56.268 INFO:teuthology.orchestra.run.smithi029.stderr:bluestore allocator dump block 2024-04-12T11:56:56.268 INFO:teuthology.orchestra.run.smithi029.stderr:Error EINVAL: invalid command 2024-04-12T11:56:56.317 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:56.351 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:56.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:56.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:56.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:56.505 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:56.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:56.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:56.607 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:56.643 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:56.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:56.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:56.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:56.778 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:56.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:56.879 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:56.907 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:56.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:56.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:57.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:57.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:57.055 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:57.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:57.157 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:57.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:57.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:57.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:57.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:57.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:57.346 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:57.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:57.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:57.494 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:57.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:57.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:57.565 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:57 smithi029 ceph-mon[97787]: pgmap v2248: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 780 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 2568/37485 objects degraded (6.851%); 1700/37485 objects misplaced (4.535%); 0 B/s, 32 objects/s recovering 2024-04-12T11:56:57.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:57 smithi029 ceph-mon[93650]: pgmap v2248: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 780 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 2568/37485 objects degraded (6.851%); 1700/37485 objects misplaced (4.535%); 0 B/s, 32 objects/s recovering 2024-04-12T11:56:57.596 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:57.638 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:57.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:57.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:57 smithi087 ceph-mon[89014]: pgmap v2248: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 780 MiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 2568/37485 objects degraded (6.851%); 1700/37485 objects misplaced (4.535%); 0 B/s, 32 objects/s recovering 2024-04-12T11:56:57.740 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:57.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:57.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:57.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:57.921 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:57.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:58.023 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:58.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:58.124 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:58.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:58.225 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:56:58.256 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:58.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:58.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:56:58.358 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:56:58.389 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:58.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:58.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:56:58.453 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:56:58.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:58.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:58.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:58.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:58.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:56:58.678 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:58.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:58.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:56:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:58 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1260681518' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:56:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:58 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:56:58.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 2568/37485 objects degraded (6.851%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:58.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:58 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1260681518' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:56:58.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:58 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:56:58.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 2568/37485 objects degraded (6.851%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:58.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:58 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1260681518' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:56:58.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:58 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:56:58.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 2568/37485 objects degraded (6.851%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:56:58.780 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:58.807 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:58.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:58.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:56:58.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:58.946 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:58.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:58.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:59.049 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:56:59.084 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:59.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:59.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:56:59.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:59.227 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:59.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:59.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:59.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:56:59.365 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:59.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:59.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:56:59.467 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:59.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:59.507 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:59.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:59.608 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:56:59.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:59.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:59.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:56:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:59 smithi087 ceph-mon[89014]: pgmap v2249: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 701 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 2568/33708 objects degraded (7.618%); 1220/33708 objects misplaced (3.619%); 0 B/s, 22 objects/s recovering 2024-04-12T11:56:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:59 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:56:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:59 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:56:59.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:56:59 smithi087 ceph-mon[89014]: osdmap e1307: 8 total, 8 up, 6 in 2024-04-12T11:56:59.750 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:56:59.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:59 smithi029 ceph-mon[93650]: pgmap v2249: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 701 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 2568/33708 objects degraded (7.618%); 1220/33708 objects misplaced (3.619%); 0 B/s, 22 objects/s recovering 2024-04-12T11:56:59.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:59 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:56:59.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:59 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:56:59.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:56:59 smithi029 ceph-mon[93650]: osdmap e1307: 8 total, 8 up, 6 in 2024-04-12T11:56:59.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:59 smithi029 ceph-mon[97787]: pgmap v2249: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 701 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 2568/33708 objects degraded (7.618%); 1220/33708 objects misplaced (3.619%); 0 B/s, 22 objects/s recovering 2024-04-12T11:56:59.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:59 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:56:59.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:59 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:56:59.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:56:59 smithi029 ceph-mon[97787]: osdmap e1307: 8 total, 8 up, 6 in 2024-04-12T11:56:59.791 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:56:59.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:59.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:56:59.876 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-12T11:56:59.877 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-04-12T11:56:59.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-04-12T11:56:59.893 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:56:59.921 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:56:59.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:56:59.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:00.023 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:00.046 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:00.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:00.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:00.148 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:00.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:00.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:00.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:00.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:00.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:00.310 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:00.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:00.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:00.452 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:00.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:00.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:00.554 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:00.588 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:00.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:00.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:00.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:00.713 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:00.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:00.815 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:00.846 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:00.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:00.948 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:00.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:00.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:00.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:01.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:01.124 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:01.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:01.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:01.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:01.270 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:01.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:01.373 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:01.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:57:01.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:01.403 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:01.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:01.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:01.532 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:01.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:01.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:01.634 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:01.696 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:01.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:01.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:01 smithi087 ceph-mon[89014]: pgmap v2251: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 503 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 2092/24174 objects degraded (8.654%); 1220/24174 objects misplaced (5.047%); 0 B/s, 32 objects/s recovering 2024-04-12T11:57:01.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:01 smithi029 ceph-mon[93650]: pgmap v2251: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 503 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 2092/24174 objects degraded (8.654%); 1220/24174 objects misplaced (5.047%); 0 B/s, 32 objects/s recovering 2024-04-12T11:57:01.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:01 smithi029 ceph-mon[97787]: pgmap v2251: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 503 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 2092/24174 objects degraded (8.654%); 1220/24174 objects misplaced (5.047%); 0 B/s, 32 objects/s recovering 2024-04-12T11:57:01.798 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:01.855 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:01.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:01.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:01.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:02.002 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:02.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:02.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:02.104 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:02.133 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:02.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:02.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:02.235 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:02.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:02.279 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:02.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:02.380 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:02.414 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:02.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:02.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:02.516 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:02.545 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:02.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:02.646 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:02.694 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:02.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:02.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:57:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:57:02.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:57:02.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:02.831 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:02.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:02.933 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:02.961 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:02.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:02.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:03.063 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:03.092 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:03.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:03.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:03.194 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:03.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:03.242 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:03.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:03.343 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:03.376 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:03.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:03.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:03.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:03.510 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:03.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:03.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:03.612 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:03.643 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:03.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:03.649 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:03 smithi029 ceph-mon[97787]: pgmap v2252: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 503 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 2092/24174 objects degraded (8.654%); 1220/24174 objects misplaced (5.047%); 0 B/s, 32 objects/s recovering 2024-04-12T11:57:03.649 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:03.649 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:03 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:03.649 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 2092/24174 objects degraded (8.654%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:57:03.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:03 smithi029 ceph-mon[93650]: pgmap v2252: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 503 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 2092/24174 objects degraded (8.654%); 1220/24174 objects misplaced (5.047%); 0 B/s, 32 objects/s recovering 2024-04-12T11:57:03.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:03.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:03 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:03.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 2092/24174 objects degraded (8.654%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:57:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:03 smithi087 ceph-mon[89014]: pgmap v2252: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 503 MiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 2092/24174 objects degraded (8.654%); 1220/24174 objects misplaced (5.047%); 0 B/s, 32 objects/s recovering 2024-04-12T11:57:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:03 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 2092/24174 objects degraded (8.654%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:57:03.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:03.798 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:03.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:03.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:03.900 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:03.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:03.925 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:03.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:04.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:04.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:04.056 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:04.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:04.158 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:04.192 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:04.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:04.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:04.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:04.344 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:04.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:04.446 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:04.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:04.509 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:04.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:04.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:04.659 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:04.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:04.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:04.760 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:04.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:04.789 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:04.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:04.866 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:57:04.889 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:04.911 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:57:04.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:04.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:04.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:04.953 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:57:04.953 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:57:04.953 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:57:04.953 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:57:04.953 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:57:04.954 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:57:04.997 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:57:05.034 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:05.039 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:57:05.064 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:05.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:05.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:05.076 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:57:05.116 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:57:05.160 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:57:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:57:04] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.43.0" 2024-04-12T11:57:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:57:05.164 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:57:05.164 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:57:05.164 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:57:05.164 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:57:05.164 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:57:05.164 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:57:05.164 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:57:05.164 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:57:05.166 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:05.175 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:05.182 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:05.193 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:05.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:05.294 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:05.323 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:05.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:05.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:05.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:05.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:05.464 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:05.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:05.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:05 smithi029 ceph-mon[97787]: pgmap v2253: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 413 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 2092/19851 objects degraded (10.539%); 640/19851 objects misplaced (3.224%); 0 B/s, 32 objects/s recovering 2024-04-12T11:57:05.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:57:05.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:57:05.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:05.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:05 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1086574270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:57:05.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:05 smithi029 ceph-mon[93650]: pgmap v2253: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 413 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 2092/19851 objects degraded (10.539%); 640/19851 objects misplaced (3.224%); 0 B/s, 32 objects/s recovering 2024-04-12T11:57:05.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:57:05.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:57:05.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:05.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:05 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1086574270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:57:05.566 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:05.605 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:05.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:05.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:05.707 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:05 smithi087 ceph-mon[89014]: pgmap v2253: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 413 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 789 op/s; 2092/19851 objects degraded (10.539%); 640/19851 objects misplaced (3.224%); 0 B/s, 32 objects/s recovering 2024-04-12T11:57:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:05 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:57:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:05 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:57:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:05 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:05 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1086574270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:57:05.737 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:05.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:05.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:05.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-04-12T11:57:05.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:05.929 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:05.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:06.030 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:06.059 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:06.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:06.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:06.161 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:06.185 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:06.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:06.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:06.287 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:06.325 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:06.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:06.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:06.427 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:06.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:06.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:06.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:06.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:06.584 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:06.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:06.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:06.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:06.725 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:06.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:06.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:06.827 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:06.862 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:06.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:06.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:07.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:07.009 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:07.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:07.110 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:07.143 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:07.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:07.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:07.267 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:07.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:07.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:07.369 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:07.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:07.403 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:07.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:07.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:07.527 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:07.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:07.629 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:07.652 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:07.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:07.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:07 smithi087 ceph-mon[89014]: pgmap v2254: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 224 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 889 op/s; 1626/10794 objects degraded (15.064%); 640/10794 objects misplaced (5.929%); 0 B/s, 34 objects/s recovering 2024-04-12T11:57:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:07.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:07 smithi029 ceph-mon[93650]: pgmap v2254: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 224 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 889 op/s; 1626/10794 objects degraded (15.064%); 640/10794 objects misplaced (5.929%); 0 B/s, 34 objects/s recovering 2024-04-12T11:57:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:07 smithi029 ceph-mon[97787]: pgmap v2254: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 224 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 889 op/s; 1626/10794 objects degraded (15.064%); 640/10794 objects misplaced (5.929%); 0 B/s, 34 objects/s recovering 2024-04-12T11:57:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:07.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:07.793 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:07.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:07.895 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:07.932 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:07.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:07.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:08.034 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:08.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:08.058 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:08.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:08.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:08.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:08.196 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:08.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:08.297 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:08.305 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 101763 objects 2024-04-12T11:57:08.305 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :140.535 2024-04-12T11:57:08.321 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:08.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:08.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:08.423 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:08.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:08.447 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:08.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:08.548 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:08.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:08.574 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:08.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:08.675 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:08.698 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:08.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:08.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 1626/10794 objects degraded (15.064%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:57:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 1626/10794 objects degraded (15.064%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:57:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 1626/10794 objects degraded (15.064%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:57:08.801 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:08.824 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:08.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:08.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:08.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:08.950 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:08.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:09.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:09.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:09.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:09.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:09.178 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:09.203 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:09.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:09.304 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:09.329 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:09.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:09.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:09.431 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:09.458 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:09.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:09.531 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-04-12T11:57:09.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-04-12T11:57:09.559 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:09.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:09.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:09.688 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:09.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:09.715 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:09.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:09 smithi087 ceph-mon[89014]: pgmap v2255: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 141 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 895 op/s; 1626/6819 objects degraded (23.845%); 79/6819 objects misplaced (1.159%); 0 B/s, 34 objects/s recovering 2024-04-12T11:57:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:09 smithi029 ceph-mon[93650]: pgmap v2255: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 141 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 895 op/s; 1626/6819 objects degraded (23.845%); 79/6819 objects misplaced (1.159%); 0 B/s, 34 objects/s recovering 2024-04-12T11:57:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:09 smithi029 ceph-mon[97787]: pgmap v2255: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 95 active+clean; 141 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 895 op/s; 1626/6819 objects degraded (23.845%); 79/6819 objects misplaced (1.159%); 0 B/s, 34 objects/s recovering 2024-04-12T11:57:09.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:09.843 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:09.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:09.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:09.945 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:09.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:09.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:10.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:10.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:10.098 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:10.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:10.199 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:10.224 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:10.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:10.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:10.326 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:10.354 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:10.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:10.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:10.456 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:10.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:10.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:10.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:10.585 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:10.612 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:10.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:10.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:10.714 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:10.743 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:10.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:10.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:10.869 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:10.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:10.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:10.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:10.996 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:10.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:11.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:11.122 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:11.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:11.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:11.224 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:11.248 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:11.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:11.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:11.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:11.374 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:11.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:11.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:11.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:11.500 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:11.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:11.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:11.602 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:11.626 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:11.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:11 smithi087 ceph-mon[89014]: pgmap v2256: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 11 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 942 op/s; 928/564 objects degraded (164.539%); 79/564 objects misplaced (14.007%); 0 B/s, 70 objects/s recovering 2024-04-12T11:57:11.728 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:11.751 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:11.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:11.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:11.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:11 smithi029 ceph-mon[93650]: pgmap v2256: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 11 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 942 op/s; 928/564 objects degraded (164.539%); 79/564 objects misplaced (14.007%); 0 B/s, 70 objects/s recovering 2024-04-12T11:57:11.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:11 smithi029 ceph-mon[97787]: pgmap v2256: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 11 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 942 op/s; 928/564 objects degraded (164.539%); 79/564 objects misplaced (14.007%); 0 B/s, 70 objects/s recovering 2024-04-12T11:57:11.853 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:11.881 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:11.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:11.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:11.983 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:12.012 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:12.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:12.113 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:12.141 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:12.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:12.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:12.243 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:12.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:12.272 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:12.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:12.373 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:12.400 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:12.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:12.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:12.502 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:12.530 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:12.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:12.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:12.632 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:12.659 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:12.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:12.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:57:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:57:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:57:12.762 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:12.788 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:12.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:12.815 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:12.890 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:12.918 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:12.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:12.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:13.020 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:13.045 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:13.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:13.147 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:13.174 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:13.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:13.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:13.303 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:13.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:13.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:13.405 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:13.432 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:13.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:13.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:13.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:13 smithi029 ceph-mon[97787]: pgmap v2257: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 11 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 928/564 objects degraded (164.539%); 79/564 objects misplaced (14.007%); 0 B/s, 54 objects/s recovering 2024-04-12T11:57:13.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:13 smithi029 ceph-mon[97787]: osdmap e1308: 8 total, 8 up, 6 in 2024-04-12T11:57:13.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 928/564 objects degraded (164.539%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:57:13.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:13 smithi029 ceph-mon[93650]: pgmap v2257: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 11 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 928/564 objects degraded (164.539%); 79/564 objects misplaced (14.007%); 0 B/s, 54 objects/s recovering 2024-04-12T11:57:13.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:13 smithi029 ceph-mon[93650]: osdmap e1308: 8 total, 8 up, 6 in 2024-04-12T11:57:13.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 928/564 objects degraded (164.539%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:57:13.534 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:13.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:13.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:13.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:13.664 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:13.697 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:13.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:13.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:13.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:13 smithi087 ceph-mon[89014]: pgmap v2257: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 11 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 928/564 objects degraded (164.539%); 79/564 objects misplaced (14.007%); 0 B/s, 54 objects/s recovering 2024-04-12T11:57:13.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:13 smithi087 ceph-mon[89014]: osdmap e1308: 8 total, 8 up, 6 in 2024-04-12T11:57:13.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 928/564 objects degraded (164.539%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-12T11:57:13.791 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:13.800 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:13.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:13.829 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:13.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:13.931 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:13.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:13.961 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:13.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:14.016 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:57:14.063 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:14.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:14.093 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:14.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:14.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:14.224 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:14.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:14.325 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:14.354 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:14.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:14.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:14.456 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:14.485 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:14.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:14.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:14.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:14.613 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:14.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:14.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:14.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:14 smithi029 ceph-mon[97787]: osdmap e1309: 8 total, 8 up, 6 in 2024-04-12T11:57:14.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:14 smithi029 ceph-mon[93650]: osdmap e1309: 8 total, 8 up, 6 in 2024-04-12T11:57:14.715 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:14.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:14 smithi087 ceph-mon[89014]: osdmap e1309: 8 total, 8 up, 6 in 2024-04-12T11:57:14.743 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:14.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:14.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:14.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:14.872 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:14.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:14.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:14.974 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:15.002 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:15.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:15.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:15.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:15.130 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:15.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:15.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:15.233 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:15.261 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:15.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:15.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:57:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:57:14] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.43.0" 2024-04-12T11:57:15.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:15.389 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:15.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:15.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:15.516 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:15.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:15.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:15.618 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:15.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:15.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:15.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:15 smithi087 ceph-mon[89014]: pgmap v2260: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 446 op/s; 928/72 objects degraded (1288.889%); 0 B/s, 60 objects/s recovering 2024-04-12T11:57:15.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:15.759 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:15 smithi029 ceph-mon[93650]: pgmap v2260: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 446 op/s; 928/72 objects degraded (1288.889%); 0 B/s, 60 objects/s recovering 2024-04-12T11:57:15.759 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:15 smithi029 ceph-mon[97787]: pgmap v2260: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 95 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 446 op/s; 928/72 objects degraded (1288.889%); 0 B/s, 60 objects/s recovering 2024-04-12T11:57:15.773 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:15.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:15.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:15.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:15.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:15.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:15.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:16.006 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:16.034 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:16.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:16.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:16.136 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:16.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:16.164 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:16.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:16.265 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:16.293 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:16.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:16.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:16.395 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:16.422 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:16.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:16.524 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:16.549 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:16.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:16.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:16.651 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:16.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:16.676 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:16.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:16.709 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:16 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 928/72 objects degraded (1288.889%), 1 pg degraded, 1 pg undersized) 2024-04-12T11:57:16.709 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:16 smithi029 ceph-mon[97787]: osdmap e1310: 8 total, 8 up, 6 in 2024-04-12T11:57:16.710 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:16 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 928/72 objects degraded (1288.889%), 1 pg degraded, 1 pg undersized) 2024-04-12T11:57:16.710 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:16 smithi029 ceph-mon[93650]: osdmap e1310: 8 total, 8 up, 6 in 2024-04-12T11:57:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:16 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 928/72 objects degraded (1288.889%), 1 pg degraded, 1 pg undersized) 2024-04-12T11:57:16.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:16 smithi087 ceph-mon[89014]: osdmap e1310: 8 total, 8 up, 6 in 2024-04-12T11:57:16.778 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:16.802 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:16.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:16.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:16.904 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:16.928 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:16.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:16.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:17.030 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:17.059 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:17.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:17.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:17.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:17.184 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:17.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:17.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:17.286 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:17.314 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:17.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:17.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:17.416 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:17.441 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:17.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:17.543 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:17.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:17.569 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:17.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:17.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:17.695 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:17.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:17 smithi087 ceph-mon[89014]: pgmap v2261: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 0 B/s, 296 objects/s recovering 2024-04-12T11:57:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:17 smithi087 ceph-mon[89014]: osdmap e1311: 8 total, 8 up, 6 in 2024-04-12T11:57:17.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:17 smithi029 ceph-mon[93650]: pgmap v2261: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 0 B/s, 296 objects/s recovering 2024-04-12T11:57:17.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:17 smithi029 ceph-mon[93650]: osdmap e1311: 8 total, 8 up, 6 in 2024-04-12T11:57:17.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:17 smithi029 ceph-mon[97787]: pgmap v2261: 99 pgs: 1 active+recovering+remapped, 2 active+remapped+backfill_wait, 96 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 281 op/s; 0 B/s, 296 objects/s recovering 2024-04-12T11:57:17.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:17 smithi029 ceph-mon[97787]: osdmap e1311: 8 total, 8 up, 6 in 2024-04-12T11:57:17.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:17.822 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:17.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:17.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:17.948 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:17.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:18.050 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:18.077 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:18.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:18.179 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:18.205 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:18.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:18.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:18.307 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:18.333 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:18.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:18.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:18.435 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:18.461 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:18.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:18.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:18.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:18.589 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:18.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:18.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:18.715 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:18.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:18.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:18.816 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:18.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:18.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:18.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:18.946 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:18.973 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:18.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:19.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:19.102 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:19.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:19.204 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:19.231 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:19.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:19.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:19.333 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:19.360 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:19.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:19.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:19.462 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:19.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:19.491 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:19.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:19.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:19.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:19.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:19.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:19.717 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:19.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:19 smithi087 ceph-mon[89014]: pgmap v2264: 99 pgs: 1 active+recovering+remapped, 98 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 0 B/s, 323 objects/s recovering 2024-04-12T11:57:19.744 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:19.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:19.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:19.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:19 smithi029 ceph-mon[93650]: pgmap v2264: 99 pgs: 1 active+recovering+remapped, 98 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 0 B/s, 323 objects/s recovering 2024-04-12T11:57:19.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:19 smithi029 ceph-mon[97787]: pgmap v2264: 99 pgs: 1 active+recovering+remapped, 98 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 0 B/s, 323 objects/s recovering 2024-04-12T11:57:19.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:19.873 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:19.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:19.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:19.975 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:20.002 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:20.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:20.104 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:20.131 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:20.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:20.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:20.232 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:20.258 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:20.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:20.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:20.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:20.389 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:20.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:20.490 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:20.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:20.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:20.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:20.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:20.646 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:20.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:20.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:20.748 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:20.777 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:20.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:20.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:20.878 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:20.906 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:20.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:20.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:20.951 INFO:teuthology.orchestra.run.smithi029.stderr:no valid command found; 10 closest matches: 2024-04-12T11:57:20.951 INFO:teuthology.orchestra.run.smithi029.stderr:0 2024-04-12T11:57:20.951 INFO:teuthology.orchestra.run.smithi029.stderr:1 2024-04-12T11:57:20.951 INFO:teuthology.orchestra.run.smithi029.stderr:2 2024-04-12T11:57:20.951 INFO:teuthology.orchestra.run.smithi029.stderr:abort 2024-04-12T11:57:20.951 INFO:teuthology.orchestra.run.smithi029.stderr:assert 2024-04-12T11:57:20.951 INFO:teuthology.orchestra.run.smithi029.stderr:bench [] [] [] [] 2024-04-12T11:57:20.952 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs debug_inject_read_zeros 2024-04-12T11:57:20.952 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs files list 2024-04-12T11:57:20.952 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs stats 2024-04-12T11:57:20.952 INFO:teuthology.orchestra.run.smithi029.stderr:bluestore allocator dump block 2024-04-12T11:57:20.952 INFO:teuthology.orchestra.run.smithi029.stderr:Error EINVAL: invalid command 2024-04-12T11:57:21.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:21.036 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:21.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:21.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:21.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:21.166 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:21.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:21.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:21.294 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:21.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:21.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:21.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:21.425 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:21.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:21.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:21.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:21.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:21.554 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:21.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:21.656 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:21.683 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:21.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:21.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:21 smithi087 ceph-mon[89014]: pgmap v2265: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 489 objects/s recovering 2024-04-12T11:57:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:21 smithi029 ceph-mon[97787]: pgmap v2265: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 489 objects/s recovering 2024-04-12T11:57:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:21 smithi029 ceph-mon[93650]: pgmap v2265: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 489 objects/s recovering 2024-04-12T11:57:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:21.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:21.813 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:21.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:21.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:21.914 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:21.943 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:21.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:22.045 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:22.073 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:22.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:22.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:22.137 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:22.162 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-12T11:57:22.162 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-04-12T11:57:22.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-04-12T11:57:22.174 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:22.230 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:22.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:22.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:22.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:22.359 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:22.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:22.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:22.461 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:22.485 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:22.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:22.587 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:22.615 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:22.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:22.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:22.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:57:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:57:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:57:22.717 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:22.745 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:22.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:22.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:22.846 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:22.876 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:22.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:22.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:22.978 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:23.002 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:23.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:23.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:23.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:23.130 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:23.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:23.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:23.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:23.255 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:23.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:23.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:23.357 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:23.384 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:23.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:23.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:23.511 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:23.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:23.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:23.613 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:23.638 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:23.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:23.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:23.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:23 smithi087 ceph-mon[89014]: pgmap v2266: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 417 objects/s recovering 2024-04-12T11:57:23.729 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:23 smithi029 ceph-mon[97787]: pgmap v2266: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 417 objects/s recovering 2024-04-12T11:57:23.729 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:23 smithi029 ceph-mon[93650]: pgmap v2266: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 417 objects/s recovering 2024-04-12T11:57:23.741 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:23.768 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:23.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:23.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:23.870 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:23.897 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:23.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:23.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:23.999 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:24.026 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:24.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:24.102 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:57:24.103 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:57:24.127 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:24.154 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:24.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:24.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:24.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:24.282 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:24.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:24.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:24.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:24.407 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:24.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:24.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:24.447 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_5' removed 2024-04-12T11:57:24.509 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:24.555 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:24.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:24.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:24.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:24.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:24.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:24 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2024-04-12T11:57:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:24 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2024-04-12T11:57:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:24 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4049794240' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:57:24.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:24 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:57:24.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:24 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2024-04-12T11:57:24.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:24 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2024-04-12T11:57:24.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:24 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4049794240' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:57:24.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:24 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:57:24.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:24 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]: dispatch 2024-04-12T11:57:24.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:24 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2024-04-12T11:57:24.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:24 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4049794240' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:57:24.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:24 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T11:57:24.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:24.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:24.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:24.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:24.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:24.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:24.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:25.035 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:25.061 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:25.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:25.079 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:57:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:57:24] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-04-12T11:57:25.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:25.186 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:25.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:25.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:25.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:25.311 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:25.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:25.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:25.413 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:25.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[93650]: pgmap v2267: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 181 objects/s recovering 2024-04-12T11:57:25.444 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:25.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:25.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:25.546 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:25.569 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:25.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:25.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:25.671 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:25.699 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:25.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:25.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:25 smithi087 ceph-mon[89014]: pgmap v2267: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 181 objects/s recovering 2024-04-12T11:57:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]': finished 2024-04-12T11:57:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2024-04-12T11:57:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:25 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:57:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:25 smithi087 ceph-mon[89014]: osdmap e1312: 8 total, 8 up, 6 in 2024-04-12T11:57:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:25 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1881383690' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:57:25.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]': finished 2024-04-12T11:57:25.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2024-04-12T11:57:25.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:57:25.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[93650]: osdmap e1312: 8 total, 8 up, 6 in 2024-04-12T11:57:25.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1881383690' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:57:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[97787]: pgmap v2267: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s, 181 objects/s recovering 2024-04-12T11:57:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [3, 2]}]': finished 2024-04-12T11:57:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2024-04-12T11:57:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T11:57:25.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[97787]: osdmap e1312: 8 total, 8 up, 6 in 2024-04-12T11:57:25.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:25 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1881383690' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:57:25.801 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:25.829 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:25.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:25.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:25.914 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-12T11:57:25.915 INFO:tasks.radosbench:Beginning radosbench... 2024-04-12T11:57:25.915 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2024-04-12T11:57:25.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool create unique_pool_6 16 2024-04-12T11:57:25.930 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:25.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:25.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:25.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:26.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:26.083 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:26.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:26.185 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:26.213 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:26.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:26.315 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:26.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:26.339 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:26.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:26.440 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:26.468 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:26.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:26.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:26.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:26 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:57:26.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:26 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1881383690' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:57:26.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:26 smithi029 ceph-mon[93650]: osdmap e1313: 8 total, 8 up, 6 in 2024-04-12T11:57:26.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:26 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:57:26.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:26 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1881383690' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:57:26.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:26 smithi029 ceph-mon[97787]: osdmap e1313: 8 total, 8 up, 6 in 2024-04-12T11:57:26.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:26.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:26.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:26.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:26.628 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:57:26.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:26 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:57:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:26 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1881383690' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:57:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:26 smithi087 ceph-mon[89014]: osdmap e1313: 8 total, 8 up, 6 in 2024-04-12T11:57:26.727 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:26.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:26.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:26.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:26.854 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:26.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:26.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:26.956 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:26.980 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:26.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:27.082 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:27.110 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:27.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:27.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:27.212 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:27.236 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:27.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:27.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:27.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:27.362 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:27.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:27.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:27.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:27.487 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:27.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:27.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:27.534 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-04-12T11:57:27.590 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:27.615 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:27.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:27.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:27 smithi087 ceph-mon[89014]: pgmap v2270: 83 pgs: 1 peering, 82 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:27 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:27.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:27.741 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:27.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:27.756 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:27 smithi029 ceph-mon[93650]: pgmap v2270: 83 pgs: 1 peering, 82 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:27 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:27 smithi029 ceph-mon[97787]: pgmap v2270: 83 pgs: 1 peering, 82 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:27 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:27.843 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:27.867 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:27.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:27.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:27.969 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:27.994 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:27.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:27.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:28.096 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:28.121 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:28.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:28.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:28.223 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:28.249 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:28.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:28.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:28.351 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:28.374 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:28.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:28.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:28.503 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:28.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:28.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:28.605 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:28.630 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:28.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:28.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:28.732 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:28.756 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:28.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:28.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:28.858 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:28.883 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:28.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:28.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:28.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:29.013 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:29.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:29.115 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:29.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:29.140 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:29.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:29.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:29.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:29.289 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:29.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:29.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:29.418 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:29.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:29.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:29.471 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:57:29.472 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:57:29.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:29.554 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:29.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:29.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:29.656 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:29.681 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:29.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:29 smithi087 ceph-mon[89014]: pgmap v2271: 83 pgs: 1 peering, 82 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:29.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:29 smithi029 ceph-mon[93650]: pgmap v2271: 83 pgs: 1 peering, 82 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:29 smithi029 ceph-mon[97787]: pgmap v2271: 83 pgs: 1 peering, 82 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:29.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:29.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:29.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:29.916 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:29.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:29.947 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:29.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:30.049 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:30.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:30.079 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:30.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:30.113 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-04-12T11:57:30.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:30.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:30.230 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:30.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:30.331 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:30.358 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:30.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:30.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:30.376 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:30.460 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:30.485 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:30.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:30.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:30.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:30.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:30.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:30.717 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:30.742 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:30.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:30.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:30.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:30.871 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:30.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:30.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:30.973 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:31.001 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:31.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:31.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:31.102 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:31.130 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:31.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:31.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:31.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:31.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:31.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:31.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:31.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:31.388 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:31.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:31.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:31.489 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:31.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:31.517 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:31.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:31.618 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:31.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:31.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:31.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:31.660 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:31 smithi087 ceph-mon[89014]: pgmap v2272: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:31.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:31.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:31 smithi029 ceph-mon[93650]: pgmap v2272: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:31 smithi029 ceph-mon[97787]: pgmap v2272: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:31.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:31.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:31.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:31.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:31.873 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:57:31.873 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:57:31.900 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:31.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:31.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:32.001 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:32.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:32.030 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:32.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:32.131 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:32.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:32.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:32.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:32.261 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:32.284 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:32.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:32.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:32.386 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:32.410 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:32.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:32.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:32.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:57:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:57:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:57:32.511 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-04-12T11:57:32.511 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:32.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:32.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:32.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:32.636 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:32.661 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:32.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:32.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:32.717 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:32.762 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:32.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:32.787 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:32.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:32.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:32.915 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:32.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:32.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:33.017 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:33.041 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:33.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:33.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:33.143 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:33.166 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:33.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:33.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:33.268 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:33.292 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:33.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:33.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:33.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:33.417 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:33.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:33.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:33.519 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:33.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:33.548 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:33.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:33.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:33.677 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:33.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:33.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:33.697 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:33 smithi087 ceph-mon[89014]: pgmap v2273: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:33 smithi029 ceph-mon[93650]: pgmap v2273: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:33.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:33 smithi029 ceph-mon[97787]: pgmap v2273: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:33.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:33.804 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:33.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:33.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:33.906 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:33.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:33.931 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:33.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:34.032 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:34.057 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:34.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:34.159 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:34.182 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:34.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:34.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:34.282 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:57:34.283 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:57:34.284 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:34.308 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:34.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:34.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:34.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:34.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:34.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:34.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:34.565 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:34.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:34.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:34.667 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:34.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:34.699 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:34.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:34.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:34.828 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:34.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:34.929 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:34.957 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:34.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:34.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-04-12T11:57:35.028 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:57:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:57:34] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-04-12T11:57:35.033 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:35.040 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:57:35.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:35.061 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:57:35.083 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:57:35.085 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:35.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:35.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:35.103 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:57:35.123 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:57:35.143 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:57:35.162 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:57:35.183 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:57:35.184 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:57:35.184 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:57:35.184 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:57:35.185 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:57:35.185 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:57:35.185 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:57:35.185 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:57:35.185 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:57:35.187 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:35.211 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:35.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:35.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:35.216 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:35.313 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:35.337 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:35.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:35.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:35.439 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:35.462 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:35.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:35.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:35.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:35.589 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:35.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:35.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:35.641 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:35 smithi087 ceph-mon[89014]: pgmap v2274: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:35.641 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:35 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3398912525' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:57:35.691 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:35.715 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:35.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:35.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:35.730 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:35 smithi029 ceph-mon[93650]: pgmap v2274: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:35.731 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:35 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3398912525' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:57:35.731 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:35 smithi029 ceph-mon[97787]: pgmap v2274: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:35.731 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:35 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3398912525' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:57:35.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:35.842 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:35.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:35.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:35.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:35.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:35.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:36.071 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:36.097 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:36.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:36.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:36.199 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:36.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:36.223 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:36.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:36.324 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:36.352 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:36.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:36.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:36.419 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:57:36.453 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:36.477 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:36.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:36.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:36.580 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:36.605 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:36.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:36.613 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:36.707 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:36.732 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:36.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:36.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:36.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:36.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:36.834 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:36.846 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:57:36.846 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:57:36.846 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T11:57:36.846 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:57:36.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:36.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:36.960 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:36.985 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:36.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:37.086 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:37.112 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:37.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:37.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:37.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:37.239 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:37.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:37.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:37.341 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:37.366 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:37.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:37.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:37.468 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:37.492 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:37.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:37.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:37.518 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-04-12T11:57:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:37 smithi029 ceph-mon[93650]: pgmap v2275: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:37.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:37 smithi029 ceph-mon[97787]: pgmap v2275: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:37.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:37.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:37.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:37.621 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:37.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:37 smithi087 ceph-mon[89014]: pgmap v2275: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 531 GiB / 536 GiB avail 2024-04-12T11:57:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:37.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:37.747 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:37.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:37.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:37.762 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:37.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:37.873 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:37.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:37.975 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:37.999 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:38.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:38.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:38.101 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:38.127 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:38.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:38.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:38.229 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:38.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:38.255 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:38.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:38.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:38.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:38.382 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:38.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:38.397 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:38.483 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:38.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:38.508 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:38.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:38.609 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:38.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:38.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:38.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:38.735 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:38.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:38.759 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:38.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:38.860 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:38.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:38.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:38.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:38.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:39.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:39.011 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:39.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:39.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:39.135 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:39.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:39.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:39.226 INFO:teuthology.orchestra.run.smithi029.stderr:no valid command found; 10 closest matches: 2024-04-12T11:57:39.226 INFO:teuthology.orchestra.run.smithi029.stderr:0 2024-04-12T11:57:39.226 INFO:teuthology.orchestra.run.smithi029.stderr:1 2024-04-12T11:57:39.226 INFO:teuthology.orchestra.run.smithi029.stderr:2 2024-04-12T11:57:39.227 INFO:teuthology.orchestra.run.smithi029.stderr:abort 2024-04-12T11:57:39.227 INFO:teuthology.orchestra.run.smithi029.stderr:assert 2024-04-12T11:57:39.227 INFO:teuthology.orchestra.run.smithi029.stderr:bench [] [] [] [] 2024-04-12T11:57:39.227 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs debug_inject_read_zeros 2024-04-12T11:57:39.227 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs files list 2024-04-12T11:57:39.227 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs stats 2024-04-12T11:57:39.227 INFO:teuthology.orchestra.run.smithi029.stderr:bluestore allocator dump block 2024-04-12T11:57:39.227 INFO:teuthology.orchestra.run.smithi029.stderr:Error EINVAL: invalid command 2024-04-12T11:57:39.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:39.262 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:39.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:39.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:39.364 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:39.370 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:57:39.370 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:57:39.370 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:57:39.370 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:57:39.371 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:57:39.389 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:39.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:39.472 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_6' created 2024-04-12T11:57:39.491 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:39.514 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:39.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:39.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:39.616 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:39.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:39.641 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:39.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:39 smithi087 ceph-mon[89014]: pgmap v2276: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:39 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3250092792' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2024-04-12T11:57:39.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:39.766 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:39.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:39.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:39 smithi029 ceph-mon[93650]: pgmap v2276: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:39.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:39 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3250092792' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2024-04-12T11:57:39.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:39 smithi029 ceph-mon[97787]: pgmap v2276: 83 pgs: 83 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:39.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:39 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3250092792' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2024-04-12T11:57:39.868 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:39.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:39.894 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:39.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:39.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:57:39.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:40.041 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:40.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:40.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:40.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:40.166 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:40.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:40.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:40.217 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-04-12T11:57:40.268 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:40.271 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-12T11:57:40.271 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-04-12T11:57:40.272 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-04-12T11:57:40.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:40.292 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:40.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:40.393 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:40.418 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:40.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:40.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:40.426 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:40.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:40.544 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:40.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:40.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:40.647 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:40.671 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:40.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:40.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:40.713 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:40 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3250092792' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-04-12T11:57:40.713 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:40 smithi029 ceph-mon[93650]: osdmap e1314: 8 total, 8 up, 6 in 2024-04-12T11:57:40.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:40 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3250092792' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-04-12T11:57:40.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:40 smithi029 ceph-mon[97787]: osdmap e1314: 8 total, 8 up, 6 in 2024-04-12T11:57:40.765 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:40 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3250092792' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-04-12T11:57:40.765 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:40 smithi087 ceph-mon[89014]: osdmap e1314: 8 total, 8 up, 6 in 2024-04-12T11:57:40.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:40.796 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:40.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:40.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:40.922 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:40.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:40.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:41.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:41.050 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:41.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:41.152 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:41.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:41.176 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:41.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:41.277 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:41.300 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:41.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:41.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:41.402 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:41.426 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:41.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:41.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:41.444 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2024-04-12T11:57:41.497 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:41.528 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:41.552 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:41.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:41.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:41.595 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:41 smithi087 ceph-mon[89014]: pgmap v2278: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:41.595 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:41 smithi087 ceph-mon[89014]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:57:41.595 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:41 smithi087 ceph-mon[89014]: osdmap e1315: 8 total, 8 up, 6 in 2024-04-12T11:57:41.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:41 smithi029 ceph-mon[97787]: pgmap v2278: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:41.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:41 smithi029 ceph-mon[97787]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:57:41.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:41 smithi029 ceph-mon[97787]: osdmap e1315: 8 total, 8 up, 6 in 2024-04-12T11:57:41.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:41 smithi029 ceph-mon[93650]: pgmap v2278: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:41.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:41 smithi029 ceph-mon[93650]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-12T11:57:41.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:41 smithi029 ceph-mon[93650]: osdmap e1315: 8 total, 8 up, 6 in 2024-04-12T11:57:41.654 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:41.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:41.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:41.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:41.779 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:41.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:41.803 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:41.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:41.904 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:41.928 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:41.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:41.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:42.030 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:42.037 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:57:42.037 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T11:57:42.037 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T11:57:42.038 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T11:57:42.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.038 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:28366 10.a 10:5cd39eea:::benchmark_data_smithi029_256330_object28365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:57:42.038 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:53:29.643641+0000", 2024-04-12T11:57:42.038 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 252.390871377, 2024-04-12T11:57:42.038 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50832011300000002, 2024-04-12T11:57:42.038 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.038 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.038 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 28366 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643641+0000", 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643641+0000", 2024-04-12T11:57:42.039 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643640+0000", 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643697+0000", 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6647999999999997e-05 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.040 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643707+0000", 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0672999999999999e-05 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643715+0000", 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3989999999999998e-06 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643764+0000", 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9574999999999999e-05 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643893+0000", 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012865899999999999 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0", 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644066+0000", 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017302299999999999 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.647119+0000", 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030533169999999998 2024-04-12T11:57:42.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.151849+0000", 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50472942399999998 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.151891+0000", 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2233e-05 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.043 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.151961+0000", 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0022000000000001e-05 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:28371 10.a 10:5449dfcb:::benchmark_data_smithi029_256330_object28370:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:53:29.644317+0000", 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 252.39019487600001, 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.51451874099999995, 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.044 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 28371 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644317+0000", 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644317+0000", 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644316+0000", 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644387+0000", 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0300000000000001e-05 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644398+0000", 2024-04-12T11:57:42.046 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1088e-05 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644405+0000", 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.233e-06 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.645328+0000", 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00092331900000000003 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.645459+0000", 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013036100000000001 2024-04-12T11:57:42.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0", 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.646076+0000", 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00061679499999999995 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.647256+0000", 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0011801470000000001 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.048 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.155833+0000", 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50857733599999999 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.158746+0000", 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0029129609999999999 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.158836+0000", 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0099999999999995e-05 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.049 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:75499 10.a 10:5ba68729:::benchmark_data_smithi029_256330_object75498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:13.175640+0000", 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 208.858872658, 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.74925757500000001, 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75499 2024-04-12T11:57:42.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175640+0000", 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175640+0000", 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175638+0000", 2024-04-12T11:57:42.051 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175686+0000", 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7534999999999998e-05 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175698+0000", 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2112e-05 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175706+0000", 2024-04-12T11:57:42.052 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1259999999999998e-06 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.177660+0000", 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0019538300000000001 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.177802+0000", 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014221200000000001 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:57:42.053 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.177959+0000", 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015694799999999999 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.188846+0000", 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.010886905000000001 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.193805+0000", 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.004958596 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.054 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.922554+0000", 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.72874928999999999 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.922606+0000", 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2528999999999998e-05 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.924897+0000", 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0022907940000000001 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.055 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:75507 10.a 10:5fb6aa19:::benchmark_data_smithi029_256330_object75506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:13.181734+0000", 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 208.85277790000001, 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.77549217999999998, 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.056 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75507 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181734+0000", 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181734+0000", 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181726+0000", 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999914 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181773+0000", 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7509999999999997e-05 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181787+0000", 2024-04-12T11:57:42.058 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.345e-05 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181795+0000", 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0909999999999993e-06 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.188918+0000", 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0071231200000000001 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.059 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.189098+0000", 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018042199999999999 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.189346+0000", 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00024795900000000002 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.191973+0000", 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0026264769999999999 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.060 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.194043+0000", 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0020700580000000001 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.955399+0000", 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.76135565699999996 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.955437+0000", 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8482999999999997e-05 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.061 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.957227+0000", 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001789433 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:75523 10.a 10:58a8ebd4:::benchmark_data_smithi029_256330_object75522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:13.200557+0000", 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 208.83395550500001, 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.80757887900000003, 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.062 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75523 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200557+0000", 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.063 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200557+0000", 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200555+0000", 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200615+0000", 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9916e-05 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.064 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.065 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200625+0000", 2024-04-12T11:57:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0291e-05 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200641+0000", 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5931999999999999e-05 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.067 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200690+0000", 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8884000000000002e-05 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200874+0000", 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000183451 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.201075+0000", 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00020140300000000001 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.068 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.203290+0000", 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0022150859999999998 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.204365+0000", 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0010749959999999999 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.962486+0000", 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.758120656 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.069 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.962525+0000", 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9167e-05 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:14.008136+0000", 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.045610629 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.070 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:75542 10.f 10:f998c42b:::benchmark_data_smithi029_256330_object75541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:13.235436+0000", 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 208.799076603, 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.68735639299999995, 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75542 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.071 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235436+0000", 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235436+0000", 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235434+0000", 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.072 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235481+0000", 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6989e-05 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235494+0000", 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3220999999999999e-05 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235501+0000", 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8369999999999999e-06 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.073 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235534+0000", 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3006999999999998e-05 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235637+0000", 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000102472 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235734+0000", 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6926e-05 2024-04-12T11:57:42.074 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.237928+0000", 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0021942060000000002 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.238542+0000", 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00061416400000000005 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.922715+0000", 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.684172907 2024-04-12T11:57:42.075 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.922758+0000", 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3232000000000002e-05 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.922792+0000", 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3797000000000002e-05 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.076 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:75543 10.a 10:599efe38:::benchmark_data_smithi029_256330_object75542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:13.236268+0000", 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 208.79824381500001, 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.77272679899999996, 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75543 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.077 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236268+0000", 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236268+0000", 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236268+0000", 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.078 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236354+0000", 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5798999999999996e-05 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236360+0000", 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0870000000000003e-06 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236364+0000", 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1520000000000002e-06 2024-04-12T11:57:42.079 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236400+0000", 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6399000000000002e-05 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236504+0000", 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010341200000000001 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:57:42.080 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236602+0000", 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7769999999999994e-05 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.238669+0000", 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0020678900000000002 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.239329+0000", 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00065918599999999997 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.081 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:14.008190+0000", 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.76886104099999997 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:14.008231+0000", 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0920000000000001e-05 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:14.008995+0000", 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00076463300000000002 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.082 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155253 10.f 10:fbaad049:::benchmark_data_smithi029_256330_object53487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:07.211877+0000", 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 94.822635449000003, 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.40654251600000002, 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155253 2024-04-12T11:57:42.083 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211877+0000", 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211877+0000", 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.084 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211876+0000", 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211880+0000", 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2230000000000001e-06 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211887+0000", 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.348e-06 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.085 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211892+0000", 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9049999999999996e-06 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.213491+0000", 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001599523 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.213583+0000", 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2151000000000006e-05 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.086 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.213663+0000", 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0251000000000001e-05 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.216046+0000", 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023823440000000002 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.230337+0000", 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014290792 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.087 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.618319+0000", 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.38798262500000003 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.618374+0000", 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4818999999999998e-05 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.618419+0000", 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5266000000000003e-05 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.088 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155260 10.f 10:fcffd604:::benchmark_data_smithi029_256330_object53494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:07.223957+0000", 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 94.8105549, 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.40551721499999999, 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155260 2024-04-12T11:57:42.089 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223957+0000", 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223957+0000", 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223956+0000", 2024-04-12T11:57:42.090 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223962+0000", 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2120000000000002e-06 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223969+0000", 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2429999999999999e-06 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223975+0000", 2024-04-12T11:57:42.091 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1360000000000002e-06 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223998+0000", 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.2841e-05 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.224097+0000", 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9648999999999994e-05 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.224178+0000", 2024-04-12T11:57:42.092 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0876000000000003e-05 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.227210+0000", 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003031499 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.230617+0000", 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034070440000000001 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.629310+0000", 2024-04-12T11:57:42.093 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.39869354800000001 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.629418+0000", 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000107297 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.629475+0000", 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6833000000000003e-05 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.094 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155858 10.a 10:5f54eb39:::benchmark_data_smithi029_256330_object54092:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:08.359538+0000", 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 93.674974177999999, 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.386920337, 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155858 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.095 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359538+0000", 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359538+0000", 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359537+0000", 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.096 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359544+0000", 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.849e-06 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359557+0000", 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3485999999999999e-05 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359565+0000", 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8199999999999997e-06 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.097 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359610+0000", 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4845999999999997e-05 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359743+0000", 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000133566 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359879+0000", 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000135952 2024-04-12T11:57:42.098 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.363523+0000", 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0036437639999999999 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.367054+0000", 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00353105 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.746372+0000", 2024-04-12T11:57:42.099 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.37931752099999999 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.746416+0000", 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4827000000000003e-05 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.746458+0000", 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1902e-05 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.100 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155863 10.b 10:d05c458e:::benchmark_data_smithi029_256330_object54097:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:08.359831+0000", 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 93.674680781000006, 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30073972900000001, 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155863 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.101 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359831+0000", 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359831+0000", 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359831+0000", 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.102 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359835+0000", 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.805e-06 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359842+0000", 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5499999999999997e-06 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359848+0000", 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8939999999999998e-06 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.103 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.363052+0000", 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0032038790000000002 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.363187+0000", 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000134542 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1,4", 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.363342+0000", 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015579499999999999 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.104 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.366996+0000", 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0036534839999999998 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.368259+0000", 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0012627440000000001 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.370377+0000", 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0021185969999999998 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.105 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.660383+0000", 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29000565699999997 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.660464+0000", 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1235000000000004e-05 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.660571+0000", 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000107151 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.106 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155867 10.a 10:563170cd:::benchmark_data_smithi029_256330_object54101:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:08.369275+0000", 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 93.665237241, 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.38112349600000001, 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155867 2024-04-12T11:57:42.107 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369275+0000", 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369275+0000", 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.108 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369274+0000", 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369279+0000", 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1220000000000003e-06 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369289+0000", 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2890000000000003e-06 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.109 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369295+0000", 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4130000000000003e-06 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369325+0000", 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.9643000000000001e-05 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369419+0000", 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4534e-05 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.110 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369501+0000", 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1775999999999998e-05 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.372239+0000", 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002737803 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.372630+0000", 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000391612 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.111 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.750289+0000", 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.3776583 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.750354+0000", 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5726000000000001e-05 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.750398+0000", 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4161e-05 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.112 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:156467 10.b 10:d03df4ba:::benchmark_data_smithi029_256330_object54701:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:09.517597+0000", 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 92.516915064000003, 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.301060577, 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 156467 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.113 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.517597+0000", 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.517597+0000", 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.517597+0000", 2024-04-12T11:57:42.114 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.517599+0000", 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.571e-06 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.517605+0000", 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5210000000000002e-06 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.517620+0000", 2024-04-12T11:57:42.115 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5152e-05 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.517744+0000", 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012410700000000001 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.517879+0000", 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013456199999999999 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1,4", 2024-04-12T11:57:42.116 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.518009+0000", 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013033399999999999 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.522649+0000", 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.004640127 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.522844+0000", 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019451200000000001 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.117 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.528977+0000", 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0061337170000000003 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.818483+0000", 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.28950576300000003 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.818538+0000", 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4752e-05 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.118 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.818658+0000", 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011985300000000001 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157799 10.f 10:fd68811f:::benchmark_data_smithi029_256330_object56033:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.713914+0000", 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 90.320598141999994, 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.408914153, 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.119 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157799 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713914+0000", 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713914+0000", 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.120 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713913+0000", 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713920+0000", 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9990000000000002e-06 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713935+0000", 2024-04-12T11:57:42.121 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5463000000000001e-05 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713944+0000", 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.799e-06 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713983+0000", 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9019e-05 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.122 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.714102+0000", 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000119529 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.714226+0000", 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000123965 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.717724+0000", 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034974189999999999 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.123 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.723864+0000", 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0061404440000000001 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122732+0000", 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.39886750599999998 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122785+0000", 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3251999999999998e-05 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.124 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122828+0000", 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3248000000000003e-05 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157802 10.f 10:fc076b5f:::benchmark_data_smithi029_256330_object56036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.718855+0000", 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 90.315657521999995, 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.404126121, 2024-04-12T11:57:42.125 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157802 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718855+0000", 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.126 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718855+0000", 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718854+0000", 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718858+0000", 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6460000000000001e-06 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.127 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718867+0000", 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4819999999999994e-06 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718872+0000", 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3000000000000001e-06 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718910+0000", 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.803e-05 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.128 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.719009+0000", 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9062999999999997e-05 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.719088+0000", 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8380000000000005e-05 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.721914+0000", 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002826631 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.129 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.728088+0000", 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0061736009999999999 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122875+0000", 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.39478680799999999 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122945+0000", 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0653000000000003e-05 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.130 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122981+0000", 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5395999999999998e-05 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157803 10.a 10:55552f02:::benchmark_data_smithi029_256330_object56037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.718879+0000", 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 90.315633422000005, 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.40287460899999999, 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.131 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157803 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718879+0000", 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.132 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718879+0000", 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718878+0000", 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718881+0000", 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.447e-06 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.133 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718888+0000", 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6420000000000001e-06 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718893+0000", 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4809999999999997e-06 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718945+0000", 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2340999999999999e-05 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.134 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.719066+0000", 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000120307 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.719172+0000", 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000106498 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.725265+0000", 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0060925709999999998 2024-04-12T11:57:42.135 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.730326+0000", 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0050610560000000004 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121641+0000", 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.39131515100000003 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.136 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121706+0000", 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4937000000000005e-05 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121753+0000", 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7559999999999999e-05 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157804 10.a 10:5305d031:::benchmark_data_smithi029_256330_object56038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.718899+0000", 2024-04-12T11:57:42.137 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 90.315613123000006, 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.40304277399999999, 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157804 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718899+0000", 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.138 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718899+0000", 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718899+0000", 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718901+0000", 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5600000000000001e-06 2024-04-12T11:57:42.139 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718908+0000", 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8120000000000002e-06 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718913+0000", 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0200000000000002e-06 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.721883+0000", 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002970014 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.140 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.721991+0000", 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000107551 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.722093+0000", 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010247 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.725415+0000", 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0033219310000000002 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.141 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.730367+0000", 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0049522259999999997 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121818+0000", 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.39145069100000002 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121902+0000", 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3788999999999993e-05 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.142 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121942+0000", 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0051e-05 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157822 10.a 10:5a13f56b:::benchmark_data_smithi029_256330_object56056:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.740827+0000", 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 90.293685408000002, 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.38124880799999999, 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.143 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157822 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740827+0000", 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740827+0000", 2024-04-12T11:57:42.144 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740825+0000", 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740832+0000", 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7089999999999996e-06 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.145 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740843+0000", 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1413000000000001e-05 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740850+0000", 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3600000000000001e-06 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740895+0000", 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5597999999999999e-05 2024-04-12T11:57:42.146 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:42.147 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:42.147 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.147 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.741056+0000", 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016123 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.741168+0000", 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000111935 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.744393+0000", 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0032241399999999999 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.148 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.746021+0000", 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0016283929999999999 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121988+0000", 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.37596682100000001 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122036+0000", 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8723000000000001e-05 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.149 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122076+0000", 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9180000000000001e-05 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157855 10.f 10:f3b223c4:::benchmark_data_smithi029_256330_object56089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.801057+0000", 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 90.2334551, 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.32633640800000002, 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.150 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157855 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.801057+0000", 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.801057+0000", 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.151 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.801056+0000", 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.801061+0000", 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6949999999999997e-06 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.801073+0000", 2024-04-12T11:57:42.152 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2092e-05 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.801084+0000", 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0288999999999999e-05 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.801151+0000", 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7829000000000004e-05 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.801316+0000", 2024-04-12T11:57:42.153 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016441200000000001 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.801417+0000", 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000101158 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.804516+0000", 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030988309999999998 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.154 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.804766+0000", 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00025023899999999997 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.127315+0000", 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.32254923000000002 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.127360+0000", 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4659000000000001e-05 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.155 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.127394+0000", 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3522000000000002e-05 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:158738 10.a 10:586bbe59:::benchmark_data_smithi029_256330_object56972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:13.066989+0000", 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 88.967522864000003, 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.326215439, 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:57:42.156 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 158738 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.066989+0000", 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.066989+0000", 2024-04-12T11:57:42.157 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.066988+0000", 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.066993+0000", 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9779999999999996e-06 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T11:57:42.158 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.067002+0000", 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.585e-06 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.067016+0000", 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4029000000000001e-05 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.067055+0000", 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9529e-05 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T11:57:42.159 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.067169+0000", 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000113726 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.067271+0000", 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000102091 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.070212+0000", 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0029407180000000002 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T11:57:42.160 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.070283+0000", 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1465999999999996e-05 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.393128+0000", 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.32284426199999999 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.393162+0000", 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3878000000000003e-05 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T11:57:42.161 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:13.393205+0000", 2024-04-12T11:57:42.162 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3217000000000003e-05 2024-04-12T11:57:42.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.162 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.162 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T11:57:42.162 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T11:57:42.162 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:57:42.248 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:42.275 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:42.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:42.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:42.377 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:42.401 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:42.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:42.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:42.503 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:42.526 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:42.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:42.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:42.555 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:42 smithi029 ceph-mon[97787]: osdmap e1316: 8 total, 8 up, 6 in 2024-04-12T11:57:42.555 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:42 smithi029 ceph-mon[93650]: osdmap e1316: 8 total, 8 up, 6 in 2024-04-12T11:57:42.581 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:42 smithi087 ceph-mon[89014]: osdmap e1316: 8 total, 8 up, 6 in 2024-04-12T11:57:42.581 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:57:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:57:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:57:42.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:42.651 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:42.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:42.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:42.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:42.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:42.780 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:42.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:42.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:42.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:42.907 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:42.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:42.949 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-04-12T11:57:43.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:43.035 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:43.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:43.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:43.137 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:43.153 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:43.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:43.162 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:43.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:43.263 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:43.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:43.288 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:43.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:43.388 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:43.413 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:43.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:43.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:43.515 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:43.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:43 smithi029 ceph-mon[93650]: pgmap v2281: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:43.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:43 smithi029 ceph-mon[97787]: pgmap v2281: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:43.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:43.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:43.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:43.599 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:43 smithi087 ceph-mon[89014]: pgmap v2281: 99 pgs: 10 creating+peering, 6 unknown, 83 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:43.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:43.664 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:43.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:43.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:43.766 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:43.789 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:43.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:43.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:43.811 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:43.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:43.916 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:43.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:44.018 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:44.042 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:44.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:44.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:44.144 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:44.168 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:44.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:44.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:44.269 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:44.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:44.294 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:44.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:44.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:44.422 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:44.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:44.494 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:57:44.524 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:44.548 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:44.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:44.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:44.640 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:44 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1991447651' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:57:44.640 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:44 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:57:44.649 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:44.710 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:44.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:44.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:44 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1991447651' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:57:44.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:44 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:57:44.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:44 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1991447651' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:57:44.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:44 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:57:44.812 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:44.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:44.837 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:44.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:44.882 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:57:44.883 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:57:44.883 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T11:57:44.883 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:57:44.938 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:44.964 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:44.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:44.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:45.066 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:45.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:45.097 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:45.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:45.198 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:45.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:45.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:45.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:45.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:57:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:57:44] "GET /metrics HTTP/1.1" 200 34181 "" "Prometheus/2.43.0" 2024-04-12T11:57:45.328 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:45.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:45.353 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:45.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:45.455 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:45.457 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-04-12T11:57:45.480 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:45.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:45.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:45.582 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:45.606 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:45.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:45.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:45.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:45 smithi029 ceph-mon[97787]: pgmap v2282: 99 pgs: 10 creating+peering, 1 unknown, 88 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:45.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:45 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:57:45.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:45 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:57:45.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:45 smithi029 ceph-mon[97787]: osdmap e1317: 8 total, 8 up, 6 in 2024-04-12T11:57:45.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:45 smithi029 ceph-mon[93650]: pgmap v2282: 99 pgs: 10 creating+peering, 1 unknown, 88 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:45.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:45 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:57:45.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:45 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:57:45.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:45 smithi029 ceph-mon[93650]: osdmap e1317: 8 total, 8 up, 6 in 2024-04-12T11:57:45.661 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:45.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:45 smithi087 ceph-mon[89014]: pgmap v2282: 99 pgs: 10 creating+peering, 1 unknown, 88 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:45 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:57:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:45 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:57:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:45 smithi087 ceph-mon[89014]: osdmap e1317: 8 total, 8 up, 6 in 2024-04-12T11:57:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:45.734 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:45.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:45.835 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:45.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:45.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:45.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:45.962 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:45.992 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:45.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:45.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:46.094 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:46.120 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:46.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:46.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:46.221 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:46.245 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:46.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:46.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:46.347 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:46.370 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:46.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:46.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:46.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:46.498 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:46.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:46.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:46.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:46.625 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:46.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:46.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:46.751 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:46.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:46.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:46.853 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:46.877 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:46.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:46.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:46.979 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:47.002 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:47.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:47.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:47.104 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:47.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:47.129 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:47.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:47.230 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:47.254 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:47.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:47.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:47.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:47.357 INFO:teuthology.orchestra.run.smithi029.stderr:no valid command found; 10 closest matches: 2024-04-12T11:57:47.358 INFO:teuthology.orchestra.run.smithi029.stderr:0 2024-04-12T11:57:47.358 INFO:teuthology.orchestra.run.smithi029.stderr:1 2024-04-12T11:57:47.358 INFO:teuthology.orchestra.run.smithi029.stderr:2 2024-04-12T11:57:47.358 INFO:teuthology.orchestra.run.smithi029.stderr:abort 2024-04-12T11:57:47.358 INFO:teuthology.orchestra.run.smithi029.stderr:assert 2024-04-12T11:57:47.358 INFO:teuthology.orchestra.run.smithi029.stderr:bench [] [] [] [] 2024-04-12T11:57:47.358 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs debug_inject_read_zeros 2024-04-12T11:57:47.358 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs files list 2024-04-12T11:57:47.358 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs stats 2024-04-12T11:57:47.359 INFO:teuthology.orchestra.run.smithi029.stderr:bluestore allocator dump block 2024-04-12T11:57:47.359 INFO:teuthology.orchestra.run.smithi029.stderr:Error EINVAL: invalid command 2024-04-12T11:57:47.379 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:47.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:47.387 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:57:47.387 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T11:57:47.387 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T11:57:47.387 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T11:57:47.387 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:57:47.481 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:47.505 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:47.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:47.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:47.607 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:47.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:47.631 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:47.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:47 smithi087 ceph-mon[89014]: pgmap v2284: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:47.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:47.757 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:47.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:47.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:47 smithi029 ceph-mon[93650]: pgmap v2284: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:47.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:47 smithi029 ceph-mon[97787]: pgmap v2284: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:47.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:47.884 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:47.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:47.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:47.981 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-04-12T11:57:47.986 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:47.992 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:57:48.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:48.011 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:48.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:48.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:48.137 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:48.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:48.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:48.186 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:48.239 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:48.262 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:48.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:48.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:48.332 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-12T11:57:48.333 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-04-12T11:57:48.333 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-04-12T11:57:48.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:48.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:48.399 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:48.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:48.500 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:48.523 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:48.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:48.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:48.625 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:48.651 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:48.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:48.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:48.753 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:48.777 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:48.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:48.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:48.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:48.906 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:48.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:48.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:49.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:49.033 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:49.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:49.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:49.080 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:49.101 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:57:49.102 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:49.123 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:57:49.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:49.143 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:57:49.158 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:49.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:49.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:49.163 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:57:49.183 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:57:49.202 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:57:49.222 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:57:49.242 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:57:49.243 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:57:49.243 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:57:49.243 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:57:49.243 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:57:49.243 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:57:49.243 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:57:49.243 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:57:49.243 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:57:49.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:49.284 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:49.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:49.386 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:49.409 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:49.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:49.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:49.499 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_6' 2024-04-12T11:57:49.510 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:49.533 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:49.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:49.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:49.624 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:49 smithi087 ceph-mon[89014]: pgmap v2285: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:49.624 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:49 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4245962063' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:57:49.624 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:49 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:57:49.624 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:49 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/20417638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:57:49.636 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:49.659 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:49.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:49.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:49.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:49 smithi029 ceph-mon[93650]: pgmap v2285: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:49 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4245962063' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:57:49.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:49 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:57:49.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:49 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/20417638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:57:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:49 smithi029 ceph-mon[97787]: pgmap v2285: 99 pgs: 99 active+clean; 634 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:49 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4245962063' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:57:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:49 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T11:57:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:49 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/20417638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:57:49.784 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:49.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:49.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:49.836 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:57:49.837 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:57:49.886 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:49.911 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:49.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:49.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:50.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:50.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:50.037 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:50.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:50.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:50.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:50.163 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:50.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:50.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:50.288 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:50.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:50.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:50.390 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:50.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:50.414 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:50.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:50.447 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-04-12T11:57:50.515 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:50.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:50 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:57:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:50 smithi029 ceph-mon[93650]: osdmap e1318: 8 total, 8 up, 6 in 2024-04-12T11:57:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:50 smithi029 ceph-mon[93650]: 1.3 scrub starts 2024-04-12T11:57:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:50 smithi029 ceph-mon[93650]: 1.3 scrub ok 2024-04-12T11:57:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:50 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:57:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:50 smithi029 ceph-mon[97787]: osdmap e1318: 8 total, 8 up, 6 in 2024-04-12T11:57:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:50 smithi029 ceph-mon[97787]: 1.3 scrub starts 2024-04-12T11:57:50.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:50 smithi029 ceph-mon[97787]: 1.3 scrub ok 2024-04-12T11:57:50.553 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:50.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:50.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:50.656 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:50.681 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:50.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:50.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:50.690 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:50 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-12T11:57:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:50 smithi087 ceph-mon[89014]: osdmap e1318: 8 total, 8 up, 6 in 2024-04-12T11:57:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:50 smithi087 ceph-mon[89014]: 1.3 scrub starts 2024-04-12T11:57:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:50 smithi087 ceph-mon[89014]: 1.3 scrub ok 2024-04-12T11:57:50.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:50.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:50.810 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:50.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:50.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:50.937 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:50.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:50.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:51.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:51.067 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:51.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:51.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:51.168 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:51.192 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:51.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:51.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:51.294 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:51.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:51.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:51.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:51.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:51.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:51.446 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:51.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:51.547 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:51.571 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:51.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:51.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:51.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:51.700 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:51.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:51.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:51 smithi087 ceph-mon[89014]: pgmap v2287: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:51 smithi087 ceph-mon[89014]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:57:51.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:51.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:51 smithi029 ceph-mon[97787]: pgmap v2287: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:51.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:51 smithi029 ceph-mon[97787]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:57:51.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:51 smithi029 ceph-mon[93650]: pgmap v2287: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:51 smithi029 ceph-mon[93650]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-12T11:57:51.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:57:51.802 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:51.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:51.827 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:51.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:51.927 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:51.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:51.951 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:51.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:52.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:57:52.052 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:52.076 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:52.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:52.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:52.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:52.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:52.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:52.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:52.209 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:57:52.209 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:57:52.302 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:52.326 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:52.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:52.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:52.428 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:52.451 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:52.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:52.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:52.467 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:57:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:57:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:57:52.553 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:52.578 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:52.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:52.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:52.705 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:52.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:52.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:52.807 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:52.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:52.832 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:52.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:52.871 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-04-12T11:57:52.933 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:52.977 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:52.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:52.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:53.079 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:53.104 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:53.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:53.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:53.134 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:53.158 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2024-04-12T11:57:53.161 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-12T11:57:53.161 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-12T11:57:53.207 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:53.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2024-04-12T11:57:53.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-12T11:57:53.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_285290 2024-04-12T11:57:53.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:57:53.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2024-04-12T11:57:53.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:53 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:53.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:53 smithi087 ceph-mon[89014]: pgmap v2288: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:53.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:53.247 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:53.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:53.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:53 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:53.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:53 smithi029 ceph-mon[93650]: pgmap v2288: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:53.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:53 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:57:53.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:53 smithi029 ceph-mon[97787]: pgmap v2288: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:53.348 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:53.386 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:53.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:53.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:53.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:53.533 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:53.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:53.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:53.635 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:53.671 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:53.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:53.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:53.773 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:53.799 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:53.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:53.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:53.900 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:53.923 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:53.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:53.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:54.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:54.056 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:54.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:54.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:54.158 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:54.173 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:57:54.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:54 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3470882130' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:57:54.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:54 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:57:54.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:54 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3470882130' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:57:54.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:54 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:57:54.202 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:54.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:54.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:54.208 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 2402 2386 149.116 149.125 0.00429406 0.00668166 2024-04-12T11:57:54.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:54.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:54.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:54.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:54.444 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:54.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:54 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3470882130' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:57:54.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:54 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:57:54.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:54.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:54.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:54.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:54.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:54.626 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:54.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:54.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:54.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:54.765 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:54.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:54.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:54.906 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:54.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:55.010 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:55.049 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:55.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:55.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:55.121 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:57:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:57:54] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.43.0" 2024-04-12T11:57:55.150 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:55.185 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:55.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:55.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:55.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 5654 5638 176.174 203.25 0.00557972 0.00566518 2024-04-12T11:57:55.287 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:no valid command found; 10 closest matches: 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:0 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:1 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:2 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:abort 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:assert 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:bench [] [] [] [] 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs debug_inject_read_zeros 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs files list 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:bluefs stats 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:bluestore allocator dump block 2024-04-12T11:57:55.303 INFO:teuthology.orchestra.run.smithi029.stderr:Error EINVAL: invalid command 2024-04-12T11:57:55.363 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:55.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:55.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:55.383 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:55 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:57:55.383 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:55 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:57:55.383 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:55 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:57:55.383 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:55 smithi087 ceph-mon[89014]: osdmap e1319: 8 total, 8 up, 6 in 2024-04-12T11:57:55.383 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:55 smithi087 ceph-mon[89014]: pgmap v2290: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:55.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:55.508 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:55.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:55.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:55.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:55 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:57:55.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:55 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:57:55.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:55 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:57:55.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:55 smithi029 ceph-mon[97787]: osdmap e1319: 8 total, 8 up, 6 in 2024-04-12T11:57:55.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:55 smithi029 ceph-mon[97787]: pgmap v2290: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:55.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:55 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:57:55.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:55 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:57:55.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:55 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:57:55.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:55 smithi029 ceph-mon[93650]: osdmap e1319: 8 total, 8 up, 6 in 2024-04-12T11:57:55.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:55 smithi029 ceph-mon[93650]: pgmap v2290: 99 pgs: 99 active+clean; 582 KiB data, 4.9 GiB used, 532 GiB / 536 GiB avail 2024-04-12T11:57:55.597 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:57:55.597 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:57:55.610 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:55.646 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:55.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:55.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:55.748 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:55.781 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:55.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:55.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:55.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:57:55.921 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:55.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:55.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:57:56.022 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:56.067 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:56.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:56.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:56.168 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:56.200 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:56.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:56.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:56.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 8824 8808 183.485 198.125 0.0044278 0.00544199 2024-04-12T11:57:56.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:56.319 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-04-12T11:57:56.341 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:56.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:56.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:56.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:56.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:56.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:56.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:56.581 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:56.619 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:56.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:56.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:56.664 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:57:56.721 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:56.736 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-12T11:57:56.736 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-04-12T11:57:56.749 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:56.763 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:56.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:56.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:56.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:56.905 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:56.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:56.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:57.007 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:57.040 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:57.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:57.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:57.142 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:57.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:57:57.180 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:57.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:57.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:57.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 11876 11860 185.297 190.75 0.00194451 0.00538646 2024-04-12T11:57:57.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:57:57.325 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:57.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:57.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:57:57.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:57.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:57 smithi087 ceph-mon[89014]: pgmap v2291: 99 pgs: 99 active+clean; 166 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 21 MiB/s wr, 328 op/s 2024-04-12T11:57:57.472 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:57.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:57.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:57 smithi029 ceph-mon[97787]: pgmap v2291: 99 pgs: 99 active+clean; 166 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 21 MiB/s wr, 328 op/s 2024-04-12T11:57:57.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:57 smithi029 ceph-mon[93650]: pgmap v2291: 99 pgs: 99 active+clean; 166 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 21 MiB/s wr, 328 op/s 2024-04-12T11:57:57.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:57.611 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:57.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:57.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:57.713 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:57.737 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:57.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:57.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:57.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:57.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:57.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:57.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:58.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:58.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:58.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:58.114 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:58.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:58.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:58.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:58.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 14221 14205 177.547 146.562 0.00477218 0.00562533 2024-04-12T11:57:58.253 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:58.315 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:58.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:58.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:58.417 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:58.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:58.454 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:58.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:58.470 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:57:58.545 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T11:57:58.546 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:57:58.555 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:58.596 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:58.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:58.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:58.698 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:58.734 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:58.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:58.836 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:57:58.869 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:58.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:58.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:57:58.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:57:59.012 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:59.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:57:59.114 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:59.155 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:59.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:59.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:59.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 16874 16858 175.589 165.812 0.0036593 0.00568844 2024-04-12T11:57:59.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-04-12T11:57:59.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:59.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:59.315 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:59.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:59.377 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:57:59.386 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:57:59 smithi029 ceph-mon[93650]: pgmap v2292: 99 pgs: 99 active+clean; 419 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 48 MiB/s wr, 768 op/s 2024-04-12T11:57:59.387 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:57:59 smithi029 ceph-mon[97787]: pgmap v2292: 99 pgs: 99 active+clean; 419 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 48 MiB/s wr, 768 op/s 2024-04-12T11:57:59.416 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:57:59.419 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:57:59.448 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:59.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:57:59.462 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:57:59.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:57:59 smithi087 ceph-mon[89014]: pgmap v2292: 99 pgs: 99 active+clean; 419 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 48 MiB/s wr, 768 op/s 2024-04-12T11:57:59.502 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:57:59.545 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:57:59.550 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:57:59.587 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:57:59.592 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:59.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:59.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:57:59.626 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:57:59.668 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:57:59.673 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:57:59.674 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:57:59.674 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:57:59.674 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:57:59.674 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:57:59.674 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:57:59.674 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:57:59.674 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:57:59.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:57:59.738 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:57:59.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:59.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:57:59.840 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:57:59.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:57:59.873 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:57:59.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:57:59.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:00.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:00.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:00.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:00.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:00.153 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:00.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:00.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:00.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 19274 19258 171.931 150 0.00644229 0.00581018 2024-04-12T11:58:00.256 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:00.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:00.295 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:00.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:00.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:00.433 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:00.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:00.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:00 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2679568709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:58:00.535 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:00 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2679568709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:58:00.536 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:00.574 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:00.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:00.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:00.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:00.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:00 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2679568709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:58:00.718 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:00.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:00.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:00.820 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:00.851 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:00.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:00.953 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:00.994 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:00.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:00.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:01.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:01.135 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:01.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:01.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:01.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 21981 21965 171.587 169.188 0.00434109 0.00582167 2024-04-12T11:58:01.237 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:01.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:01.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:01.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:01.409 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:01.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:01.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:01.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:01.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:01 smithi029 ceph-mon[93650]: pgmap v2293: 99 pgs: 99 active+clean; 1019 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-12T11:58:01.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:01 smithi029 ceph-mon[97787]: pgmap v2293: 99 pgs: 99 active+clean; 1019 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-12T11:58:01.549 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:01.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:01.654 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:01.690 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:01.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:01.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:01 smithi087 ceph-mon[89014]: pgmap v2293: 99 pgs: 99 active+clean; 1019 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-12T11:58:01.737 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 6, 7, 3] out_osds: [5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:58:01.738 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:58:01.738 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-04-12T11:58:01.738 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-04-12T11:58:01.738 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-04-12T11:58:01.738 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-04-12T11:58:01.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:01.835 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:01.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:01.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:01.936 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:01.977 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:01.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:01.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:02.041 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:58:02.080 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:02.083 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:58:02.122 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:02.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:02.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:02.209 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 24525 24509 170.187 159 0.00479055 0.00587062 2024-04-12T11:58:02.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:02.255 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:02.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:02.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:02.357 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:02.382 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:02.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:02.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:02.479 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:58:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:58:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:58:02.484 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:02.522 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:02.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:02.624 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:02.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:02.665 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:02.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:02.766 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:02.802 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:02.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:02.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:02.904 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:02.932 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:02.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:02.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:03.034 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:03.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:03.069 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:03.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:03.170 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:03.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 26230 26214 163.82 106.562 0.00610206 0.0060983 2024-04-12T11:58:03.211 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:03.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:03.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:03.313 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:03.337 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:03.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:03.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:03.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:03.481 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:03.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:03.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:03 smithi029 ceph-mon[97787]: pgmap v2294: 99 pgs: 99 active+clean; 1019 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-12T11:58:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:03 smithi029 ceph-mon[93650]: pgmap v2294: 99 pgs: 99 active+clean; 1019 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-12T11:58:03.527 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:03 smithi087 ceph-mon[89014]: pgmap v2294: 99 pgs: 99 active+clean; 1019 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-12T11:58:03.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:03.619 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:03.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:03.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:03.721 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:03.764 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:03.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:03.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:03.789 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:58:03.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:03.893 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:03.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:03.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:03.945 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:58:03.945 INFO:teuthology.orchestra.run.smithi087.stdout: "success": "" 2024-04-12T11:58:03.946 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:58:03.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:04.037 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:04.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:04.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:04.139 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:04.163 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:04.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:04.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:04.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 27508 27492 156.188 79.875 0.00343918 0.00639648 2024-04-12T11:58:04.265 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:04.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:04.305 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:04.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:04.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:04.444 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:04.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:04.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:04.546 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:04.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:04.570 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:04.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:04.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:04.700 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:04.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:04.801 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:04.837 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:04.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:04.938 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:04.962 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:04.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:04.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:05.064 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:05.087 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:05.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:05.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:05.190 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:05.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 28836 28820 150.088 83 0.0062419 0.00665552 2024-04-12T11:58:05.236 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:05.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:05.265 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:58:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:58:04] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.43.0" 2024-04-12T11:58:05.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:05.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:05.380 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:05.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:05 smithi087 ceph-mon[89014]: pgmap v2295: 99 pgs: 99 active+clean; 1.2 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 125 MiB/s wr, 2.01k op/s 2024-04-12T11:58:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:05 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:58:05.481 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:05.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:05 smithi029 ceph-mon[93650]: pgmap v2295: 99 pgs: 99 active+clean; 1.2 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 125 MiB/s wr, 2.01k op/s 2024-04-12T11:58:05.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:05 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:58:05.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:05 smithi029 ceph-mon[97787]: pgmap v2295: 99 pgs: 99 active+clean; 1.2 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 125 MiB/s wr, 2.01k op/s 2024-04-12T11:58:05.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:05 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T11:58:05.520 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:05.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:05.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:05.622 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:05.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:05.651 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:05.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:05.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:05.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:05.784 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:05.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:05.885 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:05.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:05.950 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:05.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:06.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:06.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:06.086 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:06.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:06.188 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:06.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 30344 30328 145.792 94.25 0.00163038 0.00681495 2024-04-12T11:58:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:06.213 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:06.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:06.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:06.347 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:06.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:06.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:06.450 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:06.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:06.481 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:06.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:06.582 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:06.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:06.616 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:06.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:06.717 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:06.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:06.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:06.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:06.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:06.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:06.896 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:06.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:06.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:07.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:07.024 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:07.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:07.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:07.149 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:07.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:07.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 31630 31614 141.119 80.375 0.00202979 0.00705311 2024-04-12T11:58:07.251 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:07.278 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:07.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:07.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:07.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:07.419 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:07.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:07.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:07.521 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:07.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:07.554 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:07.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:07.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:07.679 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:07.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:07.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:07 smithi087 ceph-mon[89014]: pgmap v2296: 99 pgs: 99 active+clean; 1.7 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-04-12T11:58:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:07 smithi029 ceph-mon[93650]: pgmap v2296: 99 pgs: 99 active+clean; 1.7 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-04-12T11:58:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:07 smithi029 ceph-mon[97787]: pgmap v2296: 99 pgs: 99 active+clean; 1.7 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-04-12T11:58:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:07.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:07.806 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:07.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:07.907 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:07.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:07.941 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:07.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:08.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:08.086 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:08.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:08.187 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:08.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 32778 32762 136.494 71.75 0.117504 0.00729035 2024-04-12T11:58:08.213 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:08.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:08.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:08.361 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:08.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:08.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:08.488 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:08.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:08.589 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:08.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:08.626 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:08.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:08.727 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:08.761 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:08.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:08.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:08.863 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:08.902 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:08.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:08.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:09.004 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:09.041 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:09.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:09.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:09.143 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:09.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:09.183 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:09.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:09.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 34491 34475 134.654 107.062 0.00242547 0.00741461 2024-04-12T11:58:09.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:09.319 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:09.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:09.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:09.421 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:09.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:09 smithi029 ceph-mon[97787]: pgmap v2297: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-04-12T11:58:09.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1090118039' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:58:09.435 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:09 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:58:09.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:09 smithi029 ceph-mon[93650]: pgmap v2297: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-04-12T11:58:09.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1090118039' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:58:09.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:09 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:58:09.467 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:09 smithi087 ceph-mon[89014]: pgmap v2297: 99 pgs: 99 active+clean; 1.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-04-12T11:58:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1090118039' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:58:09.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:09 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:58:09.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:09.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:09.569 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:09.617 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:09.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:09.719 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:09.748 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:09.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:09.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:09.769 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 6, 7, 3] out_osds: [5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:58:09.769 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:58:09.770 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-04-12T11:58:09.770 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.644502 2024-04-12T11:58:09.770 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 4 0.6445018906067241 2024-04-12T11:58:09.850 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:09.895 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:09.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:09.997 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:10.032 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:10.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:10.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:10.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:10.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:10.159 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:10.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:10.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 35910 35894 131.95 88.6875 0.0025683 0.00754285 2024-04-12T11:58:10.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:10.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:10.284 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:10.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:10.385 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:10.428 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:10.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:10.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:10.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:10 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:58:10.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:10 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:58:10.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:10 smithi087 ceph-mon[89014]: osdmap e1320: 8 total, 8 up, 6 in 2024-04-12T11:58:10.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:10 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:58:10.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:10 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:58:10.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:10 smithi029 ceph-mon[97787]: osdmap e1320: 8 total, 8 up, 6 in 2024-04-12T11:58:10.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:10 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:58:10.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:10 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:58:10.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:10 smithi029 ceph-mon[93650]: osdmap e1320: 8 total, 8 up, 6 in 2024-04-12T11:58:10.530 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:10.569 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:10.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:10.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:10.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:10.708 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:10.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:10.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:10.849 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:10.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:10.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:10.951 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:10.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:10.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:10.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:11.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:11.102 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:58:11.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:11.126 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:11.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:11.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 37628 37612 130.584 107.375 0.00252232 0.00763346 2024-04-12T11:58:11.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:11.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:11.260 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:11.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:11.361 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:11.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:11.386 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:11.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:11.452 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:11 smithi087 ceph-mon[89014]: pgmap v2299: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:58:11.487 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:11.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:11 smithi029 ceph-mon[93650]: pgmap v2299: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:58:11.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:11 smithi029 ceph-mon[97787]: pgmap v2299: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:58:11.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:11.528 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:11.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:11.630 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:11.668 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:11.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:11.770 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:11.795 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:11.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:11.897 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:11.922 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:11.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:12.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:12.060 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:12.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:12.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:12.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:12.193 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:12.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:12.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:12.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 38756 38740 127.421 70.5 0.0102199 0.00784285 2024-04-12T11:58:12.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:12.332 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:12.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:12.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:12.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:12.457 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:12.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:12.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:12.558 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:12.597 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:12.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:12.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:12.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:58:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:58:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:58:12.736 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:12.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:12.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:12.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:12.874 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:12.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:12.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:13.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:13.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:13.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:13.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:13.135 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:13.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:13.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:58:13.210549+0000 min lat: 0.00118232 max lat: 0.330018 avg lat: 0.00794406 2024-04-12T11:58:13.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:58:13.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 39967 39951 124.834 75.6875 0.00224372 0.00794406 2024-04-12T11:58:13.238 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:13.261 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:13.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:13.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:13.403 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:13.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:13.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:13.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:13 smithi029 ceph-mon[93650]: pgmap v2300: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:58:13.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:13 smithi029 ceph-mon[97787]: pgmap v2300: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:58:13.531 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:13.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:13.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:13.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:13.662 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:13.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:13 smithi087 ceph-mon[89014]: pgmap v2300: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-04-12T11:58:13.763 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:13.799 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:13.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:13.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:13.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:13.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:13.941 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:13.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:14.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:14.080 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:14.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:14.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:14.182 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:14.206 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:14.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:14.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 41262 41246 122.743 80.9375 0.00238616 0.00811316 2024-04-12T11:58:14.308 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:14.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:14.346 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:14.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:14.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:14.486 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:14.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:14.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:14.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:14.626 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:14.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:14.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:14.728 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:14.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:14.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:14.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:14.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:14 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:14 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:14.856 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:14.894 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:14.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:14.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:14.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:14.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:14 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:14.996 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:15.018 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:15.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:15.120 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:15.144 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:15.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:15.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:15.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 42324 42308 120.181 66.375 0.00457796 0.00828968 2024-04-12T11:58:15.245 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:15.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:58:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:58:14] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2024-04-12T11:58:15.270 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:15.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:15.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:15.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:15.395 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:15.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:15.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:15.497 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:15.521 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:15.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:15.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:15.623 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:15.655 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:15.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:15.756 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:15.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:15.781 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:15.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:15.882 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:15.920 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:15.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:15.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:15.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:15 smithi087 ceph-mon[89014]: pgmap v2301: 99 pgs: 99 active+clean; 2.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-04-12T11:58:16.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:15 smithi029 ceph-mon[97787]: pgmap v2301: 99 pgs: 99 active+clean; 2.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-04-12T11:58:16.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:15 smithi029 ceph-mon[93650]: pgmap v2301: 99 pgs: 99 active+clean; 2.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 103 MiB/s wr, 1.64k op/s 2024-04-12T11:58:16.022 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:16.056 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:16.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:16.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:16.158 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:16.194 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:16.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:16.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 43156 43140 117.216 52 0.0206327 0.00852299 2024-04-12T11:58:16.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:16.339 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:16.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:16.440 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:16.474 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:16.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:16.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:16.575 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:16.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:16.614 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:16.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:16.715 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:16.751 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:16.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:16.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:16.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:16.891 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:16.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:16.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:16.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:17.027 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:17.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:17.128 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:17.166 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:17.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:17.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:17.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 45097 45081 117.386 121.312 0.0109179 0.00851337 2024-04-12T11:58:17.269 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:17.306 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:17.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:17.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:17.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:17.449 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:17.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:17.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:17.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:17.585 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:17.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:17.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:17.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:17.737 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:17.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:17.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:17.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:17.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:17.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:17.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:17.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:17 smithi087 ceph-mon[89014]: pgmap v2302: 99 pgs: 99 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-04-12T11:58:17.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:17.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:17.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:18.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:17 smithi029 ceph-mon[93650]: pgmap v2302: 99 pgs: 99 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-04-12T11:58:18.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:17 smithi029 ceph-mon[97787]: pgmap v2302: 99 pgs: 99 active+clean; 2.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-04-12T11:58:18.088 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:18.125 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:18.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:18.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:18.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 46516 46500 116.238 88.6875 0.00407093 0.00859617 2024-04-12T11:58:18.231 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:18.269 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:18.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:18.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:18.370 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:18.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:18.406 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:18.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:18.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:18.545 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:18.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:18.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:18.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:18.685 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:18.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:18.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:18.786 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:18.821 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:18.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:18.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:18.946 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:18.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:19.048 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:19.081 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:19.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:19.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:19.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:19.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 47954 47938 115.223 89.875 0.00656953 0.00867327 2024-04-12T11:58:19.220 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:19.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:19.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:19.322 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:19.354 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:19.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:19.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:19.456 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:19.490 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:19.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:19.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:19.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:19.629 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:19.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:19.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:19.731 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:19.767 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:19.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:19.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:19.869 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:19.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:19.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:19.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:19.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:19 smithi087 ceph-mon[89014]: pgmap v2303: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T11:58:20.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:20.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:19 smithi029 ceph-mon[97787]: pgmap v2303: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T11:58:20.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:19 smithi029 ceph-mon[93650]: pgmap v2303: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T11:58:20.042 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:20.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:20.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:20.144 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:20.182 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:20.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:20.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:20.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 49777 49761 115.175 113.938 0.00724037 0.00867742 2024-04-12T11:58:20.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:20.324 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:20.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:20.426 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:20.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:20.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:20.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:20.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:20.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:20.600 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:20.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:20.635 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:58:20.701 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:20.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:20.739 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:20.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:20.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:20.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:20.880 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:20.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:20.981 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:21.011 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:21.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:21.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:21.113 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:21.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:21.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:21.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:21.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 51408 51392 114.702 101.938 0.0173024 0.00871323 2024-04-12T11:58:21.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:21.279 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:21.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:21.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:21.419 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:21.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:21.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:21.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:21.557 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:21.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:21.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:21.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:21.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:21.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:21.803 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:21.839 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:21.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:21.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:21.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:21 smithi087 ceph-mon[89014]: pgmap v2304: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-04-12T11:58:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:21.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:21.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:21.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:21.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:21 smithi029 ceph-mon[97787]: pgmap v2304: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-04-12T11:58:22.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:22.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:22.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:21 smithi029 ceph-mon[93650]: pgmap v2304: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-04-12T11:58:22.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:58:22.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:22.070 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:22.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:22.107 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:22.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:22.108 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:58:22.209 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:22.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 52917 52901 113.997 94.3125 0.00567377 0.00876536 2024-04-12T11:58:22.252 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:22.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:22.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:22.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:22.399 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:22.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:22.500 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:22.534 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:22.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:22.636 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:22.678 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:22.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:22.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:58:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:58:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:58:22.780 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:22.818 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:22.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:22.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:22.920 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:22.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:22.960 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:22.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:23.061 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:23.103 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:23.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:23.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:23.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:23.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 54553 54537 113.605 102.25 0.00944359 0.00879697 2024-04-12T11:58:23.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:23.247 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:23.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:23.348 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:23.382 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:23.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:23.484 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:23.521 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:23.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:23.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:23.593 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:23 smithi087 ceph-mon[89014]: pgmap v2305: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-04-12T11:58:23.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:23.626 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:58:23.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:23.676 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:23.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:23.721 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:58:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:58:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:55292 11.0 e1320/1314 11:0d05cbd9:::benchmark_data_smithi029_285290_object55291:head v 1320'3448, mlcod=1320'3446)", 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:58:23.711693+0000", 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0065572160000000003, 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065990320000000003, 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.711693+0000", 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.711693+0000", 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.711694+0000", 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.39e-07 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.711712+0000", 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8345000000000001e-05 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.711719+0000", 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.737e-06 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.711724+0000", 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0869999999999997e-06 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.714425+0000", 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027009439999999998 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.714445+0000", 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0332000000000001e-05 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:55289 11.2 e1320/1314 11:40dd51df:::benchmark_data_smithi029_285290_object55288:head v 1320'3462, mlcod=1320'3459)", 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:58:23.713118+0000", 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0051322370000000004, 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052481979999999999, 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713118+0000", 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713118+0000", 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713119+0000", 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2900000000000003e-07 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713144+0000", 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5622e-05 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713158+0000", 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3573e-05 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713165+0000", 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6159999999999998e-06 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.714881+0000", 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017159289999999999 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.714899+0000", 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8549000000000001e-05 2024-04-12T11:58:23.728 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:55294 11.2 e1320/1314 11:4d9121ca:::benchmark_data_smithi029_285290_object55293:head v 1320'3463, mlcod=1320'3459)", 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:58:23.713185+0000", 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0050657239999999997, 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052768479999999998, 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713185+0000", 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713185+0000", 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713185+0000", 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1199999999999999e-07 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713205+0000", 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9882e-05 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713211+0000", 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5370000000000003e-06 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.713216+0000", 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.045e-06 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:23.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.716806+0000", 2024-04-12T11:58:23.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035896859999999999 2024-04-12T11:58:23.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:23.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.716826+0000", 2024-04-12T11:58:23.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.033e-05 2024-04-12T11:58:23.732 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.732 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:55296 11.0 e1320/1314 11:057eaeaa:::benchmark_data_smithi029_285290_object55295:head v 1320'3449, mlcod=1320'3446)", 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:58:23.715695+0000", 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0025555999999999999, 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028398379999999999, 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:58:23.733 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:23.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.715695+0000", 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.715695+0000", 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.715696+0000", 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4e-07 2024-04-12T11:58:23.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.715721+0000", 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4984999999999999e-05 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.715730+0000", 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2019999999999993e-06 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.715737+0000", 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1799999999999999e-06 2024-04-12T11:58:23.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.716427+0000", 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00069017299999999998 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.716446+0000", 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9015e-05 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:55299 11.e e1320/1314 11:79b4ed64:::benchmark_data_smithi029_285290_object55298:head v 1320'3338, mlcod=1320'3336)", 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:58:23.717498+0000", 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.000752299, 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001100944, 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.717498+0000", 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.717498+0000", 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.717499+0000", 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8800000000000002e-07 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.717531+0000", 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2017999999999997e-05 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.717541+0000", 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8719999999999997e-06 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.717548+0000", 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.063e-06 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.717597+0000", 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9175000000000003e-05 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.717618+0000", 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1148000000000001e-05 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:55307 11.6 11:69d74091:::benchmark_data_smithi029_285290_object55306:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:58:23.718267+0000", 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 4294967295.9999838, 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00040185399999999999, 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:23.741 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 55307 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.718267+0000", 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.742 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.718267+0000", 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.718268+0000", 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3820000000000001e-06 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.718290+0000", 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1596000000000001e-05 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.743 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.718305+0000", 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5693e-05 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.718314+0000", 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4460000000000001e-06 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.718367+0000", 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3134000000000003e-05 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:23.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:23.745 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:23.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:23.718533+0000", 2024-04-12T11:58:23.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016588900000000001 2024-04-12T11:58:23.745 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.745 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:23.745 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.745 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:23.745 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2024-04-12T11:58:23.745 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 6 2024-04-12T11:58:23.745 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:58:23.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:23 smithi029 ceph-mon[97787]: pgmap v2305: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-04-12T11:58:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:23 smithi029 ceph-mon[93650]: pgmap v2305: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-04-12T11:58:23.776 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:23.814 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:23.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:23.916 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:23.957 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:23.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:24.059 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:24.097 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:24.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:24.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:24.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 56079 56063 113.017 95.375 0.003863 0.00884262 2024-04-12T11:58:24.225 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:24.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:24.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:24.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:24.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:24.361 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:24.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:24.462 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:24.497 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:24.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:24.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:24.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:24.631 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:24.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:24.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:24.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:24.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:24.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:24.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:24.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:24.910 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:24.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:25.012 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:25.046 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:25.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:25.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:25.148 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:25.172 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:25.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:25.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 57257 57241 111.786 73.625 0.00285587 0.00891793 2024-04-12T11:58:25.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:58:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:58:24] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.43.0" 2024-04-12T11:58:25.273 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:25.298 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:25.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:25.399 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:25.430 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:25.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:25.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:25.531 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:25.587 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:25.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:25.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:25.689 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:25.716 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:25 smithi029 ceph-mon[97787]: pgmap v2306: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-04-12T11:58:25.717 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:25 smithi029 ceph-mon[93650]: pgmap v2306: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-04-12T11:58:25.724 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:25.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:25.826 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:25.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:25.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:25.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:25.961 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:25.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:25 smithi087 ceph-mon[89014]: pgmap v2306: 99 pgs: 99 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-04-12T11:58:25.986 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:25.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:25.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:26.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:26.132 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:26.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:26.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:26.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 58317 58301 110.406 66.25 0.00649032 0.00905067 2024-04-12T11:58:26.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:26.278 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:26.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:26.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:26.379 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:26.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:26.404 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:26.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:26.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:26.528 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:26.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:26.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:26.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:26.666 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:26.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:26.768 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:26.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:26.793 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:26.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:26.894 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:26.930 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:26.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:26.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:27.032 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:27.081 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:27.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:27.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:27.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:27.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 59248 59232 108.87 58.1875 0.0123877 0.00917969 2024-04-12T11:58:27.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:27.224 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:27.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:27.322 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:58:27.324 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:27.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:27.363 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:27.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:27.363 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:58:27.404 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:58:27.447 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:58:27.463 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:27.490 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:58:27.499 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:27.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:27.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:27.530 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:58:27.573 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:58:27.574 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:27 smithi029 ceph-mon[97787]: pgmap v2307: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-04-12T11:58:27.574 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:27 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/974573241' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:58:27.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:27 smithi029 ceph-mon[93650]: pgmap v2307: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-04-12T11:58:27.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:27 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/974573241' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:58:27.601 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:27.611 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:58:27.614 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:58:27.615 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:58:27.615 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:58:27.615 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:58:27.615 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:58:27.615 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:58:27.615 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:58:27.615 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:58:27.640 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:27.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:27.742 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:27.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:27.781 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:27.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:27.882 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:27.916 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:27.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:27.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:27.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-04-12T11:58:27.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:27 smithi087 ceph-mon[89014]: pgmap v2307: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-04-12T11:58:27.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:27 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/974573241' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:58:28.018 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:28.051 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:28.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:28.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:28.152 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:28.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:28.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:28.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:28.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 60751 60735 108.443 93.9375 0.00605904 0.00921596 2024-04-12T11:58:28.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:28.326 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:28.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:28.428 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:28.464 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:28.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:28.566 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:28.601 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:28.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:28.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:28.703 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:28.741 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:28.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:28.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:28.842 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:28.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:28.877 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:28.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:28.978 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:29.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:29.020 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:29.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:29.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:29.154 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:29.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:29.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:29.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 62304 62288 108.127 97.0625 0.00935588 0.00924335 2024-04-12T11:58:29.256 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:29.290 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:29.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:29.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:29.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:29.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:29.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:29.540 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:29.576 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:29.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:29.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:29.679 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:29.722 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:29.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:29.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:29.824 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:29.863 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:29.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:29.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:29 smithi029 ceph-mon[97787]: pgmap v2308: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T11:58:29.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:29 smithi029 ceph-mon[93650]: pgmap v2308: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T11:58:29.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:29 smithi087 ceph-mon[89014]: pgmap v2308: 99 pgs: 99 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-04-12T11:58:29.999 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:30.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:30.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:30.101 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:30.136 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:30.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:30.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:30.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 63834 63818 107.789 95.625 0.0122938 0.00927269 2024-04-12T11:58:30.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:30.276 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:58:30.285 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:30.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:30.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:30.337 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:58:30.389 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:30.427 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:30.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:30.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:30.529 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:30.560 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:30.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:30.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:30.662 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:30.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:58:30.699 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:30.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:30.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:30.801 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:30.829 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:30.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:30.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:30.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:30.972 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:30.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:31.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:31.108 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:31.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:31.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:31.210 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:31.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 65140 65124 107.1 81.625 0.0128811 0.00933158 2024-04-12T11:58:31.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:31.250 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:31.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:31.351 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:31.386 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:31.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:31.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:31.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:31.540 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:31.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:31.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:31.679 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:31.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:31.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:31.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:31.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:31.822 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:31.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:31.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:31.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:31.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:31.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:31.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:31 smithi087 ceph-mon[89014]: pgmap v2309: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-12T11:58:32.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:31 smithi029 ceph-mon[93650]: pgmap v2309: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-12T11:58:32.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:31 smithi029 ceph-mon[97787]: pgmap v2309: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-04-12T11:58:32.060 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:32.083 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:32.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:32.184 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:32.208 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:32.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:32.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:32.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 66361 66345 106.31 76.3125 0.00280457 0.00937169 2024-04-12T11:58:32.310 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:32.351 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:32.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:32.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:32.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:32.476 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:32.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:32.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:32.566 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:32 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/200634198' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6445018906067241}]: dispatch 2024-04-12T11:58:32.566 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:58:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:58:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:58:32.578 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:32.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:32.605 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:32.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:32.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:32.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:32.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:32.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:32.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:32 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/200634198' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6445018906067241}]: dispatch 2024-04-12T11:58:32.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:32 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6445018906067241}]: dispatch 2024-04-12T11:58:32.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:32 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/200634198' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6445018906067241}]: dispatch 2024-04-12T11:58:32.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:32 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6445018906067241}]: dispatch 2024-04-12T11:58:32.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:32.884 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:32.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:32.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:32 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6445018906067241}]: dispatch 2024-04-12T11:58:32.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:33.009 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:33.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:33.111 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:33.141 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:33.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:33.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:33.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:58:33.212866+0000 min lat: 0.00118232 max lat: 0.578447 avg lat: 0.00952739 2024-04-12T11:58:33.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:58:33.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 67161 67145 104.903 50 0.00669519 0.00952739 2024-04-12T11:58:33.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:33.284 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:33.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:33.386 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:33.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:33.427 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:33.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:33.528 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:33.551 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:33.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:33.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:33.653 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:33.693 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:33.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:33.795 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:33.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:33.832 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:33.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:33.933 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:33.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:33 smithi087 ceph-mon[89014]: pgmap v2310: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-12T11:58:33.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:33 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6445018906067241}]': finished 2024-04-12T11:58:33.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:33 smithi087 ceph-mon[89014]: osdmap e1321: 8 total, 8 up, 6 in 2024-04-12T11:58:33.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:33.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:33.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:34.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:33 smithi029 ceph-mon[97787]: pgmap v2310: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-12T11:58:34.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:33 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6445018906067241}]': finished 2024-04-12T11:58:34.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:33 smithi029 ceph-mon[97787]: osdmap e1321: 8 total, 8 up, 6 in 2024-04-12T11:58:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:33 smithi029 ceph-mon[93650]: pgmap v2310: 99 pgs: 99 active+clean; 3.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-04-12T11:58:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:33 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6445018906067241}]': finished 2024-04-12T11:58:34.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:33 smithi029 ceph-mon[93650]: osdmap e1321: 8 total, 8 up, 6 in 2024-04-12T11:58:34.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:34.094 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:34.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:34.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:34.196 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:34.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 68312 68296 104.098 71.9375 0.0104204 0.00960113 2024-04-12T11:58:34.234 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:34.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:34.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:34.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:34.373 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:34.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:34.474 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:34.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:34.515 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:34.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:34.616 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:34.639 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:34.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:34.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:34.741 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:34.779 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:34.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:34.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:34.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:34.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:34 smithi029 ceph-mon[93650]: osdmap e1322: 8 total, 8 up, 6 in 2024-04-12T11:58:34.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:34 smithi029 ceph-mon[93650]: pgmap v2313: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-12T11:58:34.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:34 smithi029 ceph-mon[97787]: osdmap e1322: 8 total, 8 up, 6 in 2024-04-12T11:58:34.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:34 smithi029 ceph-mon[97787]: pgmap v2313: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-12T11:58:34.929 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:34.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:34.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:34 smithi087 ceph-mon[89014]: osdmap e1322: 8 total, 8 up, 6 in 2024-04-12T11:58:34.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:34 smithi087 ceph-mon[89014]: pgmap v2313: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-12T11:58:35.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:35.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:35.071 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:35.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:35.172 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:35.197 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:35.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:35.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 69338 69322 103.146 64.125 0.00323383 0.00965806 2024-04-12T11:58:35.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:58:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:58:34] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.43.0" 2024-04-12T11:58:35.299 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:35.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:35.338 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:35.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:35.439 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:35.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:35.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:35.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:35.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:35.622 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:35.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:35.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:35.724 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:35.750 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:35.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:35.852 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:35.897 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:35.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:35.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:36.000 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:36.035 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:36.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:36.137 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:36.160 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:36.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:36.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:36.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 70556 70540 102.518 76.125 0.00373019 0.00973183 2024-04-12T11:58:36.263 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:36.292 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:36.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:36.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:36.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:36.432 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:36.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:36.533 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:36.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:36.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:36.669 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:36.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:36.716 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:36.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:36.817 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:36.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:36.864 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:36.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:36.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:37.013 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:37.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:37.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:37.167 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:37.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:37.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:37.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 71811 71795 101.971 78.4375 0.00793401 0.00980156 2024-04-12T11:58:37.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:37.315 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:37.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:37.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:37.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:37.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:37.465 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:37.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:37 smithi029 ceph-mon[93650]: pgmap v2314: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-12T11:58:37.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:37.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:37 smithi029 ceph-mon[97787]: pgmap v2314: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-12T11:58:37.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:37.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:37.603 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:37.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:37.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:37.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:37 smithi087 ceph-mon[89014]: pgmap v2314: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-12T11:58:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:37.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:37.748 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:37.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:37.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:37.887 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:37.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:37.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:37.989 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:38.025 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:38.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:38.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:38.126 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:38.162 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:38.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:38.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:38.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 73204 73188 101.639 87.0625 0.00855362 0.00983393 2024-04-12T11:58:38.246 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.4 primary-affinity to 0.644502 (8422382) 2024-04-12T11:58:38.263 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:38.300 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:38.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:38.401 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:38.446 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:38.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:38.551 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:38.588 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:38.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:38.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:38.690 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:38.725 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:38.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:38.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:38.827 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:38.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:38.866 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:38.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:38.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:39.005 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:39.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:39.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:39.107 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:39.140 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:39.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:39.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:39.214 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 74602 74586 101.329 87.375 0.00610318 0.00986376 2024-04-12T11:58:39.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:39.282 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:39.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:39.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:39.384 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:39.424 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:39.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:39.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:39.513 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:39 smithi029 ceph-mon[97787]: pgmap v2315: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 62 MiB/s wr, 991 op/s 2024-04-12T11:58:39.513 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:39 smithi029 ceph-mon[93650]: pgmap v2315: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 62 MiB/s wr, 991 op/s 2024-04-12T11:58:39.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:39.562 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:39.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:39.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:39.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:39 smithi087 ceph-mon[89014]: pgmap v2315: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 62 MiB/s wr, 991 op/s 2024-04-12T11:58:39.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:39.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:39.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:39.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:39.855 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:39.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:39.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:39.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:39.995 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:39.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:39.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:40.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:40.132 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:40.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:40.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:40.144 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:58:40.215 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 75916 75900 100.916 82.125 0.0136916 0.00990374 2024-04-12T11:58:40.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:40.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:40.286 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:40.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:40.387 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:40.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:40.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:40.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:40.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:40.556 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:40.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:40.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:40.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:40.701 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:40.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:40.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:40.803 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:40.841 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:40.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:40.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:40.943 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:40.980 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:40.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:41.009 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:58:41.009 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:58:41.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:41.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:41.124 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:41.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:41.215 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 77110 77094 100.369 74.625 0.016104 0.00995837 2024-04-12T11:58:41.225 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:41.257 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:41.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:41.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:41.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:41 smithi029 ceph-mon[97787]: pgmap v2316: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-04-12T11:58:41.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:41 smithi029 ceph-mon[93650]: pgmap v2316: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-04-12T11:58:41.359 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:41.388 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:41.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:41.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:41.490 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:41.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:41.522 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:41.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:41.623 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:41.657 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:41.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:41.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:41 smithi087 ceph-mon[89014]: pgmap v2316: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-04-12T11:58:41.759 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:41.794 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:41.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:41.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:41.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:41.931 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:41.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:41.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:41.960 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-04-12T11:58:42.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:42.064 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:42.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:42.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:42.166 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:42.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:42.205 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:42.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:42.215 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 78345 78329 99.8954 77.1875 0.0167958 0.0100056 2024-04-12T11:58:42.294 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T11:58:42.306 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:42.361 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:42.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:42.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:42.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:42.503 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:42.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:42.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:42.525 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:42 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1523252686' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:58:42.525 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:42 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:58:42.525 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:42 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1523252686' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:58:42.525 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:42 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:58:42.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:42.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:42.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:42.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:58:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:58:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:58:42.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:42 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1523252686' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:58:42.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:42 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:58:42.750 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:42.793 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:42.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:42.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:42.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:42.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:42.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:42.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:43.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:43.075 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:43.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:43.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:43.216 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 79609 79593 99.4774 79 0.0141815 0.0100473 2024-04-12T11:58:43.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:43.217 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:43.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:43.319 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:43.350 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:43.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:43.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:43.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:43.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:43.502 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:43.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:43.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:43.640 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:43.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:43.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:43 smithi029 ceph-mon[97787]: pgmap v2317: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:58:43.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:43 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:58:43.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:43 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:58:43.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:43 smithi029 ceph-mon[97787]: osdmap e1323: 8 total, 8 up, 6 in 2024-04-12T11:58:43.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:43 smithi029 ceph-mon[93650]: pgmap v2317: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:58:43.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:43 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:58:43.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:43 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:58:43.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:43 smithi029 ceph-mon[93650]: osdmap e1323: 8 total, 8 up, 6 in 2024-04-12T11:58:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:43 smithi087 ceph-mon[89014]: pgmap v2317: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:58:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:43 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:58:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:43 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T11:58:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:43 smithi087 ceph-mon[89014]: osdmap e1323: 8 total, 8 up, 6 in 2024-04-12T11:58:43.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:43.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:43.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:43.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:43.871 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:43.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:43.910 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:43.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:44.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:44.039 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:44.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:44.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:44.117 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 6, 7, 3] out_osds: [5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:58:44.117 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:58:44.117 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-04-12T11:58:44.118 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.140335 2024-04-12T11:58:44.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 3 0.14033477962851815 2024-04-12T11:58:44.142 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:44.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:44.174 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:44.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:44.216 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 80859 80843 99.0587 78.125 0.0159831 0.0100895 2024-04-12T11:58:44.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:44.327 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:44.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:44.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:44.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:44.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:44.470 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:44.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:44.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:44.610 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:44.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:44.711 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:44.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:44.736 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:44.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:44.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:44.876 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:44.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:44.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:44.978 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:45.033 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:45.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:45.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:45.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:45.163 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:45.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:45.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:45.216 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 81921 81905 98.4301 66.375 0.00963846 0.0101545 2024-04-12T11:58:45.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:45.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:58:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:58:44] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.43.0" 2024-04-12T11:58:45.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:45.302 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:45.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:45.403 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:45.439 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:45.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:45.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:45.541 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:45.577 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:45.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:45.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:45.679 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:45 smithi087 ceph-mon[89014]: pgmap v2319: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-12T11:58:45.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:45.727 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:45.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:45 smithi029 ceph-mon[93650]: pgmap v2319: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-12T11:58:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:45 smithi029 ceph-mon[97787]: pgmap v2319: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-12T11:58:45.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:45.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:45.853 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:45.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:45.954 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:45.994 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:45.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:45.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:46.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:46.119 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:46.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:46.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:46.216 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 83000 82984 97.8452 67.4375 0.0122505 0.0102154 2024-04-12T11:58:46.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:46.261 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:46.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:46.363 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:46.401 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:46.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:46.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:46.503 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:46.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:46.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:46.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:46.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:46.695 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:46.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:46.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:46.824 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:46.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:46.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:46.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:46.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:46.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:47.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:47.113 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:47.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:47.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:47.215 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:47.216 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 84123 84107 97.333 70.1875 0.0036971 0.0102665 2024-04-12T11:58:47.238 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:47.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:47.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:47.340 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:47.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:47.379 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:47.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:47.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:47.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:47.517 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:47.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:47.618 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:47.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:47.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:47.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:47.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:47 smithi087 ceph-mon[89014]: pgmap v2320: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-12T11:58:47.745 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:47.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:47 smithi029 ceph-mon[93650]: pgmap v2320: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-12T11:58:47.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:47 smithi029 ceph-mon[97787]: pgmap v2320: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-12T11:58:47.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:47.771 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:47.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:47.874 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:47.910 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:47.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:47.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:48.012 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:48.048 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:48.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:48.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:48.150 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:48.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:48.187 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:48.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:48.216 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 84929 84913 96.4791 50.375 0.0134272 0.0103599 2024-04-12T11:58:48.287 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:48.324 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:48.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:48.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:48.426 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:48.460 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:48.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:48.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:48.562 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:48.595 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:48.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:48.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:48.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T11:58:48.697 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:48.729 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:48.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:48.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:48.832 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:48.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:48.899 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:48.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:49.000 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:49.032 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:49.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:49.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:49.134 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:49.169 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:49.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:49.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:49.217 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 86033 86017 95.9875 69 0.0228901 0.0104129 2024-04-12T11:58:49.270 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:49.311 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:49.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:49.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:49.413 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:49.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:49.443 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:49.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:49.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:49 smithi029 ceph-mon[97787]: pgmap v2321: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-12T11:58:49.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:49 smithi029 ceph-mon[93650]: pgmap v2321: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-12T11:58:49.544 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:49.567 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:58:49.579 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:49.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:49.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:49.604 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:58:49.638 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:58:49.680 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:58:49.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:49.716 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:58:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:49 smithi087 ceph-mon[89014]: pgmap v2321: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-12T11:58:49.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:49.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:49.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:49.754 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:58:49.795 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:58:49.823 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:49.834 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:58:49.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:58:49.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:58:49.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:58:49.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:58:49.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:58:49.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:58:49.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:58:49.838 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:58:49.862 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:49.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:49.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:49.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:49.996 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:49.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:49.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:50.098 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:50.127 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:58:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:50.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:50.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:50.217 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 87202 87186 95.5839 73.0625 0.01068 0.0104568 2024-04-12T11:58:50.236 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:50.276 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:50.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:50.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:50.377 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:50.415 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:50.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:50.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:50.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:50.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:50 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/911564137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:58:50.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:50 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/911564137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:58:50.555 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:50.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:50.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:50.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:50.693 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:50.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:50.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:50 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/911564137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:58:50.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:50.832 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:50.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:50.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:50.933 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:50.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:50.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:50.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:51.073 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:51.122 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:51.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:51.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:51.217 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 88436 88420 95.2656 77.125 0.025922 0.010492 2024-04-12T11:58:51.224 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:51.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:51.261 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:51.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:51.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:51.400 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:51.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:51.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:51.502 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:51.533 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:51.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:51.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:51.559 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:51 smithi029 ceph-mon[93650]: pgmap v2322: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-12T11:58:51.559 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:51 smithi029 ceph-mon[97787]: pgmap v2322: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-12T11:58:51.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:51.675 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:51.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:51 smithi087 ceph-mon[89014]: pgmap v2322: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-12T11:58:51.776 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:51.809 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:51.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:51.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:51.911 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:51.944 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:51.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:51.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:52.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:52.085 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:52.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:52.187 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:52.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 89649 89633 94.9346 75.8125 0.0124677 0.0105288 2024-04-12T11:58:52.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:52.219 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:52.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:52.320 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:52.358 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:52.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:52.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:52 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:52.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:52 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:52.460 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:52.492 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:52.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:52.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:52.594 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:52.618 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:58:52.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:58:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:58:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:58:52.619 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:52 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:58:52.636 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:52.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:52.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:52.738 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:52.764 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:58:52.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:52.771 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:52.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:52.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:52.906 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:52.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:52.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:53.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:53.054 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:53.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:53.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:53.156 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:53.189 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:53.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:53.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:53.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:58:53.218295+0000 min lat: 0.00118232 max lat: 0.578447 avg lat: 0.0105676 2024-04-12T11:58:53.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:58:53.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 90830 90814 94.5824 73.8125 0.00612681 0.0105676 2024-04-12T11:58:53.292 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:53.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:53.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:53.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:53.420 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:53.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:53.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:53.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:53.558 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:53.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:53.594 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:53.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:53.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:53 smithi087 ceph-mon[89014]: pgmap v2323: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-12T11:58:53.727 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:53.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:53.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:53.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:53 smithi029 ceph-mon[97787]: pgmap v2323: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-12T11:58:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:53 smithi029 ceph-mon[93650]: pgmap v2323: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-12T11:58:53.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:53.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:53.867 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:53.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:53.968 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:54.002 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:54.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:54.104 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:54.138 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:54.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:54.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:54.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 91895 91879 94.1229 66.5625 0.00412969 0.010608 2024-04-12T11:58:54.240 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:54.268 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:54.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:54.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:54.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:54.406 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:54.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:54.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:54.508 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:54.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:54.550 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:54.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:54.651 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:54.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:54.675 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:54.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:54.776 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:54.808 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:54.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:54.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:54.910 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:54.934 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:54.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:54.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:55.036 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:55.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:55.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:55.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:55.174 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:55.206 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:55.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:55.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:55.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 92671 92655 93.3871 48.5 0.00642616 0.010703 2024-04-12T11:58:55.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:58:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:58:54] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.43.0" 2024-04-12T11:58:55.308 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:55.352 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:55.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:55.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:55.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:55.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:55.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:55.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:55.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:55.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:55.627 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:55.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:55 smithi087 ceph-mon[89014]: pgmap v2324: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-04-12T11:58:55.728 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:55.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:55.751 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:55.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:55.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:55 smithi029 ceph-mon[93650]: pgmap v2324: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-04-12T11:58:55.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:55 smithi029 ceph-mon[97787]: pgmap v2324: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-04-12T11:58:55.852 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:55.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:55.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:55.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:55.978 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:56.014 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:56.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:56.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:56.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:56.162 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:56.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:56.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:56.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 93381 93365 92.6091 44.375 0.00409816 0.0107921 2024-04-12T11:58:56.263 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:56.294 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:56.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:56.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:56.392 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:58:56.392 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T11:58:56.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T11:58:56.392 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:58:56.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.392 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28344 10.6 10:6e372a0a:::benchmark_data_smithi029_256330_object28343:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:58:56.392 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.636497+0000", 2024-04-12T11:58:56.392 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 325.05097676499997, 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.514864867, 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28344 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636497+0000", 2024-04-12T11:58:56.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636497+0000", 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636498+0000", 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.031e-06 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636529+0000", 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0688999999999997e-05 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636540+0000", 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0917000000000001e-05 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636547+0000", 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4250000000000001e-06 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640097+0000", 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035499939999999999 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640239+0000", 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000142113 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4", 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640380+0000", 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000141113 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.641764+0000", 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013837739999999999 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.150979+0000", 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50921531600000003 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151037+0000", 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7224e-05 2024-04-12T11:58:56.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151362+0000", 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032527100000000002 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28359 10.8 10:1699d62b:::benchmark_data_smithi029_256330_object28358:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.640282+0000", 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 325.04719137699999, 2024-04-12T11:58:56.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51126328099999996, 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28359 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640282+0000", 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640282+0000", 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640284+0000", 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2550000000000001e-06 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640310+0000", 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6261000000000001e-05 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640321+0000", 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1199999999999999e-05 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640330+0000", 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7010000000000002e-06 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640392+0000", 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2129000000000001e-05 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640525+0000", 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013261800000000001 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640649+0000", 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012463700000000001 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.641965+0000", 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001315433 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.642936+0000", 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00097142699999999997 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151453+0000", 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50851709099999998 2024-04-12T11:58:56.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151504+0000", 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0624000000000002e-05 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151546+0000", 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1904999999999999e-05 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.404 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28362 10.8 10:1e680b68:::benchmark_data_smithi029_256330_object28361:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.643369+0000", 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 325.044105118, 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51099498499999996, 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28362 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643369+0000", 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643369+0000", 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643369+0000", 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2399999999999998e-07 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643391+0000", 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1288999999999999e-05 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643399+0000", 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6379999999999993e-06 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643406+0000", 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7789999999999998e-06 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.645541+0000", 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002134698 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.645668+0000", 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000127813 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.645850+0000", 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018152399999999999 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152136+0000", 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50628628399999998 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152428+0000", 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000291236 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154207+0000", 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017794110000000001 2024-04-12T11:58:56.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154317+0000", 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110235 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154364+0000", 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6453999999999999e-05 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28367 10.8 10:172b0297:::benchmark_data_smithi029_256330_object28366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.643647+0000", 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 325.04382698000001, 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51514234299999995, 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28367 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643647+0000", 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643647+0000", 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643648+0000", 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6000000000000002e-07 2024-04-12T11:58:56.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643669+0000", 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1191000000000001e-05 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643679+0000", 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0607e-05 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643691+0000", 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1769e-05 2024-04-12T11:58:56.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152468+0000", 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50877640800000001 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152609+0000", 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000141058 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152760+0000", 2024-04-12T11:58:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015116700000000001 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158543+0000", 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0057828669999999997 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158586+0000", 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3633000000000001e-05 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158683+0000", 2024-04-12T11:58:56.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6847000000000003e-05 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158742+0000", 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8697999999999999e-05 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158789+0000", 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7237999999999997e-05 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28370 10.6 10:6134ad43:::benchmark_data_smithi029_256330_object28369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.644366+0000", 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 325.043107389, 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51561275900000003, 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28370 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644366+0000", 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644366+0000", 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644367+0000", 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1699999999999997e-07 2024-04-12T11:58:56.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644431+0000", 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3511999999999998e-05 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644443+0000", 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2517000000000001e-05 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644451+0000", 2024-04-12T11:58:56.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7989999999999994e-06 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151408+0000", 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50695730000000006 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151604+0000", 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019551200000000001 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4", 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151805+0000", 2024-04-12T11:58:56.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000201466 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.153704+0000", 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001898409 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.159834+0000", 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061304890000000003 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.159891+0000", 2024-04-12T11:58:56.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6725000000000003e-05 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.159979+0000", 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8213000000000004e-05 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28372 10.8 10:100b9aa0:::benchmark_data_smithi029_256330_object28371:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.644503+0000", 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 325.04297095999999, 2024-04-12T11:58:56.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.514388289, 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28372 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644503+0000", 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644503+0000", 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644504+0000", 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5600000000000005e-07 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644527+0000", 2024-04-12T11:58:56.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3509000000000001e-05 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644536+0000", 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0799999999999995e-06 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644543+0000", 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8029999999999999e-06 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154400+0000", 2024-04-12T11:58:56.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50985725800000004 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154537+0000", 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013657400000000001 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154983+0000", 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00044639700000000002 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158625+0000", 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036415060000000001 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158663+0000", 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8380999999999997e-05 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158800+0000", 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000137427 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158848+0000", 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7664000000000003e-05 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158891+0000", 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2933999999999999e-05 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:75508 10.8 10:12b3b9c8:::benchmark_data_smithi029_256330_object75507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:13.182801+0000", 2024-04-12T11:58:56.428 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 281.50467248699999, 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73462143300000005, 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75508 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182801+0000", 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182801+0000", 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182803+0000", 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.516e-06 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182825+0000", 2024-04-12T11:58:56.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.251e-05 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182838+0000", 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2595000000000001e-05 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182846+0000", 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3699999999999995e-06 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182937+0000", 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0354999999999998e-05 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.183087+0000", 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015046300000000001 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.183241+0000", 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000153988 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.432 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.188275+0000", 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050336510000000001 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.190168+0000", 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001893385 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.917315+0000", 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.72714688500000002 2024-04-12T11:58:56.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.917373+0000", 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7748999999999999e-05 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.917423+0000", 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9966000000000002e-05 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:75532 10.8 10:16d45b92:::benchmark_data_smithi029_256330_object75531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:13.208605+0000", 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 281.47886838099998, 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71678397999999999, 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75532 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208605+0000", 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208605+0000", 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208607+0000", 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5069999999999999e-06 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208629+0000", 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2036e-05 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208644+0000", 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4888e-05 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208653+0000", 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6050000000000004e-06 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208707+0000", 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3154e-05 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208821+0000", 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011414200000000001 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208925+0000", 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010425900000000001 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.213198+0000", 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042726819999999999 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.213287+0000", 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9775e-05 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925086+0000", 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71179846400000002 2024-04-12T11:58:56.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925163+0000", 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7373999999999995e-05 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925389+0000", 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022609400000000001 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:75534 10.8 10:1c9cded7:::benchmark_data_smithi029_256330_object75533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:13.209813+0000", 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 281.477661189, 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71577829400000004, 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75534 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209813+0000", 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209813+0000", 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209814+0000", 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0899999999999999e-06 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209860+0000", 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6703999999999999e-05 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209872+0000", 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1678999999999999e-05 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209880+0000", 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3990000000000004e-06 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.210963+0000", 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001083028 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.211126+0000", 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000162342 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.211246+0000", 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000120622 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.213352+0000", 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002105909 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.214394+0000", 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010415229999999999 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925433+0000", 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71103895299999997 2024-04-12T11:58:56.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925549+0000", 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115869 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925591+0000", 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2175999999999997e-05 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:76046 10.6 10:685cd136:::benchmark_data_smithi029_256330_object76045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:14.743676+0000", 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 279.94379770099999, 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36810407000000001, 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76046 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.447 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743676+0000", 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743676+0000", 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743677+0000", 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2929999999999999e-06 2024-04-12T11:58:56.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743699+0000", 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1786000000000001e-05 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743711+0000", 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1473e-05 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743718+0000", 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2599999999999999e-06 2024-04-12T11:58:56.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743792+0000", 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4210999999999998e-05 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.743968+0000", 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017591500000000001 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:58:56.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.744157+0000", 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018945600000000001 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.746515+0000", 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023574239999999999 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.754397+0000", 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078825560000000006 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.108850+0000", 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.35445250900000003 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.108912+0000", 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1970999999999993e-05 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.111780+0000", 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028682159999999998 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.452 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:76053 10.6 10:6bdceeb5:::benchmark_data_smithi029_256330_object76052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:14.750992+0000", 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 279.93648199799998, 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36981416700000003, 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.453 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 76053 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.750992+0000", 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.750992+0000", 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.750993+0000", 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4980000000000001e-06 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751017+0000", 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3402000000000001e-05 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751030+0000", 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2982999999999999e-05 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751037+0000", 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7160000000000003e-06 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751157+0000", 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011981000000000001 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751349+0000", 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000191807 2024-04-12T11:58:56.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.751649+0000", 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00029982300000000001 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.754558+0000", 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029093610000000001 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.756360+0000", 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001801932 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.118087+0000", 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36172679800000002 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.118158+0000", 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1054000000000001e-05 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.120806+0000", 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002647983 2024-04-12T11:58:56.458 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:155258 10.8 10:1358a27f:::benchmark_data_smithi029_256330_object53492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:07.224617+0000", 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 167.46285687700001, 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39277075, 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.459 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 155258 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224617+0000", 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224617+0000", 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224618+0000", 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4139999999999999e-06 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224623+0000", 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.206e-06 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224632+0000", 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8849999999999994e-06 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224639+0000", 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6760000000000001e-06 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.504396+0000", 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27975708799999999 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.504744+0000", 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034789800000000002 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.504869+0000", 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000125077 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.507838+0000", 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029684680000000001 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.508396+0000", 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00055870900000000001 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.617181+0000", 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.108784301 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.617243+0000", 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2685e-05 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.617388+0000", 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014434300000000001 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.464 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:155262 10.4 10:22a6342a:::benchmark_data_smithi029_256330_object53496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:07.224678+0000", 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 167.46279605199999, 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39542548100000002, 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.465 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 155262 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224678+0000", 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224678+0000", 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224678+0000", 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5400000000000002e-07 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224680+0000", 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.181e-06 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224686+0000", 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5759999999999996e-06 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.224691+0000", 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9130000000000002e-06 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.231234+0000", 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065434869999999997 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.231347+0000", 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000112445 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4", 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.231458+0000", 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011129 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.504725+0000", 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.273266644 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.507574+0000", 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028490019999999998 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.620000+0000", 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.112426559 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.620056+0000", 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5473000000000002e-05 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:07.620103+0000", 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7556999999999999e-05 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.470 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35727.0:155855 10.2 e1303/1296 10:47783a5f:::benchmark_data_smithi029_256330_object54089:head v 1303'9826, mlcod=1303'9823)", 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:08.355540+0000", 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 166.331934139, 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39447121299999999, 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.355540+0000", 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.355540+0000", 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.355540+0000", 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7999999999999995e-07 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.355546+0000", 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5389999999999998e-06 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.355555+0000", 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8939999999999999e-06 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.355561+0000", 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.925e-06 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.655615+0000", 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30005421300000001 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.655642+0000", 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6968999999999999e-05 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.749423+0000", 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.093781286000000005 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.750011+0000", 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00058780699999999998 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:155860 10.4 10:2ca5892f:::benchmark_data_smithi029_256330_object54094:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:08.359586+0000", 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 166.32788776199999, 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.40491840499999998, 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.475 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 155860 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359586+0000", 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359586+0000", 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359588+0000", 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7010000000000001e-06 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359593+0000", 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.682e-06 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359606+0000", 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2495e-05 2024-04-12T11:58:56.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359615+0000", 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0040000000000005e-06 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.655660+0000", 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.29604528800000002 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.655790+0000", 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013018400000000001 2024-04-12T11:58:56.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4", 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.656059+0000", 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00026886 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.660450+0000", 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043910950000000002 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.663735+0000", 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003284889 2024-04-12T11:58:56.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.764398+0000", 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10066322799999999 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.764457+0000", 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9002999999999997e-05 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.764504+0000", 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6975999999999999e-05 2024-04-12T11:58:56.480 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:155861 10.6 10:60d50826:::benchmark_data_smithi029_256330_object54095:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:08.359632+0000", 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 166.32784219999999, 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.40747666500000002, 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.481 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 155861 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359632+0000", 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359632+0000", 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359632+0000", 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3000000000000001e-07 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359635+0000", 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.88e-06 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359642+0000", 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5610000000000004e-06 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.359647+0000", 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3530000000000002e-06 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.361438+0000", 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017911559999999999 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.655161+0000", 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.293723395 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.750471+0000", 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.095309968999999994 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.750572+0000", 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010087200000000001 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4,6", 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.750708+0000", 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013528100000000001 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.763244+0000", 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012536524 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.763304+0000", 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0083000000000001e-05 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.763343+0000", 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9246000000000003e-05 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.766712+0000", 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033685220000000001 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.766772+0000", 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0059999999999998e-05 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.767108+0000", 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033623300000000002 2024-04-12T11:58:56.487 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35727.0:155858 10.a e1303/1285 10:5f54eb39:::benchmark_data_smithi029_256330_object54092:head v 1303'9792, mlcod=1303'9790)", 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:08.360404+0000", 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 166.32706990400001, 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38586024600000002, 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.360404+0000", 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.360404+0000", 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.360405+0000", 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4000000000000001e-07 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.360414+0000", 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0329999999999997e-06 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.360424+0000", 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0485e-05 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.360431+0000", 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5189999999999998e-06 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.361834+0000", 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001403715 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.361853+0000", 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8573999999999999e-05 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.664036+0000", 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30218325099999999 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:08.746264+0000", 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.082227929000000005 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:157814 10.8 10:10ee586e:::benchmark_data_smithi029_256330_object56048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:11.730325+0000", 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 162.95714875900001, 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39058399199999999, 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.492 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 157814 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.730325+0000", 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.730325+0000", 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.730327+0000", 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6199999999999999e-06 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.730332+0000", 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0250000000000002e-06 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.730355+0000", 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3482e-05 2024-04-12T11:58:56.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.730363+0000", 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9340000000000006e-06 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.730419+0000", 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6156000000000002e-05 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.730553+0000", 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000133545 2024-04-12T11:58:56.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.730686+0000", 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013306799999999999 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.736573+0000", 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058868480000000001 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.736612+0000", 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9167e-05 2024-04-12T11:58:56.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:12.120802+0000", 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38419006 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:12.120858+0000", 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6391000000000003e-05 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:12.120909+0000", 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0695999999999997e-05 2024-04-12T11:58:56.497 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:157816 10.8 10:1ead3ce6:::benchmark_data_smithi029_256330_object56050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:11.732807+0000", 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 162.95466661699999, 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39578991899999999, 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T11:58:56.498 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 157816 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.732807+0000", 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.732807+0000", 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.732809+0000", 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.403e-06 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.732815+0000", 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.178e-06 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.732831+0000", 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6714999999999998e-05 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.732839+0000", 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8329999999999994e-06 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.733535+0000", 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00069543299999999997 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.733668+0000", 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013283199999999999 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.733796+0000", 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012844899999999999 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.736798+0000", 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030024259999999999 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:11.737107+0000", 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000308819 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:12.128491+0000", 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39138393300000002 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:12.128547+0000", 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5682e-05 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:12.128597+0000", 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0216000000000002e-05 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.503 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:42336 11.6 11:6fad3c1c:::benchmark_data_smithi029_285290_object42335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:58:15.218748+0000", 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 39.468726060999998, 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34096321299999999, 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 42336 2024-04-12T11:58:56.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.218748+0000", 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.218748+0000", 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.218749+0000", 2024-04-12T11:58:56.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4559999999999999e-06 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.218774+0000", 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4587999999999999e-05 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.218788+0000", 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4630000000000001e-05 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:58:56.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.218797+0000", 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6139999999999992e-06 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.218852+0000", 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4750000000000003e-05 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.218966+0000", 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114238 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2024-04-12T11:58:56.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.219074+0000", 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010813400000000001 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.220483+0000", 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014091100000000001 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.220905+0000", 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00042207600000000002 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T11:58:56.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.559596+0000", 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33869030999999999 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.559661+0000", 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5575999999999997e-05 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:58:15.559711+0000", 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9731000000000002e-05 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.509 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:58:56.510 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:58:56.510 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:58:56.510 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:56.534 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:56.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:56.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:56.635 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:56.672 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:56.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:56.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:56.774 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:56.799 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:56.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:56.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:56.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:56.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:56.926 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:56.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:57.027 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:57.059 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:57.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:57.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:57.161 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:57.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:57.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:57.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:57.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 94095 94079 91.8594 44.625 0.0033969 0.0108624 2024-04-12T11:58:57.287 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:57.321 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:57.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:57.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:57.423 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:57.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:57.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:57.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:57.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:57 smithi029 ceph-mon[93650]: pgmap v2325: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-04-12T11:58:57.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:57 smithi029 ceph-mon[97787]: pgmap v2325: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-04-12T11:58:57.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:57.585 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:57.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:57.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:57.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:57 smithi087 ceph-mon[89014]: pgmap v2325: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-04-12T11:58:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:57.718 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:57.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:57.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:57.856 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:57.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:57.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:57.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:57.998 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:57.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:57.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:58.100 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:58.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:58.141 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:58.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:58.220 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 94937 94921 91.2554 52.625 0.0180473 0.0109529 2024-04-12T11:58:58.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:58.284 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:58.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:58.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:58.386 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:58.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:58.424 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:58.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:58.525 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:58.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:58.571 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:58.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:58.672 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:58:58.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:58.716 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:58.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:58:58.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:58:58.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:58.845 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:58.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:58:58.946 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:58.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:58.979 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:58.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:59.080 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:58:59.115 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:59.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:59.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:58:59.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:58:59.220 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 95911 95895 90.7951 60.875 0.0112627 0.0110087 2024-04-12T11:58:59.245 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:59.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:59.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:58:59.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:58:59.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:59.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:59.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:58:59.471 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:59.495 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:59.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:59.597 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:58:59.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:59.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:59.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:58:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:58:59 smithi087 ceph-mon[89014]: pgmap v2326: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T11:58:59.735 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:58:59.767 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:58:59.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:59.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:58:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:58:59 smithi029 ceph-mon[97787]: pgmap v2326: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T11:58:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:58:59 smithi029 ceph-mon[93650]: pgmap v2326: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T11:58:59.836 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:58:59.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:58:59.893 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:58:59.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:58:59.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:58:59.995 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:00.034 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:00.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:00.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:00.136 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:00.173 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:00.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:00.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:00.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 96581 96565 90.0649 41.875 0.0118544 0.0110983 2024-04-12T11:59:00.275 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:00.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:00.315 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:00.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:00.417 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:00.452 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:00.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:00.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:00.554 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:00.590 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:00.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:00.692 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:00.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:00.726 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:00.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:00.827 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:00.865 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:00.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:00.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:01.007 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:01.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:01.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:01.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:01.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:01.146 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:01.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:01.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 97644 97628 89.7174 66.4375 0.00562481 0.0111401 2024-04-12T11:59:01.247 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:01.270 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:01.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:01.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:01.407 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:01.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:01.509 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:01.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:01 smithi029 ceph-mon[97787]: pgmap v2327: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T11:59:01.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:01 smithi029 ceph-mon[93650]: pgmap v2327: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T11:59:01.544 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:01.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:01.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:01.646 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:01.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:01.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:01.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:01 smithi087 ceph-mon[89014]: pgmap v2327: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-12T11:59:01.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:01.817 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:01.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:01.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:01.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:01.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:01.961 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:01.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:02.062 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:02.101 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:02.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:02.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:02.202 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:02.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 98653 98637 89.3311 63.0625 0.00724514 0.0111892 2024-04-12T11:59:02.235 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:02.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:02.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:02.337 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:02.377 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:02.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:02.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:02.479 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:02.512 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:02.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:02.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:02.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:02.653 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:02.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:02.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:02.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-04-12T11:59:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:59:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:59:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:59:02.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:02.783 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:02.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:02.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:02.885 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:02.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:02.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:03.036 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:03.078 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:03.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:03.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:03.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:03.213 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:03.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:03.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 99787 99771 89.0673 70.875 0.00679446 0.0112221 2024-04-12T11:59:03.316 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:03.353 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:03.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:03.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:03.456 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:03.491 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:03.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:03.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:03.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:03.705 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:03.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:03.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:03 smithi087 ceph-mon[89014]: pgmap v2328: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 51 MiB/s wr, 808 op/s 2024-04-12T11:59:03.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:03 smithi029 ceph-mon[97787]: pgmap v2328: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 51 MiB/s wr, 808 op/s 2024-04-12T11:59:03.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:03 smithi029 ceph-mon[93650]: pgmap v2328: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 51 MiB/s wr, 808 op/s 2024-04-12T11:59:03.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:03.844 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:03.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:03.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:03.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:03.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:03.993 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:03.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:04.094 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:04.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:04.136 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:04.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:04.220 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 100955 100939 88.8409 73 0.0193024 0.0112506 2024-04-12T11:59:04.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:04.287 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:04.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:04.389 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:04.430 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:04.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:04.531 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:04.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:04.564 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:04.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:04.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:04.688 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:04.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:04.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:04.695 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:04.773 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:04.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:04.839 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:04.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:04.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:04.941 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:04.977 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:04.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:05.079 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:05.117 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:05.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:05.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:05.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:05.220 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 101974 101958 88.4915 63.6875 0.0415355 0.011292 2024-04-12T11:59:05.257 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:05.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:05.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:05.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:59:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:59:04] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.43.0" 2024-04-12T11:59:05.359 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:05.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:05.395 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:05.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:05.496 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:05.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:05.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:05.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:05.640 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:05.677 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:05.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:05.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:05 smithi087 ceph-mon[89014]: pgmap v2329: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 58 MiB/s wr, 929 op/s 2024-04-12T11:59:05.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:05 smithi029 ceph-mon[93650]: pgmap v2329: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 58 MiB/s wr, 929 op/s 2024-04-12T11:59:05.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:05 smithi029 ceph-mon[97787]: pgmap v2329: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 58 MiB/s wr, 929 op/s 2024-04-12T11:59:05.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:05.814 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:05.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:05.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:05.916 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:05.952 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:05.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:05.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:06.054 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:06.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:06.094 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:06.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:06.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:06.220 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 103097 103081 88.2407 70.1875 0.0209874 0.0113276 2024-04-12T11:59:06.229 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:06.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:06.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:06.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:06.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:06.368 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:06.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:06.469 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:06.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:06.507 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:06.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:06.608 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:06.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:06.652 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:06.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:06.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:06.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:06.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:06.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:06.789 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:06.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:06.890 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:06.924 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:06.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:06.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:07.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:07.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:07.062 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:07.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:07.163 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:07.200 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:07.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:07.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:07.220 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 104240 104224 88.0136 71.4375 0.00843432 0.0113566 2024-04-12T11:59:07.301 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:07.336 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:07.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:07.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:07.438 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:07.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:07.491 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:07.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:07.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:07 smithi029 ceph-mon[97787]: pgmap v2330: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-12T11:59:07.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:07 smithi029 ceph-mon[93650]: pgmap v2330: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-12T11:59:07.592 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:07.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:07.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:07.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:07 smithi087 ceph-mon[89014]: pgmap v2330: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-12T11:59:07.745 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:07.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:07.780 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:07.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:07.880 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:07.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:07.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:07.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:08.013 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:08.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:08.068 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:08.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:08.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:08.224 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 105243 105227 87.6732 62.6875 0.0420185 0.0114005 2024-04-12T11:59:08.228 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:08.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:08.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:08.330 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:08.358 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:08.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:08.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:08.460 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:08.488 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:08.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:08.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:08.589 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:08.639 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:08.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:08.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:08.741 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:08 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:08 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:08 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:08 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:08.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:08.782 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:08.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:08.883 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:08.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:08.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:08.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:08 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:08.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:08 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:09.014 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:09.043 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:09.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:09.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:09.200 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:09.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:09.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:09.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 105961 105945 87.1099 44.875 0.0156995 0.0114748 2024-04-12T11:59:09.302 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:09.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:09.335 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:09.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:09.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:09.476 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:09.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:09.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:09.578 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:09.618 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:09.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:09.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:09.763 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:09.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:09.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:09.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[93650]: pgmap v2331: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s 2024-04-12T11:59:09.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:59:09.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:59:09.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:09.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/248814944' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.14033477962851815}]: dispatch 2024-04-12T11:59:09.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.14033477962851815}]: dispatch 2024-04-12T11:59:09.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3350214327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:59:09.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:59:09.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[97787]: pgmap v2331: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s 2024-04-12T11:59:09.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:59:09.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:59:09.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:09.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/248814944' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.14033477962851815}]: dispatch 2024-04-12T11:59:09.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.14033477962851815}]: dispatch 2024-04-12T11:59:09.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3350214327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:59:09.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:09 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:59:09.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:09.877 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.3 primary-affinity to 0.140335 (891962) 2024-04-12T11:59:09.884 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T11:59:09.904 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:09.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:09.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:09 smithi087 ceph-mon[89014]: pgmap v2331: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s 2024-04-12T11:59:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T11:59:09.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T11:59:09.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:09.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/248814944' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.14033477962851815}]: dispatch 2024-04-12T11:59:09.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:09 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.14033477962851815}]: dispatch 2024-04-12T11:59:09.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3350214327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:59:09.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:09 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T11:59:10.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:10.029 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:10.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:10.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:10.131 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:10.155 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:10.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:10.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:10.223 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 106685 106669 86.5662 45.25 0.00405181 0.0114973 2024-04-12T11:59:10.256 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:10.279 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:10.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:10.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:10.405 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:10.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:10.489 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:59:10.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:10.508 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:59:10.528 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:59:10.531 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:10.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:10.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:10.547 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:59:10.567 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:59:10.588 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:59:10.606 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:59:10.626 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:59:10.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:10 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:59:10.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:10 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.14033477962851815}]': finished 2024-04-12T11:59:10.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:10 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:59:10.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:10 smithi029 ceph-mon[93650]: osdmap e1324: 8 total, 8 up, 6 in 2024-04-12T11:59:10.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/106535193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:10.627 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:10 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:59:10.627 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:10 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.14033477962851815}]': finished 2024-04-12T11:59:10.627 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:10 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:59:10.627 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:10 smithi029 ceph-mon[97787]: osdmap e1324: 8 total, 8 up, 6 in 2024-04-12T11:59:10.627 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/106535193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:10.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:59:10.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:59:10.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:59:10.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:59:10.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:59:10.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:59:10.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:59:10.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:59:10.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:10.658 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:10.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:10.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:10.759 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:10.784 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:10.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:10.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:10.885 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:10.924 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:10.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:10.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:10 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:59:10.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:10 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.14033477962851815}]': finished 2024-04-12T11:59:10.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:10 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T11:59:10.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:10 smithi087 ceph-mon[89014]: osdmap e1324: 8 total, 8 up, 6 in 2024-04-12T11:59:10.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/106535193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:11.025 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:11.064 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:11.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:11.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:11.166 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:11.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:11.204 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:11.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:11.223 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 15 107074 107059 85.7689 24.375 0.0128878 0.0116544 2024-04-12T11:59:11.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:11.329 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:11.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:11.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:11.431 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:11.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:11.467 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:11.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:11.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:11.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:11.608 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:11.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:11.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:11.750 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:11.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:11.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:11.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:11.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:11.898 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:11.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:11 smithi087 ceph-mon[89014]: pgmap v2333: 99 pgs: 1 peering, 98 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 54 MiB/s wr, 870 op/s 2024-04-12T11:59:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:11 smithi087 ceph-mon[89014]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:59:11.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:11 smithi087 ceph-mon[89014]: osdmap e1325: 8 total, 8 up, 6 in 2024-04-12T11:59:11.984 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:11 smithi029 ceph-mon[93650]: pgmap v2333: 99 pgs: 1 peering, 98 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 54 MiB/s wr, 870 op/s 2024-04-12T11:59:11.984 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:11 smithi029 ceph-mon[93650]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:59:11.984 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:11 smithi029 ceph-mon[93650]: osdmap e1325: 8 total, 8 up, 6 in 2024-04-12T11:59:11.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:11 smithi029 ceph-mon[97787]: pgmap v2333: 99 pgs: 1 peering, 98 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 54 MiB/s wr, 870 op/s 2024-04-12T11:59:11.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:11 smithi029 ceph-mon[97787]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-12T11:59:11.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:11 smithi029 ceph-mon[97787]: osdmap e1325: 8 total, 8 up, 6 in 2024-04-12T11:59:11.999 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:12.033 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:12.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:12.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:12.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:12.180 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:12.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:12.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:12.223 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 108149 108133 85.5328 67.125 0.00849959 0.0116863 2024-04-12T11:59:12.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:12.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:12.324 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:12.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:12.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:12.459 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:12.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:12.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:12.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:12.599 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:12.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:12.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:59:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:59:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:59:12.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:12.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:12.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:12.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:12.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:12.871 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:12.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:12.896 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:59:12.896 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:59:12.973 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:13.013 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:13.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:13.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:13.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:13.154 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:13.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:13.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:13.223 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-04-12T11:59:13.222814+0000 min lat: 0.00118232 max lat: 1.04367 avg lat: 0.0117098 2024-04-12T11:59:13.223 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-12T11:59:13.223 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 109299 109283 85.362 71.875 0.0106062 0.0117098 2024-04-12T11:59:13.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:13.297 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:13.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:13.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:13.398 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:13.434 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:13.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:13.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:13.536 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:13.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:13.579 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:13.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:13.680 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:13.720 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:13.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:13.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:13.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:59:13.821 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:13.868 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:13.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:13.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:13 smithi087 ceph-mon[89014]: 1.7 scrub starts 2024-04-12T11:59:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:13 smithi087 ceph-mon[89014]: 1.7 scrub ok 2024-04-12T11:59:13.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:13 smithi087 ceph-mon[89014]: pgmap v2335: 99 pgs: 1 peering, 98 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 57 MiB/s wr, 906 op/s 2024-04-12T11:59:13.970 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:14.008 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:14.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:14.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:13 smithi029 ceph-mon[93650]: 1.7 scrub starts 2024-04-12T11:59:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:13 smithi029 ceph-mon[93650]: 1.7 scrub ok 2024-04-12T11:59:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:13 smithi029 ceph-mon[93650]: pgmap v2335: 99 pgs: 1 peering, 98 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 57 MiB/s wr, 906 op/s 2024-04-12T11:59:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:13 smithi029 ceph-mon[97787]: 1.7 scrub starts 2024-04-12T11:59:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:13 smithi029 ceph-mon[97787]: 1.7 scrub ok 2024-04-12T11:59:14.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:13 smithi029 ceph-mon[97787]: pgmap v2335: 99 pgs: 1 peering, 98 active+clean; 6.4 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 57 MiB/s wr, 906 op/s 2024-04-12T11:59:14.110 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:14.123 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-04-12T11:59:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:14.170 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:14.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:14.228 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 110403 110387 85.1588 69 0.0107182 0.0117378 2024-04-12T11:59:14.242 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:14.271 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:14.327 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:14.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:14.428 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:14.483 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:14.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:14.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:14.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:14.639 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:14.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:14.741 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:14.775 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:14.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:14.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:14.877 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:14.915 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:14.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:14.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:15.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:15.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:59:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:59:14] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.43.0" 2024-04-12T11:59:15.057 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:15.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:15.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:15.158 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:15.194 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:15.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:15.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:15.224 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 111500 111484 84.9563 68.5625 0.0130182 0.0117658 2024-04-12T11:59:15.296 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:15.334 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:15.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:15.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:15.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:15.474 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:15.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:15.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:15.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:15.581 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T11:59:15.619 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:15.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:15.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:15.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:15.762 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:15.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:15.863 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:15.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:15.902 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:15.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:15.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:15 smithi029 ceph-mon[97787]: pgmap v2336: 99 pgs: 1 peering, 98 active+clean; 6.5 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 10 KiB/s rd, 44 MiB/s wr, 713 op/s; 0 B/s, 0 objects/s recovering 2024-04-12T11:59:15.955 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:15 smithi029 ceph-mon[93650]: pgmap v2336: 99 pgs: 1 peering, 98 active+clean; 6.5 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 10 KiB/s rd, 44 MiB/s wr, 713 op/s; 0 B/s, 0 objects/s recovering 2024-04-12T11:59:15.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:15 smithi087 ceph-mon[89014]: pgmap v2336: 99 pgs: 1 peering, 98 active+clean; 6.5 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 10 KiB/s rd, 44 MiB/s wr, 713 op/s; 0 B/s, 0 objects/s recovering 2024-04-12T11:59:16.003 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:16.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:16.075 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:16.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:16.096 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:16.128 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:16.176 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:16.211 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:16.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:16.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:16.224 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 112566 112550 84.7354 66.625 0.0127466 0.0117966 2024-04-12T11:59:16.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:16.348 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:16.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:16.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:16.450 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:16.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:16.492 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:16.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:16.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:16.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:16.635 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:16.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:16.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:16.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:16.783 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:16.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:16.815 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:16 smithi029 ceph-mon[97787]: pgmap v2337: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 10 KiB/s rd, 57 MiB/s wr, 916 op/s 2024-04-12T11:59:16.815 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:16 smithi029 ceph-mon[97787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T11:59:16.815 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:16 smithi029 ceph-mon[93650]: pgmap v2337: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 10 KiB/s rd, 57 MiB/s wr, 916 op/s 2024-04-12T11:59:16.815 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:16 smithi029 ceph-mon[93650]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T11:59:16.885 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:16.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:16.928 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:16.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:16 smithi087 ceph-mon[89014]: pgmap v2337: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 10 KiB/s rd, 57 MiB/s wr, 916 op/s 2024-04-12T11:59:16.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:16 smithi087 ceph-mon[89014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-12T11:59:17.029 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:17.061 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:17.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:17.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:17.163 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:17.202 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:17.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:17.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:17.224 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 113629 113613 84.5175 66.4375 0.0106336 0.0118271 2024-04-12T11:59:17.297 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:59:17.297 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:59:17.297 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:59:17.297 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:59:17.297 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:59:17.297 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:59:17.297 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:59:17.297 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:59:17.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:59:17.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:59:17.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:59:17.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:59:17.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:59:17.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:59:17.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:59:17.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:59:17.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:17.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:17.341 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:17.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:17.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:17.466 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:17.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:17.554 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 6, 7, 3] out_osds: [5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:59:17.554 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:59:17.554 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-04-12T11:59:17.554 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-04-12T11:59:17.554 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-04-12T11:59:17.555 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-04-12T11:59:17.567 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:17.621 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:17.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:17.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:17.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:17.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:17.753 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:17.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:17.762 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T11:59:17.854 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:17.878 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:17.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:17.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:17.884 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:17 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/298731502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:17.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:18.015 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:18.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:18.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:18.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:17 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/298731502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:18.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:17 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/298731502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:18.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:18.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:18.160 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:18.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:18.225 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 114102 114086 83.871 29.5625 0.00533746 0.0119186 2024-04-12T11:59:18.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:18.298 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:18.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:18.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:18.400 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:18.435 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:18.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:18.538 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:18.575 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:18.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:18.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:18.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:18.710 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:18.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:18.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:18.812 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:18.842 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:18.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:18.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:18.914 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:18 smithi087 ceph-mon[89014]: 1.4 scrub starts 2024-04-12T11:59:18.914 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:18 smithi087 ceph-mon[89014]: 1.4 scrub ok 2024-04-12T11:59:18.915 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:18 smithi087 ceph-mon[89014]: pgmap v2338: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:59:18.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:18.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:18.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:19.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:18 smithi029 ceph-mon[97787]: 1.4 scrub starts 2024-04-12T11:59:19.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:18 smithi029 ceph-mon[97787]: 1.4 scrub ok 2024-04-12T11:59:19.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:18 smithi029 ceph-mon[97787]: pgmap v2338: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:59:19.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:18 smithi029 ceph-mon[93650]: 1.4 scrub starts 2024-04-12T11:59:19.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:18 smithi029 ceph-mon[93650]: 1.4 scrub ok 2024-04-12T11:59:19.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:18 smithi029 ceph-mon[93650]: pgmap v2338: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:59:19.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:19.126 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:19.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:19.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:19.225 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 115176 115160 83.6762 67.125 0.0100626 0.0119462 2024-04-12T11:59:19.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:19.260 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:19.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:19.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:19.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:19.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:19.398 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:19.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:19.499 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:19.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:19.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:19.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:19.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:19.672 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:19.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:19.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:19.774 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:19.810 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:19.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:19.912 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:19.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:19.951 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:19.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:20.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:20.073 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T11:59:20.073 INFO:teuthology.orchestra.run.smithi087.stdout: "success": "" 2024-04-12T11:59:20.073 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T11:59:20.088 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:20.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:20.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:20.190 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:20.225 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 116121 116105 83.3933 59.0625 0.0130718 0.0119858 2024-04-12T11:59:20.230 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:20.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:20.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:20.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:20.379 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:20.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:20.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:20.509 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:20.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:20.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:20.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:20.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:20.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:20.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:20.749 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:20.807 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:20.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:20.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:20.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:20.932 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:20.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:20.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:21.034 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:21.073 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:21.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:21.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:21.175 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:21.207 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:21.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:21.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:21.225 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 117024 117008 83.087 56.4375 0.0244017 0.0120299 2024-04-12T11:59:21.309 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:21.333 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:21.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:21.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:21.435 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:21.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:21.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:21.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:21 smithi029 ceph-mon[93650]: pgmap v2339: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:59:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:21 smithi029 ceph-mon[97787]: pgmap v2339: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:59:21.576 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:21.604 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:21.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:21.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:21.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:21 smithi087 ceph-mon[89014]: pgmap v2339: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-12T11:59:21.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:21.742 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:21.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:21.843 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:21.880 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:21.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:21.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:21.983 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:22.019 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:22.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:22.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:22.121 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:22.152 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:22.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:22.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:22.225 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 117863 117847 82.7426 52.4375 0.0172638 0.0120808 2024-04-12T11:59:22.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:22.296 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:22.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:22.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:22.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:22.444 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:22.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:22.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:22.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:22 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:22.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:22.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:22 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:22.546 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:22.563 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:59:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:59:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:59:22.564 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:22.564 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:22 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:22.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:22.582 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:22.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:22.683 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:22.707 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:22.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:22.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:22.808 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:22.849 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:22.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:22.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:22.951 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:22.989 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:22.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:22.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:23.091 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:23.114 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:23.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:23.216 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:23.225 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 90 16 118688 118672 82.3961 51.5625 0.015868 0.0121136 2024-04-12T11:59:23.239 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:23.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:23.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:23.340 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:23.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:23.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:23.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:23.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:23.489 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:23.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:23 smithi029 ceph-mon[93650]: pgmap v2340: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 57 MiB/s wr, 914 op/s 2024-04-12T11:59:23.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:23 smithi029 ceph-mon[97787]: pgmap v2340: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 57 MiB/s wr, 914 op/s 2024-04-12T11:59:23.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:23.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:23.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:23.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:23.717 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:23.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:23 smithi087 ceph-mon[89014]: pgmap v2340: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 57 MiB/s wr, 914 op/s 2024-04-12T11:59:23.741 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:23.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:23.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:23.843 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:23.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:23.871 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:23.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:23.972 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:24.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:24.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:24.102 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:24.142 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:24.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:24.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:24.225 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.066 2024-04-12T11:59:24.226 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 118688 2024-04-12T11:59:24.226 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2024-04-12T11:59:24.226 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2024-04-12T11:59:24.226 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 82.3619 2024-04-12T11:59:24.226 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 34.878 2024-04-12T11:59:24.226 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 203.25 2024-04-12T11:59:24.226 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 24.375 2024-04-12T11:59:24.226 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1317 2024-04-12T11:59:24.226 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 558.048 2024-04-12T11:59:24.226 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3252 2024-04-12T11:59:24.227 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 390 2024-04-12T11:59:24.227 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0121374 2024-04-12T11:59:24.227 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0217167 2024-04-12T11:59:24.227 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 1.04367 2024-04-12T11:59:24.227 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00118232 2024-04-12T11:59:24.233 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2024-04-12T11:59:24.244 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:24.278 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:24.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:24.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:24.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:24.419 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:24.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:24.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:24.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:24.546 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:24.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:24.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:24.647 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:24.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:24.681 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:24.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:24.782 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:24.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:24.816 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:24.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:24.917 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:24.951 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:24.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:24.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:25.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:59:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:59:24] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.43.0" 2024-04-12T11:59:25.053 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:25.090 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:25.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:25.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:25.192 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:25.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:25.228 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:25.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:25.295 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:25 smithi029 ceph-mon[97787]: pgmap v2341: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T11:59:25.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:25 smithi029 ceph-mon[93650]: pgmap v2341: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T11:59:25.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:25.374 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:25.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:25.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:25.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:25.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:25.518 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:25.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:25.619 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:25.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:25.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:25.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:25 smithi087 ceph-mon[89014]: pgmap v2341: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-12T11:59:25.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:25.781 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 6, 7, 3] out_osds: [5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:59:25.781 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:59:25.781 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-04-12T11:59:25.781 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.648787 2024-04-12T11:59:25.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 7 0.6487868054409824 2024-04-12T11:59:25.788 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:25.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:25.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:25.890 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:25.922 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:25.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:25.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:26.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:26.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:26.061 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:26.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:26.138 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T11:59:26.162 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:26.192 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:26.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:26.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:26.294 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:26.361 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:26.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:26.462 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:26.497 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:26.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:26.599 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:26.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:26.633 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:26.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:26.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:26.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:26.780 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:26.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:26.881 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:26.916 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:26.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:26.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:27.018 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:27.065 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:27.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:27.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:27.167 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:27.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:27.204 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:27.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:27.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:27.338 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:27.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:27.439 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:27.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:27.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:27.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:27.583 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:27.611 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:27.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:27.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:27.623 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:27 smithi029 ceph-mon[93650]: pgmap v2342: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 63 MiB/s wr, 1.02k op/s 2024-04-12T11:59:27.624 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:27 smithi029 ceph-mon[97787]: pgmap v2342: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 63 MiB/s wr, 1.02k op/s 2024-04-12T11:59:27.713 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:27 smithi087 ceph-mon[89014]: pgmap v2342: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 63 MiB/s wr, 1.02k op/s 2024-04-12T11:59:27.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:27.762 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:27.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:27.775 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:27.849 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:27.863 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:27.869 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:59:27.870 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T11:59:27.870 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T11:59:27.870 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T11:59:27.870 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:59:27.893 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:27.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:27.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:27.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:28.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:28.042 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:28.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:28.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:28.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:28.188 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:28.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:28.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:28.292 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T11:59:28.335 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:28.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:28.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:28.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:28.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:28.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:28.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:28.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:28 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2766912430' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:59:28.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:28 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2766912430' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:59:28.574 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:28.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:28.606 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:28.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:28.707 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:28.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:28 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2766912430' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T11:59:28.744 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:28.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:28.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:28.846 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:28.869 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:28.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:28.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:28.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:29.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:29.011 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:29.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:29.112 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:29.147 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:29.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:29.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:29.249 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:29.273 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:29.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:29.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:29.374 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:29.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:29.420 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:29.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:29.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:29.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:29.558 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:29.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:29.659 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:29.697 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:29.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:29.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:29 smithi087 ceph-mon[89014]: pgmap v2343: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 46 MiB/s wr, 887 op/s 2024-04-12T11:59:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:29 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:59:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:29 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T11:59:29.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:29 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2766912430' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:59:29.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:29 smithi087 ceph-mon[89014]: osdmap e1326: 8 total, 8 up, 6 in 2024-04-12T11:59:29.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:29 smithi029 ceph-mon[93650]: pgmap v2343: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 46 MiB/s wr, 887 op/s 2024-04-12T11:59:29.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:29 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:59:29.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:29 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T11:59:29.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:29 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2766912430' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:59:29.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:29 smithi029 ceph-mon[93650]: osdmap e1326: 8 total, 8 up, 6 in 2024-04-12T11:59:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:29 smithi029 ceph-mon[97787]: pgmap v2343: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 510 GiB / 536 GiB avail; 46 MiB/s wr, 887 op/s 2024-04-12T11:59:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:29 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T11:59:29.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:29 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T11:59:29.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:29 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2766912430' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T11:59:29.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:29 smithi029 ceph-mon[97787]: osdmap e1326: 8 total, 8 up, 6 in 2024-04-12T11:59:29.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:29.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:29.838 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:29.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:29.939 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:29.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:29.977 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:29.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:30.078 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:30.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:30.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:30.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:30.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:30.242 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:30.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:30.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:30.344 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:30.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:30.382 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:30.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:30.483 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:30.521 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:30.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:30.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:30.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:30.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:30.664 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:30.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:30.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:30.802 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:30.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:30.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:30.904 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:30.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:30.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:30.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:31.039 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:31.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:31.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:31.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:31.174 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:31.209 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:31.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:31.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:31.310 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:31.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:31.348 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:31.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:31.449 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:31.481 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:31.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:31.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:31.583 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:31.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:31 smithi029 ceph-mon[97787]: pgmap v2345: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 24 MiB/s wr, 859 op/s 2024-04-12T11:59:31.593 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:31 smithi029 ceph-mon[93650]: pgmap v2345: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 24 MiB/s wr, 859 op/s 2024-04-12T11:59:31.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:31.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:31.715 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:31 smithi087 ceph-mon[89014]: pgmap v2345: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 24 MiB/s wr, 859 op/s 2024-04-12T11:59:31.748 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:31.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:31.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:31.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:31.893 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:31.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:31.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:32.028 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:32.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:32.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:32.130 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:32.165 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:32.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:32.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:32.267 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:32.310 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:32.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:32.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:32.454 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:32.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:32.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:32.556 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:32.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:32.593 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:32.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:32.622 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:59:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:59:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:59:32.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:32.741 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:32.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:32.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:32.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:32.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:32.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:33.022 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:33.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:33.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:33.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-04-12T11:59:33.124 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:33.163 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:33.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:33.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:33.264 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:33.313 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:33.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:33.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:33.335 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:33 smithi029 ceph-mon[93650]: pgmap v2346: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 24 MiB/s wr, 859 op/s 2024-04-12T11:59:33.335 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:33 smithi029 ceph-mon[97787]: pgmap v2346: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 24 MiB/s wr, 859 op/s 2024-04-12T11:59:33.415 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:33.442 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:33.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:33.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:33.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T11:59:33.543 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:33.575 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:33.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:33.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:33.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:33 smithi087 ceph-mon[89014]: pgmap v2346: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 102 B/s rd, 24 MiB/s wr, 859 op/s 2024-04-12T11:59:33.727 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:33.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:33.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:33.829 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:33.866 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:33.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:33.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:33.968 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:34.000 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:34.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:34.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:34.102 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:34.142 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:34.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:34.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:34.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:34.288 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:34.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:34.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:34.410 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:34.411 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:34.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:34.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:34.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:34.608 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:34.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:34.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:34.710 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:34.748 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:34.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:34.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:34.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:34.889 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:34.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:34.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:34.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:35.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:35.050 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:35.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:35.151 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:35.197 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:35.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:35.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:35.249 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:59:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:59:34] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.43.0" 2024-04-12T11:59:35.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:35.305 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.7 primary-affinity to 0.648787 (8425182) 2024-04-12T11:59:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:35.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:35.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:35.448 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:35.472 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:35.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:35.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:35 smithi029 ceph-mon[93650]: pgmap v2347: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 102 B/s rd, 12 MiB/s wr, 897 op/s 2024-04-12T11:59:35.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:35 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1874811977' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6487868054409824}]: dispatch 2024-04-12T11:59:35.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:35 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6487868054409824}]: dispatch 2024-04-12T11:59:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:35 smithi029 ceph-mon[97787]: pgmap v2347: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 102 B/s rd, 12 MiB/s wr, 897 op/s 2024-04-12T11:59:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:35 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1874811977' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6487868054409824}]: dispatch 2024-04-12T11:59:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:35 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6487868054409824}]: dispatch 2024-04-12T11:59:35.573 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:35.597 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:35.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:35.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:35.698 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:35.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:35 smithi087 ceph-mon[89014]: pgmap v2347: 99 pgs: 99 active+clean; 6.8 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 102 B/s rd, 12 MiB/s wr, 897 op/s 2024-04-12T11:59:35.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:35 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1874811977' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6487868054409824}]: dispatch 2024-04-12T11:59:35.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:35 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6487868054409824}]: dispatch 2024-04-12T11:59:35.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:35.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:35.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:35.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:35.848 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:35.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:35.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:35.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:35.974 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:35.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:35.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:36.075 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:36.100 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:36.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:36.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:36.201 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:36.225 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:36.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:36.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:36.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:36.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:36.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:36.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:36.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:36.507 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:36.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:36.609 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:36.640 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:36.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:36.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:36.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:36 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6487868054409824}]': finished 2024-04-12T11:59:36.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:36 smithi087 ceph-mon[89014]: osdmap e1327: 8 total, 8 up, 6 in 2024-04-12T11:59:36.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:36 smithi087 ceph-mon[89014]: osdmap e1328: 8 total, 8 up, 6 in 2024-04-12T11:59:36.742 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:36.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:36 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6487868054409824}]': finished 2024-04-12T11:59:36.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:36 smithi029 ceph-mon[97787]: osdmap e1327: 8 total, 8 up, 6 in 2024-04-12T11:59:36.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:36 smithi029 ceph-mon[97787]: osdmap e1328: 8 total, 8 up, 6 in 2024-04-12T11:59:36.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:36 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.6487868054409824}]': finished 2024-04-12T11:59:36.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:36 smithi029 ceph-mon[93650]: osdmap e1327: 8 total, 8 up, 6 in 2024-04-12T11:59:36.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:36 smithi029 ceph-mon[93650]: osdmap e1328: 8 total, 8 up, 6 in 2024-04-12T11:59:36.775 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:36.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:36.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:36.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:36.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:36.916 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:36.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:37.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:37.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:37.061 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:37.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:37.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:37.203 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:37.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:37.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:37.339 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:37.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:37.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:37.441 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:37.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:37.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:37.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:37.573 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:37.612 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:37.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:37.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:37.663 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:37 smithi029 ceph-mon[93650]: pgmap v2349: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T11:59:37.663 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:37.663 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:37.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:37 smithi029 ceph-mon[97787]: pgmap v2349: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T11:59:37.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:37.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:37.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:37 smithi087 ceph-mon[89014]: pgmap v2349: 99 pgs: 99 active+clean; 6.6 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-12T11:59:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:37.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:37.755 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:37.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:37.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:37.890 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:37.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:37.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:37.992 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:38.031 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:38.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:38.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:38.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:38.173 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:38.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:38.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:38.275 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:38.308 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:38.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:38.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:38.409 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:38.445 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:38.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:38.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:38.512 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:38 smithi029 ceph-mon[93650]: 1.2 scrub starts 2024-04-12T11:59:38.512 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:38 smithi029 ceph-mon[93650]: 1.2 scrub ok 2024-04-12T11:59:38.512 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:38 smithi029 ceph-mon[93650]: 1.7 scrub starts 2024-04-12T11:59:38.512 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:38 smithi029 ceph-mon[93650]: 1.7 scrub ok 2024-04-12T11:59:38.512 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:38 smithi029 ceph-mon[97787]: 1.2 scrub starts 2024-04-12T11:59:38.512 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:38 smithi029 ceph-mon[97787]: 1.2 scrub ok 2024-04-12T11:59:38.512 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:38 smithi029 ceph-mon[97787]: 1.7 scrub starts 2024-04-12T11:59:38.512 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:38 smithi029 ceph-mon[97787]: 1.7 scrub ok 2024-04-12T11:59:38.547 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:38.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:38.585 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:38.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:38.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:38 smithi087 ceph-mon[89014]: 1.2 scrub starts 2024-04-12T11:59:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:38 smithi087 ceph-mon[89014]: 1.2 scrub ok 2024-04-12T11:59:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:38 smithi087 ceph-mon[89014]: 1.7 scrub starts 2024-04-12T11:59:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:38 smithi087 ceph-mon[89014]: 1.7 scrub ok 2024-04-12T11:59:38.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:38.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:38.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:38.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:38.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:38.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:38.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:38.964 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:38.999 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:38.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:39.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:39.099 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:59:39.101 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:39.130 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:39.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:39.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:39.138 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:59:39.184 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:59:39.225 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T11:59:39.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:39.266 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:59:39.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:39.285 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:39.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:39.309 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:59:39.359 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:59:39.386 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:39.397 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:59:39.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:39 smithi029 ceph-mon[97787]: pgmap v2351: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 866 op/s; 20 KiB/s, 0 objects/s recovering 2024-04-12T11:59:39.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:39 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/682063367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:39.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:39 smithi029 ceph-mon[93650]: pgmap v2351: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 866 op/s; 20 KiB/s, 0 objects/s recovering 2024-04-12T11:59:39.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:39 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/682063367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:39.401 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:59:39.401 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:59:39.401 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:59:39.401 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T11:59:39.401 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:59:39.401 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:59:39.401 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:59:39.401 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:59:39.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:39.419 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:39.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:39.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:39.550 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:39.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:39.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:39.652 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:39.688 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:39.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:39.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:39 smithi087 ceph-mon[89014]: pgmap v2351: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 866 op/s; 20 KiB/s, 0 objects/s recovering 2024-04-12T11:59:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:39 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/682063367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:39.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:39.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:39.830 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:39.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:39.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:39.976 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:39.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:40.078 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:40.113 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:40.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:40.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:40.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:40.265 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:40.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:40.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:40.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:40.408 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:40.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:40.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:40.510 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:40.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:40.554 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:40.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:40.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:40.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:40.701 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:40.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:40.769 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T11:59:40.769 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T11:59:40.801 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:40.831 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:40.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:40.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:40.933 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T11:59:40.974 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:40.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:40.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T11:59:41.076 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:41.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:41.112 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:41.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:41.213 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:41.253 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:41.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:41.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:41.355 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:41.378 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:41.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:41.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:41.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:41.515 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:41.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:41.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:41.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:41 smithi029 ceph-mon[97787]: pgmap v2352: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 1.28k op/s; 20 KiB/s, 0 objects/s recovering 2024-04-12T11:59:41.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:41 smithi029 ceph-mon[93650]: pgmap v2352: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 1.28k op/s; 20 KiB/s, 0 objects/s recovering 2024-04-12T11:59:41.616 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:41.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:41.651 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:41.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:41 smithi087 ceph-mon[89014]: pgmap v2352: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 1.28k op/s; 20 KiB/s, 0 objects/s recovering 2024-04-12T11:59:41.752 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:41.784 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:41.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:41.886 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:41.916 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:41.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:41.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:42.018 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:42.054 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:42.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:42.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:42.199 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:42.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:42.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:42.301 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:42.335 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:42.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:42.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:42.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:42.484 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:42.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:42.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:42.609 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:42.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:42.701 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 6, 7, 3] out_osds: [5, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:59:42.701 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T11:59:42.701 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [0, 1, 2, 3, 4, 6, 7, 5] 2024-04-12T11:59:42.701 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-12T11:59:42.701 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.3 2024-04-12T11:59:42.711 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:59:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:59:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:59:42.735 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:42.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:42.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:42.836 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:42.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:42.878 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:42.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:42.979 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:43.016 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:43.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:43.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:43.019 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:59:42 smithi029 systemd[1]: Stopping Ceph osd.3 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T11:59:43.117 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:43.142 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:43.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:43.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:43.245 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:43.292 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:43.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:43.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:43.417 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:43.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:43.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:43.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:43.548 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:43.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:43.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:43.588 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:43 smithi029 ceph-mon[97787]: pgmap v2353: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 988 op/s; 20 KiB/s, 0 objects/s recovering 2024-04-12T11:59:43.588 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:43 smithi029 ceph-mon[93650]: pgmap v2353: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 988 op/s; 20 KiB/s, 0 objects/s recovering 2024-04-12T11:59:43.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:43.683 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:43.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:43.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:43 smithi087 ceph-mon[89014]: pgmap v2353: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 988 op/s; 20 KiB/s, 0 objects/s recovering 2024-04-12T11:59:43.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:43.820 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:43.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:43.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:43.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:43.954 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:43.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:43.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:44.056 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:44.089 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:44.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:44.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:44.191 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:44.224 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:44.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:44.326 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:44.350 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:44.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:44.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:44.485 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:44.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:44.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:44.587 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:44.622 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:44.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:44.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:44.724 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:44.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:44.758 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:44.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:44.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:44.886 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:44.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:44.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:44.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:45.022 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:45.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:45.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:45.124 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:45.149 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:59:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:59:44] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-04-12T11:59:45.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:45.163 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:45.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:45.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:45.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:45.300 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:45.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:45.401 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:45.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:45.437 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:45.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:45.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:45 smithi029 ceph-mon[97787]: pgmap v2354: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 715 op/s 2024-04-12T11:59:45.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:45 smithi029 ceph-mon[93650]: pgmap v2354: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 715 op/s 2024-04-12T11:59:45.537 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:45.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:45.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:45.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:45.668 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:45.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:45.710 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:45.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:45.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:45 smithi087 ceph-mon[89014]: pgmap v2354: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 715 op/s 2024-04-12T11:59:45.811 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:45.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:45.848 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:45.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:45.949 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:45.985 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:45.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:45.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:46.087 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:46.122 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:46.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:46.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:46.224 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:46.257 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:46.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:46.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:46.359 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:46.384 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T11:59:46.390 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:46.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:46.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:46.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:46.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:46 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1486121744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:59:46.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:46 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:59:46.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:46 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1486121744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:59:46.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:46 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:59:46.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:46.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:46.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:46.629 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:46.665 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:46.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:46.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:46 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1486121744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:59:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:46 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T11:59:46.767 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:46.800 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:46.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:46.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:46.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T11:59:46.902 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:46.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:46.929 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:46.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:47.030 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:47.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:47.084 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:47.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:47.185 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:47.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:47.217 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:47.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:47.318 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:47.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:47.357 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:47.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:47.458 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:47.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:47.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:47.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:47.586 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:47.624 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:47.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:47 smithi087 ceph-mon[89014]: pgmap v2355: 99 pgs: 99 active+clean; 6.0 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 968 op/s 2024-04-12T11:59:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:47 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:59:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:47 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:59:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:47 smithi087 ceph-mon[89014]: osdmap e1329: 8 total, 8 up, 6 in 2024-04-12T11:59:47.726 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:47.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:47.760 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:47.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:47 smithi029 ceph-mon[93650]: pgmap v2355: 99 pgs: 99 active+clean; 6.0 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 968 op/s 2024-04-12T11:59:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:47 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:59:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:47 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:59:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:47 smithi029 ceph-mon[93650]: osdmap e1329: 8 total, 8 up, 6 in 2024-04-12T11:59:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:47 smithi029 ceph-mon[97787]: pgmap v2355: 99 pgs: 99 active+clean; 6.0 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 968 op/s 2024-04-12T11:59:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:47 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T11:59:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:47 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T11:59:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:47 smithi029 ceph-mon[97787]: osdmap e1329: 8 total, 8 up, 6 in 2024-04-12T11:59:47.862 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:47.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:47.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:47.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:48.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:48.019 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:48.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:48.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:48.157 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:48.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:48.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:48.259 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:48.294 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:48.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:48.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:48.396 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:48.420 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:48.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:48.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:48.521 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:48.545 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:48.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:48.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:48.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:48.685 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:48.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:48.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:48.787 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:48.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:48.829 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:48.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:48.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:48.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:48.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:48.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:49.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:49.080 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:49.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:49.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:49.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:49.217 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:49.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:49.319 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:49.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:49.351 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:49.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:49.454 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:49.487 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:49.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:49.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:49.589 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:49.625 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:49.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:49 smithi087 ceph-mon[89014]: pgmap v2357: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 970 op/s 2024-04-12T11:59:49.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:49.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:49 smithi029 ceph-mon[93650]: pgmap v2357: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 970 op/s 2024-04-12T11:59:49.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:49 smithi029 ceph-mon[97787]: pgmap v2357: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 970 op/s 2024-04-12T11:59:49.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:49.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:49.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:49.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:49.905 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:49.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:50.007 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:50.039 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:50.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:50.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:50.185 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:50.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:50.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:50.287 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:50.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:50.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:50.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:50.445 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:50.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:50.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:50.459 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-04-12T11:59:50.547 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:50.614 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:50.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:50.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:50.716 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:50.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:50.749 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:50.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:50.850 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:50.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:50.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:50.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:50.986 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:51.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:51.032 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:51.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:51.133 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:51.171 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:51.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:51.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:51.260 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:51.267 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:51.269 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[120907]: 2024-04-12T11:59:51.230+0000 7fad0b07f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T11:59:51.269 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[120907]: 2024-04-12T11:59:51.230+0000 7fad0b07f700 -1 osd.3 1329 *** Got signal Terminated *** 2024-04-12T11:59:51.269 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[120907]: 2024-04-12T11:59:51.230+0000 7fad0b07f700 -1 osd.3 1329 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T11:59:51.272 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:51.307 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:51.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:51.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:51.409 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:51.434 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:51.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:51.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:51.536 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:51.559 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:51.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:51.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:51.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-mon[97787]: pgmap v2358: 99 pgs: 99 active+clean; 5.7 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 964 op/s 2024-04-12T11:59:51.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-mon[97787]: osd.3 marked itself down and dead 2024-04-12T11:59:51.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:51.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:51.581 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-mon[93650]: pgmap v2358: 99 pgs: 99 active+clean; 5.7 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 964 op/s 2024-04-12T11:59:51.581 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-mon[93650]: osd.3 marked itself down and dead 2024-04-12T11:59:51.581 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:51.581 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:51.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:51.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:51.685 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:51.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:51 smithi087 ceph-mon[89014]: pgmap v2358: 99 pgs: 99 active+clean; 5.7 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 964 op/s 2024-04-12T11:59:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:51 smithi087 ceph-mon[89014]: osd.3 marked itself down and dead 2024-04-12T11:59:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T11:59:51.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T11:59:51.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:51.810 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:51.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:51.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:51.911 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:51.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:51.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:51.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:52.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:52.061 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:52.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:52.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:52.163 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:52.188 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:52.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:52.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:52.269 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:59:51 smithi029 podman[296028]: 2024-04-12 11:59:51.967895553 +0000 UTC m=+0.893000700 container died 1f975ff65c6b1962f6e35ea3d80c12daf0b429ae704ddda7c5fb5af1fc27376d (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409) 2024-04-12T11:59:52.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:52.313 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:52.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:52.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:52.415 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:52.456 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:52.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:52.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:52.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:52.598 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:52.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:52.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 11:59:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T11:59:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T11:59:52.618 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:52 smithi087 ceph-mon[89014]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:59:52.618 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:52 smithi087 ceph-mon[89014]: osdmap e1330: 8 total, 7 up, 6 in 2024-04-12T11:59:52.619 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:52 smithi087 ceph-mon[89014]: osdmap e1331: 8 total, 7 up, 6 in 2024-04-12T11:59:52.699 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:52.732 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:52.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:52.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:52 smithi029 ceph-mon[97787]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:59:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:52 smithi029 ceph-mon[97787]: osdmap e1330: 8 total, 7 up, 6 in 2024-04-12T11:59:52.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:52 smithi029 ceph-mon[97787]: osdmap e1331: 8 total, 7 up, 6 in 2024-04-12T11:59:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:52 smithi029 ceph-mon[93650]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T11:59:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:52 smithi029 ceph-mon[93650]: osdmap e1330: 8 total, 7 up, 6 in 2024-04-12T11:59:52.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:52 smithi029 ceph-mon[93650]: osdmap e1331: 8 total, 7 up, 6 in 2024-04-12T11:59:52.834 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:52.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:52.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:52.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:52.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:52.995 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:52.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:53.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T11:59:53.097 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:53.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:53.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:53.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:53.235 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:53.271 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:53.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:53.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:53.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:53.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:53.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:53.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:53.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:53.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:53.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:53.640 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:53.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:53 smithi029 ceph-mon[97787]: pgmap v2360: 99 pgs: 4 stale+active+clean, 95 active+clean; 5.7 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-12T11:59:53.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:53 smithi029 ceph-mon[93650]: pgmap v2360: 99 pgs: 4 stale+active+clean, 95 active+clean; 5.7 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-12T11:59:53.678 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:53.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:53.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:53 smithi087 ceph-mon[89014]: pgmap v2360: 99 pgs: 4 stale+active+clean, 95 active+clean; 5.7 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-12T11:59:53.780 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:53.819 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:53.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:53.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:53.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:53.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:53.963 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:53.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:54.018 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:59:53 smithi029 podman[296028]: 2024-04-12 11:59:53.799021838 +0000 UTC m=+2.724126988 container cleanup 1f975ff65c6b1962f6e35ea3d80c12daf0b429ae704ddda7c5fb5af1fc27376d (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-12T11:59:54.019 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:59:53 smithi029 bash[296028]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3 2024-04-12T11:59:54.064 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:54.095 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:54.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:54.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:54.197 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:54.228 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:54.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:54.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:54.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:54.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:54.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:54.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:54.452 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 11:59:54 smithi029 podman[296145]: 2024-04-12 11:59:54.061032982 +0000 UTC m=+2.159649200 container remove 1f975ff65c6b1962f6e35ea3d80c12daf0b429ae704ddda7c5fb5af1fc27376d (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, io.buildah.version=1.33.5, RELEASE=wip-rf-comb1-2cee185, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-12T11:59:54.463 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:54.497 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:54.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:54.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:54.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:54.633 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:54.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:54.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:54.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:54 smithi087 ceph-mon[89014]: Health check failed: Degraded data redundancy: 17206/279000 objects degraded (6.167%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:59:54.735 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:54.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:54 smithi029 ceph-mon[97787]: Health check failed: Degraded data redundancy: 17206/279000 objects degraded (6.167%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:59:54.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:54 smithi029 ceph-mon[93650]: Health check failed: Degraded data redundancy: 17206/279000 objects degraded (6.167%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T11:59:54.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:54.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:54.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:54.873 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:54.910 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:54.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:55.011 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:55.048 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:55.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:55.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:55.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:55.201 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:55.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:55.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:55.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 11:59:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:11:59:54] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.43.0" 2024-04-12T11:59:55.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:55.337 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:55.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:55.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:55.439 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:55.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:55.477 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:55.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:55.578 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:55.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:55.603 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:55.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:55.704 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:55 smithi087 ceph-mon[89014]: pgmap v2362: 99 pgs: 14 active+undersized, 8 active+undersized+degraded, 3 stale+active+clean, 74 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 753 op/s; 17206/279000 objects degraded (6.167%) 2024-04-12T11:59:55.733 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:55.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:55.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:55.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:55 smithi029 ceph-mon[97787]: pgmap v2362: 99 pgs: 14 active+undersized, 8 active+undersized+degraded, 3 stale+active+clean, 74 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 753 op/s; 17206/279000 objects degraded (6.167%) 2024-04-12T11:59:55.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:55 smithi029 ceph-mon[93650]: pgmap v2362: 99 pgs: 14 active+undersized, 8 active+undersized+degraded, 3 stale+active+clean, 74 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 753 op/s; 17206/279000 objects degraded (6.167%) 2024-04-12T11:59:55.835 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:55.870 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:55.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:55.971 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:56.007 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:56.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:56.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:56.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:56.150 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:56.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:56.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:56.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:56.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:56.287 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:56.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:56.388 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:56.415 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:56.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:56.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:56.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T11:59:56.541 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:56.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T11:59:56.643 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:56.676 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:56.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:56.778 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:56.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:56.814 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:56.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:56.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:56.938 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:56.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:56.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:57.040 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:57.064 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:57.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:57.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:57.165 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:57.189 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:57.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:57.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:57.291 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:57.324 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:57.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:57.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:57.426 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:57.466 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:57.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:57.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:57.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:57.591 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:57.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:57.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:57.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:57.717 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:57 smithi087 ceph-mon[89014]: pgmap v2363: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 961 op/s; 45076/269232 objects degraded (16.742%); 0 B/s, 0 objects/s recovering 2024-04-12T11:59:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:57.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:57.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:57 smithi029 ceph-mon[97787]: pgmap v2363: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 961 op/s; 45076/269232 objects degraded (16.742%); 0 B/s, 0 objects/s recovering 2024-04-12T11:59:57.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:57 smithi029 ceph-mon[93650]: pgmap v2363: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.5 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 961 op/s; 45076/269232 objects degraded (16.742%); 0 B/s, 0 objects/s recovering 2024-04-12T11:59:57.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:57.853 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:57.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:57.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:57.955 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:57.983 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:57.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:57.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:58.085 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:58.108 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:58.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:58.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:58.210 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:58.235 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:58.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:58.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:58.337 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:58.373 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:58.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:58.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:58.475 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:58.508 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:58.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:58.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:58.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:58.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:58.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:58.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:58.746 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T11:59:58.784 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:58.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:58.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T11:59:58.885 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:58.923 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:58.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:58.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:59.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:59.068 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:59.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:59.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:59.170 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:59.205 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:59.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:59.306 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T11:59:59.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:59.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T11:59:59.421 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T11:59:59.449 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T11:59:59.478 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:59.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:59.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T11:59:59.479 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T11:59:59.504 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T11:59:59.536 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T11:59:59.577 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T11:59:59.579 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T11:59:59.613 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T11:59:59.617 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:59.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:59.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T11:59:59.661 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T11:59:59.703 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T11:59:59.703 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T11:59:59.703 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T11:59:59.703 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T11:59:59.703 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-04-12T11:59:59.703 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T11:59:59.704 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T11:59:59.704 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T11:59:59.704 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T11:59:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:59 smithi087 ceph-mon[89014]: pgmap v2364: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.4 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 44026/264054 objects degraded (16.673%); 0 B/s, 0 objects/s recovering 2024-04-12T11:59:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 11:59:59 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4065955838' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:59.719 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T11:59:59.754 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T11:59:59.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:59.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T11:59:59.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:59 smithi029 ceph-mon[97787]: pgmap v2364: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.4 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 44026/264054 objects degraded (16.673%); 0 B/s, 0 objects/s recovering 2024-04-12T11:59:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 11:59:59 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4065955838' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:59 smithi029 ceph-mon[93650]: pgmap v2364: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.4 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 44026/264054 objects degraded (16.673%); 0 B/s, 0 objects/s recovering 2024-04-12T11:59:59.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 11:59:59 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4065955838' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T11:59:59.776 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T11:59:59.776 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T11:59:59.776 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.776 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:153375 11.4 e1331/1314 11:27c49199:::benchmark_data_smithi029_285290_object34684:head v 1331'9615, mlcod=1331'9613)", 2024-04-12T11:59:59.776 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:59.772817+0000", 2024-04-12T11:59:59.776 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.00105861, 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001082333, 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772817+0000", 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772817+0000", 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:59:59.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772818+0000", 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.71e-07 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772824+0000", 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0379999999999996e-06 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772834+0000", 2024-04-12T11:59:59.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9399999999999997e-06 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772841+0000", 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8820000000000001e-06 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772887+0000", 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5713000000000001e-05 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772903+0000", 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5424e-05 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:153365 11.2 11:4eea3390:::benchmark_data_smithi029_285290_object34674:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1331)", 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:59.764235+0000", 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0096401170000000001, 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0097089689999999996, 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:59:59.780 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 153365 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764235+0000", 2024-04-12T11:59:59.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764235+0000", 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764237+0000", 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5319999999999999e-06 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.782 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:59:59.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764243+0000", 2024-04-12T11:59:59.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2469999999999997e-06 2024-04-12T11:59:59.783 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.783 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.783 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:59:59.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764257+0000", 2024-04-12T11:59:59.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3978999999999999e-05 2024-04-12T11:59:59.783 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764267+0000", 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8050000000000001e-06 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764323+0000", 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6205000000000001e-05 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:59:59.784 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764465+0000", 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014231899999999999 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0", 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764596+0000", 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013048000000000001 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772076+0000", 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074804600000000004 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:59:59.785 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:153366 11.c 11:37fce523:::benchmark_data_smithi029_285290_object34675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1331)", 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:59.764905+0000", 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0089699389999999997, 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0090908689999999997, 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 153366 2024-04-12T11:59:59.786 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764905+0000", 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764905+0000", 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764906+0000", 2024-04-12T11:59:59.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5600000000000004e-07 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764911+0000", 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.33e-06 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764922+0000", 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1770000000000001e-05 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.764935+0000", 2024-04-12T11:59:59.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2568000000000001e-05 2024-04-12T11:59:59.789 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.789 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.789 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:59:59.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.765065+0000", 2024-04-12T11:59:59.789 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000129858 2024-04-12T11:59:59.789 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.789 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.789 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:59:59.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.765330+0000", 2024-04-12T11:59:59.789 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00026569700000000001 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.768655+0000", 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003324156 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772217+0000", 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035619720000000001 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:59:59.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.791 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:153369 11.2 11:4fa92a13:::benchmark_data_smithi029_285290_object34678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1331)", 2024-04-12T11:59:59.791 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:59.765188+0000", 2024-04-12T11:59:59.791 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0086870939999999994, 2024-04-12T11:59:59.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0088553269999999996, 2024-04-12T11:59:59.791 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T11:59:59.791 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T11:59:59.791 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2024-04-12T11:59:59.791 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T11:59:59.791 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 153369 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.765188+0000", 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.765188+0000", 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T11:59:59.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.765189+0000", 2024-04-12T11:59:59.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8400000000000004e-07 2024-04-12T11:59:59.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T11:59:59.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.765193+0000", 2024-04-12T11:59:59.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9659999999999998e-06 2024-04-12T11:59:59.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.765202+0000", 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5040000000000008e-06 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.765212+0000", 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9180000000000006e-06 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.768054+0000", 2024-04-12T11:59:59.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028420960000000001 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.768195+0000", 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000140495 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0", 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.768668+0000", 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00047289100000000002 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T11:59:59.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:59.772331+0000", 2024-04-12T11:59:59.796 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036628229999999999 2024-04-12T11:59:59.796 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:59:59.796 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T11:59:59.796 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:59:59.796 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T11:59:59.796 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2024-04-12T11:59:59.796 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 4 2024-04-12T11:59:59.796 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T11:59:59.856 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T11:59:59.890 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T11:59:59.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T11:59:59.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T11:59:59.992 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:00.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:00.032 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:00.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:00.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:00.167 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:00.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:00.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:00.268 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:00.301 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:00.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:00.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:00.403 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:00.441 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:00.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:00.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:00.472 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 44026/264054 objects degraded (16.673%), 16 pgs degraded 2024-04-12T12:00:00.472 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-04-12T12:00:00.472 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: [WRN] OSD_DOWN: 1 osds down 2024-04-12T12:00:00.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 44026/264054 objects degraded (16.673%), 16 pgs degraded 2024-04-12T12:00:00.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-04-12T12:00:00.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: [WRN] OSD_DOWN: 1 osds down 2024-04-12T12:00:00.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: osd.3 (root=default,host=smithi029) is down 2024-04-12T12:00:00.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: [WRN] PG_DEGRADED: Degraded data redundancy: 44026/264054 objects degraded (16.673%), 16 pgs degraded 2024-04-12T12:00:00.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 1.4 is active+undersized+degraded, acting [7,6] 2024-04-12T12:00:00.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 1.8 is active+undersized+degraded, acting [2,4] 2024-04-12T12:00:00.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 3.7 is active+undersized+degraded, acting [0,7] 2024-04-12T12:00:00.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 3.b is active+undersized+degraded, acting [0,4] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 3.d is active+undersized+degraded, acting [6,7] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 3.1d is active+undersized+degraded, acting [6,4] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 11.0 is active+undersized+degraded, acting [0,7] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 11.1 is active+undersized+degraded, acting [6,0] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 11.2 is active+undersized+degraded, acting [2,0] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 11.5 is active+undersized+degraded, acting [7,6] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 11.8 is active+undersized+degraded, acting [6,0] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 11.9 is active+undersized+degraded, acting [0,6] 2024-04-12T12:00:00.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 11.a is active+undersized+degraded, acting [2,6] 2024-04-12T12:00:00.475 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[93650]: pg 11.b is active+undersized+degraded, acting [0,4] 2024-04-12T12:00:00.543 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:00.573 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:00.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:00.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:00.675 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:00.710 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:00.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: osd.3 (root=default,host=smithi029) is down 2024-04-12T12:00:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: [WRN] PG_DEGRADED: Degraded data redundancy: 44026/264054 objects degraded (16.673%), 16 pgs degraded 2024-04-12T12:00:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 1.4 is active+undersized+degraded, acting [7,6] 2024-04-12T12:00:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 1.8 is active+undersized+degraded, acting [2,4] 2024-04-12T12:00:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 3.7 is active+undersized+degraded, acting [0,7] 2024-04-12T12:00:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 3.b is active+undersized+degraded, acting [0,4] 2024-04-12T12:00:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 3.d is active+undersized+degraded, acting [6,7] 2024-04-12T12:00:00.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-04-12T12:00:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-04-12T12:00:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 3.1d is active+undersized+degraded, acting [6,4] 2024-04-12T12:00:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 11.0 is active+undersized+degraded, acting [0,7] 2024-04-12T12:00:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 11.1 is active+undersized+degraded, acting [6,0] 2024-04-12T12:00:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 11.2 is active+undersized+degraded, acting [2,0] 2024-04-12T12:00:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 11.5 is active+undersized+degraded, acting [7,6] 2024-04-12T12:00:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 11.8 is active+undersized+degraded, acting [6,0] 2024-04-12T12:00:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 11.9 is active+undersized+degraded, acting [0,6] 2024-04-12T12:00:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 11.a is active+undersized+degraded, acting [2,6] 2024-04-12T12:00:00.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:00 smithi029 ceph-mon[97787]: pg 11.b is active+undersized+degraded, acting [0,4] 2024-04-12T12:00:00.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:00.848 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:00.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:00.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:00.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:00.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 44026/264054 objects degraded (16.673%), 16 pgs degraded 2024-04-12T12:00:00.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-04-12T12:00:00.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: [WRN] OSD_DOWN: 1 osds down 2024-04-12T12:00:00.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: osd.3 (root=default,host=smithi029) is down 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: [WRN] PG_DEGRADED: Degraded data redundancy: 44026/264054 objects degraded (16.673%), 16 pgs degraded 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 1.4 is active+undersized+degraded, acting [7,6] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 1.8 is active+undersized+degraded, acting [2,4] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 3.7 is active+undersized+degraded, acting [0,7] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 3.b is active+undersized+degraded, acting [0,4] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 3.d is active+undersized+degraded, acting [6,7] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 3.1d is active+undersized+degraded, acting [6,4] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 11.0 is active+undersized+degraded, acting [0,7] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 11.1 is active+undersized+degraded, acting [6,0] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 11.2 is active+undersized+degraded, acting [2,0] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 11.5 is active+undersized+degraded, acting [7,6] 2024-04-12T12:00:00.970 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 11.8 is active+undersized+degraded, acting [6,0] 2024-04-12T12:00:00.971 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 11.9 is active+undersized+degraded, acting [0,6] 2024-04-12T12:00:00.971 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 11.a is active+undersized+degraded, acting [2,6] 2024-04-12T12:00:00.971 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:00 smithi087 ceph-mon[89014]: pg 11.b is active+undersized+degraded, acting [0,4] 2024-04-12T12:00:00.985 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:00.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:01.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:01.117 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:01.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:01.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:01.188 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-04-12T12:00:01.188 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-04-12T12:00:01.219 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:01.254 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:01.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:01.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:01.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:01.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:01.398 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:01.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:01.472 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:01 smithi029 podman[296938]: 2024-04-12 12:00:01.24788374 +0000 UTC m=+0.023694562 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T12:00:01.498 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:01.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:01.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:01.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:01.635 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:01.674 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:01.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:01.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:01 smithi029 ceph-mon[93650]: pgmap v2365: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 42506/254109 objects degraded (16.727%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:01.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:01 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 42506/254109 objects degraded (16.727%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:01.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:01 smithi029 ceph-mon[97787]: pgmap v2365: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 42506/254109 objects degraded (16.727%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:01.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:01 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 42506/254109 objects degraded (16.727%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:01.770 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:01 smithi029 podman[296938]: 2024-04-12 12:00:01.661098679 +0000 UTC m=+0.436909476 container create 100a029a168d1c74f99d8aa2a63b939dbdd2332562438dabc42fea062442bf16 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-12T12:00:01.776 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:01.815 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:01.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:01.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:01.917 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:01.955 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:01.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:01.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:01 smithi087 ceph-mon[89014]: pgmap v2365: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s; 42506/254109 objects degraded (16.727%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:01 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 42506/254109 objects degraded (16.727%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:02.057 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:02.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:02.094 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:02.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:02.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:02.232 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:02.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:02.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:02.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:02.367 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:02.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:02.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:02.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:02.505 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:02.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:02.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:02.607 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:02.641 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:02.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:02.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:02.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-04-12T12:00:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:00:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:00:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:00:02.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:02.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:02.788 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:02.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:02.889 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:02.928 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:02.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:02.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:03.029 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:03.063 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:03.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:03.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:03.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:03.190 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T12:00:03.195 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:03.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:03.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:03.230 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:03 smithi029 podman[296938]: 2024-04-12 12:00:03.060480955 +0000 UTC m=+1.836291752 container init 100a029a168d1c74f99d8aa2a63b939dbdd2332562438dabc42fea062442bf16 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240409) 2024-04-12T12:00:03.231 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:03 smithi029 podman[296938]: 2024-04-12 12:00:03.070832929 +0000 UTC m=+1.846643729 container start 100a029a168d1c74f99d8aa2a63b939dbdd2332562438dabc42fea062442bf16 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-12T12:00:03.297 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:03.341 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:03.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:03.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:03.442 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:03.479 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:03.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:03.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:03 smithi029 ceph-mon[93650]: pgmap v2366: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 946 op/s; 42506/254109 objects degraded (16.727%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:03.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:03 smithi029 ceph-mon[97787]: pgmap v2366: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 946 op/s; 42506/254109 objects degraded (16.727%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:03.520 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:03 smithi029 podman[296938]: 2024-04-12 12:00:03.303160209 +0000 UTC m=+2.078971014 container attach 100a029a168d1c74f99d8aa2a63b939dbdd2332562438dabc42fea062442bf16 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-12T12:00:03.582 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:03.615 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:03.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:03.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:03.751 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:03.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:03.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:03.853 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:03.892 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:03.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:03.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:03 smithi087 ceph-mon[89014]: pgmap v2366: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 946 op/s; 42506/254109 objects degraded (16.727%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:03.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:04.027 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:04.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:04.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:04.129 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:04.165 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:04.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:04.241 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:00:04.266 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:04.299 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:04.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:04.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:04.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:04.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:04.435 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:04.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:04.536 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:04.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:04.571 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:04.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:04.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:04.714 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:04.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:04.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:04.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:04.850 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:04.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:04.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:04.952 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:04.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:04.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:04.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:05.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:05.121 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:05.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:05.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:05.212 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:00:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:00:04] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.43.0" 2024-04-12T12:00:05.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:05.263 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:05.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:05.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:05.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:05.398 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:05.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:05.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:05.500 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:05.537 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:05.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:05.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:05.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:05.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:05.674 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:05.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:05.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:05 smithi029 ceph-mon[97787]: pgmap v2367: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 957 op/s; 41364/248526 objects degraded (16.644%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:05.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:05 smithi029 ceph-mon[93650]: pgmap v2367: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 957 op/s; 41364/248526 objects degraded (16.644%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:05.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:05.811 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:05.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:05.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:05.913 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:05.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:05.952 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:05.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:05.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:05 smithi087 ceph-mon[89014]: pgmap v2367: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 5.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 957 op/s; 41364/248526 objects degraded (16.644%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:06.052 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:06.086 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:06.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:06.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:06.188 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:06.219 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:06.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:06.320 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:06.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:06.359 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:06.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:06.460 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:06.497 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:06.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:06.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:06.633 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:06.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:06.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:06.735 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:06.764 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:06.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:06.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:06.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:06.897 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:06.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:06.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:06.999 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:07.038 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:07.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:07.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:07.065 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:00:07.140 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:07.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:07.179 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:07.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:07.281 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:07.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:07.319 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:07.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:07.345 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T12:00:07.420 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:07.455 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:07.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:07.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:07.557 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:07.587 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:07.588 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:07 smithi029 ceph-mon[97787]: pgmap v2368: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.22k op/s; 39353/235152 objects degraded (16.735%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:07.588 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:07.588 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:07.588 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:07 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 39353/235152 objects degraded (16.735%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:07.588 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:07 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1164427901' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:00:07.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:07 smithi029 ceph-mon[93650]: pgmap v2368: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.22k op/s; 39353/235152 objects degraded (16.735%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:07.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:07.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:07.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:07 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 39353/235152 objects degraded (16.735%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:07.589 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:07 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1164427901' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:00:07.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:07.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:07.691 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:07.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:07 smithi087 ceph-mon[89014]: pgmap v2368: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.8 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.22k op/s; 39353/235152 objects degraded (16.735%); 0 B/s, 0 objects/s recovering 2024-04-12T12:00:07.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:07.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:07.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:07 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 39353/235152 objects degraded (16.735%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:07.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:07 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1164427901' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:00:07.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:07.725 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:07.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:07.827 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:07.876 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:07.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:07.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:07.978 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:08.013 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:08.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:08.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:08.115 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:08.153 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:08.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:08.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:08.255 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:08.281 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:00:08.281 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T12:00:08.281 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T12:00:08.281 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T12:00:08.281 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:00:08.291 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:08.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:08.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:08.393 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:08.426 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:08.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:08.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:08.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:08 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:00:08.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:08 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1164427901' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T12:00:08.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:08 smithi029 ceph-mon[93650]: osdmap e1332: 8 total, 7 up, 6 in 2024-04-12T12:00:08.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:08 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:00:08.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:08 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1164427901' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T12:00:08.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:08 smithi029 ceph-mon[97787]: osdmap e1332: 8 total, 7 up, 6 in 2024-04-12T12:00:08.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:08.563 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:08.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:08.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:08.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:08.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:08.704 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:08.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:08 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:00:08.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:08 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1164427901' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T12:00:08.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:08 smithi087 ceph-mon[89014]: osdmap e1332: 8 total, 7 up, 6 in 2024-04-12T12:00:08.804 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:08.843 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:08.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:08.945 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:08.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:08.984 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:08.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:09.085 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:09.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:09.121 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:09.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:09.222 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:09.259 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:09.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:09.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:09.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:09.402 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:09.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:09.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:09.504 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:09.548 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:09.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:09.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:09.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:09.681 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:09.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:09.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:09 smithi087 ceph-mon[89014]: pgmap v2370: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.7 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.16k op/s; 38171/229218 objects degraded (16.653%) 2024-04-12T12:00:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:09 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:00:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:09 smithi029 ceph-mon[93650]: pgmap v2370: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.7 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.16k op/s; 38171/229218 objects degraded (16.653%) 2024-04-12T12:00:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:09 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:00:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:09 smithi029 ceph-mon[97787]: pgmap v2370: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.7 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.16k op/s; 38171/229218 objects degraded (16.653%) 2024-04-12T12:00:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:09 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:00:09.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:09 smithi029 conmon[297083]: conmon 100a029a168d1c74f99d : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-100a029a168d1c74f99d8aa2a63b939dbdd2332562438dabc42fea062442bf16.scope/container/memory.events 2024-04-12T12:00:09.783 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:09.824 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:09.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:09.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:09.903 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-04-12T12:00:09.925 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:09.969 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:09.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:10.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:10.116 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:10.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:10.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:10.195 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:09 smithi029 podman[296938]: 2024-04-12 12:00:09.874682997 +0000 UTC m=+8.650493807 container died 100a029a168d1c74f99d8aa2a63b939dbdd2332562438dabc42fea062442bf16 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0) 2024-04-12T12:00:10.218 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:10.253 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:10.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:10.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:10.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:10.395 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:10.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:10.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:10.497 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:10.537 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:10.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:10.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:10.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:10.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:10.674 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:10.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:10.775 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:10.810 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:10.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:10.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:10.912 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:10.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:10.959 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:10.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:11.060 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:11.115 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:11.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:11.217 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:11.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:11.265 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:11.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:11.278 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.3.service' 2024-04-12T12:00:11.366 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:11.369 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:00:11.369 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-12T12:00:11.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:11.402 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:11.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:11.503 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:11 smithi029 ceph-mon[93650]: pgmap v2371: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.26k op/s; 36222/216408 objects degraded (16.738%) 2024-04-12T12:00:11.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:11 smithi029 ceph-mon[97787]: pgmap v2371: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.26k op/s; 36222/216408 objects degraded (16.738%) 2024-04-12T12:00:11.538 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:11.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:11.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:11.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:11.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:11.680 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:11.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:11.709 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:11 smithi087 ceph-mon[89014]: pgmap v2371: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.26k op/s; 36222/216408 objects degraded (16.738%) 2024-04-12T12:00:11.782 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:11.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:11.832 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:11.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:11.933 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:11.935 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:00:11.965 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:11.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:11.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:12.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:12.098 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:12.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:12.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:12.200 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:12.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:12.237 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:12.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:12.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:12.375 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:12.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:12.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:12.477 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:12.511 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:12.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:12.613 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:12.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:12.660 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:12.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:00:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:00:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:00:12.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:12.806 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:12.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:12.908 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:12.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:12.944 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:12.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:13.045 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:13.084 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:13.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:13.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:13.186 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:13.237 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:13.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:13.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:13.377 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:13.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:13.479 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:13.484 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:13 smithi029 ceph-mon[93650]: pgmap v2372: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.26k op/s; 36222/216408 objects degraded (16.738%) 2024-04-12T12:00:13.484 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 36222/216408 objects degraded (16.738%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:13.485 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:13 smithi029 ceph-mon[97787]: pgmap v2372: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.26k op/s; 36222/216408 objects degraded (16.738%) 2024-04-12T12:00:13.485 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 36222/216408 objects degraded (16.738%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:13.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:13.521 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:13.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:13.622 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:13.658 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:13.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:13.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:13 smithi087 ceph-mon[89014]: pgmap v2372: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.26k op/s; 36222/216408 objects degraded (16.738%) 2024-04-12T12:00:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 36222/216408 objects degraded (16.738%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:13.761 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:13.805 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:13.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:13.907 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:13.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:13.946 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:13.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:14.002 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T12:00:14.048 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:14.087 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:14.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:14.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:14.189 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:14.224 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:14.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:14.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:14.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:14.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:14.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:14.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:14.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:14 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2327016469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:00:14.416 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:14 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2327016469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:00:14.465 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:14.505 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:14.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:14.604 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T12:00:14.604 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T12:00:14.604 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T12:00:14.604 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T12:00:14.604 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T12:00:14.604 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T12:00:14.604 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T12:00:14.605 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T12:00:14.605 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T12:00:14.605 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T12:00:14.605 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-04-12T12:00:14.605 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T12:00:14.605 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T12:00:14.605 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T12:00:14.605 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T12:00:14.607 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:14.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:14.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:14.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:14.664 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:14 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2327016469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:00:14.746 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:14.783 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:14.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:14.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:14.884 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:14.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:14.920 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:14.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:15.021 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:15.068 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:15.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:15.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:15.170 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:15.196 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:00:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:00:14] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.43.0" 2024-04-12T12:00:15.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:15.219 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:15.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:15.320 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:15.355 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:15.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:15.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:15.458 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:15.502 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:15.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:15.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:15.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:15 smithi029 ceph-mon[97787]: pgmap v2373: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.25k op/s; 35152/210981 objects degraded (16.661%) 2024-04-12T12:00:15.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:15 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:15.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:15 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:15.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:15 smithi029 ceph-mon[93650]: pgmap v2373: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.25k op/s; 35152/210981 objects degraded (16.661%) 2024-04-12T12:00:15.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:15 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:15.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:15 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:15.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:15.642 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:15.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:15.706 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T12:00:15.706 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:00:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:15 smithi087 ceph-mon[89014]: pgmap v2373: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 1.25k op/s; 35152/210981 objects degraded (16.661%) 2024-04-12T12:00:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:15 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:15 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:15.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:15.778 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:15.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:15.879 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:15.910 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:15.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:15.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:16.012 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:16.057 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:16.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:16.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:16.158 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:16.191 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:16.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:16.293 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:16.327 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:16.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:16.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:16.370 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 6, 7, 3] out_osds: [5, 1] dead_osds: [3] live_osds: [0, 1, 2, 4, 6, 7, 5] 2024-04-12T12:00:16.370 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T12:00:16.371 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-04-12T12:00:16.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd in 1 2024-04-12T12:00:16.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:16.464 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:16.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:16.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:16.566 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:16.594 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:16.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:16.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:16.696 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:16.732 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:16.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:16.834 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:16.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:16.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:16.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:16.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:17.016 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:17.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:17.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:17.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-04-12T12:00:17.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:17.152 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:17.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:17.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:17.254 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:17.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:17.309 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:17.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:17.402 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:17 smithi087 ceph-mon[89014]: pgmap v2374: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.21k op/s; 33285/198936 objects degraded (16.732%) 2024-04-12T12:00:17.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:17.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:17.441 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:17.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:17.543 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:17.551 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-04-12T12:00:17.551 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-04-12T12:00:17.579 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:00:17.579 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:00:17.586 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:17.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:17.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:17.688 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:17.719 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:17.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:17.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:17.720 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:17 smithi029 ceph-mon[93650]: pgmap v2374: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.21k op/s; 33285/198936 objects degraded (16.732%) 2024-04-12T12:00:17.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:17 smithi029 ceph-mon[97787]: pgmap v2374: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 4.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.21k op/s; 33285/198936 objects degraded (16.732%) 2024-04-12T12:00:17.821 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:17.858 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:17.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:17.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:17.959 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:17.999 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:18.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:18.100 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:18.136 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:18.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:18.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:18.238 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:18.277 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:18.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:18.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:18.379 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:18.416 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:18.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:18.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:18.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:18.553 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:18.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:18.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:18.597 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 33285/198936 objects degraded (16.732%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:18.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:18.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:18.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:18.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:18.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 33285/198936 objects degraded (16.732%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:18.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 33285/198936 objects degraded (16.732%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:18.793 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:18.830 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:18.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:18.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:18.967 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:18.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:19.068 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:19.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:19.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:19.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:19.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:19.238 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:19.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:19.339 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:19.376 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:19.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:19.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:19.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:19.518 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:19.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:19 smithi029 ceph-mon[97787]: pgmap v2375: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.28k op/s; 32191/193416 objects degraded (16.643%) 2024-04-12T12:00:19.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:19 smithi029 ceph-mon[93650]: pgmap v2375: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.28k op/s; 32191/193416 objects degraded (16.643%) 2024-04-12T12:00:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:19.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:19.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T12:00:19.621 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:19.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:19.672 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:19.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:19.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:19 smithi087 ceph-mon[89014]: pgmap v2375: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1.28k op/s; 32191/193416 objects degraded (16.643%) 2024-04-12T12:00:19.773 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:19.813 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:19.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:19.914 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:19.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:19.939 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:19.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:20.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:20.064 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:20.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:20.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:20.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:20.192 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:20.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:20.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:20.294 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:20.331 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:20.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:20.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:20.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:20.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:20.471 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:20.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:20.572 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:20.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:20.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:20.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:20.715 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:20.738 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:20.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:20.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:20.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:20.863 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:20.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:20.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:20.996 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:20.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:20.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:21.098 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:21.137 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:21.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:21.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:21.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:21.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:21.277 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:21.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:21.378 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:21.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:21.417 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:21.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:21 smithi087 ceph-mon[89014]: pgmap v2376: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.33k op/s; 30252/181182 objects degraded (16.697%) 2024-04-12T12:00:21.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:21.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:21.524 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T12:00:21.524 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:00:21.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:21.543 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:21.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:21.644 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:21.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:21.669 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:21.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:21 smithi029 ceph-mon[97787]: pgmap v2376: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.33k op/s; 30252/181182 objects degraded (16.697%) 2024-04-12T12:00:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:21 smithi029 ceph-mon[93650]: pgmap v2376: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.33k op/s; 30252/181182 objects degraded (16.697%) 2024-04-12T12:00:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:21.769 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:21.802 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:21.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:21.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:21.904 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:21.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:21.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:21.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:22.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:22.074 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:22.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:22.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:22.176 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:22.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:22.208 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:22.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:22.309 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:22.344 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:22.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:22.445 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:22.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:22.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:22.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:22.586 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:22.598 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:00:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:00:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:00:22.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:22.628 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:22.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:22.636 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-04-12T12:00:22.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:22.758 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:22.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:22.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:22.859 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:22.910 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:22.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:22.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:23.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:23.021 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:23.038 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:23.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:23.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:23.140 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:23.178 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:23.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:23.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:23.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:23.314 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:23.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:23.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:23.416 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:23.442 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:23.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:23.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:23.544 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:23.568 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:23.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:23.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:23.669 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:23.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:23.712 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:23.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:23 smithi087 ceph-mon[89014]: pgmap v2377: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 978 op/s; 30252/181182 objects degraded (16.697%) 2024-04-12T12:00:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 30252/181182 objects degraded (16.697%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:23.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:23 smithi029 ceph-mon[97787]: pgmap v2377: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 978 op/s; 30252/181182 objects degraded (16.697%) 2024-04-12T12:00:23.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 30252/181182 objects degraded (16.697%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:23 smithi029 ceph-mon[93650]: pgmap v2377: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 978 op/s; 30252/181182 objects degraded (16.697%) 2024-04-12T12:00:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 30252/181182 objects degraded (16.697%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:23.813 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:23.845 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:23.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:23.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:23.950 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:23.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:23.992 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:23.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:24.093 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:24.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:24.129 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:24.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:24.230 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:24.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:24.255 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:24.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:24.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:24.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:24.380 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:24.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:24.481 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:24.505 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:24.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:24.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:24.607 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:24.643 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:24.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:24.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:24.768 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:24.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:24.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:24.831 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T12:00:24.831 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:00:24.870 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:24.892 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:24.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:24.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:24.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:25.017 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:25.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:25.119 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:25.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:25.161 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:25.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:25.262 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:25.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:00:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:00:24] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.43.0" 2024-04-12T12:00:25.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:25.296 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:25.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:25.397 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:25.434 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:25.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:25.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:25.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:25 smithi087 ceph-mon[89014]: pgmap v2378: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 29323/176610 objects degraded (16.603%) 2024-04-12T12:00:25.537 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:25.556 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:25 smithi029 ceph-mon[97787]: pgmap v2378: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 29323/176610 objects degraded (16.603%) 2024-04-12T12:00:25.556 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:25 smithi029 ceph-mon[93650]: pgmap v2378: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 29323/176610 objects degraded (16.603%) 2024-04-12T12:00:25.574 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:25.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:25.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:25.577 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-04-12T12:00:25.676 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:25.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:25.721 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:25.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:25.822 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:25.858 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:25.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:25.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:25.960 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:25.963 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:26.024 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:26.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:26.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:26.162 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:26.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:26.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:26.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:26.307 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:26.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:26.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:26.409 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:26.433 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:26.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:26.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:26.535 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:26.564 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:26.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:26.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:26.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:26.698 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:26.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:26.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:26.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:26.841 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:26.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:26.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:26.967 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:26.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:26.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:27.069 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:27.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:27.105 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:27.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:27.206 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:27.248 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:27.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:27.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:27.350 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:27.394 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:27.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:27.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:27.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:27.531 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:27.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:27.632 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:27.673 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:27.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:27.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:27.696 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:27 smithi087 ceph-mon[89014]: pgmap v2379: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.21k op/s; 27938/167505 objects degraded (16.679%) 2024-04-12T12:00:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:27 smithi029 ceph-mon[93650]: pgmap v2379: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.21k op/s; 27938/167505 objects degraded (16.679%) 2024-04-12T12:00:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:27 smithi029 ceph-mon[97787]: pgmap v2379: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1.21k op/s; 27938/167505 objects degraded (16.679%) 2024-04-12T12:00:27.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:27.800 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:27.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:27.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:27.849 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T12:00:27.849 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:00:27.902 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:27.933 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:27.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:28.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:28.066 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:28.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:28.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:28.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:28.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:28.193 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:28.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:28.294 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:28.325 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:28.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:28.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:28.461 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:28.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:28.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:28.510 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 27938/167505 objects degraded (16.679%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:28.548 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-04-12T12:00:28.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:28.592 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:28.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:28.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:28.695 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:28.718 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:28.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:28.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:28.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 27938/167505 objects degraded (16.679%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 27938/167505 objects degraded (16.679%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:28.820 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:28.869 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:28.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:28.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:28.908 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:28.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:29.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:29.002 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:29.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:29.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:29.127 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:29.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:29.228 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:29.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:29.254 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:29.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:29.355 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:29.389 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:29.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:29.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:29.490 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:29.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:29.527 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:29.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:29.628 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:29.663 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:29.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:29.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:29 smithi087 ceph-mon[89014]: pgmap v2380: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 986 op/s; 27117/163413 objects degraded (16.594%) 2024-04-12T12:00:29.765 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:29.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:29 smithi029 ceph-mon[97787]: pgmap v2380: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 986 op/s; 27117/163413 objects degraded (16.594%) 2024-04-12T12:00:29.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:29 smithi029 ceph-mon[93650]: pgmap v2380: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 986 op/s; 27117/163413 objects degraded (16.594%) 2024-04-12T12:00:29.800 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:29.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:29.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:29.901 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:29.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:29.938 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:29.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:30.039 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:30.063 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:30.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:30.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:30.195 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:30.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:30.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:30.297 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:30.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:30.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:30.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:30.445 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:30.468 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:30.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:30.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:30.570 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:30.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:30.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:30.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:30.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:30.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:30.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:30.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:30.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:30.872 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:30.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:30.974 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:31.005 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:31.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:31.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:31.107 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:31.146 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:31.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:31.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:31.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:31.279 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:31.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:31.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:31.352 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T12:00:31.352 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:00:31.380 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:31.420 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:31.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:31.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:31.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:31.546 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:31.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:31.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:31.648 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:31.671 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:31.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:31.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:31.696 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:31 smithi087 ceph-mon[89014]: pgmap v2381: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 25776/154194 objects degraded (16.717%) 2024-04-12T12:00:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:31 smithi029 ceph-mon[97787]: pgmap v2381: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 25776/154194 objects degraded (16.717%) 2024-04-12T12:00:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:31 smithi029 ceph-mon[93650]: pgmap v2381: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 25776/154194 objects degraded (16.717%) 2024-04-12T12:00:31.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:31.804 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:31.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:31.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:31.906 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:31.946 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:31.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:32.004 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-04-12T12:00:32.049 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:32.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:32.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:32.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:32.174 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:32.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:32.229 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:32.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:32.276 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:32.330 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:32.366 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:32.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:32.467 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:32.501 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:32.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:32.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:32.582 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:00:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:00:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:00:32.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:32.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:32.634 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:32.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:32.735 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:32.765 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:32.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:32.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:32.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:32.905 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:32.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:32.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:33.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:33.040 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:33.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:33.142 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:33.178 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:33.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:33.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:33.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:33.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:33.317 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:33.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:33.418 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:33.450 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:33.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:33.482 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:00:33.552 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:33.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:33.585 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:33.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:33.636 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:33 smithi029 ceph-mon[93650]: pgmap v2382: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 25776/154194 objects degraded (16.717%) 2024-04-12T12:00:33.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 25776/154194 objects degraded (16.717%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:33.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:33 smithi029 ceph-mon[97787]: pgmap v2382: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 25776/154194 objects degraded (16.717%) 2024-04-12T12:00:33.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 25776/154194 objects degraded (16.717%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:33.685 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:33 smithi087 ceph-mon[89014]: pgmap v2382: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 25776/154194 objects degraded (16.717%) 2024-04-12T12:00:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 25776/154194 objects degraded (16.717%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:33.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:33.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:33.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:33.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:33.856 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:33.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:33.958 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:33.990 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:33.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:33.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:34.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:34.125 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:34.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:34.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:34.227 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:34.269 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:34.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:34.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:33 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T12:00:33.955+0000 7fae24212700 -1 mon.a@0(leader).osd e1332 definitely_dead 0 2024-04-12T12:00:34.271 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T12:00:34.271 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:00:34.370 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:34.400 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:34.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:34.462 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T12:00:34.466 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.1. 2024-04-12T12:00:34.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:34.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:34.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:34.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:34.627 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:34.651 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:34.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:34.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:34.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:34 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2300494322' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T12:00:34.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:34 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/477137947' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2024-04-12T12:00:34.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:34.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:34 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2300494322' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T12:00:34.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:34 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/477137947' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2024-04-12T12:00:34.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:34 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2300494322' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T12:00:34.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:34 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/477137947' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2024-04-12T12:00:34.776 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:34.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:34.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:34.878 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:34.894 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-04-12T12:00:34.903 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:34.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:34.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:35.004 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:35.028 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:35.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:35.092 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:00:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:00:34] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.43.0" 2024-04-12T12:00:35.130 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:35.140 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:35.155 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:35.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:35.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:35.257 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:35.280 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:35.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:35.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:35.382 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:35.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:35.408 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:35.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:35.509 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:35.515 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:35 smithi087 ceph-mon[89014]: pgmap v2383: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 24951/150096 objects degraded (16.623%) 2024-04-12T12:00:35.515 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:35 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:00:35.515 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:35 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2300494322' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T12:00:35.515 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:35 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/477137947' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-04-12T12:00:35.515 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:35 smithi087 ceph-mon[89014]: osdmap e1333: 8 total, 7 up, 7 in 2024-04-12T12:00:35.533 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:35.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:35.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:35.660 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:35.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:35.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:35.762 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:35 smithi029 ceph-mon[93650]: pgmap v2383: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 24951/150096 objects degraded (16.623%) 2024-04-12T12:00:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:35 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:00:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:35 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2300494322' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T12:00:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:35 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/477137947' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-04-12T12:00:35.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:35 smithi029 ceph-mon[93650]: osdmap e1333: 8 total, 7 up, 7 in 2024-04-12T12:00:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:35 smithi029 ceph-mon[97787]: pgmap v2383: 99 pgs: 42 active+undersized, 16 active+undersized+degraded, 41 active+clean; 3.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 24951/150096 objects degraded (16.623%) 2024-04-12T12:00:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:35 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:00:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:35 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2300494322' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T12:00:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:35 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/477137947' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-04-12T12:00:35.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:35 smithi029 ceph-mon[97787]: osdmap e1333: 8 total, 7 up, 7 in 2024-04-12T12:00:35.786 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:35.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:35.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:35.888 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:35.911 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:35.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:35.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:36.013 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:36.036 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:36.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:36.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:36.138 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:36.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:36.279 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:36.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:36.380 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:36.404 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:36.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:36.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:36.557 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:36.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:36.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:36.646 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:36 smithi029 ceph-mon[93650]: osdmap e1334: 8 total, 7 up, 7 in 2024-04-12T12:00:36.646 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:36.646 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:36 smithi029 ceph-mon[93650]: osdmap e1335: 8 total, 7 up, 7 in 2024-04-12T12:00:36.646 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:36 smithi029 ceph-mon[97787]: osdmap e1334: 8 total, 7 up, 7 in 2024-04-12T12:00:36.647 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:36.647 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:36 smithi029 ceph-mon[97787]: osdmap e1335: 8 total, 7 up, 7 in 2024-04-12T12:00:36.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:36.685 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:36 smithi087 ceph-mon[89014]: osdmap e1334: 8 total, 7 up, 7 in 2024-04-12T12:00:36.685 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:36.685 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:36 smithi087 ceph-mon[89014]: osdmap e1335: 8 total, 7 up, 7 in 2024-04-12T12:00:36.697 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:36.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:36.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:36.729 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T12:00:36.763 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T12:00:36.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:36.822 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:36.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:36.824 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T12:00:36.824 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:00:36.912 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T12:00:36.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:36.951 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T12:00:36.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:36.960 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:36.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:36.991 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T12:00:37.027 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T12:00:37.061 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:37.074 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T12:00:37.074 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T12:00:37.075 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T12:00:37.075 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T12:00:37.075 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-04-12T12:00:37.075 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T12:00:37.075 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T12:00:37.075 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T12:00:37.075 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T12:00:37.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:37.099 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:37.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:37.200 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:37.235 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:37.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:37.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:37.337 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:37.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:37.380 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:37.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:37.481 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:37.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-04-12T12:00:37.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:37.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:37.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:37.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:37.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:37.682 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:37.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: pgmap v2386: 99 pgs: 17 peering, 35 active+undersized, 16 active+undersized+degraded, 31 active+clean; 3.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 24380/145887 objects degraded (16.712%) 2024-04-12T12:00:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: 11.0 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11951 2024-04-12T12:00:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: 11.e starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11687 2024-04-12T12:00:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: 11.6 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11999 2024-04-12T12:00:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: 11.3 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11939 2024-04-12T12:00:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: 11.b starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11770 2024-04-12T12:00:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: 11.f starting backfill to osd.1 from (0'0,0'0] MAX to 1332'12033 2024-04-12T12:00:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11884 2024-04-12T12:00:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11857 2024-04-12T12:00:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: 11.c starting backfill to osd.1 from (0'0,0'0] MAX to 1332'12065 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: 11.7 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11738 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2374539251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: pgmap v2386: 99 pgs: 17 peering, 35 active+undersized, 16 active+undersized+degraded, 31 active+clean; 3.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 24380/145887 objects degraded (16.712%) 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: 11.0 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11951 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: 11.e starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11687 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: 11.6 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11999 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: 11.3 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11939 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: 11.b starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11770 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: 11.f starting backfill to osd.1 from (0'0,0'0] MAX to 1332'12033 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11884 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11857 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: 11.c starting backfill to osd.1 from (0'0,0'0] MAX to 1332'12065 2024-04-12T12:00:37.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: 11.7 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11738 2024-04-12T12:00:37.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:37 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2374539251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:00:37.783 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:37.837 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:37.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:37.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:37.863 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:37.889 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-04-12T12:00:37.939 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:37.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: pgmap v2386: 99 pgs: 17 peering, 35 active+undersized, 16 active+undersized+degraded, 31 active+clean; 3.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 24380/145887 objects degraded (16.712%) 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: 11.0 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11951 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: 11.e starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11687 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: 11.6 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11999 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: 11.3 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11939 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: 11.b starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11770 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: 11.f starting backfill to osd.1 from (0'0,0'0] MAX to 1332'12033 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11884 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11857 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: 11.c starting backfill to osd.1 from (0'0,0'0] MAX to 1332'12065 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: 11.7 starting backfill to osd.1 from (0'0,0'0] MAX to 1332'11738 2024-04-12T12:00:37.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:37 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2374539251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:00:37.982 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:37.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:37.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:38.084 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:38.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T12:00:38.122 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:38.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:38.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:38.224 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:38.251 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:38.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:38.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:38.353 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:38.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:38.385 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:38.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:38.486 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:38.525 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:38.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:38.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:38.627 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:38.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:38.678 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:38.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:38 smithi087 ceph-mon[89014]: osdmap e1336: 8 total, 7 up, 7 in 2024-04-12T12:00:38.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 24380/145887 objects degraded (16.712%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:38.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:38 smithi029 ceph-mon[97787]: osdmap e1336: 8 total, 7 up, 7 in 2024-04-12T12:00:38.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 24380/145887 objects degraded (16.712%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:38.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:38 smithi029 ceph-mon[93650]: osdmap e1336: 8 total, 7 up, 7 in 2024-04-12T12:00:38.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 24380/145887 objects degraded (16.712%), 16 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:38.780 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:38.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:38.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:38.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:38.916 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:38.955 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:38.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:38.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:39.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:39.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:39.098 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:39.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:39.199 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:39.216 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-04-12T12:00:39.217 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-04-12T12:00:39.247 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:39.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:39.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:39.349 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:39.382 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:39.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:39.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:39.483 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:39.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:39.532 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:39.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:39.632 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:39.669 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:39.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:39.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:39.681 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:39 smithi087 ceph-mon[89014]: pgmap v2389: 99 pgs: 12 active+clean+remapped, 18 peering, 17 active+undersized, 1 active+remapped+backfill_wait, 15 active+undersized+degraded, 36 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 23883/143187 objects degraded (16.680%); 2932/143187 objects misplaced (2.048%); 16 KiB/s, 0 objects/s recovering 2024-04-12T12:00:39.681 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:39 smithi087 ceph-mon[89014]: osdmap e1337: 8 total, 7 up, 7 in 2024-04-12T12:00:39.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:39 smithi029 ceph-mon[97787]: pgmap v2389: 99 pgs: 12 active+clean+remapped, 18 peering, 17 active+undersized, 1 active+remapped+backfill_wait, 15 active+undersized+degraded, 36 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 23883/143187 objects degraded (16.680%); 2932/143187 objects misplaced (2.048%); 16 KiB/s, 0 objects/s recovering 2024-04-12T12:00:39.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:39 smithi029 ceph-mon[97787]: osdmap e1337: 8 total, 7 up, 7 in 2024-04-12T12:00:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:39 smithi029 ceph-mon[93650]: pgmap v2389: 99 pgs: 12 active+clean+remapped, 18 peering, 17 active+undersized, 1 active+remapped+backfill_wait, 15 active+undersized+degraded, 36 active+clean; 2.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 23883/143187 objects degraded (16.680%); 2932/143187 objects misplaced (2.048%); 16 KiB/s, 0 objects/s recovering 2024-04-12T12:00:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:39 smithi029 ceph-mon[93650]: osdmap e1337: 8 total, 7 up, 7 in 2024-04-12T12:00:39.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:39.776 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:00:39.776 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T12:00:39.776 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.776 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:193925 11.5 11:a8453fa3:::benchmark_data_smithi029_285290_object75234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1337)", 2024-04-12T12:00:39.776 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:00:39.767004+0000", 2024-04-12T12:00:39.776 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.0069323149999999997, 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0069683050000000002, 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "waiting for sub ops", 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 193925 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.767004+0000", 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:39.777 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.767004+0000", 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.767003+0000", 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.767008+0000", 2024-04-12T12:00:39.778 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9010000000000002e-06 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.767018+0000", 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0716e-05 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.767025+0000", 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3679999999999998e-06 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:39.779 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.767071+0000", 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6291000000000001e-05 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.767181+0000", 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000109566 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 6", 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.767307+0000", 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000126756 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.780 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773953+0000", 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0066457950000000003 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.45800.0:193931 11.c e1337/1334 11:30ed33c4:::benchmark_data_smithi029_285290_object75240:head v 1337'12223, mlcod=1337'12221)", 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:00:39.770610+0000", 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.0033263540000000001, 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034561090000000002, 2024-04-12T12:00:39.781 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770610+0000", 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770610+0000", 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.782 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770609+0000", 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770617+0000", 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9990000000000008e-06 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770625+0000", 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1189999999999997e-06 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.783 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770631+0000", 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8340000000000004e-06 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770683+0000", 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1665000000000001e-05 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770703+0000", 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0103e-05 2024-04-12T12:00:39.784 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.45800.0:193927 11.d e1337/1334 11:bf44eea7:::benchmark_data_smithi029_285290_object75236:head v 1337'12064, mlcod=1337'12062)", 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:00:39.770338+0000", 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.0035976739999999999, 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0038035579999999999, 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:39.785 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770338+0000", 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770338+0000", 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770338+0000", 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.786 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770345+0000", 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4649999999999997e-06 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770911+0000", 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00056604500000000005 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.770921+0000", 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9129999999999998e-06 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.787 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773418+0000", 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002496613 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773444+0000", 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5704999999999999e-05 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.788 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.45800.0:193930 11.e e1337/1334 11:726ccd0f:::benchmark_data_smithi029_285290_object75239:head v 1337'11856, mlcod=1337'11854)", 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:00:39.771180+0000", 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 0.0027566040000000002, 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030746839999999998, 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.771180+0000", 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.789 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.771180+0000", 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.771179+0000", 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:39.790 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.771185+0000", 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4450000000000001e-06 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.771199+0000", 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3618000000000001e-05 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.771206+0000", 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0670000000000003e-06 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.791 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.771275+0000", 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9144000000000001e-05 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.771301+0000", 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5374000000000001e-05 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.792 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_repop(client.45800.0:193928 11.0 e1337/1334 11:0f7da528:::benchmark_data_smithi029_285290_object75237:head v 1337'12114, mlcod=1337'12109)", 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:00:39.773860+0000", 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 7.5847999999999996e-05, 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00045520700000000001, 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773860+0000", 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773860+0000", 2024-04-12T12:00:39.793 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773859+0000", 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773873+0000", 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3849000000000001e-05 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.794 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773886+0000", 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3371e-05 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773895+0000", 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5909999999999996e-06 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773936+0000", 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1068000000000001e-05 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:39.795 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:39.796 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:39.773957+0000", 2024-04-12T12:00:39.796 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1200999999999999e-05 2024-04-12T12:00:39.796 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.796 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:39.796 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.796 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:39.796 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2024-04-12T12:00:39.796 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 5 2024-04-12T12:00:39.796 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:00:39.807 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:39.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:39.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:39.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:39.947 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:39.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:39.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:40.049 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:40.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:40.089 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:40.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:40.089 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:00:40.190 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:40.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:40.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:40.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:40.328 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:40.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:40.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:40.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:40.464 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:40.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-04-12T12:00:40.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:40.498 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:40.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:40.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:40.643 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:40.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:40.744 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:40.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:40.788 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:40.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:40.813 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:40.889 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:40.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:40.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:40.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:41.037 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:41.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:41.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:41.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:41.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:41.207 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:41.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:41.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:41.218 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T12:00:41.309 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:41.339 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:41.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:41.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:41.441 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:41.481 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:41.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:41.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:41.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:41 smithi029 ceph-mon[93650]: pgmap v2391: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 1 peering, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 21797/132303 objects degraded (16.475%); 27183/132303 objects misplaced (20.546%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T12:00:41.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:41.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:41.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:41 smithi029 ceph-mon[97787]: pgmap v2391: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 1 peering, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 21797/132303 objects degraded (16.475%); 27183/132303 objects misplaced (20.546%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T12:00:41.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:41.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:41.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:41.655 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:41.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:41.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:41 smithi087 ceph-mon[89014]: pgmap v2391: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 1 peering, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 21797/132303 objects degraded (16.475%); 27183/132303 objects misplaced (20.546%); 4.2 MiB/s, 67 objects/s recovering 2024-04-12T12:00:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:41.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:41.757 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:41.790 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:41.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:41.892 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:41.925 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:41.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:41.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:42.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:42.056 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:42.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:42.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:42.158 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:42.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:42.213 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:42.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:42.314 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:42.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:42.346 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:42.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:42.368 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:00:42.447 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:42.448 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:00:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:00:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:00:42.483 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:42.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:42.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:42.634 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:42.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:42.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:42.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:00:42.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:00:42.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:42 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:42.736 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:42.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:00:42.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:00:42.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:42 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:42.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:00:42.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:00:42.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:42 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:42.779 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:42.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:42.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:42.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:42.890 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 6, 7, 3, 1] out_osds: [5] dead_osds: [3] live_osds: [0, 1, 2, 4, 6, 7, 5] 2024-04-12T12:00:42.890 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2024-04-12T12:00:42.890 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2024-04-12T12:00:42.890 DEBUG:teuthology.orchestra.run.smithi029:osd.3> sudo journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.3.service 2024-04-12T12:00:42.917 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:42.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:42.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:42.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl start ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.3 2024-04-12T12:00:43.021 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:43.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:43.067 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:43.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:43.168 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:43.220 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:43.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:43.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:43.269 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:43 smithi029 systemd[1]: Starting Ceph osd.3 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:00:43.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:43.348 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:43.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:43.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:43.450 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:43.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:43.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:43.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:43.517 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:00:43.517 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T12:00:43.518 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T12:00:43.518 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T12:00:43.518 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:00:43.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:43.633 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:43.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:43.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:43 smithi087 ceph-mon[89014]: pgmap v2392: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 1 peering, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 21797/132303 objects degraded (16.475%); 27183/132303 objects misplaced (20.546%); 3.3 MiB/s, 53 objects/s recovering 2024-04-12T12:00:43.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 21797/132303 objects degraded (16.475%), 15 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:43.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:43 smithi029 ceph-mon[97787]: pgmap v2392: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 1 peering, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 21797/132303 objects degraded (16.475%); 27183/132303 objects misplaced (20.546%); 3.3 MiB/s, 53 objects/s recovering 2024-04-12T12:00:43.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 21797/132303 objects degraded (16.475%), 15 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:43 smithi029 ceph-mon[93650]: pgmap v2392: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 1 peering, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 2.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 754 op/s; 21797/132303 objects degraded (16.475%); 27183/132303 objects misplaced (20.546%); 3.3 MiB/s, 53 objects/s recovering 2024-04-12T12:00:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 21797/132303 objects degraded (16.475%), 15 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:43.775 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:43.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:43.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:43.917 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:43.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:44.019 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:44.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:44.055 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:44.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:44.156 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:44.187 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:44.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:44.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:44.200 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-04-12T12:00:44.289 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:44.325 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:44.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:44.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:44.427 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:44.463 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:44.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:44.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:44.522 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:44.565 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:44.592 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:44.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:44.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:44.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:44.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:44.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:44.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:44.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:44.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:44.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:45.017 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:45.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:45.119 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:45.169 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:45.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:45.237 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:00:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:00:44] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.43.0" 2024-04-12T12:00:45.271 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:45.304 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:45.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:45.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:45.406 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:45.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:45.439 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:45.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:45.540 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:45.542 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T12:00:45.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:45.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:45.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:45.669 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:45.701 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:45.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:45 smithi029 ceph-mon[93650]: pgmap v2393: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 20885/127602 objects degraded (16.367%); 26841/127602 objects misplaced (21.035%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T12:00:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:45 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1381035175' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:00:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:45 smithi029 ceph-mon[97787]: pgmap v2393: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 20885/127602 objects degraded (16.367%); 26841/127602 objects misplaced (21.035%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T12:00:45.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:45 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1381035175' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:00:45.803 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:45.850 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:45.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:45.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:45.909 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:45 smithi087 ceph-mon[89014]: pgmap v2393: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 20885/127602 objects degraded (16.367%); 26841/127602 objects misplaced (21.035%); 2.6 MiB/s, 41 objects/s recovering 2024-04-12T12:00:45.909 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:45 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1381035175' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:00:45.952 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:45.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:45.994 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:45.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:46.095 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:46.127 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:46.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:46.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:46.229 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:46.270 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:46.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:46.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:46.372 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:46.409 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:00:46.409 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T12:00:46.409 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T12:00:46.409 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:25319 10.c 10:3c9dd6fc:::benchmark_data_smithi029_256330_object25318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:53:27.136519+0000", 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 439.26764217499999, 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30098649199999999, 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 25319 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.410 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.136519+0000", 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.136519+0000", 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.411 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.136518+0000", 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.136571+0000", 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3183999999999998e-05 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.136580+0000", 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.166e-06 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.412 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.136587+0000", 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1500000000000002e-06 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.136616+0000", 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.8926999999999999e-05 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.136709+0000", 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3232000000000005e-05 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:00:46.413 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.136806+0000", 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6618000000000002e-05 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.137809+0000", 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001003175 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.138448+0000", 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00063929299999999998 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.414 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.437429+0000", 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29898063699999999 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.437476+0000", 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6925000000000002e-05 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:27.437505+0000", 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.9179999999999998e-05 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.415 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:74789 10.c 10:36b2b576:::benchmark_data_smithi029_256330_object74788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:11.764574+0000", 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 394.63958618200002, 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29244159600000003, 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 74789 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.416 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764574+0000", 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764574+0000", 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764572+0000", 2024-04-12T12:00:46.417 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999976 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764608+0000", 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6573999999999999e-05 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764625+0000", 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6806000000000001e-05 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.418 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764635+0000", 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4700000000000008e-06 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764698+0000", 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2918999999999998e-05 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764886+0000", 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018877799999999999 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:00:46.419 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.765077+0000", 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019037200000000001 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.771075+0000", 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0059978820000000004 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.771789+0000", 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00071427399999999999 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.420 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.056919+0000", 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.28513039400000001 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.056973+0000", 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3940999999999997e-05 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.057016+0000", 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2777000000000001e-05 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.421 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.422 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.422 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.422 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.422 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:74793 10.c 10:3a699fa9:::benchmark_data_smithi029_256330_object74792:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:00:46.422 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:11.767667+0000", 2024-04-12T12:00:46.422 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 394.63649337700002, 2024-04-12T12:00:46.422 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.28946128399999999, 2024-04-12T12:00:46.422 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.422 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.422 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.423 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:00:46.423 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:00:46.423 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 74793 2024-04-12T12:00:46.423 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.423 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.423 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.423 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.423 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767667+0000", 2024-04-12T12:00:46.423 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767667+0000", 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767661+0000", 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999938 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.424 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767707+0000", 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5501999999999999e-05 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767722+0000", 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4946999999999999e-05 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767731+0000", 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4150000000000005e-06 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.425 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767949+0000", 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021811500000000001 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.768139+0000", 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000189991 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.768344+0000", 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000204923 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.426 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.773225+0000", 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0048805979999999999 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.773244+0000", 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8953999999999998e-05 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.057056+0000", 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.28381212099999997 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.427 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.057092+0000", 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6514000000000003e-05 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.057129+0000", 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6424000000000002e-05 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155857 10.c 10:35522eab:::benchmark_data_smithi029_256330_object54091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:00:46.428 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:08.357014+0000", 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 278.04714646899998, 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30658700100000003, 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155857 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357014+0000", 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.429 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357014+0000", 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357012+0000", 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357022+0000", 2024-04-12T12:00:46.430 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1749999999999994e-06 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357041+0000", 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9150000000000001e-05 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357050+0000", 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7440000000000002e-06 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357116+0000", 2024-04-12T12:00:46.431 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5269000000000001e-05 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357266+0000", 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015061799999999999 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357397+0000", 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013073199999999999 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.360799+0000", 2024-04-12T12:00:46.432 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034022169999999999 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.363174+0000", 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023745099999999998 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.663507+0000", 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.300333344 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.663562+0000", 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4648999999999998e-05 2024-04-12T12:00:46.433 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.663601+0000", 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9422999999999998e-05 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:156455 10.c 10:32fc95a4:::benchmark_data_smithi029_256330_object54689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:09.508865+0000", 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 276.89529581900001, 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30657693499999999, 2024-04-12T12:00:46.434 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 156455 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508865+0000", 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508865+0000", 2024-04-12T12:00:46.435 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508863+0000", 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508873+0000", 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0380000000000001e-05 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508892+0000", 2024-04-12T12:00:46.436 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8609000000000001e-05 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508903+0000", 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0621e-05 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508964+0000", 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1135000000000003e-05 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.509120+0000", 2024-04-12T12:00:46.437 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000156088 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.509267+0000", 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014736 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523749+0000", 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014482247 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.438 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523920+0000", 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017029299999999999 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815356+0000", 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29143635699999998 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815398+0000", 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1470000000000001e-05 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.439 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815442+0000", 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4196999999999997e-05 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:156457 10.c 10:3fd59f8a:::benchmark_data_smithi029_256330_object54691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:09.511391+0000", 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 276.89276975400003, 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30413762900000002, 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.440 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 156457 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511391+0000", 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511391+0000", 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.441 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511389+0000", 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511401+0000", 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1233000000000001e-05 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511420+0000", 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9307999999999999e-05 2024-04-12T12:00:46.442 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511430+0000", 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0479000000000001e-05 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512220+0000", 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00078960399999999996 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512340+0000", 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000119793 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.443 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512499+0000", 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015880500000000001 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523780+0000", 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.011281028 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523925+0000", 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000145093 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.444 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815482+0000", 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29155695799999998 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815508+0000", 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5901000000000002e-05 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815529+0000", 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1050999999999999e-05 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.445 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:156459 10.c 10:31889fff:::benchmark_data_smithi029_256330_object54693:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:09.511443+0000", 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 276.89271736699999, 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30423812099999997, 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 156459 2024-04-12T12:00:46.446 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511443+0000", 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511443+0000", 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511443+0000", 2024-04-12T12:00:46.447 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511446+0000", 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5269999999999999e-06 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511454+0000", 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2819999999999999e-06 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.448 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511459+0000", 2024-04-12T12:00:46.449 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.93e-06 2024-04-12T12:00:46.449 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.449 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.449 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:46.450 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:46.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.515749+0000", 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0042892879999999996 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.515965+0000", 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021580199999999999 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.516108+0000", 2024-04-12T12:00:46.451 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014346500000000001 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523911+0000", 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0078030189999999996 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523932+0000", 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1024999999999999e-05 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815579+0000", 2024-04-12T12:00:46.452 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29164664299999998 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815642+0000", 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3788000000000007e-05 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815681+0000", 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8970000000000001e-05 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.453 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:156462 10.c 10:314f156c:::benchmark_data_smithi029_256330_object54696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:09.512595+0000", 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 276.89156532800001, 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.303229625, 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 156462 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.454 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512595+0000", 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512595+0000", 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512595+0000", 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.455 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512599+0000", 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2780000000000004e-06 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512613+0000", 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3898e-05 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512649+0000", 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6001000000000003e-05 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.456 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.519506+0000", 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0068574969999999997 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.519626+0000", 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011934000000000001 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.519743+0000", 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000117133 2024-04-12T12:00:46.457 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.524359+0000", 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0046159110000000003 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.525641+0000", 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001282163 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.458 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815734+0000", 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29009288300000002 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815785+0000", 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0871000000000001e-05 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815825+0000", 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0399999999999999e-05 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.459 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42292 11.f 11:f84fe9a4:::benchmark_data_smithi029_285290_object42291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:14.982397+0000", 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 151.42176332599999, 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.57496645999999996, 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42292 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.460 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982397+0000", 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982397+0000", 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982396+0000", 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.461 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982432+0000", 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5750000000000002e-05 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982443+0000", 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1129e-05 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982450+0000", 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5560000000000004e-06 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.462 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982732+0000", 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00028165900000000002 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982870+0000", 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000138663 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983071+0000", 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00020081499999999999 2024-04-12T12:00:46.463 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988302+0000", 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0052308739999999999 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988376+0000", 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3998000000000005e-05 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557253+0000", 2024-04-12T12:00:46.464 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.56887719000000003 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557316+0000", 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3207999999999995e-05 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557364+0000", 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7539e-05 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.465 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42294 11.f 11:f38b9eb2:::benchmark_data_smithi029_285290_object42293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:14.983465+0000", 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 151.42069552800001, 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.57400737099999999, 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42294 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.466 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983465+0000", 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983465+0000", 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983464+0000", 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:00:46.467 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983567+0000", 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000102444 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983576+0000", 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1109999999999997e-06 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983584+0000", 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5889999999999997e-06 2024-04-12T12:00:46.468 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984590+0000", 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0010066929999999999 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984742+0000", 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015193199999999999 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:00:46.469 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984898+0000", 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015533699999999999 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988417+0000", 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0035192769999999999 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988425+0000", 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5490000000000006e-06 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.470 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557414+0000", 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.56898828000000001 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557447+0000", 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3859000000000002e-05 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557473+0000", 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5041e-05 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.471 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42299 11.f 11:f26ba9ae:::benchmark_data_smithi029_285290_object42298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:14.983989+0000", 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 151.420171654, 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.57362241999999997, 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42299 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.472 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983989+0000", 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983989+0000", 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983988+0000", 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984062+0000", 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4051e-05 2024-04-12T12:00:46.473 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984078+0000", 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5931e-05 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984086+0000", 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8059999999999995e-06 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.986408+0000", 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023222999999999998 2024-04-12T12:00:46.474 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.986530+0000", 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012160299999999999 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.986664+0000", 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013399199999999999 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988442+0000", 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0017779790000000001 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.475 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.990171+0000", 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001729284 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557527+0000", 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.56735576799999998 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557575+0000", 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8019000000000002e-05 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.476 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557611+0000", 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6665999999999998e-05 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42301 11.f 11:f820092e:::benchmark_data_smithi029_285290_object42300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:14.984094+0000", 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 151.42006716500001, 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.57360671299999999, 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.477 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42301 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984094+0000", 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.478 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984094+0000", 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984093+0000", 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984145+0000", 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2117999999999998e-05 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.479 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984154+0000", 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2e-06 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984160+0000", 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3619999999999997e-06 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988465+0000", 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0043049309999999997 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.480 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988603+0000", 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000138465 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988748+0000", 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000145038 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.990321+0000", 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001573004 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.481 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.990829+0000", 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00050818399999999996 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557648+0000", 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.56681834499999995 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557675+0000", 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.7061000000000001e-05 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.482 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557700+0000", 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5613e-05 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42313 11.f 11:f2a54442:::benchmark_data_smithi029_285290_object42312:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:15.203953+0000", 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 151.20020793, 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.35387521399999999, 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.483 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42313 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.203953+0000", 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.203953+0000", 2024-04-12T12:00:46.484 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.203948+0000", 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999957 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204009+0000", 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0476999999999997e-05 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204023+0000", 2024-04-12T12:00:46.485 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3740000000000001e-05 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204030+0000", 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4320000000000002e-06 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204068+0000", 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8158000000000003e-05 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.486 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204185+0000", 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000116706 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204306+0000", 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000120802 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.206658+0000", 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023524689999999998 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.207824+0000", 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001166203 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557738+0000", 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34991402999999999 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557786+0000", 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7219000000000003e-05 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557828+0000", 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2357000000000001e-05 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42326 11.f 11:f45f1e17:::benchmark_data_smithi029_285290_object42325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:15.211209+0000", 2024-04-12T12:00:46.489 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 151.19295187899999, 2024-04-12T12:00:46.490 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34961550800000002, 2024-04-12T12:00:46.490 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.490 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.490 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.490 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42326 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211209+0000", 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211209+0000", 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211208+0000", 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211258+0000", 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0544999999999998e-05 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211269+0000", 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0762e-05 2024-04-12T12:00:46.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211277+0000", 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.7409999999999992e-06 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211333+0000", 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6910000000000002e-05 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211478+0000", 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000144766 2024-04-12T12:00:46.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211571+0000", 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3009000000000003e-05 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.214394+0000", 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0028226839999999998 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.215203+0000", 2024-04-12T12:00:46.494 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00080880299999999997 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.560735+0000", 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34553186200000002 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.560785+0000", 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0766000000000001e-05 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.495 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.560824+0000", 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8940000000000003e-05 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42329 11.f 11:f2d1521d:::benchmark_data_smithi029_285290_object42328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:15.211568+0000", 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 151.19259285999999, 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34974637800000002, 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.496 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42329 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211568+0000", 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211568+0000", 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211567+0000", 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211630+0000", 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2470000000000003e-05 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.498 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211640+0000", 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0207000000000001e-05 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211647+0000", 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1509999999999998e-06 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.213167+0000", 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0015203879999999999 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.499 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.213296+0000", 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012849700000000001 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.213395+0000", 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9518999999999994e-05 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.214502+0000", 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0011064580000000001 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.500 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.215439+0000", 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00093702200000000003 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.561245+0000", 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34580587099999999 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.561277+0000", 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.2107000000000003e-05 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.561314+0000", 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.7336999999999998e-05 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:113780 11.4 11:2796d0fb:::benchmark_data_smithi029_285290_object113779:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:59:17.425287+0000", 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 88.97887394, 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.52363299600000002, 2024-04-12T12:00:46.502 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 113780 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425287+0000", 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.503 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425287+0000", 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425286+0000", 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425343+0000", 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7195999999999999e-05 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425353+0000", 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0541e-05 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425359+0000", 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7080000000000002e-06 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425393+0000", 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3859000000000002e-05 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425506+0000", 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000112747 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425609+0000", 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000103063 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.429488+0000", 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0038795090000000002 2024-04-12T12:00:46.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.431822+0000", 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023336070000000001 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.948860+0000", 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.517037882 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.948895+0000", 2024-04-12T12:00:46.507 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4696000000000001e-05 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.948920+0000", 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5208999999999999e-05 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:113804 11.5 11:aaaf7d84:::benchmark_data_smithi029_285290_object113803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:59:17.443513+0000", 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 88.960647972000004, 2024-04-12T12:00:46.508 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50833982200000005, 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 113804 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443513+0000", 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443513+0000", 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443511+0000", 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443552+0000", 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0726999999999998e-05 2024-04-12T12:00:46.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443563+0000", 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0755e-05 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443569+0000", 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3380000000000001e-06 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443621+0000", 2024-04-12T12:00:46.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1409000000000002e-05 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.446090+0000", 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0024690810000000001 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.446283+0000", 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019277500000000001 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450136+0000", 2024-04-12T12:00:46.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0038537710000000002 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450272+0000", 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013551200000000001 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.951798+0000", 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.501526307 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.513 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.951829+0000", 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.1095e-05 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.951853+0000", 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3329999999999999e-05 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:113807 11.4 11:25f36c58:::benchmark_data_smithi029_285290_object113806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:59:17.447507+0000", 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 88.956653293000002, 2024-04-12T12:00:46.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50804887099999996, 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 113807 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447507+0000", 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447507+0000", 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447506+0000", 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447554+0000", 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8733999999999998e-05 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447573+0000", 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8712999999999999e-05 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447583+0000", 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3829999999999998e-06 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447674+0000", 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1161000000000004e-05 2024-04-12T12:00:46.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447835+0000", 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000161466 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.448016+0000", 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000180707 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.453806+0000", 2024-04-12T12:00:46.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0057896070000000004 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.453834+0000", 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.8801000000000001e-05 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955467+0000", 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50163289700000002 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955518+0000", 2024-04-12T12:00:46.519 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0488000000000002e-05 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955556+0000", 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8535000000000003e-05 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:113808 11.4 11:23337bbc:::benchmark_data_smithi029_285290_object113807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:59:17.450659+0000", 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 88.953501457000002, 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50499450800000001, 2024-04-12T12:00:46.520 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 113808 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450659+0000", 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450659+0000", 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450658+0000", 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450710+0000", 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2609999999999999e-05 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450724+0000", 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3519999999999999e-05 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450732+0000", 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9189999999999999e-06 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.451102+0000", 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00037062999999999999 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.451297+0000", 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019469299999999999 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.451401+0000", 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000103799 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.453953+0000", 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0025525299999999999 2024-04-12T12:00:46.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.454033+0000", 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0116000000000003e-05 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955596+0000", 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50156283599999996 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955632+0000", 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5729000000000003e-05 2024-04-12T12:00:46.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955654+0000", 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1715000000000001e-05 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:197194 11.4 11:2fcc6a73:::benchmark_data_smithi029_285290_object78503:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1337)", 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:00:43.220808+0000", 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 3.1833527560000001, 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.13795716499999999, 2024-04-12T12:00:46.526 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 197194 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.220808+0000", 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.527 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.220808+0000", 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.220807+0000", 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.220813+0000", 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8630000000000002e-06 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.220823+0000", 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.448e-06 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.220830+0000", 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6059999999999997e-06 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.221037+0000", 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000206951 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.221345+0000", 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00030787300000000001 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.221688+0000", 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00034316499999999998 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.227769+0000", 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0060809119999999999 2024-04-12T12:00:46.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.228310+0000", 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00054045799999999995 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.358679+0000", 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.13036962199999999 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.358725+0000", 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5858e-05 2024-04-12T12:00:46.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:00:46.532 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:00:46.532 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:00:46.532 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:00:43.358765+0000", 2024-04-12T12:00:46.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9802000000000003e-05 2024-04-12T12:00:46.532 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.532 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.532 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.532 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:00:46.532 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:00:46.532 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:00:46.551 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:46.585 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:46.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:46.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:46.686 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:46 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T12:00:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:46 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1381035175' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T12:00:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:46 smithi087 ceph-mon[89014]: osdmap e1338: 8 total, 7 up, 7 in 2024-04-12T12:00:46.721 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:46.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:46.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:46 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T12:00:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:46 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1381035175' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T12:00:46.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:46 smithi029 ceph-mon[97787]: osdmap e1338: 8 total, 7 up, 7 in 2024-04-12T12:00:46.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:46 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T12:00:46.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:46 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1381035175' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T12:00:46.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:46 smithi029 ceph-mon[93650]: osdmap e1338: 8 total, 7 up, 7 in 2024-04-12T12:00:46.823 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:46.859 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:46.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:46.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:46.961 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:46.997 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:46.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:46.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:47.099 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:47.136 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:47.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:47.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:47.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:47.269 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:47 smithi029 podman[301242]: 2024-04-12 12:00:47.07642047 +0000 UTC m=+0.043625238 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T12:00:47.282 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:47.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:47.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:47.383 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:47.411 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:47.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:47.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:47.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:47.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:47.559 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:47.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:47.660 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:47.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:47 smithi029 ceph-mon[93650]: pgmap v2395: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 19013/118476 objects degraded (16.048%); 24358/118476 objects misplaced (20.559%); 7.0 MiB/s, 111 objects/s recovering 2024-04-12T12:00:47.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:47 smithi029 ceph-mon[97787]: pgmap v2395: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 19013/118476 objects degraded (16.048%); 24358/118476 objects misplaced (20.559%); 7.0 MiB/s, 111 objects/s recovering 2024-04-12T12:00:47.678 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:47 smithi029 podman[301242]: 2024-04-12 12:00:47.387934732 +0000 UTC m=+0.355139486 container create b5dbaf25d2f4ea9dce6c5636b3c0f5ae09d257c91a9071bbeac409792e4aa223 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T12:00:47.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:47.691 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:47.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:47.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:47.821 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:47.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:47.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:47.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:47.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:47.953 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:47.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:47.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:47 smithi087 ceph-mon[89014]: pgmap v2395: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 19013/118476 objects degraded (16.048%); 24358/118476 objects misplaced (20.559%); 7.0 MiB/s, 111 objects/s recovering 2024-04-12T12:00:48.055 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:48.089 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:48.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:48.191 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:48.225 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:48.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:48.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:48.230 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:48 smithi029 podman[301242]: 2024-04-12 12:00:48.091084737 +0000 UTC m=+1.058289500 container init b5dbaf25d2f4ea9dce6c5636b3c0f5ae09d257c91a9071bbeac409792e4aa223 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-12T12:00:48.230 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:48 smithi029 podman[301242]: 2024-04-12 12:00:48.09821997 +0000 UTC m=+1.065424725 container start b5dbaf25d2f4ea9dce6c5636b3c0f5ae09d257c91a9071bbeac409792e4aa223 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-12T12:00:48.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:48.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:48.370 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:48.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:48.471 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:48.482 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T12:00:48.482 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:48 smithi029 podman[301242]: 2024-04-12 12:00:48.327433937 +0000 UTC m=+1.294638707 container attach b5dbaf25d2f4ea9dce6c5636b3c0f5ae09d257c91a9071bbeac409792e4aa223 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185) 2024-04-12T12:00:48.500 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:48.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:48.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:48.535 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T12:00:48.578 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T12:00:48.602 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:48.622 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T12:00:48.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:48.642 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:48.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T12:00:48.708 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T12:00:48.743 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:48.762 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T12:00:48.762 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 19013/118476 objects degraded (16.048%), 15 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:48.762 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:48 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2747009700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:00:48.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 19013/118476 objects degraded (16.048%), 15 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:48.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:48 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2747009700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:00:48.763 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T12:00:48.763 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T12:00:48.763 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T12:00:48.763 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-04-12T12:00:48.763 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T12:00:48.763 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T12:00:48.763 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T12:00:48.763 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T12:00:48.773 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:48.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:48.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:48.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:48.907 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:48.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:48.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:48.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 19013/118476 objects degraded (16.048%), 15 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:48.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:48 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2747009700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:00:49.009 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:49.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:49.035 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:49.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:49.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set noscrub 2024-04-12T12:00:49.135 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:49.160 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:49.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:49.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:49.211 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-04-12T12:00:49.262 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:49.311 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:49.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:49.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:49.435 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:49.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:49.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:49.470 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:49.537 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:49.569 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:49.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:49.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:49.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:49.711 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:49.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:49.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:49.813 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:49.843 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:49.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:49.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:49.945 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:49.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:49 smithi087 ceph-mon[89014]: pgmap v2396: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 992 op/s; 18170/114198 objects degraded (15.911%); 24046/114198 objects misplaced (21.056%); 5.7 MiB/s, 91 objects/s recovering 2024-04-12T12:00:49.989 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:49.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:49.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:50.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:49 smithi029 ceph-mon[93650]: pgmap v2396: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 992 op/s; 18170/114198 objects degraded (15.911%); 24046/114198 objects misplaced (21.056%); 5.7 MiB/s, 91 objects/s recovering 2024-04-12T12:00:50.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:49 smithi029 ceph-mon[97787]: pgmap v2396: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 992 op/s; 18170/114198 objects degraded (15.911%); 24046/114198 objects misplaced (21.056%); 5.7 MiB/s, 91 objects/s recovering 2024-04-12T12:00:50.092 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:50.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:50.127 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:50.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:50.228 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:50.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:50.264 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:50.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:50.365 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:50.404 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:50.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:50.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:50.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:50.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:50.554 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:50.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:50.655 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:50.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:50.695 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:50.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:50.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:50.829 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:50.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:50.879 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-04-12T12:00:50.880 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-04-12T12:00:50.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:50.973 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:50.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:50.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:51.019 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[301292]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T12:00:51.019 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[301292]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T12:00:51.019 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[301292]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[301292]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[301292]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[301292]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate[301292]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 bash[301242]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 bash[301242]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 bash[301242]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 bash[301242]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 bash[301242]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 bash[301242]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 bash[301242]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-12T12:00:51.020 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:50 smithi029 podman[301242]: 2024-04-12 12:00:50.85760239 +0000 UTC m=+3.824807145 container died b5dbaf25d2f4ea9dce6c5636b3c0f5ae09d257c91a9071bbeac409792e4aa223 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-12T12:00:51.077 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:51.109 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:51.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:51.210 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:51.249 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:51.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:51.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:51.351 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:51.387 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:51.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:51.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:51.489 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:51.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:51.530 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:51.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:51.631 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:51.670 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:51.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:51.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:51.772 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:51.809 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:51.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:51.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:51.816 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:00:51.822 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T12:00:51.823 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:00:51.823 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:51 smithi087 ceph-mon[89014]: pgmap v2397: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 16287/104970 objects degraded (15.516%); 21527/104970 objects misplaced (20.508%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:00:51.823 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:51.823 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:51.911 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:51.939 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:51 smithi029 podman[301641]: 2024-04-12 12:00:51.616657398 +0000 UTC m=+0.798948698 container remove b5dbaf25d2f4ea9dce6c5636b3c0f5ae09d257c91a9071bbeac409792e4aa223 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-activate, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T12:00:51.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:51 smithi029 ceph-mon[93650]: pgmap v2397: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 16287/104970 objects degraded (15.516%); 21527/104970 objects misplaced (20.508%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:00:51.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:51.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:51.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:51 smithi029 ceph-mon[97787]: pgmap v2397: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 16287/104970 objects degraded (15.516%); 21527/104970 objects misplaced (20.508%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:00:51.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:00:51.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:00:51.947 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:51.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:51.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:52.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:52.092 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:52.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:52.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:52.194 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:52.229 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:52.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:52.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:52.269 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:51 smithi029 podman[301763]: 2024-04-12 12:00:51.84264694 +0000 UTC m=+0.041090817 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T12:00:52.269 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:51 smithi029 podman[301763]: 2024-04-12 12:00:51.987656689 +0000 UTC m=+0.186100607 container create e138af00dd594968e95b8d25d262eaa1fdd2f4b32935bec948335bd830f9984b (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-12T12:00:52.332 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:52.371 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:52.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:52.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:52.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-04-12T12:00:52.473 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:52.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:52.510 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:52.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:52.611 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:52.642 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:52 smithi029 podman[301763]: 2024-04-12 12:00:52.319228523 +0000 UTC m=+0.517672389 container init e138af00dd594968e95b8d25d262eaa1fdd2f4b32935bec948335bd830f9984b (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-rf-comb1-2cee185) 2024-04-12T12:00:52.642 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:52 smithi029 podman[301763]: 2024-04-12 12:00:52.325468377 +0000 UTC m=+0.523912239 container start e138af00dd594968e95b8d25d262eaa1fdd2f4b32935bec948335bd830f9984b (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-04-12T12:00:52.643 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:52 smithi029 ceph-osd[301841]: -- 172.21.15.29:0/1764463467 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564eb4997680 con 0x564eb4961000 2024-04-12T12:00:52.643 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:52 smithi029 bash[301763]: e138af00dd594968e95b8d25d262eaa1fdd2f4b32935bec948335bd830f9984b 2024-04-12T12:00:52.643 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:52 smithi029 systemd[1]: Started Ceph osd.3 for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T12:00:52.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:52.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:52.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:52.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:00:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:00:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:00:52.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:52.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:52.805 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:52.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:52.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T12:00:52.906 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:52.975 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:52.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:52.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:53.076 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:53.108 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:53.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:53.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:53.210 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:53.251 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:53.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:53.268 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:00:52 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:00:52.959+0000 7f6b2fd407c0 -1 Falling back to public interface 2024-04-12T12:00:53.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:53.392 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:53.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:53.494 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:53.530 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:53.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:53.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:53.620 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-04-12T12:00:53.634 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:53.672 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:53.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:53.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:53.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:53.809 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:53.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:53.910 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:53.934 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:53.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:53.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:53.938 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:53 smithi087 ceph-mon[89014]: pgmap v2398: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 16287/104970 objects degraded (15.516%); 21527/104970 objects misplaced (20.508%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:00:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 16287/104970 objects degraded (15.516%), 15 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:54.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:53 smithi029 ceph-mon[93650]: pgmap v2398: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 16287/104970 objects degraded (15.516%); 21527/104970 objects misplaced (20.508%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:00:54.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 16287/104970 objects degraded (15.516%), 15 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:54.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:53 smithi029 ceph-mon[97787]: pgmap v2398: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 16287/104970 objects degraded (15.516%); 21527/104970 objects misplaced (20.508%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:00:54.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 16287/104970 objects degraded (15.516%), 15 pgs degraded (PG_DEGRADED) 2024-04-12T12:00:54.036 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:54.060 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:54.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:54.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:54.162 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:54.208 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:54.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:54.309 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:54.362 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:54.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:54.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:54.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:54.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:54.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:54.601 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:54.632 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:54.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:54.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:54.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:54.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:54.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:54.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:54.873 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:54.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:54.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:54.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:55.005 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:55.030 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:55.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:55.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:55.132 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:55.167 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:55.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:55.268 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:55.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:00:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:00:54] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.43.0" 2024-04-12T12:00:55.311 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:55.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:55.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:55.414 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:55.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:55.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:55.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:55.541 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:55.569 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:55.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:55.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:55.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:55.711 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:55.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:55 smithi087 ceph-mon[89014]: pgmap v2399: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 15419/100656 objects degraded (15.319%); 21235/100656 objects misplaced (21.097%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:00:55.814 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:55.837 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:55.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:55.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:55.939 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:55.978 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:55.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:55.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:56.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:55 smithi029 ceph-mon[97787]: pgmap v2399: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 15419/100656 objects degraded (15.319%); 21235/100656 objects misplaced (21.097%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:00:56.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:55 smithi029 ceph-mon[93650]: pgmap v2399: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 2.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 15419/100656 objects degraded (15.319%); 21235/100656 objects misplaced (21.097%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:00:56.079 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:56.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:56.112 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:56.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:56.213 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:56.243 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:56.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:56.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:56.345 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:56.377 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:56.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:56.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:56.479 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:56.507 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:56.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:56.591 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T12:00:56.592 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:00:56.609 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:56.641 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:56.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:56.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:56.742 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:56.777 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:56.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:56.878 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:56.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:56.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:56.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:57.014 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:57.043 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:57.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:57.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:57.145 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:57.181 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:57.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:57.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:57.257 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-04-12T12:00:57.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:57.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:57.318 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:57.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:57.419 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:57.443 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:57.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:57.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:57.490 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:00:57.545 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:57.582 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:57.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:57.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:57.684 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:57.720 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:57.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:57.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:57.822 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:57.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:57.848 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:57.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:57.853 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:57 smithi087 ceph-mon[89014]: pgmap v2400: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 13658/91374 objects degraded (14.947%); 18703/91374 objects misplaced (20.469%); 6.8 MiB/s, 108 objects/s recovering 2024-04-12T12:00:57.949 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:57.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:57.987 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:57.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:58.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:57 smithi029 ceph-mon[97787]: pgmap v2400: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 13658/91374 objects degraded (14.947%); 18703/91374 objects misplaced (20.469%); 6.8 MiB/s, 108 objects/s recovering 2024-04-12T12:00:58.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:57 smithi029 ceph-mon[93650]: pgmap v2400: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 844 op/s; 13658/91374 objects degraded (14.947%); 18703/91374 objects misplaced (20.469%); 6.8 MiB/s, 108 objects/s recovering 2024-04-12T12:00:58.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:00:58.126 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:58.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:58.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:00:58.228 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:58.258 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:58.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:58.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:58.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:58.405 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:58.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:58.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:58.506 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:00:58.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:58.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:58.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:00:58.636 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:58.659 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:58 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 13658/91374 objects degraded (14.947%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-04-12T12:00:58.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:58.661 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:58.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:58.762 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:58.800 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:58.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:58.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:58.902 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:00:58.930 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:58.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:58.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:00:59.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:58 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 13658/91374 objects degraded (14.947%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-04-12T12:00:59.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:58 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 13658/91374 objects degraded (14.947%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-04-12T12:00:59.032 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:59.057 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:59.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:59.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:59.159 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:00:59.184 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:59.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:59.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:00:59.264 INFO:teuthology.orchestra.run.smithi087.stdout:ERROR: (22) Invalid argument 2024-04-12T12:00:59.264 INFO:teuthology.orchestra.run.smithi087.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:00:59.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:59.312 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:59.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:59.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:59.414 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:00:59.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:59.439 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:00:59.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:00:59.540 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:00:59.567 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:59.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:59.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:00:59.669 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:59.695 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:59.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:59.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:59.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:00:59.832 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:00:59.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:00:59.869 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:59 smithi087 ceph-mon[89014]: pgmap v2401: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 12935/87804 objects degraded (14.732%); 18455/87804 objects misplaced (21.018%); 6.1 MiB/s, 97 objects/s recovering 2024-04-12T12:00:59.869 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:00:59 smithi087 ceph-mon[89014]: osdmap e1339: 8 total, 7 up, 7 in 2024-04-12T12:00:59.901 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-04-12T12:00:59.933 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:00:59.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:59 smithi029 ceph-mon[97787]: pgmap v2401: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 12935/87804 objects degraded (14.732%); 18455/87804 objects misplaced (21.018%); 6.1 MiB/s, 97 objects/s recovering 2024-04-12T12:00:59.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:00:59 smithi029 ceph-mon[97787]: osdmap e1339: 8 total, 7 up, 7 in 2024-04-12T12:00:59.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:59 smithi029 ceph-mon[93650]: pgmap v2401: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 12935/87804 objects degraded (14.732%); 18455/87804 objects misplaced (21.018%); 6.1 MiB/s, 97 objects/s recovering 2024-04-12T12:00:59.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:00:59 smithi029 ceph-mon[93650]: osdmap e1339: 8 total, 7 up, 7 in 2024-04-12T12:00:59.999 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:00.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:00.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:00.101 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:00.129 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:00.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:00.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:00.231 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:00.263 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:00.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:00.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:00.364 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:00.388 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:00.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:00.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:00.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:00.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:00.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:00.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:00.636 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:00.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:00.675 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:00.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:00.776 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:00.804 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:00.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:00.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:00.906 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:00.949 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:00.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:00.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:00.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:00 smithi087 ceph-mon[89014]: osdmap e1340: 8 total, 7 up, 7 in 2024-04-12T12:01:00.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:00 smithi087 ceph-mon[89014]: pgmap v2404: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 875 op/s; 12574/83952 objects degraded (14.978%); 17351/83952 objects misplaced (20.668%); 4.5 MiB/s, 72 objects/s recovering 2024-04-12T12:01:01.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:00 smithi029 ceph-mon[97787]: osdmap e1340: 8 total, 7 up, 7 in 2024-04-12T12:01:01.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:00 smithi029 ceph-mon[97787]: pgmap v2404: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 875 op/s; 12574/83952 objects degraded (14.978%); 17351/83952 objects misplaced (20.668%); 4.5 MiB/s, 72 objects/s recovering 2024-04-12T12:01:01.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:00 smithi029 ceph-mon[93650]: osdmap e1340: 8 total, 7 up, 7 in 2024-04-12T12:01:01.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:00 smithi029 ceph-mon[93650]: pgmap v2404: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 875 op/s; 12574/83952 objects degraded (14.978%); 17351/83952 objects misplaced (20.668%); 4.5 MiB/s, 72 objects/s recovering 2024-04-12T12:01:01.051 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:01.087 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:01.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:01.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:01.189 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:01.226 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:01.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:01.327 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:01.362 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:01.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:01.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:01.464 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:01.491 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:01.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:01.593 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:01.624 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:01.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:01.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:01.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:01.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:01.808 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:01.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:01.909 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:01.933 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:01.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:02.034 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:02.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:02.076 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:02.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:02.177 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:02.208 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:02.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:02.310 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:02.333 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:02.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:02.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:02.435 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:02.464 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:02.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:02.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:02.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:02.609 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:02.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:02.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:02.711 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:01:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:01:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:01:02.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:02.740 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:02.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:02.841 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:02.875 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:02.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:02.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:02.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:03.008 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:03.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:03.110 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:03.145 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:03.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:03.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:03.247 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:03.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:03.278 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:03.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:03.379 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:03.410 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:03.410 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:01:03.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:03.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:03.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:03.561 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:03.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:03.603 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:01:03.605 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:01:03.640 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:03 smithi029 ceph-mon[93650]: pgmap v2405: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 11565/82398 objects degraded (14.036%); 16999/82398 objects misplaced (20.630%); 9.9 MiB/s, 158 objects/s recovering 2024-04-12T12:01:03.640 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:03 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 11565/82398 objects degraded (14.036%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-04-12T12:01:03.641 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:03 smithi029 ceph-mon[97787]: pgmap v2405: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 11565/82398 objects degraded (14.036%); 16999/82398 objects misplaced (20.630%); 9.9 MiB/s, 158 objects/s recovering 2024-04-12T12:01:03.641 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:03 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 11565/82398 objects degraded (14.036%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-04-12T12:01:03.662 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:03.696 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:03.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:03.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:03 smithi087 ceph-mon[89014]: pgmap v2405: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 15 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 11565/82398 objects degraded (14.036%); 16999/82398 objects misplaced (20.630%); 9.9 MiB/s, 158 objects/s recovering 2024-04-12T12:01:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:03 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 11565/82398 objects degraded (14.036%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-04-12T12:01:03.798 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:03.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:03.826 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:03.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:03.927 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:03.950 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:03.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:03.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:04.052 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:04.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:04.090 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:04.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:04.191 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:04.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:04.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:04.327 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:04.351 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:04.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:04.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:04.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:04.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:04.502 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:04.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:04.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:04.644 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:04.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:04.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:04.746 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:04.777 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:04.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:04.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:04.879 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:04.909 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:04.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:04.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:05.011 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:05.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:05.036 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:05.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:05.137 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:05.173 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:05.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:05.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:05.249 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:01:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:01:04] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.43.0" 2024-04-12T12:01:05.275 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:05.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:05.302 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:05.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:05.403 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:05.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:05.447 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:05.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:05.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:05 smithi029 ceph-mon[93650]: pgmap v2406: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 10742/78627 objects degraded (13.662%); 16500/78627 objects misplaced (20.985%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:01:05.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:05 smithi029 ceph-mon[97787]: pgmap v2406: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 10742/78627 objects degraded (13.662%); 16500/78627 objects misplaced (20.985%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:01:05.549 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:05.584 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:05.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:05.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:05.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:05 smithi087 ceph-mon[89014]: pgmap v2406: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 10742/78627 objects degraded (13.662%); 16500/78627 objects misplaced (20.985%); 7.2 MiB/s, 115 objects/s recovering 2024-04-12T12:01:05.723 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:05.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:05.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:05.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:05.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:05.862 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:05.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:05.963 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:05.999 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:05.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:06.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:06.018 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:01:05 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:01:05.517+0000 7f6b2fd407c0 -1 osd.3 1329 log_to_monitors true 2024-04-12T12:01:06.101 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:06.135 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:06.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:06.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:06.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:06.281 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:06.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:06.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:06.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:06.421 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:06.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:06.457 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:06 smithi029 ceph-mon[97787]: from='osd.3 [v2:172.21.15.29:6826/1944387251,v1:172.21.15.29:6827/1944387251]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T12:01:06.457 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:06 smithi029 ceph-mon[97787]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T12:01:06.458 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:06 smithi029 ceph-mon[93650]: from='osd.3 [v2:172.21.15.29:6826/1944387251,v1:172.21.15.29:6827/1944387251]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T12:01:06.458 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:06 smithi029 ceph-mon[93650]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T12:01:06.458 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:01:06 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:01:06.373+0000 7f6b1d190700 -1 osd.3 1329 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T12:01:06.522 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:06.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:06.557 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:06.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:06.658 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:06.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:06.691 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:06.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:06 smithi087 ceph-mon[89014]: from='osd.3 [v2:172.21.15.29:6826/1944387251,v1:172.21.15.29:6827/1944387251]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T12:01:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:06 smithi087 ceph-mon[89014]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T12:01:06.792 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:06.825 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:06.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:06.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:06.927 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:06.958 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:06.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:07.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:07.100 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:07.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:07.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:07.202 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:07.233 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:07.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:07.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:07.335 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:07.367 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:07.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:07.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:07.469 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:07.493 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:07.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:07.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:07.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:07.619 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:07.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:07.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:07 smithi087 ceph-mon[89014]: pgmap v2407: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 9643/71868 objects degraded (13.418%); 14712/71868 objects misplaced (20.471%); 9.7 MiB/s, 154 objects/s recovering 2024-04-12T12:01:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:07 smithi087 ceph-mon[89014]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T12:01:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:07 smithi087 ceph-mon[89014]: from='osd.3 [v2:172.21.15.29:6826/1944387251,v1:172.21.15.29:6827/1944387251]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T12:01:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:07 smithi087 ceph-mon[89014]: osdmap e1341: 8 total, 7 up, 7 in 2024-04-12T12:01:07.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:07 smithi087 ceph-mon[89014]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T12:01:07.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:07.721 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:07.746 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:07.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:07.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[93650]: pgmap v2407: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 9643/71868 objects degraded (13.418%); 14712/71868 objects misplaced (20.471%); 9.7 MiB/s, 154 objects/s recovering 2024-04-12T12:01:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[93650]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T12:01:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[93650]: from='osd.3 [v2:172.21.15.29:6826/1944387251,v1:172.21.15.29:6827/1944387251]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T12:01:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[93650]: osdmap e1341: 8 total, 7 up, 7 in 2024-04-12T12:01:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[93650]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T12:01:07.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[97787]: pgmap v2407: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 7 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 1.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 9643/71868 objects degraded (13.418%); 14712/71868 objects misplaced (20.471%); 9.7 MiB/s, 154 objects/s recovering 2024-04-12T12:01:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[97787]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T12:01:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[97787]: from='osd.3 [v2:172.21.15.29:6826/1944387251,v1:172.21.15.29:6827/1944387251]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T12:01:07.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[97787]: osdmap e1341: 8 total, 7 up, 7 in 2024-04-12T12:01:07.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[97787]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-04-12T12:01:07.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:07.848 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:07.872 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:07.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:07.973 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:07.998 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:07.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:08.101 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:08.126 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:08.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:08.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:08.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:08.267 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:08.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:08.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:08.368 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:08.393 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:08.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:08.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[97787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[97787]: osd.3 [v2:172.21.15.29:6826/1944387251,v1:172.21.15.29:6827/1944387251] boot 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[97787]: osdmap e1342: 8 total, 8 up, 7 in 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 9643/71868 objects degraded (13.418%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[97787]: osdmap e1343: 8 total, 8 up, 7 in 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[93650]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[93650]: osd.3 [v2:172.21.15.29:6826/1944387251,v1:172.21.15.29:6827/1944387251] boot 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[93650]: osdmap e1342: 8 total, 8 up, 7 in 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 9643/71868 objects degraded (13.418%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-04-12T12:01:08.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:08 smithi029 ceph-mon[93650]: osdmap e1343: 8 total, 8 up, 7 in 2024-04-12T12:01:08.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:08.520 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:08.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:08.622 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:08.648 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:08.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:08 smithi087 ceph-mon[89014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T12:01:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:08 smithi087 ceph-mon[89014]: osd.3 [v2:172.21.15.29:6826/1944387251,v1:172.21.15.29:6827/1944387251] boot 2024-04-12T12:01:08.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:08 smithi087 ceph-mon[89014]: osdmap e1342: 8 total, 8 up, 7 in 2024-04-12T12:01:08.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:08 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T12:01:08.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:08 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 9643/71868 objects degraded (13.418%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-04-12T12:01:08.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:08 smithi087 ceph-mon[89014]: osdmap e1343: 8 total, 8 up, 7 in 2024-04-12T12:01:08.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:08.774 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:08.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:08.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:08.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:08.903 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:08.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:09.004 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:09.028 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:09.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:09.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:09.129 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:09.155 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:09.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:09.199 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2024-04-12T12:01:09.258 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:09.282 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:09.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:09.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:09.414 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:09.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:09.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:09.453 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[97787]: 11.b continuing backfill to osd.1 from (1325'7200,1341'13344] MIN to 1341'13344 2024-04-12T12:01:09.453 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[97787]: 11.a continuing backfill to osd.1 from (1323'5000,1341'13337] 11:5b86dcc3:::benchmark_data_smithi029_285290_object118498:head to 1341'13337 2024-04-12T12:01:09.453 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[97787]: pgmap v2411: 99 pgs: 13 remapped+peering, 15 peering, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 7 active+remapped+backfill_wait, 5 active+undersized+degraded, 45 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 4967/67587 objects degraded (7.349%); 12927/67587 objects misplaced (19.126%); 5.7 MiB/s, 91 objects/s recovering 2024-04-12T12:01:09.453 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2543011920' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T12:01:09.453 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[97787]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:01:09.453 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2543011920' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T12:01:09.453 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[97787]: osdmap e1344: 8 total, 8 up, 7 in 2024-04-12T12:01:09.453 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[93650]: 11.b continuing backfill to osd.1 from (1325'7200,1341'13344] MIN to 1341'13344 2024-04-12T12:01:09.453 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[93650]: 11.a continuing backfill to osd.1 from (1323'5000,1341'13337] 11:5b86dcc3:::benchmark_data_smithi029_285290_object118498:head to 1341'13337 2024-04-12T12:01:09.453 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[93650]: pgmap v2411: 99 pgs: 13 remapped+peering, 15 peering, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 7 active+remapped+backfill_wait, 5 active+undersized+degraded, 45 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 4967/67587 objects degraded (7.349%); 12927/67587 objects misplaced (19.126%); 5.7 MiB/s, 91 objects/s recovering 2024-04-12T12:01:09.454 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2543011920' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T12:01:09.454 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[93650]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:01:09.454 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2543011920' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T12:01:09.454 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:09 smithi029 ceph-mon[93650]: osdmap e1344: 8 total, 8 up, 7 in 2024-04-12T12:01:09.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:09.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:09.547 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:09.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:09.648 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:09.675 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:09.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:09.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:09 smithi087 ceph-mon[89014]: 11.b continuing backfill to osd.1 from (1325'7200,1341'13344] MIN to 1341'13344 2024-04-12T12:01:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:09 smithi087 ceph-mon[89014]: 11.a continuing backfill to osd.1 from (1323'5000,1341'13337] 11:5b86dcc3:::benchmark_data_smithi029_285290_object118498:head to 1341'13337 2024-04-12T12:01:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:09 smithi087 ceph-mon[89014]: pgmap v2411: 99 pgs: 13 remapped+peering, 15 peering, 1 active+undersized+degraded+remapped+backfilling, 3 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 7 active+remapped+backfill_wait, 5 active+undersized+degraded, 45 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 4967/67587 objects degraded (7.349%); 12927/67587 objects misplaced (19.126%); 5.7 MiB/s, 91 objects/s recovering 2024-04-12T12:01:09.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2543011920' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2024-04-12T12:01:09.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:09 smithi087 ceph-mon[89014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:01:09.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2543011920' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-04-12T12:01:09.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:09 smithi087 ceph-mon[89014]: osdmap e1344: 8 total, 8 up, 7 in 2024-04-12T12:01:09.777 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:09.823 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:09.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:09.925 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:09.950 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:09.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:09.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:10.052 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:10.053 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T12:01:10.053 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:01:10.078 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:10.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:10.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:10.180 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:10.205 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:10.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:10.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:10.307 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:10.352 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:10.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:10.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:10.454 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:10.481 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:10.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:10.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:10.583 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:10.608 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:10.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:10.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:10.710 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:10.736 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:10.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:10.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:10.838 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:10.865 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:10.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:10.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:10.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:10.991 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:10.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:10.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:11.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:11.136 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:11.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:11.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:11.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:11.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:11.278 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:11.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:11.379 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:11.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:11.406 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:11.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:11.507 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:11.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:11.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:11.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:11.637 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:11.663 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:11.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:11.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:11 smithi087 ceph-mon[89014]: pgmap v2413: 99 pgs: 13 remapped+peering, 15 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 4 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+undersized+degraded, 54 active+clean; 1.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 7733/64428 objects degraded (12.003%); 14401/64428 objects misplaced (22.352%); 9.7 MiB/s, 184 objects/s recovering 2024-04-12T12:01:11.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:11.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:11 smithi029 ceph-mon[97787]: pgmap v2413: 99 pgs: 13 remapped+peering, 15 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 4 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+undersized+degraded, 54 active+clean; 1.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 7733/64428 objects degraded (12.003%); 14401/64428 objects misplaced (22.352%); 9.7 MiB/s, 184 objects/s recovering 2024-04-12T12:01:11.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:11 smithi029 ceph-mon[93650]: pgmap v2413: 99 pgs: 13 remapped+peering, 15 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 4 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 2 active+undersized+degraded, 54 active+clean; 1.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 7733/64428 objects degraded (12.003%); 14401/64428 objects misplaced (22.352%); 9.7 MiB/s, 184 objects/s recovering 2024-04-12T12:01:11.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:11.815 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:11.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:11.916 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:11.943 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:11.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:11.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:12.044 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:12.072 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:12.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:12.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:12.174 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:12.203 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:12.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:12.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:12.341 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:12.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:12.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:12.442 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:12.489 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:12.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:12.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:12.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:12.627 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:12.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:01:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:01:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:01:12.729 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:12.755 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:12.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:12.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:12.857 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:12.885 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:12.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:12.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:13.033 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:13.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:13.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:13.135 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:13.163 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:13.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:13.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:13.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:13.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:13.289 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:13.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:13.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:13.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:13.434 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:13.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:13.536 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:13.562 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:13.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:13.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:13.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:13.707 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:13.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:13.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:13 smithi087 ceph-mon[89014]: pgmap v2414: 99 pgs: 1 active+recovering+undersized+remapped, 11 remapped+peering, 15 peering, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 62 active+clean; 1.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 6030/63138 objects degraded (9.551%); 17213/63138 objects misplaced (27.263%); 8.2 MiB/s, 152 objects/s recovering 2024-04-12T12:01:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:13 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 6030/63138 objects degraded (9.551%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:13 smithi029 ceph-mon[97787]: pgmap v2414: 99 pgs: 1 active+recovering+undersized+remapped, 11 remapped+peering, 15 peering, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 62 active+clean; 1.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 6030/63138 objects degraded (9.551%); 17213/63138 objects misplaced (27.263%); 8.2 MiB/s, 152 objects/s recovering 2024-04-12T12:01:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:13 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 6030/63138 objects degraded (9.551%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:13 smithi029 ceph-mon[93650]: pgmap v2414: 99 pgs: 1 active+recovering+undersized+remapped, 11 remapped+peering, 15 peering, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 62 active+clean; 1.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 6030/63138 objects degraded (9.551%); 17213/63138 objects misplaced (27.263%); 8.2 MiB/s, 152 objects/s recovering 2024-04-12T12:01:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:13 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 6030/63138 objects degraded (9.551%), 3 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:13.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:13.843 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:13.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:13.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:13.944 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:13.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:13.973 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:13.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:14.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:14.099 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:14.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:14.201 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:14.229 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:14.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:14.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:14.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:14.375 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:14.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:14.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:14.477 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:14.501 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:14.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:14.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:14.648 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:14.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:14.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:14.750 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:14.775 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:14.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:14.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:14.877 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:14.901 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:14.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:15.003 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:15.043 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:15.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:15.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:15.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:15.191 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:15.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:15.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:15.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:01:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:01:14] "GET /metrics HTTP/1.1" 200 36611 "" "Prometheus/2.43.0" 2024-04-12T12:01:15.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:15.333 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:15.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:15.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:15.435 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:15.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:15.482 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:15.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:15.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:15.622 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:15.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:15.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:15 smithi087 ceph-mon[89014]: pgmap v2415: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 23757/61197 objects degraded (38.821%); 17122/61197 objects misplaced (27.978%); 7.1 MiB/s, 131 objects/s recovering 2024-04-12T12:01:15.723 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:15.750 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:15.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:15.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:15.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:15 smithi029 ceph-mon[97787]: pgmap v2415: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 23757/61197 objects degraded (38.821%); 17122/61197 objects misplaced (27.978%); 7.1 MiB/s, 131 objects/s recovering 2024-04-12T12:01:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:15 smithi029 ceph-mon[93650]: pgmap v2415: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 23757/61197 objects degraded (38.821%); 17122/61197 objects misplaced (27.978%); 7.1 MiB/s, 131 objects/s recovering 2024-04-12T12:01:15.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:15.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:15.892 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:15.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:15.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:16.018 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:16.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:16.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:16.120 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:16.149 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:16.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:16.251 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:16.280 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:16.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:16.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:16.382 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:16.412 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:16.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:16.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:16.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:16.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:16.559 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:16.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:16.660 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:16.685 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:16.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:16.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:16.788 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:16.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:16.815 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:16.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:16.916 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:16.944 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:16.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:16.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:17.046 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:17.089 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:17.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:17.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:17.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:17.256 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:17.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:17.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:17.358 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:17.386 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:17.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:17.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:17.487 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:17.514 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:17.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:17.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:17.616 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:17.645 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:17.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:17.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:17 smithi087 ceph-mon[89014]: pgmap v2416: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 23555/56130 objects degraded (41.965%); 16195/56130 objects misplaced (28.853%); 6.1 MiB/s, 123 objects/s recovering 2024-04-12T12:01:17.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:17.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:17 smithi029 ceph-mon[97787]: pgmap v2416: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 23555/56130 objects degraded (41.965%); 16195/56130 objects misplaced (28.853%); 6.1 MiB/s, 123 objects/s recovering 2024-04-12T12:01:17.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:17 smithi029 ceph-mon[93650]: pgmap v2416: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 23555/56130 objects degraded (41.965%); 16195/56130 objects misplaced (28.853%); 6.1 MiB/s, 123 objects/s recovering 2024-04-12T12:01:17.771 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:17.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:17.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:17.873 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:17.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:17.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:17.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:18.005 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:18.031 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:18.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:18.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:18.133 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:18.163 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:18.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:18.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:18.265 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:18.290 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:18.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:18.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:18.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:18.437 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:18.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:18.538 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:18.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:18.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:18.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:18.669 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:18.694 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:18.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:18.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:18 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 23555/56130 objects degraded (41.965%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:18.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:18 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 23555/56130 objects degraded (41.965%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:18.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:18 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 23555/56130 objects degraded (41.965%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:18.796 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:18.822 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:18.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:18.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:18.924 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:18.954 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:18.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:19.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:19.093 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:19.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:19.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:19.195 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:19.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:19.220 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:19.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:19.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:19.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:19.365 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:19.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:19.466 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:19.491 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:19.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:19.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:19.593 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:19.619 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:19.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:19.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:19 smithi087 ceph-mon[89014]: pgmap v2417: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 23555/52968 objects degraded (44.470%); 16018/52968 objects misplaced (30.241%); 4.9 MiB/s, 99 objects/s recovering 2024-04-12T12:01:19.721 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:19.741 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:19 smithi029 ceph-mon[93650]: pgmap v2417: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 23555/52968 objects degraded (44.470%); 16018/52968 objects misplaced (30.241%); 4.9 MiB/s, 99 objects/s recovering 2024-04-12T12:01:19.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:19 smithi029 ceph-mon[97787]: pgmap v2417: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 23555/52968 objects degraded (44.470%); 16018/52968 objects misplaced (30.241%); 4.9 MiB/s, 99 objects/s recovering 2024-04-12T12:01:19.749 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:19.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:19.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:19.787 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-04-12T12:01:19.851 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:19.877 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:19.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:19.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:19.979 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:20.010 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:20.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:20.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:20.112 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:20.140 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:20.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:20.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:20.242 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:20.280 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:20.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:20.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:20.381 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:20.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:20.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:20.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:20.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:20.542 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:20.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:20.643 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:20.669 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:20.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:20.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:20.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:20.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:20.821 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:20.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:20.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:20.950 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:20.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:20.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:21.052 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:21.082 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:21.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:21.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:21.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:21.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:21.237 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:21.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:21.338 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:21.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:21.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:21.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:21.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:21.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:21.496 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:21.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:21 smithi029 ceph-mon[93650]: pgmap v2418: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 23371/49272 objects degraded (47.433%); 15220/49272 objects misplaced (30.890%); 4.4 MiB/s, 89 objects/s recovering 2024-04-12T12:01:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:21 smithi029 ceph-mon[97787]: pgmap v2418: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 23371/49272 objects degraded (47.433%); 15220/49272 objects misplaced (30.890%); 4.4 MiB/s, 89 objects/s recovering 2024-04-12T12:01:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:21.598 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:21.624 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:21.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:21.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:21 smithi087 ceph-mon[89014]: pgmap v2418: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 1.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 23371/49272 objects degraded (47.433%); 15220/49272 objects misplaced (30.890%); 4.4 MiB/s, 89 objects/s recovering 2024-04-12T12:01:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:21.726 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:21.752 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:21.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:21.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:21.854 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:21.896 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:21.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:21.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:21.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:22.047 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:22.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:22.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:22.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:22.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:22.187 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:22.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:22.288 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:22.324 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:22.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:22.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:22.426 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:22.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:22.474 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:22.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:22.575 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:22.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:22.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:22.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:22.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:01:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:01:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:01:22.716 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:22.745 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:22.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:22.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:22.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:22.891 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:22.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:22.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:22.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:23.033 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:23.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:23.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:23.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:23.185 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:23.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:23.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:23.287 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:23.313 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:23.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:23.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:23.363 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T12:01:23.410 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T12:01:23.414 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:23.441 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:23.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:23.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:23.462 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T12:01:23.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:23 smithi029 ceph-mon[97787]: pgmap v2419: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 991 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 23320/47592 objects degraded (49.000%); 15052/47592 objects misplaced (31.627%); 853 KiB/s, 29 objects/s recovering 2024-04-12T12:01:23.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:23 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 23320/47592 objects degraded (49.000%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:23.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:23 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1817470405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:01:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:23 smithi029 ceph-mon[93650]: pgmap v2419: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 991 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 23320/47592 objects degraded (49.000%); 15052/47592 objects misplaced (31.627%); 853 KiB/s, 29 objects/s recovering 2024-04-12T12:01:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:23 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 23320/47592 objects degraded (49.000%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:23 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1817470405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:01:23.521 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T12:01:23.542 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:23.566 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T12:01:23.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:23.572 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:23.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:23.613 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T12:01:23.661 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:01:23.661 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T12:01:23.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.661 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223894 11.0 e1344/1343 11:0f277a93:::benchmark_data_smithi029_285290_object105203:head v 1344'13992, mlcod=1329'9073)", 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.612133+0000", 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.046312299000000001, 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.046338529000000003, 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.612133+0000", 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.612133+0000", 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.612134+0000", 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0699999999999996e-07 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.612229+0000", 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4939999999999996e-05 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.612237+0000", 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1470000000000001e-06 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.612244+0000", 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9210000000000003e-06 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:23.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.640464+0000", 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.028220129 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.640492+0000", 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8461000000000001e-05 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223896 11.0 e1344/1343)", 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.617499+0000", 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.040946538999999997, 2024-04-12T12:01:23.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.041012928999999997, 2024-04-12T12:01:23.666 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.666 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.666 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T12:01:23.666 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.617499+0000", 2024-04-12T12:01:23.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.617499+0000", 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.617500+0000", 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.08e-06 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.617509+0000", 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2080000000000006e-06 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.617522+0000", 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.305e-05 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.617531+0000", 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6770000000000001e-06 2024-04-12T12:01:23.668 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223903 11.0 e1344/1343)", 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.625018+0000", 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.033427252999999997, 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033520729999999999, 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.625018+0000", 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.625018+0000", 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.625019+0000", 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1340000000000001e-06 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.625029+0000", 2024-04-12T12:01:23.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6180000000000001e-06 2024-04-12T12:01:23.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.625041+0000", 2024-04-12T12:01:23.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2852e-05 2024-04-12T12:01:23.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.625050+0000", 2024-04-12T12:01:23.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8380000000000008e-06 2024-04-12T12:01:23.674 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.674 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223902 11.b e1344/1342 11:de6763f1:::benchmark_data_smithi029_285290_object105211:head v 1344'13825, mlcod=1331'9105)", 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.631776+0000", 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.026669352, 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026793931999999999, 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631776+0000", 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631776+0000", 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631777+0000", 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2440000000000001e-06 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631784+0000", 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4869999999999996e-06 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631797+0000", 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3067e-05 2024-04-12T12:01:23.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631806+0000", 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2549999999999997e-06 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631852+0000", 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5850000000000003e-05 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631875+0000", 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3224999999999999e-05 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223904 11.2 e1344/1343)", 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.631832+0000", 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.026613271000000001, 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026770299000000001, 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.679 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.680 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T12:01:23.680 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.680 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:23.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631832+0000", 2024-04-12T12:01:23.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631832+0000", 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631832+0000", 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8299999999999997e-07 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631837+0000", 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5460000000000002e-06 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631844+0000", 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0820000000000001e-06 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.631849+0000", 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2480000000000004e-06 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223907 11.a e1344/1342)", 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.639400+0000", 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.019045001999999998, 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019230179, 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.684 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639400+0000", 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639400+0000", 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639401+0000", 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1829999999999999e-06 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639411+0000", 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3570000000000004e-06 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639431+0000", 2024-04-12T12:01:23.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9939e-05 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639440+0000", 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1880000000000008e-06 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223908 11.b e1344/1342 11:da44feb6:::benchmark_data_smithi029_285290_object105217:head v 1344'13826, mlcod=1331'9105)", 2024-04-12T12:01:23.687 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.639472+0000", 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.018972889999999999, 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019186814999999999, 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639472+0000", 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639472+0000", 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639473+0000", 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0999999999999999e-07 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639478+0000", 2024-04-12T12:01:23.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1000000000000003e-06 2024-04-12T12:01:23.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639487+0000", 2024-04-12T12:01:23.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1600000000000004e-06 2024-04-12T12:01:23.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.639492+0000", 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1499999999999998e-06 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.640360+0000", 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00086838699999999998 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:23.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.640389+0000", 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8532999999999999e-05 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223909 11.b e1344/1342)", 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.642376+0000", 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.016068855999999999, 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016314999, 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.692 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.642376+0000", 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.642376+0000", 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.642378+0000", 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.361e-06 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.642387+0000", 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4720000000000001e-06 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.642402+0000", 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4669e-05 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.642410+0000", 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7399999999999993e-06 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223910 11.b e1344/1342)", 2024-04-12T12:01:23.695 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.652789+0000", 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0056557689999999997, 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059279449999999996, 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.652789+0000", 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.652789+0000", 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.652791+0000", 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4580000000000001e-06 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.652799+0000", 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6810000000000004e-06 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.652812+0000", 2024-04-12T12:01:23.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2662e-05 2024-04-12T12:01:23.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.698 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T12:01:23.698 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:01:23.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.652821+0000", 2024-04-12T12:01:23.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4190000000000008e-06 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223913 11.5 e1344/1343 11:aaf94ff6:::benchmark_data_smithi029_285290_object105222:head v 1344'14089, mlcod=1329'9210)", 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.656700+0000", 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.001745368, 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020481850000000001, 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.700 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.656700+0000", 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.656700+0000", 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.656701+0000", 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.181e-06 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.656710+0000", 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3719999999999994e-06 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.656730+0000", 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9510000000000001e-05 2024-04-12T12:01:23.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.656741+0000", 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1267000000000001e-05 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.656808+0000", 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6742000000000006e-05 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:23.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.656832+0000", 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4587999999999999e-05 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223880 11.2 e1344/1343 11:49cc900b:::benchmark_data_smithi029_285290_object105189:head v 1344'14235, mlcod=1331'9483)", 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.597620+0000", 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.060825206, 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.061163652999999998, 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.704 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.597620+0000", 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.597620+0000", 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.597621+0000", 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3960000000000001e-06 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.597630+0000", 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7199999999999995e-06 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.597647+0000", 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6552999999999999e-05 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.597655+0000", 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7590000000000003e-06 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.597874+0000", 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000218247 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.597926+0000", 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2766999999999998e-05 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223893 11.8 e1344/1343 11:123430f5:::benchmark_data_smithi029_285290_object105202:head v 1344'14127, mlcod=1329'9145)", 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.610755+0000", 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.047690372000000002, 2024-04-12T12:01:23.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.048063411, 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610755+0000", 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610755+0000", 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610756+0000", 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4580000000000001e-06 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610767+0000", 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1126e-05 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610782+0000", 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.487e-05 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610792+0000", 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8879999999999992e-06 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.611030+0000", 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023828200000000001 2024-04-12T12:01:23.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.611075+0000", 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4484000000000003e-05 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223897 11.8 e1344/1343 11:19a4b02a:::benchmark_data_smithi029_285290_object105206:head v 1344'14128, mlcod=1329'9145)", 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.610819+0000", 2024-04-12T12:01:23.712 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.047625770999999997, 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.048032953000000003, 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610819+0000", 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610819+0000", 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610820+0000", 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.89e-07 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610824+0000", 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4719999999999997e-06 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610831+0000", 2024-04-12T12:01:23.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7889999999999997e-06 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610836+0000", 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1159999999999998e-06 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.643223+0000", 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.032387209 2024-04-12T12:01:23.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.643256+0000", 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2267999999999997e-05 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223891 11.a e1344/1342 11:55c7f7df:::benchmark_data_smithi029_285290_object105200:head v 1344'13832, mlcod=1329'8943)", 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.610853+0000", 2024-04-12T12:01:23.716 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.047592359000000001, 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.048034138999999997, 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610853+0000", 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610853+0000", 2024-04-12T12:01:23.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610854+0000", 2024-04-12T12:01:23.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8400000000000002e-07 2024-04-12T12:01:23.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610862+0000", 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8320000000000006e-06 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610873+0000", 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.152e-05 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610881+0000", 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.503e-06 2024-04-12T12:01:23.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:23.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.640703+0000", 2024-04-12T12:01:23.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029822511999999999 2024-04-12T12:01:23.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:23.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.640725+0000", 2024-04-12T12:01:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1885000000000001e-05 2024-04-12T12:01:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:223882 11.2 e1344/1343 11:402302bb:::benchmark_data_smithi029_285290_object105191:head v 1344'14236, mlcod=1331'9483)", 2024-04-12T12:01:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:23.610895+0000", 2024-04-12T12:01:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.047549930999999997, 2024-04-12T12:01:23.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.048026669000000001, 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610895+0000", 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610895+0000", 2024-04-12T12:01:23.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610896+0000", 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2599999999999998e-07 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610900+0000", 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6789999999999996e-06 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:23.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610907+0000", 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5529999999999998e-06 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.610912+0000", 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4330000000000003e-06 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.640862+0000", 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029949320000000001 2024-04-12T12:01:23.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:23.640880+0000", 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8893e-05 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 15 2024-04-12T12:01:23.725 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:01:23.727 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T12:01:23.727 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T12:01:23.727 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T12:01:23.727 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T12:01:23.727 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-04-12T12:01:23.728 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T12:01:23.728 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T12:01:23.728 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T12:01:23.728 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T12:01:23.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:23.729 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:23.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:23.729 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:23 smithi087 ceph-mon[89014]: pgmap v2419: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 991 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 23320/47592 objects degraded (49.000%); 15052/47592 objects misplaced (31.627%); 853 KiB/s, 29 objects/s recovering 2024-04-12T12:01:23.729 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:23 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 23320/47592 objects degraded (49.000%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:23.729 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:23 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1817470405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:01:23.830 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:23.857 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:23.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:23.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:23.959 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:23.987 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:23.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:23.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:24.089 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:24.116 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:24.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:24.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:24.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:24.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:24.268 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:24.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:24.369 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:24.394 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:24.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:24.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:24.498 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:24.522 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:24.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:24.624 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:24.654 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:24.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:24.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:24.756 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:24.784 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:24.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:24.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:24.832 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 0 1 2024-04-12T12:01:24.886 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:24.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:24.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:24.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:24.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd set nodeep-scrub 2024-04-12T12:01:25.014 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:25.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:25.120 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:25.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:25.221 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:25.231 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:01:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:01:24] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.43.0" 2024-04-12T12:01:25.295 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:25.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:25.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:25.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:25.449 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:25.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:25.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:25.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:25 smithi029 ceph-mon[97787]: pgmap v2420: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 925 MiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 23320/44445 objects degraded (52.469%); 14886/44445 objects misplaced (33.493%); 0 B/s, 15 objects/s recovering 2024-04-12T12:01:25.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:25 smithi029 ceph-mon[93650]: pgmap v2420: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 925 MiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 23320/44445 objects degraded (52.469%); 14886/44445 objects misplaced (33.493%); 0 B/s, 15 objects/s recovering 2024-04-12T12:01:25.551 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:25.594 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:25.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:25.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:25.696 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:25 smithi087 ceph-mon[89014]: pgmap v2420: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 925 MiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 23320/44445 objects degraded (52.469%); 14886/44445 objects misplaced (33.493%); 0 B/s, 15 objects/s recovering 2024-04-12T12:01:25.723 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:25.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:25.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:25.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:25.855 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:25.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:25.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:25.956 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:25.999 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:26.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:26.100 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:26.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:26.150 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:26.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:26.251 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:26.289 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:26.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:26.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:26.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:26.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:26.429 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:26.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:26.530 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:26.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:26.594 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:26.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:26.695 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:26.723 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:26.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:26.826 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:26.864 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:26.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:26.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:26.966 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:27.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:27.013 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:27.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:27.114 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:27.144 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:27.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:27.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:27.222 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:01:27.237 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:01:27.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:27.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:27.319 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:27.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:27.420 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:27.469 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:27.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:27.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:27.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:27 smithi029 ceph-mon[97787]: pgmap v2421: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 825 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 23106/39639 objects degraded (58.291%); 14015/39639 objects misplaced (35.357%); 0 B/s, 23 objects/s recovering 2024-04-12T12:01:27.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:27 smithi029 ceph-mon[93650]: pgmap v2421: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 825 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 23106/39639 objects degraded (58.291%); 14015/39639 objects misplaced (35.357%); 0 B/s, 23 objects/s recovering 2024-04-12T12:01:27.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:27.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:27.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:27.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:27.701 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:27 smithi087 ceph-mon[89014]: pgmap v2421: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 825 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 23106/39639 objects degraded (58.291%); 14015/39639 objects misplaced (35.357%); 0 B/s, 23 objects/s recovering 2024-04-12T12:01:27.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-04-12T12:01:27.740 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:27.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:27.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:27.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:27.891 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:27.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:27.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:28.021 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:28.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:28.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:28.123 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:28.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:28.151 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:28.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:28.252 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:28.279 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:28.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:28.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:28.381 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:28.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:28.406 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:28.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:28.507 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:28.535 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:28.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:28.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:28.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:28.676 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:28.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:28.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:28.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:28 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 23106/39639 objects degraded (58.291%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:28 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 23106/39639 objects degraded (58.291%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:28.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:28 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 23106/39639 objects degraded (58.291%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:28.778 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:28.801 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:28.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:28.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:28.903 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:28.931 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:28.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:29.033 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:29.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:29.062 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:29.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:29.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:29.217 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:29.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:29.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:29.362 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:29.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:29.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:29.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:29.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:29.509 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:29.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:29.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:29 smithi029 ceph-mon[97787]: pgmap v2422: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 760 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 23106/36546 objects degraded (63.224%); 13848/36546 objects misplaced (37.892%); 0 B/s, 16 objects/s recovering 2024-04-12T12:01:29.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:29 smithi029 ceph-mon[93650]: pgmap v2422: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 760 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 23106/36546 objects degraded (63.224%); 13848/36546 objects misplaced (37.892%); 0 B/s, 16 objects/s recovering 2024-04-12T12:01:29.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:29.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:29.650 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:29.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:29 smithi087 ceph-mon[89014]: pgmap v2422: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 760 MiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 23106/36546 objects degraded (63.224%); 13848/36546 objects misplaced (37.892%); 0 B/s, 16 objects/s recovering 2024-04-12T12:01:29.752 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:29.782 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:29.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:29.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:29.884 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:29.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:29.913 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:29.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:30.014 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:30.040 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:30.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:30.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:30.142 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:30.188 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:30.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:30.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:30.290 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:30.332 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:30.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:30.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:30.417 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:01:30.417 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T12:01:30.417 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T12:01:30.417 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T12:01:30.417 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:01:30.434 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:30.475 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:30.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:30.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:30.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:30.626 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:30.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:30.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:30.728 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:30.756 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:30.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:30.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:30.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:30.901 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:30.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:30.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:31.003 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:31.040 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:31.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:31.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:31.141 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:31.168 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:31.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:31.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:31.259 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-04-12T12:01:31.270 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:31.363 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:31.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:31.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:31.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:31.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:31 smithi029 ceph-mon[93650]: pgmap v2423: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 681 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 22943/32733 objects degraded (70.091%); 13021/32733 objects misplaced (39.779%); 0 B/s, 16 objects/s recovering 2024-04-12T12:01:31.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:31 smithi029 ceph-mon[97787]: pgmap v2423: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 681 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 22943/32733 objects degraded (70.091%); 13021/32733 objects misplaced (39.779%); 0 B/s, 16 objects/s recovering 2024-04-12T12:01:31.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:31.544 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:31.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:31.645 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:31.672 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:31.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:31.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:31 smithi087 ceph-mon[89014]: pgmap v2423: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 681 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 22943/32733 objects degraded (70.091%); 13021/32733 objects misplaced (39.779%); 0 B/s, 16 objects/s recovering 2024-04-12T12:01:31.774 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:31.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:31.807 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:31.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:31.907 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:31.934 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:31.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:31.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:32.036 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:32.064 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:32.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:32.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:32.166 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:32.196 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:32.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:32.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:32.298 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:32.330 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:32.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:32.431 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:32.460 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:32.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:32.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:32.604 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:32.632 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:32.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:32.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:01:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:01:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:01:32.734 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:32.764 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:32.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:32.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:32.866 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:32.890 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:32.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:32.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:32.992 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:33.040 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:33.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:33.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:33.179 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:33.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:33.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:33.281 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:33.326 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:33.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:33.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:33.428 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:33.460 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:33.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:33.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:33.480 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:01:33.562 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:33.589 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:33.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:33.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:33.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:33 smithi029 ceph-mon[93650]: pgmap v2424: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 647 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 22891/31089 objects degraded (73.631%); 12832/31089 objects misplaced (41.275%); 0 B/s, 26 objects/s recovering 2024-04-12T12:01:33.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:33 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 22891/31089 objects degraded (73.631%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:33.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:33 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3269330194' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:01:33.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:33 smithi029 ceph-mon[97787]: pgmap v2424: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 647 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 22891/31089 objects degraded (73.631%); 12832/31089 objects misplaced (41.275%); 0 B/s, 26 objects/s recovering 2024-04-12T12:01:33.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:33 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 22891/31089 objects degraded (73.631%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:33.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:33 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3269330194' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:01:33.691 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:33 smithi087 ceph-mon[89014]: pgmap v2424: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 647 MiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 22891/31089 objects degraded (73.631%); 12832/31089 objects misplaced (41.275%); 0 B/s, 26 objects/s recovering 2024-04-12T12:01:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:33 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 22891/31089 objects degraded (73.631%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:33.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:33 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3269330194' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:01:33.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:33.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:33.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:33.845 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:33.874 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:33.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:33.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:33.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:34.004 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:34.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:34.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:34.106 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:34.132 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:34.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:34.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:34.215 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.0 to 1 (10000) 2024-04-12T12:01:34.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:34.295 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:34.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:34.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:34.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:34.435 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:34.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:34.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:34.537 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:34.567 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:34.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:34.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:34.669 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:34.695 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:34.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:34.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:34.797 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:34.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:34.821 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:34.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:34.922 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:34.949 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:34.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:34.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:35.051 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:35.092 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:35.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:35.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:35.194 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:35.220 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:01:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:01:34] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.43.0" 2024-04-12T12:01:35.228 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:35.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:35.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:35.329 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:35.356 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:35.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:35.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:35.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:35.497 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:35.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:35.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:35 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3269330194' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-04-12T12:01:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:35 smithi029 ceph-mon[97787]: osdmap e1345: 8 total, 8 up, 7 in 2024-04-12T12:01:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:35 smithi029 ceph-mon[97787]: pgmap v2426: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 576 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 22891/27705 objects degraded (82.624%); 12633/27705 objects misplaced (45.598%); 0 B/s, 21 objects/s recovering 2024-04-12T12:01:35.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:35 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3269330194' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-04-12T12:01:35.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:35 smithi029 ceph-mon[93650]: osdmap e1345: 8 total, 8 up, 7 in 2024-04-12T12:01:35.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:35 smithi029 ceph-mon[93650]: pgmap v2426: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 576 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 22891/27705 objects degraded (82.624%); 12633/27705 objects misplaced (45.598%); 0 B/s, 21 objects/s recovering 2024-04-12T12:01:35.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:35.645 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:35.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:35.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:35.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:35 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3269330194' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-04-12T12:01:35.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:35 smithi087 ceph-mon[89014]: osdmap e1345: 8 total, 8 up, 7 in 2024-04-12T12:01:35.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:35 smithi087 ceph-mon[89014]: pgmap v2426: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 576 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 557 op/s; 22891/27705 objects degraded (82.624%); 12633/27705 objects misplaced (45.598%); 0 B/s, 21 objects/s recovering 2024-04-12T12:01:35.747 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:35.774 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:35.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:35.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:35.918 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:35.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:35.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:36.020 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:36.063 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:36.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:36.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:36.165 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:36.191 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:36.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:36.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:36.293 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:36.318 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:36.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:36.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:36.420 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:36.452 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:36.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:36.553 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:36.582 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:36.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:36.684 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:36.712 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:36.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:36.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:36.814 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:36.853 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:36.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:36.954 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:36.983 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:36.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:37.085 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:37.111 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:37.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:37.213 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:37.252 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:37.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:37.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:37.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:37.393 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:37.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:37.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:37.494 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:37.525 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:37.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:37.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:37.576 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:37 smithi029 ceph-mon[97787]: pgmap v2427: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 462 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 22677/22212 objects degraded (102.093%); 11637/22212 objects misplaced (52.391%); 0 B/s, 20 objects/s recovering 2024-04-12T12:01:37.576 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:37.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:37 smithi029 ceph-mon[93650]: pgmap v2427: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 462 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 22677/22212 objects degraded (102.093%); 11637/22212 objects misplaced (52.391%); 0 B/s, 20 objects/s recovering 2024-04-12T12:01:37.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:37.627 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:37.667 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:37.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:37.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:37 smithi087 ceph-mon[89014]: pgmap v2427: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 462 MiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 22677/22212 objects degraded (102.093%); 11637/22212 objects misplaced (52.391%); 0 B/s, 20 objects/s recovering 2024-04-12T12:01:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:37.769 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:37.798 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:37.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:37.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:37.900 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:37.935 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:37.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:37.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:38.038 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:38.065 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:38.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:38.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:38.120 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:01:38.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:38.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:38.216 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:38.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:38.317 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:38.342 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:38.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:38.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:38.444 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:38.470 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:38.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:38.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:38.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:38 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 22677/22212 objects degraded (102.093%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:38.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:38 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 22677/22212 objects degraded (102.093%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:38.572 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:38.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:38.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:38.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:38.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:38.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:38 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 22677/22212 objects degraded (102.093%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:38.749 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:38.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:38.764 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 2 1 2024-04-12T12:01:38.851 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:38.879 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:38.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:38.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:39.049 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:39.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:39.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:39.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:39.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:39.199 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:39.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:39.300 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:39.327 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:39.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:39.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:39.347 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2024-04-12T12:01:39.429 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:39.459 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:39.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:39.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:39.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:39.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:39 smithi029 ceph-mon[97787]: pgmap v2428: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 396 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 22677/19068 objects degraded (118.927%); 11456/19068 objects misplaced (60.080%); 0 B/s, 20 objects/s recovering 2024-04-12T12:01:39.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:39 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/203475022' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:01:39.586 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:39 smithi029 ceph-mon[93650]: pgmap v2428: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 396 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 22677/19068 objects degraded (118.927%); 11456/19068 objects misplaced (60.080%); 0 B/s, 20 objects/s recovering 2024-04-12T12:01:39.586 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:39 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/203475022' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:01:39.609 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:39.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:39.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:39.711 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:39 smithi087 ceph-mon[89014]: pgmap v2428: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 396 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 22677/19068 objects degraded (118.927%); 11456/19068 objects misplaced (60.080%); 0 B/s, 20 objects/s recovering 2024-04-12T12:01:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:39 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/203475022' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:01:39.739 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:39.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:39.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:39.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:39.882 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:39.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:39.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:39.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:40.023 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:40.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:40.125 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:40.153 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:40.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:40.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:40.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:40.340 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:40.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:40.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:40.442 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:40.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:40.470 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:40.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:40.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:40 smithi029 ceph-mon[97787]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:01:40.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:40 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/203475022' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T12:01:40.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:40 smithi029 ceph-mon[97787]: osdmap e1346: 8 total, 8 up, 7 in 2024-04-12T12:01:40.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:40 smithi029 ceph-mon[93650]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:01:40.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:40 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/203475022' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T12:01:40.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:40 smithi029 ceph-mon[93650]: osdmap e1346: 8 total, 8 up, 7 in 2024-04-12T12:01:40.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:40.599 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:40.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:40.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:40.701 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:40 smithi087 ceph-mon[89014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:01:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:40 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/203475022' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-04-12T12:01:40.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:40 smithi087 ceph-mon[89014]: osdmap e1346: 8 total, 8 up, 7 in 2024-04-12T12:01:40.743 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:40.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:40.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:40.846 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:40.874 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:40.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:40.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:40.976 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:41.001 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:41.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:41.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:41.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:41.152 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:41.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:41.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:41.254 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:41.282 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:41.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:41.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:41.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:41.436 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:41.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:41.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:41.538 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:41.565 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:41.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:41.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:41.666 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:41.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:41.712 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:41.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:41.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:41 smithi087 ceph-mon[89014]: pgmap v2430: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 317 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 22516/15261 objects degraded (147.539%); 10601/15261 objects misplaced (69.465%); 0 B/s, 10 objects/s recovering 2024-04-12T12:01:41.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:41 smithi029 ceph-mon[97787]: pgmap v2430: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 317 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 22516/15261 objects degraded (147.539%); 10601/15261 objects misplaced (69.465%); 0 B/s, 10 objects/s recovering 2024-04-12T12:01:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:41 smithi029 ceph-mon[93650]: pgmap v2430: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 317 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 22516/15261 objects degraded (147.539%); 10601/15261 objects misplaced (69.465%); 0 B/s, 10 objects/s recovering 2024-04-12T12:01:41.813 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:41.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:41.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:41.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:41.945 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:41.984 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:41.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:42.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:42.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:42.115 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:42.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:42.216 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:42.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:42.245 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:42.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:42.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T12:01:42.346 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:42.410 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:42.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:42.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:42.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:42.550 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:42.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:42.651 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:42.673 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:42.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:42.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:42.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:01:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:01:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:01:42.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:42.850 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:42.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:42.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:42.952 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:42.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:42.995 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:42.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:43.096 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:43.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:43.130 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:43.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:43.231 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:43.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:43.294 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:43.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:43.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:43.441 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:43.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:43.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:43.543 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:43.566 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:43.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:43.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:43.668 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:43.691 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:43.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:43.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:43 smithi087 ceph-mon[89014]: pgmap v2431: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 281 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 22457/13548 objects degraded (165.759%); 10404/13548 objects misplaced (76.794%); 0 B/s, 23 objects/s recovering 2024-04-12T12:01:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:43 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:01:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:43 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 22457/13548 objects degraded (165.759%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:43.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:43 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:43.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:43 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:43 smithi029 ceph-mon[93650]: pgmap v2431: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 281 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 22457/13548 objects degraded (165.759%); 10404/13548 objects misplaced (76.794%); 0 B/s, 23 objects/s recovering 2024-04-12T12:01:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:43 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:01:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:43 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 22457/13548 objects degraded (165.759%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:43 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:43.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:43 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:43.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:43 smithi029 ceph-mon[97787]: pgmap v2431: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 281 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 22457/13548 objects degraded (165.759%); 10404/13548 objects misplaced (76.794%); 0 B/s, 23 objects/s recovering 2024-04-12T12:01:43.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:43 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:01:43.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:43 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 22457/13548 objects degraded (165.759%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:43.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:43 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:43.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:43 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:43.793 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:43.821 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:43.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:43.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:43.923 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:43.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:43.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:44.071 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:44.094 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:44.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:44.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:44.196 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:44.241 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:44.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:44.343 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:44.379 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:44.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:44.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:44.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:44.515 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:44.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:44.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:44.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:44.641 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:44.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:44.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:44.743 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:44.778 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:44.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:44.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:44.880 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:44.912 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:44.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:44.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:45.014 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:45.042 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:45.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:45.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:45.144 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:45.188 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:45.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:45.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:45.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:01:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:01:44] "GET /metrics HTTP/1.1" 200 36609 "" "Prometheus/2.43.0" 2024-04-12T12:01:45.290 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:45.319 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:45.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:45.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:45.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:45.457 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:45.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:45.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:45.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:45 smithi087 ceph-mon[89014]: pgmap v2432: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 215 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 22457/10380 objects degraded (216.349%); 10208/10380 objects misplaced (98.343%); 0 B/s, 18 objects/s recovering 2024-04-12T12:01:45.560 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:45.588 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:45.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:45.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:45.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:45.728 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:45.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:45.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:45 smithi029 ceph-mon[97787]: pgmap v2432: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 215 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 22457/10380 objects degraded (216.349%); 10208/10380 objects misplaced (98.343%); 0 B/s, 18 objects/s recovering 2024-04-12T12:01:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:45 smithi029 ceph-mon[93650]: pgmap v2432: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 215 MiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 22457/10380 objects degraded (216.349%); 10208/10380 objects misplaced (98.343%); 0 B/s, 18 objects/s recovering 2024-04-12T12:01:45.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:45.881 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:45.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:45.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:45.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:46.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:46.023 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:46.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:46.124 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:46.156 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:46.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:46.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:46.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:46.316 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:46.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:46.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:46.418 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:46.480 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:46.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:46.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:46.582 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:46.613 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:46.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:46.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:46.714 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:46.741 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:46.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:46.843 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:46.893 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:46.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:46.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:46.994 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:47.023 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:47.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:47.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:47.125 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:47.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:47.159 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:47.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:47.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:47.300 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:47.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:47.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:47.401 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:47.450 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:47.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:47.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:47.504 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 118688 objects 2024-04-12T12:01:47.504 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :143.27 2024-04-12T12:01:47.552 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:47.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:47.577 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:47.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:47.678 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:47.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:47.702 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:47.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:47 smithi087 ceph-mon[89014]: pgmap v2433: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 116 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 22250/5607 objects degraded (396.825%); 9334/5607 objects misplaced (166.470%); 0 B/s, 19 objects/s recovering 2024-04-12T12:01:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:47.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:47 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:47 smithi029 ceph-mon[93650]: pgmap v2433: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 116 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 22250/5607 objects degraded (396.825%); 9334/5607 objects misplaced (166.470%); 0 B/s, 19 objects/s recovering 2024-04-12T12:01:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:47 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:47 smithi029 ceph-mon[97787]: pgmap v2433: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 116 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 22250/5607 objects degraded (396.825%); 9334/5607 objects misplaced (166.470%); 0 B/s, 19 objects/s recovering 2024-04-12T12:01:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:47 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:47.804 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:47.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:47.845 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:47.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:47.946 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:47.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:47.972 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:47.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:48.073 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:48.096 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:48.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:48.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:48.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:48.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:48.223 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:48.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:48.325 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:48.350 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:48.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:48.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:48.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:48.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:48 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 22250/5607 objects degraded (396.825%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:48.479 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:48.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:48.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:48.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:48 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 22250/5607 objects degraded (396.825%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:48.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:48 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 22250/5607 objects degraded (396.825%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:48.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:48.606 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:48.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:48.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:48.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:48.733 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:48.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:48.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:48.834 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:48.859 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:48.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:48.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:48.962 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:48.986 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:48.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:48.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:49.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:49.113 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:49.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:49.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:49.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:49.239 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:49.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:49.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:49.342 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:49.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:49.366 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:49.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:49.467 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:49.492 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:49.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:49.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:49.593 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:49.617 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:49.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:49.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:49 smithi087 ceph-mon[89014]: pgmap v2434: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 59 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22250/2856 objects degraded (779.062%); 9170/2856 objects misplaced (321.078%); 0 B/s, 19 objects/s recovering 2024-04-12T12:01:49.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:49 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:49.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:49 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:49.718 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:49.741 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:49.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:49.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:49.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:49 smithi029 ceph-mon[93650]: pgmap v2434: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 59 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22250/2856 objects degraded (779.062%); 9170/2856 objects misplaced (321.078%); 0 B/s, 19 objects/s recovering 2024-04-12T12:01:49.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:49 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:49.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:49 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:49 smithi029 ceph-mon[97787]: pgmap v2434: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 59 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22250/2856 objects degraded (779.062%); 9170/2856 objects misplaced (321.078%); 0 B/s, 19 objects/s recovering 2024-04-12T12:01:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:49 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:49 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:01:49.843 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:49.866 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:49.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:49.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:49.993 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:49.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:49.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:50.094 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:50.121 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:50.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:50.222 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:50.246 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:50.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:50.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:50.371 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:50.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:50.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:50.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:50.497 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:50.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:50.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:50.623 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:50.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:50.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:50.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:50.750 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:50.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:50.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:50.852 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:50.880 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:50.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:50.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:50.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:51.006 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:51.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:51.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:51.108 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:51.109 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2024-04-12T12:01:51.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2024-04-12T12:01:51.136 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:51.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:51.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:51.238 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:51.266 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:51.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:51.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:51.368 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:51.392 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:51.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:51.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:51.494 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:51.518 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:51 smithi029 ceph-mon[97787]: pgmap v2435: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 15 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 22157/783 objects degraded (2829.757%); 8711/783 objects misplaced (1112.516%); 0 B/s, 17 objects/s recovering 2024-04-12T12:01:51.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:51.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:51 smithi029 ceph-mon[93650]: pgmap v2435: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 15 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 22157/783 objects degraded (2829.757%); 8711/783 objects misplaced (1112.516%); 0 B/s, 17 objects/s recovering 2024-04-12T12:01:51.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:51.519 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:51.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:51.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:51.621 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:51.649 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:51.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:51.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:51 smithi087 ceph-mon[89014]: pgmap v2435: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 15 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 22157/783 objects degraded (2829.757%); 8711/783 objects misplaced (1112.516%); 0 B/s, 17 objects/s recovering 2024-04-12T12:01:51.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:01:51.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:51.776 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:51.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:51.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:51.878 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:51.903 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:51.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:52.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:52.029 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:52.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:52.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:52.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:52.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:52.156 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:52.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:52.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:52.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:52.282 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:52.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:52.384 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:52.412 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:52.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:52.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:52.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:52.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:52.543 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:52.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:52.619 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:01:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:01:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:01:52.644 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:52.674 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:52.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:52.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:52.776 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:52.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:52.801 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:52.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:52.902 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:52.926 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:52.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:52.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:53.028 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:53.052 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:53.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:53.154 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:53.177 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:53.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:53.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:53.279 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:53.302 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:53.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:53.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:53.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:53.427 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:53.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:53.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:53.529 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:53.553 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:53.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:53.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:53.654 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:53.681 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:53.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:53.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:53 smithi029 ceph-mon[97787]: pgmap v2436: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 46 objects/s recovering 2024-04-12T12:01:53.714 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:53 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 22137/72 objects degraded (30745.833%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:53.714 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:53 smithi029 ceph-mon[93650]: pgmap v2436: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 46 objects/s recovering 2024-04-12T12:01:53.714 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:53 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 22137/72 objects degraded (30745.833%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:53 smithi087 ceph-mon[89014]: pgmap v2436: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 46 objects/s recovering 2024-04-12T12:01:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:53 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 22137/72 objects degraded (30745.833%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T12:01:53.784 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:53.807 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:53.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:53.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:53.908 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:53.934 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:53.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:54.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:54.059 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:54.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:54.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:54.161 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:54.187 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:54.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:54.288 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:54.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:54.312 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:54.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:54.413 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:54.438 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:54.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:54.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:54.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:54.564 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:54.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:54.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:54.665 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:54.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:54.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:54.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:54.792 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:01:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T12:01:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T12:01:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T12:01:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28344 10.6 10:6e372a0a:::benchmark_data_smithi029_256330_object28343:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T12:01:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.636497+0000", 2024-04-12T12:01:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 504.07449750000001, 2024-04-12T12:01:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.514864867, 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28344 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636497+0000", 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636497+0000", 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636498+0000", 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.031e-06 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636529+0000", 2024-04-12T12:01:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0688999999999997e-05 2024-04-12T12:01:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636540+0000", 2024-04-12T12:01:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0917000000000001e-05 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.636547+0000", 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4250000000000001e-06 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640097+0000", 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035499939999999999 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640239+0000", 2024-04-12T12:01:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000142113 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4", 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.640380+0000", 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000141113 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.641764+0000", 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013837739999999999 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.150979+0000", 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50921531600000003 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151037+0000", 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7224e-05 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151362+0000", 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032527100000000002 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28367 10.8 10:172b0297:::benchmark_data_smithi029_256330_object28366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.643647+0000", 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 504.06734771499998, 2024-04-12T12:01:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51514234299999995, 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28367 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643647+0000", 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643647+0000", 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643648+0000", 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6000000000000002e-07 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643669+0000", 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1191000000000001e-05 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643679+0000", 2024-04-12T12:01:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0607e-05 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.643691+0000", 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1769e-05 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152468+0000", 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50877640800000001 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152609+0000", 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000141058 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.152760+0000", 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015116700000000001 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158543+0000", 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0057828669999999997 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158586+0000", 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3633000000000001e-05 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158683+0000", 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6847000000000003e-05 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158742+0000", 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8697999999999999e-05 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158789+0000", 2024-04-12T12:01:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7237999999999997e-05 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28370 10.6 10:6134ad43:::benchmark_data_smithi029_256330_object28369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.644366+0000", 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 504.06662812399998, 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51561275900000003, 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:01:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28370 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644366+0000", 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644366+0000", 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644367+0000", 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1699999999999997e-07 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644431+0000", 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3511999999999998e-05 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644443+0000", 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2517000000000001e-05 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644451+0000", 2024-04-12T12:01:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7989999999999994e-06 2024-04-12T12:01:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151408+0000", 2024-04-12T12:01:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50695730000000006 2024-04-12T12:01:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151604+0000", 2024-04-12T12:01:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019551200000000001 2024-04-12T12:01:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4", 2024-04-12T12:01:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.151805+0000", 2024-04-12T12:01:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000201466 2024-04-12T12:01:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.153704+0000", 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001898409 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.159834+0000", 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061304890000000003 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.159891+0000", 2024-04-12T12:01:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6725000000000003e-05 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.159979+0000", 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8213000000000004e-05 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28372 10.8 10:100b9aa0:::benchmark_data_smithi029_256330_object28371:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.644503+0000", 2024-04-12T12:01:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 504.06649169500002, 2024-04-12T12:01:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.514388289, 2024-04-12T12:01:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:01:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:01:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28372 2024-04-12T12:01:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644503+0000", 2024-04-12T12:01:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644503+0000", 2024-04-12T12:01:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644504+0000", 2024-04-12T12:01:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5600000000000005e-07 2024-04-12T12:01:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644527+0000", 2024-04-12T12:01:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3509000000000001e-05 2024-04-12T12:01:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644536+0000", 2024-04-12T12:01:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0799999999999995e-06 2024-04-12T12:01:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644543+0000", 2024-04-12T12:01:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8029999999999999e-06 2024-04-12T12:01:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154400+0000", 2024-04-12T12:01:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50985725800000004 2024-04-12T12:01:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154537+0000", 2024-04-12T12:01:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013657400000000001 2024-04-12T12:01:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T12:01:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154983+0000", 2024-04-12T12:01:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00044639700000000002 2024-04-12T12:01:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158625+0000", 2024-04-12T12:01:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036415060000000001 2024-04-12T12:01:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158663+0000", 2024-04-12T12:01:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8380999999999997e-05 2024-04-12T12:01:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158800+0000", 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000137427 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158848+0000", 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7664000000000003e-05 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.158891+0000", 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2933999999999999e-05 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:75508 10.8 10:12b3b9c8:::benchmark_data_smithi029_256330_object75507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:13.182801+0000", 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.52819322200003, 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73462143300000005, 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:01:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75508 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182801+0000", 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182801+0000", 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182803+0000", 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.516e-06 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182825+0000", 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.251e-05 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182838+0000", 2024-04-12T12:01:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2595000000000001e-05 2024-04-12T12:01:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182846+0000", 2024-04-12T12:01:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3699999999999995e-06 2024-04-12T12:01:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.182937+0000", 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0354999999999998e-05 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.183087+0000", 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015046300000000001 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.183241+0000", 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000153988 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.188275+0000", 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050336510000000001 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.190168+0000", 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001893385 2024-04-12T12:01:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.917315+0000", 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.72714688500000002 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.917373+0000", 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7748999999999999e-05 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.917423+0000", 2024-04-12T12:01:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9966000000000002e-05 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:75532 10.8 10:16d45b92:::benchmark_data_smithi029_256330_object75531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:13.208605+0000", 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.50238911600002, 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71678397999999999, 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:01:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75532 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208605+0000", 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208605+0000", 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208607+0000", 2024-04-12T12:01:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5069999999999999e-06 2024-04-12T12:01:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208629+0000", 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2036e-05 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208644+0000", 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4888e-05 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208653+0000", 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6050000000000004e-06 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208707+0000", 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3154e-05 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208821+0000", 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011414200000000001 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.208925+0000", 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010425900000000001 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.213198+0000", 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042726819999999999 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.213287+0000", 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9775e-05 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925086+0000", 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71179846400000002 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925163+0000", 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7373999999999995e-05 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925389+0000", 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022609400000000001 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:75534 10.8 10:1c9cded7:::benchmark_data_smithi029_256330_object75533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:13.209813+0000", 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.50118192399998, 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71577829400000004, 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75534 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209813+0000", 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209813+0000", 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209814+0000", 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0899999999999999e-06 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209860+0000", 2024-04-12T12:01:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6703999999999999e-05 2024-04-12T12:01:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209872+0000", 2024-04-12T12:01:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1678999999999999e-05 2024-04-12T12:01:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.209880+0000", 2024-04-12T12:01:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3990000000000004e-06 2024-04-12T12:01:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.210963+0000", 2024-04-12T12:01:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001083028 2024-04-12T12:01:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.211126+0000", 2024-04-12T12:01:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000162342 2024-04-12T12:01:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2024-04-12T12:01:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.211246+0000", 2024-04-12T12:01:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000120622 2024-04-12T12:01:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.213352+0000", 2024-04-12T12:01:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002105909 2024-04-12T12:01:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.214394+0000", 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010415229999999999 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925433+0000", 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71103895299999997 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925549+0000", 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115869 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:13.925591+0000", 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2175999999999997e-05 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:113799 11.b 11:d9e77736:::benchmark_data_smithi029_285290_object113798:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:17.438814+0000", 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 156.27218009000001, 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.514381325, 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 113799 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438814+0000", 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438814+0000", 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438815+0000", 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7399999999999991e-07 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438837+0000", 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1691e-05 2024-04-12T12:01:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438846+0000", 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7490000000000004e-06 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438852+0000", 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5150000000000003e-06 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.440798+0000", 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001945453 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.440954+0000", 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015633300000000001 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4", 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.441109+0000", 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015499000000000001 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.450534+0000", 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0094248019999999995 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.450581+0000", 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6987999999999998e-05 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.953079+0000", 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50249780099999997 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.953137+0000", 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8026000000000003e-05 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.953196+0000", 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9002999999999997e-05 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:113800 11.b 11:d25c4a0b:::benchmark_data_smithi029_285290_object113799:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:17.438863+0000", 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 156.272131231, 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51445400799999996, 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 113800 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438863+0000", 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438863+0000", 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438864+0000", 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1800000000000001e-07 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438913+0000", 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9181999999999998e-05 2024-04-12T12:01:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438920+0000", 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5870000000000004e-06 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.438927+0000", 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1120000000000001e-06 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.444294+0000", 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005367798 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.444431+0000", 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013708899999999999 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4", 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.444568+0000", 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013693000000000001 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.450540+0000", 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059711690000000001 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.450611+0000", 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1837999999999999e-05 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.953236+0000", 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50262422799999995 2024-04-12T12:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.953282+0000", 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6159000000000003e-05 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.953317+0000", 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5497999999999998e-05 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:113801 11.b 11:d5c4fd99:::benchmark_data_smithi029_285290_object113800:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:17.439236+0000", 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 156.271758175, 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51419001399999997, 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 113801 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.439236+0000", 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.439236+0000", 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.439237+0000", 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8899999999999998e-07 2024-04-12T12:01:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.439284+0000", 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6554000000000002e-05 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.439294+0000", 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0621999999999999e-05 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.439301+0000", 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1080000000000004e-06 2024-04-12T12:01:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.447465+0000", 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0081632509999999998 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.447596+0000", 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013117000000000001 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,4", 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.447740+0000", 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000144525 2024-04-12T12:01:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.450620+0000", 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028798959999999998 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.451711+0000", 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001090265 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.953351+0000", 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50164080600000005 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.953394+0000", 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2556999999999999e-05 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.953426+0000", 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2471000000000001e-05 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:113805 11.6 11:6bf06a2e:::benchmark_data_smithi029_285290_object113804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:17.444890+0000", 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 156.26610492200001, 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51247677400000002, 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 113805 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.444890+0000", 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.444890+0000", 2024-04-12T12:01:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.444891+0000", 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2580000000000001e-06 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.444940+0000", 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9184999999999997e-05 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.444951+0000", 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0825999999999999e-05 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.444958+0000", 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3320000000000003e-06 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.446750+0000", 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017914680000000001 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.446896+0000", 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014650500000000001 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.447001+0000", 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105207 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.450013+0000", 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003012126 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.450969+0000", 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00095585099999999997 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.952224+0000", 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50125439299999996 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.952284+0000", 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0018e-05 2024-04-12T12:01:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:17.957366+0000", 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050826049999999996 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130453 11.0 11:0b588480:::benchmark_data_smithi029_285290_object11762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.314337+0000", 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 138.39665708499999, 2024-04-12T12:01:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0063460319999999, 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130453 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.314337+0000", 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.314337+0000", 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.314338+0000", 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.39e-07 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.314341+0000", 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6879999999999999e-06 2024-04-12T12:01:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.314347+0000", 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8719999999999996e-06 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.314353+0000", 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0059999999999998e-06 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.315430+0000", 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010760889999999999 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.315439+0000", 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9040000000000004e-06 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.309504+0000", 2024-04-12T12:01:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99406466400000004 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.312076+0000", 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025717270000000002 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.312463+0000", 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000387263 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.317165+0000", 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047020279999999996 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.317210+0000", 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4492e-05 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.320573+0000", 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003363638 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.320628+0000", 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4595000000000002e-05 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.320683+0000", 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5627000000000001e-05 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130473 11.0 11:056f3796:::benchmark_data_smithi029_285290_object11782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.342251+0000", 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 138.36874395500001, 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98206040400000005, 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130473 2024-04-12T12:01:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342251+0000", 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342251+0000", 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342252+0000", 2024-04-12T12:01:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4529999999999999e-06 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342257+0000", 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.186e-06 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342274+0000", 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7054999999999998e-05 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342284+0000", 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3039999999999994e-06 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342393+0000", 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109503 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342405+0000", 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1633e-05 2024-04-12T12:01:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.317237+0000", 2024-04-12T12:01:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97483198400000004 2024-04-12T12:01:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.317358+0000", 2024-04-12T12:01:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000120859 2024-04-12T12:01:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:01:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.317488+0000", 2024-04-12T12:01:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000130438 2024-04-12T12:01:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.320554+0000", 2024-04-12T12:01:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030661099999999999 2024-04-12T12:01:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.324125+0000", 2024-04-12T12:01:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035711610000000002 2024-04-12T12:01:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.324206+0000", 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0513000000000007e-05 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.324267+0000", 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1093999999999995e-05 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.324311+0000", 2024-04-12T12:01:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4110999999999998e-05 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130476 11.0 11:07427874:::benchmark_data_smithi029_285290_object11785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.342820+0000", 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 138.36817465499999, 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99346918399999995, 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130476 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342820+0000", 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342820+0000", 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342821+0000", 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1100000000000005e-07 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342825+0000", 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4159999999999997e-06 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342841+0000", 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5514000000000001e-05 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342848+0000", 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5560000000000002e-06 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342898+0000", 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9598000000000002e-05 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.342914+0000", 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6429000000000001e-05 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.320717+0000", 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97780287700000001 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.320834+0000", 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000116611 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.320978+0000", 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000144621 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.324363+0000", 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033843580000000001 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.324400+0000", 2024-04-12T12:01:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6869999999999998e-05 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.336193+0000", 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011793692 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.336242+0000", 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8544999999999997e-05 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.336289+0000", 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7286000000000001e-05 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130477 11.0 11:015aa6a5:::benchmark_data_smithi029_285290_object11786:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.344238+0000", 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 138.36675663599999, 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99553225300000003, 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130477 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.344238+0000", 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.344238+0000", 2024-04-12T12:01:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.344239+0000", 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8700000000000004e-07 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.344243+0000", 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4519999999999999e-06 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.344258+0000", 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4949e-05 2024-04-12T12:01:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.344267+0000", 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3219999999999993e-06 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.346978+0000", 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027110889999999999 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.346991+0000", 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2850999999999999e-05 2024-04-12T12:01:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.324435+0000", 2024-04-12T12:01:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97744458300000003 2024-04-12T12:01:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.324575+0000", 2024-04-12T12:01:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000139648 2024-04-12T12:01:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:01:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.324650+0000", 2024-04-12T12:01:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5455000000000002e-05 2024-04-12T12:01:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.332605+0000", 2024-04-12T12:01:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0079545139999999993 2024-04-12T12:01:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.339568+0000", 2024-04-12T12:01:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069635010000000004 2024-04-12T12:01:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.339661+0000", 2024-04-12T12:01:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2689000000000007e-05 2024-04-12T12:01:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.339720+0000", 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8936e-05 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.339770+0000", 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0277e-05 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130505 11.0 11:0a5cc773:::benchmark_data_smithi029_285290_object11814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.372887+0000", 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 138.338107976, 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97990499399999997, 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130505 2024-04-12T12:01:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.372887+0000", 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.372887+0000", 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.372888+0000", 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.449e-06 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.372893+0000", 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.451e-06 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.372906+0000", 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2469999999999999e-05 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.372914+0000", 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0639999999999994e-06 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.372984+0000", 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9676000000000002e-05 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.372995+0000", 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1368e-05 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.328509+0000", 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95551387300000001 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.328625+0000", 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011627400000000001 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.328742+0000", 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000116381 2024-04-12T12:01:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.339575+0000", 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010833092000000001 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.339618+0000", 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3260999999999998e-05 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.352697+0000", 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013078645999999999 2024-04-12T12:01:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.352748+0000", 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1715999999999997e-05 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.352792+0000", 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3272999999999997e-05 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130519 11.0 11:0da2c1f4:::benchmark_data_smithi029_285290_object11828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.384713+0000", 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 138.32628185999999, 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97663162100000001, 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130519 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.384713+0000", 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.384713+0000", 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.384714+0000", 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5199999999999995e-07 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.384719+0000", 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9960000000000001e-06 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.384730+0000", 2024-04-12T12:01:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1201e-05 2024-04-12T12:01:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.384738+0000", 2024-04-12T12:01:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.371e-06 2024-04-12T12:01:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.384857+0000", 2024-04-12T12:01:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011846299999999999 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.384872+0000", 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5125e-05 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.332636+0000", 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94776447500000005 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.332769+0000", 2024-04-12T12:01:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013242899999999999 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.332901+0000", 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013244999999999999 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.340630+0000", 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077287249999999997 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.352843+0000", 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012213541 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.361239+0000", 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0083957059999999993 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.361292+0000", 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2546999999999997e-05 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.361344+0000", 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2639999999999997e-05 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130542 11.0 11:07539bed:::benchmark_data_smithi029_285290_object11851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.410650+0000", 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 138.30034475599999, 2024-04-12T12:01:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95078194800000004, 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130542 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.410650+0000", 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.410650+0000", 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.410651+0000", 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.035e-06 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.410656+0000", 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7509999999999999e-06 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.410669+0000", 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3635000000000001e-05 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.410677+0000", 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4699999999999996e-06 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.410740+0000", 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3802999999999999e-05 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.410749+0000", 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7590000000000003e-06 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.336322+0000", 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.925572544 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.336417+0000", 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5169999999999999e-05 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.336516+0000", 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9139000000000001e-05 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.345907+0000", 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0093906509999999999 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.352889+0000", 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069826970000000004 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.361353+0000", 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084632370000000002 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.361393+0000", 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0673999999999997e-05 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.361432+0000", 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8383000000000002e-05 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130545 11.0 11:052972cc:::benchmark_data_smithi029_285290_object11854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.416622+0000", 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 138.29437286300001, 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94489852399999996, 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130545 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.416622+0000", 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.416622+0000", 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.416623+0000", 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9199999999999999e-07 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.416627+0000", 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1320000000000004e-06 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.416637+0000", 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0237999999999999e-05 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.416643+0000", 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1299999999999998e-06 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.416694+0000", 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0997e-05 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.416702+0000", 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5610000000000001e-06 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.340662+0000", 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92396063799999995 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.340790+0000", 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012784100000000001 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.340913+0000", 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012323999999999999 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.345944+0000", 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050303320000000002 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.352941+0000", 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069970199999999996 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.361438+0000", 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084977200000000003 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.361482+0000", 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3961999999999997e-05 2024-04-12T12:01:54.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.361520+0000", 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7721e-05 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130546 11.0 11:04bf5095:::benchmark_data_smithi029_285290_object11855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.417358+0000", 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 138.29363643400001, 2024-04-12T12:01:54.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95072712699999995, 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130546 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.417358+0000", 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:01:54.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.417358+0000", 2024-04-12T12:01:54.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:01:54.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:01:54.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.417359+0000", 2024-04-12T12:01:54.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7100000000000001e-07 2024-04-12T12:01:54.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:01:54.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.417363+0000", 2024-04-12T12:01:54.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0110000000000002e-06 2024-04-12T12:01:54.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:01:54.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.417372+0000", 2024-04-12T12:01:54.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3530000000000001e-06 2024-04-12T12:01:54.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:01:54.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.417377+0000", 2024-04-12T12:01:54.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3750000000000002e-06 2024-04-12T12:01:54.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.417422+0000", 2024-04-12T12:01:54.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4913000000000002e-05 2024-04-12T12:01:54.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:01:54.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.417430+0000", 2024-04-12T12:01:54.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6569999999999997e-06 2024-04-12T12:01:54.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:01:54.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.345970+0000", 2024-04-12T12:01:54.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92853984899999997 2024-04-12T12:01:54.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:01:54.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.346071+0000", 2024-04-12T12:01:54.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010144100000000001 2024-04-12T12:01:54.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:01:54.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.346212+0000", 2024-04-12T12:01:54.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014026800000000001 2024-04-12T12:01:54.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.352985+0000", 2024-04-12T12:01:54.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067729180000000002 2024-04-12T12:01:54.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:01:54.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.361561+0000", 2024-04-12T12:01:54.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0085762800000000004 2024-04-12T12:01:54.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:01:54.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.367987+0000", 2024-04-12T12:01:54.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0064263039999999999 2024-04-12T12:01:54.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:01:54.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.368032+0000", 2024-04-12T12:01:54.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4756000000000003e-05 2024-04-12T12:01:54.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:01:54.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:01:54.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:01:54.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.368085+0000", 2024-04-12T12:01:54.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3331000000000002e-05 2024-04-12T12:01:54.945 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.946 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.946 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.946 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:01:54.946 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:01:54.946 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:01:54.947 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:54.972 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:54.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:55.074 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:55.102 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:55.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:55.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:55.205 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:55.233 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:55.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:55.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:55.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:01:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:01:54] "GET /metrics HTTP/1.1" 200 36606 "" "Prometheus/2.43.0" 2024-04-12T12:01:55.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:55.360 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:55.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:55.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:55.462 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:55.491 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:55.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:55.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:55.592 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:55.619 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:55.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:55.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:55 smithi087 ceph-mon[89014]: pgmap v2437: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 38 objects/s recovering 2024-04-12T12:01:55.722 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:55.748 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:55.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:55.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:55.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:55 smithi029 ceph-mon[93650]: pgmap v2437: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 38 objects/s recovering 2024-04-12T12:01:55.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:55 smithi029 ceph-mon[97787]: pgmap v2437: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 38 objects/s recovering 2024-04-12T12:01:55.850 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:55.877 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:55.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:55.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:55.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:56.004 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:56.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:56.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:56.106 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:56.130 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:56.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:56.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:56.257 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:56.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:56.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:56.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:56.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:56.384 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:56.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:56.485 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:56.513 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:56.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:56.615 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:56.643 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:56.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:56.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:56.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:56.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:56.770 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:56.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:56.871 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:56.899 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:56.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:56.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:57.001 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:57.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:57.025 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:57.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:57.126 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:57.154 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:57.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:57.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:57.256 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:57.280 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:57.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:57.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:57.381 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:57.409 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:57.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:57.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:57.511 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:57.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:57 smithi029 ceph-mon[93650]: pgmap v2438: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 286 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 94 objects/s recovering 2024-04-12T12:01:57.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:57 smithi029 ceph-mon[97787]: pgmap v2438: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 286 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 94 objects/s recovering 2024-04-12T12:01:57.539 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:57.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:57.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:57.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:57.668 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:57.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:57 smithi087 ceph-mon[89014]: pgmap v2438: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 286 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 94 objects/s recovering 2024-04-12T12:01:57.770 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:57.795 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:57.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:57.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:57.897 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:57.924 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:57.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:57.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:58.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:58.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:58.049 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:58.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:58.150 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:58.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:58.178 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:58.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:58.279 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:58.307 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:58.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:58.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:58.409 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:01:58.437 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:58.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:58.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:01:58.538 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:58.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:58.566 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:58.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:58.667 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:58.695 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:58.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:58.761 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:01:58.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:58.820 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:58.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:58.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:58.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:58.945 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:58.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:58.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:01:59.047 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:01:59.071 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:59.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:01:59.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:01:59.201 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:59.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:01:59.303 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:59.331 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:59.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:59.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:59.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:01:59.461 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:59.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:59.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:01:59.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:01:59.588 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:59.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:59.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:01:59.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:01:59.714 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:59.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:59.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:01:59.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:01:59 smithi087 ceph-mon[89014]: pgmap v2439: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 86 objects/s recovering 2024-04-12T12:01:59.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:01:59 smithi029 ceph-mon[93650]: pgmap v2439: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 86 objects/s recovering 2024-04-12T12:01:59.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:01:59 smithi029 ceph-mon[97787]: pgmap v2439: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 86 objects/s recovering 2024-04-12T12:01:59.816 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:01:59.844 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:01:59.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:59.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:01:59.945 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:01:59.969 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:01:59.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:01:59.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:00.071 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:00.094 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:00.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:00.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:00.196 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:00.219 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:00.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:00.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:00.345 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:00.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:00.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:00.447 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:00.475 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:00.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:00.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:00.576 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:00.604 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:00.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:00.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:00.706 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:00.734 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:00.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:00.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:00.835 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:00.863 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:00.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:00.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:00.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:00.989 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:00.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:00.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:01.091 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:01.116 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:01.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:01.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:01.242 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:01.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:01.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:01.345 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:01.373 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:01.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:01.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:01.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:01.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:01.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:01.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:01.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:01.624 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:01.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:01.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:01 smithi087 ceph-mon[89014]: pgmap v2440: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 86 objects/s recovering 2024-04-12T12:02:01.726 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:01.753 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:01.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:01.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:01.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:01 smithi029 ceph-mon[93650]: pgmap v2440: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 86 objects/s recovering 2024-04-12T12:02:01.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:01 smithi029 ceph-mon[97787]: pgmap v2440: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 86 objects/s recovering 2024-04-12T12:02:01.855 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:01.883 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:01.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:01.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:01.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:02.011 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:02.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:02.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:02.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:02.136 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:02.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:02.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:02.238 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:02.266 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:02.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:02.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:02.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:02.393 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:02.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:02.494 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:02.518 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:02.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:02.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:02.619 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:02.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:02.648 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:02.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:02:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:02:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:02:02.750 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:02.774 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:02.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:02.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:02.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:02.903 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:02.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:03.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:03.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:03.031 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:03.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:03.084 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-04-12T12:02:03.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:03.158 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:03.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:03.259 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:03.284 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:03.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:03.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:03.386 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:03.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:03.413 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:03.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:03.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:03.539 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:03.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:03.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:03 smithi029 ceph-mon[97787]: pgmap v2441: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 19 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 150 objects/s recovering 2024-04-12T12:02:03.593 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:03 smithi029 ceph-mon[93650]: pgmap v2441: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 19 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 150 objects/s recovering 2024-04-12T12:02:03.641 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:03.669 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:03.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:03.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:03 smithi087 ceph-mon[89014]: pgmap v2441: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 19 op/s; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 150 objects/s recovering 2024-04-12T12:02:03.771 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:03.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:03.801 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:03.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:03.902 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:03.930 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:03.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:03.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:04.032 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:04.057 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:04.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:04.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:04.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:04.184 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:04.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:04.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:04.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:04.311 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:04.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:04.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:04.413 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:04.440 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:04.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:04.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:04.566 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:04.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:04.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:04.668 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:04.693 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:04.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:04.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:04.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:04.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:04.821 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:04.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:04.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:04.947 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:04.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:04.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:05.049 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:05.076 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:05.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:05.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:05.178 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:05.203 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:05.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:05.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:02:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:02:04] "GET /metrics HTTP/1.1" 200 36606 "" "Prometheus/2.43.0" 2024-04-12T12:02:05.305 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:05.333 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:05.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:05.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:05.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T12:02:05.435 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:05.453 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T12:02:05.463 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:05.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:05.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:05.479 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T12:02:05.501 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T12:02:05.522 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T12:02:05.546 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T12:02:05.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:05 smithi029 ceph-mon[93650]: pgmap v2442: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 119 objects/s recovering 2024-04-12T12:02:05.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:05 smithi029 ceph-mon[97787]: pgmap v2442: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 119 objects/s recovering 2024-04-12T12:02:05.565 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:05.566 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T12:02:05.591 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T12:02:05.592 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:05.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:05.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:05.593 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T12:02:05.593 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T12:02:05.593 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T12:02:05.593 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T12:02:05.594 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T12:02:05.594 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T12:02:05.594 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T12:02:05.594 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T12:02:05.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:05 smithi087 ceph-mon[89014]: pgmap v2442: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 119 objects/s recovering 2024-04-12T12:02:05.720 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:05.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:05.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:05.822 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:05.849 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:05.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:05.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:05.946 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:05.951 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:05.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:05.980 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:05.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:06.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:06.107 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:06.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:06.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:06.209 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:06.235 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:06.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:06.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:06.337 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:06.365 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:06.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:06.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:06.467 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:06.495 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:06.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:06.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:06.597 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:06.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:06.625 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:06.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:06.625 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/187450362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:02:06.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:06.626 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/187450362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:02:06.626 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/187450362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:02:06.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:06.726 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:06.753 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:06.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:06.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:06.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:06.881 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:06.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:06.982 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:07.009 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:07.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:07.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:07.111 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:07.136 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:07.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:07.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:07.264 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:07.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:07.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:07.365 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:07.380 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 1 (10000) 2024-04-12T12:02:07.398 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:07.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:07.500 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:07.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:07.527 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:07.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:07.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:07.652 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:07.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:07.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:07 smithi087 ceph-mon[89014]: pgmap v2443: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 217 objects/s recovering 2024-04-12T12:02:07.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:07 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/890435727' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T12:02:07.736 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:07 smithi029 ceph-mon[97787]: pgmap v2443: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 217 objects/s recovering 2024-04-12T12:02:07.736 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:07 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/890435727' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T12:02:07.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:07 smithi029 ceph-mon[93650]: pgmap v2443: 99 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 22137/72 objects degraded (30745.833%); 8633/72 objects misplaced (11990.278%); 0 B/s, 217 objects/s recovering 2024-04-12T12:02:07.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:07 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/890435727' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T12:02:07.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:07.780 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:07.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:07.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:07.882 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:07.908 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:07.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:07.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:08.010 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:08.033 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:08.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:08.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:08.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:08.158 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:08.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:08.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:08.260 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:08.284 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:08.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:08.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:08.386 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:08.411 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:08.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:08.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:08.535 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:08.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:08.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:08.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:08.660 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:08.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:08.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:08 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/890435727' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-04-12T12:02:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:08 smithi087 ceph-mon[89014]: osdmap e1347: 8 total, 8 up, 7 in 2024-04-12T12:02:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:08 smithi087 ceph-mon[89014]: osdmap e1348: 8 total, 8 up, 7 in 2024-04-12T12:02:08.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:08 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/890435727' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-04-12T12:02:08.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:08 smithi029 ceph-mon[93650]: osdmap e1347: 8 total, 8 up, 7 in 2024-04-12T12:02:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:08 smithi029 ceph-mon[93650]: osdmap e1348: 8 total, 8 up, 7 in 2024-04-12T12:02:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:08 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/890435727' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-04-12T12:02:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:08 smithi029 ceph-mon[97787]: osdmap e1347: 8 total, 8 up, 7 in 2024-04-12T12:02:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:08 smithi029 ceph-mon[97787]: osdmap e1348: 8 total, 8 up, 7 in 2024-04-12T12:02:08.785 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:08.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:08.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:08.887 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:08.910 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:08.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:08.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:09.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:09.037 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:09.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:09.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:09.138 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:09.162 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:09.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:09.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:09.264 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:09.288 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:09.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:09.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:09.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:09.414 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:09.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:09.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:09.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:09.543 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:09.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:09.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:09.645 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:09.669 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:09.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:09.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:09 smithi087 ceph-mon[89014]: 11.d starting backfill to osd.2 from (0'0,0'0] MAX to 1346'14742 2024-04-12T12:02:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:09 smithi087 ceph-mon[89014]: pgmap v2446: 99 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17740/72 objects degraded (24638.889%); 8633/72 objects misplaced (11990.278%); 0 B/s, 244 objects/s recovering 2024-04-12T12:02:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:09 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 17740/72 objects degraded (24638.889%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T12:02:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:09 smithi087 ceph-mon[89014]: osdmap e1349: 8 total, 8 up, 7 in 2024-04-12T12:02:09.763 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-04-12T12:02:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:09 smithi029 ceph-mon[93650]: 11.d starting backfill to osd.2 from (0'0,0'0] MAX to 1346'14742 2024-04-12T12:02:09.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:09 smithi029 ceph-mon[93650]: pgmap v2446: 99 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17740/72 objects degraded (24638.889%); 8633/72 objects misplaced (11990.278%); 0 B/s, 244 objects/s recovering 2024-04-12T12:02:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:09 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 17740/72 objects degraded (24638.889%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T12:02:09.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:09 smithi029 ceph-mon[93650]: osdmap e1349: 8 total, 8 up, 7 in 2024-04-12T12:02:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:09 smithi029 ceph-mon[97787]: 11.d starting backfill to osd.2 from (0'0,0'0] MAX to 1346'14742 2024-04-12T12:02:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:09 smithi029 ceph-mon[97787]: pgmap v2446: 99 pgs: 1 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17740/72 objects degraded (24638.889%); 8633/72 objects misplaced (11990.278%); 0 B/s, 244 objects/s recovering 2024-04-12T12:02:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:09 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 17740/72 objects degraded (24638.889%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-12T12:02:09.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:09 smithi029 ceph-mon[97787]: osdmap e1349: 8 total, 8 up, 7 in 2024-04-12T12:02:09.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:09.795 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:09.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:09.896 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:09.920 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:09.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:09.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:10.022 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:10.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:10.046 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:10.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:10.147 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:10.171 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:10.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:10.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:10.273 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:10.297 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:10.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:10.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:10.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:10.423 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:10.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:10.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:10.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:10.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:10.549 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:10.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:10.651 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:10.674 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:10.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:10.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:10.776 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:10.799 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:10.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:10.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:10.857 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:10.901 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:10.925 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:10.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:10.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:11.026 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:11.050 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:11.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:11.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:11.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:11.175 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:11.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:11.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:11.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:11.301 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:11.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:11.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:11.402 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:11.426 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:11.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:11.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:11 smithi029 ceph-mon[93650]: osdmap e1350: 8 total, 8 up, 7 in 2024-04-12T12:02:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:11 smithi029 ceph-mon[93650]: pgmap v2449: 99 pgs: 2 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17740/72 objects degraded (24638.889%); 8633/72 objects misplaced (11990.278%); 145 B/s, 6 objects/s recovering 2024-04-12T12:02:11.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:11 smithi029 ceph-mon[97787]: osdmap e1350: 8 total, 8 up, 7 in 2024-04-12T12:02:11.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:11 smithi029 ceph-mon[97787]: pgmap v2449: 99 pgs: 2 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17740/72 objects degraded (24638.889%); 8633/72 objects misplaced (11990.278%); 145 B/s, 6 objects/s recovering 2024-04-12T12:02:11.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:11.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:11.553 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:11.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:11.654 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:11.678 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:11.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:11.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:11 smithi087 ceph-mon[89014]: osdmap e1350: 8 total, 8 up, 7 in 2024-04-12T12:02:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:11 smithi087 ceph-mon[89014]: pgmap v2449: 99 pgs: 2 active+recovery_wait+undersized+remapped, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 84 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17740/72 objects degraded (24638.889%); 8633/72 objects misplaced (11990.278%); 145 B/s, 6 objects/s recovering 2024-04-12T12:02:11.780 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:11.804 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:11.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:11.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:11.906 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:11.929 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:11.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:11.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:12.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:12.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:12.056 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:12.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:12.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 4 1 2024-04-12T12:02:12.158 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:12.160 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:12.186 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:12.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:12.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:12.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:12.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:12.312 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:12.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:12.413 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:12.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:12.438 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:12.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:12.539 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:12.563 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:12.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:12.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:12.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:12.689 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:12.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:12.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:02:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:02:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:02:12.791 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:12.816 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:12.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:12.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:12.918 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:12.943 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:12.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:13.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:13.068 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:13.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:13.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:13.170 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:13.194 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:13.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:13.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:13.296 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:13.320 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:13.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:13.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:13.422 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:13.446 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:13.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:13.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:13.571 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:13.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:13.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:13.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:13 smithi029 ceph-mon[97787]: pgmap v2450: 99 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17740/72 objects degraded (24638.889%); 8633/72 objects misplaced (11990.278%); 118 B/s, 178 objects/s recovering 2024-04-12T12:02:13.641 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:13 smithi029 ceph-mon[93650]: pgmap v2450: 99 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17740/72 objects degraded (24638.889%); 8633/72 objects misplaced (11990.278%); 118 B/s, 178 objects/s recovering 2024-04-12T12:02:13.673 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:13.697 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:13.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:13.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:13 smithi087 ceph-mon[89014]: pgmap v2450: 99 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17740/72 objects degraded (24638.889%); 8633/72 objects misplaced (11990.278%); 118 B/s, 178 objects/s recovering 2024-04-12T12:02:13.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:13.824 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:13.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:13.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:13.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:13.949 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:13.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:13.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:14.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:14.075 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:14.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:14.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:14.177 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:14.202 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:14.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:14.304 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:14.328 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:14.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:14.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:14.453 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:14.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:14.555 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:14.579 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:14.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:14.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:14.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:14.704 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:14.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:14.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:14.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:14 smithi087 ceph-mon[89014]: Health check update: Degraded data redundancy: 17649/72 objects degraded (24512.500%), 4 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-04-12T12:02:14.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:14 smithi029 ceph-mon[93650]: Health check update: Degraded data redundancy: 17649/72 objects degraded (24512.500%), 4 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-04-12T12:02:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:14 smithi029 ceph-mon[97787]: Health check update: Degraded data redundancy: 17649/72 objects degraded (24512.500%), 4 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-04-12T12:02:14.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:14.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:14.831 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:14.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:14.932 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:14.956 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:14.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:14.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:15.058 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:15.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:15.082 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:15.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:15.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:15.207 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:15.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:15.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:15.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:02:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:02:14] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.43.0" 2024-04-12T12:02:15.309 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:15.332 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:15.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:15.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:15.435 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:15.458 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:15.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:15.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:15.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:15.584 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:15.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:15.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:15.685 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:15.709 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:15.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:15.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:15 smithi087 ceph-mon[89014]: pgmap v2451: 99 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 96 B/s, 258 objects/s recovering 2024-04-12T12:02:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:15 smithi029 ceph-mon[93650]: pgmap v2451: 99 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 96 B/s, 258 objects/s recovering 2024-04-12T12:02:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:15 smithi029 ceph-mon[97787]: pgmap v2451: 99 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 96 B/s, 258 objects/s recovering 2024-04-12T12:02:15.811 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:15.835 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:15.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:15.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:15.937 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:15.961 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:15.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:16.063 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:16.088 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:16.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:16.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:16.190 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:16.214 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:16.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:16.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:16.315 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:16.339 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:16.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:16.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:16.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:16.467 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:16.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:16.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:16.591 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:16.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:16.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:16.693 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:16.718 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:16.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:16.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:16.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:16.844 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:16.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:16.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:16.946 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:16.970 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:16.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:17.072 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:17.096 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:17.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:17.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:17.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:17.222 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:17.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:17.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:17.347 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:17.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:17.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:17.472 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:17.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:17.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:17.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:17.599 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:17.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:17.701 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:17 smithi087 ceph-mon[89014]: pgmap v2452: 99 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 72 B/s, 195 objects/s recovering 2024-04-12T12:02:17.725 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:17.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:17.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:17.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:17 smithi029 ceph-mon[93650]: pgmap v2452: 99 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 72 B/s, 195 objects/s recovering 2024-04-12T12:02:17.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:17 smithi029 ceph-mon[97787]: pgmap v2452: 99 pgs: 2 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 72 B/s, 195 objects/s recovering 2024-04-12T12:02:17.826 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:17.850 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:17.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:17.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:17.952 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:17.975 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:17.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:17.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:18.077 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:18.101 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:18.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:18.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:18.203 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:18.227 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:18.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:18.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:18.328 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:18.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:18.353 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:18.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:18.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:18.478 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:18.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:18.580 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:18.604 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:18.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:18.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:18.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:18.729 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:18.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:18.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:18.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:18.855 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:18.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:18.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:18.957 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:18.980 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:18.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:18.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:19.082 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:19.106 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:19.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:19.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:19.208 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:19.231 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:19.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:19.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:19.333 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:19.357 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:19.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:19.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:19.459 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:19.484 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:19.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:19.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:19.586 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:19.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:19.611 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:19.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:19.711 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:19.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:19 smithi087 ceph-mon[89014]: pgmap v2453: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 64 B/s, 264 objects/s recovering 2024-04-12T12:02:19.735 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:19.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:19.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:19 smithi029 ceph-mon[93650]: pgmap v2453: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 64 B/s, 264 objects/s recovering 2024-04-12T12:02:19.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:19 smithi029 ceph-mon[97787]: pgmap v2453: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 64 B/s, 264 objects/s recovering 2024-04-12T12:02:19.837 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:19.862 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:19.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:19.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:19.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:19.988 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:19.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:19.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:20.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:20.113 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:20.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:20.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:20.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:20.239 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:20.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:20.341 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:20.364 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:20.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:20.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:20.490 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:20.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:20.592 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:20.615 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:20.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:20.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:20.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:20.740 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:20.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:20.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:20.842 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:20.865 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:20.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:20.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:20.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:20.990 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:20.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:20.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:21.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:21.116 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:21.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:21.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:21.218 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:21.242 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:21.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:21.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:21.344 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:21.367 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:21.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:21.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:21.469 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:21.493 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:21.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:21.595 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:21.620 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:21.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:21.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:21 smithi087 ceph-mon[89014]: pgmap v2454: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 0 B/s, 237 objects/s recovering 2024-04-12T12:02:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:21.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:21.745 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:21.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:21.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:21 smithi029 ceph-mon[93650]: pgmap v2454: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 0 B/s, 237 objects/s recovering 2024-04-12T12:02:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:21 smithi029 ceph-mon[97787]: pgmap v2454: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 0 B/s, 237 objects/s recovering 2024-04-12T12:02:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:21.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:21.871 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:21.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:21.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:21.973 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:21.997 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:21.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:21.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:22.099 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:22.122 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:22.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:22.224 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:22.248 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:22.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:22.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:22.350 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:22.374 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:22.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:22.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:22.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:22.499 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:22.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:22.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:22.601 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:22.618 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:02:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:02:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:02:22.625 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:22.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:22.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:22.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:22.751 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:22.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:22.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:22.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:22.877 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:22.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:22.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:22.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:23.003 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:23.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:23.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:23.104 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:23.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:23.129 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:23.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:23.219 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:23.230 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:23.259 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:23.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:23.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:23.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:23.389 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:23.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:23.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:23.491 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:23.516 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:23.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:23.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:23 smithi029 ceph-mon[93650]: pgmap v2455: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 0 B/s, 199 objects/s recovering 2024-04-12T12:02:23.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:23 smithi029 ceph-mon[97787]: pgmap v2455: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 0 B/s, 199 objects/s recovering 2024-04-12T12:02:23.618 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:23.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:23.645 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:23.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:23 smithi087 ceph-mon[89014]: pgmap v2455: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 0 B/s, 199 objects/s recovering 2024-04-12T12:02:23.747 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:23.770 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:23.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:23.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:23.872 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:23.896 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:23.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:23.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:23.998 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:24.021 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:24.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:24.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:24.123 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:24.149 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:24.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:24.250 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:24.273 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:24.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:24.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:24.375 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:24.400 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:24.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:24.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:24.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:24.526 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:24.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:24.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:24.627 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:24.652 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:24.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:24.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:24.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:24.779 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:24.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:24.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:24.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:24.904 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:24.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:24.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:25.006 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:25.018 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:02:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:02:24] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2024-04-12T12:02:25.029 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:25.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:25.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:25.131 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6 2024-04-12T12:02:25.155 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:25.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:25.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.6' 2024-04-12T12:02:25.257 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4 2024-04-12T12:02:25.280 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:25.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:25.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.4' 2024-04-12T12:02:25.382 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:25.400 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_6' removed 2024-04-12T12:02:25.405 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:25.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:25.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:25.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1 2024-04-12T12:02:25.543 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:25.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.1' 2024-04-12T12:02:25.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:25 smithi029 ceph-mon[93650]: pgmap v2456: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 0 B/s, 210 objects/s recovering 2024-04-12T12:02:25.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:25 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1365815011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T12:02:25.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:25 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T12:02:25.643 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:25 smithi029 ceph-mon[97787]: pgmap v2456: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 0 B/s, 210 objects/s recovering 2024-04-12T12:02:25.643 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:25 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1365815011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T12:02:25.643 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:25 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T12:02:25.645 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:25.677 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:25.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:25 smithi087 ceph-mon[89014]: pgmap v2456: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 86 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 17649/72 objects degraded (24512.500%); 8633/72 objects misplaced (11990.278%); 0 B/s, 210 objects/s recovering 2024-04-12T12:02:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:25 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1365815011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T12:02:25.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:25 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-12T12:02:25.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:25.803 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:25.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:25.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:25.905 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7 2024-04-12T12:02:25.931 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:25.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:25.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.7' 2024-04-12T12:02:26.032 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:26.061 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:26.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:26.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:26.162 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:26.186 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:26.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:26.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:26.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2 2024-04-12T12:02:26.312 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:26.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:26.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.2' 2024-04-12T12:02:26.414 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3 2024-04-12T12:02:26.439 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:26.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.3' 2024-04-12T12:02:26.540 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0 2024-04-12T12:02:26.563 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: docker: command not found 2024-04-12T12:02:26.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:26.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.0' 2024-04-12T12:02:26.665 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:26.693 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:26.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:26 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T12:02:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:26 smithi087 ceph-mon[89014]: osdmap e1351: 8 total, 8 up, 7 in 2024-04-12T12:02:26.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:26 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T12:02:26.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:26 smithi029 ceph-mon[97787]: osdmap e1351: 8 total, 8 up, 7 in 2024-04-12T12:02:26.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:26 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-04-12T12:02:26.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:26 smithi029 ceph-mon[93650]: osdmap e1351: 8 total, 8 up, 7 in 2024-04-12T12:02:26.795 DEBUG:teuthology.orchestra.run.smithi087:> sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5 2024-04-12T12:02:26.823 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: docker: command not found 2024-04-12T12:02:26.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:02:26.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi087 with status 1: 'sudo docker kill -s 1 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd.5' 2024-04-12T12:02:26.890 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-12T12:02:26.890 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-04-12T12:02:26.890 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-04-12T12:02:26.891 INFO:tasks.thrashosds:joining thrashosds 2024-04-12T12:02:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:27 smithi087 ceph-mon[89014]: pgmap v2458: 83 pgs: 83 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail 2024-04-12T12:02:27.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:27 smithi087 ceph-mon[89014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17649/72 objects degraded (24512.500%), 4 pgs degraded, 7 pgs undersized) 2024-04-12T12:02:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:27 smithi029 ceph-mon[93650]: pgmap v2458: 83 pgs: 83 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail 2024-04-12T12:02:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:27 smithi029 ceph-mon[93650]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17649/72 objects degraded (24512.500%), 4 pgs degraded, 7 pgs undersized) 2024-04-12T12:02:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:27 smithi029 ceph-mon[97787]: pgmap v2458: 83 pgs: 83 active+clean; 582 KiB data, 7.3 GiB used, 619 GiB / 626 GiB avail 2024-04-12T12:02:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:27 smithi029 ceph-mon[97787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17649/72 objects degraded (24512.500%), 4 pgs degraded, 7 pgs undersized) 2024-04-12T12:02:28.286 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:29.643 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:29 smithi029 ceph-mon[93650]: pgmap v2459: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:29.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:29 smithi029 ceph-mon[97787]: pgmap v2459: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:29 smithi087 ceph-mon[89014]: pgmap v2459: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:30.410 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 1 (10000) 2024-04-12T12:02:30.508 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2024-04-12T12:02:30.508 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-04-12T12:02:30.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:30 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/585402964' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:02:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:30 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/585402964' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:02:30.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:30 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/585402964' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:02:31.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 6 1 2024-04-12T12:02:31.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:31 smithi029 ceph-mon[93650]: pgmap v2460: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:31.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:31 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/585402964' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-04-12T12:02:31.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:31 smithi029 ceph-mon[93650]: osdmap e1352: 8 total, 8 up, 7 in 2024-04-12T12:02:31.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:31 smithi029 ceph-mon[97787]: pgmap v2460: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:31.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:31 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/585402964' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-04-12T12:02:31.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:31 smithi029 ceph-mon[97787]: osdmap e1352: 8 total, 8 up, 7 in 2024-04-12T12:02:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:31 smithi087 ceph-mon[89014]: pgmap v2460: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:31 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/585402964' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-04-12T12:02:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:31 smithi087 ceph-mon[89014]: osdmap e1352: 8 total, 8 up, 7 in 2024-04-12T12:02:32.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:02:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:02:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:02:32.812 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2024-04-12T12:02:32.832 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2024-04-12T12:02:32.856 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2024-04-12T12:02:32.878 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2024-04-12T12:02:32.899 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2024-04-12T12:02:32.918 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2024-04-12T12:02:32.938 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2024-04-12T12:02:32.958 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2024-04-12T12:02:32.959 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = '' 2024-04-12T12:02:32.959 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = '' 2024-04-12T12:02:32.959 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = '' 2024-04-12T12:02:32.959 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = '' 2024-04-12T12:02:32.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = '' 2024-04-12T12:02:32.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = '' 2024-04-12T12:02:32.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = '' 2024-04-12T12:02:32.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = '' 2024-04-12T12:02:33.091 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T12:02:33.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:33 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:33.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:33 smithi029 ceph-mon[93650]: pgmap v2462: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:33.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:33 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2820590239' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T12:02:33.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:33 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2915100362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:02:33.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:33 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:33.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:33 smithi029 ceph-mon[97787]: pgmap v2462: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:33.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:33 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2820590239' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T12:02:33.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:33 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2915100362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:02:33.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-04-12T12:02:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:33 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:33 smithi087 ceph-mon[89014]: pgmap v2462: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:33.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:33 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2820590239' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T12:02:33.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:33 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2915100362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2024-04-12T12:02:34.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:34 smithi087 ceph-mon[89014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:02:34.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:34 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2820590239' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T12:02:34.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:34 smithi087 ceph-mon[89014]: osdmap e1353: 8 total, 8 up, 7 in 2024-04-12T12:02:34.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:34 smithi029 ceph-mon[93650]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:02:34.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:34 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2820590239' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T12:02:34.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:34 smithi029 ceph-mon[93650]: osdmap e1353: 8 total, 8 up, 7 in 2024-04-12T12:02:34.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:34 smithi029 ceph-mon[97787]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-04-12T12:02:34.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:34 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2820590239' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T12:02:34.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:34 smithi029 ceph-mon[97787]: osdmap e1353: 8 total, 8 up, 7 in 2024-04-12T12:02:35.132 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:35 smithi029 ceph-mon[93650]: pgmap v2464: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:35.133 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:35 smithi029 ceph-mon[97787]: pgmap v2464: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:35.133 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:02:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:02:34] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2024-04-12T12:02:35.254 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:35.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:35 smithi087 ceph-mon[89014]: pgmap v2464: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:35.625 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:36.151 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset noscrub 2024-04-12T12:02:37.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:37 smithi029 ceph-mon[97787]: pgmap v2465: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:37.517 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:37.517 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:37 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:37.517 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:37 smithi029 ceph-mon[93650]: pgmap v2465: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:37.517 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:37.517 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:37 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:37.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:37 smithi087 ceph-mon[89014]: pgmap v2465: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:37.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:37.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:37 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:39.484 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:39 smithi029 ceph-mon[93650]: pgmap v2466: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:39.484 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:39 smithi029 ceph-mon[97787]: pgmap v2466: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:39.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:39 smithi087 ceph-mon[89014]: pgmap v2466: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T12:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T12:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T12:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:02:40.604 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:41.076 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.6 to 1 (10000) 2024-04-12T12:02:41.101 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-04-12T12:02:41.460 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:41.460 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:41.460 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3640635067' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:02:41.460 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:02:41.460 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[97787]: pgmap v2467: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:41.460 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:02:41.460 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:02:41.460 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:41.460 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:41.460 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:41.461 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3640635067' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:02:41.461 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:02:41.461 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[93650]: pgmap v2467: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:41.461 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:02:41.461 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:02:41.461 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:41.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:41.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:41.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:41 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3640635067' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:02:41.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:41 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:02:41.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:41 smithi087 ceph-mon[89014]: pgmap v2467: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:41.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:02:41.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:02:41.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:41.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 7 1 2024-04-12T12:02:41.894 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:42.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:42 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-04-12T12:02:42.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:42 smithi029 ceph-mon[97787]: osdmap e1354: 8 total, 8 up, 7 in 2024-04-12T12:02:42.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:42 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-04-12T12:02:42.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:42 smithi029 ceph-mon[93650]: osdmap e1354: 8 total, 8 up, 7 in 2024-04-12T12:02:42.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:42 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-04-12T12:02:42.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:42 smithi087 ceph-mon[89014]: osdmap e1354: 8 total, 8 up, 7 in 2024-04-12T12:02:42.468 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:02:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:02:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:02:43.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:43 smithi029 ceph-mon[93650]: osdmap e1355: 8 total, 8 up, 7 in 2024-04-12T12:02:43.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:43 smithi029 ceph-mon[93650]: pgmap v2470: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:43 smithi029 ceph-mon[97787]: osdmap e1355: 8 total, 8 up, 7 in 2024-04-12T12:02:43.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:43 smithi029 ceph-mon[97787]: pgmap v2470: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:43.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:43 smithi087 ceph-mon[89014]: osdmap e1355: 8 total, 8 up, 7 in 2024-04-12T12:02:43.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:43 smithi087 ceph-mon[89014]: pgmap v2470: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:43.846 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:44.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:44 smithi029 ceph-mon[97787]: osdmap e1356: 8 total, 8 up, 7 in 2024-04-12T12:02:44.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:44 smithi029 ceph-mon[97787]: osdmap e1357: 8 total, 8 up, 7 in 2024-04-12T12:02:44.393 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:44 smithi029 ceph-mon[93650]: osdmap e1356: 8 total, 8 up, 7 in 2024-04-12T12:02:44.393 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:44 smithi029 ceph-mon[93650]: osdmap e1357: 8 total, 8 up, 7 in 2024-04-12T12:02:44.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:44 smithi087 ceph-mon[89014]: osdmap e1356: 8 total, 8 up, 7 in 2024-04-12T12:02:44.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:44 smithi087 ceph-mon[89014]: osdmap e1357: 8 total, 8 up, 7 in 2024-04-12T12:02:45.216 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:02:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:02:44] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-04-12T12:02:45.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:45 smithi087 ceph-mon[89014]: osdmap e1358: 8 total, 8 up, 7 in 2024-04-12T12:02:45.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:45 smithi087 ceph-mon[89014]: pgmap v2474: 83 pgs: 1 remapped+peering, 82 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:45.470 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:45 smithi029 ceph-mon[93650]: osdmap e1358: 8 total, 8 up, 7 in 2024-04-12T12:02:45.470 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:45 smithi029 ceph-mon[93650]: pgmap v2474: 83 pgs: 1 remapped+peering, 82 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:45.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:45 smithi029 ceph-mon[97787]: osdmap e1358: 8 total, 8 up, 7 in 2024-04-12T12:02:45.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:45 smithi029 ceph-mon[97787]: pgmap v2474: 83 pgs: 1 remapped+peering, 82 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:46.227 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2024-04-12T12:02:46.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:46 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/938150301' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T12:02:46.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:46 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/938150301' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T12:02:46.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:46 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/938150301' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2024-04-12T12:02:46.930 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:02:46.930 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T12:02:46.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T12:02:46.930 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T12:02:46.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.930 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:28369 10.2 10:4ae5b963:::benchmark_data_smithi029_256330_object28368:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T12:02:46.930 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:29.644209+0000", 2024-04-12T12:02:46.930 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 557.28312780099998, 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.51128793500000003, 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 28369 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644209+0000", 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644209+0000", 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644211+0000", 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7349999999999999e-06 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644234+0000", 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3073000000000001e-05 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644247+0000", 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3263e-05 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644255+0000", 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.729e-06 2024-04-12T12:02:46.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644312+0000", 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7408999999999999e-05 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644494+0000", 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018137200000000001 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:02:46.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.644630+0000", 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000136556 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.646188+0000", 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001557952 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:29.647249+0000", 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010610400000000001 2024-04-12T12:02:46.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.154196+0000", 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.506946542 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.155434+0000", 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001237953 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:30.155497+0000", 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3311000000000004e-05 2024-04-12T12:02:46.936 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:33405 10.2 10:4829f386:::benchmark_data_smithi029_256330_object33404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:53:33.223479+0000", 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 553.70385828099995, 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38356202099999998, 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:02:46.937 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 33405 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223479+0000", 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223479+0000", 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223480+0000", 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.875e-06 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223507+0000", 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6846999999999999e-05 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223522+0000", 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4944e-05 2024-04-12T12:02:46.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223531+0000", 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1419999999999999e-06 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223593+0000", 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1894000000000001e-05 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223728+0000", 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013449299999999999 2024-04-12T12:02:46.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.223858+0000", 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000129813 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.225383+0000", 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001525857 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.225544+0000", 2024-04-12T12:02:46.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000160722 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.606916+0000", 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38137180199999998 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.606985+0000", 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9036999999999996e-05 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:53:33.607041+0000", 2024-04-12T12:02:46.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5594999999999999e-05 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35727.0:76041 10.0 e1282/1270 10:043b3560:::benchmark_data_smithi029_256330_object76040:head v 1282'4759, mlcod=1282'4757)", 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:54:14.739382+0000", 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 512.18795477900005, 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37150275999999999, 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.943 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739382+0000", 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739382+0000", 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739383+0000", 2024-04-12T12:02:46.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.003e-06 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739418+0000", 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5002000000000001e-05 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739429+0000", 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1209999999999999e-05 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:14.739436+0000", 2024-04-12T12:02:46.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0589999999999997e-06 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.099758+0000", 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36032216 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.099788+0000", 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9048e-05 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.110718+0000", 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010930949000000001 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:54:15.110885+0000", 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000166329 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35727.0:153017 10.b e1303/1296 10:d8d8c85a:::benchmark_data_smithi029_256330_object51251:head v 1303'9751, mlcod=1303'9747)", 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:04.009015+0000", 2024-04-12T12:02:46.947 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 402.91832136900001, 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15182558199999999, 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:04.009015+0000", 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:04.009015+0000", 2024-04-12T12:02:46.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:04.009017+0000", 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.062e-06 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:04.009024+0000", 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2060000000000001e-06 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:04.009034+0000", 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.061e-05 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:04.009042+0000", 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.909e-06 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:04.131069+0000", 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.122026527 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:04.131089+0000", 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0656e-05 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:04.160752+0000", 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029662713 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:04.160841+0000", 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8899000000000004e-05 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.951 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35727.0:156634 10.2 e1303/1296 10:4c7ae2c8:::benchmark_data_smithi029_256330_object54868:head v 1303'9874, mlcod=1303'9872)", 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:09.996395+0000", 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 396.93094138999999, 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.272217714, 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.996395+0000", 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.996395+0000", 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.996397+0000", 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.198e-06 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.996406+0000", 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2220000000000008e-06 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.996419+0000", 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2787e-05 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.996427+0000", 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2309999999999996e-06 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.996506+0000", 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.907e-05 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.996535+0000", 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8856e-05 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.268422+0000", 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27188763799999999 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.268613+0000", 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000190712 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35727.0:156640 10.b e1303/1296 10:d308b8da:::benchmark_data_smithi029_256330_object54874:head v 1303'9988, mlcod=1303'9986)", 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:09.999703+0000", 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 396.92763355199997, 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27335298699999999, 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.956 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999703+0000", 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999703+0000", 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999704+0000", 2024-04-12T12:02:46.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1489999999999999e-06 2024-04-12T12:02:46.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999713+0000", 2024-04-12T12:02:46.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4149999999999999e-06 2024-04-12T12:02:46.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999727+0000", 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4303e-05 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999737+0000", 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5910000000000002e-06 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999821+0000", 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4047000000000004e-05 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.263163+0000", 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26334196100000001 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.272964+0000", 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0098010779999999995 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.273056+0000", 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2442999999999996e-05 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.961 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35727.0:156637 10.6 e1303/1296 10:6666c540:::benchmark_data_smithi029_256330_object54871:head v 1303'9713, mlcod=1284'6330)", 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:09.999951+0000", 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 396.92738591400001, 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27324393400000002, 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999951+0000", 2024-04-12T12:02:46.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999951+0000", 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999952+0000", 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7000000000000004e-07 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:09.999957+0000", 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5099999999999998e-06 2024-04-12T12:02:46.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.262730+0000", 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26277307999999999 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.262745+0000", 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.507e-05 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.263261+0000", 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00051586899999999996 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.263287+0000", 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6074000000000001e-05 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.272951+0000", 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0096635349999999991 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.273195+0000", 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000244126 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.965 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35727.0:156642 10.2 e1303/1296 10:4ca5d85f:::benchmark_data_smithi029_256330_object54876:head v 1303'9875, mlcod=1303'9872)", 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:10.005261+0000", 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 396.92207549099999, 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26354454700000002, 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.005261+0000", 2024-04-12T12:02:46.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.005261+0000", 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.005263+0000", 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4169999999999999e-06 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.005273+0000", 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0413e-05 2024-04-12T12:02:46.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.262818+0000", 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25754517599999999 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.262830+0000", 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.201e-05 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.263272+0000", 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00044121999999999998 2024-04-12T12:02:46.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.263297+0000", 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5884000000000002e-05 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.268622+0000", 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053246420000000001 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:10.268806+0000", 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000183785 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:226571 11.0 11:0f0a1175:::benchmark_data_smithi029_285290_object107880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1344)", 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:28.278299+0000", 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 78.649038074000003, 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14234699200000001, 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.970 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 226571 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278299+0000", 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278299+0000", 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278301+0000", 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1739999999999999e-06 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278308+0000", 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1330000000000001e-06 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278323+0000", 2024-04-12T12:02:46.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5224e-05 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278333+0000", 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0108e-05 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278438+0000", 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010433999999999999 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278595+0000", 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015704799999999999 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278730+0000", 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000135415 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.288561+0000", 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0098308389999999992 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.292178+0000", 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036168839999999999 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.420119+0000", 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12794091399999999 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.420184+0000", 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5488999999999996e-05 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.420646+0000", 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00046142399999999999 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:226573 11.0 11:0943dc10:::benchmark_data_smithi029_285290_object107882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1344)", 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:28.278358+0000", 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 78.648978933999999, 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14263828100000001, 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.976 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 226573 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278358+0000", 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278358+0000", 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278358+0000", 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2300000000000002e-07 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278361+0000", 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0960000000000001e-06 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278368+0000", 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5629999999999997e-06 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278373+0000", 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.48e-06 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.285464+0000", 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00709043 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.285594+0000", 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013039400000000001 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.285724+0000", 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012929200000000001 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.292185+0000", 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0064611210000000002 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.419933+0000", 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12774807899999999 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.420744+0000", 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00081138200000000003 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.420800+0000", 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5384999999999999e-05 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.420996+0000", 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019663600000000001 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:226574 11.0 11:09d2e89e:::benchmark_data_smithi029_285290_object107883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1344)", 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:28.278380+0000", 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 78.648956463999994, 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.142912489, 2024-04-12T12:02:46.982 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 226574 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278380+0000", 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278380+0000", 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278381+0000", 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4700000000000002e-07 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278389+0000", 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0739999999999993e-06 2024-04-12T12:02:46.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278394+0000", 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4569999999999996e-06 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.278399+0000", 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8029999999999996e-06 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.288600+0000", 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010200485 2024-04-12T12:02:46.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.288748+0000", 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014842499999999999 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.288885+0000", 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013682399999999999 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:02:46.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.292193+0000", 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003308512 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.420700+0000", 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12850666899999999 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.421043+0000", 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034338499999999998 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.421110+0000", 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7031000000000002e-05 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.421293+0000", 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018247700000000001 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:226578 11.d e1344/1334 11:b3848f36:::benchmark_data_smithi029_285290_object107887:head v 1344'14036, mlcod=1344'14034)", 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:28.291332+0000", 2024-04-12T12:02:46.988 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 78.636004501000002, 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13510512999999999, 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.291332+0000", 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.291332+0000", 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.291333+0000", 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7399999999999998e-07 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.291338+0000", 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4140000000000002e-06 2024-04-12T12:02:46.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.291345+0000", 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0609999999999998e-06 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.291351+0000", 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2569999999999998e-06 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.419891+0000", 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12854001800000001 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.419916+0000", 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5749999999999999e-05 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.426186+0000", 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062699679999999999 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:28.426437+0000", 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00025118800000000001 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:230342 11.0 11:06e540e6:::benchmark_data_smithi029_285290_object111651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1345)", 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:34.727495+0000", 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 72.199841456000001, 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.171960481, 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:46.993 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 230342 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.727495+0000", 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.727495+0000", 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.727497+0000", 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6169999999999999e-06 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.727504+0000", 2024-04-12T12:02:46.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6290000000000004e-06 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.727515+0000", 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1464999999999999e-05 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.727523+0000", 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3280000000000006e-06 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:46.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.727582+0000", 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8310000000000002e-05 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.727736+0000", 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000154069 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.727832+0000", 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6578999999999998e-05 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.734421+0000", 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065886540000000002 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.734438+0000", 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7439999999999999e-05 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.765644+0000", 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031205310999999999 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:46.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.765674+0000", 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0358e-05 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.899456+0000", 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13378172099999999 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:230344 11.0 11:0179c16d:::benchmark_data_smithi029_285290_object111653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1345)", 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:34.728023+0000", 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 72.199313630999995, 2024-04-12T12:02:46.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17203163399999999, 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 230344 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.728023+0000", 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.728023+0000", 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.728024+0000", 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4799999999999997e-07 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.728028+0000", 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8670000000000004e-06 2024-04-12T12:02:47.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.728040+0000", 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1955e-05 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.728048+0000", 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2330000000000006e-06 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.731046+0000", 2024-04-12T12:02:47.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002998077 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.731186+0000", 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013996899999999999 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.731329+0000", 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014292600000000001 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:02:47.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.734444+0000", 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031146590000000001 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.736954+0000", 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002510268 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.899534+0000", 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.16258033299999999 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:47.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.899598+0000", 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3829000000000002e-05 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900055+0000", 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00045657 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:230356 11.0 11:066753d6:::benchmark_data_smithi029_285290_object111665:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1345)", 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:34.742152+0000", 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 72.185184405000001, 2024-04-12T12:02:47.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.158341967, 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 230356 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.742152+0000", 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.742152+0000", 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.742154+0000", 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.291e-06 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.742158+0000", 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.69e-06 2024-04-12T12:02:47.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.742169+0000", 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0308e-05 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.742176+0000", 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8050000000000001e-06 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.742237+0000", 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1174999999999996e-05 2024-04-12T12:02:47.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.742368+0000", 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013091900000000001 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.742499+0000", 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131149 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.751136+0000", 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0086374499999999996 2024-04-12T12:02:47.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.754418+0000", 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032818159999999999 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900102+0000", 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.145683697 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900158+0000", 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5843e-05 2024-04-12T12:02:47.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900494+0000", 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000336824 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:230362 11.0 11:0e33d069:::benchmark_data_smithi029_285290_object111671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1345)", 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:34.749328+0000", 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 72.178008798999997, 2024-04-12T12:02:47.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.151442097, 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 230362 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.749328+0000", 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.749328+0000", 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.749329+0000", 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.421e-06 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.749335+0000", 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.84e-06 2024-04-12T12:02:47.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.749348+0000", 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2499e-05 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.749355+0000", 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6000000000000001e-06 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.751187+0000", 2024-04-12T12:02:47.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018311309999999999 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.751311+0000", 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000124621 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.751432+0000", 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012098499999999999 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:02:47.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.754427+0000", 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002994629 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.758192+0000", 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037655560000000002 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900548+0000", 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14235539899999999 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:47.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900593+0000", 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5086999999999997e-05 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900770+0000", 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000177329 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.45800.0:230344 11.0 e1345/1343 ondisk, result = 0)", 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:34.764719+0000", 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 72.162618257999995, 2024-04-12T12:02:47.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13534228400000001, 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764719+0000", 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764719+0000", 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764719+0000", 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8000000000000002e-07 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764722+0000", 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9569999999999998e-06 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764732+0000", 2024-04-12T12:02:47.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0214000000000001e-05 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764737+0000", 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1699999999999996e-06 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.899518+0000", 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.134780445 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:47.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.899528+0000", 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0215e-05 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900061+0000", 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00053300299999999999 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.45800.0:230356 11.0 e1345/1343 ondisk, result = 0)", 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:34.764745+0000", 2024-04-12T12:02:47.021 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 72.162591872999997, 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13575438100000001, 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764745+0000", 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764745+0000", 2024-04-12T12:02:47.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764745+0000", 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4900000000000002e-07 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764748+0000", 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5280000000000002e-06 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:47.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764753+0000", 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5779999999999998e-06 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764758+0000", 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5560000000000001e-06 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900088+0000", 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.135330266 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900096+0000", 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6520000000000006e-06 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900499+0000", 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000403552 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.45800.0:230362 11.0 e1345/1343 ondisk, result = 0)", 2024-04-12T12:02:47.025 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:34.764768+0000", 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 72.162568805999996, 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13600672999999999, 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764768+0000", 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:47.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764768+0000", 2024-04-12T12:02:47.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764768+0000", 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3900000000000001e-07 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764771+0000", 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4320000000000002e-06 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764775+0000", 2024-04-12T12:02:47.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7729999999999999e-06 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.764778+0000", 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7389999999999999e-06 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900527+0000", 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.135748905 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900541+0000", 2024-04-12T12:02:47.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.428e-05 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.900775+0000", 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023336200000000001 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:230372 11.0 11:0aff5718:::benchmark_data_smithi029_285290_object111681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1345)", 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:34.765582+0000", 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 72.161755200000002, 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17792776900000001, 2024-04-12T12:02:47.030 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 230372 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.765582+0000", 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.765582+0000", 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.765583+0000", 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0920000000000001e-06 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.765586+0000", 2024-04-12T12:02:47.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4319999999999999e-06 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.765594+0000", 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1699999999999997e-06 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.765599+0000", 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1669999999999998e-06 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.901006+0000", 2024-04-12T12:02:47.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13540601799999999 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.901154+0000", 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014895800000000001 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.901288+0000", 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013334299999999999 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:47.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.910825+0000", 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095372459999999992 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.910840+0000", 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5367999999999999e-05 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.940268+0000", 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029427814 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.940470+0000", 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020142399999999999 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:34.943509+0000", 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030397369999999998 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:02:47.036 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:02:47.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:47 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/938150301' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T12:02:47.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:47 smithi029 ceph-mon[93650]: osdmap e1359: 8 total, 8 up, 7 in 2024-04-12T12:02:47.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:47 smithi029 ceph-mon[93650]: pgmap v2476: 83 pgs: 1 remapped+peering, 82 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-12T12:02:47.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:47 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/938150301' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T12:02:47.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:47 smithi029 ceph-mon[97787]: osdmap e1359: 8 total, 8 up, 7 in 2024-04-12T12:02:47.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:47 smithi029 ceph-mon[97787]: pgmap v2476: 83 pgs: 1 remapped+peering, 82 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-12T12:02:47.547 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:47 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/938150301' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-04-12T12:02:47.547 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:47 smithi087 ceph-mon[89014]: osdmap e1359: 8 total, 8 up, 7 in 2024-04-12T12:02:47.547 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:47 smithi087 ceph-mon[89014]: pgmap v2476: 83 pgs: 1 remapped+peering, 82 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-12T12:02:47.967 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 12:02:47 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a[62875]: logger=cleanup t=2024-04-12T12:02:47.54616445Z level=info msg="Completed cleanup jobs" duration=12.720905ms 2024-04-12T12:02:48.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd unset nodeep-scrub 2024-04-12T12:02:49.346 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.7 to 1 (10000) 2024-04-12T12:02:49.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-04-12T12:02:49.709 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:49 smithi029 ceph-mon[97787]: pgmap v2477: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-04-12T12:02:49.709 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:49 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2014800551' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:02:49.710 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:49 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:02:49.710 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:49 smithi029 ceph-mon[93650]: pgmap v2477: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-04-12T12:02:49.710 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:49 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2014800551' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:02:49.710 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:49 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:02:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:49 smithi087 ceph-mon[89014]: pgmap v2477: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-04-12T12:02:49.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:49 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2014800551' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:02:49.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:49 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:02:50.078 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 3 1 2024-04-12T12:02:50.110 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:50.114 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:50.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:50 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-04-12T12:02:50.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:50 smithi029 ceph-mon[93650]: osdmap e1360: 8 total, 8 up, 7 in 2024-04-12T12:02:50.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:50 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-04-12T12:02:50.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:50 smithi029 ceph-mon[97787]: osdmap e1360: 8 total, 8 up, 7 in 2024-04-12T12:02:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:50 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-04-12T12:02:50.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:50 smithi087 ceph-mon[89014]: osdmap e1360: 8 total, 8 up, 7 in 2024-04-12T12:02:51.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:51 smithi087 ceph-mon[89014]: pgmap v2479: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 42 KiB/s, 0 objects/s recovering 2024-04-12T12:02:51.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:51 smithi029 ceph-mon[93650]: pgmap v2479: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 42 KiB/s, 0 objects/s recovering 2024-04-12T12:02:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:51 smithi029 ceph-mon[97787]: pgmap v2479: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 42 KiB/s, 0 objects/s recovering 2024-04-12T12:02:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:02:52.423 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:52.619 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:02:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:02:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:02:53.146 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:02:53.146 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T12:02:53.146 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2024-04-12T12:02:53.146 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:02:53.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:53 smithi029 ceph-mon[93650]: pgmap v2480: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-04-12T12:02:53.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:53 smithi029 ceph-mon[97787]: pgmap v2480: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-04-12T12:02:53.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:53 smithi087 ceph-mon[89014]: pgmap v2480: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-04-12T12:02:55.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:02:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:02:54] "GET /metrics HTTP/1.1" 200 34147 "" "Prometheus/2.43.0" 2024-04-12T12:02:55.363 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2024-04-12T12:02:55.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:55 smithi029 ceph-mon[93650]: pgmap v2481: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:55.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:55 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1264046579' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:02:55.545 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:55 smithi029 ceph-mon[97787]: pgmap v2481: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:55.545 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:55 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1264046579' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:02:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:55 smithi087 ceph-mon[89014]: pgmap v2481: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:55 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1264046579' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2024-04-12T12:02:55.818 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-04-12T12:02:56.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:56 smithi029 ceph-mon[97787]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T12:02:56.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:56 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T12:02:56.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:56 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1264046579' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T12:02:56.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:56 smithi029 ceph-mon[97787]: osdmap e1361: 8 total, 8 up, 7 in 2024-04-12T12:02:56.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:56 smithi029 ceph-mon[93650]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T12:02:56.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:56 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T12:02:56.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:56 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1264046579' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T12:02:56.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:56 smithi029 ceph-mon[93650]: osdmap e1361: 8 total, 8 up, 7 in 2024-04-12T12:02:56.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:56 smithi087 ceph-mon[89014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-04-12T12:02:56.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:56 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T12:02:56.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:56 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1264046579' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-04-12T12:02:56.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:56 smithi087 ceph-mon[89014]: osdmap e1361: 8 total, 8 up, 7 in 2024-04-12T12:02:57.036 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:02:57.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:57 smithi029 ceph-mon[93650]: pgmap v2483: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:57.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:57 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:57.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:57 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3903165386' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:02:57.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:57 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:02:57.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:57 smithi029 ceph-mon[97787]: pgmap v2483: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:57.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:57 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:57.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:57 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3903165386' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:02:57.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:57 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:02:57.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:57 smithi087 ceph-mon[89014]: pgmap v2483: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:57.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:57 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:02:57.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:57 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3903165386' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:02:57.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:57 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:02:58.101 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.3 to 1 (10000) 2024-04-12T12:02:58.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 1 1 2024-04-12T12:02:59.231 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:59 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-04-12T12:02:59.231 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:59 smithi029 ceph-mon[97787]: osdmap e1362: 8 total, 8 up, 7 in 2024-04-12T12:02:59.231 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:02:59 smithi029 ceph-mon[97787]: pgmap v2485: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:59.232 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:59 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-04-12T12:02:59.232 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:59 smithi029 ceph-mon[93650]: osdmap e1362: 8 total, 8 up, 7 in 2024-04-12T12:02:59.232 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:02:59 smithi029 ceph-mon[93650]: pgmap v2485: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:59.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:59 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-04-12T12:02:59.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:59 smithi087 ceph-mon[89014]: osdmap e1362: 8 total, 8 up, 7 in 2024-04-12T12:02:59.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:02:59 smithi087 ceph-mon[89014]: pgmap v2485: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:02:59.490 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:02:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T12:02:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T12:02:59.595 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T12:02:59.595 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:02:59.596 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:00.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-04-12T12:03:01.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:01 smithi029 ceph-mon[93650]: pgmap v2486: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:01.607 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:01 smithi029 ceph-mon[97787]: pgmap v2486: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:01.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:01 smithi087 ceph-mon[89014]: pgmap v2486: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:02.041 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:02.353 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.1 to 1 (10000) 2024-04-12T12:03:02.455 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:02 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3253899443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:02.455 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:02 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:02.455 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:02 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3253899443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:02.456 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:02 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:02.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:03:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:03:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:03:02.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:02 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3253899443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:02.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:02 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:03.492 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:03 smithi029 ceph-mon[97787]: pgmap v2487: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:03.492 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:03 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-04-12T12:03:03.493 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:03 smithi029 ceph-mon[97787]: osdmap e1363: 8 total, 8 up, 7 in 2024-04-12T12:03:03.493 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:03 smithi029 ceph-mon[93650]: pgmap v2487: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:03.493 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:03 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-04-12T12:03:03.493 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:03 smithi029 ceph-mon[93650]: osdmap e1363: 8 total, 8 up, 7 in 2024-04-12T12:03:03.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:03 smithi087 ceph-mon[89014]: pgmap v2487: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:03.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:03 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-04-12T12:03:03.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:03 smithi087 ceph-mon[89014]: osdmap e1363: 8 total, 8 up, 7 in 2024-04-12T12:03:04.053 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-04-12T12:03:04.054 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T12:03:04.577 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:03:04.578 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T12:03:04.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T12:03:04.578 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T12:03:04.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.578 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152240 10.9 10:96f5a552:::benchmark_data_smithi029_256330_object50474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.578 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.241233+0000", 2024-04-12T12:03:04.581 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.33242801900002, 2024-04-12T12:03:04.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1357475240000001, 2024-04-12T12:03:04.582 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.582 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.582 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.582 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.582 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.583 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.583 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152240 2024-04-12T12:03:04.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.583 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.241233+0000", 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.241233+0000", 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.241234+0000", 2024-04-12T12:03:04.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.257e-06 2024-04-12T12:03:04.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.241241+0000", 2024-04-12T12:03:04.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4590000000000003e-06 2024-04-12T12:03:04.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.241254+0000", 2024-04-12T12:03:04.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3210000000000001e-05 2024-04-12T12:03:04.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.241262+0000", 2024-04-12T12:03:04.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5890000000000003e-06 2024-04-12T12:03:04.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.241333+0000", 2024-04-12T12:03:04.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0197000000000006e-05 2024-04-12T12:03:04.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2024-04-12T12:03:04.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.241340+0000", 2024-04-12T12:03:04.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4730000000000003e-06 2024-04-12T12:03:04.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.249133+0000", 2024-04-12T12:03:04.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077923669999999997 2024-04-12T12:03:04.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.249220+0000", 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7608000000000006e-05 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.372359+0000", 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.123139092 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.372472+0000", 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000112682 2024-04-12T12:03:04.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.372546+0000", 2024-04-12T12:03:04.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3670000000000004e-05 2024-04-12T12:03:04.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.376871+0000", 2024-04-12T12:03:04.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043249940000000004 2024-04-12T12:03:04.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.376911+0000", 2024-04-12T12:03:04.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0011e-05 2024-04-12T12:03:04.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.376922+0000", 2024-04-12T12:03:04.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1647999999999999e-05 2024-04-12T12:03:04.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.376955+0000", 2024-04-12T12:03:04.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2289999999999997e-05 2024-04-12T12:03:04.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.376980+0000", 2024-04-12T12:03:04.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5976999999999999e-05 2024-04-12T12:03:04.593 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.593 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.594 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.594 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152257 10.9 10:948ac246:::benchmark_data_smithi029_256330_object50491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.594 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.259069+0000", 2024-04-12T12:03:04.594 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.31459169700003, 2024-04-12T12:03:04.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.125534174, 2024-04-12T12:03:04.594 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152257 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.259069+0000", 2024-04-12T12:03:04.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.259069+0000", 2024-04-12T12:03:04.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.259071+0000", 2024-04-12T12:03:04.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3009999999999999e-06 2024-04-12T12:03:04.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.259075+0000", 2024-04-12T12:03:04.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5040000000000004e-06 2024-04-12T12:03:04.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.259091+0000", 2024-04-12T12:03:04.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6259000000000001e-05 2024-04-12T12:03:04.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.259100+0000", 2024-04-12T12:03:04.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9590000000000001e-06 2024-04-12T12:03:04.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.262306+0000", 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032061780000000001 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.262318+0000", 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1912e-05 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.377012+0000", 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1146935680000001 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.377089+0000", 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7423999999999996e-05 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.377168+0000", 2024-04-12T12:03:04.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9019999999999999e-05 2024-04-12T12:03:04.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.384461+0000", 2024-04-12T12:03:04.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072930460000000001 2024-04-12T12:03:04.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.384506+0000", 2024-04-12T12:03:04.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4042999999999998e-05 2024-04-12T12:03:04.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.384516+0000", 2024-04-12T12:03:04.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0257e-05 2024-04-12T12:03:04.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.384558+0000", 2024-04-12T12:03:04.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2299000000000003e-05 2024-04-12T12:03:04.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.384603+0000", 2024-04-12T12:03:04.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5404000000000001e-05 2024-04-12T12:03:04.605 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.605 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.605 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.606 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152266 10.9 10:9cfcd399:::benchmark_data_smithi029_256330_object50500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.606 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.270248+0000", 2024-04-12T12:03:04.606 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.30341266599999, 2024-04-12T12:03:04.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1236261009999999, 2024-04-12T12:03:04.606 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.607 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.607 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.607 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.607 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.607 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.607 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152266 2024-04-12T12:03:04.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.608 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.270248+0000", 2024-04-12T12:03:04.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.270248+0000", 2024-04-12T12:03:04.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.270250+0000", 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.573e-06 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.270256+0000", 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5509999999999996e-06 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.270270+0000", 2024-04-12T12:03:04.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.308e-05 2024-04-12T12:03:04.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.270279+0000", 2024-04-12T12:03:04.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7790000000000007e-06 2024-04-12T12:03:04.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.271715+0000", 2024-04-12T12:03:04.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001435932 2024-04-12T12:03:04.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.271727+0000", 2024-04-12T12:03:04.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.172e-05 2024-04-12T12:03:04.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.380755+0000", 2024-04-12T12:03:04.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.109027695 2024-04-12T12:03:04.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.380855+0000", 2024-04-12T12:03:04.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9945000000000006e-05 2024-04-12T12:03:04.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.380929+0000", 2024-04-12T12:03:04.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4128000000000006e-05 2024-04-12T12:03:04.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.393658+0000", 2024-04-12T12:03:04.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012729333000000001 2024-04-12T12:03:04.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.393709+0000", 2024-04-12T12:03:04.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0767999999999999e-05 2024-04-12T12:03:04.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.393764+0000", 2024-04-12T12:03:04.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.533e-05 2024-04-12T12:03:04.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.393819+0000", 2024-04-12T12:03:04.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4672000000000001e-05 2024-04-12T12:03:04.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.393874+0000", 2024-04-12T12:03:04.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5594999999999999e-05 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152280 10.9 10:947b8cbd:::benchmark_data_smithi029_256330_object50514:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.285935+0000", 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.28772644200001, 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1081029170000001, 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.618 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152280 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.285935+0000", 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.285935+0000", 2024-04-12T12:03:04.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.285937+0000", 2024-04-12T12:03:04.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0339999999999999e-06 2024-04-12T12:03:04.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.286027+0000", 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0787999999999999e-05 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.286050+0000", 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2597000000000001e-05 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.286061+0000", 2024-04-12T12:03:04.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1097000000000001e-05 2024-04-12T12:03:04.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.286137+0000", 2024-04-12T12:03:04.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6130999999999993e-05 2024-04-12T12:03:04.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.286156+0000", 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8352e-05 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.384631+0000", 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0984758939999999 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.384722+0000", 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0909999999999995e-05 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.384819+0000", 2024-04-12T12:03:04.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6742000000000003e-05 2024-04-12T12:03:04.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.393754+0000", 2024-04-12T12:03:04.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089346870000000002 2024-04-12T12:03:04.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.393922+0000", 2024-04-12T12:03:04.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016779100000000001 2024-04-12T12:03:04.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.393934+0000", 2024-04-12T12:03:04.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2334000000000001e-05 2024-04-12T12:03:04.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.393996+0000", 2024-04-12T12:03:04.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2524e-05 2024-04-12T12:03:04.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.394037+0000", 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1035999999999998e-05 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152306 10.9 10:94e5d3a8:::benchmark_data_smithi029_256330_object50540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.628 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.318631+0000", 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.25503006399998, 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0756854389999999, 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152306 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.318631+0000", 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.318631+0000", 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.318632+0000", 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.561e-06 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.318638+0000", 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8220000000000003e-06 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.318651+0000", 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2757e-05 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.318660+0000", 2024-04-12T12:03:04.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5150000000000006e-06 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.318725+0000", 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5418000000000003e-05 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.318736+0000", 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0509000000000001e-05 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.389120+0000", 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.070384459 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.389295+0000", 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000174566 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.389438+0000", 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014371800000000001 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.394048+0000", 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046093059999999996 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.394118+0000", 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0650999999999998e-05 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.394221+0000", 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103232 2024-04-12T12:03:04.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.394278+0000", 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6274999999999999e-05 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.394316+0000", 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8649999999999998e-05 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152307 10.9 10:981024cd:::benchmark_data_smithi029_256330_object50541:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.319969+0000", 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.253692484, 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0913376189999999, 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152307 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.319969+0000", 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.319969+0000", 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.319970+0000", 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.299e-06 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.319975+0000", 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5849999999999999e-06 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.319987+0000", 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1600000000000001e-05 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.319995+0000", 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6860000000000006e-06 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.320053+0000", 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8573999999999999e-05 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.320063+0000", 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0035e-05 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.394359+0000", 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0742959430000001 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.394469+0000", 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010988 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.394574+0000", 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000104511 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.402957+0000", 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0083837089999999996 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411098+0000", 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0081406859999999994 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411227+0000", 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012884699999999999 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411269+0000", 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2283000000000002e-05 2024-04-12T12:03:04.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411306+0000", 2024-04-12T12:03:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6980999999999997e-05 2024-04-12T12:03:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152322 10.9 10:919bb6f0:::benchmark_data_smithi029_256330_object50556:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.483512+0000", 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.09014902500002, 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92799271699999997, 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152322 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.483512+0000", 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.483512+0000", 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.483514+0000", 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8300000000000001e-06 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.483523+0000", 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8599999999999999e-06 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.483537+0000", 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4124000000000001e-05 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.483544+0000", 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7819999999999996e-06 2024-04-12T12:03:04.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.483626+0000", 2024-04-12T12:03:04.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2724000000000003e-05 2024-04-12T12:03:04.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.483637+0000", 2024-04-12T12:03:04.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0959e-05 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.399299+0000", 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.91566140600000001 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.399398+0000", 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8918000000000004e-05 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.399508+0000", 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110571 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411170+0000", 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011661906999999999 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411319+0000", 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000149044 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411423+0000", 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000104209 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411464+0000", 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0281000000000003e-05 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411505+0000", 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1102000000000001e-05 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.649 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152330 10.9 10:9a1de603:::benchmark_data_smithi029_256330_object50564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.510174+0000", 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.06348686199999, 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.91024621100000003, 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.650 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152330 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.510174+0000", 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.510174+0000", 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.510176+0000", 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.779e-06 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.510183+0000", 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5669999999999997e-06 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.510201+0000", 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.77e-05 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.510211+0000", 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9979999999999998e-06 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.510312+0000", 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100988 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.510323+0000", 2024-04-12T12:03:04.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1307e-05 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.402996+0000", 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.892672727 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.403112+0000", 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011582900000000001 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.403242+0000", 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013004800000000001 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411326+0000", 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0080844109999999997 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411374+0000", 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7757999999999999e-05 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420296+0000", 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00892161 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420366+0000", 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0276000000000003e-05 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420420+0000", 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4212999999999997e-05 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.656 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152352 10.9 10:9a452f86:::benchmark_data_smithi029_256330_object50586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.538791+0000", 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.03486987700001, 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.88186087499999999, 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.657 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152352 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.538791+0000", 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.538791+0000", 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.538792+0000", 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3480000000000001e-06 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.538798+0000", 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.755e-06 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.538814+0000", 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6075000000000001e-05 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.538824+0000", 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3219999999999999e-06 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.538904+0000", 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0233000000000002e-05 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.538919+0000", 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5204999999999999e-05 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.407380+0000", 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86846126999999995 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.407503+0000", 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012282300000000001 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.407591+0000", 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7622999999999998e-05 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411517+0000", 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039266140000000001 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420468+0000", 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089507189999999993 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420521+0000", 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3306000000000001e-05 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420610+0000", 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9000999999999997e-05 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420652+0000", 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1581000000000001e-05 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.663 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152363 10.9 10:993f08ac:::benchmark_data_smithi029_256330_object50597:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.552490+0000", 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.02117100100003, 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86838422800000004, 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152363 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.552490+0000", 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.552490+0000", 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.552491+0000", 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6400000000000001e-07 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.552495+0000", 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2200000000000003e-06 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.552508+0000", 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3339000000000001e-05 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.552515+0000", 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8210000000000004e-06 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.552576+0000", 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0996999999999999e-05 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.552589+0000", 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2514000000000001e-05 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411602+0000", 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85901289300000006 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411698+0000", 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6182999999999997e-05 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.411769+0000", 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1507999999999994e-05 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420479+0000", 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.008709952 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420699+0000", 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021985400000000001 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420743+0000", 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4162000000000002e-05 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420832+0000", 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8608000000000003e-05 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420874+0000", 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2313e-05 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.670 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152416 10.9 10:9c5c9a68:::benchmark_data_smithi029_256330_object50650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.626864+0000", 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 422.94679731000002, 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.80313656899999997, 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152416 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.626864+0000", 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.626864+0000", 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.626865+0000", 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3740000000000001e-06 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.626870+0000", 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2460000000000003e-06 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.626886+0000", 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5783000000000001e-05 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.626895+0000", 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9469999999999992e-06 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.626964+0000", 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8909e-05 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.626975+0000", 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0744000000000001e-05 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.416180+0000", 2024-04-12T12:03:04.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78920567500000005 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.416324+0000", 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014358900000000001 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.416440+0000", 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115568 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.429764+0000", 2024-04-12T12:03:04.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013324904 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.429813+0000", 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8324e-05 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.429896+0000", 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3047000000000006e-05 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.429954+0000", 2024-04-12T12:03:04.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7747000000000001e-05 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.430000+0000", 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6712000000000003e-05 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152418 10.9 10:90ff6269:::benchmark_data_smithi029_256330_object50652:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.630789+0000", 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 422.942872034, 2024-04-12T12:03:04.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.80388811500000001, 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152418 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.630789+0000", 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.630789+0000", 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.630790+0000", 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4750000000000001e-06 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.630797+0000", 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0589999999999997e-06 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.630814+0000", 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6620999999999999e-05 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.630823+0000", 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8380000000000008e-06 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.630882+0000", 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8734999999999999e-05 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.630890+0000", 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2439999999999993e-06 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.420907+0000", 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.79001754899999999 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.421013+0000", 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105505 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.421126+0000", 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113497 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.429832+0000", 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0087053670000000003 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.430155+0000", 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032324900000000001 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.434532+0000", 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043767049999999998 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.434631+0000", 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9592000000000005e-05 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.434677+0000", 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5679000000000001e-05 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35727.0:152441 10.9 10:985925cc:::benchmark_data_smithi029_256330_object50675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1301)", 2024-04-12T12:03:04.684 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:56:01.666562+0000", 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 422.90709938499998, 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.79326451899999995, 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35727", 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 152441 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.666562+0000", 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.666562+0000", 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.666563+0000", 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.787e-06 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.666579+0000", 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5569e-05 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.666594+0000", 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5122e-05 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.666603+0000", 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5520000000000005e-06 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.666677+0000", 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4002000000000001e-05 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:01.666687+0000", 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0702e-05 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.430190+0000", 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76350308200000006 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.430308+0000", 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117429 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.430422+0000", 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114123 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.444921+0000", 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01449864 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.444960+0000", 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9323999999999998e-05 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.459721+0000", 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014761051000000001 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.459774+0000", 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3211000000000003e-05 2024-04-12T12:03:04.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:56:02.459826+0000", 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1925000000000002e-05 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130435 11.f 11:f43abcc5:::benchmark_data_smithi029_285290_object11744:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.296407+0000", 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 209.27725352300001, 2024-04-12T12:03:04.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0151951889999999, 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130435 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.296407+0000", 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.296407+0000", 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.296409+0000", 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.663e-06 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.296415+0000", 2024-04-12T12:03:04.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.857e-06 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.296428+0000", 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2931999999999999e-05 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.296436+0000", 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8790000000000003e-06 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.300024+0000", 2024-04-12T12:03:04.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035881239999999998 2024-04-12T12:03:04.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2024-04-12T12:03:04.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.300031+0000", 2024-04-12T12:03:04.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1690000000000003e-06 2024-04-12T12:03:04.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.301253+0000", 2024-04-12T12:03:04.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001221761 2024-04-12T12:03:04.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.301265+0000", 2024-04-12T12:03:04.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1871e-05 2024-04-12T12:03:04.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.309455+0000", 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0081906199999999 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.311603+0000", 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021473130000000001 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130451 11.f 11:f164388f:::benchmark_data_smithi029_285290_object11760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.315259+0000", 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 209.25840162899999, 2024-04-12T12:03:04.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0192472349999999, 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130451 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.315259+0000", 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.315259+0000", 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.315261+0000", 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6819999999999999e-06 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.315268+0000", 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4579999999999998e-06 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.315285+0000", 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7425e-05 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.315293+0000", 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4940000000000003e-06 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.318964+0000", 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036707979999999999 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.318975+0000", 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0736e-05 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.312149+0000", 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99317399500000003 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.312834+0000", 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00068511400000000001 2024-04-12T12:03:04.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.312957+0000", 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012339800000000001 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.319549+0000", 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065917650000000003 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.319590+0000", 2024-04-12T12:03:04.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0652000000000003e-05 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334405+0000", 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014814944 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334465+0000", 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0386000000000001e-05 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334507+0000", 2024-04-12T12:03:04.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1387999999999997e-05 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130455 11.f 11:f9be9d8c:::benchmark_data_smithi029_285290_object11764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.318371+0000", 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 209.25529012300001, 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0162402020000001, 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130455 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.318371+0000", 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.318371+0000", 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.318372+0000", 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7599999999999997e-07 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.318376+0000", 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2169999999999996e-06 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.318385+0000", 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3959999999999995e-06 2024-04-12T12:03:04.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.318392+0000", 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7190000000000004e-06 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.319005+0000", 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00061302100000000005 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.319013+0000", 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2519999999999999e-06 2024-04-12T12:03:04.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.316081+0000", 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99706776699999999 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.316221+0000", 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014056299999999999 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.316350+0000", 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012815400000000001 2024-04-12T12:03:04.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.319628+0000", 2024-04-12T12:03:04.709 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:04.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032782419999999998 2024-04-12T12:03:04.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.326505+0000", 2024-04-12T12:03:04.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0068772130000000001 2024-04-12T12:03:04.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334516+0000", 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0080111880000000007 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334573+0000", 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6619000000000001e-05 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334611+0000", 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8175e-05 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.711 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130497 11.f 11:f8f1e606:::benchmark_data_smithi029_285290_object11806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.363918+0000", 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 209.20974344800001, 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97095930399999997, 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130497 2024-04-12T12:03:04.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.363918+0000", 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.363918+0000", 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.363918+0000", 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4900000000000004e-07 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.363922+0000", 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9339999999999999e-06 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.363930+0000", 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3520000000000001e-06 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.363935+0000", 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6579999999999999e-06 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.363999+0000", 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3429999999999994e-05 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.364012+0000", 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2805e-05 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.319657+0000", 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95564554800000001 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.319785+0000", 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012747499999999999 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.319903+0000", 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011858000000000001 2024-04-12T12:03:04.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.326556+0000", 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066526880000000004 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.326602+0000", 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5933999999999998e-05 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334622+0000", 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0080204569999999999 2024-04-12T12:03:04.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334832+0000", 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000209748 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334877+0000", 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4746000000000002e-05 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130502 11.f 11:f8f8e3c6:::benchmark_data_smithi029_285290_object11811:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.369303+0000", 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 209.204357974, 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96664910699999995, 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130502 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.369303+0000", 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.369303+0000", 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.369304+0000", 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1599999999999999e-06 2024-04-12T12:03:04.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.369309+0000", 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0969999999999996e-06 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.369322+0000", 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2534000000000001e-05 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.369329+0000", 2024-04-12T12:03:04.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6149999999999999e-06 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.369406+0000", 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7070000000000006e-05 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.369418+0000", 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1110999999999999e-05 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.323035+0000", 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95361783499999997 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.323170+0000", 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013406299999999999 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.323286+0000", 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011622 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334316+0000", 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011029779999999999 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334889+0000", 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00057347799999999999 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.335744+0000", 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00085477399999999999 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.335911+0000", 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000167441 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.335952+0000", 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0929000000000001e-05 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130526 11.f 11:ff87a53c:::benchmark_data_smithi029_285290_object11835:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.392710+0000", 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 209.18095091699999, 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94794170300000002, 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130526 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.392710+0000", 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.392710+0000", 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.392712+0000", 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.606e-06 2024-04-12T12:03:04.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.392717+0000", 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7450000000000001e-06 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.392730+0000", 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2449999999999999e-05 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.392744+0000", 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4450000000000001e-05 2024-04-12T12:03:04.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.392815+0000", 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0450000000000005e-05 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.392826+0000", 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1629e-05 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.326654+0000", 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93382773699999999 2024-04-12T12:03:04.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.326793+0000", 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000138753 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.326914+0000", 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012128 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334360+0000", 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074453669999999996 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.334897+0000", 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00053708900000000003 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.340544+0000", 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056478479999999996 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.340605+0000", 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0534e-05 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.340652+0000", 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6764999999999997e-05 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45800.0:130541 11.f 11:f6e33ed9:::benchmark_data_smithi029_285290_object11850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1327)", 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T11:59:35.409130+0000", 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 209.16453074, 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93422754600000002, 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45800", 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 130541 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.733 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.409130+0000", 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.409130+0000", 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.409131+0000", 2024-04-12T12:03:04.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9400000000000004e-07 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.409135+0000", 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1269999999999996e-06 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.409153+0000", 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7603000000000001e-05 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:04.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.409161+0000", 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1070000000000005e-06 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.409226+0000", 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4654999999999997e-05 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:35.409234+0000", 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0330000000000008e-06 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.329925+0000", 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92069110600000004 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.330048+0000", 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012317 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.330182+0000", 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013371300000000001 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.335696+0000", 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055142129999999996 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.340663+0000", 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049675839999999997 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.343169+0000", 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025055640000000001 2024-04-12T12:03:04.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.343310+0000", 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014058899999999999 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T11:59:36.343358+0000", 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8188e-05 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:04.739 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:03:05.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:03:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:03:04] "GET /metrics HTTP/1.1" 200 34147 "" "Prometheus/2.43.0" 2024-04-12T12:03:05.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:05 smithi029 ceph-mon[93650]: pgmap v2489: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:05.626 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:05 smithi029 ceph-mon[97787]: pgmap v2489: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:05.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:05 smithi087 ceph-mon[89014]: pgmap v2489: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:06.655 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-04-12T12:03:06.874 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:07.461 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:07 smithi029 ceph-mon[97787]: pgmap v2490: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:07.461 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:07.461 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:07.461 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:07 smithi029 ceph-mon[93650]: pgmap v2490: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:07.462 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:07.462 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:07.669 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:07 smithi087 ceph-mon[89014]: pgmap v2490: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:07.670 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:07.670 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T12:03:07.930 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T12:03:08.589 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:08.590 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T12:03:08.590 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T12:03:08.590 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:08.651 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2490,"stamp":"2024-04-12T12:03:06.288329+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":800812,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1826,"ondisk_log_size":1826,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5778468,"kb_used_data":61008,"kb_used_omap":189,"kb_used_meta":5717186,"kb_avail":650437596,"statfs":{"total":671965249536,"available":666048098304,"internally_reserved":0,"allocated":62472192,"data_stored":40720152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":193677,"internal_metadata":5854399347},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.194915"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":2099,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623411+0000","last_change":"2024-04-12T12:00:35.472642+0000","last_active":"2024-04-12T12:03:02.623411+0000","last_peered":"2024-04-12T12:03:02.623411+0000","last_clean":"2024-04-12T12:03:02.623411+0000","last_became_active":"2024-04-12T12:00:35.472258+0000","last_became_peered":"2024-04-12T12:00:35.472258+0000","last_unstale":"2024-04-12T12:03:02.623411+0000","last_undegraded":"2024-04-12T12:03:02.623411+0000","last_fullsized":"2024-04-12T12:03:02.623411+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:41:16.991252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346566+0000","last_change":"2024-04-12T12:01:09.588317+0000","last_active":"2024-04-12T12:03:02.346566+0000","last_peered":"2024-04-12T12:03:02.346566+0000","last_clean":"2024-04-12T12:03:02.346566+0000","last_became_active":"2024-04-12T12:01:09.587996+0000","last_became_peered":"2024-04-12T12:01:09.587996+0000","last_unstale":"2024-04-12T12:03:02.346566+0000","last_undegraded":"2024-04-12T12:03:02.346566+0000","last_fullsized":"2024-04-12T12:03:02.346566+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:48:36.981875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346584+0000","last_change":"2024-04-12T12:01:09.587349+0000","last_active":"2024-04-12T12:03:02.346584+0000","last_peered":"2024-04-12T12:03:02.346584+0000","last_clean":"2024-04-12T12:03:02.346584+0000","last_became_active":"2024-04-12T12:01:09.587019+0000","last_became_peered":"2024-04-12T12:01:09.587019+0000","last_unstale":"2024-04-12T12:03:02.346584+0000","last_undegraded":"2024-04-12T12:03:02.346584+0000","last_fullsized":"2024-04-12T12:03:02.346584+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:19:17.511244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2201,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347895+0000","last_change":"2024-04-12T12:01:08.211946+0000","last_active":"2024-04-12T12:03:02.347895+0000","last_peered":"2024-04-12T12:03:02.347895+0000","last_clean":"2024-04-12T12:03:02.347895+0000","last_became_active":"2024-04-12T12:01:08.209590+0000","last_became_peered":"2024-04-12T12:01:08.209590+0000","last_unstale":"2024-04-12T12:03:02.347895+0000","last_undegraded":"2024-04-12T12:03:02.347895+0000","last_fullsized":"2024-04-12T12:03:02.347895+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:08.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":749,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347715+0000","last_change":"2024-04-12T12:01:09.236804+0000","last_active":"2024-04-12T12:03:02.347715+0000","last_peered":"2024-04-12T12:03:02.347715+0000","last_clean":"2024-04-12T12:03:02.347715+0000","last_became_active":"2024-04-12T12:01:09.236485+0000","last_became_peered":"2024-04-12T12:01:09.236485+0000","last_unstale":"2024-04-12T12:03:02.347715+0000","last_undegraded":"2024-04-12T12:03:02.347715+0000","last_fullsized":"2024-04-12T12:03:02.347715+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:23.077697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1358,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.095039+0000","last_change":"2024-04-12T12:00:35.474896+0000","last_active":"2024-04-12T12:02:58.095039+0000","last_peered":"2024-04-12T12:02:58.095039+0000","last_clean":"2024-04-12T12:02:58.095039+0000","last_became_active":"2024-04-12T12:00:35.474579+0000","last_became_peered":"2024-04-12T12:00:35.474579+0000","last_unstale":"2024-04-12T12:02:58.095039+0000","last_undegraded":"2024-04-12T12:02:58.095039+0000","last_fullsized":"2024-04-12T12:02:58.095039+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:51:32.229812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2201,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348305+0000","last_change":"2024-04-12T12:02:42.081163+0000","last_active":"2024-04-12T12:03:02.348305+0000","last_peered":"2024-04-12T12:03:02.348305+0000","last_clean":"2024-04-12T12:03:02.348305+0000","last_became_active":"2024-04-12T12:02:42.080873+0000","last_became_peered":"2024-04-12T12:02:42.080873+0000","last_unstale":"2024-04-12T12:03:02.348305+0000","last_undegraded":"2024-04-12T12:03:02.348305+0000","last_fullsized":"2024-04-12T12:03:02.348305+0000","mapping_epoch":1354,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:22:43.949432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":190,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094757+0000","last_change":"2024-04-12T12:01:09.514195+0000","last_active":"2024-04-12T12:02:58.094757+0000","last_peered":"2024-04-12T12:02:58.094757+0000","last_clean":"2024-04-12T12:02:58.094757+0000","last_became_active":"2024-04-12T12:01:09.513505+0000","last_became_peered":"2024-04-12T12:01:09.513505+0000","last_unstale":"2024-04-12T12:02:58.094757+0000","last_undegraded":"2024-04-12T12:02:58.094757+0000","last_fullsized":"2024-04-12T12:02:58.094757+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:43.907048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347552+0000","last_change":"2024-04-12T12:01:09.224595+0000","last_active":"2024-04-12T12:03:02.347552+0000","last_peered":"2024-04-12T12:03:02.347552+0000","last_clean":"2024-04-12T12:03:02.347552+0000","last_became_active":"2024-04-12T12:01:09.224210+0000","last_became_peered":"2024-04-12T12:01:09.224210+0000","last_unstale":"2024-04-12T12:03:02.347552+0000","last_undegraded":"2024-04-12T12:03:02.347552+0000","last_fullsized":"2024-04-12T12:03:02.347552+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:22.161025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":575,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094490+0000","last_change":"2024-04-12T12:01:08.236429+0000","last_active":"2024-04-12T12:02:58.094490+0000","last_peered":"2024-04-12T12:02:58.094490+0000","last_clean":"2024-04-12T12:02:58.094490+0000","last_became_active":"2024-04-12T12:01:08.236170+0000","last_became_peered":"2024-04-12T12:01:08.236170+0000","last_unstale":"2024-04-12T12:02:58.094490+0000","last_undegraded":"2024-04-12T12:02:58.094490+0000","last_fullsized":"2024-04-12T12:02:58.094490+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:54:28.854893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":219,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.439062+0000","last_change":"2024-04-12T12:01:08.204030+0000","last_active":"2024-04-12T12:02:55.439062+0000","last_peered":"2024-04-12T12:02:55.439062+0000","last_clean":"2024-04-12T12:02:55.439062+0000","last_became_active":"2024-04-12T12:01:08.203790+0000","last_became_peered":"2024-04-12T12:01:08.203790+0000","last_unstale":"2024-04-12T12:02:55.439062+0000","last_undegraded":"2024-04-12T12:02:55.439062+0000","last_fullsized":"2024-04-12T12:02:55.439062+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:18:55.719738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":76,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347096+0000","last_change":"2024-04-12T12:02:42.081270+0000","last_active":"2024-04-12T12:03:02.347096+0000","last_peered":"2024-04-12T12:03:02.347096+0000","last_clean":"2024-04-12T12:03:02.347096+0000","last_became_active":"2024-04-12T12:02:42.080832+0000","last_became_peered":"2024-04-12T12:02:42.080832+0000","last_unstale":"2024-04-12T12:03:02.347096+0000","last_undegraded":"2024-04-12T12:03:02.347096+0000","last_fullsized":"2024-04-12T12:03:02.347096+0000","mapping_epoch":1354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:06.752668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":2092,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.624081+0000","last_change":"2024-04-12T11:45:53.024626+0000","last_active":"2024-04-12T12:03:02.624081+0000","last_peered":"2024-04-12T12:03:02.624081+0000","last_clean":"2024-04-12T12:03:02.624081+0000","last_became_active":"2024-04-12T11:45:52.976276+0000","last_became_peered":"2024-04-12T11:45:52.976276+0000","last_unstale":"2024-04-12T12:03:02.624081+0000","last_undegraded":"2024-04-12T12:03:02.624081+0000","last_fullsized":"2024-04-12T12:03:02.624081+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:49.929210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":2063,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.624285+0000","last_change":"2024-04-12T12:00:35.473701+0000","last_active":"2024-04-12T12:03:02.624285+0000","last_peered":"2024-04-12T12:03:02.624285+0000","last_clean":"2024-04-12T12:03:02.624285+0000","last_became_active":"2024-04-12T12:00:35.473318+0000","last_became_peered":"2024-04-12T12:00:35.473318+0000","last_unstale":"2024-04-12T12:03:02.624285+0000","last_undegraded":"2024-04-12T12:03:02.624285+0000","last_fullsized":"2024-04-12T12:03:02.624285+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:45:46.936638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":174,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.438972+0000","last_change":"2024-04-12T11:45:53.035604+0000","last_active":"2024-04-12T12:02:55.438972+0000","last_peered":"2024-04-12T12:02:55.438972+0000","last_clean":"2024-04-12T12:02:55.438972+0000","last_became_active":"2024-04-12T11:45:52.984722+0000","last_became_peered":"2024-04-12T11:45:52.984722+0000","last_unstale":"2024-04-12T12:02:55.438972+0000","last_undegraded":"2024-04-12T12:02:55.438972+0000","last_fullsized":"2024-04-12T12:02:55.438972+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:30.674395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346861+0000","last_change":"2024-04-12T12:01:09.225722+0000","last_active":"2024-04-12T12:03:02.346861+0000","last_peered":"2024-04-12T12:03:02.346861+0000","last_clean":"2024-04-12T12:03:02.346861+0000","last_became_active":"2024-04-12T12:01:09.224774+0000","last_became_peered":"2024-04-12T12:01:09.224774+0000","last_unstale":"2024-04-12T12:03:02.346861+0000","last_undegraded":"2024-04-12T12:03:02.346861+0000","last_fullsized":"2024-04-12T12:03:02.346861+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:19:05.420649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2184,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346839+0000","last_change":"2024-04-12T12:01:08.222480+0000","last_active":"2024-04-12T12:03:02.346839+0000","last_peered":"2024-04-12T12:03:02.346839+0000","last_clean":"2024-04-12T12:03:02.346839+0000","last_became_active":"2024-04-12T12:01:08.222018+0000","last_became_peered":"2024-04-12T12:01:08.222018+0000","last_unstale":"2024-04-12T12:03:02.346839+0000","last_undegraded":"2024-04-12T12:03:02.346839+0000","last_fullsized":"2024-04-12T12:03:02.346839+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:12:10.882773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1358,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.438840+0000","last_change":"2024-04-12T11:57:25.432612+0000","last_active":"2024-04-12T12:02:55.438840+0000","last_peered":"2024-04-12T12:02:55.438840+0000","last_clean":"2024-04-12T12:02:55.438840+0000","last_became_active":"2024-04-12T11:57:25.432182+0000","last_became_peered":"2024-04-12T11:57:25.432182+0000","last_unstale":"2024-04-12T12:02:55.438840+0000","last_undegraded":"2024-04-12T12:02:55.438840+0000","last_fullsized":"2024-04-12T12:02:55.438840+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:00:10.448754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1254,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622566+0000","last_change":"2024-04-12T11:59:36.297225+0000","last_active":"2024-04-12T12:03:02.622566+0000","last_peered":"2024-04-12T12:03:02.622566+0000","last_clean":"2024-04-12T12:03:02.622566+0000","last_became_active":"2024-04-12T11:59:36.296877+0000","last_became_peered":"2024-04-12T11:59:36.296877+0000","last_unstale":"2024-04-12T12:03:02.622566+0000","last_undegraded":"2024-04-12T12:03:02.622566+0000","last_fullsized":"2024-04-12T12:03:02.622566+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:26:53.114573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2184,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347981+0000","last_change":"2024-04-12T12:01:08.213742+0000","last_active":"2024-04-12T12:03:02.347981+0000","last_peered":"2024-04-12T12:03:02.347981+0000","last_clean":"2024-04-12T12:03:02.347981+0000","last_became_active":"2024-04-12T12:01:08.211529+0000","last_became_peered":"2024-04-12T12:01:08.211529+0000","last_unstale":"2024-04-12T12:03:02.347981+0000","last_undegraded":"2024-04-12T12:03:02.347981+0000","last_fullsized":"2024-04-12T12:03:02.347981+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:18.227656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1365,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094488+0000","last_change":"2024-04-12T12:01:08.218173+0000","last_active":"2024-04-12T12:02:58.094488+0000","last_peered":"2024-04-12T12:02:58.094488+0000","last_clean":"2024-04-12T12:02:58.094488+0000","last_became_active":"2024-04-12T12:01:08.217894+0000","last_became_peered":"2024-04-12T12:01:08.217894+0000","last_unstale":"2024-04-12T12:02:58.094488+0000","last_undegraded":"2024-04-12T12:02:58.094488+0000","last_fullsized":"2024-04-12T12:02:58.094488+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:11:50.138367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":189,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.096106+0000","last_change":"2024-04-12T12:01:09.226707+0000","last_active":"2024-04-12T12:02:58.096106+0000","last_peered":"2024-04-12T12:02:58.096106+0000","last_clean":"2024-04-12T12:02:58.096106+0000","last_became_active":"2024-04-12T12:01:09.226344+0000","last_became_peered":"2024-04-12T12:01:09.226344+0000","last_unstale":"2024-04-12T12:02:58.096106+0000","last_undegraded":"2024-04-12T12:02:58.096106+0000","last_fullsized":"2024-04-12T12:02:58.096106+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:50:21.461207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":172,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.439097+0000","last_change":"2024-04-12T11:45:53.028640+0000","last_active":"2024-04-12T12:02:55.439097+0000","last_peered":"2024-04-12T12:02:55.439097+0000","last_clean":"2024-04-12T12:02:55.439097+0000","last_became_active":"2024-04-12T11:45:52.983099+0000","last_became_peered":"2024-04-12T11:45:52.983099+0000","last_unstale":"2024-04-12T12:02:55.439097+0000","last_undegraded":"2024-04-12T12:02:55.439097+0000","last_fullsized":"2024-04-12T12:02:55.439097+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:43:01.897014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1307,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.438422+0000","last_change":"2024-04-12T12:02:42.081502+0000","last_active":"2024-04-12T12:02:55.438422+0000","last_peered":"2024-04-12T12:02:55.438422+0000","last_clean":"2024-04-12T12:02:55.438422+0000","last_became_active":"2024-04-12T12:02:42.081079+0000","last_became_peered":"2024-04-12T12:02:42.081079+0000","last_unstale":"2024-04-12T12:02:55.438422+0000","last_undegraded":"2024-04-12T12:02:55.438422+0000","last_fullsized":"2024-04-12T12:02:55.438422+0000","mapping_epoch":1354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:36:47.384736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":198,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.438964+0000","last_change":"2024-04-12T12:00:35.474735+0000","last_active":"2024-04-12T12:02:55.438964+0000","last_peered":"2024-04-12T12:02:55.438964+0000","last_clean":"2024-04-12T12:02:55.438964+0000","last_became_active":"2024-04-12T12:00:35.463433+0000","last_became_peered":"2024-04-12T12:00:35.463433+0000","last_unstale":"2024-04-12T12:02:55.438964+0000","last_undegraded":"2024-04-12T12:02:55.438964+0000","last_fullsized":"2024-04-12T12:02:55.438964+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:27.215562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346426+0000","last_change":"2024-04-12T12:01:09.591568+0000","last_active":"2024-04-12T12:03:02.346426+0000","last_peered":"2024-04-12T12:03:02.346426+0000","last_clean":"2024-04-12T12:03:02.346426+0000","last_became_active":"2024-04-12T12:01:09.591189+0000","last_became_peered":"2024-04-12T12:01:09.591189+0000","last_unstale":"2024-04-12T12:03:02.346426+0000","last_undegraded":"2024-04-12T12:03:02.346426+0000","last_fullsized":"2024-04-12T12:03:02.346426+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:56:06.870492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":75,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094923+0000","last_change":"2024-04-12T12:01:08.200089+0000","last_active":"2024-04-12T12:02:58.094923+0000","last_peered":"2024-04-12T12:02:58.094923+0000","last_clean":"2024-04-12T12:02:58.094923+0000","last_became_active":"2024-04-12T12:01:08.199531+0000","last_became_peered":"2024-04-12T12:01:08.199531+0000","last_unstale":"2024-04-12T12:02:58.094923+0000","last_undegraded":"2024-04-12T12:02:58.094923+0000","last_fullsized":"2024-04-12T12:02:58.094923+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:51:22.075323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1363,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094674+0000","last_change":"2024-04-12T12:00:35.474945+0000","last_active":"2024-04-12T12:02:58.094674+0000","last_peered":"2024-04-12T12:02:58.094674+0000","last_clean":"2024-04-12T12:02:58.094674+0000","last_became_active":"2024-04-12T12:00:35.474564+0000","last_became_peered":"2024-04-12T12:00:35.474564+0000","last_unstale":"2024-04-12T12:02:58.094674+0000","last_undegraded":"2024-04-12T12:02:58.094674+0000","last_fullsized":"2024-04-12T12:02:58.094674+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:13:53.178867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1366,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094715+0000","last_change":"2024-04-12T12:01:09.233104+0000","last_active":"2024-04-12T12:02:58.094715+0000","last_peered":"2024-04-12T12:02:58.094715+0000","last_clean":"2024-04-12T12:02:58.094715+0000","last_became_active":"2024-04-12T12:01:09.232861+0000","last_became_peered":"2024-04-12T12:01:09.232861+0000","last_unstale":"2024-04-12T12:02:58.094715+0000","last_undegraded":"2024-04-12T12:02:58.094715+0000","last_fullsized":"2024-04-12T12:02:58.094715+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:20:34.752472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2165,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347096+0000","last_change":"2024-04-12T12:00:35.475414+0000","last_active":"2024-04-12T12:03:02.347096+0000","last_peered":"2024-04-12T12:03:02.347096+0000","last_clean":"2024-04-12T12:03:02.347096+0000","last_became_active":"2024-04-12T12:00:35.475098+0000","last_became_peered":"2024-04-12T12:00:35.475098+0000","last_unstale":"2024-04-12T12:03:02.347096+0000","last_undegraded":"2024-04-12T12:03:02.347096+0000","last_fullsized":"2024-04-12T12:03:02.347096+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:25:12.377240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"249'161","reported_seq":1727,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.438662+0000","last_change":"2024-04-12T12:01:08.230318+0000","last_active":"2024-04-12T12:02:55.438662+0000","last_peered":"2024-04-12T12:02:55.438662+0000","last_clean":"2024-04-12T12:02:55.438662+0000","last_became_active":"2024-04-12T12:01:08.229959+0000","last_became_peered":"2024-04-12T12:01:08.229959+0000","last_unstale":"2024-04-12T12:02:55.438662+0000","last_undegraded":"2024-04-12T12:02:55.438662+0000","last_fullsized":"2024-04-12T12:02:55.438662+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:08.650841+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:08.650841+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:17:07.248029+0000","scrub_duration":0.001456279,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":251,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347200+0000","last_change":"2024-04-12T12:01:08.360054+0000","last_active":"2024-04-12T12:03:02.347200+0000","last_peered":"2024-04-12T12:03:02.347200+0000","last_clean":"2024-04-12T12:03:02.347200+0000","last_became_active":"2024-04-12T12:01:08.359716+0000","last_became_peered":"2024-04-12T12:01:08.359716+0000","last_unstale":"2024-04-12T12:03:02.347200+0000","last_undegraded":"2024-04-12T12:03:02.347200+0000","last_fullsized":"2024-04-12T12:03:02.347200+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:05.807406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1886,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094763+0000","last_change":"2024-04-12T12:01:08.223388+0000","last_active":"2024-04-12T12:02:58.094763+0000","last_peered":"2024-04-12T12:02:58.094763+0000","last_clean":"2024-04-12T12:02:58.094763+0000","last_became_active":"2024-04-12T12:01:08.222873+0000","last_became_peered":"2024-04-12T12:01:08.222873+0000","last_unstale":"2024-04-12T12:02:58.094763+0000","last_undegraded":"2024-04-12T12:02:58.094763+0000","last_fullsized":"2024-04-12T12:02:58.094763+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:02.009904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"249'161","reported_seq":862,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622404+0000","last_change":"2024-04-12T12:02:44.218756+0000","last_active":"2024-04-12T12:03:02.622404+0000","last_peered":"2024-04-12T12:03:02.622404+0000","last_clean":"2024-04-12T12:03:02.622404+0000","last_became_active":"2024-04-12T12:02:44.218302+0000","last_became_peered":"2024-04-12T12:02:44.218302+0000","last_unstale":"2024-04-12T12:03:02.622404+0000","last_undegraded":"2024-04-12T12:03:02.622404+0000","last_fullsized":"2024-04-12T12:03:02.622404+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:10.437190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:10.437190+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:36:07.969961+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1106,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623521+0000","last_change":"2024-04-12T12:00:35.472844+0000","last_active":"2024-04-12T12:03:02.623521+0000","last_peered":"2024-04-12T12:03:02.623521+0000","last_clean":"2024-04-12T12:03:02.623521+0000","last_became_active":"2024-04-12T12:00:35.472506+0000","last_became_peered":"2024-04-12T12:00:35.472506+0000","last_unstale":"2024-04-12T12:03:02.623521+0000","last_undegraded":"2024-04-12T12:03:02.623521+0000","last_fullsized":"2024-04-12T12:03:02.623521+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:01:56.528972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":2049,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623129+0000","last_change":"2024-04-12T12:00:35.468294+0000","last_active":"2024-04-12T12:03:02.623129+0000","last_peered":"2024-04-12T12:03:02.623129+0000","last_clean":"2024-04-12T12:03:02.623129+0000","last_became_active":"2024-04-12T12:00:35.467888+0000","last_became_peered":"2024-04-12T12:00:35.467888+0000","last_unstale":"2024-04-12T12:03:02.623129+0000","last_undegraded":"2024-04-12T12:03:02.623129+0000","last_fullsized":"2024-04-12T12:03:02.623129+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:35:16.687008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":2086,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623560+0000","last_change":"2024-04-12T11:45:52.976139+0000","last_active":"2024-04-12T12:03:02.623560+0000","last_peered":"2024-04-12T12:03:02.623560+0000","last_clean":"2024-04-12T12:03:02.623560+0000","last_became_active":"2024-04-12T11:45:52.975803+0000","last_became_peered":"2024-04-12T11:45:52.975803+0000","last_unstale":"2024-04-12T12:03:02.623560+0000","last_undegraded":"2024-04-12T12:03:02.623560+0000","last_fullsized":"2024-04-12T12:03:02.623560+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:46.618143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":2067,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.625034+0000","last_change":"2024-04-12T12:00:35.473079+0000","last_active":"2024-04-12T12:03:02.625034+0000","last_peered":"2024-04-12T12:03:02.625034+0000","last_clean":"2024-04-12T12:03:02.625034+0000","last_became_active":"2024-04-12T12:00:35.472773+0000","last_became_peered":"2024-04-12T12:00:35.472773+0000","last_unstale":"2024-04-12T12:03:02.625034+0000","last_undegraded":"2024-04-12T12:03:02.625034+0000","last_fullsized":"2024-04-12T12:03:02.625034+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:20:42.192043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"249'161","reported_seq":824,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622715+0000","last_change":"2024-04-12T11:53:49.137544+0000","last_active":"2024-04-12T12:03:02.622715+0000","last_peered":"2024-04-12T12:03:02.622715+0000","last_clean":"2024-04-12T12:03:02.622715+0000","last_became_active":"2024-04-12T11:53:49.137196+0000","last_became_peered":"2024-04-12T11:53:49.137196+0000","last_unstale":"2024-04-12T12:03:02.622715+0000","last_undegraded":"2024-04-12T12:03:02.622715+0000","last_fullsized":"2024-04-12T12:03:02.622715+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:11.447056+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:11.447056+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:17:12.938789+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347528+0000","last_change":"2024-04-12T12:01:09.230000+0000","last_active":"2024-04-12T12:03:02.347528+0000","last_peered":"2024-04-12T12:03:02.347528+0000","last_clean":"2024-04-12T12:03:02.347528+0000","last_became_active":"2024-04-12T12:01:09.229670+0000","last_became_peered":"2024-04-12T12:01:09.229670+0000","last_unstale":"2024-04-12T12:03:02.347528+0000","last_undegraded":"2024-04-12T12:03:02.347528+0000","last_fullsized":"2024-04-12T12:03:02.347528+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:12.886844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346864+0000","last_change":"2024-04-12T12:00:35.477779+0000","last_active":"2024-04-12T12:03:02.346864+0000","last_peered":"2024-04-12T12:03:02.346864+0000","last_clean":"2024-04-12T12:03:02.346864+0000","last_became_active":"2024-04-12T12:00:35.477462+0000","last_became_peered":"2024-04-12T12:00:35.477462+0000","last_unstale":"2024-04-12T12:03:02.346864+0000","last_undegraded":"2024-04-12T12:03:02.346864+0000","last_fullsized":"2024-04-12T12:03:02.346864+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:51:17.465133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"249'161","reported_seq":631,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.095026+0000","last_change":"2024-04-12T12:01:08.203484+0000","last_active":"2024-04-12T12:02:58.095026+0000","last_peered":"2024-04-12T12:02:58.095026+0000","last_clean":"2024-04-12T12:02:58.095026+0000","last_became_active":"2024-04-12T12:01:08.203128+0000","last_became_peered":"2024-04-12T12:01:08.203128+0000","last_unstale":"2024-04-12T12:02:58.095026+0000","last_undegraded":"2024-04-12T12:02:58.095026+0000","last_fullsized":"2024-04-12T12:02:58.095026+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:37.607274+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:37.607274+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:03:30.798799+0000","scrub_duration":0.0037173240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346966+0000","last_change":"2024-04-12T12:00:35.473414+0000","last_active":"2024-04-12T12:03:02.346966+0000","last_peered":"2024-04-12T12:03:02.346966+0000","last_clean":"2024-04-12T12:03:02.346966+0000","last_became_active":"2024-04-12T12:00:35.472846+0000","last_became_peered":"2024-04-12T12:00:35.472846+0000","last_unstale":"2024-04-12T12:03:02.346966+0000","last_undegraded":"2024-04-12T12:03:02.346966+0000","last_fullsized":"2024-04-12T12:03:02.346966+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:59:19.941070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1357,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622987+0000","last_change":"2024-04-12T12:01:08.211265+0000","last_active":"2024-04-12T12:03:02.622987+0000","last_peered":"2024-04-12T12:03:02.622987+0000","last_clean":"2024-04-12T12:03:02.622987+0000","last_became_active":"2024-04-12T12:01:08.210898+0000","last_became_peered":"2024-04-12T12:01:08.210898+0000","last_unstale":"2024-04-12T12:03:02.622987+0000","last_undegraded":"2024-04-12T12:03:02.622987+0000","last_fullsized":"2024-04-12T12:03:02.622987+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:08:47.927220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347008+0000","last_change":"2024-04-12T12:00:35.477414+0000","last_active":"2024-04-12T12:03:02.347008+0000","last_peered":"2024-04-12T12:03:02.347008+0000","last_clean":"2024-04-12T12:03:02.347008+0000","last_became_active":"2024-04-12T12:00:35.477035+0000","last_became_peered":"2024-04-12T12:00:35.477035+0000","last_unstale":"2024-04-12T12:03:02.347008+0000","last_undegraded":"2024-04-12T12:03:02.347008+0000","last_fullsized":"2024-04-12T12:03:02.347008+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:25:45.630314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"249'161","reported_seq":828,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348009+0000","last_change":"2024-04-12T11:53:49.129889+0000","last_active":"2024-04-12T12:03:02.348009+0000","last_peered":"2024-04-12T12:03:02.348009+0000","last_clean":"2024-04-12T12:03:02.348009+0000","last_became_active":"2024-04-12T11:53:49.129462+0000","last_became_peered":"2024-04-12T11:53:49.129462+0000","last_unstale":"2024-04-12T12:03:02.348009+0000","last_undegraded":"2024-04-12T12:03:02.348009+0000","last_fullsized":"2024-04-12T12:03:02.348009+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:05.639497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:05.639497+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:16:18.801501+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346902+0000","last_change":"2024-04-12T12:00:35.475473+0000","last_active":"2024-04-12T12:03:02.346902+0000","last_peered":"2024-04-12T12:03:02.346902+0000","last_clean":"2024-04-12T12:03:02.346902+0000","last_became_active":"2024-04-12T12:00:35.475125+0000","last_became_peered":"2024-04-12T12:00:35.475125+0000","last_unstale":"2024-04-12T12:03:02.346902+0000","last_undegraded":"2024-04-12T12:03:02.346902+0000","last_fullsized":"2024-04-12T12:03:02.346902+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:59:05.176022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":641,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346949+0000","last_change":"2024-04-12T11:58:33.661396+0000","last_active":"2024-04-12T12:03:02.346949+0000","last_peered":"2024-04-12T12:03:02.346949+0000","last_clean":"2024-04-12T12:03:02.346949+0000","last_became_active":"2024-04-12T11:58:33.661076+0000","last_became_peered":"2024-04-12T11:58:33.661076+0000","last_unstale":"2024-04-12T12:03:02.346949+0000","last_undegraded":"2024-04-12T12:03:02.346949+0000","last_fullsized":"2024-04-12T12:03:02.346949+0000","mapping_epoch":1321,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:31:47.577826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346962+0000","last_change":"2024-04-12T12:01:09.225968+0000","last_active":"2024-04-12T12:03:02.346962+0000","last_peered":"2024-04-12T12:03:02.346962+0000","last_clean":"2024-04-12T12:03:02.346962+0000","last_became_active":"2024-04-12T12:01:09.225273+0000","last_became_peered":"2024-04-12T12:01:09.225273+0000","last_unstale":"2024-04-12T12:03:02.346962+0000","last_undegraded":"2024-04-12T12:03:02.346962+0000","last_fullsized":"2024-04-12T12:03:02.346962+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:39.143286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"249'161","reported_seq":689,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346678+0000","last_change":"2024-04-12T11:59:36.979504+0000","last_active":"2024-04-12T12:03:02.346678+0000","last_peered":"2024-04-12T12:03:02.346678+0000","last_clean":"2024-04-12T12:03:02.346678+0000","last_became_active":"2024-04-12T11:59:36.296167+0000","last_became_peered":"2024-04-12T11:59:36.296167+0000","last_unstale":"2024-04-12T12:03:02.346678+0000","last_undegraded":"2024-04-12T12:03:02.346678+0000","last_fullsized":"2024-04-12T12:03:02.346678+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:36.979380+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:36.979380+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:42:09.481447+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2147,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622745+0000","last_change":"2024-04-12T12:01:08.241455+0000","last_active":"2024-04-12T12:03:02.622745+0000","last_peered":"2024-04-12T12:03:02.622745+0000","last_clean":"2024-04-12T12:03:02.622745+0000","last_became_active":"2024-04-12T12:01:08.240087+0000","last_became_peered":"2024-04-12T12:01:08.240087+0000","last_unstale":"2024-04-12T12:03:02.622745+0000","last_undegraded":"2024-04-12T12:03:02.622745+0000","last_fullsized":"2024-04-12T12:03:02.622745+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:45:12.951549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347371+0000","last_change":"2024-04-12T12:00:35.471479+0000","last_active":"2024-04-12T12:03:02.347371+0000","last_peered":"2024-04-12T12:03:02.347371+0000","last_clean":"2024-04-12T12:03:02.347371+0000","last_became_active":"2024-04-12T12:00:35.471015+0000","last_became_peered":"2024-04-12T12:00:35.471015+0000","last_unstale":"2024-04-12T12:03:02.347371+0000","last_undegraded":"2024-04-12T12:03:02.347371+0000","last_fullsized":"2024-04-12T12:03:02.347371+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:55:50.471132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346782+0000","last_change":"2024-04-12T12:00:35.479622+0000","last_active":"2024-04-12T12:03:02.346782+0000","last_peered":"2024-04-12T12:03:02.346782+0000","last_clean":"2024-04-12T12:03:02.346782+0000","last_became_active":"2024-04-12T12:00:35.479249+0000","last_became_peered":"2024-04-12T12:00:35.479249+0000","last_unstale":"2024-04-12T12:03:02.346782+0000","last_undegraded":"2024-04-12T12:03:02.346782+0000","last_fullsized":"2024-04-12T12:03:02.346782+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:24:26.689379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1409,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622836+0000","last_change":"2024-04-12T12:01:08.223472+0000","last_active":"2024-04-12T12:03:02.622836+0000","last_peered":"2024-04-12T12:03:02.622836+0000","last_clean":"2024-04-12T12:03:02.622836+0000","last_became_active":"2024-04-12T12:01:08.222991+0000","last_became_peered":"2024-04-12T12:01:08.222991+0000","last_unstale":"2024-04-12T12:03:02.622836+0000","last_undegraded":"2024-04-12T12:03:02.622836+0000","last_fullsized":"2024-04-12T12:03:02.622836+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:28:37.101566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1696,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.438620+0000","last_change":"2024-04-12T11:53:06.603022+0000","last_active":"2024-04-12T12:02:55.438620+0000","last_peered":"2024-04-12T12:02:55.438620+0000","last_clean":"2024-04-12T12:02:55.438620+0000","last_became_active":"2024-04-12T11:49:07.217921+0000","last_became_peered":"2024-04-12T11:49:07.217921+0000","last_unstale":"2024-04-12T12:02:55.438620+0000","last_undegraded":"2024-04-12T12:02:55.438620+0000","last_fullsized":"2024-04-12T12:02:55.438620+0000","mapping_epoch":1210,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":0,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:06.602918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:06.602918+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:40.776170+0000","scrub_duration":0.001429877,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":594,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094740+0000","last_change":"2024-04-12T12:01:08.220206+0000","last_active":"2024-04-12T12:02:58.094740+0000","last_peered":"2024-04-12T12:02:58.094740+0000","last_clean":"2024-04-12T12:02:58.094740+0000","last_became_active":"2024-04-12T12:01:08.219103+0000","last_became_peered":"2024-04-12T12:01:08.219103+0000","last_unstale":"2024-04-12T12:02:58.094740+0000","last_undegraded":"2024-04-12T12:02:58.094740+0000","last_fullsized":"2024-04-12T12:02:58.094740+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:04:53.296805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1336,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.095009+0000","last_change":"2024-04-12T11:45:53.016616+0000","last_active":"2024-04-12T12:02:58.095009+0000","last_peered":"2024-04-12T12:02:58.095009+0000","last_clean":"2024-04-12T12:02:58.095009+0000","last_became_active":"2024-04-12T11:45:53.016292+0000","last_became_peered":"2024-04-12T11:45:53.016292+0000","last_unstale":"2024-04-12T12:02:58.095009+0000","last_undegraded":"2024-04-12T12:02:58.095009+0000","last_fullsized":"2024-04-12T12:02:58.095009+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:53:29.382293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"249'161","reported_seq":736,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347232+0000","last_change":"2024-04-12T12:02:44.218750+0000","last_active":"2024-04-12T12:03:02.347232+0000","last_peered":"2024-04-12T12:03:02.347232+0000","last_clean":"2024-04-12T12:03:02.347232+0000","last_became_active":"2024-04-12T12:02:44.218240+0000","last_became_peered":"2024-04-12T12:02:44.218240+0000","last_unstale":"2024-04-12T12:03:02.347232+0000","last_undegraded":"2024-04-12T12:03:02.347232+0000","last_fullsized":"2024-04-12T12:03:02.347232+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:57:49.744017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:57:49.744017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:07.839300+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347286+0000","last_change":"2024-04-12T12:00:35.475887+0000","last_active":"2024-04-12T12:03:02.347286+0000","last_peered":"2024-04-12T12:03:02.347286+0000","last_clean":"2024-04-12T12:03:02.347286+0000","last_became_active":"2024-04-12T12:00:35.475531+0000","last_became_peered":"2024-04-12T12:00:35.475531+0000","last_unstale":"2024-04-12T12:03:02.347286+0000","last_undegraded":"2024-04-12T12:03:02.347286+0000","last_fullsized":"2024-04-12T12:03:02.347286+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:06:33.171131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2184,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348204+0000","last_change":"2024-04-12T12:01:08.241097+0000","last_active":"2024-04-12T12:03:02.348204+0000","last_peered":"2024-04-12T12:03:02.348204+0000","last_clean":"2024-04-12T12:03:02.348204+0000","last_became_active":"2024-04-12T12:01:08.240402+0000","last_became_peered":"2024-04-12T12:01:08.240402+0000","last_unstale":"2024-04-12T12:03:02.348204+0000","last_undegraded":"2024-04-12T12:03:02.348204+0000","last_fullsized":"2024-04-12T12:03:02.348204+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:30:15.712301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2163,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348066+0000","last_change":"2024-04-12T12:00:35.480505+0000","last_active":"2024-04-12T12:03:02.348066+0000","last_peered":"2024-04-12T12:03:02.348066+0000","last_clean":"2024-04-12T12:03:02.348066+0000","last_became_active":"2024-04-12T12:00:35.480136+0000","last_became_peered":"2024-04-12T12:00:35.480136+0000","last_unstale":"2024-04-12T12:03:02.348066+0000","last_undegraded":"2024-04-12T12:03:02.348066+0000","last_fullsized":"2024-04-12T12:03:02.348066+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:12:05.320400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"249'161","reported_seq":705,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094937+0000","last_change":"2024-04-12T12:01:08.209702+0000","last_active":"2024-04-12T12:02:58.094937+0000","last_peered":"2024-04-12T12:02:58.094937+0000","last_clean":"2024-04-12T12:02:58.094937+0000","last_became_active":"2024-04-12T12:01:08.209465+0000","last_became_peered":"2024-04-12T12:01:08.209465+0000","last_unstale":"2024-04-12T12:02:58.094937+0000","last_undegraded":"2024-04-12T12:02:58.094937+0000","last_fullsized":"2024-04-12T12:02:58.094937+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:05.130465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:05.130465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:53:20.786613+0000","scrub_duration":0.001397695,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":169,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.439064+0000","last_change":"2024-04-12T11:45:53.005585+0000","last_active":"2024-04-12T12:02:55.439064+0000","last_peered":"2024-04-12T12:02:55.439064+0000","last_clean":"2024-04-12T12:02:55.439064+0000","last_became_active":"2024-04-12T11:45:52.987880+0000","last_became_peered":"2024-04-12T11:45:52.987880+0000","last_unstale":"2024-04-12T12:02:55.439064+0000","last_undegraded":"2024-04-12T12:02:55.439064+0000","last_fullsized":"2024-04-12T12:02:55.439064+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:09.819861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347097+0000","last_change":"2024-04-12T12:00:35.473178+0000","last_active":"2024-04-12T12:03:02.347097+0000","last_peered":"2024-04-12T12:03:02.347097+0000","last_clean":"2024-04-12T12:03:02.347097+0000","last_became_active":"2024-04-12T12:00:35.472877+0000","last_became_peered":"2024-04-12T12:00:35.472877+0000","last_unstale":"2024-04-12T12:03:02.347097+0000","last_undegraded":"2024-04-12T12:03:02.347097+0000","last_fullsized":"2024-04-12T12:03:02.347097+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:03:28.998314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2167,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347405+0000","last_change":"2024-04-12T12:00:35.475473+0000","last_active":"2024-04-12T12:03:02.347405+0000","last_peered":"2024-04-12T12:03:02.347405+0000","last_clean":"2024-04-12T12:03:02.347405+0000","last_became_active":"2024-04-12T12:00:35.475120+0000","last_became_peered":"2024-04-12T12:00:35.475120+0000","last_unstale":"2024-04-12T12:03:02.347405+0000","last_undegraded":"2024-04-12T12:03:02.347405+0000","last_fullsized":"2024-04-12T12:03:02.347405+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:14:52.218068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"249'161","reported_seq":832,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.624761+0000","last_change":"2024-04-12T12:02:44.216787+0000","last_active":"2024-04-12T12:03:02.624761+0000","last_peered":"2024-04-12T12:03:02.624761+0000","last_clean":"2024-04-12T12:03:02.624761+0000","last_became_active":"2024-04-12T12:02:44.216350+0000","last_became_peered":"2024-04-12T12:02:44.216350+0000","last_unstale":"2024-04-12T12:03:02.624761+0000","last_undegraded":"2024-04-12T12:03:02.624761+0000","last_fullsized":"2024-04-12T12:03:02.624761+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:10.604457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:10.604457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:01:05.908018+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":135,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348410+0000","last_change":"2024-04-12T12:01:08.213894+0000","last_active":"2024-04-12T12:03:02.348410+0000","last_peered":"2024-04-12T12:03:02.348410+0000","last_clean":"2024-04-12T12:03:02.348410+0000","last_became_active":"2024-04-12T12:01:08.213309+0000","last_became_peered":"2024-04-12T12:01:08.213309+0000","last_unstale":"2024-04-12T12:03:02.348410+0000","last_undegraded":"2024-04-12T12:03:02.348410+0000","last_fullsized":"2024-04-12T12:03:02.348410+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:35:42.203466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347604+0000","last_change":"2024-04-12T12:00:35.477662+0000","last_active":"2024-04-12T12:03:02.347604+0000","last_peered":"2024-04-12T12:03:02.347604+0000","last_clean":"2024-04-12T12:03:02.347604+0000","last_became_active":"2024-04-12T12:00:35.477235+0000","last_became_peered":"2024-04-12T12:00:35.477235+0000","last_unstale":"2024-04-12T12:03:02.347604+0000","last_undegraded":"2024-04-12T12:03:02.347604+0000","last_fullsized":"2024-04-12T12:03:02.347604+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:18:34.713724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":115,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.438403+0000","last_change":"2024-04-12T12:01:08.230577+0000","last_active":"2024-04-12T12:02:55.438403+0000","last_peered":"2024-04-12T12:02:55.438403+0000","last_clean":"2024-04-12T12:02:55.438403+0000","last_became_active":"2024-04-12T12:01:08.230190+0000","last_became_peered":"2024-04-12T12:01:08.230190+0000","last_unstale":"2024-04-12T12:02:55.438403+0000","last_undegraded":"2024-04-12T12:02:55.438403+0000","last_fullsized":"2024-04-12T12:02:55.438403+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:59:11.839551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"249'161","reported_seq":1655,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-04-12T12:02:55.438857+0000","last_change":"2024-04-12T12:01:08.230547+0000","last_active":"2024-04-12T12:02:55.438857+0000","last_peered":"2024-04-12T12:02:55.438857+0000","last_clean":"2024-04-12T12:02:55.438857+0000","last_became_active":"2024-04-12T12:01:08.230123+0000","last_became_peered":"2024-04-12T12:01:08.230123+0000","last_unstale":"2024-04-12T12:02:55.438857+0000","last_undegraded":"2024-04-12T12:02:55.438857+0000","last_fullsized":"2024-04-12T12:02:55.438857+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:17.062855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:17.062855+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:06:49.991085+0000","scrub_duration":0.0078090989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1369,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.095075+0000","last_change":"2024-04-12T11:45:52.979745+0000","last_active":"2024-04-12T12:02:58.095075+0000","last_peered":"2024-04-12T12:02:58.095075+0000","last_clean":"2024-04-12T12:02:58.095075+0000","last_became_active":"2024-04-12T11:45:52.978936+0000","last_became_peered":"2024-04-12T11:45:52.978936+0000","last_unstale":"2024-04-12T12:02:58.095075+0000","last_undegraded":"2024-04-12T12:02:58.095075+0000","last_fullsized":"2024-04-12T12:02:58.095075+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:22.710430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":2174,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347784+0000","last_change":"2024-04-12T12:00:35.485498+0000","last_active":"2024-04-12T12:03:02.347784+0000","last_peered":"2024-04-12T12:03:02.347784+0000","last_clean":"2024-04-12T12:03:02.347784+0000","last_became_active":"2024-04-12T12:00:35.475885+0000","last_became_peered":"2024-04-12T12:00:35.475885+0000","last_unstale":"2024-04-12T12:03:02.347784+0000","last_undegraded":"2024-04-12T12:03:02.347784+0000","last_fullsized":"2024-04-12T12:03:02.347784+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:42:57.450495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1938,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094486+0000","last_change":"2024-04-12T11:58:33.661378+0000","last_active":"2024-04-12T12:02:58.094486+0000","last_peered":"2024-04-12T12:02:58.094486+0000","last_clean":"2024-04-12T12:02:58.094486+0000","last_became_active":"2024-04-12T11:58:33.661014+0000","last_became_peered":"2024-04-12T11:58:33.661014+0000","last_unstale":"2024-04-12T12:02:58.094486+0000","last_undegraded":"2024-04-12T12:02:58.094486+0000","last_fullsized":"2024-04-12T12:02:58.094486+0000","mapping_epoch":1321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:03:05.992776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":2154,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347305+0000","last_change":"2024-04-12T11:45:52.979689+0000","last_active":"2024-04-12T12:03:02.347305+0000","last_peered":"2024-04-12T12:03:02.347305+0000","last_clean":"2024-04-12T12:03:02.347305+0000","last_became_active":"2024-04-12T11:45:52.979286+0000","last_became_peered":"2024-04-12T11:45:52.979286+0000","last_unstale":"2024-04-12T12:03:02.347305+0000","last_undegraded":"2024-04-12T12:03:02.347305+0000","last_fullsized":"2024-04-12T12:03:02.347305+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:10.453472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":41,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623009+0000","last_change":"2024-04-12T12:02:08.214192+0000","last_active":"2024-04-12T12:03:02.623009+0000","last_peered":"2024-04-12T12:03:02.623009+0000","last_clean":"2024-04-12T12:03:02.623009+0000","last_became_active":"2024-04-12T12:02:08.213849+0000","last_became_peered":"2024-04-12T12:02:08.213849+0000","last_unstale":"2024-04-12T12:03:02.623009+0000","last_undegraded":"2024-04-12T12:03:02.623009+0000","last_fullsized":"2024-04-12T12:03:02.623009+0000","mapping_epoch":1347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:07:59.571577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":249,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.624260+0000","last_change":"2024-04-12T12:00:35.483510+0000","last_active":"2024-04-12T12:03:02.624260+0000","last_peered":"2024-04-12T12:03:02.624260+0000","last_clean":"2024-04-12T12:03:02.624260+0000","last_became_active":"2024-04-12T12:00:35.472254+0000","last_became_peered":"2024-04-12T12:00:35.472254+0000","last_unstale":"2024-04-12T12:03:02.624260+0000","last_undegraded":"2024-04-12T12:03:02.624260+0000","last_fullsized":"2024-04-12T12:03:02.624260+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:28.883365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":73,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347136+0000","last_change":"2024-04-12T12:01:09.587265+0000","last_active":"2024-04-12T12:03:02.347136+0000","last_peered":"2024-04-12T12:03:02.347136+0000","last_clean":"2024-04-12T12:03:02.347136+0000","last_became_active":"2024-04-12T12:01:09.586910+0000","last_became_peered":"2024-04-12T12:01:09.586910+0000","last_unstale":"2024-04-12T12:03:02.347136+0000","last_undegraded":"2024-04-12T12:03:02.347136+0000","last_fullsized":"2024-04-12T12:03:02.347136+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:23.708275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":85,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.095019+0000","last_change":"2024-04-12T12:00:35.481562+0000","last_active":"2024-04-12T12:02:58.095019+0000","last_peered":"2024-04-12T12:02:58.095019+0000","last_clean":"2024-04-12T12:02:58.095019+0000","last_became_active":"2024-04-12T12:00:35.475238+0000","last_became_peered":"2024-04-12T12:00:35.475238+0000","last_unstale":"2024-04-12T12:02:58.095019+0000","last_undegraded":"2024-04-12T12:02:58.095019+0000","last_fullsized":"2024-04-12T12:02:58.095019+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.893745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1932,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346565+0000","last_change":"2024-04-12T12:01:08.225762+0000","last_active":"2024-04-12T12:03:02.346565+0000","last_peered":"2024-04-12T12:03:02.346565+0000","last_clean":"2024-04-12T12:03:02.346565+0000","last_became_active":"2024-04-12T12:01:08.225072+0000","last_became_peered":"2024-04-12T12:01:08.225072+0000","last_unstale":"2024-04-12T12:03:02.346565+0000","last_undegraded":"2024-04-12T12:03:02.346565+0000","last_fullsized":"2024-04-12T12:03:02.346565+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:04.300351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":217,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623179+0000","last_change":"2024-04-12T12:02:42.080490+0000","last_active":"2024-04-12T12:03:02.623179+0000","last_peered":"2024-04-12T12:03:02.623179+0000","last_clean":"2024-04-12T12:03:02.623179+0000","last_became_active":"2024-04-12T12:02:42.080074+0000","last_became_peered":"2024-04-12T12:02:42.080074+0000","last_unstale":"2024-04-12T12:03:02.623179+0000","last_undegraded":"2024-04-12T12:03:02.623179+0000","last_fullsized":"2024-04-12T12:03:02.623179+0000","mapping_epoch":1354,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:42:31.461796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":734,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622789+0000","last_change":"2024-04-12T12:01:08.239424+0000","last_active":"2024-04-12T12:03:02.622789+0000","last_peered":"2024-04-12T12:03:02.622789+0000","last_clean":"2024-04-12T12:03:02.622789+0000","last_became_active":"2024-04-12T12:01:08.239070+0000","last_became_peered":"2024-04-12T12:01:08.239070+0000","last_unstale":"2024-04-12T12:03:02.622789+0000","last_undegraded":"2024-04-12T12:03:02.622789+0000","last_fullsized":"2024-04-12T12:03:02.622789+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:26.732811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":2211,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348315+0000","last_change":"2024-04-12T12:02:08.217720+0000","last_active":"2024-04-12T12:03:02.348315+0000","last_peered":"2024-04-12T12:03:02.348315+0000","last_clean":"2024-04-12T12:03:02.348315+0000","last_became_active":"2024-04-12T12:02:08.214163+0000","last_became_peered":"2024-04-12T12:02:08.214163+0000","last_unstale":"2024-04-12T12:03:02.348315+0000","last_undegraded":"2024-04-12T12:03:02.348315+0000","last_fullsized":"2024-04-12T12:03:02.348315+0000","mapping_epoch":1347,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:56:25.681924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2910,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":2066,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622968+0000","last_change":"2024-04-12T12:00:35.472529+0000","last_active":"2024-04-12T12:03:02.622968+0000","last_peered":"2024-04-12T12:03:02.622968+0000","last_clean":"2024-04-12T12:03:02.622968+0000","last_became_active":"2024-04-12T12:00:35.472057+0000","last_became_peered":"2024-04-12T12:00:35.472057+0000","last_unstale":"2024-04-12T12:03:02.622968+0000","last_undegraded":"2024-04-12T12:03:02.622968+0000","last_fullsized":"2024-04-12T12:03:02.622968+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:07:59.083703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":12748,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35155,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234608,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972584,"kb_used_data":8796,"kb_used_omap":1,"kb_used_meta":963774,"kb_avail":92772568,"statfs":{"total":95995035648,"available":94999109632,"internally_reserved":0,"allocated":9007104,"data_stored":5890989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":986905035},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496245,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":798116,"kb_used_data":9048,"kb_used_omap":1,"kb_used_meta":789054,"kb_avail":92947036,"statfs":{"total":95995035648,"available":95177764864,"internally_reserved":0,"allocated":9265152,"data_stored":6154113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":807991752},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536555,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770636,"kb_used_data":8448,"kb_used_omap":30,"kb_used_meta":762145,"kb_avail":92974516,"statfs":{"total":95995035648,"available":95205904384,"internally_reserved":0,"allocated":8650752,"data_stored":5558349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":780436802},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190639,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1131360,"kb_used_data":8468,"kb_used_omap":43,"kb_used_meta":1122836,"kb_avail":92613792,"statfs":{"total":95995035648,"available":94836523008,"internally_reserved":0,"allocated":8671232,"data_stored":5566942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":1149784107},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849767,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1027812,"kb_used_data":8728,"kb_used_omap":36,"kb_used_meta":1019035,"kb_avail":92717340,"statfs":{"total":95995035648,"available":94942556160,"internally_reserved":0,"allocated":8937472,"data_stored":5829642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":1043492020},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1342,"seq":5763846111256,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":126616,"kb_used_data":9036,"kb_used_omap":32,"kb_used_meta":117535,"kb_avail":93618536,"statfs":{"total":95995035648,"available":95865380864,"internally_reserved":0,"allocated":9252864,"data_stored":6148877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33145,"internal_metadata":120356487},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018332,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951344,"kb_used_data":8484,"kb_used_omap":42,"kb_used_meta":942805,"kb_avail":92793808,"statfs":{"total":95995035648,"available":95020859392,"internally_reserved":0,"allocated":8687616,"data_stored":5571240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":965433144},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":5506148073580,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10321,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4818,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T12:03:08.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T12:03:08.867 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:09.140 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-04-12T12:03:09.355 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:09.424 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:09 smithi087 ceph-mon[89014]: from='client.46025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:03:09.424 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:09 smithi087 ceph-mon[89014]: pgmap v2491: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:09.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:09 smithi029 ceph-mon[97787]: from='client.46025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:03:09.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:09 smithi029 ceph-mon[97787]: pgmap v2491: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:09 smithi029 ceph-mon[93650]: from='client.46025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:03:09.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:09 smithi029 ceph-mon[93650]: pgmap v2491: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:10.896 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:10.896 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T12:03:10.896 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T12:03:10.896 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T12:03:10.896 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:10.913 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T12:03:10.914 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T12:03:11.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:11 smithi029 ceph-mon[93650]: pgmap v2492: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:11.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:11 smithi029 ceph-mon[97787]: pgmap v2492: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:11.601 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:11 smithi087 ceph-mon[89014]: pgmap v2492: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:11.625 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-04-12T12:03:11.655 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2492,"stamp":"2024-04-12T12:03:10.289291+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":800812,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1826,"ondisk_log_size":1826,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5778516,"kb_used_data":61056,"kb_used_omap":189,"kb_used_meta":5717186,"kb_avail":650437548,"statfs":{"total":671965249536,"available":666048049152,"internally_reserved":0,"allocated":62521344,"data_stored":40752108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":193677,"internal_metadata":5854399347},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002333"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":2099,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623411+0000","last_change":"2024-04-12T12:00:35.472642+0000","last_active":"2024-04-12T12:03:02.623411+0000","last_peered":"2024-04-12T12:03:02.623411+0000","last_clean":"2024-04-12T12:03:02.623411+0000","last_became_active":"2024-04-12T12:00:35.472258+0000","last_became_peered":"2024-04-12T12:00:35.472258+0000","last_unstale":"2024-04-12T12:03:02.623411+0000","last_undegraded":"2024-04-12T12:03:02.623411+0000","last_fullsized":"2024-04-12T12:03:02.623411+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:41:16.991252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346611+0000","last_change":"2024-04-12T12:01:09.588317+0000","last_active":"2024-04-12T12:03:02.346611+0000","last_peered":"2024-04-12T12:03:02.346611+0000","last_clean":"2024-04-12T12:03:02.346611+0000","last_became_active":"2024-04-12T12:01:09.587996+0000","last_became_peered":"2024-04-12T12:01:09.587996+0000","last_unstale":"2024-04-12T12:03:02.346611+0000","last_undegraded":"2024-04-12T12:03:02.346611+0000","last_fullsized":"2024-04-12T12:03:02.346611+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:48:36.981875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346584+0000","last_change":"2024-04-12T12:01:09.587349+0000","last_active":"2024-04-12T12:03:02.346584+0000","last_peered":"2024-04-12T12:03:02.346584+0000","last_clean":"2024-04-12T12:03:02.346584+0000","last_became_active":"2024-04-12T12:01:09.587019+0000","last_became_peered":"2024-04-12T12:01:09.587019+0000","last_unstale":"2024-04-12T12:03:02.346584+0000","last_undegraded":"2024-04-12T12:03:02.346584+0000","last_fullsized":"2024-04-12T12:03:02.346584+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:19:17.511244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2201,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347895+0000","last_change":"2024-04-12T12:01:08.211946+0000","last_active":"2024-04-12T12:03:02.347895+0000","last_peered":"2024-04-12T12:03:02.347895+0000","last_clean":"2024-04-12T12:03:02.347895+0000","last_became_active":"2024-04-12T12:01:08.209590+0000","last_became_peered":"2024-04-12T12:01:08.209590+0000","last_unstale":"2024-04-12T12:03:02.347895+0000","last_undegraded":"2024-04-12T12:03:02.347895+0000","last_fullsized":"2024-04-12T12:03:02.347895+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:08.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":749,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347715+0000","last_change":"2024-04-12T12:01:09.236804+0000","last_active":"2024-04-12T12:03:02.347715+0000","last_peered":"2024-04-12T12:03:02.347715+0000","last_clean":"2024-04-12T12:03:02.347715+0000","last_became_active":"2024-04-12T12:01:09.236485+0000","last_became_peered":"2024-04-12T12:01:09.236485+0000","last_unstale":"2024-04-12T12:03:02.347715+0000","last_undegraded":"2024-04-12T12:03:02.347715+0000","last_fullsized":"2024-04-12T12:03:02.347715+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:23.077697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1358,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.095039+0000","last_change":"2024-04-12T12:00:35.474896+0000","last_active":"2024-04-12T12:02:58.095039+0000","last_peered":"2024-04-12T12:02:58.095039+0000","last_clean":"2024-04-12T12:02:58.095039+0000","last_became_active":"2024-04-12T12:00:35.474579+0000","last_became_peered":"2024-04-12T12:00:35.474579+0000","last_unstale":"2024-04-12T12:02:58.095039+0000","last_undegraded":"2024-04-12T12:02:58.095039+0000","last_fullsized":"2024-04-12T12:02:58.095039+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:51:32.229812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2202,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348353+0000","last_change":"2024-04-12T12:02:42.081163+0000","last_active":"2024-04-12T12:03:02.348353+0000","last_peered":"2024-04-12T12:03:02.348353+0000","last_clean":"2024-04-12T12:03:02.348353+0000","last_became_active":"2024-04-12T12:02:42.080873+0000","last_became_peered":"2024-04-12T12:02:42.080873+0000","last_unstale":"2024-04-12T12:03:02.348353+0000","last_undegraded":"2024-04-12T12:03:02.348353+0000","last_fullsized":"2024-04-12T12:03:02.348353+0000","mapping_epoch":1354,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:22:43.949432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":192,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623415+0000","last_change":"2024-04-12T12:01:09.514195+0000","last_active":"2024-04-12T12:03:02.623415+0000","last_peered":"2024-04-12T12:03:02.623415+0000","last_clean":"2024-04-12T12:03:02.623415+0000","last_became_active":"2024-04-12T12:01:09.513505+0000","last_became_peered":"2024-04-12T12:01:09.513505+0000","last_unstale":"2024-04-12T12:03:02.623415+0000","last_undegraded":"2024-04-12T12:03:02.623415+0000","last_fullsized":"2024-04-12T12:03:02.623415+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:43.907048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347552+0000","last_change":"2024-04-12T12:01:09.224595+0000","last_active":"2024-04-12T12:03:02.347552+0000","last_peered":"2024-04-12T12:03:02.347552+0000","last_clean":"2024-04-12T12:03:02.347552+0000","last_became_active":"2024-04-12T12:01:09.224210+0000","last_became_peered":"2024-04-12T12:01:09.224210+0000","last_unstale":"2024-04-12T12:03:02.347552+0000","last_undegraded":"2024-04-12T12:03:02.347552+0000","last_fullsized":"2024-04-12T12:03:02.347552+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:52:22.161025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":575,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094490+0000","last_change":"2024-04-12T12:01:08.236429+0000","last_active":"2024-04-12T12:02:58.094490+0000","last_peered":"2024-04-12T12:02:58.094490+0000","last_clean":"2024-04-12T12:02:58.094490+0000","last_became_active":"2024-04-12T12:01:08.236170+0000","last_became_peered":"2024-04-12T12:01:08.236170+0000","last_unstale":"2024-04-12T12:02:58.094490+0000","last_undegraded":"2024-04-12T12:02:58.094490+0000","last_fullsized":"2024-04-12T12:02:58.094490+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:54:28.854893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":220,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965846+0000","last_change":"2024-04-12T12:01:08.204030+0000","last_active":"2024-04-12T12:02:58.965846+0000","last_peered":"2024-04-12T12:02:58.965846+0000","last_clean":"2024-04-12T12:02:58.965846+0000","last_became_active":"2024-04-12T12:01:08.203790+0000","last_became_peered":"2024-04-12T12:01:08.203790+0000","last_unstale":"2024-04-12T12:02:58.965846+0000","last_undegraded":"2024-04-12T12:02:58.965846+0000","last_fullsized":"2024-04-12T12:02:58.965846+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:18:55.719738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":76,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347096+0000","last_change":"2024-04-12T12:02:42.081270+0000","last_active":"2024-04-12T12:03:02.347096+0000","last_peered":"2024-04-12T12:03:02.347096+0000","last_clean":"2024-04-12T12:03:02.347096+0000","last_became_active":"2024-04-12T12:02:42.080832+0000","last_became_peered":"2024-04-12T12:02:42.080832+0000","last_unstale":"2024-04-12T12:03:02.347096+0000","last_undegraded":"2024-04-12T12:03:02.347096+0000","last_fullsized":"2024-04-12T12:03:02.347096+0000","mapping_epoch":1354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:06.752668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":2092,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.624081+0000","last_change":"2024-04-12T11:45:53.024626+0000","last_active":"2024-04-12T12:03:02.624081+0000","last_peered":"2024-04-12T12:03:02.624081+0000","last_clean":"2024-04-12T12:03:02.624081+0000","last_became_active":"2024-04-12T11:45:52.976276+0000","last_became_peered":"2024-04-12T11:45:52.976276+0000","last_unstale":"2024-04-12T12:03:02.624081+0000","last_undegraded":"2024-04-12T12:03:02.624081+0000","last_fullsized":"2024-04-12T12:03:02.624081+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:49.929210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":2063,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.624285+0000","last_change":"2024-04-12T12:00:35.473701+0000","last_active":"2024-04-12T12:03:02.624285+0000","last_peered":"2024-04-12T12:03:02.624285+0000","last_clean":"2024-04-12T12:03:02.624285+0000","last_became_active":"2024-04-12T12:00:35.473318+0000","last_became_peered":"2024-04-12T12:00:35.473318+0000","last_unstale":"2024-04-12T12:03:02.624285+0000","last_undegraded":"2024-04-12T12:03:02.624285+0000","last_fullsized":"2024-04-12T12:03:02.624285+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:45:46.936638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":175,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965607+0000","last_change":"2024-04-12T11:45:53.035604+0000","last_active":"2024-04-12T12:02:58.965607+0000","last_peered":"2024-04-12T12:02:58.965607+0000","last_clean":"2024-04-12T12:02:58.965607+0000","last_became_active":"2024-04-12T11:45:52.984722+0000","last_became_peered":"2024-04-12T11:45:52.984722+0000","last_unstale":"2024-04-12T12:02:58.965607+0000","last_undegraded":"2024-04-12T12:02:58.965607+0000","last_fullsized":"2024-04-12T12:02:58.965607+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:30.674395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346861+0000","last_change":"2024-04-12T12:01:09.225722+0000","last_active":"2024-04-12T12:03:02.346861+0000","last_peered":"2024-04-12T12:03:02.346861+0000","last_clean":"2024-04-12T12:03:02.346861+0000","last_became_active":"2024-04-12T12:01:09.224774+0000","last_became_peered":"2024-04-12T12:01:09.224774+0000","last_unstale":"2024-04-12T12:03:02.346861+0000","last_undegraded":"2024-04-12T12:03:02.346861+0000","last_fullsized":"2024-04-12T12:03:02.346861+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:19:05.420649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2184,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346839+0000","last_change":"2024-04-12T12:01:08.222480+0000","last_active":"2024-04-12T12:03:02.346839+0000","last_peered":"2024-04-12T12:03:02.346839+0000","last_clean":"2024-04-12T12:03:02.346839+0000","last_became_active":"2024-04-12T12:01:08.222018+0000","last_became_peered":"2024-04-12T12:01:08.222018+0000","last_unstale":"2024-04-12T12:03:02.346839+0000","last_undegraded":"2024-04-12T12:03:02.346839+0000","last_fullsized":"2024-04-12T12:03:02.346839+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:12:10.882773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1359,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965618+0000","last_change":"2024-04-12T11:57:25.432612+0000","last_active":"2024-04-12T12:02:58.965618+0000","last_peered":"2024-04-12T12:02:58.965618+0000","last_clean":"2024-04-12T12:02:58.965618+0000","last_became_active":"2024-04-12T11:57:25.432182+0000","last_became_peered":"2024-04-12T11:57:25.432182+0000","last_unstale":"2024-04-12T12:02:58.965618+0000","last_undegraded":"2024-04-12T12:02:58.965618+0000","last_fullsized":"2024-04-12T12:02:58.965618+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:00:10.448754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1254,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622566+0000","last_change":"2024-04-12T11:59:36.297225+0000","last_active":"2024-04-12T12:03:02.622566+0000","last_peered":"2024-04-12T12:03:02.622566+0000","last_clean":"2024-04-12T12:03:02.622566+0000","last_became_active":"2024-04-12T11:59:36.296877+0000","last_became_peered":"2024-04-12T11:59:36.296877+0000","last_unstale":"2024-04-12T12:03:02.622566+0000","last_undegraded":"2024-04-12T12:03:02.622566+0000","last_fullsized":"2024-04-12T12:03:02.622566+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:26:53.114573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2185,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348016+0000","last_change":"2024-04-12T12:01:08.213742+0000","last_active":"2024-04-12T12:03:02.348016+0000","last_peered":"2024-04-12T12:03:02.348016+0000","last_clean":"2024-04-12T12:03:02.348016+0000","last_became_active":"2024-04-12T12:01:08.211529+0000","last_became_peered":"2024-04-12T12:01:08.211529+0000","last_unstale":"2024-04-12T12:03:02.348016+0000","last_undegraded":"2024-04-12T12:03:02.348016+0000","last_fullsized":"2024-04-12T12:03:02.348016+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:18.227656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1365,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094488+0000","last_change":"2024-04-12T12:01:08.218173+0000","last_active":"2024-04-12T12:02:58.094488+0000","last_peered":"2024-04-12T12:02:58.094488+0000","last_clean":"2024-04-12T12:02:58.094488+0000","last_became_active":"2024-04-12T12:01:08.217894+0000","last_became_peered":"2024-04-12T12:01:08.217894+0000","last_unstale":"2024-04-12T12:02:58.094488+0000","last_undegraded":"2024-04-12T12:02:58.094488+0000","last_fullsized":"2024-04-12T12:02:58.094488+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:11:50.138367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":191,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623295+0000","last_change":"2024-04-12T12:01:09.226707+0000","last_active":"2024-04-12T12:03:02.623295+0000","last_peered":"2024-04-12T12:03:02.623295+0000","last_clean":"2024-04-12T12:03:02.623295+0000","last_became_active":"2024-04-12T12:01:09.226344+0000","last_became_peered":"2024-04-12T12:01:09.226344+0000","last_unstale":"2024-04-12T12:03:02.623295+0000","last_undegraded":"2024-04-12T12:03:02.623295+0000","last_fullsized":"2024-04-12T12:03:02.623295+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:50:21.461207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":173,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965807+0000","last_change":"2024-04-12T11:45:53.028640+0000","last_active":"2024-04-12T12:02:58.965807+0000","last_peered":"2024-04-12T12:02:58.965807+0000","last_clean":"2024-04-12T12:02:58.965807+0000","last_became_active":"2024-04-12T11:45:52.983099+0000","last_became_peered":"2024-04-12T11:45:52.983099+0000","last_unstale":"2024-04-12T12:02:58.965807+0000","last_undegraded":"2024-04-12T12:02:58.965807+0000","last_fullsized":"2024-04-12T12:02:58.965807+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:43:01.897014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1308,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965268+0000","last_change":"2024-04-12T12:02:42.081502+0000","last_active":"2024-04-12T12:02:58.965268+0000","last_peered":"2024-04-12T12:02:58.965268+0000","last_clean":"2024-04-12T12:02:58.965268+0000","last_became_active":"2024-04-12T12:02:42.081079+0000","last_became_peered":"2024-04-12T12:02:42.081079+0000","last_unstale":"2024-04-12T12:02:58.965268+0000","last_undegraded":"2024-04-12T12:02:58.965268+0000","last_fullsized":"2024-04-12T12:02:58.965268+0000","mapping_epoch":1354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:36:47.384736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":199,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965729+0000","last_change":"2024-04-12T12:00:35.474735+0000","last_active":"2024-04-12T12:02:58.965729+0000","last_peered":"2024-04-12T12:02:58.965729+0000","last_clean":"2024-04-12T12:02:58.965729+0000","last_became_active":"2024-04-12T12:00:35.463433+0000","last_became_peered":"2024-04-12T12:00:35.463433+0000","last_unstale":"2024-04-12T12:02:58.965729+0000","last_undegraded":"2024-04-12T12:02:58.965729+0000","last_fullsized":"2024-04-12T12:02:58.965729+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:27.215562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346426+0000","last_change":"2024-04-12T12:01:09.591568+0000","last_active":"2024-04-12T12:03:02.346426+0000","last_peered":"2024-04-12T12:03:02.346426+0000","last_clean":"2024-04-12T12:03:02.346426+0000","last_became_active":"2024-04-12T12:01:09.591189+0000","last_became_peered":"2024-04-12T12:01:09.591189+0000","last_unstale":"2024-04-12T12:03:02.346426+0000","last_undegraded":"2024-04-12T12:03:02.346426+0000","last_fullsized":"2024-04-12T12:03:02.346426+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:56:06.870492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":75,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094923+0000","last_change":"2024-04-12T12:01:08.200089+0000","last_active":"2024-04-12T12:02:58.094923+0000","last_peered":"2024-04-12T12:02:58.094923+0000","last_clean":"2024-04-12T12:02:58.094923+0000","last_became_active":"2024-04-12T12:01:08.199531+0000","last_became_peered":"2024-04-12T12:01:08.199531+0000","last_unstale":"2024-04-12T12:02:58.094923+0000","last_undegraded":"2024-04-12T12:02:58.094923+0000","last_fullsized":"2024-04-12T12:02:58.094923+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:51:22.075323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1363,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094674+0000","last_change":"2024-04-12T12:00:35.474945+0000","last_active":"2024-04-12T12:02:58.094674+0000","last_peered":"2024-04-12T12:02:58.094674+0000","last_clean":"2024-04-12T12:02:58.094674+0000","last_became_active":"2024-04-12T12:00:35.474564+0000","last_became_peered":"2024-04-12T12:00:35.474564+0000","last_unstale":"2024-04-12T12:02:58.094674+0000","last_undegraded":"2024-04-12T12:02:58.094674+0000","last_fullsized":"2024-04-12T12:02:58.094674+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:13:53.178867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1366,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094715+0000","last_change":"2024-04-12T12:01:09.233104+0000","last_active":"2024-04-12T12:02:58.094715+0000","last_peered":"2024-04-12T12:02:58.094715+0000","last_clean":"2024-04-12T12:02:58.094715+0000","last_became_active":"2024-04-12T12:01:09.232861+0000","last_became_peered":"2024-04-12T12:01:09.232861+0000","last_unstale":"2024-04-12T12:02:58.094715+0000","last_undegraded":"2024-04-12T12:02:58.094715+0000","last_fullsized":"2024-04-12T12:02:58.094715+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:20:34.752472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2165,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347096+0000","last_change":"2024-04-12T12:00:35.475414+0000","last_active":"2024-04-12T12:03:02.347096+0000","last_peered":"2024-04-12T12:03:02.347096+0000","last_clean":"2024-04-12T12:03:02.347096+0000","last_became_active":"2024-04-12T12:00:35.475098+0000","last_became_peered":"2024-04-12T12:00:35.475098+0000","last_unstale":"2024-04-12T12:03:02.347096+0000","last_undegraded":"2024-04-12T12:03:02.347096+0000","last_fullsized":"2024-04-12T12:03:02.347096+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:25:12.377240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"249'161","reported_seq":1728,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965378+0000","last_change":"2024-04-12T12:01:08.230318+0000","last_active":"2024-04-12T12:02:58.965378+0000","last_peered":"2024-04-12T12:02:58.965378+0000","last_clean":"2024-04-12T12:02:58.965378+0000","last_became_active":"2024-04-12T12:01:08.229959+0000","last_became_peered":"2024-04-12T12:01:08.229959+0000","last_unstale":"2024-04-12T12:02:58.965378+0000","last_undegraded":"2024-04-12T12:02:58.965378+0000","last_fullsized":"2024-04-12T12:02:58.965378+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:08.650841+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:08.650841+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:17:07.248029+0000","scrub_duration":0.001456279,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":251,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347200+0000","last_change":"2024-04-12T12:01:08.360054+0000","last_active":"2024-04-12T12:03:02.347200+0000","last_peered":"2024-04-12T12:03:02.347200+0000","last_clean":"2024-04-12T12:03:02.347200+0000","last_became_active":"2024-04-12T12:01:08.359716+0000","last_became_peered":"2024-04-12T12:01:08.359716+0000","last_unstale":"2024-04-12T12:03:02.347200+0000","last_undegraded":"2024-04-12T12:03:02.347200+0000","last_fullsized":"2024-04-12T12:03:02.347200+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:31:05.807406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1886,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094763+0000","last_change":"2024-04-12T12:01:08.223388+0000","last_active":"2024-04-12T12:02:58.094763+0000","last_peered":"2024-04-12T12:02:58.094763+0000","last_clean":"2024-04-12T12:02:58.094763+0000","last_became_active":"2024-04-12T12:01:08.222873+0000","last_became_peered":"2024-04-12T12:01:08.222873+0000","last_unstale":"2024-04-12T12:02:58.094763+0000","last_undegraded":"2024-04-12T12:02:58.094763+0000","last_fullsized":"2024-04-12T12:02:58.094763+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:57:02.009904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"249'161","reported_seq":862,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622404+0000","last_change":"2024-04-12T12:02:44.218756+0000","last_active":"2024-04-12T12:03:02.622404+0000","last_peered":"2024-04-12T12:03:02.622404+0000","last_clean":"2024-04-12T12:03:02.622404+0000","last_became_active":"2024-04-12T12:02:44.218302+0000","last_became_peered":"2024-04-12T12:02:44.218302+0000","last_unstale":"2024-04-12T12:03:02.622404+0000","last_undegraded":"2024-04-12T12:03:02.622404+0000","last_fullsized":"2024-04-12T12:03:02.622404+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:10.437190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:10.437190+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:36:07.969961+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1106,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623521+0000","last_change":"2024-04-12T12:00:35.472844+0000","last_active":"2024-04-12T12:03:02.623521+0000","last_peered":"2024-04-12T12:03:02.623521+0000","last_clean":"2024-04-12T12:03:02.623521+0000","last_became_active":"2024-04-12T12:00:35.472506+0000","last_became_peered":"2024-04-12T12:00:35.472506+0000","last_unstale":"2024-04-12T12:03:02.623521+0000","last_undegraded":"2024-04-12T12:03:02.623521+0000","last_fullsized":"2024-04-12T12:03:02.623521+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:01:56.528972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":2049,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623129+0000","last_change":"2024-04-12T12:00:35.468294+0000","last_active":"2024-04-12T12:03:02.623129+0000","last_peered":"2024-04-12T12:03:02.623129+0000","last_clean":"2024-04-12T12:03:02.623129+0000","last_became_active":"2024-04-12T12:00:35.467888+0000","last_became_peered":"2024-04-12T12:00:35.467888+0000","last_unstale":"2024-04-12T12:03:02.623129+0000","last_undegraded":"2024-04-12T12:03:02.623129+0000","last_fullsized":"2024-04-12T12:03:02.623129+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:35:16.687008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":2086,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623560+0000","last_change":"2024-04-12T11:45:52.976139+0000","last_active":"2024-04-12T12:03:02.623560+0000","last_peered":"2024-04-12T12:03:02.623560+0000","last_clean":"2024-04-12T12:03:02.623560+0000","last_became_active":"2024-04-12T11:45:52.975803+0000","last_became_peered":"2024-04-12T11:45:52.975803+0000","last_unstale":"2024-04-12T12:03:02.623560+0000","last_undegraded":"2024-04-12T12:03:02.623560+0000","last_fullsized":"2024-04-12T12:03:02.623560+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:46.618143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":2067,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.625034+0000","last_change":"2024-04-12T12:00:35.473079+0000","last_active":"2024-04-12T12:03:02.625034+0000","last_peered":"2024-04-12T12:03:02.625034+0000","last_clean":"2024-04-12T12:03:02.625034+0000","last_became_active":"2024-04-12T12:00:35.472773+0000","last_became_peered":"2024-04-12T12:00:35.472773+0000","last_unstale":"2024-04-12T12:03:02.625034+0000","last_undegraded":"2024-04-12T12:03:02.625034+0000","last_fullsized":"2024-04-12T12:03:02.625034+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:20:42.192043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"249'161","reported_seq":824,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622715+0000","last_change":"2024-04-12T11:53:49.137544+0000","last_active":"2024-04-12T12:03:02.622715+0000","last_peered":"2024-04-12T12:03:02.622715+0000","last_clean":"2024-04-12T12:03:02.622715+0000","last_became_active":"2024-04-12T11:53:49.137196+0000","last_became_peered":"2024-04-12T11:53:49.137196+0000","last_unstale":"2024-04-12T12:03:02.622715+0000","last_undegraded":"2024-04-12T12:03:02.622715+0000","last_fullsized":"2024-04-12T12:03:02.622715+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:11.447056+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:11.447056+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:17:12.938789+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347528+0000","last_change":"2024-04-12T12:01:09.230000+0000","last_active":"2024-04-12T12:03:02.347528+0000","last_peered":"2024-04-12T12:03:02.347528+0000","last_clean":"2024-04-12T12:03:02.347528+0000","last_became_active":"2024-04-12T12:01:09.229670+0000","last_became_peered":"2024-04-12T12:01:09.229670+0000","last_unstale":"2024-04-12T12:03:02.347528+0000","last_undegraded":"2024-04-12T12:03:02.347528+0000","last_fullsized":"2024-04-12T12:03:02.347528+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:37:12.886844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346864+0000","last_change":"2024-04-12T12:00:35.477779+0000","last_active":"2024-04-12T12:03:02.346864+0000","last_peered":"2024-04-12T12:03:02.346864+0000","last_clean":"2024-04-12T12:03:02.346864+0000","last_became_active":"2024-04-12T12:00:35.477462+0000","last_became_peered":"2024-04-12T12:00:35.477462+0000","last_unstale":"2024-04-12T12:03:02.346864+0000","last_undegraded":"2024-04-12T12:03:02.346864+0000","last_fullsized":"2024-04-12T12:03:02.346864+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:51:17.465133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"249'161","reported_seq":631,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.095026+0000","last_change":"2024-04-12T12:01:08.203484+0000","last_active":"2024-04-12T12:02:58.095026+0000","last_peered":"2024-04-12T12:02:58.095026+0000","last_clean":"2024-04-12T12:02:58.095026+0000","last_became_active":"2024-04-12T12:01:08.203128+0000","last_became_peered":"2024-04-12T12:01:08.203128+0000","last_unstale":"2024-04-12T12:02:58.095026+0000","last_undegraded":"2024-04-12T12:02:58.095026+0000","last_fullsized":"2024-04-12T12:02:58.095026+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:37.607274+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:37.607274+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:03:30.798799+0000","scrub_duration":0.0037173240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346966+0000","last_change":"2024-04-12T12:00:35.473414+0000","last_active":"2024-04-12T12:03:02.346966+0000","last_peered":"2024-04-12T12:03:02.346966+0000","last_clean":"2024-04-12T12:03:02.346966+0000","last_became_active":"2024-04-12T12:00:35.472846+0000","last_became_peered":"2024-04-12T12:00:35.472846+0000","last_unstale":"2024-04-12T12:03:02.346966+0000","last_undegraded":"2024-04-12T12:03:02.346966+0000","last_fullsized":"2024-04-12T12:03:02.346966+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:59:19.941070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1357,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622987+0000","last_change":"2024-04-12T12:01:08.211265+0000","last_active":"2024-04-12T12:03:02.622987+0000","last_peered":"2024-04-12T12:03:02.622987+0000","last_clean":"2024-04-12T12:03:02.622987+0000","last_became_active":"2024-04-12T12:01:08.210898+0000","last_became_peered":"2024-04-12T12:01:08.210898+0000","last_unstale":"2024-04-12T12:03:02.622987+0000","last_undegraded":"2024-04-12T12:03:02.622987+0000","last_fullsized":"2024-04-12T12:03:02.622987+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:08:47.927220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347008+0000","last_change":"2024-04-12T12:00:35.477414+0000","last_active":"2024-04-12T12:03:02.347008+0000","last_peered":"2024-04-12T12:03:02.347008+0000","last_clean":"2024-04-12T12:03:02.347008+0000","last_became_active":"2024-04-12T12:00:35.477035+0000","last_became_peered":"2024-04-12T12:00:35.477035+0000","last_unstale":"2024-04-12T12:03:02.347008+0000","last_undegraded":"2024-04-12T12:03:02.347008+0000","last_fullsized":"2024-04-12T12:03:02.347008+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:25:45.630314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"249'161","reported_seq":828,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348009+0000","last_change":"2024-04-12T11:53:49.129889+0000","last_active":"2024-04-12T12:03:02.348009+0000","last_peered":"2024-04-12T12:03:02.348009+0000","last_clean":"2024-04-12T12:03:02.348009+0000","last_became_active":"2024-04-12T11:53:49.129462+0000","last_became_peered":"2024-04-12T11:53:49.129462+0000","last_unstale":"2024-04-12T12:03:02.348009+0000","last_undegraded":"2024-04-12T12:03:02.348009+0000","last_fullsized":"2024-04-12T12:03:02.348009+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:05.639497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:05.639497+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:16:18.801501+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346902+0000","last_change":"2024-04-12T12:00:35.475473+0000","last_active":"2024-04-12T12:03:02.346902+0000","last_peered":"2024-04-12T12:03:02.346902+0000","last_clean":"2024-04-12T12:03:02.346902+0000","last_became_active":"2024-04-12T12:00:35.475125+0000","last_became_peered":"2024-04-12T12:00:35.475125+0000","last_unstale":"2024-04-12T12:03:02.346902+0000","last_undegraded":"2024-04-12T12:03:02.346902+0000","last_fullsized":"2024-04-12T12:03:02.346902+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:59:05.176022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":641,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346949+0000","last_change":"2024-04-12T11:58:33.661396+0000","last_active":"2024-04-12T12:03:02.346949+0000","last_peered":"2024-04-12T12:03:02.346949+0000","last_clean":"2024-04-12T12:03:02.346949+0000","last_became_active":"2024-04-12T11:58:33.661076+0000","last_became_peered":"2024-04-12T11:58:33.661076+0000","last_unstale":"2024-04-12T12:03:02.346949+0000","last_undegraded":"2024-04-12T12:03:02.346949+0000","last_fullsized":"2024-04-12T12:03:02.346949+0000","mapping_epoch":1321,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:31:47.577826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346962+0000","last_change":"2024-04-12T12:01:09.225968+0000","last_active":"2024-04-12T12:03:02.346962+0000","last_peered":"2024-04-12T12:03:02.346962+0000","last_clean":"2024-04-12T12:03:02.346962+0000","last_became_active":"2024-04-12T12:01:09.225273+0000","last_became_peered":"2024-04-12T12:01:09.225273+0000","last_unstale":"2024-04-12T12:03:02.346962+0000","last_undegraded":"2024-04-12T12:03:02.346962+0000","last_fullsized":"2024-04-12T12:03:02.346962+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:39.143286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"249'161","reported_seq":689,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346678+0000","last_change":"2024-04-12T11:59:36.979504+0000","last_active":"2024-04-12T12:03:02.346678+0000","last_peered":"2024-04-12T12:03:02.346678+0000","last_clean":"2024-04-12T12:03:02.346678+0000","last_became_active":"2024-04-12T11:59:36.296167+0000","last_became_peered":"2024-04-12T11:59:36.296167+0000","last_unstale":"2024-04-12T12:03:02.346678+0000","last_undegraded":"2024-04-12T12:03:02.346678+0000","last_fullsized":"2024-04-12T12:03:02.346678+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:36.979380+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:36.979380+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:42:09.481447+0000","scrub_duration":17,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2147,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622745+0000","last_change":"2024-04-12T12:01:08.241455+0000","last_active":"2024-04-12T12:03:02.622745+0000","last_peered":"2024-04-12T12:03:02.622745+0000","last_clean":"2024-04-12T12:03:02.622745+0000","last_became_active":"2024-04-12T12:01:08.240087+0000","last_became_peered":"2024-04-12T12:01:08.240087+0000","last_unstale":"2024-04-12T12:03:02.622745+0000","last_undegraded":"2024-04-12T12:03:02.622745+0000","last_fullsized":"2024-04-12T12:03:02.622745+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:45:12.951549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347371+0000","last_change":"2024-04-12T12:00:35.471479+0000","last_active":"2024-04-12T12:03:02.347371+0000","last_peered":"2024-04-12T12:03:02.347371+0000","last_clean":"2024-04-12T12:03:02.347371+0000","last_became_active":"2024-04-12T12:00:35.471015+0000","last_became_peered":"2024-04-12T12:00:35.471015+0000","last_unstale":"2024-04-12T12:03:02.347371+0000","last_undegraded":"2024-04-12T12:03:02.347371+0000","last_fullsized":"2024-04-12T12:03:02.347371+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:55:50.471132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346782+0000","last_change":"2024-04-12T12:00:35.479622+0000","last_active":"2024-04-12T12:03:02.346782+0000","last_peered":"2024-04-12T12:03:02.346782+0000","last_clean":"2024-04-12T12:03:02.346782+0000","last_became_active":"2024-04-12T12:00:35.479249+0000","last_became_peered":"2024-04-12T12:00:35.479249+0000","last_unstale":"2024-04-12T12:03:02.346782+0000","last_undegraded":"2024-04-12T12:03:02.346782+0000","last_fullsized":"2024-04-12T12:03:02.346782+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:24:26.689379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1409,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622836+0000","last_change":"2024-04-12T12:01:08.223472+0000","last_active":"2024-04-12T12:03:02.622836+0000","last_peered":"2024-04-12T12:03:02.622836+0000","last_clean":"2024-04-12T12:03:02.622836+0000","last_became_active":"2024-04-12T12:01:08.222991+0000","last_became_peered":"2024-04-12T12:01:08.222991+0000","last_unstale":"2024-04-12T12:03:02.622836+0000","last_undegraded":"2024-04-12T12:03:02.622836+0000","last_fullsized":"2024-04-12T12:03:02.622836+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:28:37.101566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1697,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965515+0000","last_change":"2024-04-12T11:53:06.603022+0000","last_active":"2024-04-12T12:02:58.965515+0000","last_peered":"2024-04-12T12:02:58.965515+0000","last_clean":"2024-04-12T12:02:58.965515+0000","last_became_active":"2024-04-12T11:49:07.217921+0000","last_became_peered":"2024-04-12T11:49:07.217921+0000","last_unstale":"2024-04-12T12:02:58.965515+0000","last_undegraded":"2024-04-12T12:02:58.965515+0000","last_fullsized":"2024-04-12T12:02:58.965515+0000","mapping_epoch":1210,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":0,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:06.602918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:06.602918+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:40.776170+0000","scrub_duration":0.001429877,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":594,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094740+0000","last_change":"2024-04-12T12:01:08.220206+0000","last_active":"2024-04-12T12:02:58.094740+0000","last_peered":"2024-04-12T12:02:58.094740+0000","last_clean":"2024-04-12T12:02:58.094740+0000","last_became_active":"2024-04-12T12:01:08.219103+0000","last_became_peered":"2024-04-12T12:01:08.219103+0000","last_unstale":"2024-04-12T12:02:58.094740+0000","last_undegraded":"2024-04-12T12:02:58.094740+0000","last_fullsized":"2024-04-12T12:02:58.094740+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:04:53.296805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1336,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.095009+0000","last_change":"2024-04-12T11:45:53.016616+0000","last_active":"2024-04-12T12:02:58.095009+0000","last_peered":"2024-04-12T12:02:58.095009+0000","last_clean":"2024-04-12T12:02:58.095009+0000","last_became_active":"2024-04-12T11:45:53.016292+0000","last_became_peered":"2024-04-12T11:45:53.016292+0000","last_unstale":"2024-04-12T12:02:58.095009+0000","last_undegraded":"2024-04-12T12:02:58.095009+0000","last_fullsized":"2024-04-12T12:02:58.095009+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:53:29.382293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"249'161","reported_seq":736,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347232+0000","last_change":"2024-04-12T12:02:44.218750+0000","last_active":"2024-04-12T12:03:02.347232+0000","last_peered":"2024-04-12T12:03:02.347232+0000","last_clean":"2024-04-12T12:03:02.347232+0000","last_became_active":"2024-04-12T12:02:44.218240+0000","last_became_peered":"2024-04-12T12:02:44.218240+0000","last_unstale":"2024-04-12T12:03:02.347232+0000","last_undegraded":"2024-04-12T12:03:02.347232+0000","last_fullsized":"2024-04-12T12:03:02.347232+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:57:49.744017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:57:49.744017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:07.839300+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347286+0000","last_change":"2024-04-12T12:00:35.475887+0000","last_active":"2024-04-12T12:03:02.347286+0000","last_peered":"2024-04-12T12:03:02.347286+0000","last_clean":"2024-04-12T12:03:02.347286+0000","last_became_active":"2024-04-12T12:00:35.475531+0000","last_became_peered":"2024-04-12T12:00:35.475531+0000","last_unstale":"2024-04-12T12:03:02.347286+0000","last_undegraded":"2024-04-12T12:03:02.347286+0000","last_fullsized":"2024-04-12T12:03:02.347286+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:06:33.171131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2184,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348204+0000","last_change":"2024-04-12T12:01:08.241097+0000","last_active":"2024-04-12T12:03:02.348204+0000","last_peered":"2024-04-12T12:03:02.348204+0000","last_clean":"2024-04-12T12:03:02.348204+0000","last_became_active":"2024-04-12T12:01:08.240402+0000","last_became_peered":"2024-04-12T12:01:08.240402+0000","last_unstale":"2024-04-12T12:03:02.348204+0000","last_undegraded":"2024-04-12T12:03:02.348204+0000","last_fullsized":"2024-04-12T12:03:02.348204+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:30:15.712301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2163,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348066+0000","last_change":"2024-04-12T12:00:35.480505+0000","last_active":"2024-04-12T12:03:02.348066+0000","last_peered":"2024-04-12T12:03:02.348066+0000","last_clean":"2024-04-12T12:03:02.348066+0000","last_became_active":"2024-04-12T12:00:35.480136+0000","last_became_peered":"2024-04-12T12:00:35.480136+0000","last_unstale":"2024-04-12T12:03:02.348066+0000","last_undegraded":"2024-04-12T12:03:02.348066+0000","last_fullsized":"2024-04-12T12:03:02.348066+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:12:05.320400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"249'161","reported_seq":705,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094937+0000","last_change":"2024-04-12T12:01:08.209702+0000","last_active":"2024-04-12T12:02:58.094937+0000","last_peered":"2024-04-12T12:02:58.094937+0000","last_clean":"2024-04-12T12:02:58.094937+0000","last_became_active":"2024-04-12T12:01:08.209465+0000","last_became_peered":"2024-04-12T12:01:08.209465+0000","last_unstale":"2024-04-12T12:02:58.094937+0000","last_undegraded":"2024-04-12T12:02:58.094937+0000","last_fullsized":"2024-04-12T12:02:58.094937+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:05.130465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:05.130465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:53:20.786613+0000","scrub_duration":0.001397695,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":170,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965758+0000","last_change":"2024-04-12T11:45:53.005585+0000","last_active":"2024-04-12T12:02:58.965758+0000","last_peered":"2024-04-12T12:02:58.965758+0000","last_clean":"2024-04-12T12:02:58.965758+0000","last_became_active":"2024-04-12T11:45:52.987880+0000","last_became_peered":"2024-04-12T11:45:52.987880+0000","last_unstale":"2024-04-12T12:02:58.965758+0000","last_undegraded":"2024-04-12T12:02:58.965758+0000","last_fullsized":"2024-04-12T12:02:58.965758+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:09.819861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347097+0000","last_change":"2024-04-12T12:00:35.473178+0000","last_active":"2024-04-12T12:03:02.347097+0000","last_peered":"2024-04-12T12:03:02.347097+0000","last_clean":"2024-04-12T12:03:02.347097+0000","last_became_active":"2024-04-12T12:00:35.472877+0000","last_became_peered":"2024-04-12T12:00:35.472877+0000","last_unstale":"2024-04-12T12:03:02.347097+0000","last_undegraded":"2024-04-12T12:03:02.347097+0000","last_fullsized":"2024-04-12T12:03:02.347097+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:03:28.998314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2167,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347405+0000","last_change":"2024-04-12T12:00:35.475473+0000","last_active":"2024-04-12T12:03:02.347405+0000","last_peered":"2024-04-12T12:03:02.347405+0000","last_clean":"2024-04-12T12:03:02.347405+0000","last_became_active":"2024-04-12T12:00:35.475120+0000","last_became_peered":"2024-04-12T12:00:35.475120+0000","last_unstale":"2024-04-12T12:03:02.347405+0000","last_undegraded":"2024-04-12T12:03:02.347405+0000","last_fullsized":"2024-04-12T12:03:02.347405+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:14:52.218068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"249'161","reported_seq":832,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.624761+0000","last_change":"2024-04-12T12:02:44.216787+0000","last_active":"2024-04-12T12:03:02.624761+0000","last_peered":"2024-04-12T12:03:02.624761+0000","last_clean":"2024-04-12T12:03:02.624761+0000","last_became_active":"2024-04-12T12:02:44.216350+0000","last_became_peered":"2024-04-12T12:02:44.216350+0000","last_unstale":"2024-04-12T12:03:02.624761+0000","last_undegraded":"2024-04-12T12:03:02.624761+0000","last_fullsized":"2024-04-12T12:03:02.624761+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:10.604457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:10.604457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:01:05.908018+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":135,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348410+0000","last_change":"2024-04-12T12:01:08.213894+0000","last_active":"2024-04-12T12:03:02.348410+0000","last_peered":"2024-04-12T12:03:02.348410+0000","last_clean":"2024-04-12T12:03:02.348410+0000","last_became_active":"2024-04-12T12:01:08.213309+0000","last_became_peered":"2024-04-12T12:01:08.213309+0000","last_unstale":"2024-04-12T12:03:02.348410+0000","last_undegraded":"2024-04-12T12:03:02.348410+0000","last_fullsized":"2024-04-12T12:03:02.348410+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:35:42.203466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":70,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347604+0000","last_change":"2024-04-12T12:00:35.477662+0000","last_active":"2024-04-12T12:03:02.347604+0000","last_peered":"2024-04-12T12:03:02.347604+0000","last_clean":"2024-04-12T12:03:02.347604+0000","last_became_active":"2024-04-12T12:00:35.477235+0000","last_became_peered":"2024-04-12T12:00:35.477235+0000","last_unstale":"2024-04-12T12:03:02.347604+0000","last_undegraded":"2024-04-12T12:03:02.347604+0000","last_fullsized":"2024-04-12T12:03:02.347604+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:18:34.713724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965256+0000","last_change":"2024-04-12T12:01:08.230577+0000","last_active":"2024-04-12T12:02:58.965256+0000","last_peered":"2024-04-12T12:02:58.965256+0000","last_clean":"2024-04-12T12:02:58.965256+0000","last_became_active":"2024-04-12T12:01:08.230190+0000","last_became_peered":"2024-04-12T12:01:08.230190+0000","last_unstale":"2024-04-12T12:02:58.965256+0000","last_undegraded":"2024-04-12T12:02:58.965256+0000","last_fullsized":"2024-04-12T12:02:58.965256+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:59:11.839551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"249'161","reported_seq":1656,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.965643+0000","last_change":"2024-04-12T12:01:08.230547+0000","last_active":"2024-04-12T12:02:58.965643+0000","last_peered":"2024-04-12T12:02:58.965643+0000","last_clean":"2024-04-12T12:02:58.965643+0000","last_became_active":"2024-04-12T12:01:08.230123+0000","last_became_peered":"2024-04-12T12:01:08.230123+0000","last_unstale":"2024-04-12T12:02:58.965643+0000","last_undegraded":"2024-04-12T12:02:58.965643+0000","last_fullsized":"2024-04-12T12:02:58.965643+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:17.062855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:17.062855+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:06:49.991085+0000","scrub_duration":0.0078090989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1369,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.095075+0000","last_change":"2024-04-12T11:45:52.979745+0000","last_active":"2024-04-12T12:02:58.095075+0000","last_peered":"2024-04-12T12:02:58.095075+0000","last_clean":"2024-04-12T12:02:58.095075+0000","last_became_active":"2024-04-12T11:45:52.978936+0000","last_became_peered":"2024-04-12T11:45:52.978936+0000","last_unstale":"2024-04-12T12:02:58.095075+0000","last_undegraded":"2024-04-12T12:02:58.095075+0000","last_fullsized":"2024-04-12T12:02:58.095075+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:22.710430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":2174,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347784+0000","last_change":"2024-04-12T12:00:35.485498+0000","last_active":"2024-04-12T12:03:02.347784+0000","last_peered":"2024-04-12T12:03:02.347784+0000","last_clean":"2024-04-12T12:03:02.347784+0000","last_became_active":"2024-04-12T12:00:35.475885+0000","last_became_peered":"2024-04-12T12:00:35.475885+0000","last_unstale":"2024-04-12T12:03:02.347784+0000","last_undegraded":"2024-04-12T12:03:02.347784+0000","last_fullsized":"2024-04-12T12:03:02.347784+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:42:57.450495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1938,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.094486+0000","last_change":"2024-04-12T11:58:33.661378+0000","last_active":"2024-04-12T12:02:58.094486+0000","last_peered":"2024-04-12T12:02:58.094486+0000","last_clean":"2024-04-12T12:02:58.094486+0000","last_became_active":"2024-04-12T11:58:33.661014+0000","last_became_peered":"2024-04-12T11:58:33.661014+0000","last_unstale":"2024-04-12T12:02:58.094486+0000","last_undegraded":"2024-04-12T12:02:58.094486+0000","last_fullsized":"2024-04-12T12:02:58.094486+0000","mapping_epoch":1321,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1322,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:03:05.992776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":2154,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347305+0000","last_change":"2024-04-12T11:45:52.979689+0000","last_active":"2024-04-12T12:03:02.347305+0000","last_peered":"2024-04-12T12:03:02.347305+0000","last_clean":"2024-04-12T12:03:02.347305+0000","last_became_active":"2024-04-12T11:45:52.979286+0000","last_became_peered":"2024-04-12T11:45:52.979286+0000","last_unstale":"2024-04-12T12:03:02.347305+0000","last_undegraded":"2024-04-12T12:03:02.347305+0000","last_fullsized":"2024-04-12T12:03:02.347305+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:10.453472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":41,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623009+0000","last_change":"2024-04-12T12:02:08.214192+0000","last_active":"2024-04-12T12:03:02.623009+0000","last_peered":"2024-04-12T12:03:02.623009+0000","last_clean":"2024-04-12T12:03:02.623009+0000","last_became_active":"2024-04-12T12:02:08.213849+0000","last_became_peered":"2024-04-12T12:02:08.213849+0000","last_unstale":"2024-04-12T12:03:02.623009+0000","last_undegraded":"2024-04-12T12:03:02.623009+0000","last_fullsized":"2024-04-12T12:03:02.623009+0000","mapping_epoch":1347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:07:59.571577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":249,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.624260+0000","last_change":"2024-04-12T12:00:35.483510+0000","last_active":"2024-04-12T12:03:02.624260+0000","last_peered":"2024-04-12T12:03:02.624260+0000","last_clean":"2024-04-12T12:03:02.624260+0000","last_became_active":"2024-04-12T12:00:35.472254+0000","last_became_peered":"2024-04-12T12:00:35.472254+0000","last_unstale":"2024-04-12T12:03:02.624260+0000","last_undegraded":"2024-04-12T12:03:02.624260+0000","last_fullsized":"2024-04-12T12:03:02.624260+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:28.883365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":74,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.347188+0000","last_change":"2024-04-12T12:01:09.587265+0000","last_active":"2024-04-12T12:03:02.347188+0000","last_peered":"2024-04-12T12:03:02.347188+0000","last_clean":"2024-04-12T12:03:02.347188+0000","last_became_active":"2024-04-12T12:01:09.586910+0000","last_became_peered":"2024-04-12T12:01:09.586910+0000","last_unstale":"2024-04-12T12:03:02.347188+0000","last_undegraded":"2024-04-12T12:03:02.347188+0000","last_fullsized":"2024-04-12T12:03:02.347188+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:14:23.708275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":85,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-04-12T12:02:58.095019+0000","last_change":"2024-04-12T12:00:35.481562+0000","last_active":"2024-04-12T12:02:58.095019+0000","last_peered":"2024-04-12T12:02:58.095019+0000","last_clean":"2024-04-12T12:02:58.095019+0000","last_became_active":"2024-04-12T12:00:35.475238+0000","last_became_peered":"2024-04-12T12:00:35.475238+0000","last_unstale":"2024-04-12T12:02:58.095019+0000","last_undegraded":"2024-04-12T12:02:58.095019+0000","last_fullsized":"2024-04-12T12:02:58.095019+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:47.893745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1932,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.346565+0000","last_change":"2024-04-12T12:01:08.225762+0000","last_active":"2024-04-12T12:03:02.346565+0000","last_peered":"2024-04-12T12:03:02.346565+0000","last_clean":"2024-04-12T12:03:02.346565+0000","last_became_active":"2024-04-12T12:01:08.225072+0000","last_became_peered":"2024-04-12T12:01:08.225072+0000","last_unstale":"2024-04-12T12:03:02.346565+0000","last_undegraded":"2024-04-12T12:03:02.346565+0000","last_fullsized":"2024-04-12T12:03:02.346565+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:04.300351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":217,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.623179+0000","last_change":"2024-04-12T12:02:42.080490+0000","last_active":"2024-04-12T12:03:02.623179+0000","last_peered":"2024-04-12T12:03:02.623179+0000","last_clean":"2024-04-12T12:03:02.623179+0000","last_became_active":"2024-04-12T12:02:42.080074+0000","last_became_peered":"2024-04-12T12:02:42.080074+0000","last_unstale":"2024-04-12T12:03:02.623179+0000","last_undegraded":"2024-04-12T12:03:02.623179+0000","last_fullsized":"2024-04-12T12:03:02.623179+0000","mapping_epoch":1354,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:42:31.461796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":734,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622789+0000","last_change":"2024-04-12T12:01:08.239424+0000","last_active":"2024-04-12T12:03:02.622789+0000","last_peered":"2024-04-12T12:03:02.622789+0000","last_clean":"2024-04-12T12:03:02.622789+0000","last_became_active":"2024-04-12T12:01:08.239070+0000","last_became_peered":"2024-04-12T12:01:08.239070+0000","last_unstale":"2024-04-12T12:03:02.622789+0000","last_undegraded":"2024-04-12T12:03:02.622789+0000","last_fullsized":"2024-04-12T12:03:02.622789+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:45:26.732811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":2211,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.348315+0000","last_change":"2024-04-12T12:02:08.217720+0000","last_active":"2024-04-12T12:03:02.348315+0000","last_peered":"2024-04-12T12:03:02.348315+0000","last_clean":"2024-04-12T12:03:02.348315+0000","last_became_active":"2024-04-12T12:02:08.214163+0000","last_became_peered":"2024-04-12T12:02:08.214163+0000","last_unstale":"2024-04-12T12:03:02.348315+0000","last_undegraded":"2024-04-12T12:03:02.348315+0000","last_fullsized":"2024-04-12T12:03:02.348315+0000","mapping_epoch":1347,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:56:25.681924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2910,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":2066,"reported_epoch":1363,"state":"active+clean","last_fresh":"2024-04-12T12:03:02.622968+0000","last_change":"2024-04-12T12:00:35.472529+0000","last_active":"2024-04-12T12:03:02.622968+0000","last_peered":"2024-04-12T12:03:02.622968+0000","last_clean":"2024-04-12T12:03:02.622968+0000","last_became_active":"2024-04-12T12:00:35.472057+0000","last_became_peered":"2024-04-12T12:00:35.472057+0000","last_unstale":"2024-04-12T12:03:02.622968+0000","last_undegraded":"2024-04-12T12:03:02.622968+0000","last_fullsized":"2024-04-12T12:03:02.622968+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:07:59.083703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":12748,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35155,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234609,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972596,"kb_used_data":8808,"kb_used_omap":1,"kb_used_meta":963774,"kb_avail":92772556,"statfs":{"total":95995035648,"available":94999097344,"internally_reserved":0,"allocated":9019392,"data_stored":5898978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":986905035},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496246,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":798128,"kb_used_data":9060,"kb_used_omap":1,"kb_used_meta":789054,"kb_avail":92947024,"statfs":{"total":95995035648,"available":95177752576,"internally_reserved":0,"allocated":9277440,"data_stored":6162102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":807991752},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536556,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770648,"kb_used_data":8460,"kb_used_omap":30,"kb_used_meta":762145,"kb_avail":92974504,"statfs":{"total":95995035648,"available":95205892096,"internally_reserved":0,"allocated":8663040,"data_stored":5566338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":780436802},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190640,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1131360,"kb_used_data":8468,"kb_used_omap":43,"kb_used_meta":1122836,"kb_avail":92613792,"statfs":{"total":95995035648,"available":94836523008,"internally_reserved":0,"allocated":8671232,"data_stored":5566942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":1149784107},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849767,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1027812,"kb_used_data":8728,"kb_used_omap":36,"kb_used_meta":1019035,"kb_avail":92717340,"statfs":{"total":95995035648,"available":94942556160,"internally_reserved":0,"allocated":8937472,"data_stored":5829642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":1043492020},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1342,"seq":5763846111257,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":126628,"kb_used_data":9048,"kb_used_omap":32,"kb_used_meta":117535,"kb_avail":93618524,"statfs":{"total":95995035648,"available":95865368576,"internally_reserved":0,"allocated":9265152,"data_stored":6156866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33145,"internal_metadata":120356487},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018333,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951344,"kb_used_data":8484,"kb_used_omap":42,"kb_used_meta":942805,"kb_avail":92793808,"statfs":{"total":95995035648,"available":95020859392,"internally_reserved":0,"allocated":8687616,"data_stored":5571240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":965433144},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":5506148073581,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10321,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4818,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T12:03:11.659 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-04-12T12:03:11.659 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-04-12T12:03:11.659 INFO:tasks.thrashosds.thrasher:Adding osd 5 2024-04-12T12:03:11.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd in 5 2024-04-12T12:03:11.846 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:11.880 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:12.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:12 smithi029 ceph-mon[93650]: from='client.36138 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:03:12.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:12 smithi029 ceph-mon[97787]: from='client.36138 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:03:12.607 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:12 smithi087 ceph-mon[89014]: from='client.36138 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:03:12.607 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:03:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:03:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:03:13.485 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:13.486 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T12:03:13.486 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T12:03:13.486 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T12:03:13.486 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.486 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109311 10.e 10:75e66fde:::benchmark_data_smithi029_256330_object7545:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1286)", 2024-04-12T12:03:13.486 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.390346+0000", 2024-04-12T12:03:13.486 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 495.09178049100001, 2024-04-12T12:03:13.486 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.122322606, 2024-04-12T12:03:13.486 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.486 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109311 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.390346+0000", 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.487 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.390346+0000", 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.390348+0000", 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6509999999999999e-06 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.390356+0000", 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.447e-06 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.488 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.390371+0000", 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5201999999999999e-05 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.390381+0000", 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8099999999999992e-06 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.390440+0000", 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9985999999999998e-05 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.489 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.390584+0000", 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000143063 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.390709+0000", 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012529899999999999 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.393707+0000", 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002998105 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.490 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.393875+0000", 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016784600000000001 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.413317+0000", 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.019442668 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.413333+0000", 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5549999999999999e-05 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.491 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410418+0000", 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.99708491099999996 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410434+0000", 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6307000000000002e-05 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.512587+0000", 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.102152273 2024-04-12T12:03:13.492 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.512669+0000", 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2488e-05 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109324 10.e 10:7f3645e8:::benchmark_data_smithi029_256330_object7558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.416992+0000", 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 495.06513450099999, 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1062547149999999, 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.493 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109324 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.416992+0000", 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.416992+0000", 2024-04-12T12:03:13.494 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.416994+0000", 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4500000000000001e-06 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.416999+0000", 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3249999999999998e-06 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.417013+0000", 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3712e-05 2024-04-12T12:03:13.495 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.417022+0000", 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8200000000000003e-06 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.417072+0000", 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0173999999999998e-05 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.417082+0000", 2024-04-12T12:03:13.496 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8039999999999996e-06 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410469+0000", 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.993387727 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410478+0000", 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5229999999999995e-06 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.497 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.512884+0000", 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1024059900000001 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.512963+0000", 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8622999999999996e-05 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.513378+0000", 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000415258 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.498 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.522757+0000", 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0093790350000000008 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.522816+0000", 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8919999999999998e-05 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.522912+0000", 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.5901000000000002e-05 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.499 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523075+0000", 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016311400000000001 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523137+0000", 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2069000000000005e-05 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523247+0000", 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011027 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.500 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109327 10.e 10:71283bef:::benchmark_data_smithi029_256330_object7561:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.425280+0000", 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 495.05684726999999, 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0983179810000001, 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109327 2024-04-12T12:03:13.501 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.425280+0000", 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.425280+0000", 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.425281+0000", 2024-04-12T12:03:13.502 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.226e-06 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.425285+0000", 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6120000000000001e-06 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.425297+0000", 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1956e-05 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.503 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.425303+0000", 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8329999999999999e-06 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.425363+0000", 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9747999999999998e-05 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.425375+0000", 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2109e-05 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.504 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410505+0000", 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.98512943399999997 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410512+0000", 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9030000000000004e-06 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.516585+0000", 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1060721259999999 2024-04-12T12:03:13.505 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.516675+0000", 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0315000000000005e-05 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.516771+0000", 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6173000000000002e-05 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.506 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.522928+0000", 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0061571200000000003 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.522973+0000", 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5209e-05 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523015+0000", 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1221999999999999e-05 2024-04-12T12:03:13.507 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523497+0000", 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00048255800000000001 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523535+0000", 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.7648999999999998e-05 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523598+0000", 2024-04-12T12:03:13.508 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2787999999999996e-05 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109339 10.d 10:bbf5834d:::benchmark_data_smithi029_256330_object7573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.448763+0000", 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 495.03336383300001, 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.074595542, 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.509 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109339 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.448763+0000", 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.510 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.448763+0000", 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.448765+0000", 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8560000000000001e-06 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.448771+0000", 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2049999999999999e-06 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.511 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.448783+0000", 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2303999999999999e-05 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.448790+0000", 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8580000000000002e-06 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.448841+0000", 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1038000000000002e-05 2024-04-12T12:03:13.512 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.513 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.513 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.513 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:13 smithi087 ceph-mon[89014]: pgmap v2493: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:13.514 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.448849+0000", 2024-04-12T12:03:13.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0010000000000001e-06 2024-04-12T12:03:13.514 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.514 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.514 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.514 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410913+0000", 2024-04-12T12:03:13.514 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.96206330600000001 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410923+0000", 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0516999999999999e-05 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.513045+0000", 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1021219149999999 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.515 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.513172+0000", 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012733500000000001 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3,6", 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.513615+0000", 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00044225800000000001 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.522992+0000", 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0093770079999999992 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.516 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523035+0000", 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3816999999999997e-05 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523110+0000", 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4510000000000003e-05 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523165+0000", 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.5401e-05 2024-04-12T12:03:13.517 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523233+0000", 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7788999999999998e-05 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523359+0000", 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012542400000000001 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.518 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109348 10.e 10:77e4d412:::benchmark_data_smithi029_256330_object7582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.462139+0000", 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 495.019987743, 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.061682292, 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109348 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.519 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.462139+0000", 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.462139+0000", 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.462141+0000", 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7260000000000001e-06 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.520 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.462150+0000", 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.414e-06 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.462162+0000", 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1973999999999999e-05 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.462187+0000", 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4882999999999999e-05 2024-04-12T12:03:13.521 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.462257+0000", 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9597999999999993e-05 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.462267+0000", 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8600000000000005e-06 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.522 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410538+0000", 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.94827181100000002 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410546+0000", 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9480000000000008e-06 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.519510+0000", 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1089639410000001 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.523 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.519666+0000", 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015604400000000001 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.519813+0000", 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000146224 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.522935+0000", 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0031227400000000002 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.524 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523300+0000", 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00036493400000000003 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523453+0000", 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015315000000000001 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523648+0000", 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019467200000000001 2024-04-12T12:03:13.525 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523712+0000", 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4237000000000002e-05 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523821+0000", 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010913600000000001 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.526 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109356 10.e 10:70bb7225:::benchmark_data_smithi029_256330_object7590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.470282+0000", 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 495.01184443400001, 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.07159994, 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109356 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.527 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.470282+0000", 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.470282+0000", 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.528 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.470283+0000", 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5600000000000001e-07 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.470285+0000", 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9319999999999999e-06 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.470291+0000", 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.066e-06 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.529 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.470294+0000", 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.641e-06 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.470334+0000", 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9286999999999998e-05 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.470340+0000", 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7449999999999998e-06 2024-04-12T12:03:13.530 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410573+0000", 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.94023218399999997 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410581+0000", 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9799999999999998e-06 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523881+0000", 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1133000529999999 2024-04-12T12:03:13.531 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523993+0000", 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000112657 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.524111+0000", 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000117674 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531019+0000", 2024-04-12T12:03:13.532 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0069077979999999997 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531065+0000", 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6511000000000002e-05 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541644+0000", 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.010578557000000001 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.533 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541702+0000", 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8239000000000003e-05 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541760+0000", 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7664000000000002e-05 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541882+0000", 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012259599999999999 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.534 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109364 10.d 10:b588226b:::benchmark_data_smithi029_256330_object7598:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.485105+0000", 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 494.99702209700001, 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.046678124, 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.535 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109364 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.485105+0000", 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.485105+0000", 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.536 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.485106+0000", 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.24e-06 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.485112+0000", 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4999999999999999e-06 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.485124+0000", 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2245e-05 2024-04-12T12:03:13.537 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.485131+0000", 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3869999999999997e-06 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.485225+0000", 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3508e-05 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.538 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.485238+0000", 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.332e-05 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410949+0000", 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.92571129600000002 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410958+0000", 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.8189999999999998e-06 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.539 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.517751+0000", 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.106792676 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.517889+0000", 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000137959 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3,6", 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.518033+0000", 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014435699999999999 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.540 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523050+0000", 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0050171870000000002 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523398+0000", 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00034820400000000002 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531532+0000", 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0081336069999999993 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.541 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531582+0000", 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9672000000000001e-05 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531651+0000", 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9394999999999995e-05 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531783+0000", 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013175200000000001 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.542 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109368 10.d 10:b8f0e140:::benchmark_data_smithi029_256330_object7602:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.489026+0000", 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 494.99310118400001, 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0430306869999999, 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109368 2024-04-12T12:03:13.543 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.489026+0000", 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.489026+0000", 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.544 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.489027+0000", 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.685e-06 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.489036+0000", 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4619999999999996e-06 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.489050+0000", 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4242e-05 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.545 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.489059+0000", 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6580000000000008e-06 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.489124+0000", 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5370000000000006e-05 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.546 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.489137+0000", 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2386e-05 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410983+0000", 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.92184680200000002 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410992+0000", 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.2830000000000001e-06 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.547 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.520489+0000", 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1094971659999999 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.520609+0000", 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000120023 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3,6", 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.520687+0000", 2024-04-12T12:03:13.548 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.7781000000000006e-05 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523451+0000", 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002764865 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.528401+0000", 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0049492870000000001 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.549 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531866+0000", 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003465245 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531906+0000", 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0179999999999998e-05 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531956+0000", 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0148999999999997e-05 2024-04-12T12:03:13.550 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.532056+0000", 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000100103 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109375 10.d 10:bf31d025:::benchmark_data_smithi029_256330_object7609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.501345+0000", 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 494.98078199100001, 2024-04-12T12:03:13.551 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0309491120000001, 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109375 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.501345+0000", 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.552 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.501345+0000", 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.501346+0000", 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1009999999999999e-06 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.501351+0000", 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6920000000000001e-06 2024-04-12T12:03:13.553 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.501368+0000", 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7027000000000001e-05 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.501374+0000", 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2419999999999997e-06 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.501438+0000", 2024-04-12T12:03:13.554 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4237000000000002e-05 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.501451+0000", 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2700999999999999e-05 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.411015+0000", 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.90956421499999995 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.555 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.411023+0000", 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.055e-06 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523485+0000", 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.112462295 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523562+0000", 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6693000000000006e-05 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.556 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3,6", 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.523647+0000", 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4757000000000002e-05 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531546+0000", 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0078991319999999997 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531826+0000", 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00028042900000000001 2024-04-12T12:03:13.557 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.532101+0000", 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00027474299999999998 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.532142+0000", 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1118999999999997e-05 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.532189+0000", 2024-04-12T12:03:13.558 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6629999999999999e-05 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.532294+0000", 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010504400000000001 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109377 10.e 10:72fd3f6b:::benchmark_data_smithi029_256330_object7611:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.504084+0000", 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 494.97804289200002, 2024-04-12T12:03:13.559 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0425159509999999, 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109377 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.504084+0000", 2024-04-12T12:03:13.560 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.504084+0000", 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.504085+0000", 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5000000000000002e-07 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.561 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.504091+0000", 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6950000000000003e-06 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.504101+0000", 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7799999999999995e-06 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.504107+0000", 2024-04-12T12:03:13.562 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1070000000000001e-06 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.504171+0000", 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3947000000000003e-05 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.504182+0000", 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0553e-05 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.563 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410607+0000", 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.90642474699999998 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410615+0000", 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0420000000000003e-06 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.527098+0000", 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1164835049999999 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.564 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.527248+0000", 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000149798 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.527387+0000", 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013938700000000001 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541686+0000", 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014298237 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.565 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541897+0000", 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021152700000000001 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541948+0000", 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1001000000000002e-05 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.546385+0000", 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.004437061 2024-04-12T12:03:13.566 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.546489+0000", 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010420399999999999 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.546600+0000", 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011061 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.567 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109384 10.e 10:76e498b8:::benchmark_data_smithi029_256330_object7618:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.511451+0000", 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 494.97067569799998, 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0354854590000002, 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109384 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.568 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.511451+0000", 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.511451+0000", 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.569 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.511453+0000", 2024-04-12T12:03:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7069999999999999e-06 2024-04-12T12:03:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.511460+0000", 2024-04-12T12:03:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1019999999999999e-06 2024-04-12T12:03:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.571 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.511475+0000", 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4946999999999999e-05 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.511484+0000", 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0920000000000004e-06 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.511599+0000", 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000114509 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.572 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.511609+0000", 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0927999999999999e-05 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410641+0000", 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.89903149999999998 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410649+0000", 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9000000000000006e-06 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531096+0000", 2024-04-12T12:03:13.573 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1204467039999999 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531214+0000", 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000118879 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.531353+0000", 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013833500000000001 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541903+0000", 2024-04-12T12:03:13.574 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.01055031 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.542014+0000", 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000110561 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.542068+0000", 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4817e-05 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.575 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.546680+0000", 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.004611052 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.546830+0000", 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015022899999999999 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.546937+0000", 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010688699999999999 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.576 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109386 10.d 10:be8bf412:::benchmark_data_smithi029_256330_object7620:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.512968+0000", 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 494.96915928599998, 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.028475764, 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109386 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.577 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.512968+0000", 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.512968+0000", 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.578 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.512968+0000", 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4399999999999999e-07 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.512974+0000", 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.8270000000000003e-06 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.512984+0000", 2024-04-12T12:03:13.579 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0291e-05 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.512990+0000", 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.84e-06 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.513043+0000", 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2432000000000002e-05 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.580 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.513051+0000", 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8089999999999993e-06 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.411046+0000", 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.89799561000000006 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.411054+0000", 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0129999999999993e-06 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.581 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.528432+0000", 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1173780070000001 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.528547+0000", 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000115228 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3,6", 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.528682+0000", 2024-04-12T12:03:13.582 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013455100000000001 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.535120+0000", 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.006437852 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541082+0000", 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0059624559999999997 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541151+0000", 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8905999999999994e-05 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541253+0000", 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000102056 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541316+0000", 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2360000000000006e-05 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541443+0000", 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000127782 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109402 10.d 10:bdbb8c4f:::benchmark_data_smithi029_256330_object7636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.535006+0000", 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 494.94712057800001, 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0165019869999998, 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.585 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109402 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.535006+0000", 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.586 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.535006+0000", 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.535007+0000", 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0109999999999999e-06 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.535011+0000", 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.2590000000000001e-06 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.587 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.535019+0000", 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1659999999999994e-06 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.535024+0000", 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9690000000000002e-06 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.535071+0000", 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7743e-05 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.588 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.535080+0000", 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0979999999999994e-06 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.411077+0000", 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.87599743299999999 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.411085+0000", 2024-04-12T12:03:13.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9219999999999997e-06 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.532323+0000", 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1212382810000001 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.532429+0000", 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000105618 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3,6", 2024-04-12T12:03:13.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.532548+0000", 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011930999999999999 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541458+0000", 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0089101609999999998 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541515+0000", 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6931000000000001e-05 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.591 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.551255+0000", 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0097398759999999997 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.551308+0000", 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2850999999999999e-05 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.551374+0000", 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6049000000000004e-05 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.592 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.551508+0000", 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013430899999999999 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109410 10.d 10:b6028193:::benchmark_data_smithi029_256330_object7644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.551011+0000", 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 494.93111573200002, 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.00078647, 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.593 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109410 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.551011+0000", 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.594 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.551011+0000", 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.551013+0000", 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3990000000000001e-06 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.551017+0000", 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6840000000000004e-06 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.595 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.551027+0000", 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7799999999999995e-06 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.551034+0000", 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8279999999999997e-06 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.551086+0000", 2024-04-12T12:03:13.596 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1867000000000003e-05 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.551094+0000", 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.6300000000000004e-06 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.411108+0000", 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.86001386099999999 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.597 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.411116+0000", 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0150000000000003e-06 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.535150+0000", 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1240340639999999 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.535276+0000", 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000125525 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3,6", 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.535412+0000", 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013615 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541466+0000", 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0060538299999999996 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541856+0000", 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00038993699999999998 2024-04-12T12:03:13.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.551566+0000", 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0097098770000000004 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.551613+0000", 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7036000000000002e-05 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.551675+0000", 2024-04-12T12:03:13.600 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2773000000000004e-05 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.551798+0000", 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012221399999999999 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109413 10.e 10:71edb41c:::benchmark_data_smithi029_256330_object7647:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.555315+0000", 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 494.92681141999998, 2024-04-12T12:03:13.601 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0057333650000002, 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109413 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.555315+0000", 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.602 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.555315+0000", 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.555316+0000", 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6099999999999999e-07 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.555321+0000", 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.16e-06 2024-04-12T12:03:13.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.555332+0000", 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.099e-05 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.555340+0000", 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1440000000000003e-06 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.555387+0000", 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7683999999999999e-05 2024-04-12T12:03:13.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.555395+0000", 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2089999999999999e-06 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410673+0000", 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.85527885800000003 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.410681+0000", 2024-04-12T12:03:13.605 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9619999999999993e-06 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.538418+0000", 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.127736997 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.538533+0000", 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000114857 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.606 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3,6", 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.538663+0000", 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012941299999999999 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541964+0000", 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003301755 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.542126+0000", 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016116700000000001 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.607 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.546642+0000", 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0045159589999999999 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.560838+0000", 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014196847 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.560905+0000", 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6237999999999998e-05 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.608 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.561049+0000", 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014416400000000001 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:109421 10.d 10:bd1bfb4b:::benchmark_data_smithi029_256330_object7655:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:58.580847+0000", 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 494.90128022699997, 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9723240899999999, 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.609 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 109421 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.580847+0000", 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.580847+0000", 2024-04-12T12:03:13.610 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.580848+0000", 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.345e-06 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.580853+0000", 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5900000000000001e-06 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.611 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.580866+0000", 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2969000000000001e-05 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.580872+0000", 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4049999999999997e-06 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.580929+0000", 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7228999999999997e-05 2024-04-12T12:03:13.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:58.580938+0000", 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.388e-06 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.411140+0000", 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.83020203100000001 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for peered", 2024-04-12T12:03:13.613 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:59.411148+0000", 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1310000000000006e-06 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.538168+0000", 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.127020248 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.538280+0000", 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000111585 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.614 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3,6", 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.538402+0000", 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012197 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.541553+0000", 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0031510290000000001 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.542048+0000", 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000495737 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.615 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.552147+0000", 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.01009821 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.552954+0000", 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000807014 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.553062+0000", 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010880200000000001 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:55:00.553171+0000", 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010840699999999999 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:226578 11.d 11:b3848f36:::benchmark_data_smithi029_285290_object107887:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1344)", 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:01:28.288930+0000", 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 105.193196524, 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.140185802, 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.617 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 226578 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.288930+0000", 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.618 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.288930+0000", 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.288932+0000", 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9290000000000001e-06 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.288942+0000", 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4290000000000007e-06 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.619 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.288963+0000", 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1705e-05 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.288976+0000", 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2223e-05 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.289059+0000", 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3554999999999995e-05 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.289175+0000", 2024-04-12T12:03:13.620 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000116154 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 1,6,7", 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.289279+0000", 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000103288 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.292721+0000", 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034427339999999998 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.621 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.292964+0000", 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00024250000000000001 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.293359+0000", 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00039469100000000002 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.428986+0000", 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.135627794 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.622 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.429029+0000", 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2422999999999997e-05 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:28.429116+0000", 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.7377e-05 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:231708 11.7 11:e714f7c1:::benchmark_data_smithi029_285290_object113017:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1345)", 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:01:37.105233+0000", 2024-04-12T12:03:13.623 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 96.376894187000005, 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.13425673599999999, 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 231708 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105233+0000", 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105233+0000", 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105234+0000", 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.593e-06 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105241+0000", 2024-04-12T12:03:13.625 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6710000000000002e-06 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105255+0000", 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3947999999999999e-05 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105263+0000", 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9019999999999999e-06 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.626 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105330+0000", 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7279000000000004e-05 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105498+0000", 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016747299999999999 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1,7", 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105662+0000", 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000164573 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.627 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.112058+0000", 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0063955110000000004 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.112075+0000", 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7541e-05 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.114948+0000", 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0028726089999999999 2024-04-12T12:03:13.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.239337+0000", 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.124389255 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.239383+0000", 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6037e-05 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.239489+0000", 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000106344 2024-04-12T12:03:13.629 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:231710 11.7 11:ef0267bb:::benchmark_data_smithi029_285290_object113019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1345)", 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:01:37.105772+0000", 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 96.376355223999994, 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.13388647300000001, 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.630 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 231710 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105772+0000", 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105772+0000", 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.631 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105772+0000", 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1800000000000005e-07 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105779+0000", 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6490000000000002e-06 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105791+0000", 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1667e-05 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.632 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.105799+0000", 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9740000000000002e-06 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.108582+0000", 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0027837180000000001 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.108709+0000", 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012622100000000001 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.633 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1,7", 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.108843+0000", 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000134544 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.112119+0000", 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0032757630000000001 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.115002+0000", 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0028826780000000001 2024-04-12T12:03:13.634 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.115019+0000", 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7765e-05 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.239537+0000", 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.12451709 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.239579+0000", 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2760999999999999e-05 2024-04-12T12:03:13.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:37.239658+0000", 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8825000000000005e-05 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:234759 11.e 11:7873593e:::benchmark_data_smithi029_285290_object116068:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1346)", 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:01:42.528476+0000", 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 90.953651011000005, 2024-04-12T12:03:13.636 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.18235386200000001, 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "continuous": false, 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 234759 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.528476+0000", 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.637 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.528476+0000", 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.528478+0000", 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8700000000000001e-06 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.528487+0000", 2024-04-12T12:03:13.638 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2750000000000001e-06 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.528511+0000", 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4456000000000001e-05 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.528522+0000", 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0718e-05 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.528599+0000", 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.6916999999999996e-05 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.528763+0000", 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000163812 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,1,7", 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.528920+0000", 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015658600000000001 2024-04-12T12:03:13.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.532278+0000", 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0033582619999999999 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.532511+0000", 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000233425 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.532878+0000", 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00036697699999999999 2024-04-12T12:03:13.641 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.710631+0000", 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.17775302000000001 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.710696+0000", 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5111999999999996e-05 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:42.710830+0000", 2024-04-12T12:03:13.642 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013343200000000001 2024-04-12T12:03:13.643 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.643 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.643 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.643 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:13.643 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:13.643 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:13.643 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:13 smithi029 ceph-mon[93650]: pgmap v2493: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:13.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:13 smithi029 ceph-mon[97787]: pgmap v2493: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:14.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:13 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T12:03:13.833+0000 7fae24212700 -1 mon.a@0(leader).osd e1363 definitely_dead 0 2024-04-12T12:03:14.362 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.5. 2024-04-12T12:03:14.433 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-04-12T12:03:14.640 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:14.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:14 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/822890519' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2024-04-12T12:03:14.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:14 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/822890519' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2024-04-12T12:03:14.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:14 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/822890519' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2024-04-12T12:03:15.016 INFO:tasks.thrashosds.thrasher:Added osd 5 2024-04-12T12:03:15.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:03:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:03:14] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-04-12T12:03:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:15 smithi087 ceph-mon[89014]: pgmap v2494: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:15 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/822890519' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-04-12T12:03:15.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:15 smithi087 ceph-mon[89014]: osdmap e1364: 8 total, 8 up, 8 in 2024-04-12T12:03:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:15 smithi029 ceph-mon[93650]: pgmap v2494: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:15 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/822890519' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-04-12T12:03:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:15 smithi029 ceph-mon[93650]: osdmap e1364: 8 total, 8 up, 8 in 2024-04-12T12:03:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:15 smithi029 ceph-mon[97787]: pgmap v2494: 83 pgs: 83 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2024-04-12T12:03:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:15 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/822890519' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-04-12T12:03:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:15 smithi029 ceph-mon[97787]: osdmap e1364: 8 total, 8 up, 8 in 2024-04-12T12:03:16.092 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T12:03:16.092 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T12:03:16.092 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:16 smithi087 ceph-mon[89014]: osdmap e1365: 8 total, 8 up, 8 in 2024-04-12T12:03:16.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:16 smithi087 ceph-mon[89014]: osdmap e1366: 8 total, 8 up, 8 in 2024-04-12T12:03:16.755 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-04-12T12:03:16.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:16 smithi029 ceph-mon[93650]: osdmap e1365: 8 total, 8 up, 8 in 2024-04-12T12:03:16.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:16 smithi029 ceph-mon[93650]: osdmap e1366: 8 total, 8 up, 8 in 2024-04-12T12:03:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:16 smithi029 ceph-mon[97787]: osdmap e1365: 8 total, 8 up, 8 in 2024-04-12T12:03:16.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:16 smithi029 ceph-mon[97787]: osdmap e1366: 8 total, 8 up, 8 in 2024-04-12T12:03:16.961 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:17 smithi087 ceph-mon[89014]: pgmap v2497: 83 pgs: 15 peering, 4 unknown, 64 active+clean; 5.4 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:17 smithi087 ceph-mon[89014]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-12T12:03:17.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:17 smithi029 ceph-mon[93650]: pgmap v2497: 83 pgs: 15 peering, 4 unknown, 64 active+clean; 5.4 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:17.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:17 smithi029 ceph-mon[93650]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-12T12:03:17.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:17 smithi029 ceph-mon[97787]: pgmap v2497: 83 pgs: 15 peering, 4 unknown, 64 active+clean; 5.4 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:17.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:17 smithi029 ceph-mon[97787]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-04-12T12:03:18.445 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:18 smithi087 ceph-mon[89014]: osdmap e1367: 8 total, 8 up, 8 in 2024-04-12T12:03:18.565 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:18.565 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T12:03:18.565 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T12:03:18.566 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T12:03:18.566 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:18.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:18 smithi029 ceph-mon[97787]: osdmap e1367: 8 total, 8 up, 8 in 2024-04-12T12:03:18.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:18 smithi029 ceph-mon[93650]: osdmap e1367: 8 total, 8 up, 8 in 2024-04-12T12:03:19.179 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-04-12T12:03:19.385 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:19.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:19 smithi087 ceph-mon[89014]: pgmap v2500: 83 pgs: 1 active+undersized+remapped, 15 peering, 3 unknown, 64 active+clean; 325 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:03:19.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:19 smithi087 ceph-mon[89014]: osdmap e1368: 8 total, 8 up, 8 in 2024-04-12T12:03:19.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:19 smithi029 ceph-mon[93650]: pgmap v2500: 83 pgs: 1 active+undersized+remapped, 15 peering, 3 unknown, 64 active+clean; 325 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:03:19.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:19 smithi029 ceph-mon[93650]: osdmap e1368: 8 total, 8 up, 8 in 2024-04-12T12:03:19.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:19 smithi029 ceph-mon[97787]: pgmap v2500: 83 pgs: 1 active+undersized+remapped, 15 peering, 3 unknown, 64 active+clean; 325 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:03:19.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:19 smithi029 ceph-mon[97787]: osdmap e1368: 8 total, 8 up, 8 in 2024-04-12T12:03:20.018 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-04-12T12:03:20.241 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:20.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:20 smithi029 ceph-mon[93650]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-04-12T12:03:20.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:20 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T12:03:20.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:20 smithi029 ceph-mon[97787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-04-12T12:03:20.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:20 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T12:03:20.707 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:20 smithi087 ceph-mon[89014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-04-12T12:03:20.707 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:20 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T12:03:20.935 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:20.935 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T12:03:20.935 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T12:03:20.935 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T12:03:20.935 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.935 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:28366 10.a 10:5cd39eea:::benchmark_data_smithi029_256330_object28365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:53:29.643641+0000", 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 591.28838664600005, 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50832011300000002, 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 28366 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.936 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643641+0000", 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643641+0000", 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643640+0000", 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.937 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643697+0000", 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6647999999999997e-05 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643707+0000", 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0672999999999999e-05 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643715+0000", 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3989999999999998e-06 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.938 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643764+0000", 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9574999999999999e-05 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.643893+0000", 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012865899999999999 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0", 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644066+0000", 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017302299999999999 2024-04-12T12:03:20.939 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.647119+0000", 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0030533169999999998 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.151849+0000", 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50472942399999998 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.151891+0000", 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2233e-05 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.940 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.151961+0000", 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0022000000000001e-05 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:28371 10.a 10:5449dfcb:::benchmark_data_smithi029_256330_object28370:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:53:29.644317+0000", 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 591.28771014500001, 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.51451874099999995, 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.941 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 28371 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644317+0000", 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.942 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644317+0000", 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644316+0000", 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644387+0000", 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0300000000000001e-05 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.943 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644398+0000", 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1088e-05 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.644405+0000", 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.233e-06 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.645328+0000", 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00092331900000000003 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.944 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.645459+0000", 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013036100000000001 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0", 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.646076+0000", 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00061679499999999995 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:29.647256+0000", 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0011801470000000001 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.945 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.155833+0000", 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50857733599999999 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.158746+0000", 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0029129609999999999 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:53:30.158836+0000", 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.0099999999999995e-05 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.946 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:75499 10.a 10:5ba68729:::benchmark_data_smithi029_256330_object75498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:13.175640+0000", 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 547.75638792699999, 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.74925757500000001, 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75499 2024-04-12T12:03:20.947 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.948 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.948 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.948 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.948 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175640+0000", 2024-04-12T12:03:20.948 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.948 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.948 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.948 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175640+0000", 2024-04-12T12:03:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175638+0000", 2024-04-12T12:03:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.949 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175686+0000", 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7534999999999998e-05 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175698+0000", 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2112e-05 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.175706+0000", 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1259999999999998e-06 2024-04-12T12:03:20.950 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.177660+0000", 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0019538300000000001 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.177802+0000", 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014221200000000001 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:20.951 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.177959+0000", 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015694799999999999 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.188846+0000", 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.010886905000000001 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.193805+0000", 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.004958596 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.952 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.922554+0000", 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.72874928999999999 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.922606+0000", 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2528999999999998e-05 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.924897+0000", 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0022907940000000001 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.953 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:75507 10.a 10:5fb6aa19:::benchmark_data_smithi029_256330_object75506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:13.181734+0000", 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 547.75029316899997, 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.77549217999999998, 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.954 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75507 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181734+0000", 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181734+0000", 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.955 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181726+0000", 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999914 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181773+0000", 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7509999999999997e-05 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181787+0000", 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.345e-05 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.956 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.181795+0000", 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0909999999999993e-06 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.188918+0000", 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0071231200000000001 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.189098+0000", 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018042199999999999 2024-04-12T12:03:20.957 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.189346+0000", 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00024795900000000002 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.191973+0000", 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0026264769999999999 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.958 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.194043+0000", 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0020700580000000001 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.955399+0000", 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.76135565699999996 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.955437+0000", 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8482999999999997e-05 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.959 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.957227+0000", 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001789433 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:75523 10.a 10:58a8ebd4:::benchmark_data_smithi029_256330_object75522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:13.200557+0000", 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 547.73147077399994, 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.80757887900000003, 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.960 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75523 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200557+0000", 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200557+0000", 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.961 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200555+0000", 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200615+0000", 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9916e-05 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200625+0000", 2024-04-12T12:03:20.962 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0291e-05 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200641+0000", 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5931999999999999e-05 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200690+0000", 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8884000000000002e-05 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:20.963 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.200874+0000", 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000183451 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.201075+0000", 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00020140300000000001 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.203290+0000", 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0022150859999999998 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.964 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.204365+0000", 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0010749959999999999 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.962486+0000", 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.758120656 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.962525+0000", 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9167e-05 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:14.008136+0000", 2024-04-12T12:03:20.965 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.045610629 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:75542 10.f 10:f998c42b:::benchmark_data_smithi029_256330_object75541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:13.235436+0000", 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 547.696591872, 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.68735639299999995, 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.966 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75542 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235436+0000", 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235436+0000", 2024-04-12T12:03:20.967 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.969 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.969 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.969 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.969 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235434+0000", 2024-04-12T12:03:20.969 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:20.969 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.969 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.969 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.969 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235481+0000", 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6989e-05 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235494+0000", 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3220999999999999e-05 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235501+0000", 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8369999999999999e-06 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.970 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235534+0000", 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3006999999999998e-05 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235637+0000", 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000102472 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.235734+0000", 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.6926e-05 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.971 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.237928+0000", 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0021942060000000002 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.238542+0000", 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00061416400000000005 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.922715+0000", 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.684172907 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.972 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.922758+0000", 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3232000000000002e-05 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.922792+0000", 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3797000000000002e-05 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:75543 10.a 10:599efe38:::benchmark_data_smithi029_256330_object75542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:03:20.973 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:13.236268+0000", 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 547.69575908399997, 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.77272679899999996, 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 75543 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236268+0000", 2024-04-12T12:03:20.974 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236268+0000", 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236268+0000", 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.975 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236354+0000", 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5798999999999996e-05 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236360+0000", 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0870000000000003e-06 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236364+0000", 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1520000000000002e-06 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:20.976 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236400+0000", 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6399000000000002e-05 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236504+0000", 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010341200000000001 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.236602+0000", 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7769999999999994e-05 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.977 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.238669+0000", 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0020678900000000002 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:13.239329+0000", 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00065918599999999997 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:14.008190+0000", 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.76886104099999997 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.978 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:14.008231+0000", 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0920000000000001e-05 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:14.008995+0000", 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00076463300000000002 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.979 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155253 10.f 10:fbaad049:::benchmark_data_smithi029_256330_object53487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:07.211877+0000", 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 433.72015071800001, 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.40654251600000002, 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155253 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211877+0000", 2024-04-12T12:03:20.980 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211877+0000", 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211876+0000", 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.981 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211880+0000", 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2230000000000001e-06 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211887+0000", 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.348e-06 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.211892+0000", 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9049999999999996e-06 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:20.982 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.213491+0000", 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001599523 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.213583+0000", 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2151000000000006e-05 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.213663+0000", 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0251000000000001e-05 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:20.983 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.216046+0000", 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023823440000000002 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.230337+0000", 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014290792 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.618319+0000", 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.38798262500000003 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:20.984 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.618374+0000", 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4818999999999998e-05 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.618419+0000", 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5266000000000003e-05 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155260 10.f 10:fcffd604:::benchmark_data_smithi029_256330_object53494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:07.223957+0000", 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 433.708070169, 2024-04-12T12:03:20.985 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.40551721499999999, 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155260 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223957+0000", 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.986 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223957+0000", 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223956+0000", 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223962+0000", 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2120000000000002e-06 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.987 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223969+0000", 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2429999999999999e-06 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223975+0000", 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1360000000000002e-06 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.223998+0000", 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.2841e-05 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.988 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.224097+0000", 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9648999999999994e-05 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.224178+0000", 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0876000000000003e-05 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.227210+0000", 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003031499 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.989 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.230617+0000", 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034070440000000001 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.629310+0000", 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.39869354800000001 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.629418+0000", 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000107297 2024-04-12T12:03:20.990 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:07.629475+0000", 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6833000000000003e-05 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155858 10.a 10:5f54eb39:::benchmark_data_smithi029_256330_object54092:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:08.359538+0000", 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 432.57248944700001, 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.386920337, 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.991 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155858 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359538+0000", 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.992 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359538+0000", 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359537+0000", 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359544+0000", 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.849e-06 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.993 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359557+0000", 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3485999999999999e-05 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359565+0000", 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8199999999999997e-06 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359610+0000", 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4845999999999997e-05 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.994 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359743+0000", 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000133566 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.359879+0000", 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000135952 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.363523+0000", 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0036437639999999999 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.995 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.367054+0000", 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00353105 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.746372+0000", 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.37931752099999999 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.746416+0000", 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4827000000000003e-05 2024-04-12T12:03:20.996 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.746458+0000", 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1902e-05 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155867 10.a 10:563170cd:::benchmark_data_smithi029_256330_object54101:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:08.369275+0000", 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 432.56275251, 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.38112349600000001, 2024-04-12T12:03:20.997 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155867 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369275+0000", 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.998 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369275+0000", 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369274+0000", 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369279+0000", 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1220000000000003e-06 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:20.999 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369289+0000", 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2890000000000003e-06 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369295+0000", 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4130000000000003e-06 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369325+0000", 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.9643000000000001e-05 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.000 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369419+0000", 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4534e-05 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.369501+0000", 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1775999999999998e-05 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.372239+0000", 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002737803 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.001 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.372630+0000", 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000391612 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.750289+0000", 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.3776583 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.750354+0000", 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5726000000000001e-05 2024-04-12T12:03:21.002 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.003 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.003 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:21.003 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.750398+0000", 2024-04-12T12:03:21.003 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4161e-05 2024-04-12T12:03:21.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.003 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.003 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.003 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.003 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.003 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157799 10.f 10:fd68811f:::benchmark_data_smithi029_256330_object56033:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:21.004 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.713914+0000", 2024-04-12T12:03:21.004 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 429.21811341099999, 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.408914153, 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157799 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713914+0000", 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.005 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713914+0000", 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713913+0000", 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.006 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713920+0000", 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.9990000000000002e-06 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713935+0000", 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5463000000000001e-05 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713944+0000", 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.799e-06 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.007 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.713983+0000", 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9019e-05 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.714102+0000", 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000119529 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.714226+0000", 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000123965 2024-04-12T12:03:21.008 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.717724+0000", 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034974189999999999 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.723864+0000", 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0061404440000000001 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122732+0000", 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.39886750599999998 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.009 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122785+0000", 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3251999999999998e-05 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122828+0000", 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.3248000000000003e-05 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.010 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157802 10.f 10:fc076b5f:::benchmark_data_smithi029_256330_object56036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.718855+0000", 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 429.21317279099998, 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.404126121, 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157802 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:21.011 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718855+0000", 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718855+0000", 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718854+0000", 2024-04-12T12:03:21.012 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718858+0000", 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.6460000000000001e-06 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718867+0000", 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.4819999999999994e-06 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.013 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718872+0000", 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3000000000000001e-06 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718910+0000", 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.803e-05 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.719009+0000", 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9062999999999997e-05 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.014 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,3", 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.719088+0000", 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8380000000000005e-05 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.721914+0000", 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002826631 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.728088+0000", 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0061736009999999999 2024-04-12T12:03:21.015 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122875+0000", 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.39478680799999999 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122945+0000", 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.0653000000000003e-05 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122981+0000", 2024-04-12T12:03:21.016 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5395999999999998e-05 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157803 10.a 10:55552f02:::benchmark_data_smithi029_256330_object56037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.718879+0000", 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 429.21314869100001, 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.40287460899999999, 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:21.017 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157803 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718879+0000", 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718879+0000", 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.018 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718878+0000", 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718881+0000", 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.447e-06 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:21.019 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718888+0000", 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.6420000000000001e-06 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718893+0000", 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4809999999999997e-06 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718945+0000", 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2340999999999999e-05 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.020 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.719066+0000", 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000120307 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.719172+0000", 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000106498 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.725265+0000", 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0060925709999999998 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.021 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.730326+0000", 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0050610560000000004 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121641+0000", 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.39131515100000003 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121706+0000", 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4937000000000005e-05 2024-04-12T12:03:21.022 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121753+0000", 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7559999999999999e-05 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157804 10.a 10:5305d031:::benchmark_data_smithi029_256330_object56038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.718899+0000", 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 429.21312839199999, 2024-04-12T12:03:21.023 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.40304277399999999, 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157804 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718899+0000", 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.024 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718899+0000", 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718899+0000", 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718901+0000", 2024-04-12T12:03:21.025 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5600000000000001e-06 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718908+0000", 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.8120000000000002e-06 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.718913+0000", 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0200000000000002e-06 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.721883+0000", 2024-04-12T12:03:21.026 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.002970014 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.721991+0000", 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000107551 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.722093+0000", 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00010247 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.027 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.725415+0000", 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0033219310000000002 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.730367+0000", 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0049522259999999997 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121818+0000", 2024-04-12T12:03:21.028 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.39145069100000002 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121902+0000", 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.3788999999999993e-05 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121942+0000", 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0051e-05 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.029 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:157822 10.a 10:5a13f56b:::benchmark_data_smithi029_256330_object56056:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:11.740827+0000", 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 429.19120067699998, 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.38124880799999999, 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 157822 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.030 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740827+0000", 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740827+0000", 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740825+0000", 2024-04-12T12:03:21.031 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740832+0000", 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7089999999999996e-06 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740843+0000", 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1413000000000001e-05 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:21.032 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740850+0000", 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3600000000000001e-06 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.740895+0000", 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5597999999999999e-05 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.741056+0000", 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016123 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.033 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.741168+0000", 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000111935 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.744393+0000", 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0032241399999999999 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:11.746021+0000", 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0016283929999999999 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.034 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.121988+0000", 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.37596682100000001 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122036+0000", 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8723000000000001e-05 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:12.122076+0000", 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9180000000000001e-05 2024-04-12T12:03:21.035 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42321 11.3 11:c1daa23e:::benchmark_data_smithi029_285290_object42320:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:15.208225+0000", 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 305.723802982, 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.35528464999999998, 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:21.036 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42321 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.208225+0000", 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.208225+0000", 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:21.037 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.208223+0000", 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.208258+0000", 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5654000000000001e-05 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.208273+0000", 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4847e-05 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.038 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.208281+0000", 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.7179999999999996e-06 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.208325+0000", 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4428000000000002e-05 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.208437+0000", 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000111789 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.039 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,7", 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.208577+0000", 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013963699999999999 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.210399+0000", 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001821862 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.210961+0000", 2024-04-12T12:03:21.040 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00056230199999999998 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.563417+0000", 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.35245596099999998 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.563468+0000", 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0985000000000001e-05 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.563509+0000", 2024-04-12T12:03:21.041 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1176e-05 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42325 11.3 11:c5222d3b:::benchmark_data_smithi029_285290_object42324:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:15.211130+0000", 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 305.720897476, 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.35274655500000002, 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:21.042 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42325 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211130+0000", 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211130+0000", 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.043 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211128+0000", 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211179+0000", 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0997e-05 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211229+0000", 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9716000000000003e-05 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.044 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211244+0000", 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5038999999999999e-05 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211326+0000", 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.1775999999999998e-05 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211455+0000", 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012900499999999999 2024-04-12T12:03:21.045 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,7", 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211567+0000", 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000111733 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.213042+0000", 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001475376 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.046 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.213508+0000", 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00046625099999999999 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.563763+0000", 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.35025499399999999 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.563836+0000", 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2482000000000002e-05 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.047 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.563877+0000", 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0861000000000001e-05 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:113798 11.1 11:81fdce62:::benchmark_data_smithi029_285290_object113797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:59:17.437163+0000", 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 243.49486415199999, 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50584712499999995, 2024-04-12T12:03:21.048 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 113798 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.437163+0000", 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.049 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.437163+0000", 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.437159+0000", 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999957 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.437211+0000", 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1996000000000002e-05 2024-04-12T12:03:21.050 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.437232+0000", 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0557e-05 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.437239+0000", 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7299999999999999e-06 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:21.051 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.437273+0000", 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4072999999999997e-05 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.437385+0000", 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000112779 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,3", 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.437509+0000", 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012360299999999999 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.441625+0000", 2024-04-12T12:03:21.052 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0041163629999999996 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.445514+0000", 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0038888619999999999 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.942940+0000", 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.49742573499999998 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:21.053 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.942982+0000", 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2239e-05 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.943010+0000", 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.8303e-05 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:113802 11.3 11:c3a10797:::benchmark_data_smithi029_285290_object113801:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:59:17.442411+0000", 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 243.489616657, 2024-04-12T12:03:21.054 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50507740000000001, 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 113802 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.442411+0000", 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.442411+0000", 2024-04-12T12:03:21.055 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.442410+0000", 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.442459+0000", 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.9774999999999997e-05 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.442471+0000", 2024-04-12T12:03:21.056 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1813e-05 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.442481+0000", 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9760000000000007e-06 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.442526+0000", 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5309999999999998e-05 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.442665+0000", 2024-04-12T12:03:21.057 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000138446 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,7", 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.442821+0000", 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000156312 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.446679+0000", 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.003857606 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:21.058 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447609+0000", 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000930086 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.947412+0000", 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.49980340200000001 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.947448+0000", 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5967999999999999e-05 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.947488+0000", 2024-04-12T12:03:21.059 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0049999999999998e-05 2024-04-12T12:03:21.060 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.060 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.060 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.060 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:21.060 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:21.060 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:21.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:21 smithi029 ceph-mon[97787]: pgmap v2502: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 337 KiB/s, 3 objects/s recovering 2024-04-12T12:03:21.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:21.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:21 smithi029 ceph-mon[93650]: pgmap v2502: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 337 KiB/s, 3 objects/s recovering 2024-04-12T12:03:21.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:21 smithi087 ceph-mon[89014]: pgmap v2502: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 337 KiB/s, 3 objects/s recovering 2024-04-12T12:03:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:21.831 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-04-12T12:03:22.033 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:22.044 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a: {} 2024-04-12T12:03:22.054 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b: {} 2024-04-12T12:03:22.064 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c: {} 2024-04-12T12:03:22.065 INFO:teuthology.orchestra.run.smithi029.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-04-12T12:03:22.065 INFO:teuthology.orchestra.run.smithi029.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-04-12T12:03:22.065 INFO:teuthology.orchestra.run.smithi029.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-04-12T12:03:22.427 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:03:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:03:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:03:22.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:22 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1271817869' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T12:03:22.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:22 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1271817869' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T12:03:22.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:22 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1271817869' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T12:03:23.161 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 0 1 2024-04-12T12:03:23.374 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:23.663 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:23 smithi087 ceph-mon[89014]: pgmap v2503: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 278 KiB/s, 2 objects/s recovering 2024-04-12T12:03:23.695 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:23.695 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T12:03:23.695 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T12:03:23.695 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:23.727 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:23 smithi029 ceph-mon[93650]: pgmap v2503: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 278 KiB/s, 2 objects/s recovering 2024-04-12T12:03:23.728 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:23 smithi029 ceph-mon[97787]: pgmap v2503: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 278 KiB/s, 2 objects/s recovering 2024-04-12T12:03:24.340 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-04-12T12:03:24.548 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:25.073 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:03:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:03:24] "GET /metrics HTTP/1.1" 200 34150 "" "Prometheus/2.43.0" 2024-04-12T12:03:25.444 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.0 to 1 (10000) 2024-04-12T12:03:25.649 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:25 smithi087 ceph-mon[89014]: pgmap v2504: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 210 KiB/s, 1 objects/s recovering 2024-04-12T12:03:25.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-12T12:03:25.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-12T12:03:25.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:25 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-12T12:03:25.650 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:25 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3412743833' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:03:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:25 smithi029 ceph-mon[97787]: pgmap v2504: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 210 KiB/s, 1 objects/s recovering 2024-04-12T12:03:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-12T12:03:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-12T12:03:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:25 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-12T12:03:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:25 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3412743833' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:03:25.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:25 smithi029 ceph-mon[93650]: pgmap v2504: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 210 KiB/s, 1 objects/s recovering 2024-04-12T12:03:25.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-12T12:03:25.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-04-12T12:03:25.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:25 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-12T12:03:25.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:25 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3412743833' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:03:26.111 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 0 1 2024-04-12T12:03:26.156 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:26.156 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T12:03:26.156 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T12:03:26.156 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T12:03:26.156 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:26.325 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:26.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:26 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-12T12:03:26.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:26 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-12T12:03:26.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:26 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-12T12:03:26.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:26 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3412743833' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-04-12T12:03:26.628 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:26 smithi029 ceph-mon[93650]: osdmap e1369: 8 total, 8 up, 8 in 2024-04-12T12:03:26.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:26 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-12T12:03:26.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:26 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-12T12:03:26.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:26 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-12T12:03:26.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:26 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3412743833' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-04-12T12:03:26.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:26 smithi029 ceph-mon[97787]: osdmap e1369: 8 total, 8 up, 8 in 2024-04-12T12:03:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:26 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-12T12:03:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:26 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-04-12T12:03:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:26 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-12T12:03:26.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:26 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3412743833' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-04-12T12:03:26.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:26 smithi087 ceph-mon[89014]: osdmap e1369: 8 total, 8 up, 8 in 2024-04-12T12:03:26.786 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-04-12T12:03:26.992 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:27.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:27 smithi029 ceph-mon[93650]: pgmap v2506: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 128 KiB/s, 1 objects/s recovering 2024-04-12T12:03:27.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:27 smithi029 ceph-mon[93650]: osdmap e1370: 8 total, 8 up, 8 in 2024-04-12T12:03:27.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:27 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:27.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:27 smithi029 ceph-mon[97787]: pgmap v2506: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 128 KiB/s, 1 objects/s recovering 2024-04-12T12:03:27.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:27 smithi029 ceph-mon[97787]: osdmap e1370: 8 total, 8 up, 8 in 2024-04-12T12:03:27.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:27 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:27 smithi087 ceph-mon[89014]: pgmap v2506: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 128 KiB/s, 1 objects/s recovering 2024-04-12T12:03:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:27 smithi087 ceph-mon[89014]: osdmap e1370: 8 total, 8 up, 8 in 2024-04-12T12:03:27.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:27 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:28.455 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-04-12T12:03:28.581 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:28.581 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T12:03:28.581 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T12:03:28.581 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T12:03:28.581 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.581 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:74789 10.c 10:36b2b576:::benchmark_data_smithi029_256330_object74788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:11.764574+0000", 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 556.81310988099995, 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29244159600000003, 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 74789 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.582 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764574+0000", 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764574+0000", 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.583 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764572+0000", 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999976 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764608+0000", 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6573999999999999e-05 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764625+0000", 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6806000000000001e-05 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.584 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764635+0000", 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4700000000000008e-06 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764698+0000", 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2918999999999998e-05 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.764886+0000", 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00018877799999999999 2024-04-12T12:03:28.585 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.765077+0000", 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019037200000000001 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.771075+0000", 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0059978820000000004 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.586 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.771789+0000", 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00071427399999999999 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.056919+0000", 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.28513039400000001 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.056973+0000", 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3940999999999997e-05 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.587 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.057016+0000", 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2777000000000001e-05 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:74793 10.c 10:3a699fa9:::benchmark_data_smithi029_256330_object74792:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:54:11.767667+0000", 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 556.81001707600001, 2024-04-12T12:03:28.588 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.28946128399999999, 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 74793 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767667+0000", 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.589 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767667+0000", 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767661+0000", 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999938 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767707+0000", 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5501999999999999e-05 2024-04-12T12:03:28.590 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767722+0000", 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4946999999999999e-05 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767731+0000", 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.4150000000000005e-06 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.767949+0000", 2024-04-12T12:03:28.591 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021811500000000001 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.768139+0000", 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000189991 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.768344+0000", 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000204923 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.592 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.773225+0000", 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0048805979999999999 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:11.773244+0000", 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8953999999999998e-05 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.057056+0000", 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.28381212099999997 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.593 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.057092+0000", 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6514000000000003e-05 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:54:12.057129+0000", 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6424000000000002e-05 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.594 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:155857 10.c 10:35522eab:::benchmark_data_smithi029_256330_object54091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:28.595 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:08.357014+0000", 2024-04-12T12:03:28.595 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 440.22067016800003, 2024-04-12T12:03:28.595 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30658700100000003, 2024-04-12T12:03:28.595 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.595 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.595 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.595 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:28.596 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:28.596 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 155857 2024-04-12T12:03:28.596 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.596 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.596 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.596 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.596 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357014+0000", 2024-04-12T12:03:28.596 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.596 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357014+0000", 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357012+0000", 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357022+0000", 2024-04-12T12:03:28.597 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1749999999999994e-06 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357041+0000", 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9150000000000001e-05 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357050+0000", 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7440000000000002e-06 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.598 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357116+0000", 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5269000000000001e-05 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357266+0000", 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015061799999999999 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.357397+0000", 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013073199999999999 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.599 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.360799+0000", 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0034022169999999999 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.363174+0000", 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023745099999999998 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.663507+0000", 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.300333344 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.600 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.663562+0000", 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4648999999999998e-05 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:08.663601+0000", 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9422999999999998e-05 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:156455 10.c 10:32fc95a4:::benchmark_data_smithi029_256330_object54689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:28.601 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:09.508865+0000", 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 439.068819518, 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30657693499999999, 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 156455 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508865+0000", 2024-04-12T12:03:28.602 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508865+0000", 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508863+0000", 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999981 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.603 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508873+0000", 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0380000000000001e-05 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508892+0000", 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8609000000000001e-05 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508903+0000", 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0621e-05 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.604 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.508964+0000", 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.1135000000000003e-05 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.509120+0000", 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000156088 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.509267+0000", 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014736 2024-04-12T12:03:28.605 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523749+0000", 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.014482247 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523920+0000", 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017029299999999999 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815356+0000", 2024-04-12T12:03:28.606 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29143635699999998 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815398+0000", 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.1470000000000001e-05 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815442+0000", 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4196999999999997e-05 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.607 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:156457 10.c 10:3fd59f8a:::benchmark_data_smithi029_256330_object54691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:09.511391+0000", 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 439.06629345300001, 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30413762900000002, 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 156457 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.608 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511391+0000", 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511391+0000", 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511389+0000", 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.609 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511401+0000", 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1233000000000001e-05 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511420+0000", 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.9307999999999999e-05 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.610 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511430+0000", 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0479000000000001e-05 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512220+0000", 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00078960399999999996 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512340+0000", 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000119793 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.611 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512499+0000", 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015880500000000001 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523780+0000", 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.011281028 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.612 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523925+0000", 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000145093 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815482+0000", 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29155695799999998 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815508+0000", 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5901000000000002e-05 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.613 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815529+0000", 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1050999999999999e-05 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:156459 10.c 10:31889fff:::benchmark_data_smithi029_256330_object54693:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:09.511443+0000", 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 439.06624106599998, 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.30423812099999997, 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.614 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 156459 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511443+0000", 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511443+0000", 2024-04-12T12:03:28.615 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511443+0000", 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511446+0000", 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5269999999999999e-06 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.616 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511454+0000", 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.2819999999999999e-06 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.511459+0000", 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.93e-06 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.515749+0000", 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0042892879999999996 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.617 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.515965+0000", 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021580199999999999 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.516108+0000", 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014346500000000001 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523911+0000", 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0078030189999999996 2024-04-12T12:03:28.618 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.523932+0000", 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1024999999999999e-05 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815579+0000", 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29164664299999998 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815642+0000", 2024-04-12T12:03:28.619 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3788000000000007e-05 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815681+0000", 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8970000000000001e-05 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.35727.0:156462 10.c 10:314f156c:::benchmark_data_smithi029_256330_object54696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:56:09.512595+0000", 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 439.065089027, 2024-04-12T12:03:28.620 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.303229625, 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.35727", 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/1743114763", 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 156462 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512595+0000", 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.621 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512595+0000", 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512595+0000", 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512599+0000", 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2780000000000004e-06 2024-04-12T12:03:28.622 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512613+0000", 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3898e-05 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.512649+0000", 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6001000000000003e-05 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.519506+0000", 2024-04-12T12:03:28.623 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0068574969999999997 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.519626+0000", 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011934000000000001 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,4", 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.519743+0000", 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000117133 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.624 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.524359+0000", 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0046159110000000003 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.525641+0000", 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001282163 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.625 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815734+0000", 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.29009288300000002 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815785+0000", 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0871000000000001e-05 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:56:09.815825+0000", 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0399999999999999e-05 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.626 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42292 11.f 11:f84fe9a4:::benchmark_data_smithi029_285290_object42291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:14.982397+0000", 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 313.595287025, 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.57496645999999996, 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.627 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42292 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982397+0000", 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982397+0000", 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.628 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982396+0000", 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982432+0000", 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5750000000000002e-05 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982443+0000", 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1129e-05 2024-04-12T12:03:28.629 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982450+0000", 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.5560000000000004e-06 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982732+0000", 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00028165900000000002 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.630 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.982870+0000", 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000138663 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983071+0000", 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00020081499999999999 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988302+0000", 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0052308739999999999 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.631 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988376+0000", 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.3998000000000005e-05 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557253+0000", 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.56887719000000003 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557316+0000", 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3207999999999995e-05 2024-04-12T12:03:28.632 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557364+0000", 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7539e-05 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42294 11.f 11:f38b9eb2:::benchmark_data_smithi029_285290_object42293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:14.983465+0000", 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 313.594219227, 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.57400737099999999, 2024-04-12T12:03:28.633 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42294 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983465+0000", 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.634 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983465+0000", 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983464+0000", 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983567+0000", 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000102444 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.635 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983576+0000", 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1109999999999997e-06 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983584+0000", 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5889999999999997e-06 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984590+0000", 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0010066929999999999 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.636 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984742+0000", 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015193199999999999 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984898+0000", 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00015533699999999999 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.637 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988417+0000", 2024-04-12T12:03:28.638 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:28 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4026665630' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:03:28.638 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:28 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:03:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0035192769999999999 2024-04-12T12:03:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988425+0000", 2024-04-12T12:03:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5490000000000006e-06 2024-04-12T12:03:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.639 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557414+0000", 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.56898828000000001 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557447+0000", 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3859000000000002e-05 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557473+0000", 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5041e-05 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.640 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42299 11.f 11:f26ba9ae:::benchmark_data_smithi029_285290_object42298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:14.983989+0000", 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 313.59369535299999, 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.57362241999999997, 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42299 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.641 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983989+0000", 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983989+0000", 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.983988+0000", 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:28.642 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984062+0000", 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4051e-05 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984078+0000", 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5931e-05 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984086+0000", 2024-04-12T12:03:28.643 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.8059999999999995e-06 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.986408+0000", 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023222999999999998 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.986530+0000", 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012160299999999999 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.644 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.986664+0000", 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013399199999999999 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988442+0000", 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0017779790000000001 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.990171+0000", 2024-04-12T12:03:28.645 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001729284 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557527+0000", 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.56735576799999998 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557575+0000", 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8019000000000002e-05 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.646 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557611+0000", 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.6665999999999998e-05 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42301 11.f 11:f820092e:::benchmark_data_smithi029_285290_object42300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:14.984094+0000", 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 313.59359086400002, 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.57360671299999999, 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.647 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42301 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984094+0000", 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.648 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984094+0000", 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984093+0000", 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984145+0000", 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2117999999999998e-05 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.649 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984154+0000", 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.2e-06 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.984160+0000", 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.3619999999999997e-06 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988465+0000", 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0043049309999999997 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.650 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988603+0000", 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000138465 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.988748+0000", 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000145038 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.990321+0000", 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001573004 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.651 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:14.990829+0000", 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00050818399999999996 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557648+0000", 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.56681834499999995 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557675+0000", 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.7061000000000001e-05 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.652 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557700+0000", 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5613e-05 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42313 11.f 11:f2a54442:::benchmark_data_smithi029_285290_object42312:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:15.203953+0000", 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 313.37373162900002, 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.35387521399999999, 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.653 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42313 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.203953+0000", 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.654 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.203953+0000", 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.203948+0000", 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999957 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204009+0000", 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.0476999999999997e-05 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204023+0000", 2024-04-12T12:03:28.655 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3740000000000001e-05 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204030+0000", 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4320000000000002e-06 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204068+0000", 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8158000000000003e-05 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.656 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204185+0000", 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000116706 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.204306+0000", 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000120802 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.206658+0000", 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023524689999999998 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.657 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.207824+0000", 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.001166203 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557738+0000", 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34991402999999999 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557786+0000", 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.7219000000000003e-05 2024-04-12T12:03:28.658 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.557828+0000", 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.2357000000000001e-05 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42326 11.f 11:f45f1e17:::benchmark_data_smithi029_285290_object42325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:15.211209+0000", 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 313.36647557800001, 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34961550800000002, 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.659 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42326 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211209+0000", 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211209+0000", 2024-04-12T12:03:28.660 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211208+0000", 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211258+0000", 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0544999999999998e-05 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211269+0000", 2024-04-12T12:03:28.661 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0762e-05 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211277+0000", 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.7409999999999992e-06 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211333+0000", 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6910000000000002e-05 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.662 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211478+0000", 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000144766 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211571+0000", 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3009000000000003e-05 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.214394+0000", 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0028226839999999998 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.663 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.215203+0000", 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00080880299999999997 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.560735+0000", 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34553186200000002 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.560785+0000", 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0766000000000001e-05 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.664 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.560824+0000", 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8940000000000003e-05 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:42329 11.f 11:f2d1521d:::benchmark_data_smithi029_285290_object42328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:58:15.211568+0000", 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 313.36611655899998, 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34974637800000002, 2024-04-12T12:03:28.665 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 42329 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211568+0000", 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.666 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211568+0000", 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211567+0000", 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211630+0000", 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.2470000000000003e-05 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.667 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211640+0000", 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0207000000000001e-05 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.211647+0000", 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.1509999999999998e-06 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.213167+0000", 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0015203879999999999 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.668 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.213296+0000", 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012849700000000001 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 0,2", 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.213395+0000", 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.9518999999999994e-05 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.214502+0000", 2024-04-12T12:03:28.669 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0011064580000000001 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.215439+0000", 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00093702200000000003 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.561245+0000", 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.34580587099999999 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.670 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.561277+0000", 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.2107000000000003e-05 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:58:15.561314+0000", 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.7336999999999998e-05 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:113780 11.4 11:2796d0fb:::benchmark_data_smithi029_285290_object113779:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:59:17.425287+0000", 2024-04-12T12:03:28.671 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 251.15239763899999, 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.52363299600000002, 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 113780 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425287+0000", 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.672 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425287+0000", 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425286+0000", 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425343+0000", 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7195999999999999e-05 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.673 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425353+0000", 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0541e-05 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425359+0000", 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.7080000000000002e-06 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425393+0000", 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.3859000000000002e-05 2024-04-12T12:03:28.674 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425506+0000", 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000112747 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.425609+0000", 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000103063 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.429488+0000", 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0038795090000000002 2024-04-12T12:03:28.675 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.431822+0000", 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0023336070000000001 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.948860+0000", 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.517037882 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.948895+0000", 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.4696000000000001e-05 2024-04-12T12:03:28.676 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.948920+0000", 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.5208999999999999e-05 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:113804 11.5 11:aaaf7d84:::benchmark_data_smithi029_285290_object113803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:59:17.443513+0000", 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 251.13417167099999, 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50833982200000005, 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.677 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 113804 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443513+0000", 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.678 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443513+0000", 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443511+0000", 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443552+0000", 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.0726999999999998e-05 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.679 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443563+0000", 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0755e-05 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443569+0000", 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3380000000000001e-06 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.443621+0000", 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1409000000000002e-05 2024-04-12T12:03:28.680 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.446090+0000", 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0024690810000000001 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 3,6", 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.446283+0000", 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019277500000000001 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.681 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450136+0000", 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0038537710000000002 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450272+0000", 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00013551200000000001 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.951798+0000", 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.501526307 2024-04-12T12:03:28.682 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.951829+0000", 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.1095e-05 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.951853+0000", 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3329999999999999e-05 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.683 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:113807 11.4 11:25f36c58:::benchmark_data_smithi029_285290_object113806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:59:17.447507+0000", 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 251.130176992, 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50804887099999996, 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 113807 2024-04-12T12:03:28.684 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447507+0000", 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447507+0000", 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.685 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447506+0000", 2024-04-12T12:03:28.686 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:28.686 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.686 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.686 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.686 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447554+0000", 2024-04-12T12:03:28.686 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.8733999999999998e-05 2024-04-12T12:03:28.686 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.686 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.686 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.686 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447573+0000", 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.8712999999999999e-05 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447583+0000", 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3829999999999998e-06 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447674+0000", 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1161000000000004e-05 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.687 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.447835+0000", 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000161466 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.448016+0000", 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000180707 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.453806+0000", 2024-04-12T12:03:28.688 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0057896070000000004 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.453834+0000", 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.8801000000000001e-05 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955467+0000", 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50163289700000002 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.689 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955518+0000", 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0488000000000002e-05 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955556+0000", 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.8535000000000003e-05 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.690 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:113808 11.4 11:23337bbc:::benchmark_data_smithi029_285290_object113807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1325)", 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T11:59:17.450659+0000", 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 251.127025156, 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50499450800000001, 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 113808 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.691 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450659+0000", 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450659+0000", 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450658+0000", 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:28.692 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450710+0000", 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.2609999999999999e-05 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450724+0000", 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3519999999999999e-05 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.450732+0000", 2024-04-12T12:03:28.693 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.9189999999999999e-06 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.451102+0000", 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00037062999999999999 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.451297+0000", 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019469299999999999 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.694 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.451401+0000", 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000103799 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.453953+0000", 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0025525299999999999 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.454033+0000", 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.0116000000000003e-05 2024-04-12T12:03:28.695 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955596+0000", 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.50156283599999996 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955632+0000", 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5729000000000003e-05 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T11:59:17.955654+0000", 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.1715000000000001e-05 2024-04-12T12:03:28.696 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.45800.0:235470 11.4 11:2e710a53:::benchmark_data_smithi029_285290_object116779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1346)", 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:01:44.052184+0000", 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 104.52549999599999, 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.14990488499999999, 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.45800", 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3582356312", 2024-04-12T12:03:28.697 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 235470 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.052184+0000", 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.052184+0000", 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.698 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.052184+0000", 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.052187+0000", 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.6199999999999999e-06 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.052193+0000", 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.178e-06 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.699 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.052198+0000", 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.0649999999999998e-06 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.058919+0000", 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0067212419999999997 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.059048+0000", 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00012900300000000001 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.700 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "waiting for subops from 2,6", 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.059168+0000", 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000119467 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "op_commit", 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.063561+0000", 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0043930169999999999 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.066180+0000", 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.0026195580000000001 2024-04-12T12:03:28.701 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "sub_op_commit_rec", 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.202025+0000", 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.13584474699999999 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "commit_sent", 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.202060+0000", 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.5627999999999998e-05 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:01:44.202089+0000", 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.8796e-05 2024-04-12T12:03:28.702 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.703 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.703 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.703 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:28.703 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:28.703 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:28.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:28 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4026665630' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:03:28.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:28 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:03:28.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:28 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4026665630' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:03:28.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:28 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2024-04-12T12:03:29.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 1 1 2024-04-12T12:03:29.367 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:29.478 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-04-12T12:03:29.681 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:29.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:29 smithi087 ceph-mon[89014]: pgmap v2508: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:29.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:29 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-04-12T12:03:29.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:29 smithi087 ceph-mon[89014]: osdmap e1371: 8 total, 8 up, 8 in 2024-04-12T12:03:29.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:29 smithi029 ceph-mon[97787]: pgmap v2508: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:29.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:29 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-04-12T12:03:29.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:29 smithi029 ceph-mon[97787]: osdmap e1371: 8 total, 8 up, 8 in 2024-04-12T12:03:29.742 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:29 smithi029 ceph-mon[93650]: pgmap v2508: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:29.742 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:29 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-04-12T12:03:29.742 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:29 smithi029 ceph-mon[93650]: osdmap e1371: 8 total, 8 up, 8 in 2024-04-12T12:03:31.291 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:31.292 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T12:03:31.292 INFO:teuthology.orchestra.run.smithi087.stdout: "num_ops": 0 2024-04-12T12:03:31.292 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:31.458 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.1 to 1 (10000) 2024-04-12T12:03:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:31 smithi087 ceph-mon[89014]: pgmap v2510: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:31.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:31 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1681143623' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:31.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:31 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:31 smithi029 ceph-mon[93650]: pgmap v2510: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:31 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1681143623' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:31 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:31 smithi029 ceph-mon[97787]: pgmap v2510: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:31 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1681143623' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:31 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:31.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-04-12T12:03:32.183 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:32.242 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 1 1 2024-04-12T12:03:32.472 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:32.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:32 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-04-12T12:03:32.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:32 smithi029 ceph-mon[93650]: osdmap e1372: 8 total, 8 up, 8 in 2024-04-12T12:03:32.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:32 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-04-12T12:03:32.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:32 smithi029 ceph-mon[97787]: osdmap e1372: 8 total, 8 up, 8 in 2024-04-12T12:03:32.554 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:03:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:03:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:03:32.555 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:32 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-04-12T12:03:32.555 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:32 smithi087 ceph-mon[89014]: osdmap e1372: 8 total, 8 up, 8 in 2024-04-12T12:03:33.557 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:33 smithi087 ceph-mon[89014]: pgmap v2512: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:33.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:33 smithi029 ceph-mon[93650]: pgmap v2512: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:33.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:33 smithi029 ceph-mon[97787]: pgmap v2512: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:33.797 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:33.797 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [], 2024-04-12T12:03:33.797 INFO:teuthology.orchestra.run.smithi087.stdout: "complaint_time": 30, 2024-04-12T12:03:33.797 INFO:teuthology.orchestra.run.smithi087.stdout: "num_blocked_ops": 0 2024-04-12T12:03:33.798 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:34.428 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-04-12T12:03:34.475 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-04-12T12:03:34.637 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.b/config 2024-04-12T12:03:34.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:34 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/206570728' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:34.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:34 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/206570728' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:34.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:34 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/206570728' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2024-04-12T12:03:35.055 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:03:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:03:34] "GET /metrics HTTP/1.1" 200 34150 "" "Prometheus/2.43.0" 2024-04-12T12:03:35.056 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 2 1 2024-04-12T12:03:35.268 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:35.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:35 smithi029 ceph-mon[93650]: pgmap v2513: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:35.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:35 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/206570728' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-04-12T12:03:35.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:35 smithi029 ceph-mon[93650]: osdmap e1373: 8 total, 8 up, 8 in 2024-04-12T12:03:35.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:35 smithi029 ceph-mon[97787]: pgmap v2513: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:35.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:35 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/206570728' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-04-12T12:03:35.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:35 smithi029 ceph-mon[97787]: osdmap e1373: 8 total, 8 up, 8 in 2024-04-12T12:03:35.745 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:35 smithi087 ceph-mon[89014]: pgmap v2513: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:35.745 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:35 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/206570728' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-04-12T12:03:35.745 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:35 smithi087 ceph-mon[89014]: osdmap e1373: 8 total, 8 up, 8 in 2024-04-12T12:03:36.259 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "size": 20, 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 600, 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "ops": [ 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(3.10 1365/1364 [PushOp(3:0d82a743:::600.00000000:head, version: 68'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:0d82a743:::600.00000000:head@68'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:15.365104+0000", 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 20.892029231999999, 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00071445999999999999, 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.260 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.365104+0000", 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.365104+0000", 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.365104+0000", 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.261 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.365119+0000", 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5002e-05 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.365132+0000", 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2995e-05 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.365149+0000", 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7322000000000001e-05 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.262 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.365606+0000", 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00045620300000000002 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.365623+0000", 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7345000000000001e-05 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.365819+0000", 2024-04-12T12:03:36.263 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019599799999999999 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(3.1d 1365/1364 [PushOp(3:b871830b:::605.00000000:head, version: 68'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:b871830b:::605.00000000:head@68'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:15.366417+0000", 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 20.890717209999998, 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00033015699999999998, 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.264 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366417+0000", 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366417+0000", 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366416+0000", 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.265 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366428+0000", 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2242000000000001e-05 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366440+0000", 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1803e-05 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366465+0000", 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.4589000000000001e-05 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.266 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366522+0000", 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6888999999999997e-05 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366536+0000", 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4183e-05 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366747+0000", 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000210836 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.267 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(3.10 1365/1364 [PushOp(3:09401a81:::601.00000000:head, version: 68'4, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:09401a81:::601.00000000:head@68'4, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:15.366856+0000", 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 20.890278200000001, 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00042789800000000002, 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366856+0000", 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.268 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366856+0000", 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366855+0000", 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366878+0000", 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.3091e-05 2024-04-12T12:03:36.269 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366891+0000", 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2584e-05 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.366904+0000", 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3175e-05 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.367080+0000", 2024-04-12T12:03:36.270 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017623699999999999 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.367091+0000", 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1192999999999999e-05 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.367283+0000", 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00019225399999999999 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.271 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(3.10 1365/1364 [PushOp(3:0bd6d154:::602.00000000:head, version: 68'6, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:0bd6d154:::602.00000000:head@68'6, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:15.368309+0000", 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 20.888824935999999, 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00017759300000000001, 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368309+0000", 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.272 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368309+0000", 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368308+0000", 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368319+0000", 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0988999999999999e-05 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.273 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368332+0000", 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2768e-05 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368345+0000", 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2524000000000001e-05 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368389+0000", 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4598999999999998e-05 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.274 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368395+0000", 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.959e-06 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368486+0000", 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.1269999999999999e-05 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(3.1c 1365/1364 [PushOp(3:3ed09add:::607.00000000:head, version: 68'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:3ed09add:::607.00000000:head@68'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.275 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:15.368658+0000", 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 20.888476010000002, 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00038721399999999998, 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368658+0000", 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368658+0000", 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.276 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368657+0000", 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368669+0000", 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.1986e-05 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368685+0000", 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.5483000000000001e-05 2024-04-12T12:03:36.277 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368699+0000", 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4285e-05 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368739+0000", 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 3.9631000000000001e-05 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.368745+0000", 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.3890000000000002e-06 2024-04-12T12:03:36.278 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.369045+0000", 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00029997699999999999 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(3.10 1365/1364 [PushOp(3:0d89b25e:::603.00000000:head, version: 68'8, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:0d89b25e:::603.00000000:head@68'8, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.279 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:15.369294+0000", 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 20.887839301, 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00021803600000000001, 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.369294+0000", 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.369294+0000", 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.280 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.369294+0000", 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.369304+0000", 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.047e-05 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.369317+0000", 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2126999999999999e-05 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.281 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.369328+0000", 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.13e-05 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.369383+0000", 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.4706000000000001e-05 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.369393+0000", 2024-04-12T12:03:36.282 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0312000000000001e-05 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.369512+0000", 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00011962 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(3.d 1365/1364 [PushOp(3:b50e409b:::mds_snaptable:head, version: 68'1, data_included: [0~70], data_size: 70, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:b50e409b:::mds_snaptable:head@68'1, size: 70, copy_subset: [0~70], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:70, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:15.370718+0000", 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 20.886415394, 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00041608299999999998, 2024-04-12T12:03:36.283 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.370718+0000", 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.370718+0000", 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.284 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.370718+0000", 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.370728+0000", 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7729999999999994e-06 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.370741+0000", 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3335e-05 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.285 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.370755+0000", 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4147e-05 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.370812+0000", 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.6473e-05 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.286 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.370822+0000", 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0207000000000001e-05 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.371134+0000", 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00031258600000000001 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(3.1f 1365/1364 [PushOp(3:ff5b34d6:::1.00000000:head, version: 68'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:ff5b34d6:::1.00000000:head@68'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:15.371043+0000", 2024-04-12T12:03:36.287 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 20.886090657, 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00026922099999999999, 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.371043+0000", 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.371043+0000", 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.288 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.371042+0000", 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.999999 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.371060+0000", 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.7912999999999999e-05 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.371077+0000", 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6784999999999999e-05 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.289 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.371098+0000", 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 2.0925e-05 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.371158+0000", 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.9636000000000003e-05 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.371169+0000", 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.0991e-05 2024-04-12T12:03:36.290 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.371312+0000", 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00014376600000000001 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(3.1f 1365/1364 [PushOp(3:fe07bba1:::100.00000000.inode:head, version: 68'3, data_included: [0~582], data_size: 582, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:fe07bba1:::100.00000000.inode:head@68'3, size: 582, copy_subset: [0~582], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:582, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:15.372323+0000", 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 20.884811024000001, 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000223713, 2024-04-12T12:03:36.291 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.372323+0000", 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.372323+0000", 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.292 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.372322+0000", 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.372334+0000", 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2031000000000001e-05 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.372351+0000", 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.6738000000000001e-05 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.293 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.294 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.294 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.372365+0000", 2024-04-12T12:03:36.294 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4059999999999999e-05 2024-04-12T12:03:36.294 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.372416+0000", 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 5.1461000000000001e-05 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.372424+0000", 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.4170000000000003e-06 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:15.372546+0000", 2024-04-12T12:03:36.295 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000122607 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(1.4 1366/1365 [PushOp(1:30306672:devicehealth::main.db.0000000000000000:head, version: 249'161, data_included: [0~327680], data_size: 327680, omap_header_size: 0, omap_entries_size: 0, attrset_size: 10, recovery_info: ObjectRecoveryInfo(1:30306672:devicehealth::main.db.0000000000000000:head@249'161, size: 327680, copy_subset: [0~327680], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:327680, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:16.371006+0000", 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 19.886127729999998, 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00079828400000000002, 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.296 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.371006+0000", 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.371006+0000", 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.371006+0000", 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.297 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.371381+0000", 2024-04-12T12:03:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00037550099999999998 2024-04-12T12:03:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.371395+0000", 2024-04-12T12:03:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.3658999999999999e-05 2024-04-12T12:03:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.298 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.371407+0000", 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.2347000000000001e-05 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.371471+0000", 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.4060000000000007e-05 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.371486+0000", 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4912000000000001e-05 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.299 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.371804+0000", 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00031818100000000001 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "MOSDPGPush(1.8 1366/1365 [PushOp(1:1b0e96ae:devicehealth::main.db-journal.0000000000000000:head, version: 21'116, data_included: [0~262688], data_size: 262688, omap_header_size: 0, omap_entries_size: 0, attrset_size: 9, recovery_info: ObjectRecoveryInfo(1:1b0e96ae:devicehealth::main.db-journal.0000000000000000:head@21'116, size: 262688, copy_subset: [0~262688], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:262688, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:16.373104+0000", 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 19.884029713, 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.000639239, 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "started", 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.300 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.373104+0000", 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.373104+0000", 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.373104+0000", 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999995 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.301 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.373424+0000", 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00032005500000000002 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.373438+0000", 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 1.4915e-05 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.373448+0000", 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.8849999999999994e-06 2024-04-12T12:03:36.302 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.373493+0000", 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.4733e-05 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "started", 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.373501+0000", 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 7.5619999999999998e-06 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.303 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:16.373743+0000", 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00024257799999999999 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "description": "osd_op(client.34131.0:95 2.2 2:4e99cc3e:::rbd_mirror_snapshot_schedule:head [omap-get-vals] snapc 0=[] RETRY=1 ondisk+retry+read+known_if_redirected+supports_pool_eio e1368)", 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "initiated_at": "2024-04-12T12:03:21.688653+0000", 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "age": 14.568481102, 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0.00016599, 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "type_data": { 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "flag_point": "reached pg", 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "client_info": { 2024-04-12T12:03:36.304 INFO:teuthology.orchestra.run.smithi087.stdout: "client": "client.34131", 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "client_addr": "172.21.15.29:0/3454151607", 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "tid": 95 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "events": [ 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "initiated", 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:21.688653+0000", 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "throttled", 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:21.688653+0000", 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 0 2024-04-12T12:03:36.305 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "header_read", 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:21.688651+0000", 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4294967295.9999986 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "all_read", 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:21.688658+0000", 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 6.7100000000000001e-06 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "dispatched", 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:21.688667+0000", 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 8.5839999999999995e-06 2024-04-12T12:03:36.306 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "queued_for_pg", 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:21.688676+0000", 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.3249999999999997e-06 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "reached_pg", 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:21.688721+0000", 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 4.5286e-05 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "event": "done", 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "time": "2024-04-12T12:03:21.688819+0000", 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: "duration": 9.7374999999999995e-05 2024-04-12T12:03:36.307 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.308 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.308 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.308 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-12T12:03:36.308 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-12T12:03:36.308 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-12T12:03:36.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:36.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:36.916 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:37.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-04-12T12:03:37.314 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:37.482 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 1 (10000) 2024-04-12T12:03:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:37 smithi029 ceph-mon[93650]: pgmap v2515: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:37 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3837533593' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T12:03:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:37 smithi029 ceph-mon[97787]: pgmap v2515: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:37 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3837533593' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T12:03:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:37 smithi087 ceph-mon[89014]: pgmap v2515: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:37 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3837533593' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T12:03:38.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 2 1 2024-04-12T12:03:38.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:38 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3837533593' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-04-12T12:03:38.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:38 smithi029 ceph-mon[93650]: osdmap e1374: 8 total, 8 up, 8 in 2024-04-12T12:03:38.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:38 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3837533593' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-04-12T12:03:38.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:38 smithi029 ceph-mon[97787]: osdmap e1374: 8 total, 8 up, 8 in 2024-04-12T12:03:38.773 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:38.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:38 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3837533593' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-04-12T12:03:38.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:38 smithi087 ceph-mon[89014]: osdmap e1374: 8 total, 8 up, 8 in 2024-04-12T12:03:39.665 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:39 smithi029 ceph-mon[97787]: pgmap v2517: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:39.665 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:39 smithi029 ceph-mon[93650]: pgmap v2517: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:39.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:39 smithi087 ceph-mon[89014]: pgmap v2517: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:40.008 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:03:40.008 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T12:03:40.008 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2024-04-12T12:03:40.008 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:03:40.656 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-04-12T12:03:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:41 smithi029 ceph-mon[93650]: pgmap v2518: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:41.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:41 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:03:41.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:41 smithi029 ceph-mon[97787]: pgmap v2518: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:41.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:41 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:03:41.775 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:41 smithi087 ceph-mon[89014]: pgmap v2518: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:41.775 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:41 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:03:42.189 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:42.602 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:03:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:03:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:03:43.497 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-04-12T12:03:43.765 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:43 smithi029 ceph-mon[97787]: pgmap v2519: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:43.765 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:43 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4184806761' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T12:03:43.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:43 smithi029 ceph-mon[93650]: pgmap v2519: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:43.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:43 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4184806761' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T12:03:43.834 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:43 smithi087 ceph-mon[89014]: pgmap v2519: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:43.835 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:43 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4184806761' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2024-04-12T12:03:44.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:44 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4184806761' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-04-12T12:03:44.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:44 smithi029 ceph-mon[97787]: osdmap e1375: 8 total, 8 up, 8 in 2024-04-12T12:03:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:44 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4184806761' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-04-12T12:03:44.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:44 smithi029 ceph-mon[93650]: osdmap e1375: 8 total, 8 up, 8 in 2024-04-12T12:03:44.771 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:44 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4184806761' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-04-12T12:03:44.771 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:44 smithi087 ceph-mon[89014]: osdmap e1375: 8 total, 8 up, 8 in 2024-04-12T12:03:45.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:03:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:03:44] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-04-12T12:03:45.023 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:03:45.023 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2024-04-12T12:03:45.024 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2024-04-12T12:03:45.024 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2024-04-12T12:03:45.024 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:03:45.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 4 1 2024-04-12T12:03:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:45 smithi029 ceph-mon[93650]: pgmap v2521: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:45 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:45 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:45 smithi029 ceph-mon[97787]: pgmap v2521: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:45 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:45 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:45 smithi087 ceph-mon[89014]: pgmap v2521: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:45.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:45 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:45.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:45 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:46.155 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:46.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-04-12T12:03:46.930 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:47.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:47 smithi029 ceph-mon[97787]: pgmap v2522: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:47.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:47 smithi029 ceph-mon[93650]: pgmap v2522: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:47.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:47 smithi087 ceph-mon[89014]: pgmap v2522: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:49.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:49 smithi029 ceph-mon[97787]: pgmap v2523: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:49.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:49 smithi029 ceph-mon[93650]: pgmap v2523: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:49.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:49 smithi087 ceph-mon[89014]: pgmap v2523: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:50.517 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 1 (10000) 2024-04-12T12:03:50.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:50 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/752916728' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:03:50.605 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:50 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/752916728' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:03:50.618 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-04-12T12:03:50.618 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2024-04-12T12:03:50.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2024-04-12T12:03:50.618 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2024-04-12T12:03:50.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.619 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217064 11.a e1344/1342 11:50a0b1b4:::benchmark_data_smithi029_285290_object98373:head v 1344'13396, mlcod=1329'8943)", 2024-04-12T12:03:50.619 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:11.182607+0000", 2024-04-12T12:03:50.619 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 159.43346840699999, 2024-04-12T12:03:50.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15865848499999999, 2024-04-12T12:03:50.619 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.619 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.182607+0000", 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.182607+0000", 2024-04-12T12:03:50.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.182607+0000", 2024-04-12T12:03:50.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9499999999999998e-07 2024-04-12T12:03:50.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.182612+0000", 2024-04-12T12:03:50.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2899999999999996e-06 2024-04-12T12:03:50.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.182618+0000", 2024-04-12T12:03:50.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6900000000000003e-06 2024-04-12T12:03:50.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.182623+0000", 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9300000000000002e-06 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.207182+0000", 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024559161999999999 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.207217+0000", 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4709000000000003e-05 2024-04-12T12:03:50.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.341226+0000", 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.134008671 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.341265+0000", 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9638000000000002e-05 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.624 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217080 11.2 e1344/1343 11:43b0a7b5:::benchmark_data_smithi029_285290_object98389:head v 1344'13810, mlcod=1331'9253)", 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:11.188864+0000", 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 159.42721166000001, 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14291649000000001, 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.625 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.188864+0000", 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.188864+0000", 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.188865+0000", 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5600000000000004e-07 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.188873+0000", 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3629999999999994e-06 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.188889+0000", 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6271000000000001e-05 2024-04-12T12:03:50.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.188897+0000", 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6249999999999998e-06 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.205336+0000", 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016439347999999999 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.205399+0000", 2024-04-12T12:03:50.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2947999999999994e-05 2024-04-12T12:03:50.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.331726+0000", 2024-04-12T12:03:50.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12632663899999999 2024-04-12T12:03:50.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.331780+0000", 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4339999999999998e-05 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217091 11.2 e1344/1343 11:4a62f87f:::benchmark_data_smithi029_285290_object98400:head v 1344'13811, mlcod=1331'9253)", 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:11.205721+0000", 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 159.41035449699999, 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.144967086, 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.630 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.205721+0000", 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.205721+0000", 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.205721+0000", 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5499999999999998e-07 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.205726+0000", 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8940000000000001e-06 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.205732+0000", 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1979999999999998e-06 2024-04-12T12:03:50.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.205737+0000", 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.69e-06 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.220171+0000", 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014434255999999999 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.220222+0000", 2024-04-12T12:03:50.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0312000000000002e-05 2024-04-12T12:03:50.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.350389+0000", 2024-04-12T12:03:50.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.130167686 2024-04-12T12:03:50.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.350688+0000", 2024-04-12T12:03:50.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00029859500000000002 2024-04-12T12:03:50.635 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.635 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.635 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.635 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217097 11.a e1344/1342 11:5627301d:::benchmark_data_smithi029_285290_object98406:head v 1344'13397, mlcod=1329'8943)", 2024-04-12T12:03:50.636 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:11.217233+0000", 2024-04-12T12:03:50.637 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 159.398842677, 2024-04-12T12:03:50.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13369096499999999, 2024-04-12T12:03:50.637 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.637 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.637 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.637 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.217233+0000", 2024-04-12T12:03:50.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.217233+0000", 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.217234+0000", 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2389999999999999e-06 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.217243+0000", 2024-04-12T12:03:50.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5179999999999993e-06 2024-04-12T12:03:50.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.217260+0000", 2024-04-12T12:03:50.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6393999999999999e-05 2024-04-12T12:03:50.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.217279+0000", 2024-04-12T12:03:50.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8763e-05 2024-04-12T12:03:50.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.229325+0000", 2024-04-12T12:03:50.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012046573 2024-04-12T12:03:50.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.229353+0000", 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7503000000000001e-05 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.350879+0000", 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.121526088 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.350924+0000", 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4886999999999999e-05 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.641 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217098 11.a e1344/1342 11:55c26401:::benchmark_data_smithi029_285290_object98407:head v 1344'13398, mlcod=1329'8943)", 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:11.222071+0000", 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 159.39400406199999, 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12893668999999999, 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.642 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.222071+0000", 2024-04-12T12:03:50.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.222071+0000", 2024-04-12T12:03:50.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.222072+0000", 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0400000000000005e-07 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.222079+0000", 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9789999999999996e-06 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.222092+0000", 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2945000000000001e-05 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.222099+0000", 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9029999999999998e-06 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.238560+0000", 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016461048999999998 2024-04-12T12:03:50.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.238584+0000", 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4017999999999999e-05 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.350926+0000", 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.11234219400000001 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.351008+0000", 2024-04-12T12:03:50.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1798000000000005e-05 2024-04-12T12:03:50.647 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.647 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.648 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.648 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217100 11.a e1344/1342 11:5f855502:::benchmark_data_smithi029_285290_object98409:head v 1344'13399, mlcod=1329'8943)", 2024-04-12T12:03:50.648 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:11.229653+0000", 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 159.38642239399999, 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12141637700000001, 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.229653+0000", 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.229653+0000", 2024-04-12T12:03:50.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.229654+0000", 2024-04-12T12:03:50.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.234e-06 2024-04-12T12:03:50.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.229665+0000", 2024-04-12T12:03:50.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1075e-05 2024-04-12T12:03:50.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.229678+0000", 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2434e-05 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.229687+0000", 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3360000000000001e-06 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.248022+0000", 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018334816 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.248046+0000", 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.461e-05 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.351010+0000", 2024-04-12T12:03:50.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.102963892 2024-04-12T12:03:50.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:11.351069+0000", 2024-04-12T12:03:50.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8980000000000001e-05 2024-04-12T12:03:50.657 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.657 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.657 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.658 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217576 11.a e1344/1342 11:5434df98:::benchmark_data_smithi029_285290_object98885:head v 1344'13429, mlcod=1329'8943)", 2024-04-12T12:03:50.658 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:12.131908+0000", 2024-04-12T12:03:50.658 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.48416722100001, 2024-04-12T12:03:50.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.130206092, 2024-04-12T12:03:50.658 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.658 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.658 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.658 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131908+0000", 2024-04-12T12:03:50.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131908+0000", 2024-04-12T12:03:50.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131909+0000", 2024-04-12T12:03:50.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.155e-06 2024-04-12T12:03:50.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131918+0000", 2024-04-12T12:03:50.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5879999999999998e-06 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131939+0000", 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1333999999999999e-05 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131948+0000", 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6400000000000003e-06 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.149755+0000", 2024-04-12T12:03:50.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017807040999999999 2024-04-12T12:03:50.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.149782+0000", 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7284000000000002e-05 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.262008+0000", 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.112225914 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.262114+0000", 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000106136 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.663 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.664 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217579 11.a e1344/1342 11:5ecf2f83:::benchmark_data_smithi029_285290_object98888:head v 1344'13430, mlcod=1329'8943)", 2024-04-12T12:03:50.664 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:12.131972+0000", 2024-04-12T12:03:50.664 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.48410292400001, 2024-04-12T12:03:50.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.130218313, 2024-04-12T12:03:50.664 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.664 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.664 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.664 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131972+0000", 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131972+0000", 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131973+0000", 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8099999999999998e-07 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131977+0000", 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2710000000000003e-06 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131983+0000", 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2040000000000002e-06 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.131988+0000", 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7199999999999997e-06 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.163286+0000", 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031298429000000003 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.163315+0000", 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8880000000000001e-05 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.262119+0000", 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.098803558 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.262191+0000", 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1870000000000001e-05 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.668 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217582 11.a e1344/1342 11:5b405e07:::benchmark_data_smithi029_285290_object98891:head v 1344'13431, mlcod=1329'8943)", 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:12.136657+0000", 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.479418087, 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.125589124, 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.136657+0000", 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.136657+0000", 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.136658+0000", 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.54e-07 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.136664+0000", 2024-04-12T12:03:50.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2199999999999997e-06 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.136675+0000", 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0511e-05 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.136682+0000", 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.815e-06 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.173431+0000", 2024-04-12T12:03:50.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036749859000000003 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.173457+0000", 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5185999999999999e-05 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.262193+0000", 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.088736608999999994 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.262246+0000", 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3170000000000001e-05 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217588 11.2 e1344/1343 11:4f105e05:::benchmark_data_smithi029_285290_object98897:head v 1344'13834, mlcod=1331'9273)", 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:12.146204+0000", 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.469870886, 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12972347400000001, 2024-04-12T12:03:50.673 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.146204+0000", 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.146204+0000", 2024-04-12T12:03:50.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.146205+0000", 2024-04-12T12:03:50.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7499999999999999e-07 2024-04-12T12:03:50.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.146211+0000", 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8799999999999996e-06 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.146223+0000", 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1394e-05 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.146230+0000", 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3819999999999998e-06 2024-04-12T12:03:50.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.173416+0000", 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027185870000000001 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.173446+0000", 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9998999999999999e-05 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.275880+0000", 2024-04-12T12:03:50.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10243429599999999 2024-04-12T12:03:50.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.275928+0000", 2024-04-12T12:03:50.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7877999999999998e-05 2024-04-12T12:03:50.678 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217592 11.2 e1344/1343 11:4e352cc2:::benchmark_data_smithi029_285290_object98901:head v 1344'13835, mlcod=1331'9273)", 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:12.150584+0000", 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.46549095899999, 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12537272599999999, 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.679 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.150584+0000", 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.150584+0000", 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.150585+0000", 2024-04-12T12:03:50.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0699999999999999e-06 2024-04-12T12:03:50.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.150593+0000", 2024-04-12T12:03:50.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8779999999999998e-06 2024-04-12T12:03:50.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.150605+0000", 2024-04-12T12:03:50.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1219e-05 2024-04-12T12:03:50.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.150613+0000", 2024-04-12T12:03:50.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1920000000000005e-06 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.241674+0000", 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.091061171999999996 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.241694+0000", 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9843e-05 2024-04-12T12:03:50.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.275931+0000", 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.034237375 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.275957+0000", 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5976999999999999e-05 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.684 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.685 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.685 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217656 11.a e1344/1342 11:56d6af53:::benchmark_data_smithi029_285290_object98965:head v 1344'13435, mlcod=1329'8943)", 2024-04-12T12:03:50.685 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:12.251395+0000", 2024-04-12T12:03:50.685 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.364680064, 2024-04-12T12:03:50.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.133194013, 2024-04-12T12:03:50.685 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.251395+0000", 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.251395+0000", 2024-04-12T12:03:50.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.251396+0000", 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.113e-06 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.251404+0000", 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.401e-06 2024-04-12T12:03:50.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.251419+0000", 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5662e-05 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.251428+0000", 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7970000000000006e-06 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.334211+0000", 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.082782224000000001 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.334237+0000", 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6222000000000001e-05 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.384483+0000", 2024-04-12T12:03:50.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.050246688999999997 2024-04-12T12:03:50.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.384589+0000", 2024-04-12T12:03:50.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105905 2024-04-12T12:03:50.690 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.691 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.691 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.692 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:217659 11.2 e1344/1343 11:4a12c53e:::benchmark_data_smithi029_285290_object98968:head v 1344'13840, mlcod=1331'9273)", 2024-04-12T12:03:50.692 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:12.255599+0000", 2024-04-12T12:03:50.692 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.36047635200001, 2024-04-12T12:03:50.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.133413596, 2024-04-12T12:03:50.692 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.255599+0000", 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.255599+0000", 2024-04-12T12:03:50.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.255600+0000", 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6599999999999994e-07 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.255607+0000", 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4919999999999996e-06 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.255620+0000", 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.288e-05 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.255629+0000", 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3850000000000002e-06 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.335393+0000", 2024-04-12T12:03:50.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.079764373999999999 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.335422+0000", 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8407000000000001e-05 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.388961+0000", 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.053539382000000003 2024-04-12T12:03:50.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:12.389013+0000", 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1709999999999998e-05 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:222816 11.2 e1344/1343 11:4a4a535b:::benchmark_data_smithi029_285290_object104125:head v 1344'14166, mlcod=1331'9450)", 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:21.463438+0000", 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.15263729, 2024-04-12T12:03:50.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.136148397, 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.463438+0000", 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.463438+0000", 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.463439+0000", 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0440000000000001e-06 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.463450+0000", 2024-04-12T12:03:50.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.065e-05 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.463463+0000", 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3644e-05 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.463473+0000", 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4290000000000007e-06 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.468224+0000", 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004751351 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.468251+0000", 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6843999999999999e-05 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.479405+0000", 2024-04-12T12:03:50.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011153587 2024-04-12T12:03:50.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.599586+0000", 2024-04-12T12:03:50.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12018184799999999 2024-04-12T12:03:50.702 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.702 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.702 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:222824 11.1 e1344/1343 11:8ae5f161:::benchmark_data_smithi029_285290_object104133:head v 1344'13995, mlcod=1329'9146)", 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:21.476233+0000", 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.13984250300001, 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15700929499999999, 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476233+0000", 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476233+0000", 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476234+0000", 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9299999999999996e-07 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476242+0000", 2024-04-12T12:03:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8599999999999993e-06 2024-04-12T12:03:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476252+0000", 2024-04-12T12:03:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0875e-05 2024-04-12T12:03:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476260+0000", 2024-04-12T12:03:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6629999999999993e-06 2024-04-12T12:03:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.477286+0000", 2024-04-12T12:03:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001025932 2024-04-12T12:03:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.477306+0000", 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0134999999999999e-05 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.633180+0000", 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.155873498 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.633242+0000", 2024-04-12T12:03:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2439000000000003e-05 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:222822 11.b e1344/1342 11:d70f5c2a:::benchmark_data_smithi029_285290_object104131:head v 1344'13754, mlcod=1331'9105)", 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:21.476317+0000", 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.13975831600001, 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12736256000000001, 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476317+0000", 2024-04-12T12:03:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476317+0000", 2024-04-12T12:03:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476318+0000", 2024-04-12T12:03:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9100000000000004e-07 2024-04-12T12:03:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476323+0000", 2024-04-12T12:03:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.835e-06 2024-04-12T12:03:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476331+0000", 2024-04-12T12:03:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5240000000000003e-06 2024-04-12T12:03:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476337+0000", 2024-04-12T12:03:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6010000000000002e-06 2024-04-12T12:03:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476409+0000", 2024-04-12T12:03:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2019000000000003e-05 2024-04-12T12:03:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.476433+0000", 2024-04-12T12:03:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4658999999999999e-05 2024-04-12T12:03:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.603632+0000", 2024-04-12T12:03:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12719914099999999 2024-04-12T12:03:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.603680+0000", 2024-04-12T12:03:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7190000000000001e-05 2024-04-12T12:03:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:222825 11.1 e1344/1343 11:81073cb3:::benchmark_data_smithi029_285290_object104134:head v 1344'13996, mlcod=1329'9146)", 2024-04-12T12:03:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:21.479369+0000", 2024-04-12T12:03:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.13670683500001, 2024-04-12T12:03:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.154031635, 2024-04-12T12:03:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.479369+0000", 2024-04-12T12:03:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.479369+0000", 2024-04-12T12:03:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.479369+0000", 2024-04-12T12:03:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1900000000000001e-07 2024-04-12T12:03:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.479378+0000", 2024-04-12T12:03:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2099999999999993e-06 2024-04-12T12:03:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.479389+0000", 2024-04-12T12:03:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1279e-05 2024-04-12T12:03:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.600552+0000", 2024-04-12T12:03:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.121163085 2024-04-12T12:03:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.608495+0000", 2024-04-12T12:03:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0079433539999999997 2024-04-12T12:03:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.608522+0000", 2024-04-12T12:03:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6917999999999999e-05 2024-04-12T12:03:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.633333+0000", 2024-04-12T12:03:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024811178 2024-04-12T12:03:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.633400+0000", 2024-04-12T12:03:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6692000000000005e-05 2024-04-12T12:03:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:222829 11.5 e1344/1343 11:a60d7fdf:::benchmark_data_smithi029_285290_object104138:head v 1344'14021, mlcod=1329'9210)", 2024-04-12T12:03:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:21.480261+0000", 2024-04-12T12:03:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.13581463400001, 2024-04-12T12:03:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12829589599999999, 2024-04-12T12:03:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.480261+0000", 2024-04-12T12:03:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.480261+0000", 2024-04-12T12:03:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.480262+0000", 2024-04-12T12:03:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2600000000000001e-07 2024-04-12T12:03:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.480270+0000", 2024-04-12T12:03:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4589999999999998e-06 2024-04-12T12:03:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.599558+0000", 2024-04-12T12:03:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.11928778700000001 2024-04-12T12:03:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.599571+0000", 2024-04-12T12:03:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2899e-05 2024-04-12T12:03:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.599676+0000", 2024-04-12T12:03:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105148 2024-04-12T12:03:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.599709+0000", 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2830999999999998e-05 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.608402+0000", 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0086930719999999996 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.608557+0000", 2024-04-12T12:03:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015477399999999999 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:222817 11.9 e1344/1343 11:9c8ce9cb:::benchmark_data_smithi029_285290_object104126:head v 1344'14000, mlcod=1329'9145)", 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:21.482917+0000", 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.13315823900001, 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.1504122, 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.482917+0000", 2024-04-12T12:03:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.482917+0000", 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.482918+0000", 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0410000000000001e-06 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.482928+0000", 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6760000000000002e-06 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.599557+0000", 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.116629027 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.599571+0000", 2024-04-12T12:03:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3876000000000001e-05 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.599699+0000", 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000128262 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.599722+0000", 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3142e-05 2024-04-12T12:03:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.633247+0000", 2024-04-12T12:03:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033524985 2024-04-12T12:03:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:21.633329+0000", 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2191e-05 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45800.0:224218 11.9 e1344/1343 11:9b35c3ba:::benchmark_data_smithi029_285290_object105527:head v 1344'14089, mlcod=1329'9145)", 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2024-04-12T12:01:24.277010+0000", 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 146.33906585599999, 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.123505852, 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "continuous": false, 2024-04-12T12:03:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2024-04-12T12:03:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2024-04-12T12:03:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2024-04-12T12:03:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2024-04-12T12:03:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:24.277010+0000", 2024-04-12T12:03:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2024-04-12T12:03:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:24.277010+0000", 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:24.277010+0000", 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4200000000000002e-07 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:24.277014+0000", 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9040000000000002e-06 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:24.277020+0000", 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3899999999999998e-06 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:24.277025+0000", 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0730000000000004e-06 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2024-04-12T12:03:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:24.310992+0000", 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033966288999999997 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:24.311019+0000", 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7637e-05 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:24.400402+0000", 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.089382769000000001 2024-04-12T12:03:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-04-12T12:03:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2024-04-12T12:03:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2024-04-12T12:03:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2024-04-12T12:01:24.400515+0000", 2024-04-12T12:03:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113448 2024-04-12T12:03:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-04-12T12:03:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-04-12T12:03:50.750 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-04-12T12:03:50.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:50 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/752916728' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:03:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:51 smithi029 ceph-mon[93650]: pgmap v2524: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:51 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/752916728' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-04-12T12:03:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:51 smithi029 ceph-mon[93650]: osdmap e1376: 8 total, 8 up, 8 in 2024-04-12T12:03:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:51 smithi029 ceph-mon[97787]: pgmap v2524: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:51.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:51 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/752916728' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-04-12T12:03:51.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:51 smithi029 ceph-mon[97787]: osdmap e1376: 8 total, 8 up, 8 in 2024-04-12T12:03:51.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:51 smithi087 ceph-mon[89014]: pgmap v2524: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:51.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:51 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/752916728' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-04-12T12:03:51.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:51 smithi087 ceph-mon[89014]: osdmap e1376: 8 total, 8 up, 8 in 2024-04-12T12:03:51.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:03:51.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 4 1 2024-04-12T12:03:52.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:03:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:03:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:03:53.699 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:53 smithi029 ceph-mon[93650]: pgmap v2526: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:53 smithi029 ceph-mon[97787]: pgmap v2526: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:53.917 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:53 smithi087 ceph-mon[89014]: pgmap v2526: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:54.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:54.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:54 smithi087 ceph-mon[89014]: pgmap v2527: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:54.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:03:54.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:03:54.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:54 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:55.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:55.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:55.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[93650]: pgmap v2527: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:55.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:03:55.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:03:55.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:55.020 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:03:54] "GET /metrics HTTP/1.1" 200 34137 "" "Prometheus/2.43.0" 2024-04-12T12:03:55.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:55.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:55.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[97787]: pgmap v2527: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:55.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:03:55.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:03:55.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:54 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:03:56.125 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-04-12T12:03:56.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:56 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2238000932' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:03:56.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:56 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:03:56.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:56 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2238000932' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:03:56.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:56 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:03:56.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:56 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2238000932' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:03:56.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:56 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2024-04-12T12:03:56.966 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 6 1 2024-04-12T12:03:57.183 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:03:57.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:57 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-04-12T12:03:57.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:57 smithi029 ceph-mon[93650]: osdmap e1377: 8 total, 8 up, 8 in 2024-04-12T12:03:57.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:57 smithi029 ceph-mon[93650]: pgmap v2529: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:57.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:57 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-04-12T12:03:57.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:57 smithi029 ceph-mon[97787]: osdmap e1377: 8 total, 8 up, 8 in 2024-04-12T12:03:57.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:57 smithi029 ceph-mon[97787]: pgmap v2529: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:57.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:57 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-04-12T12:03:57.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:57 smithi087 ceph-mon[89014]: osdmap e1377: 8 total, 8 up, 8 in 2024-04-12T12:03:57.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:57 smithi087 ceph-mon[89014]: pgmap v2529: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:58.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:58 smithi087 ceph-mon[89014]: osdmap e1378: 8 total, 8 up, 8 in 2024-04-12T12:03:58.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:58 smithi029 ceph-mon[93650]: osdmap e1378: 8 total, 8 up, 8 in 2024-04-12T12:03:58.474 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:58 smithi029 ceph-mon[97787]: osdmap e1378: 8 total, 8 up, 8 in 2024-04-12T12:03:59.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:59 smithi029 ceph-mon[93650]: pgmap v2531: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:59.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:59 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2075050054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:03:59.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:03:59 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:03:59.145 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.6 to 1 (10000) 2024-04-12T12:03:59.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:59 smithi087 ceph-mon[89014]: pgmap v2531: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:59.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:59 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2075050054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:03:59.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:03:59 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:03:59.508 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:59 smithi029 ceph-mon[97787]: pgmap v2531: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:03:59.508 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:59 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2075050054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:03:59.509 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:03:59 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:03:59.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 6 1 2024-04-12T12:04:00.059 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:00.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:00 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-04-12T12:04:00.435 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:00 smithi029 ceph-mon[93650]: osdmap e1379: 8 total, 8 up, 8 in 2024-04-12T12:04:00.435 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:00 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-04-12T12:04:00.435 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:00 smithi029 ceph-mon[97787]: osdmap e1379: 8 total, 8 up, 8 in 2024-04-12T12:04:00.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:00 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-04-12T12:04:00.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:00 smithi087 ceph-mon[89014]: osdmap e1379: 8 total, 8 up, 8 in 2024-04-12T12:04:01.442 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:01 smithi029 ceph-mon[93650]: pgmap v2533: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:01.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:01 smithi029 ceph-mon[97787]: pgmap v2533: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:01.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:01 smithi087 ceph-mon[89014]: pgmap v2533: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:02.153 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-04-12T12:04:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:02 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4080434767' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:04:02.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:02 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:04:02.468 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:04:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:04:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:04:02.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:02 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4080434767' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:04:02.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:02 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:04:02.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:02 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4080434767' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:04:02.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:02 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2024-04-12T12:04:02.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 7 1 2024-04-12T12:04:03.124 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:03.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:03 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-04-12T12:04:03.409 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:03 smithi029 ceph-mon[93650]: osdmap e1380: 8 total, 8 up, 8 in 2024-04-12T12:04:03.409 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:03 smithi029 ceph-mon[93650]: pgmap v2535: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:03.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:03 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-04-12T12:04:03.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:03 smithi029 ceph-mon[97787]: osdmap e1380: 8 total, 8 up, 8 in 2024-04-12T12:04:03.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:03 smithi029 ceph-mon[97787]: pgmap v2535: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:03.421 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:03 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-04-12T12:04:03.422 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:03 smithi087 ceph-mon[89014]: osdmap e1380: 8 total, 8 up, 8 in 2024-04-12T12:04:03.422 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:03 smithi087 ceph-mon[89014]: pgmap v2535: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:04.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:04 smithi029 ceph-mon[93650]: osdmap e1381: 8 total, 8 up, 8 in 2024-04-12T12:04:04.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:04 smithi029 ceph-mon[97787]: osdmap e1381: 8 total, 8 up, 8 in 2024-04-12T12:04:04.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:04 smithi087 ceph-mon[89014]: osdmap e1381: 8 total, 8 up, 8 in 2024-04-12T12:04:05.019 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:04:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:04:04] "GET /metrics HTTP/1.1" 200 34137 "" "Prometheus/2.43.0" 2024-04-12T12:04:05.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:05 smithi087 ceph-mon[89014]: pgmap v2537: 83 pgs: 2 peering, 81 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:05.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:05 smithi029 ceph-mon[93650]: pgmap v2537: 83 pgs: 2 peering, 81 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:05.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:05 smithi029 ceph-mon[97787]: pgmap v2537: 83 pgs: 2 peering, 81 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:06.172 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.7 to 1 (10000) 2024-04-12T12:04:06.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:06 smithi087 ceph-mon[89014]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-12T12:04:06.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:06 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1599531901' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:04:06.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:06 smithi029 ceph-mon[93650]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-12T12:04:06.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:06 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1599531901' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:04:06.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:06 smithi029 ceph-mon[97787]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-12T12:04:06.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:06 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1599531901' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:04:06.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 7 1 2024-04-12T12:04:07.212 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:07.467 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:07 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1599531901' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-04-12T12:04:07.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:07 smithi087 ceph-mon[89014]: osdmap e1382: 8 total, 8 up, 8 in 2024-04-12T12:04:07.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:07 smithi087 ceph-mon[89014]: pgmap v2539: 83 pgs: 2 peering, 81 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:07.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:04:07.468 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:07 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:07.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:07 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1599531901' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-04-12T12:04:07.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:07 smithi029 ceph-mon[97787]: osdmap e1382: 8 total, 8 up, 8 in 2024-04-12T12:04:07.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:07 smithi029 ceph-mon[97787]: pgmap v2539: 83 pgs: 2 peering, 81 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:07.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:04:07.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:07 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:07.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:07 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1599531901' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-04-12T12:04:07.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:07 smithi029 ceph-mon[93650]: osdmap e1382: 8 total, 8 up, 8 in 2024-04-12T12:04:07.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:07 smithi029 ceph-mon[93650]: pgmap v2539: 83 pgs: 2 peering, 81 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:07.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:04:07.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:07 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:08 smithi087 ceph-mon[89014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-12T12:04:08.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:08 smithi087 ceph-mon[89014]: Cluster is now healthy 2024-04-12T12:04:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:08 smithi029 ceph-mon[93650]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-12T12:04:08.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:08 smithi029 ceph-mon[93650]: Cluster is now healthy 2024-04-12T12:04:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:08 smithi029 ceph-mon[97787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-12T12:04:08.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:08 smithi029 ceph-mon[97787]: Cluster is now healthy 2024-04-12T12:04:09.413 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-04-12T12:04:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:09 smithi087 ceph-mon[89014]: pgmap v2540: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:09.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2903559935' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:04:09.733 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:09 smithi029 ceph-mon[93650]: pgmap v2540: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:09.733 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2903559935' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:04:09.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:09 smithi029 ceph-mon[97787]: pgmap v2540: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:09.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2903559935' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2024-04-12T12:04:10.130 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 5 1 2024-04-12T12:04:10.343 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:10.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:10 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2903559935' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-04-12T12:04:10.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:10 smithi087 ceph-mon[89014]: osdmap e1383: 8 total, 8 up, 8 in 2024-04-12T12:04:10.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:10 smithi087 ceph-mon[89014]: osdmap e1384: 8 total, 8 up, 8 in 2024-04-12T12:04:10.728 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:10 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2903559935' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-04-12T12:04:10.728 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:10 smithi029 ceph-mon[93650]: osdmap e1383: 8 total, 8 up, 8 in 2024-04-12T12:04:10.728 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:10 smithi029 ceph-mon[93650]: osdmap e1384: 8 total, 8 up, 8 in 2024-04-12T12:04:10.728 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:10 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2903559935' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-04-12T12:04:10.728 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:10 smithi029 ceph-mon[97787]: osdmap e1383: 8 total, 8 up, 8 in 2024-04-12T12:04:10.728 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:10 smithi029 ceph-mon[97787]: osdmap e1384: 8 total, 8 up, 8 in 2024-04-12T12:04:11.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:11 smithi087 ceph-mon[89014]: pgmap v2542: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:11.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:11 smithi029 ceph-mon[93650]: pgmap v2542: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:11.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:11 smithi029 ceph-mon[97787]: pgmap v2542: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:04:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:04:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:04:13.436 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.5 to 1 (10000) 2024-04-12T12:04:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:13 smithi087 ceph-mon[89014]: pgmap v2544: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:13.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:13 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2195815212' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T12:04:13.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:13 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T12:04:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:13 smithi029 ceph-mon[93650]: pgmap v2544: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:13 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2195815212' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T12:04:13.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:13 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T12:04:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:13 smithi029 ceph-mon[97787]: pgmap v2544: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:13 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2195815212' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T12:04:13.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:13 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T12:04:14.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 5 1 2024-04-12T12:04:14.439 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:14.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:14 smithi029 ceph-mon[93650]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-04-12T12:04:14.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:14 smithi029 ceph-mon[93650]: osdmap e1385: 8 total, 8 up, 8 in 2024-04-12T12:04:14.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:14 smithi029 ceph-mon[97787]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-04-12T12:04:14.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:14 smithi029 ceph-mon[97787]: osdmap e1385: 8 total, 8 up, 8 in 2024-04-12T12:04:14.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:14 smithi087 ceph-mon[89014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-04-12T12:04:14.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:14 smithi087 ceph-mon[89014]: osdmap e1385: 8 total, 8 up, 8 in 2024-04-12T12:04:15.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:04:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:04:14] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2024-04-12T12:04:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:15 smithi087 ceph-mon[89014]: pgmap v2546: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:15.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:15 smithi029 ceph-mon[93650]: pgmap v2546: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:15.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:15 smithi029 ceph-mon[97787]: pgmap v2546: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:17.448 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-04-12T12:04:17.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:17 smithi087 ceph-mon[89014]: pgmap v2547: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:17.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:17 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2897552676' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T12:04:17.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:17 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:04:17.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:17 smithi029 ceph-mon[93650]: pgmap v2547: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:17.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:17 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2897552676' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T12:04:17.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:17 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:04:17.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:17 smithi029 ceph-mon[97787]: pgmap v2547: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:17.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:17 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2897552676' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2024-04-12T12:04:17.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:17 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:04:18.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd reweight 3 1 2024-04-12T12:04:18.441 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:18.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:18 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2897552676' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-04-12T12:04:18.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:18 smithi087 ceph-mon[89014]: osdmap e1386: 8 total, 8 up, 8 in 2024-04-12T12:04:18.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:18 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2897552676' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-04-12T12:04:18.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:18 smithi029 ceph-mon[97787]: osdmap e1386: 8 total, 8 up, 8 in 2024-04-12T12:04:18.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:18 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2897552676' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-04-12T12:04:18.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:18 smithi029 ceph-mon[93650]: osdmap e1386: 8 total, 8 up, 8 in 2024-04-12T12:04:19.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:19 smithi029 ceph-mon[93650]: pgmap v2549: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:19.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:19 smithi029 ceph-mon[97787]: pgmap v2549: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:19.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:19 smithi087 ceph-mon[89014]: pgmap v2549: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-04-12T12:04:20.458 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.3 to 1 (10000) 2024-04-12T12:04:20.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:20 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2332603234' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:04:20.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:20 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2332603234' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:04:20.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:20 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2332603234' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:04:21.275 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd primary-affinity 3 1 2024-04-12T12:04:21.497 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:21 smithi029 ceph-mon[93650]: pgmap v2550: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:21 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2332603234' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-04-12T12:04:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:21 smithi029 ceph-mon[93650]: osdmap e1387: 8 total, 8 up, 8 in 2024-04-12T12:04:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:04:21.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:21 smithi029 ceph-mon[97787]: pgmap v2550: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:21 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2332603234' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-04-12T12:04:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:21 smithi029 ceph-mon[97787]: osdmap e1387: 8 total, 8 up, 8 in 2024-04-12T12:04:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:04:21.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:21 smithi087 ceph-mon[89014]: pgmap v2550: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:21 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2332603234' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-04-12T12:04:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:21 smithi087 ceph-mon[89014]: osdmap e1387: 8 total, 8 up, 8 in 2024-04-12T12:04:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:04:21.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:22.619 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:04:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:04:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:04:23.465 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-04-12T12:04:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:23 smithi087 ceph-mon[89014]: pgmap v2552: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:23.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:23 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1975215391' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:04:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:23 smithi029 ceph-mon[93650]: pgmap v2552: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:23.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:23 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1975215391' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:04:23.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:23 smithi029 ceph-mon[97787]: pgmap v2552: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:23.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:23 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1975215391' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2024-04-12T12:04:24.137 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-04-12T12:04:24.137 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-04-12T12:04:24.137 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-04-12T12:04:24.137 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-04-12T12:04:24.137 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-12T12:04:24.138 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T12:04:24.352 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:24.468 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:24 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1975215391' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-04-12T12:04:24.469 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:24 smithi029 ceph-mon[93650]: osdmap e1388: 8 total, 8 up, 8 in 2024-04-12T12:04:24.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:24 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1975215391' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-04-12T12:04:24.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:24 smithi029 ceph-mon[97787]: osdmap e1388: 8 total, 8 up, 8 in 2024-04-12T12:04:24.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:24 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1975215391' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-04-12T12:04:24.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:24 smithi087 ceph-mon[89014]: osdmap e1388: 8 total, 8 up, 8 in 2024-04-12T12:04:25.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:04:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:04:24] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-04-12T12:04:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:25 smithi029 ceph-mon[97787]: pgmap v2554: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:25.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:25 smithi029 ceph-mon[97787]: osdmap e1389: 8 total, 8 up, 8 in 2024-04-12T12:04:25.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:25 smithi029 ceph-mon[93650]: pgmap v2554: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:25.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:25 smithi029 ceph-mon[93650]: osdmap e1389: 8 total, 8 up, 8 in 2024-04-12T12:04:25.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:25 smithi087 ceph-mon[89014]: pgmap v2554: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:25.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:25 smithi087 ceph-mon[89014]: osdmap e1389: 8 total, 8 up, 8 in 2024-04-12T12:04:26.683 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T12:04:26.683 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":1389,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T12:04:24.462548+0000","last_up_change":"2024-04-12T12:01:07.324373+0000","last_in_change":"2024-04-12T12:03:14.342599+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1264","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1240","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T11:11:48.103219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T11:12:56.493343+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T11:12:57.001342+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":503,"up_from":514,"up_thru":1366,"down_at":504,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6803","nonce":3899727946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6805","nonce":3899727946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6809","nonce":3899727946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6807","nonce":3899727946}]},"public_addr":"172.21.15.29:6803/3899727946","cluster_addr":"172.21.15.29:6805/3899727946","heartbeat_back_addr":"172.21.15.29:6809/3899727946","heartbeat_front_addr":"172.21.15.29:6807/3899727946","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":523,"up_from":534,"up_thru":1364,"down_at":524,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6811","nonce":840649505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6813","nonce":840649505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6817","nonce":840649505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6815","nonce":840649505}]},"public_addr":"172.21.15.29:6811/840649505","cluster_addr":"172.21.15.29:6813/840649505","heartbeat_back_addr":"172.21.15.29:6817/840649505","heartbeat_front_addr":"172.21.15.29:6815/840649505","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":544,"up_from":555,"up_thru":1369,"down_at":545,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6819","nonce":840062687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6821","nonce":840062687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6825","nonce":840062687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6823","nonce":840062687}]},"public_addr":"172.21.15.29:6819/840062687","cluster_addr":"172.21.15.29:6821/840062687","heartbeat_back_addr":"172.21.15.29:6825/840062687","heartbeat_front_addr":"172.21.15.29:6823/840062687","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":590,"last_clean_end":1329,"up_from":1342,"up_thru":1388,"down_at":1330,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":1944387251},{"type":"v1","addr":"172.21.15.29:6827","nonce":1944387251}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":1944387251},{"type":"v1","addr":"172.21.15.29:6829","nonce":1944387251}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":1944387251},{"type":"v1","addr":"172.21.15.29:6833","nonce":1944387251}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":1944387251},{"type":"v1","addr":"172.21.15.29:6831","nonce":1944387251}]},"public_addr":"172.21.15.29:6827/1944387251","cluster_addr":"172.21.15.29:6829/1944387251","heartbeat_back_addr":"172.21.15.29:6833/1944387251","heartbeat_front_addr":"172.21.15.29:6831/1944387251","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":600,"up_from":611,"up_thru":1377,"down_at":601,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6801","nonce":3230425210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6803","nonce":3230425210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6807","nonce":3230425210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6805","nonce":3230425210}]},"public_addr":"172.21.15.87:6801/3230425210","cluster_addr":"172.21.15.87:6803/3230425210","heartbeat_back_addr":"172.21.15.87:6807/3230425210","heartbeat_front_addr":"172.21.15.87:6805/3230425210","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":1265,"up_from":1282,"up_thru":1367,"down_at":1265,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3350310311},{"type":"v1","addr":"172.21.15.87:6809","nonce":3350310311}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3350310311},{"type":"v1","addr":"172.21.15.87:6811","nonce":3350310311}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3350310311},{"type":"v1","addr":"172.21.15.87:6815","nonce":3350310311}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3350310311},{"type":"v1","addr":"172.21.15.87:6813","nonce":3350310311}]},"public_addr":"172.21.15.87:6809/3350310311","cluster_addr":"172.21.15.87:6811/3350310311","heartbeat_back_addr":"172.21.15.87:6815/3350310311","heartbeat_front_addr":"172.21.15.87:6813/3350310311","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":1380,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":1383,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":503},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":523},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":544},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":1329},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":600},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":1264},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.11","mappings":[{"from":4,"to":2}]},{"pgid":"4.f","mappings":[{"from":3,"to":2}]},{"pgid":"4.15","mappings":[{"from":3,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.87:0/943640318":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/3128860153":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/2463058069":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2946468060":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/600676740":"2024-04-13T11:17:21.160240+0000","172.21.15.29:6800/686523287":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2241771384":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/3254157714":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/659258741":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.87:6834/1005928369":"2024-04-13T11:16:36.316651+0000","172.21.15.29:6801/3931254419":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/4202867789":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.87:0/1543620546":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000","172.21.15.29:6800/3931254419":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:6801/686523287":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/1221327863":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/1314515623":"2024-04-13T11:17:21.160240+0000","172.21.15.87:6835/1005928369":"2024-04-13T11:16:36.316651+0000","172.21.15.87:0/220307911":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/4218166463":"2024-04-13T11:17:21.160240+0000","172.21.15.87:0/2414938593":"2024-04-13T11:16:36.316651+0000","172.21.15.29:6801/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6800/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.87:0/3681614798":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/364858296":"2024-04-13T11:11:58.108258+0000","172.21.15.87:0/67786467":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/683850481":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/273190388":"2024-04-13T11:17:21.160240+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T12:04:27.499 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-12T12:04:27.499 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd dump --format=json 2024-04-12T12:04:27.708 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:27 smithi029 ceph-mon[97787]: pgmap v2556: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:27.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:27 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3536913384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T12:04:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:27 smithi029 ceph-mon[93650]: pgmap v2556: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:27.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:27 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3536913384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T12:04:27.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:27 smithi087 ceph-mon[89014]: pgmap v2556: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:27.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:27 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3536913384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T12:04:29.551 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:29 smithi029 ceph-mon[93650]: pgmap v2557: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:29.551 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:29 smithi029 ceph-mon[97787]: pgmap v2557: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:29.768 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T12:04:29.769 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":1389,"fsid":"80efffc6-f8bc-11ee-bc8c-c7b262605968","created":"2024-04-12T11:06:06.410593+0000","modified":"2024-04-12T12:04:24.462548+0000","last_up_change":"2024-04-12T12:01:07.324373+0000","last_in_change":"2024-04-12T12:03:14.342599+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T11:09:21.719063+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1264","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1240","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-12T11:11:48.103219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-12T11:12:56.493343+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-12T11:12:57.001342+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4c3c7583-fa1d-4a1d-95ea-77be7e31737e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":503,"up_from":514,"up_thru":1366,"down_at":504,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6803","nonce":3899727946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6805","nonce":3899727946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6809","nonce":3899727946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":3899727946},{"type":"v1","addr":"172.21.15.29:6807","nonce":3899727946}]},"public_addr":"172.21.15.29:6803/3899727946","cluster_addr":"172.21.15.29:6805/3899727946","heartbeat_back_addr":"172.21.15.29:6809/3899727946","heartbeat_front_addr":"172.21.15.29:6807/3899727946","state":["exists","up"]},{"osd":1,"uuid":"58da0c82-e2ec-4067-82d5-05c379001ae5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":523,"up_from":534,"up_thru":1364,"down_at":524,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6811","nonce":840649505}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6813","nonce":840649505}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6817","nonce":840649505}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":840649505},{"type":"v1","addr":"172.21.15.29:6815","nonce":840649505}]},"public_addr":"172.21.15.29:6811/840649505","cluster_addr":"172.21.15.29:6813/840649505","heartbeat_back_addr":"172.21.15.29:6817/840649505","heartbeat_front_addr":"172.21.15.29:6815/840649505","state":["exists","up"]},{"osd":2,"uuid":"53b6c920-2e05-4905-be45-0f33131d358d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":544,"up_from":555,"up_thru":1369,"down_at":545,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6819","nonce":840062687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6821","nonce":840062687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6825","nonce":840062687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":840062687},{"type":"v1","addr":"172.21.15.29:6823","nonce":840062687}]},"public_addr":"172.21.15.29:6819/840062687","cluster_addr":"172.21.15.29:6821/840062687","heartbeat_back_addr":"172.21.15.29:6825/840062687","heartbeat_front_addr":"172.21.15.29:6823/840062687","state":["exists","up"]},{"osd":3,"uuid":"5c33079c-a506-4d4f-97bc-11df1245e5ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":590,"last_clean_end":1329,"up_from":1342,"up_thru":1388,"down_at":1330,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":1944387251},{"type":"v1","addr":"172.21.15.29:6827","nonce":1944387251}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":1944387251},{"type":"v1","addr":"172.21.15.29:6829","nonce":1944387251}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":1944387251},{"type":"v1","addr":"172.21.15.29:6833","nonce":1944387251}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":1944387251},{"type":"v1","addr":"172.21.15.29:6831","nonce":1944387251}]},"public_addr":"172.21.15.29:6827/1944387251","cluster_addr":"172.21.15.29:6829/1944387251","heartbeat_back_addr":"172.21.15.29:6833/1944387251","heartbeat_front_addr":"172.21.15.29:6831/1944387251","state":["exists","up"]},{"osd":4,"uuid":"f776683c-bddc-4a6f-a85d-a43639afadee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":600,"up_from":611,"up_thru":1377,"down_at":601,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6801","nonce":3230425210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6803","nonce":3230425210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6807","nonce":3230425210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":3230425210},{"type":"v1","addr":"172.21.15.87:6805","nonce":3230425210}]},"public_addr":"172.21.15.87:6801/3230425210","cluster_addr":"172.21.15.87:6803/3230425210","heartbeat_back_addr":"172.21.15.87:6807/3230425210","heartbeat_front_addr":"172.21.15.87:6805/3230425210","state":["exists","up"]},{"osd":5,"uuid":"ca436dff-e509-439d-a17b-0f4efe4d4693","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":1265,"up_from":1282,"up_thru":1367,"down_at":1265,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":3350310311},{"type":"v1","addr":"172.21.15.87:6809","nonce":3350310311}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":3350310311},{"type":"v1","addr":"172.21.15.87:6811","nonce":3350310311}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":3350310311},{"type":"v1","addr":"172.21.15.87:6815","nonce":3350310311}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":3350310311},{"type":"v1","addr":"172.21.15.87:6813","nonce":3350310311}]},"public_addr":"172.21.15.87:6809/3350310311","cluster_addr":"172.21.15.87:6811/3350310311","heartbeat_back_addr":"172.21.15.87:6815/3350310311","heartbeat_front_addr":"172.21.15.87:6813/3350310311","state":["exists","up"]},{"osd":6,"uuid":"3daf2021-0642-495d-803d-ad11f134a765","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":1380,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6817","nonce":1395988062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6819","nonce":1395988062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6823","nonce":1395988062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":1395988062},{"type":"v1","addr":"172.21.15.87:6821","nonce":1395988062}]},"public_addr":"172.21.15.87:6817/1395988062","cluster_addr":"172.21.15.87:6819/1395988062","heartbeat_back_addr":"172.21.15.87:6823/1395988062","heartbeat_front_addr":"172.21.15.87:6821/1395988062","state":["exists","up"]},{"osd":7,"uuid":"2e39cb8f-dfe5-462f-8fea-30eb1eb84b5e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":1383,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6825","nonce":2616284515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6827","nonce":2616284515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6831","nonce":2616284515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":2616284515},{"type":"v1","addr":"172.21.15.87:6829","nonce":2616284515}]},"public_addr":"172.21.15.87:6825/2616284515","cluster_addr":"172.21.15.87:6827/2616284515","heartbeat_back_addr":"172.21.15.87:6831/2616284515","heartbeat_front_addr":"172.21.15.87:6829/2616284515","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:29.526967+0000","dead_epoch":503},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:08:54.471186+0000","dead_epoch":523},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:18.587875+0000","dead_epoch":544},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:09:43.477136+0000","dead_epoch":1329},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:05.698244+0000","dead_epoch":600},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:29.812661+0000","dead_epoch":1264},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:10:54.386371+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T11:11:18.479172+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.11","mappings":[{"from":4,"to":2}]},{"pgid":"4.f","mappings":[{"from":3,"to":2}]},{"pgid":"4.15","mappings":[{"from":3,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.87:0/943640318":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/3128860153":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/2463058069":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6800/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2946468060":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/2511162340":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/600676740":"2024-04-13T11:17:21.160240+0000","172.21.15.29:6800/686523287":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/3638696241":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/2241771384":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/3254157714":"2024-04-13T11:11:58.108258+0000","172.21.15.29:0/659258741":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6801/4219816068":"2024-04-13T11:07:09.398471+0000","172.21.15.87:6834/1005928369":"2024-04-13T11:16:36.316651+0000","172.21.15.29:6801/3931254419":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/4202867789":"2024-04-13T11:16:01.391032+0000","172.21.15.29:6800/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.87:0/1543620546":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/4278995423":"2024-04-13T11:06:44.508932+0000","172.21.15.29:6800/3931254419":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/4198974780":"2024-04-13T11:07:09.398471+0000","172.21.15.29:6801/848294730":"2024-04-13T11:06:44.508932+0000","172.21.15.29:6801/686523287":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/2957874151":"2024-04-13T11:07:09.398471+0000","172.21.15.29:0/1221327863":"2024-04-13T11:16:01.391032+0000","172.21.15.29:0/1314515623":"2024-04-13T11:17:21.160240+0000","172.21.15.87:6835/1005928369":"2024-04-13T11:16:36.316651+0000","172.21.15.87:0/220307911":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/4218166463":"2024-04-13T11:17:21.160240+0000","172.21.15.87:0/2414938593":"2024-04-13T11:16:36.316651+0000","172.21.15.29:6801/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.29:6800/1422272459":"2024-04-13T11:11:58.108258+0000","172.21.15.87:0/3681614798":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/3763274316":"2024-04-13T11:06:44.508932+0000","172.21.15.29:0/364858296":"2024-04-13T11:11:58.108258+0000","172.21.15.87:0/67786467":"2024-04-13T11:16:36.316651+0000","172.21.15.29:0/683850481":"2024-04-13T11:17:21.160240+0000","172.21.15.29:0/273190388":"2024-04-13T11:17:21.160240+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T12:04:29.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:29 smithi087 ceph-mon[89014]: pgmap v2557: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:30.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-12T12:04:30.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-12T12:04:30.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-12T12:04:30.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-12T12:04:30.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-12T12:04:30.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-12T12:04:30.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-12T12:04:30.507 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-12T12:04:30.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:30 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/604169926' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T12:04:30.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:30 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/604169926' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T12:04:30.899 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:30.923 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:30.945 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:30.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:30 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/604169926' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T12:04:31.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:31 smithi029 ceph-mon[93650]: pgmap v2558: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:31.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:31 smithi029 ceph-mon[97787]: pgmap v2558: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:31.874 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:31.875 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:31.969 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:31 smithi087 ceph-mon[89014]: pgmap v2558: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:04:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:04:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:04:33.611 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:33 smithi029 ceph-mon[97787]: pgmap v2559: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:33.611 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:33 smithi029 ceph-mon[93650]: pgmap v2559: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:33.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:33 smithi087 ceph-mon[89014]: pgmap v2559: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:35.266 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:04:34 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:04:34] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.43.0" 2024-04-12T12:04:35.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:35 smithi029 ceph-mon[93650]: pgmap v2560: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:35.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:35 smithi029 ceph-mon[97787]: pgmap v2560: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:35.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:35 smithi087 ceph-mon[89014]: pgmap v2560: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:36.231 INFO:teuthology.orchestra.run.smithi029.stdout:5763846111275 2024-04-12T12:04:36.231 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-12T12:04:36.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:36 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:36.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:36 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:36.841 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:36.843 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:36.844 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:36.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:36 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:37.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:37 smithi029 ceph-mon[97787]: pgmap v2561: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:37.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:37 smithi029 ceph-mon[93650]: pgmap v2561: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:37.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:37 smithi087 ceph-mon[89014]: pgmap v2561: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:39 smithi029 ceph-mon[93650]: pgmap v2562: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:39.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:39 smithi029 ceph-mon[97787]: pgmap v2562: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:39.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:39 smithi087 ceph-mon[89014]: pgmap v2562: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:41.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:41 smithi029 ceph-mon[93650]: pgmap v2563: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:41.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:41 smithi029 ceph-mon[97787]: pgmap v2563: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:41.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:41 smithi087 ceph-mon[89014]: pgmap v2563: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:42.595 INFO:teuthology.orchestra.run.smithi029.stdout:231928234628 2024-04-12T12:04:42.596 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-12T12:04:42.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:04:42 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:04:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:04:43.711 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:43 smithi029 ceph-mon[97787]: pgmap v2564: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:43.711 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:43 smithi029 ceph-mon[93650]: pgmap v2564: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:43.738 INFO:teuthology.orchestra.run.smithi029.stdout:197568496265 2024-04-12T12:04:43.739 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-12T12:04:43.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:43 smithi087 ceph-mon[89014]: pgmap v2564: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:44.204 INFO:teuthology.orchestra.run.smithi029.stdout:2624225018351 2024-04-12T12:04:44.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-12T12:04:44.815 INFO:teuthology.orchestra.run.smithi029.stdout:5506148073599 2024-04-12T12:04:44.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-12T12:04:45.233 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:04:44 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:04:44] "GET /metrics HTTP/1.1" 200 34135 "" "Prometheus/2.43.0" 2024-04-12T12:04:45.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:45 smithi029 ceph-mon[93650]: pgmap v2565: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:45.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:45 smithi029 ceph-mon[97787]: pgmap v2565: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:45.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:45 smithi087 ceph-mon[89014]: pgmap v2565: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:46.520 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:47.063 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:47.720 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:47 smithi029 ceph-mon[93650]: pgmap v2566: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:47.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:47 smithi029 ceph-mon[97787]: pgmap v2566: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:47.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:47 smithi087 ceph-mon[89014]: pgmap v2566: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:49.022 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:49.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:49 smithi029 ceph-mon[93650]: pgmap v2567: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:49.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:49 smithi029 ceph-mon[97787]: pgmap v2567: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:49.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:49 smithi087 ceph-mon[89014]: pgmap v2567: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:51.596 INFO:teuthology.orchestra.run.smithi029.stdout:2293512536577 2024-04-12T12:04:51.596 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-12T12:04:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:51 smithi029 ceph-mon[93650]: pgmap v2568: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:51.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:51 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:51 smithi029 ceph-mon[97787]: pgmap v2568: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:51.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:51 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:51.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:51 smithi087 ceph-mon[89014]: pgmap v2568: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:51.968 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:51 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:04:52.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:04:52 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:04:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:04:52.901 INFO:teuthology.orchestra.run.smithi029.stdout:5763846111279 2024-04-12T12:04:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:53 smithi029 ceph-mon[93650]: pgmap v2569: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:53.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:53 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/3628144380' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T12:04:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:53 smithi029 ceph-mon[97787]: pgmap v2569: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:53.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:53 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/3628144380' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T12:04:53.818 INFO:teuthology.orchestra.run.smithi029.stdout:2624225018354 2024-04-12T12:04:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:53 smithi087 ceph-mon[89014]: pgmap v2569: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:53.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:53 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/3628144380' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T12:04:54.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:54 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2970801594' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T12:04:54.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:54 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2970801594' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T12:04:54.854 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5763846111275 got 5763846111279 for osd.3 2024-04-12T12:04:54.854 DEBUG:teuthology.parallel:result is None 2024-04-12T12:04:54.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:54 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2970801594' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T12:04:55.269 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:04:54 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:04:54] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-04-12T12:04:55.333 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:55.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:55 smithi029 ceph-mon[93650]: pgmap v2570: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:55.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:55 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:04:55.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:55 smithi029 ceph-mon[97787]: pgmap v2570: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:55.553 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:55 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:04:55.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:55 smithi087 ceph-mon[89014]: pgmap v2570: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:55.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:55 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T12:04:55.806 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2624225018351 got 2624225018354 for osd.4 2024-04-12T12:04:55.806 DEBUG:teuthology.parallel:result is None 2024-04-12T12:04:56.764 INFO:teuthology.orchestra.run.smithi029.stdout:197568496268 2024-04-12T12:04:57.327 INFO:teuthology.orchestra.run.smithi029.stdout:2207613190661 2024-04-12T12:04:57.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-12T12:04:57.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:57 smithi029 ceph-mon[93650]: pgmap v2571: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:57.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:57 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1597622631' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T12:04:57.616 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:57 smithi029 ceph-mon[97787]: pgmap v2571: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:57.616 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:57 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1597622631' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T12:04:57.651 INFO:teuthology.orchestra.run.smithi029.stdout:2383706849788 2024-04-12T12:04:57.651 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-12T12:04:57.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:57 smithi087 ceph-mon[89014]: pgmap v2571: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:57.719 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:57 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1597622631' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T12:04:58.152 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496265 got 197568496268 for osd.6 2024-04-12T12:04:58.152 DEBUG:teuthology.parallel:result is None 2024-04-12T12:04:58.764 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:58.764 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:58.796 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:04:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:59 smithi087 ceph-mon[89014]: pgmap v2572: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:04:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:04:59.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:04:59 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:00.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:59 smithi029 ceph-mon[97787]: pgmap v2572: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:00.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:00.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:04:59 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:00.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:59 smithi029 ceph-mon[93650]: pgmap v2572: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:00.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:00.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:04:59 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:01.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:01 smithi029 ceph-mon[93650]: pgmap v2573: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:01.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:01 smithi029 ceph-mon[97787]: pgmap v2573: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:01.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:01 smithi087 ceph-mon[89014]: pgmap v2573: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:02.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:05:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:05:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:05:03.156 INFO:teuthology.orchestra.run.smithi029.stdout:231928234633 2024-04-12T12:05:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:03 smithi087 ceph-mon[89014]: pgmap v2574: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:03.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:03 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2087719798' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T12:05:03.972 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:03 smithi029 ceph-mon[97787]: pgmap v2574: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:03.972 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:03 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2087719798' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T12:05:03.972 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:03 smithi029 ceph-mon[93650]: pgmap v2574: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:03.972 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:03 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2087719798' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T12:05:04.295 INFO:teuthology.orchestra.run.smithi029.stdout:2293512536579 2024-04-12T12:05:04.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:04 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/584830913' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T12:05:04.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:04 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/584830913' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T12:05:04.958 INFO:teuthology.orchestra.run.smithi029.stdout:5506148073604 2024-04-12T12:05:04.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:04 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/584830913' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T12:05:05.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:04 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:05:04] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.43.0" 2024-04-12T12:05:05.220 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234628 got 231928234633 for osd.7 2024-04-12T12:05:05.221 DEBUG:teuthology.parallel:result is None 2024-04-12T12:05:05.448 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:05:05.861 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:05 smithi029 ceph-mon[97787]: pgmap v2575: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:05.861 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:05 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/2190433072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T12:05:05.861 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:05 smithi029 ceph-mon[93650]: pgmap v2575: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:05.861 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:05 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/2190433072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T12:05:05.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:05 smithi087 ceph-mon[89014]: pgmap v2575: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:05.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:05 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/2190433072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T12:05:06.299 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5506148073599 got 5506148073604 for osd.5 2024-04-12T12:05:06.299 DEBUG:teuthology.parallel:result is None 2024-04-12T12:05:06.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:06 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:05:07.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:06 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:05:07.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:06 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:05:07.797 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:07 smithi029 ceph-mon[97787]: pgmap v2576: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:07.797 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:07 smithi029 ceph-mon[93650]: pgmap v2576: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:07.870 INFO:teuthology.orchestra.run.smithi029.stdout:2207613190664 2024-04-12T12:05:07.967 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:07 smithi087 ceph-mon[89014]: pgmap v2576: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:08.232 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2293512536577 got 2293512536579 for osd.1 2024-04-12T12:05:08.232 DEBUG:teuthology.parallel:result is None 2024-04-12T12:05:08.791 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:08 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/4169284740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T12:05:08.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:08 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/4169284740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T12:05:09.108 INFO:teuthology.orchestra.run.smithi029.stdout:2383706849792 2024-04-12T12:05:09.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:08 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/4169284740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T12:05:09.451 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2207613190661 got 2207613190664 for osd.0 2024-04-12T12:05:09.452 DEBUG:teuthology.parallel:result is None 2024-04-12T12:05:10.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:09 smithi029 ceph-mon[93650]: pgmap v2577: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:10.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:09 smithi029 ceph-mon[93650]: from='client.? 172.21.15.29:0/1352763866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T12:05:10.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:09 smithi029 ceph-mon[97787]: pgmap v2577: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:10.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:09 smithi029 ceph-mon[97787]: from='client.? 172.21.15.29:0/1352763866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T12:05:10.088 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2383706849788 got 2383706849792 for osd.2 2024-04-12T12:05:10.088 DEBUG:teuthology.parallel:result is None 2024-04-12T12:05:10.088 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-04-12T12:05:10.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T12:05:10.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:09 smithi087 ceph-mon[89014]: pgmap v2577: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:10.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:09 smithi087 ceph-mon[89014]: from='client.? 172.21.15.29:0/1352763866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T12:05:10.830 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:05:10.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:10 smithi029 ceph-mon[93650]: pgmap v2578: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:10.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:10 smithi029 ceph-mon[97787]: pgmap v2578: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:11.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:10 smithi087 ceph-mon[89014]: pgmap v2578: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:12.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:05:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:05:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:05:12.977 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T12:05:12.978 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T12:05:13.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:13 smithi029 ceph-mon[97787]: pgmap v2579: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:13.585 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:13 smithi029 ceph-mon[93650]: pgmap v2579: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:13.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:13 smithi087 ceph-mon[89014]: pgmap v2579: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:14.005 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2579,"stamp":"2024-04-12T12:05:12.318283+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":1719652,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1826,"ondisk_log_size":1826,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5847880,"kb_used_data":72012,"kb_used_omap":310,"kb_used_meta":5775433,"kb_avail":744113336,"statfs":{"total":767960285184,"available":761972056064,"internally_reserved":0,"allocated":73740288,"data_stored":48004362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":318107,"internal_metadata":5914043749},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003155"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":2152,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473414+0000","last_change":"2024-04-12T12:00:35.472642+0000","last_active":"2024-04-12T12:04:24.473414+0000","last_peered":"2024-04-12T12:04:24.473414+0000","last_clean":"2024-04-12T12:04:24.473414+0000","last_became_active":"2024-04-12T12:00:35.472258+0000","last_became_peered":"2024-04-12T12:00:35.472258+0000","last_unstale":"2024-04-12T12:04:24.473414+0000","last_undegraded":"2024-04-12T12:04:24.473414+0000","last_fullsized":"2024-04-12T12:04:24.473414+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:41:16.991252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":648,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477726+0000","last_change":"2024-04-12T12:04:24.477296+0000","last_active":"2024-04-12T12:04:24.477726+0000","last_peered":"2024-04-12T12:04:24.477726+0000","last_clean":"2024-04-12T12:04:24.477726+0000","last_became_active":"2024-04-12T12:04:24.476920+0000","last_became_peered":"2024-04-12T12:04:24.476920+0000","last_unstale":"2024-04-12T12:04:24.477726+0000","last_undegraded":"2024-04-12T12:04:24.477726+0000","last_fullsized":"2024-04-12T12:04:24.477726+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:30:33.305524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":126,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475552+0000","last_change":"2024-04-12T12:01:09.587349+0000","last_active":"2024-04-12T12:04:24.475552+0000","last_peered":"2024-04-12T12:04:24.475552+0000","last_clean":"2024-04-12T12:04:24.475552+0000","last_became_active":"2024-04-12T12:01:09.587019+0000","last_became_peered":"2024-04-12T12:01:09.587019+0000","last_unstale":"2024-04-12T12:04:24.475552+0000","last_undegraded":"2024-04-12T12:04:24.475552+0000","last_fullsized":"2024-04-12T12:04:24.475552+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:19:17.511244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2254,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470068+0000","last_change":"2024-04-12T12:01:08.211946+0000","last_active":"2024-04-12T12:04:24.470068+0000","last_peered":"2024-04-12T12:04:24.470068+0000","last_clean":"2024-04-12T12:04:24.470068+0000","last_became_active":"2024-04-12T12:01:08.209590+0000","last_became_peered":"2024-04-12T12:01:08.209590+0000","last_unstale":"2024-04-12T12:04:24.470068+0000","last_undegraded":"2024-04-12T12:04:24.470068+0000","last_fullsized":"2024-04-12T12:04:24.470068+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:08.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":713,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477767+0000","last_change":"2024-04-12T12:04:24.477356+0000","last_active":"2024-04-12T12:04:24.477767+0000","last_peered":"2024-04-12T12:04:24.477767+0000","last_clean":"2024-04-12T12:04:24.477767+0000","last_became_active":"2024-04-12T12:04:24.477019+0000","last_became_peered":"2024-04-12T12:04:24.477019+0000","last_unstale":"2024-04-12T12:04:24.477767+0000","last_undegraded":"2024-04-12T12:04:24.477767+0000","last_fullsized":"2024-04-12T12:04:24.477767+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:52:40.627145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1394,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477670+0000","last_change":"2024-04-12T12:03:15.361645+0000","last_active":"2024-04-12T12:04:24.477670+0000","last_peered":"2024-04-12T12:04:24.477670+0000","last_clean":"2024-04-12T12:04:24.477670+0000","last_became_active":"2024-04-12T12:03:15.361336+0000","last_became_peered":"2024-04-12T12:03:15.361336+0000","last_unstale":"2024-04-12T12:04:24.477670+0000","last_undegraded":"2024-04-12T12:04:24.477670+0000","last_fullsized":"2024-04-12T12:04:24.477670+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:04:01.224150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2260,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470385+0000","last_change":"2024-04-12T12:03:15.358780+0000","last_active":"2024-04-12T12:04:24.470385+0000","last_peered":"2024-04-12T12:04:24.470385+0000","last_clean":"2024-04-12T12:04:24.470385+0000","last_became_active":"2024-04-12T12:03:15.358388+0000","last_became_peered":"2024-04-12T12:03:15.358388+0000","last_unstale":"2024-04-12T12:04:24.470385+0000","last_undegraded":"2024-04-12T12:04:24.470385+0000","last_fullsized":"2024-04-12T12:04:24.470385+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:25:02.753436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":245,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.472183+0000","last_change":"2024-04-12T12:01:09.514195+0000","last_active":"2024-04-12T12:04:24.472183+0000","last_peered":"2024-04-12T12:04:24.472183+0000","last_clean":"2024-04-12T12:04:24.472183+0000","last_became_active":"2024-04-12T12:01:09.513505+0000","last_became_peered":"2024-04-12T12:01:09.513505+0000","last_unstale":"2024-04-12T12:04:24.472183+0000","last_undegraded":"2024-04-12T12:04:24.472183+0000","last_fullsized":"2024-04-12T12:04:24.472183+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:43.907048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.464548+0000","last_change":"2024-04-12T12:03:15.365051+0000","last_active":"2024-04-12T12:04:23.464548+0000","last_peered":"2024-04-12T12:04:23.464548+0000","last_clean":"2024-04-12T12:04:23.464548+0000","last_became_active":"2024-04-12T12:03:15.364638+0000","last_became_peered":"2024-04-12T12:03:15.364638+0000","last_unstale":"2024-04-12T12:04:23.464548+0000","last_undegraded":"2024-04-12T12:04:23.464548+0000","last_fullsized":"2024-04-12T12:04:23.464548+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:10:57.519022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":559,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.479994+0000","last_change":"2024-04-12T12:04:24.479569+0000","last_active":"2024-04-12T12:04:24.479994+0000","last_peered":"2024-04-12T12:04:24.479994+0000","last_clean":"2024-04-12T12:04:24.479994+0000","last_became_active":"2024-04-12T12:04:24.479219+0000","last_became_peered":"2024-04-12T12:04:24.479219+0000","last_unstale":"2024-04-12T12:04:24.479994+0000","last_undegraded":"2024-04-12T12:04:24.479994+0000","last_fullsized":"2024-04-12T12:04:24.479994+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:49:20.667505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":247,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477729+0000","last_change":"2024-04-12T12:01:08.204030+0000","last_active":"2024-04-12T12:04:24.477729+0000","last_peered":"2024-04-12T12:04:24.477729+0000","last_clean":"2024-04-12T12:04:24.477729+0000","last_became_active":"2024-04-12T12:01:08.203790+0000","last_became_peered":"2024-04-12T12:01:08.203790+0000","last_unstale":"2024-04-12T12:04:24.477729+0000","last_undegraded":"2024-04-12T12:04:24.477729+0000","last_fullsized":"2024-04-12T12:04:24.477729+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:18:55.719738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":128,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475968+0000","last_change":"2024-04-12T12:02:42.081270+0000","last_active":"2024-04-12T12:04:24.475968+0000","last_peered":"2024-04-12T12:04:24.475968+0000","last_clean":"2024-04-12T12:04:24.475968+0000","last_became_active":"2024-04-12T12:02:42.080832+0000","last_became_peered":"2024-04-12T12:02:42.080832+0000","last_unstale":"2024-04-12T12:04:24.475968+0000","last_undegraded":"2024-04-12T12:04:24.475968+0000","last_fullsized":"2024-04-12T12:04:24.475968+0000","mapping_epoch":1354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:06.752668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":2145,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473853+0000","last_change":"2024-04-12T11:45:53.024626+0000","last_active":"2024-04-12T12:04:24.473853+0000","last_peered":"2024-04-12T12:04:24.473853+0000","last_clean":"2024-04-12T12:04:24.473853+0000","last_became_active":"2024-04-12T11:45:52.976276+0000","last_became_peered":"2024-04-12T11:45:52.976276+0000","last_unstale":"2024-04-12T12:04:24.473853+0000","last_undegraded":"2024-04-12T12:04:24.473853+0000","last_fullsized":"2024-04-12T12:04:24.473853+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:49.929210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":2116,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473609+0000","last_change":"2024-04-12T12:00:35.473701+0000","last_active":"2024-04-12T12:04:24.473609+0000","last_peered":"2024-04-12T12:04:24.473609+0000","last_clean":"2024-04-12T12:04:24.473609+0000","last_became_active":"2024-04-12T12:00:35.473318+0000","last_became_peered":"2024-04-12T12:00:35.473318+0000","last_unstale":"2024-04-12T12:04:24.473609+0000","last_undegraded":"2024-04-12T12:04:24.473609+0000","last_fullsized":"2024-04-12T12:04:24.473609+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:45:46.936638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":202,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477649+0000","last_change":"2024-04-12T11:45:53.035604+0000","last_active":"2024-04-12T12:04:24.477649+0000","last_peered":"2024-04-12T12:04:24.477649+0000","last_clean":"2024-04-12T12:04:24.477649+0000","last_became_active":"2024-04-12T11:45:52.984722+0000","last_became_peered":"2024-04-12T11:45:52.984722+0000","last_unstale":"2024-04-12T12:04:24.477649+0000","last_undegraded":"2024-04-12T12:04:24.477649+0000","last_fullsized":"2024-04-12T12:04:24.477649+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:30.674395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":606,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.480387+0000","last_change":"2024-04-12T12:04:24.478981+0000","last_active":"2024-04-12T12:04:24.480387+0000","last_peered":"2024-04-12T12:04:24.480387+0000","last_clean":"2024-04-12T12:04:24.480387+0000","last_became_active":"2024-04-12T12:04:24.478453+0000","last_became_peered":"2024-04-12T12:04:24.478453+0000","last_unstale":"2024-04-12T12:04:24.480387+0000","last_undegraded":"2024-04-12T12:04:24.480387+0000","last_fullsized":"2024-04-12T12:04:24.480387+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:09:19.217383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2237,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.469158+0000","last_change":"2024-04-12T12:01:08.222480+0000","last_active":"2024-04-12T12:04:24.469158+0000","last_peered":"2024-04-12T12:04:24.469158+0000","last_clean":"2024-04-12T12:04:24.469158+0000","last_became_active":"2024-04-12T12:01:08.222018+0000","last_became_peered":"2024-04-12T12:01:08.222018+0000","last_unstale":"2024-04-12T12:04:24.469158+0000","last_undegraded":"2024-04-12T12:04:24.469158+0000","last_fullsized":"2024-04-12T12:04:24.469158+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:12:10.882773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465096+0000","last_change":"2024-04-12T12:03:15.365615+0000","last_active":"2024-04-12T12:04:23.465096+0000","last_peered":"2024-04-12T12:04:23.465096+0000","last_clean":"2024-04-12T12:04:23.465096+0000","last_became_active":"2024-04-12T12:03:15.365188+0000","last_became_peered":"2024-04-12T12:03:15.365188+0000","last_unstale":"2024-04-12T12:04:23.465096+0000","last_undegraded":"2024-04-12T12:04:23.465096+0000","last_fullsized":"2024-04-12T12:04:23.465096+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:03:15.046301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1190,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465081+0000","last_change":"2024-04-12T12:04:10.630821+0000","last_active":"2024-04-12T12:04:23.465081+0000","last_peered":"2024-04-12T12:04:23.465081+0000","last_clean":"2024-04-12T12:04:23.465081+0000","last_became_active":"2024-04-12T12:04:10.630430+0000","last_became_peered":"2024-04-12T12:04:10.630430+0000","last_unstale":"2024-04-12T12:04:23.465081+0000","last_undegraded":"2024-04-12T12:04:23.465081+0000","last_fullsized":"2024-04-12T12:04:23.465081+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:21:01.037409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2237,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470153+0000","last_change":"2024-04-12T12:01:08.213742+0000","last_active":"2024-04-12T12:04:24.470153+0000","last_peered":"2024-04-12T12:04:24.470153+0000","last_clean":"2024-04-12T12:04:24.470153+0000","last_became_active":"2024-04-12T12:01:08.211529+0000","last_became_peered":"2024-04-12T12:01:08.211529+0000","last_unstale":"2024-04-12T12:04:24.470153+0000","last_undegraded":"2024-04-12T12:04:24.470153+0000","last_fullsized":"2024-04-12T12:04:24.470153+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:18.227656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1411,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467067+0000","last_change":"2024-04-12T12:03:15.370352+0000","last_active":"2024-04-12T12:04:23.467067+0000","last_peered":"2024-04-12T12:04:23.467067+0000","last_clean":"2024-04-12T12:04:23.467067+0000","last_became_active":"2024-04-12T12:03:15.357513+0000","last_became_peered":"2024-04-12T12:03:15.357513+0000","last_unstale":"2024-04-12T12:04:23.467067+0000","last_undegraded":"2024-04-12T12:04:23.467067+0000","last_fullsized":"2024-04-12T12:04:23.467067+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:03:34.376607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":250,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.472443+0000","last_change":"2024-04-12T12:03:26.502220+0000","last_active":"2024-04-12T12:04:24.472443+0000","last_peered":"2024-04-12T12:04:24.472443+0000","last_clean":"2024-04-12T12:04:24.472443+0000","last_became_active":"2024-04-12T12:03:26.501584+0000","last_became_peered":"2024-04-12T12:03:26.501584+0000","last_unstale":"2024-04-12T12:04:24.472443+0000","last_undegraded":"2024-04-12T12:04:24.472443+0000","last_fullsized":"2024-04-12T12:04:24.472443+0000","mapping_epoch":1369,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:47:25.800012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":200,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466317+0000","last_change":"2024-04-12T11:45:53.028640+0000","last_active":"2024-04-12T12:04:23.466317+0000","last_peered":"2024-04-12T12:04:23.466317+0000","last_clean":"2024-04-12T12:04:23.466317+0000","last_became_active":"2024-04-12T11:45:52.983099+0000","last_became_peered":"2024-04-12T11:45:52.983099+0000","last_unstale":"2024-04-12T12:04:23.466317+0000","last_undegraded":"2024-04-12T12:04:23.466317+0000","last_fullsized":"2024-04-12T12:04:23.466317+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:43:01.897014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465274+0000","last_change":"2024-04-12T12:03:15.361270+0000","last_active":"2024-04-12T12:04:23.465274+0000","last_peered":"2024-04-12T12:04:23.465274+0000","last_clean":"2024-04-12T12:04:23.465274+0000","last_became_active":"2024-04-12T12:03:15.360962+0000","last_became_peered":"2024-04-12T12:03:15.360962+0000","last_unstale":"2024-04-12T12:04:23.465274+0000","last_undegraded":"2024-04-12T12:04:23.465274+0000","last_fullsized":"2024-04-12T12:04:23.465274+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:32:12.780528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":226,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466369+0000","last_change":"2024-04-12T12:00:35.474735+0000","last_active":"2024-04-12T12:04:23.466369+0000","last_peered":"2024-04-12T12:04:23.466369+0000","last_clean":"2024-04-12T12:04:23.466369+0000","last_became_active":"2024-04-12T12:00:35.463433+0000","last_became_peered":"2024-04-12T12:00:35.463433+0000","last_unstale":"2024-04-12T12:04:23.466369+0000","last_undegraded":"2024-04-12T12:04:23.466369+0000","last_fullsized":"2024-04-12T12:04:23.466369+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:27.215562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":2059,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473394+0000","last_change":"2024-04-12T12:03:57.139522+0000","last_active":"2024-04-12T12:04:24.473394+0000","last_peered":"2024-04-12T12:04:24.473394+0000","last_clean":"2024-04-12T12:04:24.473394+0000","last_became_active":"2024-04-12T12:03:57.139257+0000","last_became_peered":"2024-04-12T12:03:57.139257+0000","last_unstale":"2024-04-12T12:04:24.473394+0000","last_undegraded":"2024-04-12T12:04:24.473394+0000","last_fullsized":"2024-04-12T12:04:24.473394+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:48:56.189762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":171,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466384+0000","last_change":"2024-04-12T12:04:10.417903+0000","last_active":"2024-04-12T12:04:23.466384+0000","last_peered":"2024-04-12T12:04:23.466384+0000","last_clean":"2024-04-12T12:04:23.466384+0000","last_became_active":"2024-04-12T12:04:10.417540+0000","last_became_peered":"2024-04-12T12:04:10.417540+0000","last_unstale":"2024-04-12T12:04:23.466384+0000","last_undegraded":"2024-04-12T12:04:23.466384+0000","last_fullsized":"2024-04-12T12:04:23.466384+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:11.426128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1390,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476758+0000","last_change":"2024-04-12T12:00:35.474945+0000","last_active":"2024-04-12T12:04:24.476758+0000","last_peered":"2024-04-12T12:04:24.476758+0000","last_clean":"2024-04-12T12:04:24.476758+0000","last_became_active":"2024-04-12T12:00:35.474564+0000","last_became_peered":"2024-04-12T12:00:35.474564+0000","last_unstale":"2024-04-12T12:04:24.476758+0000","last_undegraded":"2024-04-12T12:04:24.476758+0000","last_fullsized":"2024-04-12T12:04:24.476758+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:13:53.178867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466088+0000","last_change":"2024-04-12T12:03:15.359143+0000","last_active":"2024-04-12T12:04:23.466088+0000","last_peered":"2024-04-12T12:04:23.466088+0000","last_clean":"2024-04-12T12:04:23.466088+0000","last_became_active":"2024-04-12T12:03:15.358807+0000","last_became_peered":"2024-04-12T12:03:15.358807+0000","last_unstale":"2024-04-12T12:04:23.466088+0000","last_undegraded":"2024-04-12T12:04:23.466088+0000","last_fullsized":"2024-04-12T12:04:23.466088+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:24:08.985614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2218,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.469380+0000","last_change":"2024-04-12T12:00:35.475414+0000","last_active":"2024-04-12T12:04:24.469380+0000","last_peered":"2024-04-12T12:04:24.469380+0000","last_clean":"2024-04-12T12:04:24.469380+0000","last_became_active":"2024-04-12T12:00:35.475098+0000","last_became_peered":"2024-04-12T12:00:35.475098+0000","last_unstale":"2024-04-12T12:04:24.469380+0000","last_undegraded":"2024-04-12T12:04:24.469380+0000","last_fullsized":"2024-04-12T12:04:24.469380+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:25:12.377240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"249'161","reported_seq":1755,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465250+0000","last_change":"2024-04-12T12:01:08.230318+0000","last_active":"2024-04-12T12:04:23.465250+0000","last_peered":"2024-04-12T12:04:23.465250+0000","last_clean":"2024-04-12T12:04:23.465250+0000","last_became_active":"2024-04-12T12:01:08.229959+0000","last_became_peered":"2024-04-12T12:01:08.229959+0000","last_unstale":"2024-04-12T12:04:23.465250+0000","last_undegraded":"2024-04-12T12:04:23.465250+0000","last_fullsized":"2024-04-12T12:04:23.465250+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:08.650841+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:08.650841+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:17:07.248029+0000","scrub_duration":0.001456279,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":206,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476401+0000","last_change":"2024-04-12T12:04:24.476114+0000","last_active":"2024-04-12T12:04:24.476401+0000","last_peered":"2024-04-12T12:04:24.476401+0000","last_clean":"2024-04-12T12:04:24.476401+0000","last_became_active":"2024-04-12T12:04:24.475751+0000","last_became_peered":"2024-04-12T12:04:24.475751+0000","last_unstale":"2024-04-12T12:04:24.476401+0000","last_undegraded":"2024-04-12T12:04:24.476401+0000","last_fullsized":"2024-04-12T12:04:24.476401+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:38.380779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":2034,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473834+0000","last_change":"2024-04-12T12:03:57.175404+0000","last_active":"2024-04-12T12:04:24.473834+0000","last_peered":"2024-04-12T12:04:24.473834+0000","last_clean":"2024-04-12T12:04:24.473834+0000","last_became_active":"2024-04-12T12:03:57.174610+0000","last_became_peered":"2024-04-12T12:03:57.174610+0000","last_unstale":"2024-04-12T12:04:24.473834+0000","last_undegraded":"2024-04-12T12:04:24.473834+0000","last_fullsized":"2024-04-12T12:04:24.473834+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:26:49.922463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"249'161","reported_seq":896,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465356+0000","last_change":"2024-04-12T12:03:18.377882+0000","last_active":"2024-04-12T12:04:23.465356+0000","last_peered":"2024-04-12T12:04:23.465356+0000","last_clean":"2024-04-12T12:04:23.465356+0000","last_became_active":"2024-04-12T12:03:18.377558+0000","last_became_peered":"2024-04-12T12:03:18.377558+0000","last_unstale":"2024-04-12T12:04:23.465356+0000","last_undegraded":"2024-04-12T12:04:23.465356+0000","last_fullsized":"2024-04-12T12:04:23.465356+0000","mapping_epoch":1367,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1368,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:10.437190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:10.437190+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:29:03.934380+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1087,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467151+0000","last_change":"2024-04-12T12:04:03.162515+0000","last_active":"2024-04-12T12:04:23.467151+0000","last_peered":"2024-04-12T12:04:23.467151+0000","last_clean":"2024-04-12T12:04:23.467151+0000","last_became_active":"2024-04-12T12:04:03.162228+0000","last_became_peered":"2024-04-12T12:04:03.162228+0000","last_unstale":"2024-04-12T12:04:23.467151+0000","last_undegraded":"2024-04-12T12:04:23.467151+0000","last_fullsized":"2024-04-12T12:04:23.467151+0000","mapping_epoch":1380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:45:43.194555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":2038,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.474024+0000","last_change":"2024-04-12T12:03:57.175845+0000","last_active":"2024-04-12T12:04:24.474024+0000","last_peered":"2024-04-12T12:04:24.474024+0000","last_clean":"2024-04-12T12:04:24.474024+0000","last_became_active":"2024-04-12T12:03:57.175454+0000","last_became_peered":"2024-04-12T12:03:57.175454+0000","last_unstale":"2024-04-12T12:04:24.474024+0000","last_undegraded":"2024-04-12T12:04:24.474024+0000","last_fullsized":"2024-04-12T12:04:24.474024+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:27:03.901168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":2139,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473570+0000","last_change":"2024-04-12T11:45:52.976139+0000","last_active":"2024-04-12T12:04:24.473570+0000","last_peered":"2024-04-12T12:04:24.473570+0000","last_clean":"2024-04-12T12:04:24.473570+0000","last_became_active":"2024-04-12T11:45:52.975803+0000","last_became_peered":"2024-04-12T11:45:52.975803+0000","last_unstale":"2024-04-12T12:04:24.473570+0000","last_undegraded":"2024-04-12T12:04:24.473570+0000","last_fullsized":"2024-04-12T12:04:24.473570+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:46.618143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":2120,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.474076+0000","last_change":"2024-04-12T12:00:35.473079+0000","last_active":"2024-04-12T12:04:24.474076+0000","last_peered":"2024-04-12T12:04:24.474076+0000","last_clean":"2024-04-12T12:04:24.474076+0000","last_became_active":"2024-04-12T12:00:35.472773+0000","last_became_peered":"2024-04-12T12:00:35.472773+0000","last_unstale":"2024-04-12T12:04:24.474076+0000","last_undegraded":"2024-04-12T12:04:24.474076+0000","last_fullsized":"2024-04-12T12:04:24.474076+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:20:42.192043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"249'161","reported_seq":858,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465013+0000","last_change":"2024-04-12T12:03:18.378083+0000","last_active":"2024-04-12T12:04:23.465013+0000","last_peered":"2024-04-12T12:04:23.465013+0000","last_clean":"2024-04-12T12:04:23.465013+0000","last_became_active":"2024-04-12T12:03:18.377735+0000","last_became_peered":"2024-04-12T12:03:18.377735+0000","last_unstale":"2024-04-12T12:04:23.465013+0000","last_undegraded":"2024-04-12T12:04:23.465013+0000","last_fullsized":"2024-04-12T12:04:23.465013+0000","mapping_epoch":1367,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1368,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:11.447056+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:11.447056+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:48:05.161883+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":605,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.479624+0000","last_change":"2024-04-12T12:04:24.478786+0000","last_active":"2024-04-12T12:04:24.479624+0000","last_peered":"2024-04-12T12:04:24.479624+0000","last_clean":"2024-04-12T12:04:24.479624+0000","last_became_active":"2024-04-12T12:04:24.478377+0000","last_became_peered":"2024-04-12T12:04:24.478377+0000","last_unstale":"2024-04-12T12:04:24.479624+0000","last_undegraded":"2024-04-12T12:04:24.479624+0000","last_fullsized":"2024-04-12T12:04:24.479624+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:26:00.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":123,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475984+0000","last_change":"2024-04-12T12:00:35.477779+0000","last_active":"2024-04-12T12:04:24.475984+0000","last_peered":"2024-04-12T12:04:24.475984+0000","last_clean":"2024-04-12T12:04:24.475984+0000","last_became_active":"2024-04-12T12:00:35.477462+0000","last_became_peered":"2024-04-12T12:00:35.477462+0000","last_unstale":"2024-04-12T12:04:24.475984+0000","last_undegraded":"2024-04-12T12:04:24.475984+0000","last_fullsized":"2024-04-12T12:04:24.475984+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:51:17.465133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"249'161","reported_seq":658,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467277+0000","last_change":"2024-04-12T12:01:08.203484+0000","last_active":"2024-04-12T12:04:23.467277+0000","last_peered":"2024-04-12T12:04:23.467277+0000","last_clean":"2024-04-12T12:04:23.467277+0000","last_became_active":"2024-04-12T12:01:08.203128+0000","last_became_peered":"2024-04-12T12:01:08.203128+0000","last_unstale":"2024-04-12T12:04:23.467277+0000","last_undegraded":"2024-04-12T12:04:23.467277+0000","last_fullsized":"2024-04-12T12:04:23.467277+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:37.607274+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:37.607274+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:03:30.798799+0000","scrub_duration":0.0037173240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":123,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476396+0000","last_change":"2024-04-12T12:00:35.473414+0000","last_active":"2024-04-12T12:04:24.476396+0000","last_peered":"2024-04-12T12:04:24.476396+0000","last_clean":"2024-04-12T12:04:24.476396+0000","last_became_active":"2024-04-12T12:00:35.472846+0000","last_became_peered":"2024-04-12T12:00:35.472846+0000","last_unstale":"2024-04-12T12:04:24.476396+0000","last_undegraded":"2024-04-12T12:04:24.476396+0000","last_fullsized":"2024-04-12T12:04:24.476396+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:59:19.941070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465176+0000","last_change":"2024-04-12T12:03:15.365784+0000","last_active":"2024-04-12T12:04:23.465176+0000","last_peered":"2024-04-12T12:04:23.465176+0000","last_clean":"2024-04-12T12:04:23.465176+0000","last_became_active":"2024-04-12T12:03:15.365270+0000","last_became_peered":"2024-04-12T12:03:15.365270+0000","last_unstale":"2024-04-12T12:04:23.465176+0000","last_undegraded":"2024-04-12T12:04:23.465176+0000","last_fullsized":"2024-04-12T12:04:23.465176+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:42:06.789362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":129,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475914+0000","last_change":"2024-04-12T12:03:15.365465+0000","last_active":"2024-04-12T12:04:24.475914+0000","last_peered":"2024-04-12T12:04:24.475914+0000","last_clean":"2024-04-12T12:04:24.475914+0000","last_became_active":"2024-04-12T12:03:15.365077+0000","last_became_peered":"2024-04-12T12:03:15.365077+0000","last_unstale":"2024-04-12T12:04:24.475914+0000","last_undegraded":"2024-04-12T12:04:24.475914+0000","last_fullsized":"2024-04-12T12:04:24.475914+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:34:35.106519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"249'161","reported_seq":862,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.464583+0000","last_change":"2024-04-12T12:03:18.377941+0000","last_active":"2024-04-12T12:04:23.464583+0000","last_peered":"2024-04-12T12:04:23.464583+0000","last_clean":"2024-04-12T12:04:23.464583+0000","last_became_active":"2024-04-12T12:03:18.377623+0000","last_became_peered":"2024-04-12T12:03:18.377623+0000","last_unstale":"2024-04-12T12:04:23.464583+0000","last_undegraded":"2024-04-12T12:04:23.464583+0000","last_fullsized":"2024-04-12T12:04:23.464583+0000","mapping_epoch":1367,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1368,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:05.639497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:05.639497+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:05:27.861150+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465097+0000","last_change":"2024-04-12T12:03:15.365182+0000","last_active":"2024-04-12T12:04:23.465097+0000","last_peered":"2024-04-12T12:04:23.465097+0000","last_clean":"2024-04-12T12:04:23.465097+0000","last_became_active":"2024-04-12T12:03:15.364842+0000","last_became_peered":"2024-04-12T12:03:15.364842+0000","last_unstale":"2024-04-12T12:04:23.465097+0000","last_undegraded":"2024-04-12T12:04:23.465097+0000","last_fullsized":"2024-04-12T12:04:23.465097+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:11:41.126247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":2019,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473493+0000","last_change":"2024-04-12T12:03:57.175432+0000","last_active":"2024-04-12T12:04:24.473493+0000","last_peered":"2024-04-12T12:04:24.473493+0000","last_clean":"2024-04-12T12:04:24.473493+0000","last_became_active":"2024-04-12T12:03:57.175010+0000","last_became_peered":"2024-04-12T12:03:57.175010+0000","last_unstale":"2024-04-12T12:04:24.473493+0000","last_undegraded":"2024-04-12T12:04:24.473493+0000","last_fullsized":"2024-04-12T12:04:24.473493+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:41:03.170360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":127,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476356+0000","last_change":"2024-04-12T12:01:09.225968+0000","last_active":"2024-04-12T12:04:24.476356+0000","last_peered":"2024-04-12T12:04:24.476356+0000","last_clean":"2024-04-12T12:04:24.476356+0000","last_became_active":"2024-04-12T12:01:09.225273+0000","last_became_peered":"2024-04-12T12:01:09.225273+0000","last_unstale":"2024-04-12T12:04:24.476356+0000","last_undegraded":"2024-04-12T12:04:24.476356+0000","last_fullsized":"2024-04-12T12:04:24.476356+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:39.143286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"249'161","reported_seq":1692,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.464970+0000","last_change":"2024-04-12T12:04:10.419038+0000","last_active":"2024-04-12T12:04:23.464970+0000","last_peered":"2024-04-12T12:04:23.464970+0000","last_clean":"2024-04-12T12:04:23.464970+0000","last_became_active":"2024-04-12T12:04:10.418687+0000","last_became_peered":"2024-04-12T12:04:10.418687+0000","last_unstale":"2024-04-12T12:04:23.464970+0000","last_undegraded":"2024-04-12T12:04:23.464970+0000","last_fullsized":"2024-04-12T12:04:23.464970+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1384,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:36.979380+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:36.979380+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:24:24.960842+0000","scrub_duration":0.0018052980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2200,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.468737+0000","last_change":"2024-04-12T12:01:08.241455+0000","last_active":"2024-04-12T12:04:24.468737+0000","last_peered":"2024-04-12T12:04:24.468737+0000","last_clean":"2024-04-12T12:04:24.468737+0000","last_became_active":"2024-04-12T12:01:08.240087+0000","last_became_peered":"2024-04-12T12:01:08.240087+0000","last_unstale":"2024-04-12T12:04:24.468737+0000","last_undegraded":"2024-04-12T12:04:24.468737+0000","last_fullsized":"2024-04-12T12:04:24.468737+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:45:12.951549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":123,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475960+0000","last_change":"2024-04-12T12:00:35.471479+0000","last_active":"2024-04-12T12:04:24.475960+0000","last_peered":"2024-04-12T12:04:24.475960+0000","last_clean":"2024-04-12T12:04:24.475960+0000","last_became_active":"2024-04-12T12:00:35.471015+0000","last_became_peered":"2024-04-12T12:00:35.471015+0000","last_unstale":"2024-04-12T12:04:24.475960+0000","last_undegraded":"2024-04-12T12:04:24.475960+0000","last_fullsized":"2024-04-12T12:04:24.475960+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:55:50.471132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":129,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475793+0000","last_change":"2024-04-12T12:03:15.365552+0000","last_active":"2024-04-12T12:04:24.475793+0000","last_peered":"2024-04-12T12:04:24.475793+0000","last_clean":"2024-04-12T12:04:24.475793+0000","last_became_active":"2024-04-12T12:03:15.365134+0000","last_became_peered":"2024-04-12T12:03:15.365134+0000","last_unstale":"2024-04-12T12:04:24.475793+0000","last_undegraded":"2024-04-12T12:04:24.475793+0000","last_fullsized":"2024-04-12T12:04:24.475793+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:03:25.869378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466429+0000","last_change":"2024-04-12T12:03:15.361282+0000","last_active":"2024-04-12T12:04:23.466429+0000","last_peered":"2024-04-12T12:04:23.466429+0000","last_clean":"2024-04-12T12:04:23.466429+0000","last_became_active":"2024-04-12T12:03:15.360980+0000","last_became_peered":"2024-04-12T12:03:15.360980+0000","last_unstale":"2024-04-12T12:04:23.466429+0000","last_undegraded":"2024-04-12T12:04:23.466429+0000","last_fullsized":"2024-04-12T12:04:23.466429+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:00:27.301014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1724,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466036+0000","last_change":"2024-04-12T11:53:06.603022+0000","last_active":"2024-04-12T12:04:23.466036+0000","last_peered":"2024-04-12T12:04:23.466036+0000","last_clean":"2024-04-12T12:04:23.466036+0000","last_became_active":"2024-04-12T11:49:07.217921+0000","last_became_peered":"2024-04-12T11:49:07.217921+0000","last_unstale":"2024-04-12T12:04:23.466036+0000","last_undegraded":"2024-04-12T12:04:23.466036+0000","last_fullsized":"2024-04-12T12:04:23.466036+0000","mapping_epoch":1210,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":0,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:06.602918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:06.602918+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:40.776170+0000","scrub_duration":0.001429877,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":576,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.478779+0000","last_change":"2024-04-12T12:04:24.478415+0000","last_active":"2024-04-12T12:04:24.478779+0000","last_peered":"2024-04-12T12:04:24.478779+0000","last_clean":"2024-04-12T12:04:24.478779+0000","last_became_active":"2024-04-12T12:04:24.478091+0000","last_became_peered":"2024-04-12T12:04:24.478091+0000","last_unstale":"2024-04-12T12:04:24.478779+0000","last_undegraded":"2024-04-12T12:04:24.478779+0000","last_fullsized":"2024-04-12T12:04:24.478779+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:10:24.813865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1363,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467268+0000","last_change":"2024-04-12T11:45:53.016616+0000","last_active":"2024-04-12T12:04:23.467268+0000","last_peered":"2024-04-12T12:04:23.467268+0000","last_clean":"2024-04-12T12:04:23.467268+0000","last_became_active":"2024-04-12T11:45:53.016292+0000","last_became_peered":"2024-04-12T11:45:53.016292+0000","last_unstale":"2024-04-12T12:04:23.467268+0000","last_undegraded":"2024-04-12T12:04:23.467268+0000","last_fullsized":"2024-04-12T12:04:23.467268+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:53:29.382293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"249'161","reported_seq":789,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475840+0000","last_change":"2024-04-12T12:02:44.218750+0000","last_active":"2024-04-12T12:04:24.475840+0000","last_peered":"2024-04-12T12:04:24.475840+0000","last_clean":"2024-04-12T12:04:24.475840+0000","last_became_active":"2024-04-12T12:02:44.218240+0000","last_became_peered":"2024-04-12T12:02:44.218240+0000","last_unstale":"2024-04-12T12:04:24.475840+0000","last_undegraded":"2024-04-12T12:04:24.475840+0000","last_fullsized":"2024-04-12T12:04:24.475840+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:57:49.744017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:57:49.744017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:07.839300+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":554,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465942+0000","last_change":"2024-04-12T12:04:10.631148+0000","last_active":"2024-04-12T12:04:23.465942+0000","last_peered":"2024-04-12T12:04:23.465942+0000","last_clean":"2024-04-12T12:04:23.465942+0000","last_became_active":"2024-04-12T12:04:10.630769+0000","last_became_peered":"2024-04-12T12:04:10.630769+0000","last_unstale":"2024-04-12T12:04:23.465942+0000","last_undegraded":"2024-04-12T12:04:23.465942+0000","last_fullsized":"2024-04-12T12:04:23.465942+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:30:57.067503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2237,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470269+0000","last_change":"2024-04-12T12:01:08.241097+0000","last_active":"2024-04-12T12:04:24.470269+0000","last_peered":"2024-04-12T12:04:24.470269+0000","last_clean":"2024-04-12T12:04:24.470269+0000","last_became_active":"2024-04-12T12:01:08.240402+0000","last_became_peered":"2024-04-12T12:01:08.240402+0000","last_unstale":"2024-04-12T12:04:24.470269+0000","last_undegraded":"2024-04-12T12:04:24.470269+0000","last_fullsized":"2024-04-12T12:04:24.470269+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:30:15.712301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2216,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470214+0000","last_change":"2024-04-12T12:00:35.480505+0000","last_active":"2024-04-12T12:04:24.470214+0000","last_peered":"2024-04-12T12:04:24.470214+0000","last_clean":"2024-04-12T12:04:24.470214+0000","last_became_active":"2024-04-12T12:00:35.480136+0000","last_became_peered":"2024-04-12T12:00:35.480136+0000","last_unstale":"2024-04-12T12:04:24.470214+0000","last_undegraded":"2024-04-12T12:04:24.470214+0000","last_fullsized":"2024-04-12T12:04:24.470214+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:12:05.320400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"249'161","reported_seq":732,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467652+0000","last_change":"2024-04-12T12:01:08.209702+0000","last_active":"2024-04-12T12:04:23.467652+0000","last_peered":"2024-04-12T12:04:23.467652+0000","last_clean":"2024-04-12T12:04:23.467652+0000","last_became_active":"2024-04-12T12:01:08.209465+0000","last_became_peered":"2024-04-12T12:01:08.209465+0000","last_unstale":"2024-04-12T12:04:23.467652+0000","last_undegraded":"2024-04-12T12:04:23.467652+0000","last_fullsized":"2024-04-12T12:04:23.467652+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:05.130465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:05.130465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:53:20.786613+0000","scrub_duration":0.001397695,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":197,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477656+0000","last_change":"2024-04-12T11:45:53.005585+0000","last_active":"2024-04-12T12:04:24.477656+0000","last_peered":"2024-04-12T12:04:24.477656+0000","last_clean":"2024-04-12T12:04:24.477656+0000","last_became_active":"2024-04-12T11:45:52.987880+0000","last_became_peered":"2024-04-12T11:45:52.987880+0000","last_unstale":"2024-04-12T12:04:24.477656+0000","last_undegraded":"2024-04-12T12:04:24.477656+0000","last_fullsized":"2024-04-12T12:04:24.477656+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:09.819861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":129,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476461+0000","last_change":"2024-04-12T12:03:15.366238+0000","last_active":"2024-04-12T12:04:24.476461+0000","last_peered":"2024-04-12T12:04:24.476461+0000","last_clean":"2024-04-12T12:04:24.476461+0000","last_became_active":"2024-04-12T12:03:15.365749+0000","last_became_peered":"2024-04-12T12:03:15.365749+0000","last_unstale":"2024-04-12T12:04:24.476461+0000","last_undegraded":"2024-04-12T12:04:24.476461+0000","last_fullsized":"2024-04-12T12:04:24.476461+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:57:20.333454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2226,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.469647+0000","last_change":"2024-04-12T12:03:15.362063+0000","last_active":"2024-04-12T12:04:24.469647+0000","last_peered":"2024-04-12T12:04:24.469647+0000","last_clean":"2024-04-12T12:04:24.469647+0000","last_became_active":"2024-04-12T12:03:15.361691+0000","last_became_peered":"2024-04-12T12:03:15.361691+0000","last_unstale":"2024-04-12T12:04:24.469647+0000","last_undegraded":"2024-04-12T12:04:24.469647+0000","last_fullsized":"2024-04-12T12:04:24.469647+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:11:02.269246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"249'161","reported_seq":885,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473899+0000","last_change":"2024-04-12T12:02:44.216787+0000","last_active":"2024-04-12T12:04:24.473899+0000","last_peered":"2024-04-12T12:04:24.473899+0000","last_clean":"2024-04-12T12:04:24.473899+0000","last_became_active":"2024-04-12T12:02:44.216350+0000","last_became_peered":"2024-04-12T12:02:44.216350+0000","last_unstale":"2024-04-12T12:04:24.473899+0000","last_undegraded":"2024-04-12T12:04:24.473899+0000","last_fullsized":"2024-04-12T12:04:24.473899+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:10.604457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:10.604457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:01:05.908018+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":206,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.480437+0000","last_change":"2024-04-12T12:04:24.480110+0000","last_active":"2024-04-12T12:04:24.480437+0000","last_peered":"2024-04-12T12:04:24.480437+0000","last_clean":"2024-04-12T12:04:24.480437+0000","last_became_active":"2024-04-12T12:04:24.479426+0000","last_became_peered":"2024-04-12T12:04:24.479426+0000","last_unstale":"2024-04-12T12:04:24.480437+0000","last_undegraded":"2024-04-12T12:04:24.480437+0000","last_fullsized":"2024-04-12T12:04:24.480437+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:09:59.442971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":123,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476058+0000","last_change":"2024-04-12T12:00:35.477662+0000","last_active":"2024-04-12T12:04:24.476058+0000","last_peered":"2024-04-12T12:04:24.476058+0000","last_clean":"2024-04-12T12:04:24.476058+0000","last_became_active":"2024-04-12T12:00:35.477235+0000","last_became_peered":"2024-04-12T12:00:35.477235+0000","last_unstale":"2024-04-12T12:04:24.476058+0000","last_undegraded":"2024-04-12T12:04:24.476058+0000","last_fullsized":"2024-04-12T12:04:24.476058+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:18:34.713724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":203,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.480281+0000","last_change":"2024-04-12T12:04:24.479467+0000","last_active":"2024-04-12T12:04:24.480281+0000","last_peered":"2024-04-12T12:04:24.480281+0000","last_clean":"2024-04-12T12:04:24.480281+0000","last_became_active":"2024-04-12T12:04:24.479148+0000","last_became_peered":"2024-04-12T12:04:24.479148+0000","last_unstale":"2024-04-12T12:04:24.480281+0000","last_undegraded":"2024-04-12T12:04:24.480281+0000","last_fullsized":"2024-04-12T12:04:24.480281+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:18.428976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"249'161","reported_seq":769,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.480253+0000","last_change":"2024-04-12T12:04:24.480253+0000","last_active":"2024-04-12T12:04:24.480253+0000","last_peered":"2024-04-12T12:04:24.480253+0000","last_clean":"2024-04-12T12:04:24.480253+0000","last_became_active":"2024-04-12T12:04:24.479393+0000","last_became_peered":"2024-04-12T12:04:24.479393+0000","last_unstale":"2024-04-12T12:04:24.480253+0000","last_undegraded":"2024-04-12T12:04:24.480253+0000","last_fullsized":"2024-04-12T12:04:24.480253+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:17.062855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:17.062855+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:12:45.686867+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1396,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467451+0000","last_change":"2024-04-12T11:45:52.979745+0000","last_active":"2024-04-12T12:04:23.467451+0000","last_peered":"2024-04-12T12:04:23.467451+0000","last_clean":"2024-04-12T12:04:23.467451+0000","last_became_active":"2024-04-12T11:45:52.978936+0000","last_became_peered":"2024-04-12T11:45:52.978936+0000","last_unstale":"2024-04-12T12:04:23.467451+0000","last_undegraded":"2024-04-12T12:04:23.467451+0000","last_fullsized":"2024-04-12T12:04:23.467451+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:22.710430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":2227,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470011+0000","last_change":"2024-04-12T12:00:35.485498+0000","last_active":"2024-04-12T12:04:24.470011+0000","last_peered":"2024-04-12T12:04:24.470011+0000","last_clean":"2024-04-12T12:04:24.470011+0000","last_became_active":"2024-04-12T12:00:35.475885+0000","last_became_peered":"2024-04-12T12:00:35.475885+0000","last_unstale":"2024-04-12T12:04:24.470011+0000","last_undegraded":"2024-04-12T12:04:24.470011+0000","last_fullsized":"2024-04-12T12:04:24.470011+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:42:57.450495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":2015,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473266+0000","last_change":"2024-04-12T12:03:57.175691+0000","last_active":"2024-04-12T12:04:24.473266+0000","last_peered":"2024-04-12T12:04:24.473266+0000","last_clean":"2024-04-12T12:04:24.473266+0000","last_became_active":"2024-04-12T12:03:57.175279+0000","last_became_peered":"2024-04-12T12:03:57.175279+0000","last_unstale":"2024-04-12T12:04:24.473266+0000","last_undegraded":"2024-04-12T12:04:24.473266+0000","last_fullsized":"2024-04-12T12:04:24.473266+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:36:18.856007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":2207,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.469593+0000","last_change":"2024-04-12T11:45:52.979689+0000","last_active":"2024-04-12T12:04:24.469593+0000","last_peered":"2024-04-12T12:04:24.469593+0000","last_clean":"2024-04-12T12:04:24.469593+0000","last_became_active":"2024-04-12T11:45:52.979286+0000","last_became_peered":"2024-04-12T11:45:52.979286+0000","last_unstale":"2024-04-12T12:04:24.469593+0000","last_undegraded":"2024-04-12T12:04:24.469593+0000","last_fullsized":"2024-04-12T12:04:24.469593+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:10.453472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":100,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.468956+0000","last_change":"2024-04-12T12:03:26.501312+0000","last_active":"2024-04-12T12:04:24.468956+0000","last_peered":"2024-04-12T12:04:24.468956+0000","last_clean":"2024-04-12T12:04:24.468956+0000","last_became_active":"2024-04-12T12:03:26.500922+0000","last_became_peered":"2024-04-12T12:03:26.500922+0000","last_unstale":"2024-04-12T12:04:24.468956+0000","last_undegraded":"2024-04-12T12:04:24.468956+0000","last_fullsized":"2024-04-12T12:04:24.468956+0000","mapping_epoch":1369,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:03.245058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":39,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465998+0000","last_change":"2024-04-12T12:03:15.369500+0000","last_active":"2024-04-12T12:04:23.465998+0000","last_peered":"2024-04-12T12:04:23.465998+0000","last_clean":"2024-04-12T12:04:23.465998+0000","last_became_active":"2024-04-12T12:03:15.364137+0000","last_became_peered":"2024-04-12T12:03:15.364137+0000","last_unstale":"2024-04-12T12:04:23.465998+0000","last_undegraded":"2024-04-12T12:04:23.465998+0000","last_fullsized":"2024-04-12T12:04:23.465998+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:37:27.339272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":2056,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473747+0000","last_change":"2024-04-12T12:03:57.174685+0000","last_active":"2024-04-12T12:04:24.473747+0000","last_peered":"2024-04-12T12:04:24.473747+0000","last_clean":"2024-04-12T12:04:24.473747+0000","last_became_active":"2024-04-12T12:03:57.174257+0000","last_became_peered":"2024-04-12T12:03:57.174257+0000","last_unstale":"2024-04-12T12:04:24.473747+0000","last_undegraded":"2024-04-12T12:04:24.473747+0000","last_fullsized":"2024-04-12T12:04:24.473747+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:59:24.846007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":39,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465014+0000","last_change":"2024-04-12T12:03:15.367502+0000","last_active":"2024-04-12T12:04:23.465014+0000","last_peered":"2024-04-12T12:04:23.465014+0000","last_clean":"2024-04-12T12:04:23.465014+0000","last_became_active":"2024-04-12T12:03:15.360797+0000","last_became_peered":"2024-04-12T12:03:15.360797+0000","last_unstale":"2024-04-12T12:04:23.465014+0000","last_undegraded":"2024-04-12T12:04:23.465014+0000","last_fullsized":"2024-04-12T12:04:23.465014+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:38:31.827928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":2047,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473516+0000","last_change":"2024-04-12T12:03:57.139730+0000","last_active":"2024-04-12T12:04:24.473516+0000","last_peered":"2024-04-12T12:04:24.473516+0000","last_clean":"2024-04-12T12:04:24.473516+0000","last_became_active":"2024-04-12T12:03:57.139400+0000","last_became_peered":"2024-04-12T12:03:57.139400+0000","last_unstale":"2024-04-12T12:04:24.473516+0000","last_undegraded":"2024-04-12T12:04:24.473516+0000","last_fullsized":"2024-04-12T12:04:24.473516+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:33:08.870396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":215,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.479556+0000","last_change":"2024-04-12T12:04:24.478621+0000","last_active":"2024-04-12T12:04:24.479556+0000","last_peered":"2024-04-12T12:04:24.479556+0000","last_clean":"2024-04-12T12:04:24.479556+0000","last_became_active":"2024-04-12T12:04:24.478241+0000","last_became_peered":"2024-04-12T12:04:24.478241+0000","last_unstale":"2024-04-12T12:04:24.479556+0000","last_undegraded":"2024-04-12T12:04:24.479556+0000","last_fullsized":"2024-04-12T12:04:24.479556+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:52:26.522994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":696,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.474330+0000","last_change":"2024-04-12T12:04:24.473961+0000","last_active":"2024-04-12T12:04:24.474330+0000","last_peered":"2024-04-12T12:04:24.474330+0000","last_clean":"2024-04-12T12:04:24.474330+0000","last_became_active":"2024-04-12T12:04:24.473641+0000","last_became_peered":"2024-04-12T12:04:24.473641+0000","last_unstale":"2024-04-12T12:04:24.474330+0000","last_undegraded":"2024-04-12T12:04:24.474330+0000","last_fullsized":"2024-04-12T12:04:24.474330+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:25.247747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":2276,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470441+0000","last_change":"2024-04-12T12:03:15.373337+0000","last_active":"2024-04-12T12:04:24.470441+0000","last_peered":"2024-04-12T12:04:24.470441+0000","last_clean":"2024-04-12T12:04:24.470441+0000","last_became_active":"2024-04-12T12:03:15.357293+0000","last_became_peered":"2024-04-12T12:03:15.357293+0000","last_unstale":"2024-04-12T12:04:24.470441+0000","last_undegraded":"2024-04-12T12:04:24.470441+0000","last_fullsized":"2024-04-12T12:04:24.470441+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:42:47.903268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3492,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":2119,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473295+0000","last_change":"2024-04-12T12:00:35.472529+0000","last_active":"2024-04-12T12:04:24.473295+0000","last_peered":"2024-04-12T12:04:24.473295+0000","last_clean":"2024-04-12T12:04:24.473295+0000","last_became_active":"2024-04-12T12:00:35.472057+0000","last_became_peered":"2024-04-12T12:00:35.472057+0000","last_unstale":"2024-04-12T12:04:24.473295+0000","last_undegraded":"2024-04-12T12:04:24.473295+0000","last_fullsized":"2024-04-12T12:04:24.473295+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:07:59.083703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":13540,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35155,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1706112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234635,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972912,"kb_used_data":9120,"kb_used_omap":1,"kb_used_meta":963774,"kb_avail":92772240,"statfs":{"total":95995035648,"available":94998773760,"internally_reserved":0,"allocated":9338880,"data_stored":6109719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":986905035},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496271,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":797860,"kb_used_data":8788,"kb_used_omap":1,"kb_used_meta":789054,"kb_avail":92947292,"statfs":{"total":95995035648,"available":95178027008,"internally_reserved":0,"allocated":8998912,"data_stored":5781893,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":807991752},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536581,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770964,"kb_used_data":8772,"kb_used_omap":30,"kb_used_meta":762145,"kb_avail":92974188,"statfs":{"total":95995035648,"available":95205568512,"internally_reserved":0,"allocated":8982528,"data_stored":5777079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":780436802},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190665,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1131676,"kb_used_data":8780,"kb_used_omap":43,"kb_used_meta":1122836,"kb_avail":92613476,"statfs":{"total":95995035648,"available":94836199424,"internally_reserved":0,"allocated":8990720,"data_stored":5777683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":1149784107},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849793,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1011488,"kb_used_data":9040,"kb_used_omap":36,"kb_used_meta":1002395,"kb_avail":92733664,"statfs":{"total":95995035648,"available":94959271936,"internally_reserved":0,"allocated":9256960,"data_stored":6040383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":1026452660},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1342,"seq":5763846111283,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":126936,"kb_used_data":9356,"kb_used_omap":32,"kb_used_meta":117535,"kb_avail":93618216,"statfs":{"total":95995035648,"available":95865053184,"internally_reserved":0,"allocated":9580544,"data_stored":6367537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33145,"internal_metadata":120356487},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018358,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951660,"kb_used_data":8796,"kb_used_omap":42,"kb_used_meta":942805,"kb_avail":92793492,"statfs":{"total":95995035648,"available":95020535808,"internally_reserved":0,"allocated":9007104,"data_stored":5781981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":965433144},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":1282,"seq":5506148073606,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":84384,"kb_used_data":9360,"kb_used_omap":121,"kb_used_meta":74886,"kb_avail":93660768,"statfs":{"total":95995035648,"available":95908626432,"internally_reserved":0,"allocated":9584640,"data_stored":6368087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124430,"internal_metadata":76683762},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10321,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4818,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T12:05:14.010 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 -- ceph pg dump --format=json 2024-04-12T12:05:14.428 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/mon.a/config 2024-04-12T12:05:14.655 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:14 smithi029 ceph-mon[97787]: from='client.36366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:05:14.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:14 smithi029 ceph-mon[93650]: from='client.36366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:05:14.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:14 smithi087 ceph-mon[89014]: from='client.36366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:05:15.211 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:14 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:05:14] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.43.0" 2024-04-12T12:05:15.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:15 smithi029 ceph-mon[97787]: pgmap v2580: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:15.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:15 smithi029 ceph-mon[93650]: pgmap v2580: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:15.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:15 smithi087 ceph-mon[89014]: pgmap v2580: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:17.506 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:17 smithi029 ceph-mon[97787]: pgmap v2581: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:17.506 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:17 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:17.506 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:17 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:17.506 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:17 smithi029 ceph-mon[93650]: pgmap v2581: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:17.506 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:17 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:17.506 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:17 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:17 smithi087 ceph-mon[89014]: pgmap v2581: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:17 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:17.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:17 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:18.474 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:18 smithi029 ceph-mon[93650]: from='client.46265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:05:18.474 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:18 smithi029 ceph-mon[97787]: from='client.46265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-04-12T12:05:18.690 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-04-12T12:05:18.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:18 smithi087 ceph-mon[89014]: from='client.46265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T12:05:19.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:19 smithi087 ceph-mon[89014]: pgmap v2582: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:19.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:05:19.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:05:19.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:19 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:19.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:19 smithi029 ceph-mon[93650]: pgmap v2582: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:19.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:05:19.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:05:19.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:19 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:19.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:19 smithi029 ceph-mon[97787]: pgmap v2582: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:19.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T12:05:19.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T12:05:19.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:19 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' 2024-04-12T12:05:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:21 smithi087 ceph-mon[89014]: pgmap v2583: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:21.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:21 smithi087 ceph-mon[89014]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:05:21.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:21 smithi029 ceph-mon[93650]: pgmap v2583: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:21.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:21 smithi029 ceph-mon[93650]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:05:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:21 smithi029 ceph-mon[97787]: pgmap v2583: 83 pgs: 83 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-04-12T12:05:21.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:21 smithi029 ceph-mon[97787]: from='mgr.25202 172.21.15.29:0/1568863976' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T12:05:22.352 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2581,"stamp":"2024-04-12T12:05:16.319418+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":1719652,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1826,"ondisk_log_size":1826,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5847880,"kb_used_data":72012,"kb_used_omap":310,"kb_used_meta":5775433,"kb_avail":744113336,"statfs":{"total":767960285184,"available":761972056064,"internally_reserved":0,"allocated":73740288,"data_stored":48004362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":318107,"internal_metadata":5914043749},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003380"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":2152,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473414+0000","last_change":"2024-04-12T12:00:35.472642+0000","last_active":"2024-04-12T12:04:24.473414+0000","last_peered":"2024-04-12T12:04:24.473414+0000","last_clean":"2024-04-12T12:04:24.473414+0000","last_became_active":"2024-04-12T12:00:35.472258+0000","last_became_peered":"2024-04-12T12:00:35.472258+0000","last_unstale":"2024-04-12T12:04:24.473414+0000","last_undegraded":"2024-04-12T12:04:24.473414+0000","last_fullsized":"2024-04-12T12:04:24.473414+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:41:16.991252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":648,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477726+0000","last_change":"2024-04-12T12:04:24.477296+0000","last_active":"2024-04-12T12:04:24.477726+0000","last_peered":"2024-04-12T12:04:24.477726+0000","last_clean":"2024-04-12T12:04:24.477726+0000","last_became_active":"2024-04-12T12:04:24.476920+0000","last_became_peered":"2024-04-12T12:04:24.476920+0000","last_unstale":"2024-04-12T12:04:24.477726+0000","last_undegraded":"2024-04-12T12:04:24.477726+0000","last_fullsized":"2024-04-12T12:04:24.477726+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:30:33.305524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":126,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475552+0000","last_change":"2024-04-12T12:01:09.587349+0000","last_active":"2024-04-12T12:04:24.475552+0000","last_peered":"2024-04-12T12:04:24.475552+0000","last_clean":"2024-04-12T12:04:24.475552+0000","last_became_active":"2024-04-12T12:01:09.587019+0000","last_became_peered":"2024-04-12T12:01:09.587019+0000","last_unstale":"2024-04-12T12:04:24.475552+0000","last_undegraded":"2024-04-12T12:04:24.475552+0000","last_fullsized":"2024-04-12T12:04:24.475552+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:19:17.511244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2254,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470068+0000","last_change":"2024-04-12T12:01:08.211946+0000","last_active":"2024-04-12T12:04:24.470068+0000","last_peered":"2024-04-12T12:04:24.470068+0000","last_clean":"2024-04-12T12:04:24.470068+0000","last_became_active":"2024-04-12T12:01:08.209590+0000","last_became_peered":"2024-04-12T12:01:08.209590+0000","last_unstale":"2024-04-12T12:04:24.470068+0000","last_undegraded":"2024-04-12T12:04:24.470068+0000","last_fullsized":"2024-04-12T12:04:24.470068+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:40:08.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":713,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477767+0000","last_change":"2024-04-12T12:04:24.477356+0000","last_active":"2024-04-12T12:04:24.477767+0000","last_peered":"2024-04-12T12:04:24.477767+0000","last_clean":"2024-04-12T12:04:24.477767+0000","last_became_active":"2024-04-12T12:04:24.477019+0000","last_became_peered":"2024-04-12T12:04:24.477019+0000","last_unstale":"2024-04-12T12:04:24.477767+0000","last_undegraded":"2024-04-12T12:04:24.477767+0000","last_fullsized":"2024-04-12T12:04:24.477767+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:52:40.627145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1394,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477670+0000","last_change":"2024-04-12T12:03:15.361645+0000","last_active":"2024-04-12T12:04:24.477670+0000","last_peered":"2024-04-12T12:04:24.477670+0000","last_clean":"2024-04-12T12:04:24.477670+0000","last_became_active":"2024-04-12T12:03:15.361336+0000","last_became_peered":"2024-04-12T12:03:15.361336+0000","last_unstale":"2024-04-12T12:04:24.477670+0000","last_undegraded":"2024-04-12T12:04:24.477670+0000","last_fullsized":"2024-04-12T12:04:24.477670+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:04:01.224150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2260,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470385+0000","last_change":"2024-04-12T12:03:15.358780+0000","last_active":"2024-04-12T12:04:24.470385+0000","last_peered":"2024-04-12T12:04:24.470385+0000","last_clean":"2024-04-12T12:04:24.470385+0000","last_became_active":"2024-04-12T12:03:15.358388+0000","last_became_peered":"2024-04-12T12:03:15.358388+0000","last_unstale":"2024-04-12T12:04:24.470385+0000","last_undegraded":"2024-04-12T12:04:24.470385+0000","last_fullsized":"2024-04-12T12:04:24.470385+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:25:02.753436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":245,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.472183+0000","last_change":"2024-04-12T12:01:09.514195+0000","last_active":"2024-04-12T12:04:24.472183+0000","last_peered":"2024-04-12T12:04:24.472183+0000","last_clean":"2024-04-12T12:04:24.472183+0000","last_became_active":"2024-04-12T12:01:09.513505+0000","last_became_peered":"2024-04-12T12:01:09.513505+0000","last_unstale":"2024-04-12T12:04:24.472183+0000","last_undegraded":"2024-04-12T12:04:24.472183+0000","last_fullsized":"2024-04-12T12:04:24.472183+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:43.907048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.464548+0000","last_change":"2024-04-12T12:03:15.365051+0000","last_active":"2024-04-12T12:04:23.464548+0000","last_peered":"2024-04-12T12:04:23.464548+0000","last_clean":"2024-04-12T12:04:23.464548+0000","last_became_active":"2024-04-12T12:03:15.364638+0000","last_became_peered":"2024-04-12T12:03:15.364638+0000","last_unstale":"2024-04-12T12:04:23.464548+0000","last_undegraded":"2024-04-12T12:04:23.464548+0000","last_fullsized":"2024-04-12T12:04:23.464548+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:10:57.519022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":559,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.479994+0000","last_change":"2024-04-12T12:04:24.479569+0000","last_active":"2024-04-12T12:04:24.479994+0000","last_peered":"2024-04-12T12:04:24.479994+0000","last_clean":"2024-04-12T12:04:24.479994+0000","last_became_active":"2024-04-12T12:04:24.479219+0000","last_became_peered":"2024-04-12T12:04:24.479219+0000","last_unstale":"2024-04-12T12:04:24.479994+0000","last_undegraded":"2024-04-12T12:04:24.479994+0000","last_fullsized":"2024-04-12T12:04:24.479994+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:49:20.667505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":247,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477729+0000","last_change":"2024-04-12T12:01:08.204030+0000","last_active":"2024-04-12T12:04:24.477729+0000","last_peered":"2024-04-12T12:04:24.477729+0000","last_clean":"2024-04-12T12:04:24.477729+0000","last_became_active":"2024-04-12T12:01:08.203790+0000","last_became_peered":"2024-04-12T12:01:08.203790+0000","last_unstale":"2024-04-12T12:04:24.477729+0000","last_undegraded":"2024-04-12T12:04:24.477729+0000","last_fullsized":"2024-04-12T12:04:24.477729+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:18:55.719738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":128,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475968+0000","last_change":"2024-04-12T12:02:42.081270+0000","last_active":"2024-04-12T12:04:24.475968+0000","last_peered":"2024-04-12T12:04:24.475968+0000","last_clean":"2024-04-12T12:04:24.475968+0000","last_became_active":"2024-04-12T12:02:42.080832+0000","last_became_peered":"2024-04-12T12:02:42.080832+0000","last_unstale":"2024-04-12T12:04:24.475968+0000","last_undegraded":"2024-04-12T12:04:24.475968+0000","last_fullsized":"2024-04-12T12:04:24.475968+0000","mapping_epoch":1354,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:01:06.752668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"331'22","reported_seq":2145,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473853+0000","last_change":"2024-04-12T11:45:53.024626+0000","last_active":"2024-04-12T12:04:24.473853+0000","last_peered":"2024-04-12T12:04:24.473853+0000","last_clean":"2024-04-12T12:04:24.473853+0000","last_became_active":"2024-04-12T11:45:52.976276+0000","last_became_peered":"2024-04-12T11:45:52.976276+0000","last_unstale":"2024-04-12T12:04:24.473853+0000","last_undegraded":"2024-04-12T12:04:24.473853+0000","last_fullsized":"2024-04-12T12:04:24.473853+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:43:49.929210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":2116,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473609+0000","last_change":"2024-04-12T12:00:35.473701+0000","last_active":"2024-04-12T12:04:24.473609+0000","last_peered":"2024-04-12T12:04:24.473609+0000","last_clean":"2024-04-12T12:04:24.473609+0000","last_became_active":"2024-04-12T12:00:35.473318+0000","last_became_peered":"2024-04-12T12:00:35.473318+0000","last_unstale":"2024-04-12T12:04:24.473609+0000","last_undegraded":"2024-04-12T12:04:24.473609+0000","last_fullsized":"2024-04-12T12:04:24.473609+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:45:46.936638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":202,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477649+0000","last_change":"2024-04-12T11:45:53.035604+0000","last_active":"2024-04-12T12:04:24.477649+0000","last_peered":"2024-04-12T12:04:24.477649+0000","last_clean":"2024-04-12T12:04:24.477649+0000","last_became_active":"2024-04-12T11:45:52.984722+0000","last_became_peered":"2024-04-12T11:45:52.984722+0000","last_unstale":"2024-04-12T12:04:24.477649+0000","last_undegraded":"2024-04-12T12:04:24.477649+0000","last_fullsized":"2024-04-12T12:04:24.477649+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:30.674395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":606,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.480387+0000","last_change":"2024-04-12T12:04:24.478981+0000","last_active":"2024-04-12T12:04:24.480387+0000","last_peered":"2024-04-12T12:04:24.480387+0000","last_clean":"2024-04-12T12:04:24.480387+0000","last_became_active":"2024-04-12T12:04:24.478453+0000","last_became_peered":"2024-04-12T12:04:24.478453+0000","last_unstale":"2024-04-12T12:04:24.480387+0000","last_undegraded":"2024-04-12T12:04:24.480387+0000","last_fullsized":"2024-04-12T12:04:24.480387+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:09:19.217383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2237,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.469158+0000","last_change":"2024-04-12T12:01:08.222480+0000","last_active":"2024-04-12T12:04:24.469158+0000","last_peered":"2024-04-12T12:04:24.469158+0000","last_clean":"2024-04-12T12:04:24.469158+0000","last_became_active":"2024-04-12T12:01:08.222018+0000","last_became_peered":"2024-04-12T12:01:08.222018+0000","last_unstale":"2024-04-12T12:04:24.469158+0000","last_undegraded":"2024-04-12T12:04:24.469158+0000","last_fullsized":"2024-04-12T12:04:24.469158+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:12:10.882773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465096+0000","last_change":"2024-04-12T12:03:15.365615+0000","last_active":"2024-04-12T12:04:23.465096+0000","last_peered":"2024-04-12T12:04:23.465096+0000","last_clean":"2024-04-12T12:04:23.465096+0000","last_became_active":"2024-04-12T12:03:15.365188+0000","last_became_peered":"2024-04-12T12:03:15.365188+0000","last_unstale":"2024-04-12T12:04:23.465096+0000","last_undegraded":"2024-04-12T12:04:23.465096+0000","last_fullsized":"2024-04-12T12:04:23.465096+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:03:15.046301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1190,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465081+0000","last_change":"2024-04-12T12:04:10.630821+0000","last_active":"2024-04-12T12:04:23.465081+0000","last_peered":"2024-04-12T12:04:23.465081+0000","last_clean":"2024-04-12T12:04:23.465081+0000","last_became_active":"2024-04-12T12:04:10.630430+0000","last_became_peered":"2024-04-12T12:04:10.630430+0000","last_unstale":"2024-04-12T12:04:23.465081+0000","last_undegraded":"2024-04-12T12:04:23.465081+0000","last_fullsized":"2024-04-12T12:04:23.465081+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:21:01.037409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2237,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470153+0000","last_change":"2024-04-12T12:01:08.213742+0000","last_active":"2024-04-12T12:04:24.470153+0000","last_peered":"2024-04-12T12:04:24.470153+0000","last_clean":"2024-04-12T12:04:24.470153+0000","last_became_active":"2024-04-12T12:01:08.211529+0000","last_became_peered":"2024-04-12T12:01:08.211529+0000","last_unstale":"2024-04-12T12:04:24.470153+0000","last_undegraded":"2024-04-12T12:04:24.470153+0000","last_fullsized":"2024-04-12T12:04:24.470153+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:18.227656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":1411,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467067+0000","last_change":"2024-04-12T12:03:15.370352+0000","last_active":"2024-04-12T12:04:23.467067+0000","last_peered":"2024-04-12T12:04:23.467067+0000","last_clean":"2024-04-12T12:04:23.467067+0000","last_became_active":"2024-04-12T12:03:15.357513+0000","last_became_peered":"2024-04-12T12:03:15.357513+0000","last_unstale":"2024-04-12T12:04:23.467067+0000","last_undegraded":"2024-04-12T12:04:23.467067+0000","last_fullsized":"2024-04-12T12:04:23.467067+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:03:34.376607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":250,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.472443+0000","last_change":"2024-04-12T12:03:26.502220+0000","last_active":"2024-04-12T12:04:24.472443+0000","last_peered":"2024-04-12T12:04:24.472443+0000","last_clean":"2024-04-12T12:04:24.472443+0000","last_became_active":"2024-04-12T12:03:26.501584+0000","last_became_peered":"2024-04-12T12:03:26.501584+0000","last_unstale":"2024-04-12T12:04:24.472443+0000","last_undegraded":"2024-04-12T12:04:24.472443+0000","last_fullsized":"2024-04-12T12:04:24.472443+0000","mapping_epoch":1369,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:47:25.800012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":200,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466317+0000","last_change":"2024-04-12T11:45:53.028640+0000","last_active":"2024-04-12T12:04:23.466317+0000","last_peered":"2024-04-12T12:04:23.466317+0000","last_clean":"2024-04-12T12:04:23.466317+0000","last_became_active":"2024-04-12T11:45:52.983099+0000","last_became_peered":"2024-04-12T11:45:52.983099+0000","last_unstale":"2024-04-12T12:04:23.466317+0000","last_undegraded":"2024-04-12T12:04:23.466317+0000","last_fullsized":"2024-04-12T12:04:23.466317+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:43:01.897014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465274+0000","last_change":"2024-04-12T12:03:15.361270+0000","last_active":"2024-04-12T12:04:23.465274+0000","last_peered":"2024-04-12T12:04:23.465274+0000","last_clean":"2024-04-12T12:04:23.465274+0000","last_became_active":"2024-04-12T12:03:15.360962+0000","last_became_peered":"2024-04-12T12:03:15.360962+0000","last_unstale":"2024-04-12T12:04:23.465274+0000","last_undegraded":"2024-04-12T12:04:23.465274+0000","last_fullsized":"2024-04-12T12:04:23.465274+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:32:12.780528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":226,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466369+0000","last_change":"2024-04-12T12:00:35.474735+0000","last_active":"2024-04-12T12:04:23.466369+0000","last_peered":"2024-04-12T12:04:23.466369+0000","last_clean":"2024-04-12T12:04:23.466369+0000","last_became_active":"2024-04-12T12:00:35.463433+0000","last_became_peered":"2024-04-12T12:00:35.463433+0000","last_unstale":"2024-04-12T12:04:23.466369+0000","last_undegraded":"2024-04-12T12:04:23.466369+0000","last_fullsized":"2024-04-12T12:04:23.466369+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:27.215562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":2059,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473394+0000","last_change":"2024-04-12T12:03:57.139522+0000","last_active":"2024-04-12T12:04:24.473394+0000","last_peered":"2024-04-12T12:04:24.473394+0000","last_clean":"2024-04-12T12:04:24.473394+0000","last_became_active":"2024-04-12T12:03:57.139257+0000","last_became_peered":"2024-04-12T12:03:57.139257+0000","last_unstale":"2024-04-12T12:04:24.473394+0000","last_undegraded":"2024-04-12T12:04:24.473394+0000","last_fullsized":"2024-04-12T12:04:24.473394+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:48:56.189762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":171,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466384+0000","last_change":"2024-04-12T12:04:10.417903+0000","last_active":"2024-04-12T12:04:23.466384+0000","last_peered":"2024-04-12T12:04:23.466384+0000","last_clean":"2024-04-12T12:04:23.466384+0000","last_became_active":"2024-04-12T12:04:10.417540+0000","last_became_peered":"2024-04-12T12:04:10.417540+0000","last_unstale":"2024-04-12T12:04:23.466384+0000","last_undegraded":"2024-04-12T12:04:23.466384+0000","last_fullsized":"2024-04-12T12:04:23.466384+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:57:11.426128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1390,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476758+0000","last_change":"2024-04-12T12:00:35.474945+0000","last_active":"2024-04-12T12:04:24.476758+0000","last_peered":"2024-04-12T12:04:24.476758+0000","last_clean":"2024-04-12T12:04:24.476758+0000","last_became_active":"2024-04-12T12:00:35.474564+0000","last_became_peered":"2024-04-12T12:00:35.474564+0000","last_unstale":"2024-04-12T12:04:24.476758+0000","last_undegraded":"2024-04-12T12:04:24.476758+0000","last_fullsized":"2024-04-12T12:04:24.476758+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:13:53.178867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466088+0000","last_change":"2024-04-12T12:03:15.359143+0000","last_active":"2024-04-12T12:04:23.466088+0000","last_peered":"2024-04-12T12:04:23.466088+0000","last_clean":"2024-04-12T12:04:23.466088+0000","last_became_active":"2024-04-12T12:03:15.358807+0000","last_became_peered":"2024-04-12T12:03:15.358807+0000","last_unstale":"2024-04-12T12:04:23.466088+0000","last_undegraded":"2024-04-12T12:04:23.466088+0000","last_fullsized":"2024-04-12T12:04:23.466088+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:24:08.985614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2218,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.469380+0000","last_change":"2024-04-12T12:00:35.475414+0000","last_active":"2024-04-12T12:04:24.469380+0000","last_peered":"2024-04-12T12:04:24.469380+0000","last_clean":"2024-04-12T12:04:24.469380+0000","last_became_active":"2024-04-12T12:00:35.475098+0000","last_became_peered":"2024-04-12T12:00:35.475098+0000","last_unstale":"2024-04-12T12:04:24.469380+0000","last_undegraded":"2024-04-12T12:04:24.469380+0000","last_fullsized":"2024-04-12T12:04:24.469380+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:25:12.377240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"249'161","reported_seq":1755,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465250+0000","last_change":"2024-04-12T12:01:08.230318+0000","last_active":"2024-04-12T12:04:23.465250+0000","last_peered":"2024-04-12T12:04:23.465250+0000","last_clean":"2024-04-12T12:04:23.465250+0000","last_became_active":"2024-04-12T12:01:08.229959+0000","last_became_peered":"2024-04-12T12:01:08.229959+0000","last_unstale":"2024-04-12T12:04:23.465250+0000","last_undegraded":"2024-04-12T12:04:23.465250+0000","last_fullsized":"2024-04-12T12:04:23.465250+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:08.650841+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:08.650841+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:17:07.248029+0000","scrub_duration":0.001456279,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":206,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476401+0000","last_change":"2024-04-12T12:04:24.476114+0000","last_active":"2024-04-12T12:04:24.476401+0000","last_peered":"2024-04-12T12:04:24.476401+0000","last_clean":"2024-04-12T12:04:24.476401+0000","last_became_active":"2024-04-12T12:04:24.475751+0000","last_became_peered":"2024-04-12T12:04:24.475751+0000","last_unstale":"2024-04-12T12:04:24.476401+0000","last_undegraded":"2024-04-12T12:04:24.476401+0000","last_fullsized":"2024-04-12T12:04:24.476401+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:14:38.380779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":2034,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473834+0000","last_change":"2024-04-12T12:03:57.175404+0000","last_active":"2024-04-12T12:04:24.473834+0000","last_peered":"2024-04-12T12:04:24.473834+0000","last_clean":"2024-04-12T12:04:24.473834+0000","last_became_active":"2024-04-12T12:03:57.174610+0000","last_became_peered":"2024-04-12T12:03:57.174610+0000","last_unstale":"2024-04-12T12:04:24.473834+0000","last_undegraded":"2024-04-12T12:04:24.473834+0000","last_fullsized":"2024-04-12T12:04:24.473834+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:26:49.922463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"249'161","reported_seq":896,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465356+0000","last_change":"2024-04-12T12:03:18.377882+0000","last_active":"2024-04-12T12:04:23.465356+0000","last_peered":"2024-04-12T12:04:23.465356+0000","last_clean":"2024-04-12T12:04:23.465356+0000","last_became_active":"2024-04-12T12:03:18.377558+0000","last_became_peered":"2024-04-12T12:03:18.377558+0000","last_unstale":"2024-04-12T12:04:23.465356+0000","last_undegraded":"2024-04-12T12:04:23.465356+0000","last_fullsized":"2024-04-12T12:04:23.465356+0000","mapping_epoch":1367,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1368,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:10.437190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:10.437190+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:29:03.934380+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1087,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467151+0000","last_change":"2024-04-12T12:04:03.162515+0000","last_active":"2024-04-12T12:04:23.467151+0000","last_peered":"2024-04-12T12:04:23.467151+0000","last_clean":"2024-04-12T12:04:23.467151+0000","last_became_active":"2024-04-12T12:04:03.162228+0000","last_became_peered":"2024-04-12T12:04:03.162228+0000","last_unstale":"2024-04-12T12:04:23.467151+0000","last_undegraded":"2024-04-12T12:04:23.467151+0000","last_fullsized":"2024-04-12T12:04:23.467151+0000","mapping_epoch":1380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:45:43.194555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":2038,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.474024+0000","last_change":"2024-04-12T12:03:57.175845+0000","last_active":"2024-04-12T12:04:24.474024+0000","last_peered":"2024-04-12T12:04:24.474024+0000","last_clean":"2024-04-12T12:04:24.474024+0000","last_became_active":"2024-04-12T12:03:57.175454+0000","last_became_peered":"2024-04-12T12:03:57.175454+0000","last_unstale":"2024-04-12T12:04:24.474024+0000","last_undegraded":"2024-04-12T12:04:24.474024+0000","last_fullsized":"2024-04-12T12:04:24.474024+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:27:03.901168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":2139,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473570+0000","last_change":"2024-04-12T11:45:52.976139+0000","last_active":"2024-04-12T12:04:24.473570+0000","last_peered":"2024-04-12T12:04:24.473570+0000","last_clean":"2024-04-12T12:04:24.473570+0000","last_became_active":"2024-04-12T11:45:52.975803+0000","last_became_peered":"2024-04-12T11:45:52.975803+0000","last_unstale":"2024-04-12T12:04:24.473570+0000","last_undegraded":"2024-04-12T12:04:24.473570+0000","last_fullsized":"2024-04-12T12:04:24.473570+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:22:46.618143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":2120,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.474076+0000","last_change":"2024-04-12T12:00:35.473079+0000","last_active":"2024-04-12T12:04:24.474076+0000","last_peered":"2024-04-12T12:04:24.474076+0000","last_clean":"2024-04-12T12:04:24.474076+0000","last_became_active":"2024-04-12T12:00:35.472773+0000","last_became_peered":"2024-04-12T12:00:35.472773+0000","last_unstale":"2024-04-12T12:04:24.474076+0000","last_undegraded":"2024-04-12T12:04:24.474076+0000","last_fullsized":"2024-04-12T12:04:24.474076+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:20:42.192043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"249'161","reported_seq":858,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465013+0000","last_change":"2024-04-12T12:03:18.378083+0000","last_active":"2024-04-12T12:04:23.465013+0000","last_peered":"2024-04-12T12:04:23.465013+0000","last_clean":"2024-04-12T12:04:23.465013+0000","last_became_active":"2024-04-12T12:03:18.377735+0000","last_became_peered":"2024-04-12T12:03:18.377735+0000","last_unstale":"2024-04-12T12:04:23.465013+0000","last_undegraded":"2024-04-12T12:04:23.465013+0000","last_fullsized":"2024-04-12T12:04:23.465013+0000","mapping_epoch":1367,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1368,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:11.447056+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:11.447056+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:48:05.161883+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":605,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.479624+0000","last_change":"2024-04-12T12:04:24.478786+0000","last_active":"2024-04-12T12:04:24.479624+0000","last_peered":"2024-04-12T12:04:24.479624+0000","last_clean":"2024-04-12T12:04:24.479624+0000","last_became_active":"2024-04-12T12:04:24.478377+0000","last_became_peered":"2024-04-12T12:04:24.478377+0000","last_unstale":"2024-04-12T12:04:24.479624+0000","last_undegraded":"2024-04-12T12:04:24.479624+0000","last_fullsized":"2024-04-12T12:04:24.479624+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:26:00.520176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":123,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475984+0000","last_change":"2024-04-12T12:00:35.477779+0000","last_active":"2024-04-12T12:04:24.475984+0000","last_peered":"2024-04-12T12:04:24.475984+0000","last_clean":"2024-04-12T12:04:24.475984+0000","last_became_active":"2024-04-12T12:00:35.477462+0000","last_became_peered":"2024-04-12T12:00:35.477462+0000","last_unstale":"2024-04-12T12:04:24.475984+0000","last_undegraded":"2024-04-12T12:04:24.475984+0000","last_fullsized":"2024-04-12T12:04:24.475984+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:51:17.465133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"249'161","reported_seq":658,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467277+0000","last_change":"2024-04-12T12:01:08.203484+0000","last_active":"2024-04-12T12:04:23.467277+0000","last_peered":"2024-04-12T12:04:23.467277+0000","last_clean":"2024-04-12T12:04:23.467277+0000","last_became_active":"2024-04-12T12:01:08.203128+0000","last_became_peered":"2024-04-12T12:01:08.203128+0000","last_unstale":"2024-04-12T12:04:23.467277+0000","last_undegraded":"2024-04-12T12:04:23.467277+0000","last_fullsized":"2024-04-12T12:04:23.467277+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:37.607274+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:37.607274+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:03:30.798799+0000","scrub_duration":0.0037173240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":123,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476396+0000","last_change":"2024-04-12T12:00:35.473414+0000","last_active":"2024-04-12T12:04:24.476396+0000","last_peered":"2024-04-12T12:04:24.476396+0000","last_clean":"2024-04-12T12:04:24.476396+0000","last_became_active":"2024-04-12T12:00:35.472846+0000","last_became_peered":"2024-04-12T12:00:35.472846+0000","last_unstale":"2024-04-12T12:04:24.476396+0000","last_undegraded":"2024-04-12T12:04:24.476396+0000","last_fullsized":"2024-04-12T12:04:24.476396+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:59:19.941070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465176+0000","last_change":"2024-04-12T12:03:15.365784+0000","last_active":"2024-04-12T12:04:23.465176+0000","last_peered":"2024-04-12T12:04:23.465176+0000","last_clean":"2024-04-12T12:04:23.465176+0000","last_became_active":"2024-04-12T12:03:15.365270+0000","last_became_peered":"2024-04-12T12:03:15.365270+0000","last_unstale":"2024-04-12T12:04:23.465176+0000","last_undegraded":"2024-04-12T12:04:23.465176+0000","last_fullsized":"2024-04-12T12:04:23.465176+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:42:06.789362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":129,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475914+0000","last_change":"2024-04-12T12:03:15.365465+0000","last_active":"2024-04-12T12:04:24.475914+0000","last_peered":"2024-04-12T12:04:24.475914+0000","last_clean":"2024-04-12T12:04:24.475914+0000","last_became_active":"2024-04-12T12:03:15.365077+0000","last_became_peered":"2024-04-12T12:03:15.365077+0000","last_unstale":"2024-04-12T12:04:24.475914+0000","last_undegraded":"2024-04-12T12:04:24.475914+0000","last_fullsized":"2024-04-12T12:04:24.475914+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:34:35.106519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"249'161","reported_seq":862,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.464583+0000","last_change":"2024-04-12T12:03:18.377941+0000","last_active":"2024-04-12T12:04:23.464583+0000","last_peered":"2024-04-12T12:04:23.464583+0000","last_clean":"2024-04-12T12:04:23.464583+0000","last_became_active":"2024-04-12T12:03:18.377623+0000","last_became_peered":"2024-04-12T12:03:18.377623+0000","last_unstale":"2024-04-12T12:04:23.464583+0000","last_undegraded":"2024-04-12T12:04:23.464583+0000","last_fullsized":"2024-04-12T12:04:23.464583+0000","mapping_epoch":1367,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1368,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:05.639497+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:05.639497+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:05:27.861150+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465097+0000","last_change":"2024-04-12T12:03:15.365182+0000","last_active":"2024-04-12T12:04:23.465097+0000","last_peered":"2024-04-12T12:04:23.465097+0000","last_clean":"2024-04-12T12:04:23.465097+0000","last_became_active":"2024-04-12T12:03:15.364842+0000","last_became_peered":"2024-04-12T12:03:15.364842+0000","last_unstale":"2024-04-12T12:04:23.465097+0000","last_undegraded":"2024-04-12T12:04:23.465097+0000","last_fullsized":"2024-04-12T12:04:23.465097+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:11:41.126247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":2019,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473493+0000","last_change":"2024-04-12T12:03:57.175432+0000","last_active":"2024-04-12T12:04:24.473493+0000","last_peered":"2024-04-12T12:04:24.473493+0000","last_clean":"2024-04-12T12:04:24.473493+0000","last_became_active":"2024-04-12T12:03:57.175010+0000","last_became_peered":"2024-04-12T12:03:57.175010+0000","last_unstale":"2024-04-12T12:04:24.473493+0000","last_undegraded":"2024-04-12T12:04:24.473493+0000","last_fullsized":"2024-04-12T12:04:24.473493+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:41:03.170360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":127,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476356+0000","last_change":"2024-04-12T12:01:09.225968+0000","last_active":"2024-04-12T12:04:24.476356+0000","last_peered":"2024-04-12T12:04:24.476356+0000","last_clean":"2024-04-12T12:04:24.476356+0000","last_became_active":"2024-04-12T12:01:09.225273+0000","last_became_peered":"2024-04-12T12:01:09.225273+0000","last_unstale":"2024-04-12T12:04:24.476356+0000","last_undegraded":"2024-04-12T12:04:24.476356+0000","last_fullsized":"2024-04-12T12:04:24.476356+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:31:39.143286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"249'161","reported_seq":1692,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.464970+0000","last_change":"2024-04-12T12:04:10.419038+0000","last_active":"2024-04-12T12:04:23.464970+0000","last_peered":"2024-04-12T12:04:23.464970+0000","last_clean":"2024-04-12T12:04:23.464970+0000","last_became_active":"2024-04-12T12:04:10.418687+0000","last_became_peered":"2024-04-12T12:04:10.418687+0000","last_unstale":"2024-04-12T12:04:23.464970+0000","last_undegraded":"2024-04-12T12:04:23.464970+0000","last_fullsized":"2024-04-12T12:04:23.464970+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1384,"parent":"0.0","parent_split_bits":4,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:36.979380+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:36.979380+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:24:24.960842+0000","scrub_duration":0.0018052980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2200,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.468737+0000","last_change":"2024-04-12T12:01:08.241455+0000","last_active":"2024-04-12T12:04:24.468737+0000","last_peered":"2024-04-12T12:04:24.468737+0000","last_clean":"2024-04-12T12:04:24.468737+0000","last_became_active":"2024-04-12T12:01:08.240087+0000","last_became_peered":"2024-04-12T12:01:08.240087+0000","last_unstale":"2024-04-12T12:04:24.468737+0000","last_undegraded":"2024-04-12T12:04:24.468737+0000","last_fullsized":"2024-04-12T12:04:24.468737+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:45:12.951549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":123,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475960+0000","last_change":"2024-04-12T12:00:35.471479+0000","last_active":"2024-04-12T12:04:24.475960+0000","last_peered":"2024-04-12T12:04:24.475960+0000","last_clean":"2024-04-12T12:04:24.475960+0000","last_became_active":"2024-04-12T12:00:35.471015+0000","last_became_peered":"2024-04-12T12:00:35.471015+0000","last_unstale":"2024-04-12T12:04:24.475960+0000","last_undegraded":"2024-04-12T12:04:24.475960+0000","last_fullsized":"2024-04-12T12:04:24.475960+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:55:50.471132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":129,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475793+0000","last_change":"2024-04-12T12:03:15.365552+0000","last_active":"2024-04-12T12:04:24.475793+0000","last_peered":"2024-04-12T12:04:24.475793+0000","last_clean":"2024-04-12T12:04:24.475793+0000","last_became_active":"2024-04-12T12:03:15.365134+0000","last_became_peered":"2024-04-12T12:03:15.365134+0000","last_unstale":"2024-04-12T12:04:24.475793+0000","last_undegraded":"2024-04-12T12:04:24.475793+0000","last_fullsized":"2024-04-12T12:04:24.475793+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:03:25.869378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":35,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466429+0000","last_change":"2024-04-12T12:03:15.361282+0000","last_active":"2024-04-12T12:04:23.466429+0000","last_peered":"2024-04-12T12:04:23.466429+0000","last_clean":"2024-04-12T12:04:23.466429+0000","last_became_active":"2024-04-12T12:03:15.360980+0000","last_became_peered":"2024-04-12T12:03:15.360980+0000","last_unstale":"2024-04-12T12:04:23.466429+0000","last_undegraded":"2024-04-12T12:04:23.466429+0000","last_fullsized":"2024-04-12T12:04:23.466429+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:00:27.301014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"249'161","reported_seq":1724,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.466036+0000","last_change":"2024-04-12T11:53:06.603022+0000","last_active":"2024-04-12T12:04:23.466036+0000","last_peered":"2024-04-12T12:04:23.466036+0000","last_clean":"2024-04-12T12:04:23.466036+0000","last_became_active":"2024-04-12T11:49:07.217921+0000","last_became_peered":"2024-04-12T11:49:07.217921+0000","last_unstale":"2024-04-12T12:04:23.466036+0000","last_undegraded":"2024-04-12T12:04:23.466036+0000","last_fullsized":"2024-04-12T12:04:23.466036+0000","mapping_epoch":1210,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":0,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:06.602918+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:06.602918+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:23:40.776170+0000","scrub_duration":0.001429877,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":576,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.478779+0000","last_change":"2024-04-12T12:04:24.478415+0000","last_active":"2024-04-12T12:04:24.478779+0000","last_peered":"2024-04-12T12:04:24.478779+0000","last_clean":"2024-04-12T12:04:24.478779+0000","last_became_active":"2024-04-12T12:04:24.478091+0000","last_became_peered":"2024-04-12T12:04:24.478091+0000","last_unstale":"2024-04-12T12:04:24.478779+0000","last_undegraded":"2024-04-12T12:04:24.478779+0000","last_fullsized":"2024-04-12T12:04:24.478779+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:10:24.813865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1363,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467268+0000","last_change":"2024-04-12T11:45:53.016616+0000","last_active":"2024-04-12T12:04:23.467268+0000","last_peered":"2024-04-12T12:04:23.467268+0000","last_clean":"2024-04-12T12:04:23.467268+0000","last_became_active":"2024-04-12T11:45:53.016292+0000","last_became_peered":"2024-04-12T11:45:53.016292+0000","last_unstale":"2024-04-12T12:04:23.467268+0000","last_undegraded":"2024-04-12T12:04:23.467268+0000","last_fullsized":"2024-04-12T12:04:23.467268+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:53:29.382293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"249'161","reported_seq":789,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.475840+0000","last_change":"2024-04-12T12:02:44.218750+0000","last_active":"2024-04-12T12:04:24.475840+0000","last_peered":"2024-04-12T12:04:24.475840+0000","last_clean":"2024-04-12T12:04:24.475840+0000","last_became_active":"2024-04-12T12:02:44.218240+0000","last_became_peered":"2024-04-12T12:02:44.218240+0000","last_unstale":"2024-04-12T12:04:24.475840+0000","last_undegraded":"2024-04-12T12:04:24.475840+0000","last_fullsized":"2024-04-12T12:04:24.475840+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:57:49.744017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:57:49.744017+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:13:07.839300+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":554,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465942+0000","last_change":"2024-04-12T12:04:10.631148+0000","last_active":"2024-04-12T12:04:23.465942+0000","last_peered":"2024-04-12T12:04:23.465942+0000","last_clean":"2024-04-12T12:04:23.465942+0000","last_became_active":"2024-04-12T12:04:10.630769+0000","last_became_peered":"2024-04-12T12:04:10.630769+0000","last_unstale":"2024-04-12T12:04:23.465942+0000","last_undegraded":"2024-04-12T12:04:23.465942+0000","last_fullsized":"2024-04-12T12:04:23.465942+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:30:57.067503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2237,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470269+0000","last_change":"2024-04-12T12:01:08.241097+0000","last_active":"2024-04-12T12:04:24.470269+0000","last_peered":"2024-04-12T12:04:24.470269+0000","last_clean":"2024-04-12T12:04:24.470269+0000","last_became_active":"2024-04-12T12:01:08.240402+0000","last_became_peered":"2024-04-12T12:01:08.240402+0000","last_unstale":"2024-04-12T12:04:24.470269+0000","last_undegraded":"2024-04-12T12:04:24.470269+0000","last_fullsized":"2024-04-12T12:04:24.470269+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:30:15.712301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2216,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470214+0000","last_change":"2024-04-12T12:00:35.480505+0000","last_active":"2024-04-12T12:04:24.470214+0000","last_peered":"2024-04-12T12:04:24.470214+0000","last_clean":"2024-04-12T12:04:24.470214+0000","last_became_active":"2024-04-12T12:00:35.480136+0000","last_became_peered":"2024-04-12T12:00:35.480136+0000","last_unstale":"2024-04-12T12:04:24.470214+0000","last_undegraded":"2024-04-12T12:04:24.470214+0000","last_fullsized":"2024-04-12T12:04:24.470214+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:12:05.320400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"249'161","reported_seq":732,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467652+0000","last_change":"2024-04-12T12:01:08.209702+0000","last_active":"2024-04-12T12:04:23.467652+0000","last_peered":"2024-04-12T12:04:23.467652+0000","last_clean":"2024-04-12T12:04:23.467652+0000","last_became_active":"2024-04-12T12:01:08.209465+0000","last_became_peered":"2024-04-12T12:01:08.209465+0000","last_unstale":"2024-04-12T12:04:23.467652+0000","last_undegraded":"2024-04-12T12:04:23.467652+0000","last_fullsized":"2024-04-12T12:04:23.467652+0000","mapping_epoch":1342,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1343,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:05.130465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:05.130465+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:53:20.786613+0000","scrub_duration":0.001397695,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":197,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.477656+0000","last_change":"2024-04-12T11:45:53.005585+0000","last_active":"2024-04-12T12:04:24.477656+0000","last_peered":"2024-04-12T12:04:24.477656+0000","last_clean":"2024-04-12T12:04:24.477656+0000","last_became_active":"2024-04-12T11:45:52.987880+0000","last_became_peered":"2024-04-12T11:45:52.987880+0000","last_unstale":"2024-04-12T12:04:24.477656+0000","last_undegraded":"2024-04-12T12:04:24.477656+0000","last_fullsized":"2024-04-12T12:04:24.477656+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:28:09.819861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":129,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476461+0000","last_change":"2024-04-12T12:03:15.366238+0000","last_active":"2024-04-12T12:04:24.476461+0000","last_peered":"2024-04-12T12:04:24.476461+0000","last_clean":"2024-04-12T12:04:24.476461+0000","last_became_active":"2024-04-12T12:03:15.365749+0000","last_became_peered":"2024-04-12T12:03:15.365749+0000","last_unstale":"2024-04-12T12:04:24.476461+0000","last_undegraded":"2024-04-12T12:04:24.476461+0000","last_fullsized":"2024-04-12T12:04:24.476461+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:57:20.333454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2226,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.469647+0000","last_change":"2024-04-12T12:03:15.362063+0000","last_active":"2024-04-12T12:04:24.469647+0000","last_peered":"2024-04-12T12:04:24.469647+0000","last_clean":"2024-04-12T12:04:24.469647+0000","last_became_active":"2024-04-12T12:03:15.361691+0000","last_became_peered":"2024-04-12T12:03:15.361691+0000","last_unstale":"2024-04-12T12:04:24.469647+0000","last_undegraded":"2024-04-12T12:04:24.469647+0000","last_fullsized":"2024-04-12T12:04:24.469647+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:11:02.269246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"249'161","reported_seq":885,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473899+0000","last_change":"2024-04-12T12:02:44.216787+0000","last_active":"2024-04-12T12:04:24.473899+0000","last_peered":"2024-04-12T12:04:24.473899+0000","last_clean":"2024-04-12T12:04:24.473899+0000","last_became_active":"2024-04-12T12:02:44.216350+0000","last_became_peered":"2024-04-12T12:02:44.216350+0000","last_unstale":"2024-04-12T12:04:24.473899+0000","last_undegraded":"2024-04-12T12:04:24.473899+0000","last_fullsized":"2024-04-12T12:04:24.473899+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:53:10.604457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:53:10.604457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:01:05.908018+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":206,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.480437+0000","last_change":"2024-04-12T12:04:24.480110+0000","last_active":"2024-04-12T12:04:24.480437+0000","last_peered":"2024-04-12T12:04:24.480437+0000","last_clean":"2024-04-12T12:04:24.480437+0000","last_became_active":"2024-04-12T12:04:24.479426+0000","last_became_peered":"2024-04-12T12:04:24.479426+0000","last_unstale":"2024-04-12T12:04:24.480437+0000","last_undegraded":"2024-04-12T12:04:24.480437+0000","last_fullsized":"2024-04-12T12:04:24.480437+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T18:09:59.442971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":123,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.476058+0000","last_change":"2024-04-12T12:00:35.477662+0000","last_active":"2024-04-12T12:04:24.476058+0000","last_peered":"2024-04-12T12:04:24.476058+0000","last_clean":"2024-04-12T12:04:24.476058+0000","last_became_active":"2024-04-12T12:00:35.477235+0000","last_became_peered":"2024-04-12T12:00:35.477235+0000","last_unstale":"2024-04-12T12:04:24.476058+0000","last_undegraded":"2024-04-12T12:04:24.476058+0000","last_fullsized":"2024-04-12T12:04:24.476058+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:18:34.713724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":203,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.480281+0000","last_change":"2024-04-12T12:04:24.479467+0000","last_active":"2024-04-12T12:04:24.480281+0000","last_peered":"2024-04-12T12:04:24.480281+0000","last_clean":"2024-04-12T12:04:24.480281+0000","last_became_active":"2024-04-12T12:04:24.479148+0000","last_became_peered":"2024-04-12T12:04:24.479148+0000","last_unstale":"2024-04-12T12:04:24.480281+0000","last_undegraded":"2024-04-12T12:04:24.480281+0000","last_fullsized":"2024-04-12T12:04:24.480281+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T12:45:18.428976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"249'161","reported_seq":769,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.480253+0000","last_change":"2024-04-12T12:04:24.480253+0000","last_active":"2024-04-12T12:04:24.480253+0000","last_peered":"2024-04-12T12:04:24.480253+0000","last_clean":"2024-04-12T12:04:24.480253+0000","last_became_active":"2024-04-12T12:04:24.479393+0000","last_became_peered":"2024-04-12T12:04:24.479393+0000","last_unstale":"2024-04-12T12:04:24.480253+0000","last_undegraded":"2024-04-12T12:04:24.480253+0000","last_fullsized":"2024-04-12T12:04:24.480253+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":1240,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":3,"last_scrub":"249'161","last_scrub_stamp":"2024-04-12T11:59:17.062855+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:09:21.719506+0000","last_clean_scrub_stamp":"2024-04-12T11:59:17.062855+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T16:12:45.686867+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1396,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.467451+0000","last_change":"2024-04-12T11:45:52.979745+0000","last_active":"2024-04-12T12:04:23.467451+0000","last_peered":"2024-04-12T12:04:23.467451+0000","last_clean":"2024-04-12T12:04:23.467451+0000","last_became_active":"2024-04-12T11:45:52.978936+0000","last_became_peered":"2024-04-12T11:45:52.978936+0000","last_unstale":"2024-04-12T12:04:23.467451+0000","last_undegraded":"2024-04-12T12:04:23.467451+0000","last_fullsized":"2024-04-12T12:04:23.467451+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:11:48.514791+0000","last_clean_scrub_stamp":"2024-04-12T11:11:48.514791+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:01:22.710430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":2227,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470011+0000","last_change":"2024-04-12T12:00:35.485498+0000","last_active":"2024-04-12T12:04:24.470011+0000","last_peered":"2024-04-12T12:04:24.470011+0000","last_clean":"2024-04-12T12:04:24.470011+0000","last_became_active":"2024-04-12T12:00:35.475885+0000","last_became_peered":"2024-04-12T12:00:35.475885+0000","last_unstale":"2024-04-12T12:04:24.470011+0000","last_undegraded":"2024-04-12T12:04:24.470011+0000","last_fullsized":"2024-04-12T12:04:24.470011+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:42:57.450495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":2015,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473266+0000","last_change":"2024-04-12T12:03:57.175691+0000","last_active":"2024-04-12T12:04:24.473266+0000","last_peered":"2024-04-12T12:04:24.473266+0000","last_clean":"2024-04-12T12:04:24.473266+0000","last_became_active":"2024-04-12T12:03:57.175279+0000","last_became_peered":"2024-04-12T12:03:57.175279+0000","last_unstale":"2024-04-12T12:04:24.473266+0000","last_undegraded":"2024-04-12T12:04:24.473266+0000","last_fullsized":"2024-04-12T12:04:24.473266+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:36:18.856007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":2207,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.469593+0000","last_change":"2024-04-12T11:45:52.979689+0000","last_active":"2024-04-12T12:04:24.469593+0000","last_peered":"2024-04-12T12:04:24.469593+0000","last_clean":"2024-04-12T12:04:24.469593+0000","last_became_active":"2024-04-12T11:45:52.979286+0000","last_became_peered":"2024-04-12T11:45:52.979286+0000","last_unstale":"2024-04-12T12:04:24.469593+0000","last_undegraded":"2024-04-12T12:04:24.469593+0000","last_fullsized":"2024-04-12T12:04:24.469593+0000","mapping_epoch":1209,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:43:10.453472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":100,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.468956+0000","last_change":"2024-04-12T12:03:26.501312+0000","last_active":"2024-04-12T12:04:24.468956+0000","last_peered":"2024-04-12T12:04:24.468956+0000","last_clean":"2024-04-12T12:04:24.468956+0000","last_became_active":"2024-04-12T12:03:26.500922+0000","last_became_peered":"2024-04-12T12:03:26.500922+0000","last_unstale":"2024-04-12T12:04:24.468956+0000","last_undegraded":"2024-04-12T12:04:24.468956+0000","last_fullsized":"2024-04-12T12:04:24.468956+0000","mapping_epoch":1369,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T15:33:03.245058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":39,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465998+0000","last_change":"2024-04-12T12:03:15.369500+0000","last_active":"2024-04-12T12:04:23.465998+0000","last_peered":"2024-04-12T12:04:23.465998+0000","last_clean":"2024-04-12T12:04:23.465998+0000","last_became_active":"2024-04-12T12:03:15.364137+0000","last_became_peered":"2024-04-12T12:03:15.364137+0000","last_unstale":"2024-04-12T12:04:23.465998+0000","last_undegraded":"2024-04-12T12:04:23.465998+0000","last_fullsized":"2024-04-12T12:04:23.465998+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T19:37:27.339272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":2056,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473747+0000","last_change":"2024-04-12T12:03:57.174685+0000","last_active":"2024-04-12T12:04:24.473747+0000","last_peered":"2024-04-12T12:04:24.473747+0000","last_clean":"2024-04-12T12:04:24.473747+0000","last_became_active":"2024-04-12T12:03:57.174257+0000","last_became_peered":"2024-04-12T12:03:57.174257+0000","last_unstale":"2024-04-12T12:04:24.473747+0000","last_undegraded":"2024-04-12T12:04:24.473747+0000","last_fullsized":"2024-04-12T12:04:24.473747+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T14:59:24.846007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":39,"reported_epoch":1388,"state":"active+clean","last_fresh":"2024-04-12T12:04:23.465014+0000","last_change":"2024-04-12T12:03:15.367502+0000","last_active":"2024-04-12T12:04:23.465014+0000","last_peered":"2024-04-12T12:04:23.465014+0000","last_clean":"2024-04-12T12:04:23.465014+0000","last_became_active":"2024-04-12T12:03:15.360797+0000","last_became_peered":"2024-04-12T12:03:15.360797+0000","last_unstale":"2024-04-12T12:04:23.465014+0000","last_undegraded":"2024-04-12T12:04:23.465014+0000","last_fullsized":"2024-04-12T12:04:23.465014+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T13:38:31.827928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":2047,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473516+0000","last_change":"2024-04-12T12:03:57.139730+0000","last_active":"2024-04-12T12:04:24.473516+0000","last_peered":"2024-04-12T12:04:24.473516+0000","last_clean":"2024-04-12T12:04:24.473516+0000","last_became_active":"2024-04-12T12:03:57.139400+0000","last_became_peered":"2024-04-12T12:03:57.139400+0000","last_unstale":"2024-04-12T12:04:24.473516+0000","last_undegraded":"2024-04-12T12:04:24.473516+0000","last_fullsized":"2024-04-12T12:04:24.473516+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T20:33:08.870396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":215,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.479556+0000","last_change":"2024-04-12T12:04:24.478621+0000","last_active":"2024-04-12T12:04:24.479556+0000","last_peered":"2024-04-12T12:04:24.479556+0000","last_clean":"2024-04-12T12:04:24.479556+0000","last_became_active":"2024-04-12T12:04:24.478241+0000","last_became_peered":"2024-04-12T12:04:24.478241+0000","last_unstale":"2024-04-12T12:04:24.479556+0000","last_undegraded":"2024-04-12T12:04:24.479556+0000","last_fullsized":"2024-04-12T12:04:24.479556+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T22:52:26.522994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":696,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.474330+0000","last_change":"2024-04-12T12:04:24.473961+0000","last_active":"2024-04-12T12:04:24.474330+0000","last_peered":"2024-04-12T12:04:24.474330+0000","last_clean":"2024-04-12T12:04:24.474330+0000","last_became_active":"2024-04-12T12:04:24.473641+0000","last_became_peered":"2024-04-12T12:04:24.473641+0000","last_unstale":"2024-04-12T12:04:24.474330+0000","last_undegraded":"2024-04-12T12:04:24.474330+0000","last_fullsized":"2024-04-12T12:04:24.474330+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T17:57:25.247747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":2276,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.470441+0000","last_change":"2024-04-12T12:03:15.373337+0000","last_active":"2024-04-12T12:04:24.470441+0000","last_peered":"2024-04-12T12:04:24.470441+0000","last_clean":"2024-04-12T12:04:24.470441+0000","last_became_active":"2024-04-12T12:03:15.357293+0000","last_became_peered":"2024-04-12T12:03:15.357293+0000","last_unstale":"2024-04-12T12:04:24.470441+0000","last_undegraded":"2024-04-12T12:04:24.470441+0000","last_fullsized":"2024-04-12T12:04:24.470441+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:56.740220+0000","last_clean_scrub_stamp":"2024-04-12T11:12:56.740220+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T11:42:47.903268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3492,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":2119,"reported_epoch":1389,"state":"active+clean","last_fresh":"2024-04-12T12:04:24.473295+0000","last_change":"2024-04-12T12:00:35.472529+0000","last_active":"2024-04-12T12:04:24.473295+0000","last_peered":"2024-04-12T12:04:24.473295+0000","last_clean":"2024-04-12T12:04:24.473295+0000","last_became_active":"2024-04-12T12:00:35.472057+0000","last_became_peered":"2024-04-12T12:00:35.472057+0000","last_unstale":"2024-04-12T12:04:24.473295+0000","last_undegraded":"2024-04-12T12:04:24.473295+0000","last_fullsized":"2024-04-12T12:04:24.473295+0000","mapping_epoch":1333,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":1334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T11:12:57.743798+0000","last_clean_scrub_stamp":"2024-04-12T11:12:57.743798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T21:07:59.083703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":13540,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35155,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1706112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234635,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972912,"kb_used_data":9120,"kb_used_omap":1,"kb_used_meta":963774,"kb_avail":92772240,"statfs":{"total":95995035648,"available":94998773760,"internally_reserved":0,"allocated":9338880,"data_stored":6109719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":986905035},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496272,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":797860,"kb_used_data":8788,"kb_used_omap":1,"kb_used_meta":789054,"kb_avail":92947292,"statfs":{"total":95995035648,"available":95178027008,"internally_reserved":0,"allocated":8998912,"data_stored":5781893,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":807991752},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":534,"seq":2293512536582,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770964,"kb_used_data":8772,"kb_used_omap":30,"kb_used_meta":762145,"kb_avail":92974188,"statfs":{"total":95995035648,"available":95205568512,"internally_reserved":0,"allocated":8982528,"data_stored":5777079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31422,"internal_metadata":780436802},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":514,"seq":2207613190666,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1131676,"kb_used_data":8780,"kb_used_omap":43,"kb_used_meta":1122836,"kb_avail":92613476,"statfs":{"total":95995035648,"available":94836199424,"internally_reserved":0,"allocated":8990720,"data_stored":5777683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45013,"internal_metadata":1149784107},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":555,"seq":2383706849794,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1011488,"kb_used_data":9040,"kb_used_omap":36,"kb_used_meta":1002395,"kb_avail":92733664,"statfs":{"total":95995035648,"available":94959271936,"internally_reserved":0,"allocated":9256960,"data_stored":6040383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37708,"internal_metadata":1026452660},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1342,"seq":5763846111283,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":126936,"kb_used_data":9356,"kb_used_omap":32,"kb_used_meta":117535,"kb_avail":93618216,"statfs":{"total":95995035648,"available":95865053184,"internally_reserved":0,"allocated":9580544,"data_stored":6367537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33145,"internal_metadata":120356487},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":611,"seq":2624225018359,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951660,"kb_used_data":8796,"kb_used_omap":42,"kb_used_meta":942805,"kb_avail":92793492,"statfs":{"total":95995035648,"available":95020535808,"internally_reserved":0,"allocated":9007104,"data_stored":5781981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43208,"internal_metadata":965433144},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":1282,"seq":5506148073607,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":84384,"kb_used_data":9360,"kb_used_omap":121,"kb_used_meta":74886,"kb_avail":93660768,"statfs":{"total":95995035648,"available":95908626432,"internally_reserved":0,"allocated":9584640,"data_stored":6368087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":124430,"internal_metadata":76683762},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6088,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5754,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1512,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10321,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6662,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4818,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T12:05:22.355 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-04-12T12:05:22.355 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 82, in run_one_task task = get_task(taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 35, in get_task module = _import('tasks', module_name, task_name, fail_on_import_error=True) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 54, in _import module = __import__( ModuleNotFoundError: No module named 'tasks.workunit' 2024-04-12T12:05:22.577 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=5cadaa7286a046cf9a4f8308372c8379 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 82, in run_one_task task = get_task(taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 35, in get_task module = _import('tasks', module_name, task_name, fail_on_import_error=True) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 54, in _import module = __import__( ModuleNotFoundError: No module named 'tasks.workunit' 2024-04-12T12:05:22.580 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-12T12:05:22.591 INFO:tasks.cephadm:Teardown begin 2024-04-12T12:05:22.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T12:05:22.619 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:05:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:05:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:05:22.631 DEBUG:teuthology.orchestra.run.smithi087:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T12:05:22.667 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-12T12:05:22.667 DEBUG:teuthology.orchestra.run.smithi029:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-12T12:05:22.699 DEBUG:teuthology.orchestra.run.smithi087:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-12T12:05:22.740 INFO:tasks.cephadm:Stopping all daemons... 2024-04-12T12:05:22.741 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-12T12:05:22.741 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.a 2024-04-12T12:05:23.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:22 smithi029 systemd[1]: Stopping Ceph mon.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:05:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T12:05:23.087+0000 7fae29a1d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:05:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a[93646]: 2024-04-12T12:05:23.087+0000 7fae29a1d700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-12T12:05:23.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:23 smithi029 podman[320842]: 2024-04-12 12:05:23.333120624 +0000 UTC m=+0.385405848 container died d1506def0998659ccd935211736c68234566f534f7e49f954fde3a94f4c75644 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-12T12:05:23.520 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:23] ENGINE Bus STOPPING 2024-04-12T12:05:23.520 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T12:05:23.520 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:23] ENGINE Bus STOPPED 2024-04-12T12:05:23.520 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:23] ENGINE Bus STARTING 2024-04-12T12:05:23.520 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:23] ENGINE Serving on http://:::9283 2024-04-12T12:05:23.520 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:23 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:23] ENGINE Bus STARTED 2024-04-12T12:05:24.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:23 smithi029 podman[320842]: 2024-04-12 12:05:23.742662744 +0000 UTC m=+0.794948013 container cleanup d1506def0998659ccd935211736c68234566f534f7e49f954fde3a94f4c75644 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-12T12:05:24.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Apr 12 12:05:23 smithi029 bash[320842]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-a 2024-04-12T12:05:24.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.a.service' 2024-04-12T12:05:24.194 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:05:24.194 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-12T12:05:24.195 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-12T12:05:24.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.c 2024-04-12T12:05:24.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:24 smithi029 systemd[1]: Stopping Ceph mon.c for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:05:24.927 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:24] ENGINE Bus STOPPING 2024-04-12T12:05:24.927 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-12T12:05:24.928 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:24] ENGINE Bus STOPPED 2024-04-12T12:05:24.928 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:24] ENGINE Bus STARTING 2024-04-12T12:05:24.928 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:24] ENGINE Serving on http://:::9283 2024-04-12T12:05:24.928 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: [12/Apr/2024:12:05:24] ENGINE Bus STARTED 2024-04-12T12:05:24.929 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: ::ffff:172.21.15.87 - - [12/Apr/2024:12:05:24] "GET /metrics HTTP/1.1" 200 34137 "" "Prometheus/2.43.0" 2024-04-12T12:05:24.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c[97783]: 2024-04-12T12:05:24.539+0000 7f7c06bf5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:05:24.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:24 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c[97783]: 2024-04-12T12:05:24.539+0000 7f7c06bf5700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-12T12:05:24.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:24 smithi029 podman[320973]: 2024-04-12 12:05:24.683428229 +0000 UTC m=+0.271536380 container died 24b3f58e1d6692c78136ded59902d8d19032a359698a90ca0fac8168aa230e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, ceph=True) 2024-04-12T12:05:25.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:25 smithi029 podman[320973]: 2024-04-12 12:05:25.111607147 +0000 UTC m=+0.699715284 container cleanup 24b3f58e1d6692c78136ded59902d8d19032a359698a90ca0fac8168aa230e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, GIT_CLEAN=True, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-12T12:05:25.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:25 smithi029 bash[320973]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c 2024-04-12T12:05:25.549 INFO:journalctl@ceph.mon.c.smithi029.stdout:Apr 12 12:05:25 smithi029 podman[320998]: 2024-04-12 12:05:25.331749909 +0000 UTC m=+0.646986376 container remove 24b3f58e1d6692c78136ded59902d8d19032a359698a90ca0fac8168aa230e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-c, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0) 2024-04-12T12:05:25.571 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.c.service' 2024-04-12T12:05:25.608 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:05:25.608 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-12T12:05:25.608 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-12T12:05:25.608 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.b 2024-04-12T12:05:25.940 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:25 smithi087 systemd[1]: Stopping Ceph mon.b for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:05:26.217 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:25 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b[89010]: 2024-04-12T12:05:25.937+0000 7f0e7b030700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:05:26.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:25 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b[89010]: 2024-04-12T12:05:25.937+0000 7f0e7b030700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-12T12:05:26.218 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:26 smithi087 podman[245522]: 2024-04-12 12:05:26.122429376 +0000 UTC m=+0.300489604 container died ae22114389c9fe0161e2f8e45d4753a9066723b5b05b3c50b1640e0faec7590f (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409) 2024-04-12T12:05:26.717 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:26 smithi087 podman[245522]: 2024-04-12 12:05:26.639410851 +0000 UTC m=+0.817471079 container cleanup ae22114389c9fe0161e2f8e45d4753a9066723b5b05b3c50b1640e0faec7590f (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T12:05:26.718 INFO:journalctl@ceph.mon.b.smithi087.stdout:Apr 12 12:05:26 smithi087 bash[245522]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mon-b 2024-04-12T12:05:27.115 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mon.b.service' 2024-04-12T12:05:27.159 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:05:27.160 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-12T12:05:27.160 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-12T12:05:27.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.y 2024-04-12T12:05:27.481 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:27 smithi029 systemd[1]: Stopping Ceph mgr.y for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:05:27.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:27 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T12:05:27.478+0000 7f4c5dfaa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:05:27.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:27 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y[84546]: 2024-04-12T12:05:27.478+0000 7f4c5dfaa700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-12T12:05:27.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:27 smithi029 podman[321105]: 2024-04-12 12:05:27.511151076 +0000 UTC m=+0.155082660 container died 05c1874892626f277133aa21c0c3bc2c0ba76a92c2bfce5fc5785af0d7dbb257 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-12T12:05:28.205 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:27 smithi029 podman[321105]: 2024-04-12 12:05:27.940514963 +0000 UTC m=+0.584446538 container cleanup 05c1874892626f277133aa21c0c3bc2c0ba76a92c2bfce5fc5785af0d7dbb257 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-12T12:05:28.205 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Apr 12 12:05:27 smithi029 bash[321105]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-y 2024-04-12T12:05:28.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.y.service' 2024-04-12T12:05:28.448 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:05:28.449 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-12T12:05:28.449 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-12T12:05:28.449 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.x 2024-04-12T12:05:28.768 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 12:05:28 smithi087 systemd[1]: Stopping Ceph mgr.x for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:05:29.063 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 12:05:28 smithi087 podman[245643]: 2024-04-12 12:05:28.766515463 +0000 UTC m=+0.126099768 container died 5a46135dfdbc3347e8cb43cdef1220f8ca45f69854c290c1c106e6ad098589ec (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409) 2024-04-12T12:05:29.467 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 12:05:29 smithi087 podman[245643]: 2024-04-12 12:05:29.244151539 +0000 UTC m=+0.603735851 container cleanup 5a46135dfdbc3347e8cb43cdef1220f8ca45f69854c290c1c106e6ad098589ec (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409) 2024-04-12T12:05:29.467 INFO:journalctl@ceph.mgr.x.smithi087.stdout:Apr 12 12:05:29 smithi087 bash[245643]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-mgr-x 2024-04-12T12:05:29.652 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@mgr.x.service' 2024-04-12T12:05:29.689 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:05:29.689 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-12T12:05:29.689 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-12T12:05:29.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.0 2024-04-12T12:05:30.000 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:29 smithi029 systemd[1]: Stopping Ceph osd.0 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:05:30.269 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[106271]: 2024-04-12T12:05:29.998+0000 7f4b2ab70700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:05:30.269 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[106271]: 2024-04-12T12:05:29.998+0000 7f4b2ab70700 -1 osd.0 1389 *** Got signal Terminated *** 2024-04-12T12:05:30.269 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:29 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0[106271]: 2024-04-12T12:05:29.998+0000 7f4b2ab70700 -1 osd.0 1389 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T12:05:32.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:05:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:05:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi029:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"80efffc6-f8bc-11ee-bc8c-c7b262605968\", instance=\"smithi029:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-12T12:05:36.014 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:35 smithi029 podman[321228]: 2024-04-12 12:05:35.687564024 +0000 UTC m=+5.809736082 container died 78ce9733fe38346de4f4bc0a3f3a08409a968bf9c477438680ad11dbb22f8102 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240409) 2024-04-12T12:05:36.269 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:36 smithi029 podman[321228]: 2024-04-12 12:05:36.20446498 +0000 UTC m=+6.326637030 container cleanup 78ce9733fe38346de4f4bc0a3f3a08409a968bf9c477438680ad11dbb22f8102 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185) 2024-04-12T12:05:36.269 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:36 smithi029 bash[321228]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0 2024-04-12T12:05:36.769 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:36 smithi029 podman[321247]: 2024-04-12 12:05:36.491092679 +0000 UTC m=+0.795751503 container remove 78ce9733fe38346de4f4bc0a3f3a08409a968bf9c477438680ad11dbb22f8102 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-rf-comb1-2cee185, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T12:05:37.092 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:36 smithi029 podman[321338]: 2024-04-12 12:05:36.72684935 +0000 UTC m=+0.019873536 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T12:05:37.092 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:36 smithi029 podman[321338]: 2024-04-12 12:05:36.85163927 +0000 UTC m=+0.144663498 container create 1c8c8f680875b54a0c91f73b82594896b039ecc1f51a40516748c203ac123cc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-04-12T12:05:37.519 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:37 smithi029 podman[321338]: 2024-04-12 12:05:37.090785203 +0000 UTC m=+0.383809389 container init 1c8c8f680875b54a0c91f73b82594896b039ecc1f51a40516748c203ac123cc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-12T12:05:37.519 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:37 smithi029 podman[321338]: 2024-04-12 12:05:37.095089439 +0000 UTC m=+0.388113632 container start 1c8c8f680875b54a0c91f73b82594896b039ecc1f51a40516748c203ac123cc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-rf-comb1-2cee185) 2024-04-12T12:05:37.519 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:37 smithi029 podman[321338]: 2024-04-12 12:05:37.2250097 +0000 UTC m=+0.518033955 container attach 1c8c8f680875b54a0c91f73b82594896b039ecc1f51a40516748c203ac123cc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-12T12:05:38.473 INFO:journalctl@ceph.osd.0.smithi029.stdout:Apr 12 12:05:38 smithi029 podman[321338]: 2024-04-12 12:05:38.1597045 +0000 UTC m=+1.452728694 container died 1c8c8f680875b54a0c91f73b82594896b039ecc1f51a40516748c203ac123cc3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-0-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-rf-comb1-2cee185, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD) 2024-04-12T12:05:38.942 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.0.service' 2024-04-12T12:05:38.989 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:05:38.989 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-12T12:05:38.989 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-12T12:05:38.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.1 2024-04-12T12:05:39.265 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:39 smithi029 systemd[1]: Stopping Ceph osd.1 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:05:39.519 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[111318]: 2024-04-12T12:05:39.262+0000 7fd5b4e62700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:05:39.519 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[111318]: 2024-04-12T12:05:39.262+0000 7fd5b4e62700 -1 osd.1 1389 *** Got signal Terminated *** 2024-04-12T12:05:39.519 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1[111318]: 2024-04-12T12:05:39.262+0000 7fd5b4e62700 -1 osd.1 1389 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T12:05:44.994 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:44 smithi029 podman[321445]: 2024-04-12 12:05:44.689607304 +0000 UTC m=+5.519229225 container died 260ced18c216f31b0394d7c1f6bdf8540c18349534482ec31848288db75daffc (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-12T12:05:45.269 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:45 smithi029 podman[321445]: 2024-04-12 12:05:45.168159557 +0000 UTC m=+5.997781465 container cleanup 260ced18c216f31b0394d7c1f6bdf8540c18349534482ec31848288db75daffc (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD) 2024-04-12T12:05:45.269 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:45 smithi029 bash[321445]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1 2024-04-12T12:05:45.740 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:45 smithi029 podman[321462]: 2024-04-12 12:05:45.404886411 +0000 UTC m=+0.718518733 container remove 260ced18c216f31b0394d7c1f6bdf8540c18349534482ec31848288db75daffc (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409) 2024-04-12T12:05:45.996 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:45 smithi029 podman[321548]: 2024-04-12 12:05:45.642340893 +0000 UTC m=+0.018003217 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T12:05:45.997 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:45 smithi029 podman[321548]: 2024-04-12 12:05:45.749664733 +0000 UTC m=+0.125327048 container create 90fff53813828a167ec2b0f8eaff348b44a4f19e39abc1bb031ee4631b080ea7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-12T12:05:45.997 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:45 smithi029 podman[321548]: 2024-04-12 12:05:45.995615917 +0000 UTC m=+0.371278233 container init 90fff53813828a167ec2b0f8eaff348b44a4f19e39abc1bb031ee4631b080ea7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-12T12:05:46.269 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:45 smithi029 podman[321548]: 2024-04-12 12:05:45.999002438 +0000 UTC m=+0.374664759 container start 90fff53813828a167ec2b0f8eaff348b44a4f19e39abc1bb031ee4631b080ea7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T12:05:46.269 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:46 smithi029 podman[321548]: 2024-04-12 12:05:46.112933416 +0000 UTC m=+0.488595758 container attach 90fff53813828a167ec2b0f8eaff348b44a4f19e39abc1bb031ee4631b080ea7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-12T12:05:46.576 INFO:journalctl@ceph.osd.1.smithi029.stdout:Apr 12 12:05:46 smithi029 podman[321548]: 2024-04-12 12:05:46.2936585 +0000 UTC m=+0.669320822 container died 90fff53813828a167ec2b0f8eaff348b44a4f19e39abc1bb031ee4631b080ea7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-12T12:05:46.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.1.service' 2024-04-12T12:05:46.995 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:05:46.995 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-12T12:05:46.995 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-12T12:05:46.996 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.2 2024-04-12T12:05:47.269 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:47 smithi029 systemd[1]: Stopping Ceph osd.2 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:05:47.769 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:47 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[116047]: 2024-04-12T12:05:47.324+0000 7fbe6f060700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:05:47.769 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:47 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[116047]: 2024-04-12T12:05:47.324+0000 7fbe6f060700 -1 osd.2 1389 *** Got signal Terminated *** 2024-04-12T12:05:47.769 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:47 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2[116047]: 2024-04-12T12:05:47.324+0000 7fbe6f060700 -1 osd.2 1389 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T12:05:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:05:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:05:51.350Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-04-12T12:05:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:05:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:05:51.350Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-04-12T12:05:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:05:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:05:51.351Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-04-12T12:05:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:05:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:05:51.354Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-04-12T12:05:51.718 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:05:51 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:05:51.354Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-04-12T12:05:53.235 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:52 smithi029 podman[321651]: 2024-04-12 12:05:52.953133896 +0000 UTC m=+5.737093045 container died db6a92e3e974206b07f65766d558d949ca670fdcfc52de985ba9c8a6072b7ef7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2) 2024-04-12T12:05:53.519 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:53 smithi029 podman[321651]: 2024-04-12 12:05:53.437844714 +0000 UTC m=+6.221803863 container cleanup db6a92e3e974206b07f65766d558d949ca670fdcfc52de985ba9c8a6072b7ef7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2, ceph=True, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-12T12:05:53.519 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:53 smithi029 bash[321651]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2 2024-04-12T12:05:54.019 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:53 smithi029 podman[321668]: 2024-04-12 12:05:53.765972552 +0000 UTC m=+0.812894117 container remove db6a92e3e974206b07f65766d558d949ca670fdcfc52de985ba9c8a6072b7ef7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2, RELEASE=wip-rf-comb1-2cee185, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-12T12:05:54.391 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:54 smithi029 podman[321758]: 2024-04-12 12:05:53.969796639 +0000 UTC m=+0.017922614 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T12:05:54.391 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:54 smithi029 podman[321758]: 2024-04-12 12:05:54.07601846 +0000 UTC m=+0.124144418 container create 058fab128ab79b79af7ce2a0f8cc1c2b2f5f8bf54b354838b9266b5bc5867b92 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-12T12:05:54.392 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:54 smithi029 podman[321758]: 2024-04-12 12:05:54.298389639 +0000 UTC m=+0.346515599 container init 058fab128ab79b79af7ce2a0f8cc1c2b2f5f8bf54b354838b9266b5bc5867b92 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-12T12:05:54.392 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:54 smithi029 podman[321758]: 2024-04-12 12:05:54.301621478 +0000 UTC m=+0.349747438 container start 058fab128ab79b79af7ce2a0f8cc1c2b2f5f8bf54b354838b9266b5bc5867b92 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-12T12:05:54.392 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:54 smithi029 podman[321758]: 2024-04-12 12:05:54.390469621 +0000 UTC m=+0.438595596 container attach 058fab128ab79b79af7ce2a0f8cc1c2b2f5f8bf54b354838b9266b5bc5867b92 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-12T12:05:54.769 INFO:journalctl@ceph.osd.2.smithi029.stdout:Apr 12 12:05:54 smithi029 podman[321758]: 2024-04-12 12:05:54.59825363 +0000 UTC m=+0.646379598 container died 058fab128ab79b79af7ce2a0f8cc1c2b2f5f8bf54b354838b9266b5bc5867b92 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T12:05:55.180 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.2.service' 2024-04-12T12:05:55.225 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:05:55.226 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-12T12:05:55.226 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-12T12:05:55.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.3 2024-04-12T12:05:55.458 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:05:55 smithi029 systemd[1]: Stopping Ceph osd.3 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:05:55.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:05:55 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:05:55.569+0000 7f6b263a7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:05:55.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:05:55 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:05:55.569+0000 7f6b263a7700 -1 osd.3 1389 *** Got signal Terminated *** 2024-04-12T12:05:55.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:05:55 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:05:55.569+0000 7f6b263a7700 -1 osd.3 1389 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T12:05:56.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:05:56 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:05:56.520+0000 7f6b1e993700 -1 osd.3 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.682754+0000 front 2024-04-12T12:05:31.682722+0000 (oldest deadline 2024-04-12T12:05:55.782832+0000) 2024-04-12T12:05:57.217 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:05:56 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:05:56.918+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:05:57.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:05:57 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:05:57.483+0000 7f6b1e993700 -1 osd.3 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.682754+0000 front 2024-04-12T12:05:31.682722+0000 (oldest deadline 2024-04-12T12:05:55.782832+0000) 2024-04-12T12:05:58.217 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:05:57 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:05:57.871+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:05:58.658 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:05:58 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:05:58.469+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:05:58.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:05:58 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:05:58.443+0000 7f6b1e993700 -1 osd.3 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.682754+0000 front 2024-04-12T12:05:31.682722+0000 (oldest deadline 2024-04-12T12:05:55.782832+0000) 2024-04-12T12:05:58.967 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:05:58 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:05:58.894+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:05:58.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:05:58 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:05:58.656+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:05:59.449 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:05:59 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:05:59.172+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:05:59.717 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:05:59 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:05:59.448+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:05:59.718 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:05:59 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:05:59.629+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:05:59.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:05:59 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:05:59.461+0000 7f6b1e993700 -1 osd.3 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.682754+0000 front 2024-04-12T12:05:31.682722+0000 (oldest deadline 2024-04-12T12:05:55.782832+0000) 2024-04-12T12:06:00.217 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:00.162+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:00.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:05:59 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:05:59.942+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:00.717 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:00.611+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:00.718 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:00.415+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:06:00.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:00 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3[301820]: 2024-04-12T12:06:00.435+0000 7f6b1e993700 -1 osd.3 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.682754+0000 front 2024-04-12T12:05:31.682722+0000 (oldest deadline 2024-04-12T12:05:55.782832+0000) 2024-04-12T12:06:01.217 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:01.145+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:01.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:00 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:00.965+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:01.342 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:01 smithi029 podman[321863]: 2024-04-12 12:06:01.042122442 +0000 UTC m=+5.617635970 container died e138af00dd594968e95b8d25d262eaa1fdd2f4b32935bec948335bd830f9984b (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-rf-comb1-2cee185, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-12T12:06:01.677 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:01 smithi029 podman[321863]: 2024-04-12 12:06:01.465222404 +0000 UTC m=+6.040735927 container cleanup e138af00dd594968e95b8d25d262eaa1fdd2f4b32935bec948335bd830f9984b (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-12T12:06:01.677 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:01 smithi029 bash[321863]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3 2024-04-12T12:06:01.717 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:01.395+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:06:01.717 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:01.643+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:02.019 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:01 smithi029 podman[321893]: 2024-04-12 12:06:01.67621223 +0000 UTC m=+0.636703376 container remove e138af00dd594968e95b8d25d262eaa1fdd2f4b32935bec948335bd830f9984b (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, ceph=True) 2024-04-12T12:06:02.408 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:02 smithi029 podman[321985]: 2024-04-12 12:06:01.991363744 +0000 UTC m=+0.017919531 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T12:06:02.408 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:02 smithi029 podman[321985]: 2024-04-12 12:06:02.10319238 +0000 UTC m=+0.129748156 container create 2587899b1a5c9cfb44d5b7c4eca54cc469ca878370f78455e7b68ebe557c4dc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409, RELEASE=wip-rf-comb1-2cee185, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-12T12:06:02.408 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:02 smithi029 podman[321985]: 2024-04-12 12:06:02.3240108 +0000 UTC m=+0.350566588 container init 2587899b1a5c9cfb44d5b7c4eca54cc469ca878370f78455e7b68ebe557c4dc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-rf-comb1-2cee185, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T12:06:02.408 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:02 smithi029 podman[321985]: 2024-04-12 12:06:02.32742289 +0000 UTC m=+0.353978663 container start 2587899b1a5c9cfb44d5b7c4eca54cc469ca878370f78455e7b68ebe557c4dc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2) 2024-04-12T12:06:02.409 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:02 smithi029 podman[321985]: 2024-04-12 12:06:02.407153989 +0000 UTC m=+0.433709770 container attach 2587899b1a5c9cfb44d5b7c4eca54cc469ca878370f78455e7b68ebe557c4dc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0) 2024-04-12T12:06:02.444 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:02.100+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:02.444 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:01 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:01.975+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:02.717 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:02.646+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:02.718 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:02 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:02.441+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:06:02.769 INFO:journalctl@ceph.osd.3.smithi029.stdout:Apr 12 12:06:02 smithi029 podman[321985]: 2024-04-12 12:06:02.624745677 +0000 UTC m=+0.651301501 container died 2587899b1a5c9cfb44d5b7c4eca54cc469ca878370f78455e7b68ebe557c4dc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409) 2024-04-12T12:06:03.315 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.3.service' 2024-04-12T12:06:03.359 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:06:03.359 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-12T12:06:03.360 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-12T12:06:03.360 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.4 2024-04-12T12:06:03.391 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:03.088+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:03.391 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:03.007+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:03.674 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:03 smithi087 systemd[1]: Stopping Ceph osd.4 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:06:03.674 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:03.445+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:06:03.967 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:03.687+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:03.968 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:03.671+0000 7f7eaad9b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:06:03.968 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:03.671+0000 7f7eaad9b700 -1 osd.4 1389 *** Got signal Terminated *** 2024-04-12T12:06:03.968 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:03.671+0000 7f7eaad9b700 -1 osd.4 1389 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T12:06:04.467 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:03 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:03.967+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:04.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:04 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:04.098+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:04.724 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:04 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:04.465+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:06:05.132 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:04 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:04.722+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:05.133 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:04 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:04.929+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:05.435 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:05.131+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:05.708 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:05.433+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:06:05.967 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:05.706+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:05.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:05.943+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:05.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:05 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:05.943+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:06.393 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:06.140+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:06.670 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:06.391+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:06:06.965 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:06.669+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:07.217 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:07 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:07.118+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:07.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:06.963+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:07.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:06 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:06.963+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:07.655 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:07 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:07.397+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:06:07.928 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:07 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:07.653+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:08.218 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:08.114+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:08.218 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:08.114+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:43.792709+0000 front 2024-04-12T12:05:43.792821+0000 (oldest deadline 2024-04-12T12:06:07.292572+0000) 2024-04-12T12:06:08.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:07 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:07.926+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:08.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:07 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:07.926+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:08.651 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:08.400+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:32.471486+0000 front 2024-04-12T12:05:32.471529+0000 (oldest deadline 2024-04-12T12:05:58.371265+0000) 2024-04-12T12:06:08.652 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4[95751]: 2024-04-12T12:06:08.400+0000 7f7ea738f700 -1 osd.4 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:41.773019+0000 front 2024-04-12T12:05:41.773068+0000 (oldest deadline 2024-04-12T12:06:07.672979+0000) 2024-04-12T12:06:08.967 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:08.650+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:08.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:08.898+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:08.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:08 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:08.898+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:09.416 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:09 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:09.144+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:09.417 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:09 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:09.144+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:43.792709+0000 front 2024-04-12T12:05:43.792821+0000 (oldest deadline 2024-04-12T12:06:07.292572+0000) 2024-04-12T12:06:09.417 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:09 smithi087 podman[245782]: 2024-04-12 12:06:09.157795083 +0000 UTC m=+5.599593687 container died fb553cefdd39034be0f8dfca2e163cde9d03ab9d40572f7279cfe8007f4fccb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-rf-comb1-2cee185, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2) 2024-04-12T12:06:09.675 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:09 smithi087 podman[245782]: 2024-04-12 12:06:09.580935258 +0000 UTC m=+6.022733855 container cleanup fb553cefdd39034be0f8dfca2e163cde9d03ab9d40572f7279cfe8007f4fccb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-rf-comb1-2cee185, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5) 2024-04-12T12:06:09.675 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:09 smithi087 bash[245782]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4 2024-04-12T12:06:09.967 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:09 smithi087 podman[245802]: 2024-04-12 12:06:09.815829 +0000 UTC m=+0.656520842 container remove fb553cefdd39034be0f8dfca2e163cde9d03ab9d40572f7279cfe8007f4fccb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4, GIT_CLEAN=True, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T12:06:09.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:09 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:09.869+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:09.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:09 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:09.869+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:09.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:09 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:09.673+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:09.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:09 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:09.673+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:10.404 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:10.143+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:10.404 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:10.143+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:43.792709+0000 front 2024-04-12T12:05:43.792821+0000 (oldest deadline 2024-04-12T12:06:07.292572+0000) 2024-04-12T12:06:10.404 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:10 smithi087 podman[245890]: 2024-04-12 12:06:10.129674234 +0000 UTC m=+0.017798401 image pull 4a0e59462345bfae27c0a8963ac6c42048934670eacf0bd882c36ca51384bc4d quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d 2024-04-12T12:06:10.404 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:10 smithi087 podman[245890]: 2024-04-12 12:06:10.266801103 +0000 UTC m=+0.154925273 container create 885ca1fc817ed52915cf78a66764ee8abc09d96c89cf6e15646503ca3ffa3de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-rf-comb1-2cee185, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-12T12:06:10.666 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:10 smithi087 podman[245890]: 2024-04-12 12:06:10.494401265 +0000 UTC m=+0.382525433 container init 885ca1fc817ed52915cf78a66764ee8abc09d96c89cf6e15646503ca3ffa3de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-rf-comb1-2cee185, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-12T12:06:10.666 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:10 smithi087 podman[245890]: 2024-04-12 12:06:10.497552815 +0000 UTC m=+0.385676983 container start 885ca1fc817ed52915cf78a66764ee8abc09d96c89cf6e15646503ca3ffa3de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=wip-rf-comb1-2cee185, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T12:06:10.666 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:10 smithi087 podman[245890]: 2024-04-12 12:06:10.627624234 +0000 UTC m=+0.515748401 container attach 885ca1fc817ed52915cf78a66764ee8abc09d96c89cf6e15646503ca3ffa3de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-rf-comb1-2cee185, io.buildah.version=1.33.5) 2024-04-12T12:06:10.967 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:10.663+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:10.967 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:10.663+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:10.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:10.869+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:10.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:10 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:10.869+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:11.467 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:11 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:11.173+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:11.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:11 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:11.173+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:43.792709+0000 front 2024-04-12T12:05:43.792821+0000 (oldest deadline 2024-04-12T12:06:07.292572+0000) 2024-04-12T12:06:11.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:11 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:11.712+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:11.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:11 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:11.712+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:11.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:11 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:11.894+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:11.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:11 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:11.894+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:12.468 INFO:journalctl@ceph.osd.4.smithi087.stdout:Apr 12 12:06:12 smithi087 podman[245890]: 2024-04-12 12:06:12.196538069 +0000 UTC m=+2.084662228 container died 885ca1fc817ed52915cf78a66764ee8abc09d96c89cf6e15646503ca3ffa3de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:d8e027e1be27399468ac02e4a9dae6525a2ab91ecb3ce73c9d2cb1ad1204958d, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, RELEASE=wip-rf-comb1-2cee185, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-12T12:06:12.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:12.206+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:12.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:12.206+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:43.792709+0000 front 2024-04-12T12:05:43.792821+0000 (oldest deadline 2024-04-12T12:06:07.292572+0000) 2024-04-12T12:06:12.748 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:12.714+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:12.748 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:12.714+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:12.770 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.4.service' 2024-04-12T12:06:12.856 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:06:12.856 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-12T12:06:12.857 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-12T12:06:12.857 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.5 2024-04-12T12:06:13.135 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:12 smithi087 systemd[1]: Stopping Ceph osd.5 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:06:13.135 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:12.915+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:13.136 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:12 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:12.915+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:13.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:13.132+0000 7fbd3ef6f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:06:13.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:13.132+0000 7fbd3ef6f700 -1 osd.5 1389 *** Got signal Terminated *** 2024-04-12T12:06:13.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:13.132+0000 7fbd3ef6f700 -1 osd.5 1389 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T12:06:13.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:13.165+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:13.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:13.165+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:43.792709+0000 front 2024-04-12T12:05:43.792821+0000 (oldest deadline 2024-04-12T12:06:07.292572+0000) 2024-04-12T12:06:13.967 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:13.675+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:13.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:13.675+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:13.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:13.877+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:13.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:13 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:13.877+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:14.467 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:14 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:14.143+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:14.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:14 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:14.143+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:43.792709+0000 front 2024-04-12T12:05:43.792821+0000 (oldest deadline 2024-04-12T12:06:07.292572+0000) 2024-04-12T12:06:15.145 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:14 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:14.723+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:15.145 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:14 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:14.723+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:15.145 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:14 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:14.723+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:15.146 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:14 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:14.837+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:15.146 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:14 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:14.837+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:15.146 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:14 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:14.837+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:15.467 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:15.144+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:15.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:15.144+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:43.792709+0000 front 2024-04-12T12:05:43.792821+0000 (oldest deadline 2024-04-12T12:06:07.292572+0000) 2024-04-12T12:06:16.199 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:15.844+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:16.199 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:15.844+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:16.199 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:15.844+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:16.200 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:15.761+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:16.200 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:15.761+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:16.200 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:15 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:15.761+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:16.467 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:16.193+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:16.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:16.193+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:43.792709+0000 front 2024-04-12T12:05:43.792821+0000 (oldest deadline 2024-04-12T12:06:07.292572+0000) 2024-04-12T12:06:16.967 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:16.716+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:16.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:16.716+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:16.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:16.716+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:16.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:16.823+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:16.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:16.823+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:16.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:16 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:16.823+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:17.467 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:17.203+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:33.689264+0000 front 2024-04-12T12:05:33.689307+0000 (oldest deadline 2024-04-12T12:05:58.989272+0000) 2024-04-12T12:06:17.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:17.203+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:43.792709+0000 front 2024-04-12T12:05:43.792821+0000 (oldest deadline 2024-04-12T12:06:07.292572+0000) 2024-04-12T12:06:17.468 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5[199511]: 2024-04-12T12:06:17.203+0000 7fbd3b565700 -1 osd.5 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:51.294001+0000 front 2024-04-12T12:05:51.293982+0000 (oldest deadline 2024-04-12T12:06:16.593820+0000) 2024-04-12T12:06:17.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:17.695+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:17.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:17.695+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:17.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:17.695+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:17.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:17.809+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:17.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:17.809+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:17.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:17 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:17.809+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:18.911 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:18 smithi087 podman[245995]: 2024-04-12 12:06:18.618512907 +0000 UTC m=+5.581905671 container died 081b534f5a76daffb3c3618bbdbc8fa96af3199d5d4bc9c0c45192ddf84f21f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=reef-d540eba, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-12T12:06:18.911 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:18 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:18.677+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:18.911 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:18 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:18.677+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:18.911 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:18 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:18.677+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:18.912 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:18 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:18.857+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:18.912 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:18 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:18.857+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:18.912 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:18 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:18.857+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:19.217 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:18 smithi087 podman[245995]: 2024-04-12 12:06:18.984109144 +0000 UTC m=+5.947501907 container cleanup 081b534f5a76daffb3c3618bbdbc8fa96af3199d5d4bc9c0c45192ddf84f21f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=reef-d540eba, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-12T12:06:19.218 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:18 smithi087 bash[245995]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5 2024-04-12T12:06:19.566 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:19 smithi087 podman[246012]: 2024-04-12 12:06:19.268559859 +0000 UTC m=+0.647694309 container remove 081b534f5a76daffb3c3618bbdbc8fa96af3199d5d4bc9c0c45192ddf84f21f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=reef-d540eba, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-04-12T12:06:19.823 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:19 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:19.634+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:19.823 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:19 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:19.634+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:19.824 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:19 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:19.634+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:19.824 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:19 smithi087 podman[246105]: 2024-04-12 12:06:19.469072243 +0000 UTC m=+0.018453635 image pull bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3 2024-04-12T12:06:19.824 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:19 smithi087 podman[246105]: 2024-04-12 12:06:19.577905424 +0000 UTC m=+0.127286845 container create 909e372a86e844a1175b9d558b80c6f34396358acf51e5096d70ed25fecfc3da (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=reef-d540eba, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True) 2024-04-12T12:06:19.824 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:19 smithi087 podman[246105]: 2024-04-12 12:06:19.788578438 +0000 UTC m=+0.337959814 container init 909e372a86e844a1175b9d558b80c6f34396358acf51e5096d70ed25fecfc3da (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=reef-d540eba, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-12T12:06:19.824 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:19 smithi087 podman[246105]: 2024-04-12 12:06:19.791632649 +0000 UTC m=+0.341014027 container start 909e372a86e844a1175b9d558b80c6f34396358acf51e5096d70ed25fecfc3da (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=reef-d540eba) 2024-04-12T12:06:19.825 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:19 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:19.822+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:19.825 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:19 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:19.822+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:19.825 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:19 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:19.822+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:20.143 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:19 smithi087 podman[246105]: 2024-04-12 12:06:19.888984689 +0000 UTC m=+0.438366080 container attach 909e372a86e844a1175b9d558b80c6f34396358acf51e5096d70ed25fecfc3da (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=reef-d540eba, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-12T12:06:20.467 INFO:journalctl@ceph.osd.5.smithi087.stdout:Apr 12 12:06:20 smithi087 podman[246105]: 2024-04-12 12:06:20.143458398 +0000 UTC m=+0.692839789 container died 909e372a86e844a1175b9d558b80c6f34396358acf51e5096d70ed25fecfc3da (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-5-deactivate, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=reef-d540eba, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T12:06:20.752 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:20.683+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:20.752 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:20.683+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:20.752 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:20.683+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:20.773 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.5.service' 2024-04-12T12:06:20.812 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:06:20.813 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-12T12:06:20.813 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-12T12:06:20.813 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.6 2024-04-12T12:06:21.155 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:20 smithi087 systemd[1]: Stopping Ceph osd.6 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:06:21.156 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:20.804+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:21.156 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:20.804+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:21.156 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:20 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:20.804+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:21.467 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:21.154+0000 7f5630e64700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:06:21.468 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:21.154+0000 7f5630e64700 -1 osd.6 1389 *** Got signal Terminated *** 2024-04-12T12:06:21.468 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:21.154+0000 7f5630e64700 -1 osd.6 1389 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T12:06:21.967 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:21.803+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:21.967 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:21.803+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:21.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:21.803+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:21.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:21.713+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:21.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:21.713+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:21.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:21 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:21.713+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:22.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:22.692+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:22.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:22.692+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:22.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:22.692+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:22.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:22.771+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:22.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:22.771+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:22.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:22.771+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:22.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:22 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:22.771+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:23.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:23.672+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:23.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:23.672+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:23.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:23.672+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:23.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:23.731+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:23.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:23.731+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:23.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:23.731+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:23.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:23 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:23.731+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:25.217 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:24 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:24.720+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:25.218 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:24 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:24.720+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:25.218 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:24 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:24.720+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:25.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:24 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:24.769+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:25.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:24 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:24.769+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:25.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:24 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:24.769+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:25.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:24 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:24.769+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:25.967 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:25 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:25.695+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:34.841523+0000 front 2024-04-12T12:05:34.841619+0000 (oldest deadline 2024-04-12T12:05:57.741622+0000) 2024-04-12T12:06:25.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:25 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:25.695+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:44.142842+0000 front 2024-04-12T12:05:44.142963+0000 (oldest deadline 2024-04-12T12:06:09.343305+0000) 2024-04-12T12:06:25.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:25 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:25.695+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:52.244401+0000 front 2024-04-12T12:05:52.244493+0000 (oldest deadline 2024-04-12T12:06:13.944210+0000) 2024-04-12T12:06:25.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:25 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6[49737]: 2024-04-12T12:06:25.695+0000 7f562d45a700 -1 osd.6 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:06:00.345698+0000 front 2024-04-12T12:06:00.345782+0000 (oldest deadline 2024-04-12T12:06:25.645580+0000) 2024-04-12T12:06:25.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:25 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:25.772+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:25.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:25 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:25.773+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:25.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:25 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:25.773+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:25.968 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:25 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:25.773+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:26.939 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:26 smithi087 podman[246211]: 2024-04-12 12:06:26.662773015 +0000 UTC m=+5.636070040 container died 2b67d3f477dd7f9d1690fa8d8ff270dd0aea640072b5be81a4f0c13768b7fbd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6, RELEASE=reef-d540eba, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403) 2024-04-12T12:06:26.939 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:26 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:26.816+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:26.939 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:26 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:26.816+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:26.940 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:26 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:26.816+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:26.940 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:26 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:26.816+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:27.217 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:27 smithi087 podman[246211]: 2024-04-12 12:06:27.012102719 +0000 UTC m=+5.985399744 container cleanup 2b67d3f477dd7f9d1690fa8d8ff270dd0aea640072b5be81a4f0c13768b7fbd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6, RELEASE=reef-d540eba, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T12:06:27.217 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:27 smithi087 bash[246211]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6 2024-04-12T12:06:27.663 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:27 smithi087 podman[246228]: 2024-04-12 12:06:27.246823155 +0000 UTC m=+0.575421998 container remove 2b67d3f477dd7f9d1690fa8d8ff270dd0aea640072b5be81a4f0c13768b7fbd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=reef-d540eba, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T12:06:27.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:27 smithi087 podman[246320]: 2024-04-12 12:06:27.565842781 +0000 UTC m=+0.018225330 image pull bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3 2024-04-12T12:06:27.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:27 smithi087 podman[246320]: 2024-04-12 12:06:27.701451554 +0000 UTC m=+0.153834103 container create 7526d1bafb5c99c9778dcca9990412657d56e18b0f0390dbd0991ec6aa945065 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=reef-d540eba, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T12:06:27.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:27 smithi087 podman[246320]: 2024-04-12 12:06:27.908925086 +0000 UTC m=+0.361307635 container init 7526d1bafb5c99c9778dcca9990412657d56e18b0f0390dbd0991ec6aa945065 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240403, RELEASE=reef-d540eba, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-12T12:06:27.968 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:27 smithi087 podman[246320]: 2024-04-12 12:06:27.912028547 +0000 UTC m=+0.364411101 container start 7526d1bafb5c99c9778dcca9990412657d56e18b0f0390dbd0991ec6aa945065 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-d540eba, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0) 2024-04-12T12:06:27.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:27 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:27.822+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:27.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:27 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:27.822+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:27.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:27 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:27.822+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:27.969 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:27 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:27.822+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:28.467 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:27 smithi087 podman[246320]: 2024-04-12 12:06:27.993286382 +0000 UTC m=+0.445668947 container attach 7526d1bafb5c99c9778dcca9990412657d56e18b0f0390dbd0991ec6aa945065 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, RELEASE=reef-d540eba, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-12T12:06:28.468 INFO:journalctl@ceph.osd.6.smithi087.stdout:Apr 12 12:06:28 smithi087 podman[246320]: 2024-04-12 12:06:28.199060707 +0000 UTC m=+0.651443257 container died 7526d1bafb5c99c9778dcca9990412657d56e18b0f0390dbd0991ec6aa945065 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=reef-d540eba, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux ) 2024-04-12T12:06:28.773 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.6.service' 2024-04-12T12:06:28.814 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:06:28.814 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-12T12:06:28.814 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-12T12:06:28.815 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.7 2024-04-12T12:06:29.150 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:28 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:28.817+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:29.150 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:28 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:28.817+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:29.151 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:28 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:28.817+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:29.151 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:28 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:28.817+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:29.151 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:28 smithi087 systemd[1]: Stopping Ceph osd.7 for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:06:29.418 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:29.148+0000 7f2afc06a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T12:06:29.419 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:29.148+0000 7f2afc06a700 -1 osd.7 1389 *** Got signal Terminated *** 2024-04-12T12:06:29.419 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:29.148+0000 7f2afc06a700 -1 osd.7 1389 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T12:06:30.217 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:29.814+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:30.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:29.814+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:30.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:29.814+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:30.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:29 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:29.814+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:31.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:30 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:30.820+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:31.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:30 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:30.820+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:31.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:30 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:30.820+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:31.219 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:30 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:30.820+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:32.217 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:31 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:31.779+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:32.217 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:31 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:31.779+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:32.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:31 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:31.779+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:32.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:31 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:31.779+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:33.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:32.744+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:33.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:32.744+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:33.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:32.744+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:33.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:32 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:32.744+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:34.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:33.745+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-04-12T12:05:31.562394+0000 front 2024-04-12T12:05:31.562327+0000 (oldest deadline 2024-04-12T12:05:56.262265+0000) 2024-04-12T12:06:34.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:33.745+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-04-12T12:05:40.363420+0000 front 2024-04-12T12:05:40.363306+0000 (oldest deadline 2024-04-12T12:06:05.663162+0000) 2024-04-12T12:06:34.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:33.745+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-04-12T12:05:50.264755+0000 front 2024-04-12T12:05:50.264677+0000 (oldest deadline 2024-04-12T12:06:14.364553+0000) 2024-04-12T12:06:34.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:33.745+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-04-12T12:05:56.065626+0000 front 2024-04-12T12:05:56.065516+0000 (oldest deadline 2024-04-12T12:06:21.965585+0000) 2024-04-12T12:06:34.218 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:33 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7[55767]: 2024-04-12T12:06:33.745+0000 7f2af4658700 -1 osd.7 1389 heartbeat_check: no reply from 172.21.15.87:6804 osd.4 since back 2024-04-12T12:06:08.366822+0000 front 2024-04-12T12:06:08.366834+0000 (oldest deadline 2024-04-12T12:06:33.666735+0000) 2024-04-12T12:06:35.012 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:34 smithi087 podman[246435]: 2024-04-12 12:06:34.740126248 +0000 UTC m=+5.695575402 container died 9c484b9f07907514b494d8216f4da238e0141c8ce25f8af5bb9e741f082d16b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, RELEASE=reef-d540eba, CEPH_POINT_RELEASE=) 2024-04-12T12:06:35.409 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:35 smithi087 podman[246435]: 2024-04-12 12:06:35.107079013 +0000 UTC m=+6.062528157 container cleanup 9c484b9f07907514b494d8216f4da238e0141c8ce25f8af5bb9e741f082d16b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-d540eba, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-12T12:06:35.409 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:35 smithi087 bash[246435]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7 2024-04-12T12:06:35.717 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:35 smithi087 podman[246453]: 2024-04-12 12:06:35.408111067 +0000 UTC m=+0.665501982 container remove 9c484b9f07907514b494d8216f4da238e0141c8ce25f8af5bb9e741f082d16b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=reef-d540eba, ceph=True, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-12T12:06:35.996 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:35 smithi087 podman[246545]: 2024-04-12 12:06:35.717438021 +0000 UTC m=+0.108097441 container create 59549ffeb75d713e1c2dbea907dfff61dfbe7d42cc5e4136e12c721d55e6b67f (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=reef-d540eba, CEPH_POINT_RELEASE=) 2024-04-12T12:06:35.996 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:35 smithi087 podman[246545]: 2024-04-12 12:06:35.627395616 +0000 UTC m=+0.018055005 image pull bb2c5eac26a47e91f49419364028af9c3bea25d6fad65850c7d6f8f2c7b91b28 quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3 2024-04-12T12:06:35.996 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:35 smithi087 podman[246545]: 2024-04-12 12:06:35.920323448 +0000 UTC m=+0.310982828 container init 59549ffeb75d713e1c2dbea907dfff61dfbe7d42cc5e4136e12c721d55e6b67f (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=reef-d540eba, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-12T12:06:35.996 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:35 smithi087 podman[246545]: 2024-04-12 12:06:35.923430751 +0000 UTC m=+0.314090127 container start 59549ffeb75d713e1c2dbea907dfff61dfbe7d42cc5e4136e12c721d55e6b67f (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=reef-d540eba, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T12:06:35.996 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:35 smithi087 podman[246545]: 2024-04-12 12:06:35.995113442 +0000 UTC m=+0.385772831 container attach 59549ffeb75d713e1c2dbea907dfff61dfbe7d42cc5e4136e12c721d55e6b67f (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=reef-d540eba, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0) 2024-04-12T12:06:36.467 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:36 smithi087 podman[246545]: 2024-04-12 12:06:36.211012031 +0000 UTC m=+0.601671417 container died 59549ffeb75d713e1c2dbea907dfff61dfbe7d42cc5e4136e12c721d55e6b67f (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7-deactivate, RELEASE=reef-d540eba, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-12T12:06:36.813 INFO:journalctl@ceph.osd.7.smithi087.stdout:Apr 12 12:06:36 smithi087 podman[246572]: 2024-04-12 12:06:36.798749125 +0000 UTC m=+0.582102090 container remove 59549ffeb75d713e1c2dbea907dfff61dfbe7d42cc5e4136e12c721d55e6b67f (image=quay.ceph.io/ceph-ci/ceph@sha256:72f2afc3fa84ea067411fbf5c1562b80cbcb72ca6ea6f1e44b170e45174046f3, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-osd-7-deactivate, RELEASE=reef-d540eba, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T12:06:36.818 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@osd.7.service' 2024-04-12T12:06:36.858 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:06:36.858 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-12T12:06:36.858 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-12T12:06:36.858 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@prometheus.a 2024-04-12T12:06:37.177 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:36 smithi087 systemd[1]: Stopping Ceph prometheus.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:06:37.452 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.176Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-12T12:06:37.452 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.176Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-12T12:06:37.453 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.176Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-12T12:06:37.453 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.176Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-12T12:06:37.453 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.176Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-12T12:06:37.453 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.177Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-12T12:06:37.453 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.177Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-12T12:06:37.453 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.177Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-12T12:06:37.453 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.177Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-12T12:06:37.453 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.178Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-12T12:06:37.453 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.178Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-12T12:06:37.454 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a[78087]: ts=2024-04-12T12:06:37.178Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-12T12:06:37.454 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 podman[246651]: 2024-04-12 12:06:37.18687257 +0000 UTC m=+0.106603526 container died 569140fc5a5359a99754861b08c344a1a019ea6bb52f543817e907a3e0effbac (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T12:06:37.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 podman[246651]: 2024-04-12 12:06:37.559493368 +0000 UTC m=+0.479224319 container cleanup 569140fc5a5359a99754861b08c344a1a019ea6bb52f543817e907a3e0effbac (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-12T12:06:37.717 INFO:journalctl@ceph.prometheus.a.smithi087.stdout:Apr 12 12:06:37 smithi087 bash[246651]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-prometheus-a 2024-04-12T12:06:37.977 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@prometheus.a.service' 2024-04-12T12:06:38.014 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T12:06:38.014 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-12T12:06:38.014 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 --force --keep-logs 2024-04-12T12:06:38.176 INFO:teuthology.orchestra.run.smithi029.stdout:Deleting cluster with fsid: 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T12:06:39.928 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 12:06:39 smithi029 systemd[1]: Stopping Ceph alertmanager.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:06:40.269 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 12:06:39 smithi029 ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a[77511]: ts=2024-04-12T12:06:39.927Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-12T12:06:40.269 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 12:06:39 smithi029 podman[322392]: 2024-04-12 12:06:39.988157842 +0000 UTC m=+0.176265638 container died 5f36b9c37cec222056a033e8e274cf62192e7cb3a466cd8cf261d35bdbc305c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T12:06:40.735 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 12:06:40 smithi029 podman[322392]: 2024-04-12 12:06:40.506783374 +0000 UTC m=+0.694891170 container cleanup 5f36b9c37cec222056a033e8e274cf62192e7cb3a466cd8cf261d35bdbc305c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T12:06:40.735 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 12:06:40 smithi029 bash[322392]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a 2024-04-12T12:06:41.019 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 12:06:40 smithi029 podman[322407]: 2024-04-12 12:06:40.734289933 +0000 UTC m=+0.744224722 container remove 5f36b9c37cec222056a033e8e274cf62192e7cb3a466cd8cf261d35bdbc305c2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-12T12:06:41.019 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 12:06:40 smithi029 podman[322407]: 2024-04-12 12:06:40.827021874 +0000 UTC m=+0.836956629 volume remove ed109fb9dd2ad9e389d5fe825664a413b61df8d12f1a530d4487886def9dbfdd 2024-04-12T12:06:41.518 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 12:06:41 smithi029 systemd[1]: Stopping Ceph node-exporter.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:06:41.518 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 12:06:41 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-12T12:06:41.519 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 12:06:41 smithi029 systemd[1]: Stopped Ceph alertmanager.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T12:06:41.519 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Apr 12 12:06:41 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@alertmanager.a.service: Consumed 3.527s CPU time. 2024-04-12T12:06:41.769 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 12:06:41 smithi029 podman[322513]: 2024-04-12 12:06:41.517349469 +0000 UTC m=+0.121895168 container died 32124106e91232b059a65c0e575644bb1158c18157b23e3e78ec63a78abb91ee (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T12:06:42.136 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 12:06:41 smithi029 podman[322513]: 2024-04-12 12:06:41.933547459 +0000 UTC m=+0.538093161 container cleanup 32124106e91232b059a65c0e575644bb1158c18157b23e3e78ec63a78abb91ee (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T12:06:42.137 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 12:06:41 smithi029 bash[322513]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a 2024-04-12T12:06:42.519 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 12:06:42 smithi029 podman[322527]: 2024-04-12 12:06:42.135880902 +0000 UTC m=+0.622042249 container remove 32124106e91232b059a65c0e575644bb1158c18157b23e3e78ec63a78abb91ee (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-12T12:06:42.519 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 12:06:42 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-12T12:06:42.519 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 12:06:42 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-12T12:06:42.520 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 12:06:42 smithi029 systemd[1]: Stopped Ceph node-exporter.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T12:06:42.520 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Apr 12 12:06:42 smithi029 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@node-exporter.a.service: Consumed 8.105s CPU time. 2024-04-12T12:06:51.411 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 --force --keep-logs 2024-04-12T12:06:51.568 INFO:teuthology.orchestra.run.smithi087.stdout:Deleting cluster with fsid: 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T12:06:53.218 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 12:06:52 smithi087 systemd[1]: Stopping Ceph grafana.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:06:53.218 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 12:06:53 smithi087 podman[247061]: 2024-04-12 12:06:53.173708283 +0000 UTC m=+0.110423443 container died d4ba4d587fdb414b40bb283109da73d4b744eab8f3bb18236cb4ec45dad4f472 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a, vcs-type=git, io.openshift.expose-services=, name=ubi8, vendor=Red Hat, Inc., distribution-scope=public, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, version=8.5, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-12T12:06:53.717 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 12:06:53 smithi087 podman[247061]: 2024-04-12 12:06:53.51342184 +0000 UTC m=+0.450137044 container cleanup d4ba4d587fdb414b40bb283109da73d4b744eab8f3bb18236cb4ec45dad4f472 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Paul Cuzner , release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, io.openshift.expose-services=, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, com.redhat.component=ubi8-container, description=Ceph Grafana Container, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, version=8.5) 2024-04-12T12:06:53.718 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 12:06:53 smithi087 bash[247061]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a 2024-04-12T12:06:54.136 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 12:06:53 smithi087 podman[247075]: 2024-04-12 12:06:53.739483568 +0000 UTC m=+0.566205537 container remove d4ba4d587fdb414b40bb283109da73d4b744eab8f3bb18236cb4ec45dad4f472 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, version=8.5, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, release=239.1651231664, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.openshift.tags=base rhel8) 2024-04-12T12:06:54.136 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 12:06:53 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-12T12:06:54.136 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 12:06:53 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-12T12:06:54.136 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 12:06:53 smithi087 systemd[1]: Stopped Ceph grafana.a for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T12:06:54.136 INFO:journalctl@ceph.grafana.a.smithi087.stdout:Apr 12 12:06:53 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@grafana.a.service: Consumed 33.288s CPU time. 2024-04-12T12:06:54.404 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 12:06:54 smithi087 systemd[1]: Stopping Ceph node-exporter.b for 80efffc6-f8bc-11ee-bc8c-c7b262605968... 2024-04-12T12:06:54.717 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 12:06:54 smithi087 podman[247183]: 2024-04-12 12:06:54.403030996 +0000 UTC m=+0.132827830 container died 78910b2d30c688a6fe964f8df008770efccb0b0e369df975994ff3cc5e26ef8a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T12:06:55.175 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 12:06:54 smithi087 podman[247183]: 2024-04-12 12:06:54.731431042 +0000 UTC m=+0.461227883 container cleanup 78910b2d30c688a6fe964f8df008770efccb0b0e369df975994ff3cc5e26ef8a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T12:06:55.176 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 12:06:54 smithi087 bash[247183]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b 2024-04-12T12:06:55.176 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 12:06:54 smithi087 podman[247197]: 2024-04-12 12:06:54.960537159 +0000 UTC m=+0.561955059 container remove 78910b2d30c688a6fe964f8df008770efccb0b0e369df975994ff3cc5e26ef8a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-12T12:06:55.176 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 12:06:54 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-12T12:06:55.467 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 12:06:55 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-12T12:06:55.467 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 12:06:55 smithi087 systemd[1]: Stopped Ceph node-exporter.b for 80efffc6-f8bc-11ee-bc8c-c7b262605968. 2024-04-12T12:06:55.468 INFO:journalctl@ceph.node-exporter.b.smithi087.stdout:Apr 12 12:06:55 smithi087 systemd[1]: ceph-80efffc6-f8bc-11ee-bc8c-c7b262605968@node-exporter.b.service: Consumed 7.290s CPU time. 2024-04-12T12:07:01.071 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T12:07:01.106 DEBUG:teuthology.orchestra.run.smithi087:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T12:07:01.137 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-12T12:07:01.142 DEBUG:teuthology.misc:Transferring archived files from smithi029:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/crash to /home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791/remote/smithi029/crash 2024-04-12T12:07:01.144 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/crash -- . 2024-04-12T12:07:01.197 INFO:teuthology.orchestra.run.smithi029.stderr:tar: /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/crash: Cannot open: No such file or directory 2024-04-12T12:07:01.197 INFO:teuthology.orchestra.run.smithi029.stderr:tar: Error is not recoverable: exiting now 2024-04-12T12:07:01.199 DEBUG:teuthology.misc:Transferring archived files from smithi087:/var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/crash to /home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791/remote/smithi087/crash 2024-04-12T12:07:01.200 DEBUG:teuthology.orchestra.run.smithi087:> sudo tar c -f - -C /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/crash -- . 2024-04-12T12:07:01.255 INFO:teuthology.orchestra.run.smithi087.stderr:tar: /var/lib/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/crash: Cannot open: No such file or directory 2024-04-12T12:07:01.255 INFO:teuthology.orchestra.run.smithi087.stderr:tar: Error is not recoverable: exiting now 2024-04-12T12:07:01.257 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-12T12:07:01.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-12T12:07:01.356 INFO:teuthology.orchestra.run.smithi029.stdout:2024-04-12T11:21:48.606223+0000 mon.a (mon.0) 594 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T12:07:01.359 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-12T12:07:01.360 INFO:tasks.cephadm:Compressing logs... 2024-04-12T12:07:01.360 DEBUG:teuthology.orchestra.run.smithi029:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-12T12:07:01.362 DEBUG:teuthology.orchestra.run.smithi087:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-12T12:07:01.407 INFO:teuthology.orchestra.run.smithi029.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-12T12:07:01.407 INFO:teuthology.orchestra.run.smithi029.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-12T12:07:01.408 INFO:teuthology.orchestra.run.smithi029.stderr:gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mon.a.log 2024-04-12T12:07:01.408 INFO:teuthology.orchestra.run.smithi029.stderr:gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.audit.log 2024-04-12T12:07:01.408 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.1.log 2024-04-12T12:07:01.409 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.3.log 2024-04-12T12:07:01.411 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi029.scbuux.log 2024-04-12T12:07:01.411 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mgr.y.log 2024-04-12T12:07:01.411 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi029.scbuux.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.0.log 2024-04-12T12:07:01.411 INFO:teuthology.orchestra.run.smithi029.stderr: 85.9% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi029.scbuux.log.gz 2024-04-12T12:07:01.411 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.cephadm.log 2024-04-12T12:07:01.411 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.log 2024-04-12T12:07:01.412 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.2.log 2024-04-12T12:07:01.412 INFO:teuthology.orchestra.run.smithi029.stderr:gzip/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.log: -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-volume.log 2024-04-12T12:07:01.412 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mon.c.log 2024-04-12T12:07:01.413 INFO:teuthology.orchestra.run.smithi087.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-12T12:07:01.413 INFO:teuthology.orchestra.run.smithi087.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-12T12:07:01.414 INFO:teuthology.orchestra.run.smithi087.stderr:gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.6.log 2024-04-12T12:07:01.414 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.audit.log 2024-04-12T12:07:01.415 INFO:teuthology.orchestra.run.smithi087.stderr:gzip/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.6.log: -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.4.log 2024-04-12T12:07:01.415 INFO:teuthology.orchestra.run.smithi087.stderr:gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.7.log 2024-04-12T12:07:01.417 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.audit.log: /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.5.log 2024-04-12T12:07:01.417 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mgr.x.log 2024-04-12T12:07:01.417 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi087.mbflzu.log 2024-04-12T12:07:01.417 INFO:teuthology.orchestra.run.smithi087.stderr:gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.cephadm.log 2024-04-12T12:07:01.417 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mgr.x.log: /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi087.mbflzu.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.log 2024-04-12T12:07:01.417 INFO:teuthology.orchestra.run.smithi087.stderr: 84.5% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mds.foo.smithi087.mbflzu.log.gz 2024-04-12T12:07:01.418 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-volume.log 2024-04-12T12:07:01.418 INFO:teuthology.orchestra.run.smithi087.stderr:gzip -5 --verbose -- /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mon.b.log 2024-04-12T12:07:01.423 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-volume.log: /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mon.c.log: 92.7% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.log.gz 2024-04-12T12:07:01.584 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.log: /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-volume.log: /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mon.b.log: 84.1% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.cephadm.log.gz 2024-04-12T12:07:01.752 INFO:teuthology.orchestra.run.smithi087.stderr: 91.1% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-volume.log.gz 2024-04-12T12:07:01.798 INFO:teuthology.orchestra.run.smithi029.stderr: 94.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-12T12:07:01.840 INFO:teuthology.orchestra.run.smithi029.stderr: 94.1% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.audit.log.gz 2024-04-12T12:07:01.887 INFO:teuthology.orchestra.run.smithi087.stderr: 86.9% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.log.gz 2024-04-12T12:07:02.038 INFO:teuthology.orchestra.run.smithi087.stderr: 90.2% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.audit.log.gz 2024-04-12T12:07:02.245 INFO:teuthology.orchestra.run.smithi029.stderr: 90.7% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph.cephadm.log.gz 2024-04-12T12:07:02.271 INFO:teuthology.orchestra.run.smithi087.stderr: 87.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-12T12:07:02.354 INFO:teuthology.orchestra.run.smithi087.stderr: 89.8% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mgr.x.log.gz 2024-04-12T12:07:02.426 INFO:teuthology.orchestra.run.smithi029.stderr: 91.0% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-volume.log.gz 2024-04-12T12:07:05.908 INFO:teuthology.orchestra.run.smithi029.stderr: 89.2% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mgr.y.log.gz 2024-04-12T12:07:18.039 INFO:teuthology.orchestra.run.smithi087.stderr: 92.3% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mon.b.log.gz 2024-04-12T12:07:28.745 INFO:teuthology.orchestra.run.smithi029.stderr: 91.8% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mon.c.log.gz 2024-04-12T12:07:48.321 INFO:teuthology.orchestra.run.smithi029.stderr: 90.3% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-mon.a.log.gz 2024-04-12T12:22:23.777 INFO:teuthology.orchestra.run.smithi087.stderr: 93.8% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.7.log.gz 2024-04-12T12:26:32.648 INFO:teuthology.orchestra.run.smithi087.stderr: 92.8% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.6.log.gz 2024-04-12T12:29:12.975 INFO:teuthology.orchestra.run.smithi087.stderr: 91.9% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.5.log.gz 2024-04-12T12:29:19.512 INFO:teuthology.orchestra.run.smithi087.stderr: 92.5% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.4.log.gz 2024-04-12T12:29:19.520 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-12T12:29:19.521 INFO:teuthology.orchestra.run.smithi087.stderr:real 22m18.143s 2024-04-12T12:29:19.521 INFO:teuthology.orchestra.run.smithi087.stderr:user 7m16.836s 2024-04-12T12:29:19.521 INFO:teuthology.orchestra.run.smithi087.stderr:sys 0m23.003s 2024-04-12T12:30:00.496 INFO:teuthology.orchestra.run.smithi029.stderr: 93.3% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.1.log.gz 2024-04-12T12:32:20.037 INFO:teuthology.orchestra.run.smithi029.stderr: 93.2% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.3.log.gz 2024-04-12T12:33:17.851 INFO:teuthology.orchestra.run.smithi029.stderr: 92.7% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.2.log.gz 2024-04-12T12:35:31.483 INFO:teuthology.orchestra.run.smithi029.stderr: 92.3% -- replaced with /var/log/ceph/80efffc6-f8bc-11ee-bc8c-c7b262605968/ceph-osd.0.log.gz 2024-04-12T12:35:31.499 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-04-12T12:35:31.499 INFO:teuthology.orchestra.run.smithi029.stderr:real 28m30.122s 2024-04-12T12:35:31.499 INFO:teuthology.orchestra.run.smithi029.stderr:user 9m19.434s 2024-04-12T12:35:31.499 INFO:teuthology.orchestra.run.smithi029.stderr:sys 0m27.157s 2024-04-12T12:35:31.500 INFO:tasks.cephadm:Archiving logs... 2024-04-12T12:35:31.502 DEBUG:teuthology.misc:Transferring archived files from smithi029:/var/log/ceph to /home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791/remote/smithi029/log 2024-04-12T12:35:31.503 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-12T12:36:51.780 DEBUG:teuthology.misc:Transferring archived files from smithi087:/var/log/ceph to /home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791/remote/smithi087/log 2024-04-12T12:36:51.782 DEBUG:teuthology.orchestra.run.smithi087:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-12T12:38:03.598 INFO:tasks.cephadm:Removing cluster... 2024-04-12T12:38:03.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 --force 2024-04-12T12:38:04.101 INFO:teuthology.orchestra.run.smithi029.stdout:Deleting cluster with fsid: 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T12:38:04.983 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 80efffc6-f8bc-11ee-bc8c-c7b262605968 --force 2024-04-12T12:38:05.165 INFO:teuthology.orchestra.run.smithi087.stdout:Deleting cluster with fsid: 80efffc6-f8bc-11ee-bc8c-c7b262605968 2024-04-12T12:38:05.925 INFO:tasks.cephadm:Removing cephadm ... 2024-04-12T12:38:05.926 DEBUG:teuthology.orchestra.run.smithi029:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-12T12:38:05.946 DEBUG:teuthology.orchestra.run.smithi087:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-12T12:38:05.967 INFO:tasks.cephadm:Teardown complete 2024-04-12T12:38:05.968 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-12T12:38:05.979 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-12T12:38:05.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-12T12:38:05.988 DEBUG:teuthology.orchestra.run.smithi087:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-12T12:38:06.096 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-12T12:38:06.106 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-12T12:38:06.107 DEBUG:teuthology.orchestra.run.smithi029:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T12:38:06.110 DEBUG:teuthology.orchestra.run.smithi087:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T12:38:06.123 INFO:teuthology.orchestra.run.smithi029.stderr:bash: line 1: ntpq: command not found 2024-04-12T12:38:06.124 INFO:teuthology.orchestra.run.smithi087.stderr:bash: line 1: ntpq: command not found 2024-04-12T12:38:06.147 INFO:teuthology.orchestra.run.smithi087.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T12:38:06.147 INFO:teuthology.orchestra.run.smithi087.stdout:=============================================================================== 2024-04-12T12:38:06.147 INFO:teuthology.orchestra.run.smithi087.stdout:^* hv01.front.sepia.ceph.com 3 6 377 21 +406us[ +410us] +/- 42ms 2024-04-12T12:38:06.147 INFO:teuthology.orchestra.run.smithi087.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 20 -2266us[-2266us] +/- 64ms 2024-04-12T12:38:06.147 INFO:teuthology.orchestra.run.smithi087.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 20 +2213us[+2213us] +/- 79ms 2024-04-12T12:38:06.148 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T12:38:06.215 INFO:teuthology.orchestra.run.smithi029.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T12:38:06.215 INFO:teuthology.orchestra.run.smithi029.stdout:=============================================================================== 2024-04-12T12:38:06.215 INFO:teuthology.orchestra.run.smithi029.stdout:^* hv01.front.sepia.ceph.com 3 6 377 18 +348us[ +377us] +/- 42ms 2024-04-12T12:38:06.216 INFO:teuthology.orchestra.run.smithi029.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 19 -2323us[-2293us] +/- 64ms 2024-04-12T12:38:06.216 INFO:teuthology.orchestra.run.smithi029.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 16 +2162us[+2162us] +/- 79ms 2024-04-12T12:38:06.216 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T12:38:06.242 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-12T12:38:06.253 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-12T12:38:06.254 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-12T12:38:06.301 DEBUG:teuthology.orchestra.run.smithi029:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-12T12:38:06.536 DEBUG:teuthology.orchestra.run.smithi087:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-12T12:38:06.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-12T12:38:06.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:38:06.692 DEBUG:teuthology.orchestra.run.smithi087:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-12T12:38:06.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:38:06.723 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-12T12:38:06.734 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-12T12:38:06.787 INFO:teuthology.task.internal:Duration was 6076.721308 seconds 2024-04-12T12:38:06.788 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-12T12:38:06.833 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-12T12:38:06.834 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-12T12:38:06.836 DEBUG:teuthology.orchestra.run.smithi087:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-12T12:38:06.909 INFO:teuthology.orchestra.run.smithi087.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T12:38:06.965 INFO:teuthology.orchestra.run.smithi029.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T12:38:07.354 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-12T12:38:07.354 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi029.front.sepia.ceph.com 2024-04-12T12:38:07.355 DEBUG:teuthology.orchestra.run.smithi029:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-12T12:38:07.383 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi087.front.sepia.ceph.com 2024-04-12T12:38:07.383 DEBUG:teuthology.orchestra.run.smithi087:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-12T12:38:07.408 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-12T12:38:07.408 DEBUG:teuthology.orchestra.run.smithi029:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-12T12:38:07.427 DEBUG:teuthology.orchestra.run.smithi087:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-12T12:38:07.477 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-12T12:38:07.477 DEBUG:teuthology.orchestra.run.smithi029:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-12T12:38:07.479 DEBUG:teuthology.orchestra.run.smithi087:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-12T12:38:09.081 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-12T12:38:09.092 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-12T12:38:09.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-12T12:38:09.130 DEBUG:teuthology.orchestra.run.smithi087:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-12T12:38:09.160 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-12T12:38:09.171 DEBUG:teuthology.orchestra.run.smithi029:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-12T12:38:09.173 DEBUG:teuthology.orchestra.run.smithi087:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-12T12:38:09.220 INFO:teuthology.orchestra.run.smithi029.stdout:kernel.core_pattern = core 2024-04-12T12:38:09.232 INFO:teuthology.orchestra.run.smithi087.stdout:kernel.core_pattern = core 2024-04-12T12:38:09.446 DEBUG:teuthology.orchestra.run.smithi029:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-12T12:38:09.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:38:09.460 DEBUG:teuthology.orchestra.run.smithi087:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-12T12:38:09.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T12:38:09.475 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-12T12:38:09.486 INFO:teuthology.task.internal:Transferring archived files... 2024-04-12T12:38:09.487 DEBUG:teuthology.misc:Transferring archived files from smithi029:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791/remote/smithi029 2024-04-12T12:38:09.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-12T12:38:09.580 DEBUG:teuthology.misc:Transferring archived files from smithi087:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791/remote/smithi087 2024-04-12T12:38:09.581 DEBUG:teuthology.orchestra.run.smithi087:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-12T12:38:09.630 INFO:teuthology.task.internal:Removing archive directory... 2024-04-12T12:38:09.631 DEBUG:teuthology.orchestra.run.smithi029:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-12T12:38:09.633 DEBUG:teuthology.orchestra.run.smithi087:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-12T12:38:09.676 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-12T12:38:09.687 INFO:teuthology.task.internal:Not uploading archives. 2024-04-12T12:38:09.687 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-12T12:38:09.729 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-12T12:38:09.730 DEBUG:teuthology.orchestra.run.smithi029:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-12T12:38:09.732 DEBUG:teuthology.orchestra.run.smithi087:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-12T12:38:09.743 INFO:teuthology.orchestra.run.smithi029.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 12 12:38 /home/ubuntu/cephtest 2024-04-12T12:38:09.747 INFO:teuthology.orchestra.run.smithi087.stdout: 265957 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 12 12:38 /home/ubuntu/cephtest 2024-04-12T12:38:09.748 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-12T12:38:09.766 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-12T12:38:09.887 INFO:teuthology.nuke:Checking targets against current locks 2024-04-12T12:38:09.912 DEBUG:teuthology.nuke:shortname: smithi029 2024-04-12T12:38:09.912 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T12:38:09.936 DEBUG:teuthology.nuke:shortname: smithi087 2024-04-12T12:38:09.936 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T12:38:09.939 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi029.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 10:48:06.617256', 'locked_by': 'scheduled_rfriedma@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBZ7FJKB+L6PIG6hkHMSXaF0oPRSlDO80Q8kJy72mhlk4GmXo9CD5wGZEMBwVpOk5ICmzPHSuiLYbmjWR5gpuuY='} 2024-04-12T12:38:09.960 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi087.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rfriedma-2024-04-12_09:02:49-upgrade-wip-rf-comb1-distro-default-smithi/7653791', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 10:48:06.618934', 'locked_by': 'scheduled_rfriedma@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLdz/kUVoFbK0olMhupH5PXgGApvaUOeBM2pv4/kTcfQKbC9VpgdpNDQA/yOMVLBnY5CM+eB0qC5ZzwYlatsAjQ='} 2024-04-12T12:38:09.992 INFO:teuthology.orchestra.console.smithi029:Power off 2024-04-12T12:38:09.992 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-12T12:38:10.018 DEBUG:teuthology.orchestra.console.smithi029:power off output: Chassis Power Control: Down/Off 2024-04-12T12:38:10.018 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T12:38:10.035 INFO:teuthology.orchestra.console.smithi087:Power off 2024-04-12T12:38:10.035 DEBUG:teuthology.orchestra.console.smithi087:pexpect command: ipmitool -H smithi087.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-12T12:38:10.053 DEBUG:teuthology.orchestra.console.smithi029:check power output: Chassis Power is on 2024-04-12T12:38:10.059 DEBUG:teuthology.orchestra.console.smithi087:power off output: Chassis Power Control: Down/Off 2024-04-12T12:38:10.059 DEBUG:teuthology.orchestra.console.smithi087:pexpect command: ipmitool -H smithi087.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T12:38:10.083 DEBUG:teuthology.orchestra.console.smithi087:check power output: Chassis Power is on 2024-04-12T12:38:14.053 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T12:38:14.085 DEBUG:teuthology.orchestra.console.smithi087:pexpect command: ipmitool -H smithi087.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T12:38:14.167 DEBUG:teuthology.orchestra.console.smithi029:check power output: Chassis Power is on 2024-04-12T12:38:14.195 DEBUG:teuthology.orchestra.console.smithi087:check power output: Chassis Power is on 2024-04-12T12:38:18.170 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T12:38:18.196 DEBUG:teuthology.orchestra.console.smithi087:pexpect command: ipmitool -H smithi087.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T12:38:18.281 DEBUG:teuthology.orchestra.console.smithi029:check power output: Chassis Power is off 2024-04-12T12:38:18.305 DEBUG:teuthology.orchestra.console.smithi087:check power output: Chassis Power is off 2024-04-12T12:38:18.383 INFO:teuthology.orchestra.console.smithi029:Power off completed 2024-04-12T12:38:18.407 INFO:teuthology.orchestra.console.smithi087:Power off completed 2024-04-12T12:38:18.512 INFO:teuthology.run:Summary data: description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 6076.721307516098 failure_reason: No module named 'tasks.workunit' flavor: default owner: scheduled_rfriedma@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=5cadaa7286a046cf9a4f8308372c8379 status: fail success: false 2024-04-12T12:38:18.512 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-12T12:38:18.590 INFO:teuthology.run:FAIL